From 5c10bdc06d47cf402b01416dd799fe0ef1e139d8 Mon Sep 17 00:00:00 2001 From: type-two Date: Sun, 21 Jun 2026 13:16:56 +1000 Subject: [PATCH] =?UTF-8?q?feat:=20self-contained=20enrichment=20=E2=80=94?= =?UTF-8?q?=20disc=5Fcache=20(own=20catalog=20copy),=20no=20external=20DB?= =?UTF-8?q?=20dependency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - schema: disc_cache (release_id→title/artist/thumb/weight); build_disc_cache.py populates it from the full Discogs dump on ultra (30,694 stocked releases). - app/mirror.py: enrich() now reads RecordGod's OWN disc_cache via the main DB — no live discogs_full / dealgod-DB dependency. DealGod reached only via API key. Co-Authored-By: Claude Opus 4.8 --- app/mirror.py | 27 +++++++++------------------ build_disc_cache.py | 36 ++++++++++++++++++++++++++++++++++++ schema.sql | 11 +++++++++++ 3 files changed, 56 insertions(+), 18 deletions(-) create mode 100644 build_disc_cache.py diff --git a/app/mirror.py b/app/mirror.py index 53d827d..db2e965 100644 --- a/app/mirror.py +++ b/app/mirror.py @@ -1,31 +1,22 @@ -import os - from sqlalchemy import text -from .db import MirrorSession +from .db import SessionLocal -# Enrichment from the Discogs mirror. Two known shapes, selected by env so the same code runs -# on ultra (the full dump) and on the VPS (dealgod's discogs_release subset): -# full → table `release` cols artists_sort / thumb / estimated_weight (ultra) -# vps → table `discogs_release` cols artist (no cover/weight) (dealgod-db) -KIND = os.getenv("DISCOGS_MIRROR_KIND", "full") -if KIND == "vps": - _SQL = ("SELECT id, title, artist, NULL::text AS thumb, NULL::int AS weight " - "FROM discogs_release WHERE id = ANY(:ids)") -else: - _SQL = ("SELECT id, title, artists_sort AS artist, " - "COALESCE(thumb_local_url, thumb) AS thumb, estimated_weight AS weight " - "FROM release WHERE id = ANY(:ids)") +# Enrichment from RecordGod's OWN cached catalog (disc_cache) — self-contained, no live +# dependency on the full Discogs dump or dealgod's DB. Populate via build_disc_cache.py. +# DealGod is reached only via its API (for market value), never its database. async def enrich(ids): - """release_id(s) → {id: {title, artist, thumb, weight}}. Best-effort: mirror down → {}.""" + """release_id(s) → {id: {title, artist, thumb, weight}}. Best-effort.""" ids = [i for i in set(ids) if i is not None] if not ids: return {} try: - async with MirrorSession() as m: - rows = await m.execute(text(_SQL), {"ids": ids}) + async with SessionLocal() as s: + rows = await s.execute(text( + "SELECT release_id AS id, title, artist, thumb, weight " + "FROM disc_cache WHERE release_id = ANY(:ids)"), {"ids": ids}) return {r["id"]: dict(r) for r in rows.mappings()} except Exception: return {} diff --git a/build_disc_cache.py b/build_disc_cache.py new file mode 100644 index 0000000..2d9f672 --- /dev/null +++ b/build_disc_cache.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Populate recordgod.disc_cache from the full Discogs mirror — RecordGod's own copy of just +the releases it stocks, so enrichment has no live cross-DB dependency. Run on ultra (where +discogs_full lives); re-run anytime to refresh for current inventory.""" +import os + +import psycopg + +RG = os.getenv("RECORDGOD_DSN", "postgresql://localhost/recordgod") +DG = os.getenv("DISCOGS_FULL_DSN", "postgresql://localhost/discogs_full") + + +def main(): + rg = psycopg.connect(RG) + dg = psycopg.connect(DG) + ids = [r[0] for r in rg.execute( + "SELECT DISTINCT release_id FROM inventory WHERE release_id IS NOT NULL").fetchall()] + rg.execute("""CREATE TABLE IF NOT EXISTS disc_cache ( + release_id bigint PRIMARY KEY, title text, artist text, thumb text, weight int)""") + rg.execute("TRUNCATE disc_cache") + n = 0 + for i in range(0, len(ids), 5000): + rows = dg.execute( + "SELECT id, title, artists_sort, COALESCE(thumb_local_url, thumb), estimated_weight " + "FROM release WHERE id = ANY(%s)", (ids[i:i + 5000],)).fetchall() + with rg.cursor() as c: + c.executemany( + "INSERT INTO disc_cache (release_id, title, artist, thumb, weight) " + "VALUES (%s,%s,%s,%s,%s) ON CONFLICT (release_id) DO NOTHING", rows) + n += len(rows) + rg.commit() + print(f"disc_cache: {n} of {len(ids)} stocked releases cached") + + +if __name__ == "__main__": + main() diff --git a/schema.sql b/schema.sql index 7a59b5a..dd1fae1 100644 --- a/schema.sql +++ b/schema.sql @@ -86,6 +86,17 @@ CREATE TABLE IF NOT EXISTS sale_items ( CREATE INDEX IF NOT EXISTS sale_items_sale_idx ON sale_items (sale_id); CREATE INDEX IF NOT EXISTS sale_items_release_idx ON sale_items (release_id); +-- Cached Discogs catalog for enrichment — RecordGod's OWN copy of just the releases it stocks, +-- so the store is self-contained: no live dependency on the full Discogs dump OR dealgod's DB. +-- Populated by build_disc_cache.py from the full mirror on ultra. +CREATE TABLE IF NOT EXISTS disc_cache ( + release_id bigint PRIMARY KEY, + title text, + artist text, + thumb text, + weight int +); + -- Encrypted credential vault. Values are Fernet ciphertext; the key lives in the environment -- (RECORDGOD_SECRET_KEY), never here. Woo/Discogs/Cloudflare/DealGod tokens land here via /settings. CREATE TABLE IF NOT EXISTS app_secret (