feat: self-contained enrichment — disc_cache (own catalog copy), no external DB dependency

- schema: disc_cache (release_id→title/artist/thumb/weight); build_disc_cache.py populates it
  from the full Discogs dump on ultra (30,694 stocked releases).
- app/mirror.py: enrich() now reads RecordGod's OWN disc_cache via the main DB — no live
  discogs_full / dealgod-DB dependency. DealGod reached only via API key.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
type-two 2026-06-21 13:16:56 +10:00
parent 9b8ecdb84d
commit 5c10bdc06d
3 changed files with 56 additions and 18 deletions

View File

@ -1,31 +1,22 @@
import os
from sqlalchemy import text
from .db import MirrorSession
from .db import SessionLocal
# Enrichment from the Discogs mirror. Two known shapes, selected by env so the same code runs
# on ultra (the full dump) and on the VPS (dealgod's discogs_release subset):
# full → table `release` cols artists_sort / thumb / estimated_weight (ultra)
# vps → table `discogs_release` cols artist (no cover/weight) (dealgod-db)
KIND = os.getenv("DISCOGS_MIRROR_KIND", "full")
if KIND == "vps":
_SQL = ("SELECT id, title, artist, NULL::text AS thumb, NULL::int AS weight "
"FROM discogs_release WHERE id = ANY(:ids)")
else:
_SQL = ("SELECT id, title, artists_sort AS artist, "
"COALESCE(thumb_local_url, thumb) AS thumb, estimated_weight AS weight "
"FROM release WHERE id = ANY(:ids)")
# Enrichment from RecordGod's OWN cached catalog (disc_cache) — self-contained, no live
# dependency on the full Discogs dump or dealgod's DB. Populate via build_disc_cache.py.
# DealGod is reached only via its API (for market value), never its database.
async def enrich(ids):
"""release_id(s) → {id: {title, artist, thumb, weight}}. Best-effort: mirror down → {}."""
"""release_id(s) → {id: {title, artist, thumb, weight}}. Best-effort."""
ids = [i for i in set(ids) if i is not None]
if not ids:
return {}
try:
async with MirrorSession() as m:
rows = await m.execute(text(_SQL), {"ids": ids})
async with SessionLocal() as s:
rows = await s.execute(text(
"SELECT release_id AS id, title, artist, thumb, weight "
"FROM disc_cache WHERE release_id = ANY(:ids)"), {"ids": ids})
return {r["id"]: dict(r) for r in rows.mappings()}
except Exception:
return {}

36
build_disc_cache.py Normal file
View File

@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Populate recordgod.disc_cache from the full Discogs mirror — RecordGod's own copy of just
the releases it stocks, so enrichment has no live cross-DB dependency. Run on ultra (where
discogs_full lives); re-run anytime to refresh for current inventory."""
import os
import psycopg
RG = os.getenv("RECORDGOD_DSN", "postgresql://localhost/recordgod")
DG = os.getenv("DISCOGS_FULL_DSN", "postgresql://localhost/discogs_full")
def main():
rg = psycopg.connect(RG)
dg = psycopg.connect(DG)
ids = [r[0] for r in rg.execute(
"SELECT DISTINCT release_id FROM inventory WHERE release_id IS NOT NULL").fetchall()]
rg.execute("""CREATE TABLE IF NOT EXISTS disc_cache (
release_id bigint PRIMARY KEY, title text, artist text, thumb text, weight int)""")
rg.execute("TRUNCATE disc_cache")
n = 0
for i in range(0, len(ids), 5000):
rows = dg.execute(
"SELECT id, title, artists_sort, COALESCE(thumb_local_url, thumb), estimated_weight "
"FROM release WHERE id = ANY(%s)", (ids[i:i + 5000],)).fetchall()
with rg.cursor() as c:
c.executemany(
"INSERT INTO disc_cache (release_id, title, artist, thumb, weight) "
"VALUES (%s,%s,%s,%s,%s) ON CONFLICT (release_id) DO NOTHING", rows)
n += len(rows)
rg.commit()
print(f"disc_cache: {n} of {len(ids)} stocked releases cached")
if __name__ == "__main__":
main()

View File

@ -86,6 +86,17 @@ CREATE TABLE IF NOT EXISTS sale_items (
CREATE INDEX IF NOT EXISTS sale_items_sale_idx ON sale_items (sale_id);
CREATE INDEX IF NOT EXISTS sale_items_release_idx ON sale_items (release_id);
-- Cached Discogs catalog for enrichment — RecordGod's OWN copy of just the releases it stocks,
-- so the store is self-contained: no live dependency on the full Discogs dump OR dealgod's DB.
-- Populated by build_disc_cache.py from the full mirror on ultra.
CREATE TABLE IF NOT EXISTS disc_cache (
release_id bigint PRIMARY KEY,
title text,
artist text,
thumb text,
weight int
);
-- Encrypted credential vault. Values are Fernet ciphertext; the key lives in the environment
-- (RECORDGOD_SECRET_KEY), never here. Woo/Discogs/Cloudflare/DealGod tokens land here via /settings.
CREATE TABLE IF NOT EXISTS app_secret (