feat: self-contained enrichment — disc_cache (own catalog copy), no external DB dependency
- schema: disc_cache (release_id→title/artist/thumb/weight); build_disc_cache.py populates it from the full Discogs dump on ultra (30,694 stocked releases). - app/mirror.py: enrich() now reads RecordGod's OWN disc_cache via the main DB — no live discogs_full / dealgod-DB dependency. DealGod reached only via API key. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
9b8ecdb84d
commit
5c10bdc06d
@ -1,31 +1,22 @@
|
||||
import os
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from .db import MirrorSession
|
||||
from .db import SessionLocal
|
||||
|
||||
# Enrichment from the Discogs mirror. Two known shapes, selected by env so the same code runs
|
||||
# on ultra (the full dump) and on the VPS (dealgod's discogs_release subset):
|
||||
# full → table `release` cols artists_sort / thumb / estimated_weight (ultra)
|
||||
# vps → table `discogs_release` cols artist (no cover/weight) (dealgod-db)
|
||||
KIND = os.getenv("DISCOGS_MIRROR_KIND", "full")
|
||||
if KIND == "vps":
|
||||
_SQL = ("SELECT id, title, artist, NULL::text AS thumb, NULL::int AS weight "
|
||||
"FROM discogs_release WHERE id = ANY(:ids)")
|
||||
else:
|
||||
_SQL = ("SELECT id, title, artists_sort AS artist, "
|
||||
"COALESCE(thumb_local_url, thumb) AS thumb, estimated_weight AS weight "
|
||||
"FROM release WHERE id = ANY(:ids)")
|
||||
# Enrichment from RecordGod's OWN cached catalog (disc_cache) — self-contained, no live
|
||||
# dependency on the full Discogs dump or dealgod's DB. Populate via build_disc_cache.py.
|
||||
# DealGod is reached only via its API (for market value), never its database.
|
||||
|
||||
|
||||
async def enrich(ids):
|
||||
"""release_id(s) → {id: {title, artist, thumb, weight}}. Best-effort: mirror down → {}."""
|
||||
"""release_id(s) → {id: {title, artist, thumb, weight}}. Best-effort."""
|
||||
ids = [i for i in set(ids) if i is not None]
|
||||
if not ids:
|
||||
return {}
|
||||
try:
|
||||
async with MirrorSession() as m:
|
||||
rows = await m.execute(text(_SQL), {"ids": ids})
|
||||
async with SessionLocal() as s:
|
||||
rows = await s.execute(text(
|
||||
"SELECT release_id AS id, title, artist, thumb, weight "
|
||||
"FROM disc_cache WHERE release_id = ANY(:ids)"), {"ids": ids})
|
||||
return {r["id"]: dict(r) for r in rows.mappings()}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
36
build_disc_cache.py
Normal file
36
build_disc_cache.py
Normal file
@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Populate recordgod.disc_cache from the full Discogs mirror — RecordGod's own copy of just
|
||||
the releases it stocks, so enrichment has no live cross-DB dependency. Run on ultra (where
|
||||
discogs_full lives); re-run anytime to refresh for current inventory."""
|
||||
import os
|
||||
|
||||
import psycopg
|
||||
|
||||
RG = os.getenv("RECORDGOD_DSN", "postgresql://localhost/recordgod")
|
||||
DG = os.getenv("DISCOGS_FULL_DSN", "postgresql://localhost/discogs_full")
|
||||
|
||||
|
||||
def main():
|
||||
rg = psycopg.connect(RG)
|
||||
dg = psycopg.connect(DG)
|
||||
ids = [r[0] for r in rg.execute(
|
||||
"SELECT DISTINCT release_id FROM inventory WHERE release_id IS NOT NULL").fetchall()]
|
||||
rg.execute("""CREATE TABLE IF NOT EXISTS disc_cache (
|
||||
release_id bigint PRIMARY KEY, title text, artist text, thumb text, weight int)""")
|
||||
rg.execute("TRUNCATE disc_cache")
|
||||
n = 0
|
||||
for i in range(0, len(ids), 5000):
|
||||
rows = dg.execute(
|
||||
"SELECT id, title, artists_sort, COALESCE(thumb_local_url, thumb), estimated_weight "
|
||||
"FROM release WHERE id = ANY(%s)", (ids[i:i + 5000],)).fetchall()
|
||||
with rg.cursor() as c:
|
||||
c.executemany(
|
||||
"INSERT INTO disc_cache (release_id, title, artist, thumb, weight) "
|
||||
"VALUES (%s,%s,%s,%s,%s) ON CONFLICT (release_id) DO NOTHING", rows)
|
||||
n += len(rows)
|
||||
rg.commit()
|
||||
print(f"disc_cache: {n} of {len(ids)} stocked releases cached")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
11
schema.sql
11
schema.sql
@ -86,6 +86,17 @@ CREATE TABLE IF NOT EXISTS sale_items (
|
||||
CREATE INDEX IF NOT EXISTS sale_items_sale_idx ON sale_items (sale_id);
|
||||
CREATE INDEX IF NOT EXISTS sale_items_release_idx ON sale_items (release_id);
|
||||
|
||||
-- Cached Discogs catalog for enrichment — RecordGod's OWN copy of just the releases it stocks,
|
||||
-- so the store is self-contained: no live dependency on the full Discogs dump OR dealgod's DB.
|
||||
-- Populated by build_disc_cache.py from the full mirror on ultra.
|
||||
CREATE TABLE IF NOT EXISTS disc_cache (
|
||||
release_id bigint PRIMARY KEY,
|
||||
title text,
|
||||
artist text,
|
||||
thumb text,
|
||||
weight int
|
||||
);
|
||||
|
||||
-- Encrypted credential vault. Values are Fernet ciphertext; the key lives in the environment
|
||||
-- (RECORDGOD_SECRET_KEY), never here. Woo/Discogs/Cloudflare/DealGod tokens land here via /settings.
|
||||
CREATE TABLE IF NOT EXISTS app_secret (
|
||||
|
||||
Loading…
Reference in New Issue
Block a user