feat: self-contained enrichment — disc_cache (own catalog copy), no external DB dependency
- schema: disc_cache (release_id→title/artist/thumb/weight); build_disc_cache.py populates it from the full Discogs dump on ultra (30,694 stocked releases). - app/mirror.py: enrich() now reads RecordGod's OWN disc_cache via the main DB — no live discogs_full / dealgod-DB dependency. DealGod reached only via API key. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
9b8ecdb84d
commit
5c10bdc06d
@ -1,31 +1,22 @@
|
|||||||
import os
|
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
from .db import MirrorSession
|
from .db import SessionLocal
|
||||||
|
|
||||||
# Enrichment from the Discogs mirror. Two known shapes, selected by env so the same code runs
|
# Enrichment from RecordGod's OWN cached catalog (disc_cache) — self-contained, no live
|
||||||
# on ultra (the full dump) and on the VPS (dealgod's discogs_release subset):
|
# dependency on the full Discogs dump or dealgod's DB. Populate via build_disc_cache.py.
|
||||||
# full → table `release` cols artists_sort / thumb / estimated_weight (ultra)
|
# DealGod is reached only via its API (for market value), never its database.
|
||||||
# vps → table `discogs_release` cols artist (no cover/weight) (dealgod-db)
|
|
||||||
KIND = os.getenv("DISCOGS_MIRROR_KIND", "full")
|
|
||||||
if KIND == "vps":
|
|
||||||
_SQL = ("SELECT id, title, artist, NULL::text AS thumb, NULL::int AS weight "
|
|
||||||
"FROM discogs_release WHERE id = ANY(:ids)")
|
|
||||||
else:
|
|
||||||
_SQL = ("SELECT id, title, artists_sort AS artist, "
|
|
||||||
"COALESCE(thumb_local_url, thumb) AS thumb, estimated_weight AS weight "
|
|
||||||
"FROM release WHERE id = ANY(:ids)")
|
|
||||||
|
|
||||||
|
|
||||||
async def enrich(ids):
|
async def enrich(ids):
|
||||||
"""release_id(s) → {id: {title, artist, thumb, weight}}. Best-effort: mirror down → {}."""
|
"""release_id(s) → {id: {title, artist, thumb, weight}}. Best-effort."""
|
||||||
ids = [i for i in set(ids) if i is not None]
|
ids = [i for i in set(ids) if i is not None]
|
||||||
if not ids:
|
if not ids:
|
||||||
return {}
|
return {}
|
||||||
try:
|
try:
|
||||||
async with MirrorSession() as m:
|
async with SessionLocal() as s:
|
||||||
rows = await m.execute(text(_SQL), {"ids": ids})
|
rows = await s.execute(text(
|
||||||
|
"SELECT release_id AS id, title, artist, thumb, weight "
|
||||||
|
"FROM disc_cache WHERE release_id = ANY(:ids)"), {"ids": ids})
|
||||||
return {r["id"]: dict(r) for r in rows.mappings()}
|
return {r["id"]: dict(r) for r in rows.mappings()}
|
||||||
except Exception:
|
except Exception:
|
||||||
return {}
|
return {}
|
||||||
|
|||||||
36
build_disc_cache.py
Normal file
36
build_disc_cache.py
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Populate recordgod.disc_cache from the full Discogs mirror — RecordGod's own copy of just
|
||||||
|
the releases it stocks, so enrichment has no live cross-DB dependency. Run on ultra (where
|
||||||
|
discogs_full lives); re-run anytime to refresh for current inventory."""
|
||||||
|
import os
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
RG = os.getenv("RECORDGOD_DSN", "postgresql://localhost/recordgod")
|
||||||
|
DG = os.getenv("DISCOGS_FULL_DSN", "postgresql://localhost/discogs_full")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
rg = psycopg.connect(RG)
|
||||||
|
dg = psycopg.connect(DG)
|
||||||
|
ids = [r[0] for r in rg.execute(
|
||||||
|
"SELECT DISTINCT release_id FROM inventory WHERE release_id IS NOT NULL").fetchall()]
|
||||||
|
rg.execute("""CREATE TABLE IF NOT EXISTS disc_cache (
|
||||||
|
release_id bigint PRIMARY KEY, title text, artist text, thumb text, weight int)""")
|
||||||
|
rg.execute("TRUNCATE disc_cache")
|
||||||
|
n = 0
|
||||||
|
for i in range(0, len(ids), 5000):
|
||||||
|
rows = dg.execute(
|
||||||
|
"SELECT id, title, artists_sort, COALESCE(thumb_local_url, thumb), estimated_weight "
|
||||||
|
"FROM release WHERE id = ANY(%s)", (ids[i:i + 5000],)).fetchall()
|
||||||
|
with rg.cursor() as c:
|
||||||
|
c.executemany(
|
||||||
|
"INSERT INTO disc_cache (release_id, title, artist, thumb, weight) "
|
||||||
|
"VALUES (%s,%s,%s,%s,%s) ON CONFLICT (release_id) DO NOTHING", rows)
|
||||||
|
n += len(rows)
|
||||||
|
rg.commit()
|
||||||
|
print(f"disc_cache: {n} of {len(ids)} stocked releases cached")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
11
schema.sql
11
schema.sql
@ -86,6 +86,17 @@ CREATE TABLE IF NOT EXISTS sale_items (
|
|||||||
CREATE INDEX IF NOT EXISTS sale_items_sale_idx ON sale_items (sale_id);
|
CREATE INDEX IF NOT EXISTS sale_items_sale_idx ON sale_items (sale_id);
|
||||||
CREATE INDEX IF NOT EXISTS sale_items_release_idx ON sale_items (release_id);
|
CREATE INDEX IF NOT EXISTS sale_items_release_idx ON sale_items (release_id);
|
||||||
|
|
||||||
|
-- Cached Discogs catalog for enrichment — RecordGod's OWN copy of just the releases it stocks,
|
||||||
|
-- so the store is self-contained: no live dependency on the full Discogs dump OR dealgod's DB.
|
||||||
|
-- Populated by build_disc_cache.py from the full mirror on ultra.
|
||||||
|
CREATE TABLE IF NOT EXISTS disc_cache (
|
||||||
|
release_id bigint PRIMARY KEY,
|
||||||
|
title text,
|
||||||
|
artist text,
|
||||||
|
thumb text,
|
||||||
|
weight int
|
||||||
|
);
|
||||||
|
|
||||||
-- Encrypted credential vault. Values are Fernet ciphertext; the key lives in the environment
|
-- Encrypted credential vault. Values are Fernet ciphertext; the key lives in the environment
|
||||||
-- (RECORDGOD_SECRET_KEY), never here. Woo/Discogs/Cloudflare/DealGod tokens land here via /settings.
|
-- (RECORDGOD_SECRET_KEY), never here. Woo/Discogs/Cloudflare/DealGod tokens land here via /settings.
|
||||||
CREATE TABLE IF NOT EXISTS app_secret (
|
CREATE TABLE IF NOT EXISTS app_secret (
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user