#!/usr/bin/env python3 """Populate recordgod.disc_cache from the full Discogs mirror — RecordGod's own copy of just the releases it stocks, so enrichment has no live cross-DB dependency. Run on ultra (where discogs_full lives); re-run anytime to refresh for current inventory.""" import os import psycopg RG = os.getenv("RECORDGOD_DSN", "postgresql://localhost/recordgod") DG = os.getenv("DISCOGS_FULL_DSN", "postgresql://localhost/discogs_full") def main(): rg = psycopg.connect(RG) dg = psycopg.connect(DG) ids = [r[0] for r in rg.execute( "SELECT DISTINCT release_id FROM inventory WHERE release_id IS NOT NULL").fetchall()] rg.execute("""CREATE TABLE IF NOT EXISTS disc_cache ( release_id bigint PRIMARY KEY, title text, artist text, thumb text, weight int)""") rg.execute("TRUNCATE disc_cache") n = 0 for i in range(0, len(ids), 5000): # artists_sort is NULL for most of this dump (~9.1M of 10.4M releases), and reading it # alone silently cached a NULL artist — which is not merely a blank column in the admin # list: dc.artist is also what the admin SEARCH and SORT run on, so an affected record # becomes unfindable by artist name. Seen live on release 447087, cached as title # "Mezcal" with no artist while its genre/style/label/country all populated, because # those read disc_release_* and only the artist reads this cache. # # The name is in release_artist regardless. extra=0 is the RELEASE artist; extra=1 rows # are credits — 447087 also lists Carl Clarke and Jim Eliot as Producers, and folding # those into the artist name would be worse than leaving it blank. # # Comma-joined rather than reconstructing Discogs' join_string ("&", "Featuring"): this # only fires where artists_sort is already NULL, and a predictable "A, B" beats both NULL # and a dangling separator. rows = dg.execute( "SELECT r.id, r.title, " " COALESCE(r.artists_sort, (" " SELECT string_agg(ra.artist_name, ', ' ORDER BY ra.position, ra.id) " " FROM release_artist ra " " WHERE ra.release_id = r.id AND ra.extra = 0)), " " COALESCE(r.thumb_local_url, r.thumb), r.estimated_weight " "FROM release r WHERE r.id = ANY(%s)", (ids[i:i + 5000],)).fetchall() with rg.cursor() as c: c.executemany( "INSERT INTO disc_cache (release_id, title, artist, thumb, weight) " "VALUES (%s,%s,%s,%s,%s) ON CONFLICT (release_id) DO NOTHING", rows) n += len(rows) rg.commit() print(f"disc_cache: {n} of {len(ids)} stocked releases cached") if __name__ == "__main__": main()