From 10af3464aca2d0c106ea46ebb383a5dc0ef6e3aa Mon Sep 17 00:00:00 2001 From: type-two Date: Mon, 22 Jun 2026 21:19:47 +1000 Subject: [PATCH] feat(disc): mirror WowPlatter Discogs metadata into recordgod (disc_sync.py) + locator matches on collection genre/style ids - disc_sync.py: MariaDB clone -> recordgod disc_* (30.8k releases, genre/style/label/format/identifier/artist/track/master), keeping WowPlatter's own genre/style ids - locator reads disc_release.genre_ids/style_ids + parses collection 'ID,0' id format - images stay with DealGod; masters sparse (backfill from Ultra later) Co-Authored-By: Claude Opus 4.8 --- app/collections_routes.py | 19 +++- disc_sync.py | 182 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 disc_sync.py diff --git a/app/collections_routes.py b/app/collections_routes.py index 2cf1ada..48e5dc4 100644 --- a/app/collections_routes.py +++ b/app/collections_routes.py @@ -23,6 +23,17 @@ def _arr(v): return [] +def _ids(v): + """Collection genre_ids/style_ids are stored as "ID,0" strings (WowPlatter's own ids) — take the lead int.""" + out = [] + for x in _arr(v): + try: + out.append(int(str(x).split(",")[0])) + except Exception: + pass + return out + + class CollectionIn(BaseModel): id: int | None = None name: str @@ -146,12 +157,12 @@ async def locate(release_id: int | None = Query(None), sku: str | None = Query(N # fallback: match the record against active collections by priority. genre/style needs the disc_cache # enrichment (added but backfilled separately) — until then a style-ruled collection won't match. meta = (await db.execute(text( - "SELECT year, genre_ids, style_ids FROM disc_cache WHERE release_id=:r" + "SELECT year, genre_ids, style_ids FROM disc_release WHERE id=:r" ), {"r": inv.get("release_id")})).mappings().first() if inv.get("release_id") else None price = float(inv["price"]) if inv.get("price") is not None else None year = meta["year"] if meta else None - rg = set(meta["genre_ids"] or []) if meta and "genre_ids" in meta.keys() else set() - rs = set(meta["style_ids"] or []) if meta and "style_ids" in meta.keys() else set() + rg = set(meta["genre_ids"] or []) if meta else set() # the record's genre ids (WowPlatter id space) + rs = set(meta["style_ids"] or []) if meta else set() suggestion = None cols = (await db.execute(text(""" @@ -169,7 +180,7 @@ async def locate(release_id: int | None = Query(None), sku: str | None = Query(N continue if c["year_max"] is not None and (year is None or year > c["year_max"]): continue - cgen, csty = set(_arr(c["genre_ids"])), set(_arr(c["style_ids"])) + cgen, csty = set(_ids(c["genre_ids"])), set(_ids(c["style_ids"])) if cgen and not (rg & cgen): continue if csty and not (rs & csty): diff --git a/disc_sync.py b/disc_sync.py new file mode 100644 index 0000000..75844e2 --- /dev/null +++ b/disc_sync.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""disc_sync.py — mirror the WowPlatter Discogs metadata clone into RecordGod (Postgres). + +Reads the MariaDB clone (the ~30.8k releases the shop stocks) and emits a Postgres load +(schema + COPY data) on stdout. Pipe it into recordgod-db: + + python disc_sync.py | gzip | ssh root@ 'gunzip | docker exec -i recordgod-db psql -U recordgod -d recordgod -q' + +Convention (per John, 2026-06-22): + • genre/style ids are WowPlatter's OWN incremental ids — Discogs has no genre/style tables. + They're copied FAITHFULLY because the collections (filing rules) reference them. Names ride along + for display only. (Collections store them as "ID,0" strings — the LOCATOR parses that, not this.) + • release_id / artist_id / label_id / master_id ARE real Discogs ids — the stable join keys. + • Augmented WowPlatter columns (beatport/apple/wiki/etc.) are dropped here; we keep what RecordGod + needs for locator + navigator + display + barcode. Images come from DealGod (not stored here). + • Masters are barely populated in WowPlatter (72) — copied as-is; backfill from the Ultra later. +""" +import pathlib +import re +import sys + +import pymysql + +WP_CONFIG = "/opt/homebrew/var/www/monsterrobot.localsite/wp-config.php" +SOCK = "/opt/homebrew/var/mysql/mysql.sock" + +SCHEMA = """ +SET synchronous_commit = off; +DROP TABLE IF EXISTS disc_release, disc_release_genre, disc_release_style, disc_release_label, + disc_release_format, disc_release_identifier, disc_release_artist, disc_release_track, + disc_genre, disc_style, disc_label, disc_artist, disc_master, disc_master_genre, disc_master_style CASCADE; +CREATE TABLE disc_genre (id int PRIMARY KEY, name text); +CREATE TABLE disc_style (id int PRIMARY KEY, parent_genre_id int, name text); +CREATE TABLE disc_label (id bigint PRIMARY KEY, name text, profile text); +CREATE TABLE disc_artist (id bigint PRIMARY KEY, name text, realname text); +CREATE TABLE disc_master (id bigint PRIMARY KEY, title text, year int, main_release bigint); +CREATE TABLE disc_release ( + id bigint PRIMARY KEY, title text, artists_sort text, released text, country text, year int, + notes text, format_quantity int, description text, apple_id text, estimated_weight numeric, + master_id bigint, thumb text, rating_count int, rating_average numeric, search_text text, + genre_ids int[], style_ids int[]); +CREATE TABLE disc_release_genre (release_id bigint, genre_id int, genre_name text); +CREATE TABLE disc_release_style (release_id bigint, style_id int, style_name text); +CREATE TABLE disc_release_label (release_id bigint, label_id bigint, label_name text, catno text); +CREATE TABLE disc_release_format (release_id bigint, name text, qty int, descriptions text); +CREATE TABLE disc_release_identifier (release_id bigint, type text, value text); +CREATE TABLE disc_release_artist (release_id bigint, artist_id bigint, artist_name text, role text, join_string text, position text); +CREATE TABLE disc_master_genre (master_id bigint, genre_id int, genre_name text); +CREATE TABLE disc_master_style (master_id bigint, style_id int, style_name text); +CREATE TABLE disc_release_track (release_id bigint, sequence int, position text, title text, duration text, track_id text, apple_id text, bpm numeric, musical_key text); +""" + +INDEXES = """ +CREATE INDEX ON disc_release USING gin (genre_ids); +CREATE INDEX ON disc_release USING gin (style_ids); +CREATE INDEX ON disc_release (master_id); +CREATE INDEX ON disc_release_genre (release_id); +CREATE INDEX ON disc_release_style (release_id); +CREATE INDEX ON disc_release_label (release_id); +CREATE INDEX ON disc_release_identifier (value); +CREATE INDEX ON disc_release_artist (release_id); +CREATE INDEX ON disc_release_artist (artist_id); +CREATE INDEX ON disc_release_track (release_id); +ANALYZE disc_release; +""" + + +def creds(): + t = pathlib.Path(WP_CONFIG).read_text() + g = lambda k: re.search(rf"'{k}',\s*'([^']*)'", t).group(1) + return g("DB_NAME"), g("DB_USER"), g("DB_PASSWORD") + + +def cp(v): + """Format one value for COPY text format.""" + if v is None or v == "": + return r"\N" + return (str(v).replace("\\", "\\\\").replace("\t", "\\t") + .replace("\n", "\\n").replace("\r", "\\r")) + + +def iarr(ids): + """int[] COPY literal — {2,6} — or NULL. Skips any null ids in the list.""" + vals = [str(int(i)) for i in (ids or []) if i is not None] + return "{" + ",".join(vals) + "}" if vals else r"\N" + + +def copy_block(out, table, cols, rows, fmt): + out.write(f"COPY {table} ({','.join(cols)}) FROM stdin;\n") + for r in rows: + out.write("\t".join(fmt(r)) + "\n") + out.write("\\.\n\n") + print(f" {table:26} {len(rows):>7,}", file=sys.stderr) + + +def main(): + name, user, pw = creds() + my = pymysql.connect(unix_socket=SOCK, user=user, password=pw, database=name, + cursorclass=pymysql.cursors.DictCursor) + c = my.cursor() + out = sys.stdout + out.write(SCHEMA) + P = "wp_rmp_disc_" + print("emitting:", file=sys.stderr) + + # genre / style lookups (WowPlatter's own ids) + c.execute(f"SELECT id, name FROM {P}genre") + copy_block(out, "disc_genre", ["id", "name"], c.fetchall(), lambda r: [cp(r["id"]), cp(r["name"])]) + c.execute(f"SELECT id, parent_genre_id, name FROM {P}style") + copy_block(out, "disc_style", ["id", "parent_genre_id", "name"], c.fetchall(), + lambda r: [cp(r["id"]), cp(r["parent_genre_id"]), cp(r["name"])]) + + # per-release genre/style id aggregation (for the fast-match arrays on disc_release) + rg, rs = {}, {} + c.execute(f"SELECT release_id, genre_id FROM {P}release_genre") + for r in c.fetchall(): + rg.setdefault(r["release_id"], []).append(r["genre_id"]) + c.execute(f"SELECT release_id, style_id FROM {P}release_style") + for r in c.fetchall(): + rs.setdefault(r["release_id"], []).append(r["style_id"]) + + # releases + c.execute(f"""SELECT id, title, artists_sort, released, country, year, notes, format_quantity, + description, apple_id, estimated_weight, master_id, thumb, rating_count, + rating_average, search_text FROM {P}release""") + cols = ["id", "title", "artists_sort", "released", "country", "year", "notes", "format_quantity", + "description", "apple_id", "estimated_weight", "master_id", "thumb", "rating_count", + "rating_average", "search_text", "genre_ids", "style_ids"] + copy_block(out, "disc_release", cols, c.fetchall(), + lambda r: [cp(r[k]) for k in cols[:-2]] + [iarr(rg.get(r["id"])), iarr(rs.get(r["id"]))]) + + # join + lookup tables + c.execute(f"SELECT release_id, genre_id, genre_name FROM {P}release_genre") + copy_block(out, "disc_release_genre", ["release_id", "genre_id", "genre_name"], c.fetchall(), + lambda r: [cp(r["release_id"]), cp(r["genre_id"]), cp(r["genre_name"])]) + c.execute(f"SELECT release_id, style_id, style_name FROM {P}release_style") + copy_block(out, "disc_release_style", ["release_id", "style_id", "style_name"], c.fetchall(), + lambda r: [cp(r["release_id"]), cp(r["style_id"]), cp(r["style_name"])]) + c.execute(f"SELECT release_id, label_id, label_name, catno FROM {P}release_label") + copy_block(out, "disc_release_label", ["release_id", "label_id", "label_name", "catno"], c.fetchall(), + lambda r: [cp(r["release_id"]), cp(r["label_id"]), cp(r["label_name"]), cp(r["catno"])]) + c.execute(f"SELECT id, name, profile FROM {P}label") + copy_block(out, "disc_label", ["id", "name", "profile"], c.fetchall(), + lambda r: [cp(r["id"]), cp(r["name"]), cp(r["profile"])]) + c.execute(f"SELECT release_id, name, qty, descriptions FROM {P}release_format") + copy_block(out, "disc_release_format", ["release_id", "name", "qty", "descriptions"], c.fetchall(), + lambda r: [cp(r["release_id"]), cp(r["name"]), cp(r["qty"]), cp(r["descriptions"])]) + c.execute(f"SELECT release_id, type, value FROM {P}release_identifier") + copy_block(out, "disc_release_identifier", ["release_id", "type", "value"], c.fetchall(), + lambda r: [cp(r["release_id"]), cp(r["type"]), cp(r["value"])]) + c.execute(f"SELECT release_id, artist_id, artist_name, role, join_string, position FROM {P}release_artist") + copy_block(out, "disc_release_artist", + ["release_id", "artist_id", "artist_name", "role", "join_string", "position"], c.fetchall(), + lambda r: [cp(r["release_id"]), cp(r["artist_id"]), cp(r["artist_name"]), + cp(r["role"]), cp(r["join_string"]), cp(r["position"])]) + # only artists referenced by our releases (the full artist table is 226k) + c.execute(f"SELECT id, name, realname FROM {P}artist WHERE id IN (SELECT DISTINCT artist_id FROM {P}release_artist)") + copy_block(out, "disc_artist", ["id", "name", "realname"], c.fetchall(), + lambda r: [cp(r["id"]), cp(r["name"]), cp(r["realname"])]) + c.execute(f"SELECT release_id, sequence, position, title, duration, track_id, apple_id, bpm, musical_key FROM {P}release_track") + copy_block(out, "disc_release_track", + ["release_id", "sequence", "position", "title", "duration", "track_id", "apple_id", "bpm", "musical_key"], + c.fetchall(), + lambda r: [cp(r["release_id"]), cp(r["sequence"]), cp(r["position"]), cp(r["title"]), + cp(r["duration"]), cp(r["track_id"]), cp(r["apple_id"]), cp(r["bpm"]), cp(r["musical_key"])]) + # masters (sparse — copy what exists) + c.execute(f"SELECT id, title, year, main_release FROM {P}master") + copy_block(out, "disc_master", ["id", "title", "year", "main_release"], c.fetchall(), + lambda r: [cp(r["id"]), cp(r["title"]), cp(r["year"]), cp(r["main_release"])]) + c.execute(f"SELECT master_id, genre_id, genre_name FROM {P}master_genre") + copy_block(out, "disc_master_genre", ["master_id", "genre_id", "genre_name"], c.fetchall(), + lambda r: [cp(r["master_id"]), cp(r["genre_id"]), cp(r["genre_name"])]) + c.execute(f"SELECT master_id, style_id, style_name FROM {P}master_style") + copy_block(out, "disc_master_style", ["master_id", "style_id", "style_name"], c.fetchall(), + lambda r: [cp(r["master_id"]), cp(r["style_id"]), cp(r["style_name"])]) + + out.write(INDEXES) + print("done.", file=sys.stderr) + + +if __name__ == "__main__": + main()