feat(disc): mirror WowPlatter Discogs metadata into recordgod (disc_sync.py) + locator matches on collection genre/style ids
- disc_sync.py: MariaDB clone -> recordgod disc_* (30.8k releases, genre/style/label/format/identifier/artist/track/master), keeping WowPlatter's own genre/style ids - locator reads disc_release.genre_ids/style_ids + parses collection 'ID,0' id format - images stay with DealGod; masters sparse (backfill from Ultra later) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
type-two
parent
a1b4538954
commit
10af3464ac
@ -23,6 +23,17 @@ def _arr(v):
|
||||
return []
|
||||
|
||||
|
||||
def _ids(v):
|
||||
"""Collection genre_ids/style_ids are stored as "ID,0" strings (WowPlatter's own ids) — take the lead int."""
|
||||
out = []
|
||||
for x in _arr(v):
|
||||
try:
|
||||
out.append(int(str(x).split(",")[0]))
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
class CollectionIn(BaseModel):
|
||||
id: int | None = None
|
||||
name: str
|
||||
@ -146,12 +157,12 @@ async def locate(release_id: int | None = Query(None), sku: str | None = Query(N
|
||||
# fallback: match the record against active collections by priority. genre/style needs the disc_cache
|
||||
# enrichment (added but backfilled separately) — until then a style-ruled collection won't match.
|
||||
meta = (await db.execute(text(
|
||||
"SELECT year, genre_ids, style_ids FROM disc_cache WHERE release_id=:r"
|
||||
"SELECT year, genre_ids, style_ids FROM disc_release WHERE id=:r"
|
||||
), {"r": inv.get("release_id")})).mappings().first() if inv.get("release_id") else None
|
||||
price = float(inv["price"]) if inv.get("price") is not None else None
|
||||
year = meta["year"] if meta else None
|
||||
rg = set(meta["genre_ids"] or []) if meta and "genre_ids" in meta.keys() else set()
|
||||
rs = set(meta["style_ids"] or []) if meta and "style_ids" in meta.keys() else set()
|
||||
rg = set(meta["genre_ids"] or []) if meta else set() # the record's genre ids (WowPlatter id space)
|
||||
rs = set(meta["style_ids"] or []) if meta else set()
|
||||
|
||||
suggestion = None
|
||||
cols = (await db.execute(text("""
|
||||
@ -169,7 +180,7 @@ async def locate(release_id: int | None = Query(None), sku: str | None = Query(N
|
||||
continue
|
||||
if c["year_max"] is not None and (year is None or year > c["year_max"]):
|
||||
continue
|
||||
cgen, csty = set(_arr(c["genre_ids"])), set(_arr(c["style_ids"]))
|
||||
cgen, csty = set(_ids(c["genre_ids"])), set(_ids(c["style_ids"]))
|
||||
if cgen and not (rg & cgen):
|
||||
continue
|
||||
if csty and not (rs & csty):
|
||||
|
||||
182
disc_sync.py
Normal file
182
disc_sync.py
Normal file
@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""disc_sync.py — mirror the WowPlatter Discogs metadata clone into RecordGod (Postgres).
|
||||
|
||||
Reads the MariaDB clone (the ~30.8k releases the shop stocks) and emits a Postgres load
|
||||
(schema + COPY data) on stdout. Pipe it into recordgod-db:
|
||||
|
||||
python disc_sync.py | gzip | ssh root@<vps> 'gunzip | docker exec -i recordgod-db psql -U recordgod -d recordgod -q'
|
||||
|
||||
Convention (per John, 2026-06-22):
|
||||
• genre/style ids are WowPlatter's OWN incremental ids — Discogs has no genre/style tables.
|
||||
They're copied FAITHFULLY because the collections (filing rules) reference them. Names ride along
|
||||
for display only. (Collections store them as "ID,0" strings — the LOCATOR parses that, not this.)
|
||||
• release_id / artist_id / label_id / master_id ARE real Discogs ids — the stable join keys.
|
||||
• Augmented WowPlatter columns (beatport/apple/wiki/etc.) are dropped here; we keep what RecordGod
|
||||
needs for locator + navigator + display + barcode. Images come from DealGod (not stored here).
|
||||
• Masters are barely populated in WowPlatter (72) — copied as-is; backfill from the Ultra later.
|
||||
"""
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
import pymysql
|
||||
|
||||
WP_CONFIG = "/opt/homebrew/var/www/monsterrobot.localsite/wp-config.php"
|
||||
SOCK = "/opt/homebrew/var/mysql/mysql.sock"
|
||||
|
||||
SCHEMA = """
|
||||
SET synchronous_commit = off;
|
||||
DROP TABLE IF EXISTS disc_release, disc_release_genre, disc_release_style, disc_release_label,
|
||||
disc_release_format, disc_release_identifier, disc_release_artist, disc_release_track,
|
||||
disc_genre, disc_style, disc_label, disc_artist, disc_master, disc_master_genre, disc_master_style CASCADE;
|
||||
CREATE TABLE disc_genre (id int PRIMARY KEY, name text);
|
||||
CREATE TABLE disc_style (id int PRIMARY KEY, parent_genre_id int, name text);
|
||||
CREATE TABLE disc_label (id bigint PRIMARY KEY, name text, profile text);
|
||||
CREATE TABLE disc_artist (id bigint PRIMARY KEY, name text, realname text);
|
||||
CREATE TABLE disc_master (id bigint PRIMARY KEY, title text, year int, main_release bigint);
|
||||
CREATE TABLE disc_release (
|
||||
id bigint PRIMARY KEY, title text, artists_sort text, released text, country text, year int,
|
||||
notes text, format_quantity int, description text, apple_id text, estimated_weight numeric,
|
||||
master_id bigint, thumb text, rating_count int, rating_average numeric, search_text text,
|
||||
genre_ids int[], style_ids int[]);
|
||||
CREATE TABLE disc_release_genre (release_id bigint, genre_id int, genre_name text);
|
||||
CREATE TABLE disc_release_style (release_id bigint, style_id int, style_name text);
|
||||
CREATE TABLE disc_release_label (release_id bigint, label_id bigint, label_name text, catno text);
|
||||
CREATE TABLE disc_release_format (release_id bigint, name text, qty int, descriptions text);
|
||||
CREATE TABLE disc_release_identifier (release_id bigint, type text, value text);
|
||||
CREATE TABLE disc_release_artist (release_id bigint, artist_id bigint, artist_name text, role text, join_string text, position text);
|
||||
CREATE TABLE disc_master_genre (master_id bigint, genre_id int, genre_name text);
|
||||
CREATE TABLE disc_master_style (master_id bigint, style_id int, style_name text);
|
||||
CREATE TABLE disc_release_track (release_id bigint, sequence int, position text, title text, duration text, track_id text, apple_id text, bpm numeric, musical_key text);
|
||||
"""
|
||||
|
||||
INDEXES = """
|
||||
CREATE INDEX ON disc_release USING gin (genre_ids);
|
||||
CREATE INDEX ON disc_release USING gin (style_ids);
|
||||
CREATE INDEX ON disc_release (master_id);
|
||||
CREATE INDEX ON disc_release_genre (release_id);
|
||||
CREATE INDEX ON disc_release_style (release_id);
|
||||
CREATE INDEX ON disc_release_label (release_id);
|
||||
CREATE INDEX ON disc_release_identifier (value);
|
||||
CREATE INDEX ON disc_release_artist (release_id);
|
||||
CREATE INDEX ON disc_release_artist (artist_id);
|
||||
CREATE INDEX ON disc_release_track (release_id);
|
||||
ANALYZE disc_release;
|
||||
"""
|
||||
|
||||
|
||||
def creds():
|
||||
t = pathlib.Path(WP_CONFIG).read_text()
|
||||
g = lambda k: re.search(rf"'{k}',\s*'([^']*)'", t).group(1)
|
||||
return g("DB_NAME"), g("DB_USER"), g("DB_PASSWORD")
|
||||
|
||||
|
||||
def cp(v):
|
||||
"""Format one value for COPY text format."""
|
||||
if v is None or v == "":
|
||||
return r"\N"
|
||||
return (str(v).replace("\\", "\\\\").replace("\t", "\\t")
|
||||
.replace("\n", "\\n").replace("\r", "\\r"))
|
||||
|
||||
|
||||
def iarr(ids):
|
||||
"""int[] COPY literal — {2,6} — or NULL. Skips any null ids in the list."""
|
||||
vals = [str(int(i)) for i in (ids or []) if i is not None]
|
||||
return "{" + ",".join(vals) + "}" if vals else r"\N"
|
||||
|
||||
|
||||
def copy_block(out, table, cols, rows, fmt):
|
||||
out.write(f"COPY {table} ({','.join(cols)}) FROM stdin;\n")
|
||||
for r in rows:
|
||||
out.write("\t".join(fmt(r)) + "\n")
|
||||
out.write("\\.\n\n")
|
||||
print(f" {table:26} {len(rows):>7,}", file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
name, user, pw = creds()
|
||||
my = pymysql.connect(unix_socket=SOCK, user=user, password=pw, database=name,
|
||||
cursorclass=pymysql.cursors.DictCursor)
|
||||
c = my.cursor()
|
||||
out = sys.stdout
|
||||
out.write(SCHEMA)
|
||||
P = "wp_rmp_disc_"
|
||||
print("emitting:", file=sys.stderr)
|
||||
|
||||
# genre / style lookups (WowPlatter's own ids)
|
||||
c.execute(f"SELECT id, name FROM {P}genre")
|
||||
copy_block(out, "disc_genre", ["id", "name"], c.fetchall(), lambda r: [cp(r["id"]), cp(r["name"])])
|
||||
c.execute(f"SELECT id, parent_genre_id, name FROM {P}style")
|
||||
copy_block(out, "disc_style", ["id", "parent_genre_id", "name"], c.fetchall(),
|
||||
lambda r: [cp(r["id"]), cp(r["parent_genre_id"]), cp(r["name"])])
|
||||
|
||||
# per-release genre/style id aggregation (for the fast-match arrays on disc_release)
|
||||
rg, rs = {}, {}
|
||||
c.execute(f"SELECT release_id, genre_id FROM {P}release_genre")
|
||||
for r in c.fetchall():
|
||||
rg.setdefault(r["release_id"], []).append(r["genre_id"])
|
||||
c.execute(f"SELECT release_id, style_id FROM {P}release_style")
|
||||
for r in c.fetchall():
|
||||
rs.setdefault(r["release_id"], []).append(r["style_id"])
|
||||
|
||||
# releases
|
||||
c.execute(f"""SELECT id, title, artists_sort, released, country, year, notes, format_quantity,
|
||||
description, apple_id, estimated_weight, master_id, thumb, rating_count,
|
||||
rating_average, search_text FROM {P}release""")
|
||||
cols = ["id", "title", "artists_sort", "released", "country", "year", "notes", "format_quantity",
|
||||
"description", "apple_id", "estimated_weight", "master_id", "thumb", "rating_count",
|
||||
"rating_average", "search_text", "genre_ids", "style_ids"]
|
||||
copy_block(out, "disc_release", cols, c.fetchall(),
|
||||
lambda r: [cp(r[k]) for k in cols[:-2]] + [iarr(rg.get(r["id"])), iarr(rs.get(r["id"]))])
|
||||
|
||||
# join + lookup tables
|
||||
c.execute(f"SELECT release_id, genre_id, genre_name FROM {P}release_genre")
|
||||
copy_block(out, "disc_release_genre", ["release_id", "genre_id", "genre_name"], c.fetchall(),
|
||||
lambda r: [cp(r["release_id"]), cp(r["genre_id"]), cp(r["genre_name"])])
|
||||
c.execute(f"SELECT release_id, style_id, style_name FROM {P}release_style")
|
||||
copy_block(out, "disc_release_style", ["release_id", "style_id", "style_name"], c.fetchall(),
|
||||
lambda r: [cp(r["release_id"]), cp(r["style_id"]), cp(r["style_name"])])
|
||||
c.execute(f"SELECT release_id, label_id, label_name, catno FROM {P}release_label")
|
||||
copy_block(out, "disc_release_label", ["release_id", "label_id", "label_name", "catno"], c.fetchall(),
|
||||
lambda r: [cp(r["release_id"]), cp(r["label_id"]), cp(r["label_name"]), cp(r["catno"])])
|
||||
c.execute(f"SELECT id, name, profile FROM {P}label")
|
||||
copy_block(out, "disc_label", ["id", "name", "profile"], c.fetchall(),
|
||||
lambda r: [cp(r["id"]), cp(r["name"]), cp(r["profile"])])
|
||||
c.execute(f"SELECT release_id, name, qty, descriptions FROM {P}release_format")
|
||||
copy_block(out, "disc_release_format", ["release_id", "name", "qty", "descriptions"], c.fetchall(),
|
||||
lambda r: [cp(r["release_id"]), cp(r["name"]), cp(r["qty"]), cp(r["descriptions"])])
|
||||
c.execute(f"SELECT release_id, type, value FROM {P}release_identifier")
|
||||
copy_block(out, "disc_release_identifier", ["release_id", "type", "value"], c.fetchall(),
|
||||
lambda r: [cp(r["release_id"]), cp(r["type"]), cp(r["value"])])
|
||||
c.execute(f"SELECT release_id, artist_id, artist_name, role, join_string, position FROM {P}release_artist")
|
||||
copy_block(out, "disc_release_artist",
|
||||
["release_id", "artist_id", "artist_name", "role", "join_string", "position"], c.fetchall(),
|
||||
lambda r: [cp(r["release_id"]), cp(r["artist_id"]), cp(r["artist_name"]),
|
||||
cp(r["role"]), cp(r["join_string"]), cp(r["position"])])
|
||||
# only artists referenced by our releases (the full artist table is 226k)
|
||||
c.execute(f"SELECT id, name, realname FROM {P}artist WHERE id IN (SELECT DISTINCT artist_id FROM {P}release_artist)")
|
||||
copy_block(out, "disc_artist", ["id", "name", "realname"], c.fetchall(),
|
||||
lambda r: [cp(r["id"]), cp(r["name"]), cp(r["realname"])])
|
||||
c.execute(f"SELECT release_id, sequence, position, title, duration, track_id, apple_id, bpm, musical_key FROM {P}release_track")
|
||||
copy_block(out, "disc_release_track",
|
||||
["release_id", "sequence", "position", "title", "duration", "track_id", "apple_id", "bpm", "musical_key"],
|
||||
c.fetchall(),
|
||||
lambda r: [cp(r["release_id"]), cp(r["sequence"]), cp(r["position"]), cp(r["title"]),
|
||||
cp(r["duration"]), cp(r["track_id"]), cp(r["apple_id"]), cp(r["bpm"]), cp(r["musical_key"])])
|
||||
# masters (sparse — copy what exists)
|
||||
c.execute(f"SELECT id, title, year, main_release FROM {P}master")
|
||||
copy_block(out, "disc_master", ["id", "title", "year", "main_release"], c.fetchall(),
|
||||
lambda r: [cp(r["id"]), cp(r["title"]), cp(r["year"]), cp(r["main_release"])])
|
||||
c.execute(f"SELECT master_id, genre_id, genre_name FROM {P}master_genre")
|
||||
copy_block(out, "disc_master_genre", ["master_id", "genre_id", "genre_name"], c.fetchall(),
|
||||
lambda r: [cp(r["master_id"]), cp(r["genre_id"]), cp(r["genre_name"])])
|
||||
c.execute(f"SELECT master_id, style_id, style_name FROM {P}master_style")
|
||||
copy_block(out, "disc_master_style", ["master_id", "style_id", "style_name"], c.fetchall(),
|
||||
lambda r: [cp(r["master_id"]), cp(r["style_id"]), cp(r["style_name"])])
|
||||
|
||||
out.write(INDEXES)
|
||||
print("done.", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user