#!/usr/bin/env python3 """Lane G — G2a: per-shop REAL stock atlases (tier 1) for `?stock=real`. THE FIRST REAL CRATE. One real shop's real crate — real sleeves, real titles, real prices — baked to a static atlas keyed by godverse shop id. No server, no runtime query: tier 1 is files. Two stages, split at THE SNAPSHOT BOUNDARY (the charter's determinism rule, and the same discipline build_towns.py applies to Overpass raw): snapshot POS DB + image map -> pipeline/_godverse/_.snapshot.json pack snapshot + images -> web/assets/stock_godverse//stock_record_*.{json,webp} The snapshot is the artifact of record: it runs where the databases are, pins every source row AND every cover's sha256, and is committed. The pack stage is pure — same snapshot + same image bytes => byte-identical atlas, on any machine, with no DB. That split is what lets Lane E's validate_atlas gate G's output on m3 without the dealgod DB. python3 pipeline/godverse_stock.py snapshot --crate 550 --image-map ~/Documents/MODELBEAST/venvs/mflux/bin/python pipeline/godverse_stock.py pack \ pipeline/_godverse/monsterrobot_550.snapshot.json # needs PIL (house convention) THE SHOP (John's ruling, R23): `monsterrobot` — a real, trading record shop at 147 Musgrave Road, Red Hill, Brisbane QLD. It is the same shop in three datasets, which is what makes tier 1 honest: recordgod its POS: real crates, real slot numbers, real prices, real grading monsterrobot.party its own website — the covers are the SHOP'S OWN product shots dealgod store 3962749 the scrape of that site; the cover cache + the godverse shop id It is NOT in thriftgod's OSM census (see G-progress R23), so its godverse id is its dealgod store id 3962749 — real, stable, and clear of the census id space (thriftgod max shop id = 2992). DETERMINISM: ordered queries, no RNG, no sampling, no clock in the output. Prices/titles/artists are FACTS copied from the POS, not minted (thriftgod's mint() is TABLESAMPLE + unseeded random — it cannot produce tier 1; see G3_ECONOMY_DESIGN.md §2). LICENCE (charter law #3): covers are the shop's own product shots from the shop's own domain, cached by dealgod. In-house 🟢. FLAG BEFORE ANY PUBLIC/COMMERCIAL RELEASE. Real titles/artists ship at tier 1 per John's R23 amendment to the no-real-trademarks law (in-house green, same flag). Provenance rides in every index. PII FENCE: recordgod holds real customer/staff/sales tables — this pipeline reads ONLY inventory/crate/disc_cache and must never read the others. """ import argparse, hashlib, json, os, subprocess, sys ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SNAP_DIR = os.path.join(ROOT, "pipeline", "_godverse") STOCK_DIR = os.path.join(ROOT, "web", "assets", "stock_godverse") IMG_CACHE = os.environ.get("DEALGOD_IMG_CACHE", "/Volumes/johnking/dealgod_backup/images") # The shop, per John's R23 ruling. `godverse_id` = dealgod store id (see module docstring). SHOP = { "godverse_id": 3962749, "slug": "monsterrobot", "name": "Monster Robot Party", "address": "147 Musgrave Road", "suburb": "Red Hill", "city": "Brisbane", "state": "QLD", "country": "AU", "type": "record", } OWN_PHOTO_HOST = "monsterrobot.party" # the licence test: the shop's own domain, nothing else CELL_W, CELL_H, ATLAS_PX = 256, 256, 2048 # Lane E's record contract (build_stock_pack.py) STOCK_TYPES = ("record", "book", "toy") # C §7.1 — the only types the loader can fetch def price_band(price): """Lane E's record bands, verbatim — C's dig UI reads this (grail => `rare`).""" p = price if price is not None else 4 return "bargain" if p < 8 else "standard" if p < 25 else "collector" if p < 60 else "grail" def psql(db, sql): """Read-only query -> list of field-lists. psql CLI, not psycopg2: the m3 has no psycopg2, and a snapshot stage that shells out has one less dep between the DB and the frozen file.""" out = subprocess.run(["psql", "-d", db, "-tAF\t", "-c", sql], capture_output=True, text=True, check=True).stdout return [ln.split("\t") for ln in out.splitlines() if ln.strip()] def _n(v): return None if v in ("", "\\N") else v # ── stage 1: snapshot ──────────────────────────────────────────────────────────────────── def load_image_map(path): """release_id -> local_image, restricted to the SHOP'S OWN photos (the licence test). On ultra (dealgod present) this is a query. On the m3 dealgod is a 2 GB dump and never restored, so the map is streamed out of it once — the command is recorded in the snapshot's provenance so the extraction is reproducible: gzcat dealgod-.sql.gz | awk -F'\\t' '/^COPY public.products \\(/{c=1;next} c && /^\\\\.$/{c=0;next} c && $50!="\\\\N"{print $50"\\t"$5"\\t"$34"\\t"$52"\\t"$56}' (fields: 50=release_id 5=image_url 34=local_image 52=current_price 56=discogs_artist_name) """ m = {} with open(path) as fh: for ln in fh: f = ln.rstrip("\n").split("\t") if len(f) < 3: continue rel, url, local = f[0], f[1], f[2] if OWN_PHOTO_HOST not in url or not local.startswith("/img/"): continue m.setdefault(rel, local) # first wins; ids are sorted upstream => stable return m def crate_menu(): """The G3 §4 crate-rotation menu, baked statically — the shop's real crates, with real labels. NOTE (→ C, R26): this is METADATA, not stock, and it cannot be otherwise under the contract as written. The loader fetches ONE index per base (`stock__index.json`) and C §7.6 caps a shop at 2 atlases = 128 items. Monster Robot has 357 crates / 24,646 baked-able records. So "the crate menu, baked statically" can ship as the shop's real crate LIST (below) — what it cannot ship as is per-crate stock, which needs a per-crate base (`…/3962749/crate_550/`) and therefore a C contract line. Filed, not invented: the menu is honest metadata today, and tier 2 is where rotation actually lives (G3 §4 `/crates`). """ rows = psql("recordgod", """ SELECT c.id, c.name, COALESCE(c.label_text,''), count(i.sku) FROM crate c JOIN inventory i ON i.crate_id = c.id AND i.in_stock GROUP BY c.id, c.name, c.label_text HAVING count(i.sku) > 0 ORDER BY c.id""") return [{"id": int(r[0]), "name": r[1], "label": r[2], "count": int(r[3])} for r in rows] # ── the MINT baker (R26 ledger #2) ─────────────────────────────────────────────────────── # The G3 doc's own note, promoted: a keyed shop with NO POS gets a deterministic seeded sample of # real dealgod listings, so its crate is distinct and plausible — never claimed as its stock. # # DETERMINISM: `random.Random(shop_id)` over an ID-ORDERED pool. Same shop_id + same pool => # byte-identical bake, forever. No TABLESAMPLE, no unseeded random — the two things that # disqualified thriftgod's own mint() from tier 1 (G3 §2). The shop_id IS the seed, so two shops # can never draw the same crate, which is what F's distinctness gate asserts. # # HONESTY: the snapshot is marked `sourcing: "mint"` and carries NO POS-claim field — no `crate`, # no `pos_db`, no `pos_dump` (E's POS_CLAIM_FIELDS). A mint crate is structurally incapable of # claiming a shop's real inventory: the fields that would say so are absent, not merely false. MINT_KINDS = { # pool row -> the stock type it can furnish "record": lambda r: r["release_id"] != "\\N", "book": lambda r: r["cat"].split("/")[0] == "books", "toy": lambda r: r["cat"].split("/")[0] in ("collectables-hobbies-toys", "gaming"), } MINT_COUNT = 16 # ≤16 items => ONE 1024² atlas (C §7.6: ≤1024² when it fits; # 1024²/256² = 16 cells). See G-progress R26 on why not more. def load_pool(path): """The dealgod mint pool — a frozen projection of the same dated dump the real tier uses.""" pool = [] with open(path) as fh: for ln in fh: f = ln.rstrip("\n").split("\t") if len(f) < 9: continue pool.append({"id": f[0], "title": f[1], "artist": f[2], "price": f[3], "cond": f[4], "cover": f[5], "ptype": f[6], "cat": f[7], "release_id": f[8]}) pool.sort(key=lambda r: int(r["id"])) # ID-ORDERED: the sample's determinism rests on this return pool def mint(args): import random pool = [r for r in load_pool(args.pool) if MINT_KINDS[args.type](r)] seen, uniq = set(), [] for r in pool: # de-dupe by cover so a crate never doubles a sleeve if r["cover"] in seen: continue seen.add(r["cover"]) uniq.append(r) rng = random.Random(args.shop) # THE SEED IS THE SHOP — deterministic, per-shop distinct picks, items, dropped = rng.sample(uniq, min(len(uniq), args.count * 4)), [], {"file_missing": 0} for r in picks: if len(items) >= args.count: break path = os.path.join(IMG_CACHE, r["cover"][len("/img/"):]) if not os.path.isfile(path): dropped["file_missing"] += 1 continue with open(path, "rb") as fh: sha = hashlib.sha256(fh.read()).hexdigest() try: price = float(r["price"]) except ValueError: continue items.append({ # C §7.2a: derived from the source's OWN stable key, never position. The source here is # dealgod, so the key is its listing id — NOT `sku_`. See G-progress R26: a POS sku and a # dealgod product id are DIFFERENT NAMESPACES, and tier-2's sold-means-gone will look up # POS skus. An id shaped like a sku that no POS can resolve is the R26 standing note # (plan ids vs godverse ids) waiting to happen in the economy. "mint_id": r["id"], "artist": r["artist"] if r["artist"] != "\\N" else "", "title": r["title"], "price": price, # §7.2c: emit `condition` ONLY if the listing genuinely carries one — an invented grade is # exactly the fiction `sourcing` exists to prevent. "condition": r["cond"] if r["cond"] not in ("\\N", "") else "", "cover": r["cover"], "cover_sha256": sha, }) items.sort(key=lambda i: int(i["mint_id"])) snap = { "version": 1, "kind": args.type, "tier": 1, "sourcing": "mint", "shop": {"godverse_id": args.shop, "slug": args.slug, "name": args.name, "suburb": args.suburb, "state": args.state, "type": args.type}, "counts": {"pool": len(uniq), "baked": len(items), "dropped": dropped}, "source": { # NO pos_db / pos_dump — a mint crate has no POS to claim "feed_db": "dealgod", "feed_dump": args.feed_dump, "image_cache": IMG_CACHE, "captured": args.captured, "seed": args.shop, "sample": f"random.Random({args.shop}) over an id-ordered pool of {len(uniq)} listings", }, "license": { "covers": "real dealgod listing photos (the sellers' own product shots), cached locally", "zone": "in-house-green", "flag": "FLAG BEFORE ANY PUBLIC/COMMERCIAL RELEASE", "metadata": "real titles/artists — John's R23 amendment to the no-real-trademarks law", "sourcing": "MINT — a seeded sample standing in for a shop with no POS. Plausible, and " "NEVER this shop's actual stock. Nothing here is a claim about this business.", }, "items": items, } os.makedirs(SNAP_DIR, exist_ok=True) dst = os.path.join(SNAP_DIR, f"mint_{args.shop}_{args.type}.snapshot.json") with open(dst, "w") as fh: json.dump(snap, fh, indent=1) fh.write("\n") print(f"mint: shop {args.shop} ({args.name}) — {len(items)} {args.type} items " f"seeded random.Random({args.shop}) from {len(uniq)} listings -> {dst}") def snapshot(args): crate = psql("recordgod", f"SELECT id,name,COALESCE(label_text,'') FROM crate WHERE id={args.crate}") if not crate: sys.exit(f"crate {args.crate} not found in recordgod") cid, cname, clabel = crate[0][0], crate[0][1], crate[0][2] # ONLY inventory/crate/disc_cache — the PII fence (see module docstring). rows = psql("recordgod", f""" SELECT i.sku, COALESCE(i.slot_number::text,''), i.release_id, COALESCE(d.artist,''), COALESCE(d.title, COALESCE(i.title,'')), i.price, COALESCE(i.condition,''), COALESCE(i.sleeve_cond,'') FROM inventory i JOIN crate c ON c.id = i.crate_id LEFT JOIN disc_cache d USING (release_id) WHERE i.in_stock AND c.id = {args.crate} AND i.release_id IS NOT NULL ORDER BY i.slot_number NULLS LAST, i.sku""") imgs = load_image_map(args.image_map) items, dropped = [], {"no_own_photo": 0, "file_missing": 0, "no_price": 0} for sku, slot, rel, artist, title, price, cond, sleeve in rows: local = imgs.get(rel) if not local: dropped["no_own_photo"] += 1 continue path = os.path.join(IMG_CACHE, local[len("/img/"):]) if not os.path.isfile(path): dropped["file_missing"] += 1 continue if not _n(price): dropped["no_price"] += 1 continue with open(path, "rb") as fh: sha = hashlib.sha256(fh.read()).hexdigest() items.append({"sku": sku, "slot": int(slot) if slot else None, "release_id": int(rel), "artist": artist, "title": title, "price": float(price), "condition": cond, "sleeve_cond": sleeve, "cover": local, "cover_sha256": sha}) if args.max_items: items = items[:args.max_items] snap = { "version": 1, "kind": "record", "tier": 1, "sourcing": "real", "shop": SHOP, "crate": {"id": int(cid), "name": cname, "label": clabel}, "crate_menu": crate_menu(), # G3 §4's menu, baked as metadata — see crate_menu() "counts": {"in_crate": len(rows), "baked": len(items), "dropped": dropped}, "source": { "pos_db": "recordgod", "pos_dump": args.pos_dump, "image_map_dump": args.image_map_dump, "image_cache": IMG_CACHE, "captured": args.captured, "extraction": "see load_image_map() docstring — the products projection command", }, # US `license` throughout (R24: British `licence` retired per C's contract §7.3; E's # validator reads `provenance.licence or provenance.license`, so US passes the gate). "license": { "covers": "the shop's own product shots (monsterrobot.party), cached by dealgod", "zone": "in-house-green", "flag": "FLAG BEFORE ANY PUBLIC/COMMERCIAL RELEASE", "metadata": "real titles/artists — John's R23 amendment to the no-real-trademarks law", "pii": "none — inventory/crate/disc_cache only; customer/staff/sales never read", }, "items": items, } os.makedirs(SNAP_DIR, exist_ok=True) dst = os.path.join(SNAP_DIR, f"{SHOP['slug']}_{cid}.snapshot.json") with open(dst, "w") as fh: json.dump(snap, fh, indent=1, sort_keys=False) fh.write("\n") print(f"snapshot: crate {cid} '{clabel or cname}' — {len(items)} real records " f"(of {len(rows)} in crate; dropped {dropped}) -> {dst}") # ── stage 2: pack (pure: snapshot + image bytes -> atlas) ───────────────────────────────── def pack(args): from PIL import Image snap = json.load(open(args.snapshot)) shop, items = snap["shop"], snap["items"] kind, sourcing = snap.get("kind", "record"), snap["sourcing"] out_dir = os.path.join(STOCK_DIR, str(shop["godverse_id"])) os.makedirs(out_dir, exist_ok=True) # C §7.6: ≤2048², and ≤1024² whenever the crate fits it. 1024²/256² = 16 cells; 2048² = 64. # A 16-item mint crate is ONE 1024² sheet; the 120-record real crate needs 2×2048² (C's own # worked example). Cell stays 256² either way — C: never shrink it, the dig renders it close-up. px = ATLAS_PX if len(items) > (1024 // CELL_H) ** 2 else 1024 grid, per_atlas = px // CELL_W, (px // CELL_W) * (px // CELL_H) idx_items, atlases = [], [] for ai in range((len(items) + per_atlas - 1) // per_atlas): chunk = items[ai * per_atlas:(ai + 1) * per_atlas] W = H = px sheet = Image.new("RGB", (W, H), (28, 28, 32)) aname = f"stock_{kind}_atlas_{ai:02d}.webp" for j, it in enumerate(chunk): gx, gy = j % grid, j // grid path = os.path.join(IMG_CACHE, it["cover"][len("/img/"):]) with open(path, "rb") as fh: raw = fh.read() if hashlib.sha256(raw).hexdigest() != it["cover_sha256"]: sys.exit(f"cover drift: {it['cover']} — snapshot sha mismatch, re-snapshot") import io cov = Image.open(io.BytesIO(raw)).convert("RGB").resize((CELL_W, CELL_H), Image.LANCZOS) sheet.paste(cov, (gx * CELL_W, gy * CELL_H)) u0, v0 = gx * CELL_W / W, gy * CELL_H / H # C §7.2a: derived from the source's own stable key, never position. `rec_NNNN` is the # generic packs' namespace and a real-stock pack must not emit into it. And position # isn't identity: sell one record and `rec_0050` becomes a DIFFERENT record next bake — # exactly what tier-2's sold-means-gone must never do. # real → `sku_` (the shop's own identity for that COPY) # mint → `mint_` (dealgod's own key; NOT a sku — see G-progress) slot_id = f"sku_{it['sku']}" if sourcing == "real" else f"mint_{it['mint_id']}" item = { "id": slot_id, "title": it["title"], "artist": it["artist"], "price": int(round(it["price"])), "price_band": price_band(it["price"]), "atlas": aname, "uv": [round(u0, 4), round(v0, 4), round(u0 + CELL_W / W, 4), round(v0 + CELL_H / H, 4)], } # C §7.2c: the price card is a record-shop sticker — artist/title/condition/price. # Emit only what the source actually has; C's card omits what's absent, and an invented # grade is the fiction `sourcing` exists to prevent. if it.get("condition"): item["condition"] = it["condition"] if it.get("sleeve_cond"): item["sleeve_cond"] = it["sleeve_cond"] idx_items.append(item) sheet.save(os.path.join(out_dir, aname), "WEBP", quality=88, method=6) atlases.append(aname) with open(args.snapshot, "rb") as fh: snap_sha = hashlib.sha256(fh.read()).hexdigest() lic = snap["license"] generator, snapshot_name = "pipeline/godverse_stock.py", os.path.basename(args.snapshot) if sourcing == "real": attribution = (f"Stock, prices + covers: {shop['name']} ({OWN_PHOTO_HOST}) — the shop's own " f"product shots · cover cache: GODVERSE/dealgod · POS: recordgod") else: # The attribution must not imply this shop supplied anything — it didn't. Say what it is. attribution = (f"MINT (seeded sample, not {shop['name']}'s stock): real dealgod listings — " f"the sellers' own product shots · cover cache: GODVERSE/dealgod") index = { "version": 1, "kind": kind, "tier": 1, # C §7.2b: `sourcing` is the real-vs-mint axis; `tier` stays the charter's ladder rung (both # are rung 1 — same mechanism, different origin). One name, one meaning. "sourcing": sourcing, "atlas_px": px, "cell_w": CELL_W, "cell_h": CELL_H, "count": len(idx_items), "atlases": atlases, # TOP LEVEL, per C's contract §7.3 + the house convention of the shipped town caches # (`license` is a short US-spelled string there, `attribution` its companion). "license": f"{lic['zone']} — {lic['covers']}; {lic['flag']}", "attribution": attribution, "generator": generator, "snapshot": snapshot_name, "shop": {k: shop[k] for k in ("godverse_id", "slug", "name", "suburb", "state")}, # NESTED — the rich block C calls colour; the four gate fields are top level above. "provenance": { "generator": generator, "source": snap["source"], "license": lic, "snapshot": snapshot_name, "snapshot_sha256": snap_sha, "determinism": ("pure bake — same snapshot + same cover bytes => byte-identical atlas; " "covers pinned by sha256, no RNG, no clock" if sourcing == "real" else f"pure bake; the sample itself is seeded random.Random({shop['godverse_id']}) " f"over an id-ordered pool => same shop id + same pool = same crate, forever"), }, "items": idx_items, } # C §7.2b / E's POS_CLAIM_FIELDS: a mint index carries NO field that claims a shop's real # inventory — `crate` and the pos_* source keys are ABSENT, not merely false. The separation is # structural: there is no line here that could make a mint crate read as real. if sourcing == "real": index["crate"] = snap["crate"] index["crate_menu"] = snap.get("crate_menu", []) dst = os.path.join(out_dir, f"stock_{kind}_index.json") with open(dst, "w") as fh: json.dump(index, fh, indent=1) fh.write("\n") where = (f"crate {snap['crate']['id']} '{snap['crate']['label'] or snap['crate']['name']}'" if sourcing == "real" else f"seeded random.Random({shop['godverse_id']})") print(f"pack[{sourcing}]: {len(idx_items)} {kind} items -> {len(atlases)}×{px}² atlas(es) " f"in {os.path.relpath(out_dir, ROOT)}") print(f" shop {shop['godverse_id']} ({shop['slug']}) · {where}") # ── the atlas manifest (R26 ledger #6, C §7.2d) ─────────────────────────────────────────── def manifest(args): """Existence DECLARED, never probed. F's wire consults this before fetching, so a shop with no atlas is never fetched — no 404, and the zero-console-errors law holds without the attribution exception that was tolerating 8 of them. Derived from the files on disk, so it cannot drift: the manifest IS the directory, read back.""" shops = [] for d in sorted(os.listdir(STOCK_DIR)) if os.path.isdir(STOCK_DIR) else []: sd = os.path.join(STOCK_DIR, d) if not (os.path.isdir(sd) and d.isdigit()): continue entry = {"godverseShopId": int(d), "types": [], "sourcing": None} for t in sorted(STOCK_TYPES): p = os.path.join(sd, f"stock_{t}_index.json") if not os.path.isfile(p): continue idx = json.load(open(p)) entry["types"].append(t) entry["sourcing"] = idx.get("sourcing") if entry["types"]: shops.append(entry) man = {"version": 1, "shops": shops} dst = os.path.join(STOCK_DIR, "index.json") with open(dst, "w") as fh: json.dump(man, fh, indent=1) fh.write("\n") real = sum(1 for s in shops if s["sourcing"] == "real") print(f"manifest: {len(shops)} shop(s) carry atlases ({real} real, {len(shops) - real} mint) " f"-> {os.path.relpath(dst, ROOT)}") ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) s1 = sub.add_parser("snapshot", help="POS + image map -> frozen snapshot (runs where the DBs are)") s1.add_argument("--crate", type=int, required=True) s1.add_argument("--image-map", required=True, help="TSV: release_id, image_url, local_image, ...") s1.add_argument("--max-items", type=int, default=0, help="cap (atlas budget — charter risk #2)") s1.add_argument("--pos-dump", default="", help="recordgod dump filename, for provenance") s1.add_argument("--image-map-dump", default="", help="dealgod dump filename, for provenance") s1.add_argument("--captured", default="", help="snapshot date (YYYY-MM-DD), for provenance") s1.set_defaults(func=snapshot) s2 = sub.add_parser("pack", help="snapshot -> atlas + index (pure; needs PIL, no DB)") s2.add_argument("snapshot") s2.set_defaults(func=pack) s3 = sub.add_parser("mint", help="seeded sample -> mint snapshot (a keyed shop with no POS)") s3.add_argument("--shop", type=int, required=True, help="godverseShopId — AND the seed") s3.add_argument("--type", required=True, choices=sorted(MINT_KINDS)) s3.add_argument("--pool", required=True, help="TSV projection of dealgod's listings") s3.add_argument("--count", type=int, default=MINT_COUNT) s3.add_argument("--slug", default="") s3.add_argument("--name", default="") s3.add_argument("--suburb", default="") s3.add_argument("--state", default="") s3.add_argument("--feed-dump", default="", help="dealgod dump filename, for provenance") s3.add_argument("--captured", default="") s3.set_defaults(func=mint) s4 = sub.add_parser("manifest", help="emit stock_godverse/index.json from the atlases on disk") s4.set_defaults(func=manifest) args = ap.parse_args() args.func(args)