feat(intake): Heal sweep — backfill missing catalog metadata (salvaged self-healing pass)
The WowPlatter self-heal idea, freed from its 3-hour import: one button scans in-stock items for
gaps (title/weight/cover) and backfills. Phase 1 = local backfill from the disc_cache mirror
(instant, unbounded UPDATE…FROM); Phase 2 = bounded Discogs re-fetch for release_ids missing from
the mirror or without cover art (rate-limited, re-run for the rest). Refactored _enrich → _fetch_release
so Heal can force-refetch (not just cache-miss). New /admin/intake/heal/{scan,run} + a 🩹 Heal admin view.
Live first run: 30,202 rows had no title/weight (migration never denormalized them → broke shipping
quotes); 30,193 healed instantly from the mirror, 239 needed Discogs (60/batch). Real win.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
type-two
parent
beabfe1297
commit
9a0bf41fbe
@ -71,6 +71,12 @@ async def _enrich(db, rid):
|
||||
hit = await _cache_hit(db, rid)
|
||||
if hit:
|
||||
return dict(hit)
|
||||
return await _fetch_release(db, rid)
|
||||
|
||||
|
||||
async def _fetch_release(db, rid):
|
||||
"""ALWAYS hit the Discogs API for this release and grow the mirror (release+artist+label+thumb),
|
||||
ignoring any existing cache row. Used by Heal to backfill missing cover art / metadata."""
|
||||
try:
|
||||
c, ok = await _client(db)
|
||||
async with c:
|
||||
@ -514,6 +520,82 @@ async def gsheet_import(ident=Depends(require_token), db=Depends(get_db)):
|
||||
return {"ok": True, "staged": staged, "skipped": skipped}
|
||||
|
||||
|
||||
# --- Heal: backfill missing catalog metadata (the salvaged self-healing pass) ----
|
||||
# WowPlatter healed gaps inline during a 3-hour import; on metal it's a bounded one-button sweep.
|
||||
# Phase 1 = LOCAL backfill from the mirror (free, instant). Phase 2 = bounded Discogs re-fetch for
|
||||
# release_ids missing from the mirror or without cover art (rate-limited → re-run for the rest).
|
||||
|
||||
# in-stock rows in this store whose release_id is missing from the mirror, or whose cached cover is blank
|
||||
_NEEDS_API = """release_id IS NOT NULL AND (
|
||||
NOT EXISTS (SELECT 1 FROM disc_cache dc WHERE dc.release_id=i.release_id)
|
||||
OR EXISTS (SELECT 1 FROM disc_cache dc WHERE dc.release_id=i.release_id
|
||||
AND (dc.thumb IS NULL OR dc.thumb='')))"""
|
||||
|
||||
|
||||
@router.get("/heal/scan")
|
||||
async def heal_scan(ident=Depends(require_token), db=Depends(get_db)):
|
||||
"""Dry-run: count the gaps before healing anything."""
|
||||
sid = {"sid": ident["store_id"]}
|
||||
base = "FROM inventory i WHERE i.store_id=:sid AND i.in_stock"
|
||||
|
||||
async def n(extra):
|
||||
return (await db.execute(text(f"SELECT count(*) {base} AND {extra}"), sid)).scalar()
|
||||
|
||||
async def nd(extra):
|
||||
return (await db.execute(text(f"SELECT count(DISTINCT i.release_id) {base} AND {extra}"), sid)).scalar()
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"no_release_id": await n("i.release_id IS NULL"), # can't auto-heal (needs matching)
|
||||
"missing_title": await n("i.release_id IS NOT NULL AND i.title IS NULL"),
|
||||
"missing_weight": await n("i.release_id IS NOT NULL AND i.weight_g IS NULL"),
|
||||
"local_fixable": await n("i.release_id IS NOT NULL AND (i.title IS NULL OR i.weight_g IS NULL) "
|
||||
"AND EXISTS (SELECT 1 FROM disc_cache dc WHERE dc.release_id=i.release_id "
|
||||
"AND (dc.title IS NOT NULL OR dc.weight IS NOT NULL))"),
|
||||
"api_needed": await nd(_NEEDS_API), # distinct releases needing Discogs
|
||||
}
|
||||
|
||||
|
||||
class HealIn(BaseModel):
|
||||
limit: int = 60 # cap Discogs fetches per run (rate limit) — re-run for the remainder
|
||||
|
||||
|
||||
@router.post("/heal/run")
|
||||
async def heal_run(body: HealIn = HealIn(), ident=Depends(require_token), db=Depends(get_db)):
|
||||
sid = ident["store_id"]
|
||||
# Phase 1 — local backfill from the mirror (instant, unbounded)
|
||||
local = (await db.execute(text("""
|
||||
UPDATE inventory i SET title = COALESCE(i.title, dc.title),
|
||||
weight_g = COALESCE(i.weight_g, dc.weight), updated_at = now()
|
||||
FROM disc_cache dc
|
||||
WHERE dc.release_id = i.release_id AND i.store_id = :sid AND i.in_stock
|
||||
AND (i.title IS NULL OR i.weight_g IS NULL)
|
||||
AND (dc.title IS NOT NULL OR dc.weight IS NOT NULL)"""), {"sid": sid})).rowcount
|
||||
await db.commit()
|
||||
|
||||
# Phase 2 — bounded Discogs re-fetch for releases missing from the mirror / without cover
|
||||
rids = [r[0] for r in (await db.execute(text(
|
||||
f"SELECT DISTINCT i.release_id FROM inventory i "
|
||||
f"WHERE i.store_id=:sid AND i.in_stock AND {_NEEDS_API} LIMIT :lim"),
|
||||
{"sid": sid, "lim": body.limit}))]
|
||||
enriched = 0
|
||||
for rid in rids:
|
||||
meta = await _fetch_release(db, rid) # always fetches → fills cover + grows mirror
|
||||
if meta:
|
||||
await db.execute(text(
|
||||
"UPDATE inventory SET title=COALESCE(title,:t), weight_g=COALESCE(weight_g,:w), "
|
||||
"updated_at=now() WHERE release_id=:r AND store_id=:sid AND in_stock"),
|
||||
{"t": meta.get("title"), "w": meta.get("weight"), "r": rid, "sid": sid})
|
||||
enriched += 1
|
||||
await asyncio.sleep(0.2) # ponytail: gentle on the Discogs rate limit
|
||||
await db.commit()
|
||||
|
||||
remaining = (await db.execute(text(
|
||||
f"SELECT count(DISTINCT i.release_id) FROM inventory i "
|
||||
f"WHERE i.store_id=:sid AND i.in_stock AND {_NEEDS_API}"), {"sid": sid})).scalar()
|
||||
return {"ok": True, "local_healed": local, "api_enriched": enriched, "remaining_api": remaining}
|
||||
|
||||
|
||||
def _selfcheck():
|
||||
# SKU from a Google-Form timestamp (both ISO and en-AU slash formats) → YYYYMMDDHHMMSS
|
||||
assert _sku_from_ts("2025-01-30T14:01:02.821Z") == "20250130140102"
|
||||
|
||||
@ -71,6 +71,7 @@
|
||||
<a data-v="wantlist">📝 Wantlist</a>
|
||||
<div class="sec">Catalog</div>
|
||||
<a data-v="intake">📥 Intake</a>
|
||||
<a data-v="heal">🩹 Heal</a>
|
||||
<a data-v="storemap">🗺️ Store map</a>
|
||||
<a data-v="crates">🗄️ Crates</a>
|
||||
<a data-v="reports">📈 Reports</a>
|
||||
@ -108,7 +109,7 @@ document.querySelectorAll('nav a[data-v]').forEach(a => a.onclick = () => go(a.d
|
||||
function go(v){
|
||||
if(ROLE!=='admin' && (v==='connections'||v==='staff'||v==='system')) v='dashboard'; // staff can't reach admin views
|
||||
document.querySelectorAll('nav a[data-v]').forEach(a => a.classList.toggle('on', a.dataset.v===v));
|
||||
({dashboard:vDash, inventory:vInv, orders:vOrders, customers:vCustomers, wantlist:vWantlist, intake:vIntake, storemap:vStoreMap, crates:vCrates, reports:vReports, connections:vConn, staff:vStaff, system:vSystem}[v])();
|
||||
({dashboard:vDash, inventory:vInv, orders:vOrders, customers:vCustomers, wantlist:vWantlist, intake:vIntake, heal:vHeal, storemap:vStoreMap, crates:vCrates, reports:vReports, connections:vConn, staff:vStaff, system:vSystem}[v])();
|
||||
}
|
||||
|
||||
// ---------- Dashboard (fast tiles first, charts lazy) ----------
|
||||
@ -672,6 +673,39 @@ async function inkImport(){
|
||||
m.innerHTML = d.ok ? `<span style="color:var(--ok)">✓ staged ${d.staged} new</span>${d.skipped?` · ${d.skipped} already stocked`:''}` : '<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>';
|
||||
}
|
||||
|
||||
// ---------- Heal (backfill missing catalog metadata) ----------
|
||||
function vHeal(){
|
||||
$('#content').innerHTML = '<h2>Heal <span class="muted" style="font-size:13px">· backfill missing catalog metadata</span></h2>'
|
||||
+ '<div class="card" style="max-width:680px">'
|
||||
+ '<p class="muted" style="margin-top:0">Scans in-stock items for gaps — missing title, weight, or cover art — and backfills from your local Discogs mirror. Anything the mirror doesn’t have is re-fetched from Discogs (rate-limited, so it heals in batches).</p>'
|
||||
+ '<button onclick="healScan()">Scan for gaps</button> <button onclick="healRun()" id="heal_run" style="display:none">🩹 Heal now</button>'
|
||||
+ '<div id="heal_out" style="margin-top:14px"></div></div>';
|
||||
}
|
||||
async function healScan(){
|
||||
const o=$('#heal_out'); o.innerHTML='<span class="muted">scanning…</span>';
|
||||
const d=await(await fetch('/admin/intake/heal/scan',{headers:hdr()})).json();
|
||||
if(!d.ok){ o.innerHTML='<span style="color:var(--warn)">scan failed</span>'; return; }
|
||||
const row=(label,n,note)=>`<tr style="border-top:1px solid var(--line)"><td style="padding:5px 10px 5px 0">${label}</td><td style="text-align:right;font-weight:600">${n}</td><td class="muted" style="padding-left:12px;font-size:12px">${note||''}</td></tr>`;
|
||||
o.innerHTML='<table style="border-collapse:collapse">'
|
||||
+ row('Missing title', d.missing_title, d.local_fixable?`${d.local_fixable} fixable instantly from the mirror`:'')
|
||||
+ row('Missing weight', d.missing_weight, 'blocks shipping quotes')
|
||||
+ row('Need Discogs re-fetch', d.api_needed, 'not in mirror / no cover art')
|
||||
+ row('No release_id', d.no_release_id, 'can’t auto-heal — needs matching')
|
||||
+ '</table>';
|
||||
const fixable = d.missing_title + d.missing_weight + d.api_needed;
|
||||
$('#heal_run').style.display = fixable ? '' : 'none';
|
||||
if(!fixable) o.innerHTML += '<div style="color:var(--ok);margin-top:10px">✓ Nothing to heal — catalog is clean.</div>';
|
||||
}
|
||||
async function healRun(){
|
||||
const o=$('#heal_out'), b=$('#heal_run'); b.disabled=true; b.textContent='healing…';
|
||||
const d=await(await fetch('/admin/intake/heal/run',{method:'POST',headers:hdr(),body:'{}'})).json();
|
||||
b.disabled=false; b.textContent='🩹 Heal now';
|
||||
if(!d.ok){ o.innerHTML='<span style="color:var(--warn)">heal failed</span>'; return; }
|
||||
o.innerHTML=`<div style="color:var(--ok)">✓ ${d.local_healed} healed locally · ${d.api_enriched} re-fetched from Discogs</div>`
|
||||
+ (d.remaining_api ? `<div style="margin-top:8px">${d.remaining_api} still need Discogs (rate-limited) — <button onclick="healRun()">Heal next batch</button></div>`
|
||||
: '<div class="muted" style="margin-top:8px">All gaps closed. Re-scan to confirm.</div>');
|
||||
}
|
||||
|
||||
// ---------- Crates ----------
|
||||
async function vCrates(){
|
||||
const m = $('#content'); m.innerHTML = '<h2>Crates</h2><div class="muted">loading…</div>';
|
||||
|
||||
Loading…
Reference in New Issue
Block a user