diff --git a/app/intake_routes.py b/app/intake_routes.py index dc65e0c..fd28b56 100644 --- a/app/intake_routes.py +++ b/app/intake_routes.py @@ -5,6 +5,7 @@ import io import json import re import secrets +import time import httpx from fastapi import APIRouter, Depends, Query @@ -314,7 +315,8 @@ def _detect(headers): def _sku_from_ts(ts: str) -> str | None: - m = re.match(r"(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})", ts or "") + # separator between date and time is T, space, or ", " depending on the sheet's locale format + m = re.match(r"(\d{4})-(\d{2})-(\d{2})[T,\s]+(\d{2}):(\d{2}):(\d{2})", ts or "") if m: return "".join(m.groups()) # YYYYMMDDHHMMSS — matches WowPlatter / migrated stock m = re.match(r"(\d{1,2})/(\d{1,2})/(\d{4})\s+(\d{1,2}):(\d{2}):(\d{2})", ts or "") @@ -324,20 +326,18 @@ def _sku_from_ts(ts: str) -> str | None: return None -def _parse_sheet(text_csv): - rows = list(csv.reader(io.StringIO(text_csv))) - if not rows: - return [], {} - col = _detect(rows[0]) - if "release_id" not in col: - return [], col +def _parse_rows(rows, col): + """rows[0] is the header (skipped); col maps our field → 0-indexed column. Shared by the + public-CSV path (col from header detection) and the service-account path (col from the + stored Google column_mappings).""" out = [] for r in rows[1:]: def g(k): i = col.get(k) - return r[i].strip() if i is not None and i < len(r) and r[i] else None + v = r[i] if i is not None and i < len(r) else None + return v.strip() if isinstance(v, str) and v.strip() else None rid = g("release_id") - if not rid or not rid.isdigit(): + if not rid or not str(rid).isdigit(): continue price = g("price") try: @@ -348,7 +348,17 @@ def _parse_sheet(text_csv): out.append({"release_id": int(rid), "sku": sku, "price": price, "media": g("media") or "VG+", "sleeve": g("sleeve"), "notes": g("notes")}) - return out, col + return out + + +def _parse_sheet(text_csv): + rows = list(csv.reader(io.StringIO(text_csv))) + if not rows: + return [], {} + col = _detect(rows[0]) + if "release_id" not in col: + return [], col + return _parse_rows(rows, col), col async def _fetch_sheet(url): @@ -377,27 +387,137 @@ async def sheet_preview(body: SheetIn, ident=Depends(require_token), db=Depends( return {"ok": True, "total": len(rows), "columns": col, "sample": rows[:10]} -@router.post("/sheet/import") -async def sheet_import(body: SheetIn, ident=Depends(require_token), db=Depends(get_db)): - csv_text, err = await _fetch_sheet(body.url) - if err: - return {"ok": False, "error": err} - rows, _ = _parse_sheet(csv_text) - staged = 0 +async def _bulk_stage(db, store_id, rows): + """Stage sheet rows, SKIPPING any SKU already in inventory — so re-importing a 34k form-log + only touches genuinely-new stock (the bulk is already-migrated). Rows without a derived SKU + always stage (can't dedup them). Returns (staged, skipped).""" + skus = [r["sku"] for r in rows if r["sku"]] + existing = set() + if skus: + existing = {x[0] for x in (await db.execute( + text("SELECT sku FROM inventory WHERE sku = ANY(:s)"), {"s": skus}))} + staged = skipped = 0 for row in rows: - await _stage(db, ident["store_id"], row["release_id"], row["sku"], row["media"], + if row["sku"] and row["sku"] in existing: + skipped += 1 + continue + await _stage(db, store_id, row["release_id"], row["sku"], row["media"], row["sleeve"], row["price"], row["notes"]) staged += 1 if staged % 25 == 0: await db.commit() await asyncio.sleep(0.2) # ponytail: gentle on the Discogs API for cache-miss enrich await db.commit() - return {"ok": True, "staged": staged} + return staged, skipped + + +@router.post("/sheet/import") +async def sheet_import(body: SheetIn, ident=Depends(require_token), db=Depends(get_db)): + csv_text, err = await _fetch_sheet(body.url) + if err: + return {"ok": False, "error": err} + rows, _ = _parse_sheet(csv_text) + staged, skipped = await _bulk_stage(db, ident["store_id"], rows) + return {"ok": True, "staged": staged, "skipped": skipped} + + +# --- 3b. the CONNECTED Google sheet via service account (private, no public share) --- + +_gtoken = {} + + +async def _google_token(db): + """Service-account access token for the Sheets API (signed JWT → OAuth token). Cached ~hour.""" + if _gtoken.get("exp", 0) > time.time() + 60: + return _gtoken["tok"] + email = await vault.get_secret(db, "google_sa_email") + pem = await vault.get_secret(db, "google_sa_private_key") + if not (email and pem): + return None + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding + now = int(time.time()) + b64 = lambda d: base64.urlsafe_b64encode(json.dumps(d, separators=(",", ":")).encode()).rstrip(b"=") + seg = b64({"alg": "RS256", "typ": "JWT"}) + b"." + b64( + {"iss": email, "scope": "https://www.googleapis.com/auth/spreadsheets.readonly", + "aud": "https://oauth2.googleapis.com/token", "iat": now, "exp": now + 3600}) + key = serialization.load_pem_private_key(pem.encode(), password=None) + sig = base64.urlsafe_b64encode(key.sign(seg, padding.PKCS1v15(), hashes.SHA256())).rstrip(b"=") + async with httpx.AsyncClient(timeout=20) as c: + r = await c.post("https://oauth2.googleapis.com/token", data={ + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "assertion": (seg + b"." + sig).decode()}) + if r.status_code != 200: + return None + j = r.json() + _gtoken.update(tok=j["access_token"], exp=now + j.get("expires_in", 3600)) + return _gtoken["tok"] + + +# WowPlatter's stored column_mappings keys → our fields +_GCOL = {"release_id": "release_id", "media_condition": "media", "sleeve_condition": "sleeve", + "price": "price", "comment": "notes", "timestamp": "timestamp", "sku": "sku"} + + +async def _configured_sheet(db): + """Read the connected sheet via the service account → (items, error). Uses the stored + column map (1-indexed) when present, else header auto-detect.""" + tok = await _google_token(db) + if not tok: + return None, "Google service account not connected (save its creds in Connections)" + sid = await vault.get_secret(db, "google_sheet_id") + if not sid: + return None, "no google_sheet_id saved" + name = await vault.get_secret(db, "google_sheet_name") or "Sheet1" + raw = await vault.get_secret(db, "google_column_map") + colmap = {} + if raw: + for gk, idx in json.loads(raw).items(): + ours = _GCOL.get(gk) + if ours and str(idx).isdigit(): + colmap[ours] = int(idx) - 1 + async with httpx.AsyncClient(timeout=30) as c: + r = await c.get(f"https://sheets.googleapis.com/v4/spreadsheets/{sid}/values/{name}", + headers={"Authorization": "Bearer " + tok}) + if r.status_code != 200: + return None, f"Sheets API HTTP {r.status_code}" + values = r.json().get("values", []) + if not values: + return [], None + return _parse_rows(values, colmap or _detect(values[0])), None + + +async def _count_new(db, rows): + skus = [r["sku"] for r in rows if r["sku"]] + existing = set() + if skus: + existing = {x[0] for x in (await db.execute( + text("SELECT sku FROM inventory WHERE sku = ANY(:s)"), {"s": skus}))} + return sum(1 for r in rows if not (r["sku"] and r["sku"] in existing)) + + +@router.get("/gsheet/preview") +async def gsheet_preview(ident=Depends(require_token), db=Depends(get_db)): + rows, err = await _configured_sheet(db) + if err: + return {"ok": False, "error": err} + return {"ok": True, "total": len(rows), "new": await _count_new(db, rows), "sample": rows[:10], + "sheet": await vault.get_secret(db, "google_sheet_name") or "Sheet1"} + + +@router.post("/gsheet/import") +async def gsheet_import(ident=Depends(require_token), db=Depends(get_db)): + rows, err = await _configured_sheet(db) + if err: + return {"ok": False, "error": err} + staged, skipped = await _bulk_stage(db, ident["store_id"], rows) + return {"ok": True, "staged": staged, "skipped": skipped} def _selfcheck(): # SKU from a Google-Form timestamp (both ISO and en-AU slash formats) → YYYYMMDDHHMMSS assert _sku_from_ts("2025-01-30T14:01:02.821Z") == "20250130140102" + assert _sku_from_ts("2025-01-30, 14:01:02") == "20250130140102" # Sheet1's locale format assert _sku_from_ts("30/01/2025 14:01:02") == "20250130140102" assert _sku_from_ts("") is None # CSV export URL derivation (id + gid) diff --git a/app/settings_routes.py b/app/settings_routes.py index fa542a8..2543f32 100644 --- a/app/settings_routes.py +++ b/app/settings_routes.py @@ -32,6 +32,11 @@ KNOWN = { "square_location_id": "Square location ID (e.g. S5D3EN3YM2AN8)", "square_environment": "Square environment — production or sandbox", "bridge_key": "WordPress bridge shared secret — the WP plugin sends this to post back orders", + "google_sa_email": "Google service-account email (…@…iam.gserviceaccount.com) — reads the intake sheet", + "google_sa_private_key": "Google service-account private key (PEM) — signs the Sheets API token", + "google_sheet_id": "Intake Google Sheet ID (from its URL)", + "google_sheet_name": "Intake sheet/tab name (e.g. Sheet1)", + "google_column_map": "Intake sheet column map JSON (Google-Form positions)", } diff --git a/site/admin.html b/site/admin.html index 9a13238..4e2d105 100644 --- a/site/admin.html +++ b/site/admin.html @@ -627,12 +627,33 @@ async function inkStageAll(id){ // --- 3. Google Sheet (public CSV) --- function inkSheet(){ - inkBody('
' + inkBody('
' + +'
📋 Connected sheet
Your private intake sheet, read via the Google service account — uses the saved column map.
' + +'
' + +'
' + +'
' +'' +'
' - +'
Auto-detects columns: release id · media/sleeve condition · price · comment · timestamp (→ SKU). Your Google-Form responses sheet works as-is.
' + +'
Auto-detects columns: release id · media/sleeve condition · price · comment · timestamp (→ SKU).
' +'
'); } +function inkSampleTable(rows){ + return '' + +rows.map(r=>``).join('')+'
release_idskumediasleevepricenotes
${r.release_id}${escapeH(r.sku||'auto')}${escapeH(r.media||'')}${escapeH(r.sleeve||'')}${r.price??''}${escapeH(r.notes||'')}
'; +} +async function inkGPreview(){ + $('#ik_gsheet').innerHTML='
reading connected sheet…
'; + const d=await(await fetch('/admin/intake/gsheet/preview',{headers:hdr()})).json(); + if(!d.ok){ $('#ik_gsheet').innerHTML=''+escapeH(d.error||'failed')+''; return; } + $('#ik_gsheet').innerHTML=`
${d.new} new of ${d.total} rows in ${escapeH(d.sheet)} (already-stocked SKUs are skipped)
` + +inkSampleTable(d.sample) + +'
'; +} +async function inkGImport(){ + const m=$('#ik_gimp'); m.textContent='importing… (enriching new titles, please wait)'; + const d=await(await fetch('/admin/intake/gsheet/import',{method:'POST',headers:hdr()})).json(); + m.innerHTML=d.ok?`✓ staged ${d.staged} new${d.skipped?` · ${d.skipped} already stocked`:''}`:''+escapeH(d.error||'failed')+''; +} async function inkPreview(){ const url=$('#ik_url').value.trim(); if(!url)return; $('#ik_sheet').innerHTML='
reading sheet…
'; @@ -648,7 +669,7 @@ async function inkPreview(){ async function inkImport(){ const url=$('#ik_sheet').dataset.url, m=$('#ik_imp'); m.textContent='importing… (enriching new titles, please wait)'; const d=await(await fetch('/admin/intake/sheet/import',{method:'POST',headers:hdr(),body:JSON.stringify({url})})).json(); - m.innerHTML = d.ok ? `✓ staged ${d.staged} rows` : ''+escapeH(d.error||'failed')+''; + m.innerHTML = d.ok ? `✓ staged ${d.staged} new${d.skipped?` · ${d.skipped} already stocked`:''}` : ''+escapeH(d.error||'failed')+''; } // ---------- Crates ----------