feat(intake): connected Google sheet via service account + skip-existing bulk stage

The intake sheet is read straight from the stored Google service account (discotag@…
inventory-sheets, migrated out of WowPlatter's wp_rmp_disc_google_service into the vault) — a
signed JWT (cryptography, no new dep) → Sheets API → private sheet, no public-share needed. Uses
the saved column_mappings (timestamp/release_id/media/sleeve/price/comment, 1-indexed). SKU from
the sheet's locale timestamp '2025-01-30, 14:01:02' → 20250130140102 (matches migrated stock).

Bulk import (both connected + paste-URL paths) now SKIPS SKUs already in inventory, so importing
the 34,752-row form log only stages the genuinely-new rows (live: 2,006 new / 32,746 skipped).
Preview shows new-of-total. Endpoints: GET/POST /admin/intake/gsheet/{preview,import}.

Also: .dockerignore now excludes disc_images/ (volume-mounted, was 916MB) — build context
918MB→1MB, deploy 13min(hung)→24s. The COPY . . was baking the image store into every image.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/intake_routes.py

@@ -5,6 +5,7 @@ import io
 import json
 import re
 import secrets
+import time
 
 import httpx
 from fastapi import APIRouter, Depends, Query
@@ -314,7 +315,8 @@ def _detect(headers):
 
 
 def _sku_from_ts(ts: str) -> str | None:
-    m = re.match(r"(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})", ts or "")
+    # separator between date and time is T, space, or ", " depending on the sheet's locale format
+    m = re.match(r"(\d{4})-(\d{2})-(\d{2})[T,\s]+(\d{2}):(\d{2}):(\d{2})", ts or "")
     if m:
         return "".join(m.groups())            # YYYYMMDDHHMMSS — matches WowPlatter / migrated stock
     m = re.match(r"(\d{1,2})/(\d{1,2})/(\d{4})\s+(\d{1,2}):(\d{2}):(\d{2})", ts or "")
@@ -324,20 +326,18 @@ def _sku_from_ts(ts: str) -> str | None:
     return None
 
 
-def _parse_sheet(text_csv):
+def _parse_rows(rows, col):
-    rows = list(csv.reader(io.StringIO(text_csv)))
+    """rows[0] is the header (skipped); col maps our field → 0-indexed column. Shared by the
-    if not rows:
+    public-CSV path (col from header detection) and the service-account path (col from the
-        return [], {}
+    stored Google column_mappings)."""
-    col = _detect(rows[0])
-    if "release_id" not in col:
-        return [], col
     out = []
     for r in rows[1:]:
         def g(k):
             i = col.get(k)
-            return r[i].strip() if i is not None and i < len(r) and r[i] else None
+            v = r[i] if i is not None and i < len(r) else None
+            return v.strip() if isinstance(v, str) and v.strip() else None
         rid = g("release_id")
-        if not rid or not rid.isdigit():
+        if not rid or not str(rid).isdigit():
             continue
         price = g("price")
         try:
@@ -348,7 +348,17 @@ def _parse_sheet(text_csv):
         out.append({"release_id": int(rid), "sku": sku, "price": price,
                     "media": g("media") or "VG+", "sleeve": g("sleeve"),
                     "notes": g("notes")})
-    return out, col
+    return out
+
+
+def _parse_sheet(text_csv):
+    rows = list(csv.reader(io.StringIO(text_csv)))
+    if not rows:
+        return [], {}
+    col = _detect(rows[0])
+    if "release_id" not in col:
+        return [], col
+    return _parse_rows(rows, col), col
 
 
 async def _fetch_sheet(url):
@@ -377,27 +387,137 @@ async def sheet_preview(body: SheetIn, ident=Depends(require_token), db=Depends(
     return {"ok": True, "total": len(rows), "columns": col, "sample": rows[:10]}
 
 
-@router.post("/sheet/import")
+async def _bulk_stage(db, store_id, rows):
-async def sheet_import(body: SheetIn, ident=Depends(require_token), db=Depends(get_db)):
+    """Stage sheet rows, SKIPPING any SKU already in inventory — so re-importing a 34k form-log
-    csv_text, err = await _fetch_sheet(body.url)
+    only touches genuinely-new stock (the bulk is already-migrated). Rows without a derived SKU
-    if err:
+    always stage (can't dedup them). Returns (staged, skipped)."""
-        return {"ok": False, "error": err}
+    skus = [r["sku"] for r in rows if r["sku"]]
-    rows, _ = _parse_sheet(csv_text)
+    existing = set()
-    staged = 0
+    if skus:
+        existing = {x[0] for x in (await db.execute(
+            text("SELECT sku FROM inventory WHERE sku = ANY(:s)"), {"s": skus}))}
+    staged = skipped = 0
     for row in rows:
-        await _stage(db, ident["store_id"], row["release_id"], row["sku"], row["media"],
+        if row["sku"] and row["sku"] in existing:
+            skipped += 1
+            continue
+        await _stage(db, store_id, row["release_id"], row["sku"], row["media"],
                      row["sleeve"], row["price"], row["notes"])
         staged += 1
         if staged % 25 == 0:
             await db.commit()
             await asyncio.sleep(0.2)          # ponytail: gentle on the Discogs API for cache-miss enrich
     await db.commit()
-    return {"ok": True, "staged": staged}
+    return staged, skipped
+
+
+@router.post("/sheet/import")
+async def sheet_import(body: SheetIn, ident=Depends(require_token), db=Depends(get_db)):
+    csv_text, err = await _fetch_sheet(body.url)
+    if err:
+        return {"ok": False, "error": err}
+    rows, _ = _parse_sheet(csv_text)
+    staged, skipped = await _bulk_stage(db, ident["store_id"], rows)
+    return {"ok": True, "staged": staged, "skipped": skipped}
+
+
+# --- 3b. the CONNECTED Google sheet via service account (private, no public share) ---
+
+_gtoken = {}
+
+
+async def _google_token(db):
+    """Service-account access token for the Sheets API (signed JWT → OAuth token). Cached ~hour."""
+    if _gtoken.get("exp", 0) > time.time() + 60:
+        return _gtoken["tok"]
+    email = await vault.get_secret(db, "google_sa_email")
+    pem = await vault.get_secret(db, "google_sa_private_key")
+    if not (email and pem):
+        return None
+    from cryptography.hazmat.primitives import hashes, serialization
+    from cryptography.hazmat.primitives.asymmetric import padding
+    now = int(time.time())
+    b64 = lambda d: base64.urlsafe_b64encode(json.dumps(d, separators=(",", ":")).encode()).rstrip(b"=")
+    seg = b64({"alg": "RS256", "typ": "JWT"}) + b"." + b64(
+        {"iss": email, "scope": "https://www.googleapis.com/auth/spreadsheets.readonly",
+         "aud": "https://oauth2.googleapis.com/token", "iat": now, "exp": now + 3600})
+    key = serialization.load_pem_private_key(pem.encode(), password=None)
+    sig = base64.urlsafe_b64encode(key.sign(seg, padding.PKCS1v15(), hashes.SHA256())).rstrip(b"=")
+    async with httpx.AsyncClient(timeout=20) as c:
+        r = await c.post("https://oauth2.googleapis.com/token", data={
+            "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
+            "assertion": (seg + b"." + sig).decode()})
+    if r.status_code != 200:
+        return None
+    j = r.json()
+    _gtoken.update(tok=j["access_token"], exp=now + j.get("expires_in", 3600))
+    return _gtoken["tok"]
+
+
+# WowPlatter's stored column_mappings keys → our fields
+_GCOL = {"release_id": "release_id", "media_condition": "media", "sleeve_condition": "sleeve",
+         "price": "price", "comment": "notes", "timestamp": "timestamp", "sku": "sku"}
+
+
+async def _configured_sheet(db):
+    """Read the connected sheet via the service account → (items, error). Uses the stored
+    column map (1-indexed) when present, else header auto-detect."""
+    tok = await _google_token(db)
+    if not tok:
+        return None, "Google service account not connected (save its creds in Connections)"
+    sid = await vault.get_secret(db, "google_sheet_id")
+    if not sid:
+        return None, "no google_sheet_id saved"
+    name = await vault.get_secret(db, "google_sheet_name") or "Sheet1"
+    raw = await vault.get_secret(db, "google_column_map")
+    colmap = {}
+    if raw:
+        for gk, idx in json.loads(raw).items():
+            ours = _GCOL.get(gk)
+            if ours and str(idx).isdigit():
+                colmap[ours] = int(idx) - 1
+    async with httpx.AsyncClient(timeout=30) as c:
+        r = await c.get(f"https://sheets.googleapis.com/v4/spreadsheets/{sid}/values/{name}",
+                        headers={"Authorization": "Bearer " + tok})
+    if r.status_code != 200:
+        return None, f"Sheets API HTTP {r.status_code}"
+    values = r.json().get("values", [])
+    if not values:
+        return [], None
+    return _parse_rows(values, colmap or _detect(values[0])), None
+
+
+async def _count_new(db, rows):
+    skus = [r["sku"] for r in rows if r["sku"]]
+    existing = set()
+    if skus:
+        existing = {x[0] for x in (await db.execute(
+            text("SELECT sku FROM inventory WHERE sku = ANY(:s)"), {"s": skus}))}
+    return sum(1 for r in rows if not (r["sku"] and r["sku"] in existing))
+
+
+@router.get("/gsheet/preview")
+async def gsheet_preview(ident=Depends(require_token), db=Depends(get_db)):
+    rows, err = await _configured_sheet(db)
+    if err:
+        return {"ok": False, "error": err}
+    return {"ok": True, "total": len(rows), "new": await _count_new(db, rows), "sample": rows[:10],
+            "sheet": await vault.get_secret(db, "google_sheet_name") or "Sheet1"}
+
+
+@router.post("/gsheet/import")
+async def gsheet_import(ident=Depends(require_token), db=Depends(get_db)):
+    rows, err = await _configured_sheet(db)
+    if err:
+        return {"ok": False, "error": err}
+    staged, skipped = await _bulk_stage(db, ident["store_id"], rows)
+    return {"ok": True, "staged": staged, "skipped": skipped}
 
 
 def _selfcheck():
     # SKU from a Google-Form timestamp (both ISO and en-AU slash formats) → YYYYMMDDHHMMSS
     assert _sku_from_ts("2025-01-30T14:01:02.821Z") == "20250130140102"
+    assert _sku_from_ts("2025-01-30, 14:01:02") == "20250130140102"   # Sheet1's locale format
     assert _sku_from_ts("30/01/2025 14:01:02") == "20250130140102"
     assert _sku_from_ts("") is None
     # CSV export URL derivation (id + gid)

app/settings_routes.py

@@ -32,6 +32,11 @@ KNOWN = {
     "square_location_id": "Square location ID (e.g. S5D3EN3YM2AN8)",
     "square_environment": "Square environment — production or sandbox",
     "bridge_key": "WordPress bridge shared secret — the WP plugin sends this to post back orders",
+    "google_sa_email": "Google service-account email (…@…iam.gserviceaccount.com) — reads the intake sheet",
+    "google_sa_private_key": "Google service-account private key (PEM) — signs the Sheets API token",
+    "google_sheet_id": "Intake Google Sheet ID (from its URL)",
+    "google_sheet_name": "Intake sheet/tab name (e.g. Sheet1)",
+    "google_column_map": "Intake sheet column map JSON (Google-Form positions)",
 }
 
 

site/admin.html

@@ -627,12 +627,33 @@ async function inkStageAll(id){
 
 // --- 3. Google Sheet (public CSV) ---
 function inkSheet(){
-  inkBody('<div class="card"><label>Google Sheet URL <span class="muted" style="font-weight:400">(Share → anyone with link can view)</span></label>'
+  inkBody('<div class="card"><div style="display:flex;justify-content:space-between;align-items:center;gap:10px;flex-wrap:wrap">'
+    +'<div><strong>📋 Connected sheet</strong><div class="muted" style="font-size:12px">Your private intake sheet, read via the Google service account — uses the saved column map.</div></div>'
+    +'<button onclick="inkGPreview()">Preview connected sheet</button></div>'
+    +'<div id="ik_gsheet" style="margin-top:10px"></div></div>'
+    +'<div class="card"><label>…or paste a Google Sheet URL <span class="muted" style="font-weight:400">(Share → anyone with link can view)</span></label>'
     +'<input id="ik_url" placeholder="https://docs.google.com/spreadsheets/d/…" style="width:100%">'
     +'<div style="margin-top:10px"><button onclick="inkPreview()">Preview</button></div>'
-    +'<div class="muted" style="font-size:12px;margin-top:8px">Auto-detects columns: release id · media/sleeve condition · price · comment · timestamp (→ SKU). Your Google-Form responses sheet works as-is.</div></div>'
+    +'<div class="muted" style="font-size:12px;margin-top:8px">Auto-detects columns: release id · media/sleeve condition · price · comment · timestamp (→ SKU).</div></div>'
     +'<div id="ik_sheet"></div>');
 }
+function inkSampleTable(rows){
+  return '<table style="width:100%;font-size:12px;border-collapse:collapse"><tr style="text-align:left;color:var(--mut)"><th>release_id</th><th>sku</th><th>media</th><th>sleeve</th><th>price</th><th>notes</th></tr>'
+    +rows.map(r=>`<tr style="border-top:1px solid var(--line)"><td>${r.release_id}</td><td>${escapeH(r.sku||'auto')}</td><td>${escapeH(r.media||'')}</td><td>${escapeH(r.sleeve||'')}</td><td>${r.price??''}</td><td>${escapeH(r.notes||'')}</td></tr>`).join('')+'</table>';
+}
+async function inkGPreview(){
+  $('#ik_gsheet').innerHTML='<div class="muted">reading connected sheet…</div>';
+  const d=await(await fetch('/admin/intake/gsheet/preview',{headers:hdr()})).json();
+  if(!d.ok){ $('#ik_gsheet').innerHTML='<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>'; return; }
+  $('#ik_gsheet').innerHTML=`<div style="margin-bottom:8px"><strong>${d.new}</strong> new of ${d.total} rows in <em>${escapeH(d.sheet)}</em> <span class="muted">(already-stocked SKUs are skipped)</span></div>`
+    +inkSampleTable(d.sample)
+    +'<div style="margin-top:12px"><button onclick="inkGImport()">Import '+d.new+' new rows</button> <span id="ik_gimp" class="muted"></span></div>';
+}
+async function inkGImport(){
+  const m=$('#ik_gimp'); m.textContent='importing… (enriching new titles, please wait)';
+  const d=await(await fetch('/admin/intake/gsheet/import',{method:'POST',headers:hdr()})).json();
+  m.innerHTML=d.ok?`<span style="color:var(--ok)">✓ staged ${d.staged} new</span>${d.skipped?` · ${d.skipped} already stocked`:''}`:'<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>';
+}
 async function inkPreview(){
   const url=$('#ik_url').value.trim(); if(!url)return;
   $('#ik_sheet').innerHTML='<div class="muted" style="padding:12px">reading sheet…</div>';
@@ -648,7 +669,7 @@ async function inkPreview(){
 async function inkImport(){
   const url=$('#ik_sheet').dataset.url, m=$('#ik_imp'); m.textContent='importing… (enriching new titles, please wait)';
   const d=await(await fetch('/admin/intake/sheet/import',{method:'POST',headers:hdr(),body:JSON.stringify({url})})).json();
-  m.innerHTML = d.ok ? `<span style="color:var(--ok)">✓ staged ${d.staged} rows</span>` : '<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>';
+  m.innerHTML = d.ok ? `<span style="color:var(--ok)">✓ staged ${d.staged} new</span>${d.skipped?` · ${d.skipped} already stocked`:''}` : '<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>';
 }
 
 // ---------- Crates ----------