feat(intake): connected Google sheet via service account + skip-existing bulk stage
The intake sheet is read straight from the stored Google service account (discotag@…
inventory-sheets, migrated out of WowPlatter's wp_rmp_disc_google_service into the vault) — a
signed JWT (cryptography, no new dep) → Sheets API → private sheet, no public-share needed. Uses
the saved column_mappings (timestamp/release_id/media/sleeve/price/comment, 1-indexed). SKU from
the sheet's locale timestamp '2025-01-30, 14:01:02' → 20250130140102 (matches migrated stock).
Bulk import (both connected + paste-URL paths) now SKIPS SKUs already in inventory, so importing
the 34,752-row form log only stages the genuinely-new rows (live: 2,006 new / 32,746 skipped).
Preview shows new-of-total. Endpoints: GET/POST /admin/intake/gsheet/{preview,import}.
Also: .dockerignore now excludes disc_images/ (volume-mounted, was 916MB) — build context
918MB→1MB, deploy 13min(hung)→24s. The COPY . . was baking the image store into every image.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
type-two
parent
88172924eb
commit
25127b1b72
@ -5,6 +5,7 @@ import io
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
@ -314,7 +315,8 @@ def _detect(headers):
|
||||
|
||||
|
||||
def _sku_from_ts(ts: str) -> str | None:
|
||||
m = re.match(r"(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})", ts or "")
|
||||
# separator between date and time is T, space, or ", " depending on the sheet's locale format
|
||||
m = re.match(r"(\d{4})-(\d{2})-(\d{2})[T,\s]+(\d{2}):(\d{2}):(\d{2})", ts or "")
|
||||
if m:
|
||||
return "".join(m.groups()) # YYYYMMDDHHMMSS — matches WowPlatter / migrated stock
|
||||
m = re.match(r"(\d{1,2})/(\d{1,2})/(\d{4})\s+(\d{1,2}):(\d{2}):(\d{2})", ts or "")
|
||||
@ -324,20 +326,18 @@ def _sku_from_ts(ts: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_sheet(text_csv):
|
||||
rows = list(csv.reader(io.StringIO(text_csv)))
|
||||
if not rows:
|
||||
return [], {}
|
||||
col = _detect(rows[0])
|
||||
if "release_id" not in col:
|
||||
return [], col
|
||||
def _parse_rows(rows, col):
|
||||
"""rows[0] is the header (skipped); col maps our field → 0-indexed column. Shared by the
|
||||
public-CSV path (col from header detection) and the service-account path (col from the
|
||||
stored Google column_mappings)."""
|
||||
out = []
|
||||
for r in rows[1:]:
|
||||
def g(k):
|
||||
i = col.get(k)
|
||||
return r[i].strip() if i is not None and i < len(r) and r[i] else None
|
||||
v = r[i] if i is not None and i < len(r) else None
|
||||
return v.strip() if isinstance(v, str) and v.strip() else None
|
||||
rid = g("release_id")
|
||||
if not rid or not rid.isdigit():
|
||||
if not rid or not str(rid).isdigit():
|
||||
continue
|
||||
price = g("price")
|
||||
try:
|
||||
@ -348,7 +348,17 @@ def _parse_sheet(text_csv):
|
||||
out.append({"release_id": int(rid), "sku": sku, "price": price,
|
||||
"media": g("media") or "VG+", "sleeve": g("sleeve"),
|
||||
"notes": g("notes")})
|
||||
return out, col
|
||||
return out
|
||||
|
||||
|
||||
def _parse_sheet(text_csv):
|
||||
rows = list(csv.reader(io.StringIO(text_csv)))
|
||||
if not rows:
|
||||
return [], {}
|
||||
col = _detect(rows[0])
|
||||
if "release_id" not in col:
|
||||
return [], col
|
||||
return _parse_rows(rows, col), col
|
||||
|
||||
|
||||
async def _fetch_sheet(url):
|
||||
@ -377,27 +387,137 @@ async def sheet_preview(body: SheetIn, ident=Depends(require_token), db=Depends(
|
||||
return {"ok": True, "total": len(rows), "columns": col, "sample": rows[:10]}
|
||||
|
||||
|
||||
@router.post("/sheet/import")
|
||||
async def sheet_import(body: SheetIn, ident=Depends(require_token), db=Depends(get_db)):
|
||||
csv_text, err = await _fetch_sheet(body.url)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
rows, _ = _parse_sheet(csv_text)
|
||||
staged = 0
|
||||
async def _bulk_stage(db, store_id, rows):
|
||||
"""Stage sheet rows, SKIPPING any SKU already in inventory — so re-importing a 34k form-log
|
||||
only touches genuinely-new stock (the bulk is already-migrated). Rows without a derived SKU
|
||||
always stage (can't dedup them). Returns (staged, skipped)."""
|
||||
skus = [r["sku"] for r in rows if r["sku"]]
|
||||
existing = set()
|
||||
if skus:
|
||||
existing = {x[0] for x in (await db.execute(
|
||||
text("SELECT sku FROM inventory WHERE sku = ANY(:s)"), {"s": skus}))}
|
||||
staged = skipped = 0
|
||||
for row in rows:
|
||||
await _stage(db, ident["store_id"], row["release_id"], row["sku"], row["media"],
|
||||
if row["sku"] and row["sku"] in existing:
|
||||
skipped += 1
|
||||
continue
|
||||
await _stage(db, store_id, row["release_id"], row["sku"], row["media"],
|
||||
row["sleeve"], row["price"], row["notes"])
|
||||
staged += 1
|
||||
if staged % 25 == 0:
|
||||
await db.commit()
|
||||
await asyncio.sleep(0.2) # ponytail: gentle on the Discogs API for cache-miss enrich
|
||||
await db.commit()
|
||||
return {"ok": True, "staged": staged}
|
||||
return staged, skipped
|
||||
|
||||
|
||||
@router.post("/sheet/import")
|
||||
async def sheet_import(body: SheetIn, ident=Depends(require_token), db=Depends(get_db)):
|
||||
csv_text, err = await _fetch_sheet(body.url)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
rows, _ = _parse_sheet(csv_text)
|
||||
staged, skipped = await _bulk_stage(db, ident["store_id"], rows)
|
||||
return {"ok": True, "staged": staged, "skipped": skipped}
|
||||
|
||||
|
||||
# --- 3b. the CONNECTED Google sheet via service account (private, no public share) ---
|
||||
|
||||
_gtoken = {}
|
||||
|
||||
|
||||
async def _google_token(db):
|
||||
"""Service-account access token for the Sheets API (signed JWT → OAuth token). Cached ~hour."""
|
||||
if _gtoken.get("exp", 0) > time.time() + 60:
|
||||
return _gtoken["tok"]
|
||||
email = await vault.get_secret(db, "google_sa_email")
|
||||
pem = await vault.get_secret(db, "google_sa_private_key")
|
||||
if not (email and pem):
|
||||
return None
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import padding
|
||||
now = int(time.time())
|
||||
b64 = lambda d: base64.urlsafe_b64encode(json.dumps(d, separators=(",", ":")).encode()).rstrip(b"=")
|
||||
seg = b64({"alg": "RS256", "typ": "JWT"}) + b"." + b64(
|
||||
{"iss": email, "scope": "https://www.googleapis.com/auth/spreadsheets.readonly",
|
||||
"aud": "https://oauth2.googleapis.com/token", "iat": now, "exp": now + 3600})
|
||||
key = serialization.load_pem_private_key(pem.encode(), password=None)
|
||||
sig = base64.urlsafe_b64encode(key.sign(seg, padding.PKCS1v15(), hashes.SHA256())).rstrip(b"=")
|
||||
async with httpx.AsyncClient(timeout=20) as c:
|
||||
r = await c.post("https://oauth2.googleapis.com/token", data={
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
||||
"assertion": (seg + b"." + sig).decode()})
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
j = r.json()
|
||||
_gtoken.update(tok=j["access_token"], exp=now + j.get("expires_in", 3600))
|
||||
return _gtoken["tok"]
|
||||
|
||||
|
||||
# WowPlatter's stored column_mappings keys → our fields
|
||||
_GCOL = {"release_id": "release_id", "media_condition": "media", "sleeve_condition": "sleeve",
|
||||
"price": "price", "comment": "notes", "timestamp": "timestamp", "sku": "sku"}
|
||||
|
||||
|
||||
async def _configured_sheet(db):
|
||||
"""Read the connected sheet via the service account → (items, error). Uses the stored
|
||||
column map (1-indexed) when present, else header auto-detect."""
|
||||
tok = await _google_token(db)
|
||||
if not tok:
|
||||
return None, "Google service account not connected (save its creds in Connections)"
|
||||
sid = await vault.get_secret(db, "google_sheet_id")
|
||||
if not sid:
|
||||
return None, "no google_sheet_id saved"
|
||||
name = await vault.get_secret(db, "google_sheet_name") or "Sheet1"
|
||||
raw = await vault.get_secret(db, "google_column_map")
|
||||
colmap = {}
|
||||
if raw:
|
||||
for gk, idx in json.loads(raw).items():
|
||||
ours = _GCOL.get(gk)
|
||||
if ours and str(idx).isdigit():
|
||||
colmap[ours] = int(idx) - 1
|
||||
async with httpx.AsyncClient(timeout=30) as c:
|
||||
r = await c.get(f"https://sheets.googleapis.com/v4/spreadsheets/{sid}/values/{name}",
|
||||
headers={"Authorization": "Bearer " + tok})
|
||||
if r.status_code != 200:
|
||||
return None, f"Sheets API HTTP {r.status_code}"
|
||||
values = r.json().get("values", [])
|
||||
if not values:
|
||||
return [], None
|
||||
return _parse_rows(values, colmap or _detect(values[0])), None
|
||||
|
||||
|
||||
async def _count_new(db, rows):
|
||||
skus = [r["sku"] for r in rows if r["sku"]]
|
||||
existing = set()
|
||||
if skus:
|
||||
existing = {x[0] for x in (await db.execute(
|
||||
text("SELECT sku FROM inventory WHERE sku = ANY(:s)"), {"s": skus}))}
|
||||
return sum(1 for r in rows if not (r["sku"] and r["sku"] in existing))
|
||||
|
||||
|
||||
@router.get("/gsheet/preview")
|
||||
async def gsheet_preview(ident=Depends(require_token), db=Depends(get_db)):
|
||||
rows, err = await _configured_sheet(db)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
return {"ok": True, "total": len(rows), "new": await _count_new(db, rows), "sample": rows[:10],
|
||||
"sheet": await vault.get_secret(db, "google_sheet_name") or "Sheet1"}
|
||||
|
||||
|
||||
@router.post("/gsheet/import")
|
||||
async def gsheet_import(ident=Depends(require_token), db=Depends(get_db)):
|
||||
rows, err = await _configured_sheet(db)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
staged, skipped = await _bulk_stage(db, ident["store_id"], rows)
|
||||
return {"ok": True, "staged": staged, "skipped": skipped}
|
||||
|
||||
|
||||
def _selfcheck():
|
||||
# SKU from a Google-Form timestamp (both ISO and en-AU slash formats) → YYYYMMDDHHMMSS
|
||||
assert _sku_from_ts("2025-01-30T14:01:02.821Z") == "20250130140102"
|
||||
assert _sku_from_ts("2025-01-30, 14:01:02") == "20250130140102" # Sheet1's locale format
|
||||
assert _sku_from_ts("30/01/2025 14:01:02") == "20250130140102"
|
||||
assert _sku_from_ts("") is None
|
||||
# CSV export URL derivation (id + gid)
|
||||
|
||||
@ -32,6 +32,11 @@ KNOWN = {
|
||||
"square_location_id": "Square location ID (e.g. S5D3EN3YM2AN8)",
|
||||
"square_environment": "Square environment — production or sandbox",
|
||||
"bridge_key": "WordPress bridge shared secret — the WP plugin sends this to post back orders",
|
||||
"google_sa_email": "Google service-account email (…@…iam.gserviceaccount.com) — reads the intake sheet",
|
||||
"google_sa_private_key": "Google service-account private key (PEM) — signs the Sheets API token",
|
||||
"google_sheet_id": "Intake Google Sheet ID (from its URL)",
|
||||
"google_sheet_name": "Intake sheet/tab name (e.g. Sheet1)",
|
||||
"google_column_map": "Intake sheet column map JSON (Google-Form positions)",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -627,12 +627,33 @@ async function inkStageAll(id){
|
||||
|
||||
// --- 3. Google Sheet (public CSV) ---
|
||||
function inkSheet(){
|
||||
inkBody('<div class="card"><label>Google Sheet URL <span class="muted" style="font-weight:400">(Share → anyone with link can view)</span></label>'
|
||||
inkBody('<div class="card"><div style="display:flex;justify-content:space-between;align-items:center;gap:10px;flex-wrap:wrap">'
|
||||
+'<div><strong>📋 Connected sheet</strong><div class="muted" style="font-size:12px">Your private intake sheet, read via the Google service account — uses the saved column map.</div></div>'
|
||||
+'<button onclick="inkGPreview()">Preview connected sheet</button></div>'
|
||||
+'<div id="ik_gsheet" style="margin-top:10px"></div></div>'
|
||||
+'<div class="card"><label>…or paste a Google Sheet URL <span class="muted" style="font-weight:400">(Share → anyone with link can view)</span></label>'
|
||||
+'<input id="ik_url" placeholder="https://docs.google.com/spreadsheets/d/…" style="width:100%">'
|
||||
+'<div style="margin-top:10px"><button onclick="inkPreview()">Preview</button></div>'
|
||||
+'<div class="muted" style="font-size:12px;margin-top:8px">Auto-detects columns: release id · media/sleeve condition · price · comment · timestamp (→ SKU). Your Google-Form responses sheet works as-is.</div></div>'
|
||||
+'<div class="muted" style="font-size:12px;margin-top:8px">Auto-detects columns: release id · media/sleeve condition · price · comment · timestamp (→ SKU).</div></div>'
|
||||
+'<div id="ik_sheet"></div>');
|
||||
}
|
||||
function inkSampleTable(rows){
|
||||
return '<table style="width:100%;font-size:12px;border-collapse:collapse"><tr style="text-align:left;color:var(--mut)"><th>release_id</th><th>sku</th><th>media</th><th>sleeve</th><th>price</th><th>notes</th></tr>'
|
||||
+rows.map(r=>`<tr style="border-top:1px solid var(--line)"><td>${r.release_id}</td><td>${escapeH(r.sku||'auto')}</td><td>${escapeH(r.media||'')}</td><td>${escapeH(r.sleeve||'')}</td><td>${r.price??''}</td><td>${escapeH(r.notes||'')}</td></tr>`).join('')+'</table>';
|
||||
}
|
||||
async function inkGPreview(){
|
||||
$('#ik_gsheet').innerHTML='<div class="muted">reading connected sheet…</div>';
|
||||
const d=await(await fetch('/admin/intake/gsheet/preview',{headers:hdr()})).json();
|
||||
if(!d.ok){ $('#ik_gsheet').innerHTML='<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>'; return; }
|
||||
$('#ik_gsheet').innerHTML=`<div style="margin-bottom:8px"><strong>${d.new}</strong> new of ${d.total} rows in <em>${escapeH(d.sheet)}</em> <span class="muted">(already-stocked SKUs are skipped)</span></div>`
|
||||
+inkSampleTable(d.sample)
|
||||
+'<div style="margin-top:12px"><button onclick="inkGImport()">Import '+d.new+' new rows</button> <span id="ik_gimp" class="muted"></span></div>';
|
||||
}
|
||||
async function inkGImport(){
|
||||
const m=$('#ik_gimp'); m.textContent='importing… (enriching new titles, please wait)';
|
||||
const d=await(await fetch('/admin/intake/gsheet/import',{method:'POST',headers:hdr()})).json();
|
||||
m.innerHTML=d.ok?`<span style="color:var(--ok)">✓ staged ${d.staged} new</span>${d.skipped?` · ${d.skipped} already stocked`:''}`:'<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>';
|
||||
}
|
||||
async function inkPreview(){
|
||||
const url=$('#ik_url').value.trim(); if(!url)return;
|
||||
$('#ik_sheet').innerHTML='<div class="muted" style="padding:12px">reading sheet…</div>';
|
||||
@ -648,7 +669,7 @@ async function inkPreview(){
|
||||
async function inkImport(){
|
||||
const url=$('#ik_sheet').dataset.url, m=$('#ik_imp'); m.textContent='importing… (enriching new titles, please wait)';
|
||||
const d=await(await fetch('/admin/intake/sheet/import',{method:'POST',headers:hdr(),body:JSON.stringify({url})})).json();
|
||||
m.innerHTML = d.ok ? `<span style="color:var(--ok)">✓ staged ${d.staged} rows</span>` : '<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>';
|
||||
m.innerHTML = d.ok ? `<span style="color:var(--ok)">✓ staged ${d.staged} new</span>${d.skipped?` · ${d.skipped} already stocked`:''}` : '<span style="color:var(--warn)">'+escapeH(d.error||'failed')+'</span>';
|
||||
}
|
||||
|
||||
// ---------- Crates ----------
|
||||
|
||||
Loading…
Reference in New Issue
Block a user