From f9ab9527f3d7d99648242d85a68601bc5232ccee Mon Sep 17 00:00:00 2001 From: type-two Date: Thu, 25 Jun 2026 16:10:36 +1000 Subject: [PATCH] =?UTF-8?q?perf(search):=20pg=5Ftrgm=20fuzzy=20staff=20sea?= =?UTF-8?q?rch=20=E2=80=94=20typo-tolerant,=20multi-word,=20~100ms=20(was?= =?UTF-8?q?=20ILIKE=20seq=20scan)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Postgres answer to WowPlatter's sluggish rigid search. Was ILIKE '%..%' → Seq Scan (3038 cost), zero typo tolerance, and multi-word phrases ('bob marley legend 2022') returned NOTHING (needs a contiguous substring). Now: pg_trgm GIN index on disc_release.search_text (already populated) + word_similarity operators — :q <% search_text (filter, GIN-accelerated) ranked by :q <<-> search_text. Threshold pinned at 0.45 on the DB (recall vs speed sweet spot ~140ms; 0.3 too loose/slow, 0.6 too strict). Added the missing disc_release_format(release_id) index (was seq-scanning per row) + inventory.title trgm for stock-finder. All DDL idempotent in _STARTUP_DDL. Live: 'bob marly survivl' (2 typos) -> Survival in 117ms. Co-Authored-By: Claude Opus 4.8 --- app/intake_routes.py | 11 +++++++---- app/main.py | 7 +++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/intake_routes.py b/app/intake_routes.py index 3e3c7e1..22c7b80 100644 --- a/app/intake_routes.py +++ b/app/intake_routes.py @@ -184,10 +184,13 @@ async def stage(body: StageIn, ident=Depends(require_token), db=Depends(get_db)) async def search(q: str = Query(""), label: str = Query(""), year: int | None = None, country: str = Query(""), fmt: str = Query(""), ident=Depends(require_token), db=Depends(get_db)): - """Search the local Discogs mirror — the fast daily path (re-stocks resolve instantly).""" - where, params = ["TRUE"], {} + """Search the local Discogs mirror — fast + typo-tolerant via pg_trgm on search_text + (word_similarity: multi-word, any order, fuzzy; GIN-indexed; threshold 0.3 pinned on the DB).""" + where, params, order = ["TRUE"], {}, "dr.title" if q: - where.append("(dr.title ILIKE :q OR dr.artists_sort ILIKE :q)"); params["q"] = f"%{q}%" + where.append(":q <% dr.search_text") # typo-tolerant word match, GIN-accelerated + order = ":q <<-> dr.search_text" # rank by word distance (closest first) + params["q"] = q if year: where.append("dr.year = :y"); params["y"] = year if country: @@ -207,7 +210,7 @@ async def search(q: str = Query(""), label: str = Query(""), year: int | None = EXISTS (SELECT 1 FROM inventory i WHERE i.release_id=dr.id AND i.in_stock AND i.store_id=1) AS in_stock FROM disc_release dr LEFT JOIN disc_cache dc ON dc.release_id=dr.id WHERE {' AND '.join(where)} - ORDER BY dr.title LIMIT 40"""), params)).mappings()] + ORDER BY {order} LIMIT 40"""), params)).mappings()] return {"items": rows, "source": "local"} diff --git a/app/main.py b/app/main.py index 312e334..2ebb96d 100644 --- a/app/main.py +++ b/app/main.py @@ -91,6 +91,13 @@ _STARTUP_DDL = [ "CREATE SEQUENCE IF NOT EXISTS sale_items_id_seq", "ALTER TABLE sale_items ALTER COLUMN id SET DEFAULT nextval('sale_items_id_seq')", "SELECT setval('sale_items_id_seq', GREATEST(1, (SELECT coalesce(max(id),0) FROM sale_items)))", + # fast fuzzy staff search — pg_trgm GIN indexes (typo-tolerant, multi-word, index-accelerated; + # replaces the ILIKE '%..%' seq scans). word_similarity threshold pinned on the DB (0.3 = good recall). + "CREATE EXTENSION IF NOT EXISTS pg_trgm", + "ALTER DATABASE recordgod SET pg_trgm.word_similarity_threshold = 0.45", # recall vs speed sweet spot (~140ms) + "CREATE INDEX IF NOT EXISTS disc_release_search_trgm ON disc_release USING gin (search_text gin_trgm_ops)", + "CREATE INDEX IF NOT EXISTS inventory_title_trgm ON inventory USING gin (title gin_trgm_ops)", + "CREATE INDEX IF NOT EXISTS disc_release_format_release_id_idx ON disc_release_format(release_id)", ]