From eb831756802a137b9a069cd3c4c5833049771b8e Mon Sep 17 00:00:00 2001
From: Filip Strand <strand.filip@gmail.com>
Date: Thu, 27 Nov 2025 18:15:56 +0100
Subject: [PATCH] Fix: Auto-download tokenizer files for FIBO-vlm when missing
 (#283)

---
 CHANGELOG.md                                  |  8 +++
 pyproject.toml                                |  2 +-
 .../models/fibo_vlm/fibo_vlm_initializer.py   | 56 +++++++++++++------
 uv.lock                                       |  4 +-
 4 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 85ba9c9..82b4394 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.12.1] - 2025-11-27
+
+### 🐛 Bug Fixes
+
+- **FIBO VLM Tokenizer Download**: Fixed an issue where the FIBO VLM tokenizer files would not download automatically when the model weights were cached but tokenizer files were missing. The initializer now properly checks for tokenizer file existence and downloads them if needed.
+
+---
+
 ## [0.12.0] - 2025-11-27
 
 # MFLUX v.0.12.0 Release Notes
diff --git a/pyproject.toml b/pyproject.toml
index ef66ce8..5bb71d6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,7 +17,7 @@ source-exclude = [
 
 [project]
 name = "mflux"
-version = "0.12.0"
+version = "0.12.1"
 description = "A MLX port of FLUX based on the Huggingface Diffusers implementation."
 readme = "README.md"
 keywords = ["diffusers", "flux", "mlx"]
diff --git a/src/mflux/models/fibo_vlm/fibo_vlm_initializer.py b/src/mflux/models/fibo_vlm/fibo_vlm_initializer.py
index 88c0529..9643307 100644
--- a/src/mflux/models/fibo_vlm/fibo_vlm_initializer.py
+++ b/src/mflux/models/fibo_vlm/fibo_vlm_initializer.py
@@ -44,22 +44,7 @@ class FIBOVLMInitializer:
         if local_path:
             root_path = Path(local_path)
         else:
-            # Use snapshot_download to get cached path - download ALL files without filtering
-            # This ensures we get the complete snapshot directory structure
-            try:
-                root_path = Path(
-                    snapshot_download(
-                        repo_id=model_id,
-                        local_files_only=True,
-                    )
-                )
-            except (FileNotFoundError, OSError):
-                # Model not in cache, allow download if online
-                root_path = Path(
-                    snapshot_download(
-                        repo_id=model_id,
-                    )
-                )
+            root_path = FIBOVLMInitializer._get_model_path(model_id)
 
         # Try different possible tokenizer paths (like FiboTokenizerHandler does)
         tokenizer_path = root_path / "tokenizer"
@@ -86,3 +71,42 @@ class FIBOVLMInitializer:
                 os.environ["HF_HUB_OFFLINE"] = old_offline
 
         return tokenizer
+
+    @staticmethod
+    def _get_model_path(model_id: str) -> Path:
+        # Try to use cached path first
+        try:
+            root_path = Path(
+                snapshot_download(
+                    repo_id=model_id,
+                    local_files_only=True,
+                )
+            )
+            # Check if tokenizer files actually exist - the model weights might be cached
+            # but tokenizer files may not have been downloaded yet
+            if not FIBOVLMInitializer._tokenizer_files_exist(root_path):
+                # Tokenizer files missing, need to download them
+                root_path = Path(
+                    snapshot_download(
+                        repo_id=model_id,
+                        local_files_only=False,
+                    )
+                )
+        except (FileNotFoundError, OSError):
+            # Model not in cache, allow download if online
+            root_path = Path(
+                snapshot_download(
+                    repo_id=model_id,
+                )
+            )
+        return root_path
+
+    @staticmethod
+    def _tokenizer_files_exist(root_path: Path) -> bool:
+        # Check for vocab.json in possible tokenizer locations
+        possible_paths = [
+            root_path / "tokenizer" / "vocab.json",
+            root_path / "text_encoder" / "vocab.json",
+            root_path / "vocab.json",
+        ]
+        return any(p.exists() for p in possible_paths)
diff --git a/uv.lock b/uv.lock
index 7076ea8..bc704c2 100644
--- a/uv.lock
+++ b/uv.lock
@@ -769,7 +769,7 @@ wheels = [
 
 [[package]]
 name = "mflux"
-version = "0.12.0.dev0"
+version = "0.12.1"
 source = { editable = "." }
 dependencies = [
     { name = "accelerate" },
@@ -813,7 +813,7 @@ requires-dist = [
     { name = "huggingface-hub", specifier = ">=0.24.5,<1.0" },
     { name = "matplotlib", specifier = ">=3.9.2,<4.0" },
     { name = "matplotlib", marker = "extra == 'dev'", specifier = ">3.10,<4.0" },
-    { name = "mlx", specifier = ">=0.27.0,<0.30.0" },
+    { name = "mlx", specifier = ">=0.27.0,<0.31.0" },
     { name = "mlx", marker = "extra == 'dev'", specifier = "==0.29.2" },
     { name = "numpy", specifier = ">=2.0.1,<3.0" },
     { name = "opencv-python", specifier = ">=4.10.0,<5.0" },