Fix: Auto-download tokenizer files for FIBO-vlm when missing (#283)

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.12.1] - 2025-11-27
+
+### 🐛 Bug Fixes
+
+- **FIBO VLM Tokenizer Download**: Fixed an issue where the FIBO VLM tokenizer files would not download automatically when the model weights were cached but tokenizer files were missing. The initializer now properly checks for tokenizer file existence and downloads them if needed.
+
+---
+
 ## [0.12.0] - 2025-11-27
 
 # MFLUX v.0.12.0 Release Notes

pyproject.toml

@@ -17,7 +17,7 @@ source-exclude = [
 
 [project]
 name = "mflux"
-version = "0.12.0"
+version = "0.12.1"
 description = "A MLX port of FLUX based on the Huggingface Diffusers implementation."
 readme = "README.md"
 keywords = ["diffusers", "flux", "mlx"]

src/mflux/models/fibo_vlm/fibo_vlm_initializer.py

@@ -44,22 +44,7 @@ class FIBOVLMInitializer:
         if local_path:
             root_path = Path(local_path)
         else:
-            # Use snapshot_download to get cached path - download ALL files without filtering
-            # This ensures we get the complete snapshot directory structure
-            try:
-                root_path = Path(
-                    snapshot_download(
-                        repo_id=model_id,
-                        local_files_only=True,
-                    )
-                )
-            except (FileNotFoundError, OSError):
-                # Model not in cache, allow download if online
-                root_path = Path(
-                    snapshot_download(
-                        repo_id=model_id,
-                    )
-                )
+            root_path = FIBOVLMInitializer._get_model_path(model_id)
 
         # Try different possible tokenizer paths (like FiboTokenizerHandler does)
         tokenizer_path = root_path / "tokenizer"
@@ -86,3 +71,42 @@ class FIBOVLMInitializer:
                 os.environ["HF_HUB_OFFLINE"] = old_offline
 
         return tokenizer
+
+    @staticmethod
+    def _get_model_path(model_id: str) -> Path:
+        # Try to use cached path first
+        try:
+            root_path = Path(
+                snapshot_download(
+                    repo_id=model_id,
+                    local_files_only=True,
+                )
+            )
+            # Check if tokenizer files actually exist - the model weights might be cached
+            # but tokenizer files may not have been downloaded yet
+            if not FIBOVLMInitializer._tokenizer_files_exist(root_path):
+                # Tokenizer files missing, need to download them
+                root_path = Path(
+                    snapshot_download(
+                        repo_id=model_id,
+                        local_files_only=False,
+                    )
+                )
+        except (FileNotFoundError, OSError):
+            # Model not in cache, allow download if online
+            root_path = Path(
+                snapshot_download(
+                    repo_id=model_id,
+                )
+            )
+        return root_path
+
+    @staticmethod
+    def _tokenizer_files_exist(root_path: Path) -> bool:
+        # Check for vocab.json in possible tokenizer locations
+        possible_paths = [
+            root_path / "tokenizer" / "vocab.json",
+            root_path / "text_encoder" / "vocab.json",
+            root_path / "vocab.json",
+        ]
+        return any(p.exists() for p in possible_paths)

uv.lock

@@ -769,7 +769,7 @@ wheels = [
 
 [[package]]
 name = "mflux"
-version = "0.12.0.dev0"
+version = "0.12.1"
 source = { editable = "." }
 dependencies = [
     { name = "accelerate" },
@@ -813,7 +813,7 @@ requires-dist = [
     { name = "huggingface-hub", specifier = ">=0.24.5,<1.0" },
     { name = "matplotlib", specifier = ">=3.9.2,<4.0" },
     { name = "matplotlib", marker = "extra == 'dev'", specifier = ">3.10,<4.0" },
-    { name = "mlx", specifier = ">=0.27.0,<0.30.0" },
+    { name = "mlx", specifier = ">=0.27.0,<0.31.0" },
     { name = "mlx", marker = "extra == 'dev'", specifier = "==0.29.2" },
     { name = "numpy", specifier = ">=2.0.1,<3.0" },
     { name = "opencv-python", specifier = ">=4.10.0,<5.0" },