Update dependencies (#288)

2025-12-03 22:13:55 +01:00 · 2025-12-03 22:13:55 +01:00 · 6551e9f6a1
commit 6551e9f6a1
parent 37d202d0df
10 changed files with 207 additions and 51 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -76,8 +76,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 - **`--path` flag removed**: The deprecated `--path` flag for loading models has been removed. Use `--model` instead for local paths, HuggingFace repos, or predefined model names.

+### 📦 Dependency Updates
+
+- **Updated `huggingface-hub`** from `>=0.24.5,<1.0` to `>=1.1.6,<2.0`
+  - v1.1.6 includes fix for incomplete file listing in `snapshot_download` which could cause cache corruption
+  - Removed explicit `accelerate` and `filelock` dependencies (pulled in as transitive dependencies)
+- **Updated `transformers`** from `>=4.57,<5.0` to `>=5.0.0rc0,<6.0`
+  - Required for `huggingface-hub` 1.x compatibility
+  - Added workaround for `Qwen2Tokenizer` bug in transformers 5.0.0rc0 where vocab/merges files are not loaded correctly via `from_pretrained()`
+
 ### 🐛 Bug Fixes

+- **Qwen empty negative prompt crash**: Fixed crash when running Qwen models without a `--negative-prompt` argument. Empty prompts now use a space as fallback to ensure valid tokenization.
+
 - **`--model` flag not working**: Fixed bug where the `--model` argument wasn't being used for loading models from HuggingFace or local paths. All CLI commands now correctly use `--model` for model path resolution.
 - **Model Saving Index File**: Fixed issue where locally saved models (via `mflux-save`) would fail to load when uploaded to HuggingFace, due to missing `model.safetensors.index.json`. The model saver now generates this index file alongside the safetensor shards, ensuring compatibility with both mflux and standard HuggingFace loading paths. (see [#285](https://github.com/filipstrand/mflux/issues/285))

--- a/pyproject.toml
+++ b/pyproject.toml
@ -14,7 +14,7 @@ source-exclude = [

 [project]
 name = "mflux"
-version = "0.13.0.dev0"
+version = "0.13.0.dev1"
 description = "A MLX port of FLUX based on the Huggingface Diffusers implementation."
 readme = "README.md"
 keywords = ["diffusers", "flux", "mlx"]
@ -23,9 +23,7 @@ maintainers = [{ name = "Filip Strand", email = "strand.filip@gmail.com" }]
 license = { file = "LICENSE" }
 requires-python = ">=3.10"
 dependencies = [
-    "accelerate>=0.31.0",
-    "filelock>=3.18.0",
-    "huggingface-hub>=0.24.5,<1.0",
+    "huggingface-hub>=1.1.6,<2.0",
    "matplotlib>=3.9.2,<4.0",
    "mlx>=0.27.0,<0.31.0",
    "numpy>=2.0.1,<3.0",
@ -46,7 +44,7 @@ dependencies = [
    "torch>=2.3.1,<3.0; python_version<'3.13'",
    "torch>=2.8.0,<3.0; python_version>='3.13'",
    "tqdm>=4.66.5,<5.0",
-    "transformers>=4.57,<5.0",
+    "transformers>=5.0.0rc0,<6.0",
    "twine>=6.1.0,<7.0",
 ]
 classifiers = [
--- a/src/mflux/models/common/tokenizer/tokenizer_loader.py
+++ b/src/mflux/models/common/tokenizer/tokenizer_loader.py
@ -108,11 +108,84 @@ class TokenizerLoader:
        else:
            raise ValueError(f"Unknown tokenizer class: {tokenizer_class}")

+        # Workaround for Qwen2Tokenizer bug in transformers 5.0.0rc0
+        # The tokenizer doesn't properly load vocab/merges from files, need to pass them directly
+        if tokenizer_class == "Qwen2Tokenizer":
+            return TokenizerLoader._load_qwen2_tokenizer_workaround(tokenizer_path, cls)
+
        return cls.from_pretrained(
            pretrained_model_name_or_path=str(tokenizer_path),
            local_files_only=True,
        )

+    @staticmethod
+    def _load_qwen2_tokenizer_workaround(tokenizer_path: Path, cls):
+        """Load Qwen2Tokenizer with explicit vocab and merges data.
+
+        In transformers 5.0, Qwen2Tokenizer builds its internal BPE tokenizer from the
+        `vocab` and `merges` parameters BEFORE calling super().__init__(). The `vocab_file`
+        and `merges_file` parameters are only passed to the parent class for saving purposes
+        and are NOT automatically loaded. We must load and parse these files ourselves.
+
+        We also need to load special tokens from tokenizer_config.json to ensure tokens like
+        <|im_start|>, <|im_end|>, <|vision_start|>, etc. are properly registered.
+        """
+        import json
+
+        from tokenizers import AddedToken
+
+        vocab_file = tokenizer_path / "vocab.json"
+        merges_file = tokenizer_path / "merges.txt"
+        config_file = tokenizer_path / "tokenizer_config.json"
+
+        if not vocab_file.exists() or not merges_file.exists():
+            # Fall back to standard loading if files don't exist
+            return cls.from_pretrained(
+                pretrained_model_name_or_path=str(tokenizer_path),
+                local_files_only=True,
+            )
+
+        # Load vocab as dict[str, int]
+        with open(vocab_file, encoding="utf-8") as f:
+            vocab = json.load(f)
+
+        # Load merges as list of tuples (pair of strings)
+        # The merges.txt file has format: "token1 token2" per line (skip header if present)
+        merges = []
+        with open(merges_file, encoding="utf-8") as f:
+            for line in f:
+                line = line.rstrip()
+                # Skip empty lines and the optional version header (e.g., "#version: 0.2")
+                if not line or line.startswith("#"):
+                    continue
+                parts = line.split()
+                if len(parts) == 2:
+                    merges.append((parts[0], parts[1]))
+
+        # Load tokenizer config for special tokens
+        config_kwargs = {}
+        if config_file.exists():
+            with open(config_file, encoding="utf-8") as f:
+                config = json.load(f)
+
+            # Extract added_tokens_decoder to add special tokens
+            added_tokens_decoder = config.get("added_tokens_decoder", {})
+            if added_tokens_decoder:
+                # Convert to the format expected by the tokenizer
+                config_kwargs["added_tokens_decoder"] = {
+                    int(k): AddedToken(
+                        content=v["content"],
+                        lstrip=v.get("lstrip", False),
+                        rstrip=v.get("rstrip", False),
+                        single_word=v.get("single_word", False),
+                        normalized=v.get("normalized", False),
+                        special=v.get("special", True),
+                    )
+                    for k, v in added_tokens_decoder.items()
+                }
+
+        return cls(vocab=vocab, merges=merges, **config_kwargs)
+
    @staticmethod
    def _create_tokenizer(
        raw_tokenizer,
--- a/src/mflux/models/common/weights/loading/weight_loader.py
+++ b/src/mflux/models/common/weights/loading/weight_loader.py
@ -314,4 +314,4 @@ class WeightLoader:

    @staticmethod
    def _convert_precision(weights: dict[str, mx.array], precision: mx.Dtype) -> dict[str, mx.array]:
-        return {k: v.astype(precision) for k, v in weights.items()}
+        return {k: v if v.dtype == precision else v.astype(precision) for k, v in weights.items()}
--- a/src/mflux/models/qwen/model/qwen_text_encoder/qwen_prompt_encoder.py
+++ b/src/mflux/models/qwen/model/qwen_text_encoder/qwen_prompt_encoder.py
@ -13,6 +13,10 @@ class QwenPromptEncoder:
        qwen_tokenizer: Tokenizer,
        qwen_text_encoder: QwenTextEncoder,
    ) -> tuple[mx.array, mx.array, mx.array, mx.array]:
+        # Use a space as fallback for empty negative prompt to ensure valid tokenization
+        if not negative_prompt or not negative_prompt.strip():
+            negative_prompt = " "
+
        # 0. Create a cache key that combines both prompts
        cache_key = f"{prompt}|NEG|{negative_prompt}"

@ -22,15 +26,17 @@ class QwenPromptEncoder:

        # 2. Encode the positive prompt
        pos_output = qwen_tokenizer.tokenize(prompt)
-        neg_output = qwen_tokenizer.tokenize(negative_prompt)
        prompt_embeds, prompt_mask = qwen_text_encoder(
            input_ids=pos_output.input_ids, attention_mask=pos_output.attention_mask
        )
+
+        # 3. Encode the negative prompt
+        neg_output = qwen_tokenizer.tokenize(negative_prompt)
        neg_prompt_embeds, neg_prompt_mask = qwen_text_encoder(
            input_ids=neg_output.input_ids, attention_mask=neg_output.attention_mask
        )

-        # 3. Cache the result (all 4 values)
+        # 4. Cache the result (all 4 values)
        result = (prompt_embeds, prompt_mask, neg_prompt_embeds, neg_prompt_mask)
        prompt_cache[cache_key] = result
        return result
--- a/tests/image_generation/helpers/image_generation_edit_test_helper.py
+++ b/tests/image_generation/helpers/image_generation_edit_test_helper.py
@ -2,6 +2,7 @@ import os
 from pathlib import Path
 from typing import Any, Type

+from mflux.callbacks.instances.memory_saver import MemorySaver
 from mflux.callbacks.instances.stepwise_handler import StepwiseHandler
 from mflux.models.common.config import ModelConfig
 from mflux.models.qwen.latent_creator.qwen_latent_creator import QwenLatentCreator
@ -28,6 +29,7 @@ class ImageGeneratorEditTestHelper:
        lora_paths: list[str] | None = None,
        lora_scales: list[float] | None = None,
        mismatch_threshold: float | None = None,
+        low_memory: bool = False,
    ):
        # resolve paths
        reference_image_path = ImageGeneratorEditTestHelper.resolve_path(reference_image_path)
@ -81,6 +83,11 @@ class ImageGeneratorEditTestHelper:
            handler = StepwiseHandler(model=model, output_dir=temp_dir, latent_creator=QwenLatentCreator)
            model.callbacks.register(handler)

+            # Register MemorySaver callback if low_memory mode is enabled
+            if low_memory:
+                memory_saver = MemorySaver(model=model)
+                model.callbacks.register(memory_saver)
+
            image = model.generate_image(**generate_kwargs)
            image.save(path=output_image_path, overwrite=True)

--- a/tests/image_generation/helpers/image_generation_test_helper.py
+++ b/tests/image_generation/helpers/image_generation_test_helper.py
@ -2,6 +2,7 @@ import os
 from pathlib import Path
 from typing import Type, Union

+from mflux.callbacks.instances.memory_saver import MemorySaver
 from mflux.models.common.config import ModelConfig
 from mflux.models.flux.variants.txt2img.flux import Flux1
 from mflux.models.qwen.variants.txt2img.qwen_image import QwenImage
@ -28,6 +29,7 @@ class ImageGeneratorTestHelper:
        negative_prompt: str | None = None,
        guidance: float | None = None,
        mismatch_threshold: float | None = None,
+        low_memory: bool = False,
    ):
        # resolve paths
        reference_image_path = ImageGeneratorTestHelper.resolve_path(reference_image_path)
@ -63,6 +65,11 @@ class ImageGeneratorTestHelper:
            if model_class == QwenImage and negative_prompt is not None:
                generate_kwargs["negative_prompt"] = negative_prompt

+            # Register MemorySaver callback if low_memory mode is enabled
+            if low_memory:
+                memory_saver = MemorySaver(model=model)
+                model.callbacks.register(memory_saver)
+
            # when
            image = model.generate_image(**generate_kwargs)
            image.save(path=output_image_path, overwrite=True)
--- a/tests/image_generation/test_generate_image_qwen_image.py
+++ b/tests/image_generation/test_generate_image_qwen_image.py
@ -21,6 +21,7 @@ class TestImageGeneratorQwenImage:
            prompt="Luxury food photograph",
            negative_prompt="ugly, blurry, low quality",
            mismatch_threshold=0.35,  # Qwen models produce visually similar images with minor pixel differences
+            low_memory=True,
        )

    @pytest.mark.slow
@ -40,6 +41,7 @@ class TestImageGeneratorQwenImage:
            prompt="Luxury food photograph of a burger",
            negative_prompt="ugly, blurry, low quality",
            mismatch_threshold=0.35,  # Qwen models produce visually similar images with minor pixel differences
+            low_memory=True,
        )

    @pytest.mark.slow
@ -60,4 +62,5 @@ class TestImageGeneratorQwenImage:
            lora_paths=["lightx2v/Qwen-Image-Lightning:Qwen-Image-Lightning-4steps-V2.0.safetensors"],
            lora_scales=[1.0],
            mismatch_threshold=0.65,  # LoRA tests have higher variance due to model updates
+            low_memory=True,
        )
--- a/tests/image_generation/test_generate_image_qwen_image_edit.py
+++ b/tests/image_generation/test_generate_image_qwen_image_edit.py
@ -18,10 +18,11 @@ class TestImageGeneratorQwenImageEdit:
            height=384,
            width=640,
            guidance=2.5,
-            quantize=8,  # We should probably use at least 8-bit, but it doesn't run on 32GB machines
+            quantize=6,  # We should probably use at least 8-bit, but it doesn't run on 32GB machines
            prompt="Make the hand fistbump the camera instead of showing a flat palm",
            image_path="reference_upscaled.png",
            mismatch_threshold=0.25,
+            low_memory=True,
        )

    @pytest.mark.slow
--- a/uv.lock
+++ b/uv.lock
@ -40,22 +40,17 @@ resolution-markers = [
 ]

 [[package]]
-name = "accelerate"
-version = "1.10.0"
+name = "anyio"
+version = "4.12.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "huggingface-hub" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.13'" },
-    { name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" },
-    { name = "packaging" },
-    { name = "psutil" },
-    { name = "pyyaml" },
-    { name = "safetensors" },
-    { name = "torch" },
+    { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
+    { name = "idna" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f7/66/be171836d86dc5b8698b3a9bf4b9eb10cb53369729939f88bf650167588b/accelerate-1.10.0.tar.gz", hash = "sha256:8270568fda9036b5cccdc09703fef47872abccd56eb5f6d53b54ea5fb7581496", size = 392261 }
+sdist = { url = "https://files.pythonhosted.org/packages/16/ce/8a777047513153587e5434fd752e89334ac33e379aa3497db860eeb60377/anyio-4.12.0.tar.gz", hash = "sha256:73c693b567b0c55130c104d0b43a9baf3aa6a31fc6110116509f27bf75e21ec0", size = 228266 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/30/dd/0107f0aa179869ee9f47ef5a2686abd5e022fdc82af901d535e52fe91ce1/accelerate-1.10.0-py3-none-any.whl", hash = "sha256:260a72b560e100e839b517a331ec85ed495b3889d12886e79d1913071993c5a3", size = 374718 },
+    { url = "https://files.pythonhosted.org/packages/7f/9c/36c5c37947ebfb8c7f22e0eb6e4d188ee2d53aa3880f3f2744fb894f0cb1/anyio-4.12.0-py3-none-any.whl", hash = "sha256:dad2376a628f98eeca4881fc56cd06affd18f659b17a747d3ff0307ced94b1bb", size = 113362 },
 ]

 [[package]]
@ -178,6 +173,18 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626 },
 ]

+[[package]]
+name = "click"
+version = "8.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "platform_system == 'Windows'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274 },
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@ -381,6 +388,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597 },
 ]

+[[package]]
+name = "h11"
+version = "0.16.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 },
+]
+
 [[package]]
 name = "hf-xet"
 version = "1.2.0"
@ -410,23 +426,53 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/cb/44/870d44b30e1dcfb6a65932e3e1506c103a8a5aea9103c337e7a53180322c/hf_xet-1.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:e6584a52253f72c9f52f9e549d5895ca7a471608495c4ecaa6cc73dba2b24d69", size = 2905735 },
 ]

+[[package]]
+name = "httpcore"
+version = "1.0.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784 },
+]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "certifi" },
+    { name = "httpcore" },
+    { name = "idna" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 },
+]
+
 [[package]]
 name = "huggingface-hub"
-version = "0.36.0"
+version = "1.1.7"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
    { name = "filelock" },
    { name = "fsspec" },
-    { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
+    { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
+    { name = "httpx" },
    { name = "packaging" },
    { name = "pyyaml" },
-    { name = "requests" },
+    { name = "shellingham" },
    { name = "tqdm" },
+    { name = "typer-slim" },
    { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/98/63/4910c5fa9128fdadf6a9c5ac138e8b1b6cee4ca44bf7915bbfbce4e355ee/huggingface_hub-0.36.0.tar.gz", hash = "sha256:47b3f0e2539c39bf5cde015d63b72ec49baff67b6931c3d97f3f84532e2b8d25", size = 463358 }
+sdist = { url = "https://files.pythonhosted.org/packages/6f/fa/a1a94c55637f2b7cfeb05263ac3881aa87c82df92d8b4b31c909079f4419/huggingface_hub-1.1.7.tar.gz", hash = "sha256:3c84b6283caca928595f08fd42e9a572f17ec3501dec508c3f2939d94bfbd9d2", size = 607537 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/cb/bd/1a875e0d592d447cbc02805fd3fe0f497714d6a2583f59d14fa9ebad96eb/huggingface_hub-0.36.0-py3-none-any.whl", hash = "sha256:7bcc9ad17d5b3f07b57c78e79d527102d08313caa278a641993acddcb894548d", size = 566094 },
+    { url = "https://files.pythonhosted.org/packages/dd/4f/82e5ab009089a2c48472bf4248391fe4091cf0b9c3e951dbb8afe3b23d76/huggingface_hub-1.1.7-py3-none-any.whl", hash = "sha256:f3efa4779f4890e44c957bbbb0f197e6028887ad09f0cf95a21659fa7753605d", size = 516239 },
 ]

 [[package]]
@ -768,11 +814,9 @@ wheels = [

 [[package]]
 name = "mflux"
-version = "0.13.0.dev0"
+version = "0.13.0.dev1"
 source = { editable = "." }
 dependencies = [
-    { name = "accelerate" },
-    { name = "filelock" },
    { name = "huggingface-hub" },
    { name = "matplotlib" },
    { name = "mlx" },
@ -806,9 +850,7 @@ dev = [

 [package.metadata]
 requires-dist = [
-    { name = "accelerate", specifier = ">=0.31.0" },
-    { name = "filelock", specifier = ">=3.18.0" },
-    { name = "huggingface-hub", specifier = ">=0.24.5,<1.0" },
+    { name = "huggingface-hub", specifier = ">=1.1.6,<2.0" },
    { name = "matplotlib", specifier = ">=3.9.2,<4.0" },
    { name = "matplotlib", marker = "extra == 'dev'", specifier = ">3.10,<4.0" },
    { name = "mlx", specifier = ">=0.27.0,<0.31.0" },
@ -833,7 +875,7 @@ requires-dist = [
    { name = "torch", marker = "python_full_version < '3.13'", specifier = ">=2.3.1,<3.0" },
    { name = "torch", marker = "python_full_version >= '3.13'", specifier = ">=2.8.0,<3.0" },
    { name = "tqdm", specifier = ">=4.66.5,<5.0" },
-    { name = "transformers", specifier = ">=4.57,<5.0" },
+    { name = "transformers", specifier = ">=5.0.0rc0,<6.0" },
    { name = "twine", specifier = ">=6.1.0,<7.0" },
 ]

@ -1497,21 +1539,6 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 },
 ]

-[[package]]
-name = "psutil"
-version = "7.0.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051 },
-    { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535 },
-    { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004 },
-    { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986 },
-    { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544 },
-    { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053 },
-    { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885 },
-]
-
 [[package]]
 name = "pycparser"
 version = "2.22"
@ -1947,6 +1974,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486 },
 ]

+[[package]]
+name = "shellingham"
+version = "1.5.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 },
+]
+
 [[package]]
 name = "six"
 version = "1.17.0"
@ -2115,7 +2151,7 @@ wheels = [

 [[package]]
 name = "transformers"
-version = "4.57.3"
+version = "5.0.0rc0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
    { name = "filelock" },
@ -2129,10 +2165,11 @@ dependencies = [
    { name = "safetensors" },
    { name = "tokenizers" },
    { name = "tqdm" },
+    { name = "typer-slim" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/dd/70/d42a739e8dfde3d92bb2fff5819cbf331fe9657323221e79415cd5eb65ee/transformers-4.57.3.tar.gz", hash = "sha256:df4945029aaddd7c09eec5cad851f30662f8bd1746721b34cc031d70c65afebc", size = 10139680 }
+sdist = { url = "https://files.pythonhosted.org/packages/94/29/835a5522b672c394b1c808b4f94a121b95fc8a8c58dc61404d69668f9d3e/transformers-5.0.0rc0.tar.gz", hash = "sha256:bb427000caa4a88943704f80448b2323ad8c6a2f4f13c1433e27d0a1f690c975", size = 8181669 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/6a/6b/2f416568b3c4c91c96e5a365d164f8a4a4a88030aa8ab4644181fdadce97/transformers-4.57.3-py3-none-any.whl", hash = "sha256:c77d353a4851b1880191603d36acb313411d3577f6e2897814f333841f7003f4", size = 11993463 },
+    { url = "https://files.pythonhosted.org/packages/8e/a9/fc60ee518b57fefca1551b15b36a5e934a101148327003fd18741c732b5b/transformers-5.0.0rc0-py3-none-any.whl", hash = "sha256:1935f8b396891c93b8520d951d4385da1b1b778914e1d79ed151ddbd32d83a22", size = 9815900 },
 ]

 [[package]]
@ -2176,6 +2213,19 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/7c/b6/74e927715a285743351233f33ea3c684528a0d374d2e43ff9ce9585b73fe/twine-6.1.0-py3-none-any.whl", hash = "sha256:a47f973caf122930bf0fbbf17f80b83bc1602c9ce393c7845f289a3001dc5384", size = 40791 },
 ]

+[[package]]
+name = "typer-slim"
+version = "0.20.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8e/45/81b94a52caed434b94da65729c03ad0fb7665fab0f7db9ee54c94e541403/typer_slim-0.20.0.tar.gz", hash = "sha256:9fc6607b3c6c20f5c33ea9590cbeb17848667c51feee27d9e314a579ab07d1a3", size = 106561 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5e/dd/5cbf31f402f1cc0ab087c94d4669cfa55bd1e818688b910631e131d74e75/typer_slim-0.20.0-py3-none-any.whl", hash = "sha256:f42a9b7571a12b97dddf364745d29f12221865acef7a2680065f9bb29c7dc89d", size = 47087 },
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.14.1"