From 6551e9f6a1ff7245d6652c7c26e1c677297526ef Mon Sep 17 00:00:00 2001 From: Filip Strand Date: Wed, 3 Dec 2025 22:13:55 +0100 Subject: [PATCH] Update dependencies (#288) --- CHANGELOG.md | 11 ++ pyproject.toml | 8 +- .../common/tokenizer/tokenizer_loader.py | 73 ++++++++++ .../common/weights/loading/weight_loader.py | 2 +- .../qwen_text_encoder/qwen_prompt_encoder.py | 10 +- .../image_generation_edit_test_helper.py | 7 + .../helpers/image_generation_test_helper.py | 7 + .../test_generate_image_qwen_image.py | 3 + .../test_generate_image_qwen_image_edit.py | 3 +- uv.lock | 134 ++++++++++++------ 10 files changed, 207 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 700d3c0..6953b04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,8 +76,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`--path` flag removed**: The deprecated `--path` flag for loading models has been removed. Use `--model` instead for local paths, HuggingFace repos, or predefined model names. +### 📦 Dependency Updates + +- **Updated `huggingface-hub`** from `>=0.24.5,<1.0` to `>=1.1.6,<2.0` + - v1.1.6 includes fix for incomplete file listing in `snapshot_download` which could cause cache corruption + - Removed explicit `accelerate` and `filelock` dependencies (pulled in as transitive dependencies) +- **Updated `transformers`** from `>=4.57,<5.0` to `>=5.0.0rc0,<6.0` + - Required for `huggingface-hub` 1.x compatibility + - Added workaround for `Qwen2Tokenizer` bug in transformers 5.0.0rc0 where vocab/merges files are not loaded correctly via `from_pretrained()` + ### 🐛 Bug Fixes +- **Qwen empty negative prompt crash**: Fixed crash when running Qwen models without a `--negative-prompt` argument. Empty prompts now use a space as fallback to ensure valid tokenization. + - **`--model` flag not working**: Fixed bug where the `--model` argument wasn't being used for loading models from HuggingFace or local paths. All CLI commands now correctly use `--model` for model path resolution. - **Model Saving Index File**: Fixed issue where locally saved models (via `mflux-save`) would fail to load when uploaded to HuggingFace, due to missing `model.safetensors.index.json`. The model saver now generates this index file alongside the safetensor shards, ensuring compatibility with both mflux and standard HuggingFace loading paths. (see [#285](https://github.com/filipstrand/mflux/issues/285)) diff --git a/pyproject.toml b/pyproject.toml index a7f06b1..e0fdb0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ source-exclude = [ [project] name = "mflux" -version = "0.13.0.dev0" +version = "0.13.0.dev1" description = "A MLX port of FLUX based on the Huggingface Diffusers implementation." readme = "README.md" keywords = ["diffusers", "flux", "mlx"] @@ -23,9 +23,7 @@ maintainers = [{ name = "Filip Strand", email = "strand.filip@gmail.com" }] license = { file = "LICENSE" } requires-python = ">=3.10" dependencies = [ - "accelerate>=0.31.0", - "filelock>=3.18.0", - "huggingface-hub>=0.24.5,<1.0", + "huggingface-hub>=1.1.6,<2.0", "matplotlib>=3.9.2,<4.0", "mlx>=0.27.0,<0.31.0", "numpy>=2.0.1,<3.0", @@ -46,7 +44,7 @@ dependencies = [ "torch>=2.3.1,<3.0; python_version<'3.13'", "torch>=2.8.0,<3.0; python_version>='3.13'", "tqdm>=4.66.5,<5.0", - "transformers>=4.57,<5.0", + "transformers>=5.0.0rc0,<6.0", "twine>=6.1.0,<7.0", ] classifiers = [ diff --git a/src/mflux/models/common/tokenizer/tokenizer_loader.py b/src/mflux/models/common/tokenizer/tokenizer_loader.py index 319cf9c..7def970 100644 --- a/src/mflux/models/common/tokenizer/tokenizer_loader.py +++ b/src/mflux/models/common/tokenizer/tokenizer_loader.py @@ -108,11 +108,84 @@ class TokenizerLoader: else: raise ValueError(f"Unknown tokenizer class: {tokenizer_class}") + # Workaround for Qwen2Tokenizer bug in transformers 5.0.0rc0 + # The tokenizer doesn't properly load vocab/merges from files, need to pass them directly + if tokenizer_class == "Qwen2Tokenizer": + return TokenizerLoader._load_qwen2_tokenizer_workaround(tokenizer_path, cls) + return cls.from_pretrained( pretrained_model_name_or_path=str(tokenizer_path), local_files_only=True, ) + @staticmethod + def _load_qwen2_tokenizer_workaround(tokenizer_path: Path, cls): + """Load Qwen2Tokenizer with explicit vocab and merges data. + + In transformers 5.0, Qwen2Tokenizer builds its internal BPE tokenizer from the + `vocab` and `merges` parameters BEFORE calling super().__init__(). The `vocab_file` + and `merges_file` parameters are only passed to the parent class for saving purposes + and are NOT automatically loaded. We must load and parse these files ourselves. + + We also need to load special tokens from tokenizer_config.json to ensure tokens like + <|im_start|>, <|im_end|>, <|vision_start|>, etc. are properly registered. + """ + import json + + from tokenizers import AddedToken + + vocab_file = tokenizer_path / "vocab.json" + merges_file = tokenizer_path / "merges.txt" + config_file = tokenizer_path / "tokenizer_config.json" + + if not vocab_file.exists() or not merges_file.exists(): + # Fall back to standard loading if files don't exist + return cls.from_pretrained( + pretrained_model_name_or_path=str(tokenizer_path), + local_files_only=True, + ) + + # Load vocab as dict[str, int] + with open(vocab_file, encoding="utf-8") as f: + vocab = json.load(f) + + # Load merges as list of tuples (pair of strings) + # The merges.txt file has format: "token1 token2" per line (skip header if present) + merges = [] + with open(merges_file, encoding="utf-8") as f: + for line in f: + line = line.rstrip() + # Skip empty lines and the optional version header (e.g., "#version: 0.2") + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) == 2: + merges.append((parts[0], parts[1])) + + # Load tokenizer config for special tokens + config_kwargs = {} + if config_file.exists(): + with open(config_file, encoding="utf-8") as f: + config = json.load(f) + + # Extract added_tokens_decoder to add special tokens + added_tokens_decoder = config.get("added_tokens_decoder", {}) + if added_tokens_decoder: + # Convert to the format expected by the tokenizer + config_kwargs["added_tokens_decoder"] = { + int(k): AddedToken( + content=v["content"], + lstrip=v.get("lstrip", False), + rstrip=v.get("rstrip", False), + single_word=v.get("single_word", False), + normalized=v.get("normalized", False), + special=v.get("special", True), + ) + for k, v in added_tokens_decoder.items() + } + + return cls(vocab=vocab, merges=merges, **config_kwargs) + @staticmethod def _create_tokenizer( raw_tokenizer, diff --git a/src/mflux/models/common/weights/loading/weight_loader.py b/src/mflux/models/common/weights/loading/weight_loader.py index c483dda..572ba2a 100644 --- a/src/mflux/models/common/weights/loading/weight_loader.py +++ b/src/mflux/models/common/weights/loading/weight_loader.py @@ -314,4 +314,4 @@ class WeightLoader: @staticmethod def _convert_precision(weights: dict[str, mx.array], precision: mx.Dtype) -> dict[str, mx.array]: - return {k: v.astype(precision) for k, v in weights.items()} + return {k: v if v.dtype == precision else v.astype(precision) for k, v in weights.items()} diff --git a/src/mflux/models/qwen/model/qwen_text_encoder/qwen_prompt_encoder.py b/src/mflux/models/qwen/model/qwen_text_encoder/qwen_prompt_encoder.py index 29508e5..8bb3afa 100644 --- a/src/mflux/models/qwen/model/qwen_text_encoder/qwen_prompt_encoder.py +++ b/src/mflux/models/qwen/model/qwen_text_encoder/qwen_prompt_encoder.py @@ -13,6 +13,10 @@ class QwenPromptEncoder: qwen_tokenizer: Tokenizer, qwen_text_encoder: QwenTextEncoder, ) -> tuple[mx.array, mx.array, mx.array, mx.array]: + # Use a space as fallback for empty negative prompt to ensure valid tokenization + if not negative_prompt or not negative_prompt.strip(): + negative_prompt = " " + # 0. Create a cache key that combines both prompts cache_key = f"{prompt}|NEG|{negative_prompt}" @@ -22,15 +26,17 @@ class QwenPromptEncoder: # 2. Encode the positive prompt pos_output = qwen_tokenizer.tokenize(prompt) - neg_output = qwen_tokenizer.tokenize(negative_prompt) prompt_embeds, prompt_mask = qwen_text_encoder( input_ids=pos_output.input_ids, attention_mask=pos_output.attention_mask ) + + # 3. Encode the negative prompt + neg_output = qwen_tokenizer.tokenize(negative_prompt) neg_prompt_embeds, neg_prompt_mask = qwen_text_encoder( input_ids=neg_output.input_ids, attention_mask=neg_output.attention_mask ) - # 3. Cache the result (all 4 values) + # 4. Cache the result (all 4 values) result = (prompt_embeds, prompt_mask, neg_prompt_embeds, neg_prompt_mask) prompt_cache[cache_key] = result return result diff --git a/tests/image_generation/helpers/image_generation_edit_test_helper.py b/tests/image_generation/helpers/image_generation_edit_test_helper.py index 7643b81..d9600b6 100644 --- a/tests/image_generation/helpers/image_generation_edit_test_helper.py +++ b/tests/image_generation/helpers/image_generation_edit_test_helper.py @@ -2,6 +2,7 @@ import os from pathlib import Path from typing import Any, Type +from mflux.callbacks.instances.memory_saver import MemorySaver from mflux.callbacks.instances.stepwise_handler import StepwiseHandler from mflux.models.common.config import ModelConfig from mflux.models.qwen.latent_creator.qwen_latent_creator import QwenLatentCreator @@ -28,6 +29,7 @@ class ImageGeneratorEditTestHelper: lora_paths: list[str] | None = None, lora_scales: list[float] | None = None, mismatch_threshold: float | None = None, + low_memory: bool = False, ): # resolve paths reference_image_path = ImageGeneratorEditTestHelper.resolve_path(reference_image_path) @@ -81,6 +83,11 @@ class ImageGeneratorEditTestHelper: handler = StepwiseHandler(model=model, output_dir=temp_dir, latent_creator=QwenLatentCreator) model.callbacks.register(handler) + # Register MemorySaver callback if low_memory mode is enabled + if low_memory: + memory_saver = MemorySaver(model=model) + model.callbacks.register(memory_saver) + image = model.generate_image(**generate_kwargs) image.save(path=output_image_path, overwrite=True) diff --git a/tests/image_generation/helpers/image_generation_test_helper.py b/tests/image_generation/helpers/image_generation_test_helper.py index bb9e695..ed35d28 100644 --- a/tests/image_generation/helpers/image_generation_test_helper.py +++ b/tests/image_generation/helpers/image_generation_test_helper.py @@ -2,6 +2,7 @@ import os from pathlib import Path from typing import Type, Union +from mflux.callbacks.instances.memory_saver import MemorySaver from mflux.models.common.config import ModelConfig from mflux.models.flux.variants.txt2img.flux import Flux1 from mflux.models.qwen.variants.txt2img.qwen_image import QwenImage @@ -28,6 +29,7 @@ class ImageGeneratorTestHelper: negative_prompt: str | None = None, guidance: float | None = None, mismatch_threshold: float | None = None, + low_memory: bool = False, ): # resolve paths reference_image_path = ImageGeneratorTestHelper.resolve_path(reference_image_path) @@ -63,6 +65,11 @@ class ImageGeneratorTestHelper: if model_class == QwenImage and negative_prompt is not None: generate_kwargs["negative_prompt"] = negative_prompt + # Register MemorySaver callback if low_memory mode is enabled + if low_memory: + memory_saver = MemorySaver(model=model) + model.callbacks.register(memory_saver) + # when image = model.generate_image(**generate_kwargs) image.save(path=output_image_path, overwrite=True) diff --git a/tests/image_generation/test_generate_image_qwen_image.py b/tests/image_generation/test_generate_image_qwen_image.py index e9b2d71..e04cc3e 100644 --- a/tests/image_generation/test_generate_image_qwen_image.py +++ b/tests/image_generation/test_generate_image_qwen_image.py @@ -21,6 +21,7 @@ class TestImageGeneratorQwenImage: prompt="Luxury food photograph", negative_prompt="ugly, blurry, low quality", mismatch_threshold=0.35, # Qwen models produce visually similar images with minor pixel differences + low_memory=True, ) @pytest.mark.slow @@ -40,6 +41,7 @@ class TestImageGeneratorQwenImage: prompt="Luxury food photograph of a burger", negative_prompt="ugly, blurry, low quality", mismatch_threshold=0.35, # Qwen models produce visually similar images with minor pixel differences + low_memory=True, ) @pytest.mark.slow @@ -60,4 +62,5 @@ class TestImageGeneratorQwenImage: lora_paths=["lightx2v/Qwen-Image-Lightning:Qwen-Image-Lightning-4steps-V2.0.safetensors"], lora_scales=[1.0], mismatch_threshold=0.65, # LoRA tests have higher variance due to model updates + low_memory=True, ) diff --git a/tests/image_generation/test_generate_image_qwen_image_edit.py b/tests/image_generation/test_generate_image_qwen_image_edit.py index 327ae1a..5c1a948 100644 --- a/tests/image_generation/test_generate_image_qwen_image_edit.py +++ b/tests/image_generation/test_generate_image_qwen_image_edit.py @@ -18,10 +18,11 @@ class TestImageGeneratorQwenImageEdit: height=384, width=640, guidance=2.5, - quantize=8, # We should probably use at least 8-bit, but it doesn't run on 32GB machines + quantize=6, # We should probably use at least 8-bit, but it doesn't run on 32GB machines prompt="Make the hand fistbump the camera instead of showing a flat palm", image_path="reference_upscaled.png", mismatch_threshold=0.25, + low_memory=True, ) @pytest.mark.slow diff --git a/uv.lock b/uv.lock index b176b1f..a3894ab 100644 --- a/uv.lock +++ b/uv.lock @@ -40,22 +40,17 @@ resolution-markers = [ ] [[package]] -name = "accelerate" -version = "1.10.0" +name = "anyio" +version = "4.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.13'" }, - { name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "packaging" }, - { name = "psutil" }, - { name = "pyyaml" }, - { name = "safetensors" }, - { name = "torch" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f7/66/be171836d86dc5b8698b3a9bf4b9eb10cb53369729939f88bf650167588b/accelerate-1.10.0.tar.gz", hash = "sha256:8270568fda9036b5cccdc09703fef47872abccd56eb5f6d53b54ea5fb7581496", size = 392261 } +sdist = { url = "https://files.pythonhosted.org/packages/16/ce/8a777047513153587e5434fd752e89334ac33e379aa3497db860eeb60377/anyio-4.12.0.tar.gz", hash = "sha256:73c693b567b0c55130c104d0b43a9baf3aa6a31fc6110116509f27bf75e21ec0", size = 228266 } wheels = [ - { url = "https://files.pythonhosted.org/packages/30/dd/0107f0aa179869ee9f47ef5a2686abd5e022fdc82af901d535e52fe91ce1/accelerate-1.10.0-py3-none-any.whl", hash = "sha256:260a72b560e100e839b517a331ec85ed495b3889d12886e79d1913071993c5a3", size = 374718 }, + { url = "https://files.pythonhosted.org/packages/7f/9c/36c5c37947ebfb8c7f22e0eb6e4d188ee2d53aa3880f3f2744fb894f0cb1/anyio-4.12.0-py3-none-any.whl", hash = "sha256:dad2376a628f98eeca4881fc56cd06affd18f659b17a747d3ff0307ced94b1bb", size = 113362 }, ] [[package]] @@ -178,6 +173,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626 }, ] +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "platform_system == 'Windows'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274 }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -381,6 +388,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597 }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 }, +] + [[package]] name = "hf-xet" version = "1.2.0" @@ -410,23 +426,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/44/870d44b30e1dcfb6a65932e3e1506c103a8a5aea9103c337e7a53180322c/hf_xet-1.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:e6584a52253f72c9f52f9e549d5895ca7a471608495c4ecaa6cc73dba2b24d69", size = 2905735 }, ] +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784 }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, +] + [[package]] name = "huggingface-hub" -version = "0.36.0" +version = "1.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, - { name = "requests" }, + { name = "shellingham" }, { name = "tqdm" }, + { name = "typer-slim" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/98/63/4910c5fa9128fdadf6a9c5ac138e8b1b6cee4ca44bf7915bbfbce4e355ee/huggingface_hub-0.36.0.tar.gz", hash = "sha256:47b3f0e2539c39bf5cde015d63b72ec49baff67b6931c3d97f3f84532e2b8d25", size = 463358 } +sdist = { url = "https://files.pythonhosted.org/packages/6f/fa/a1a94c55637f2b7cfeb05263ac3881aa87c82df92d8b4b31c909079f4419/huggingface_hub-1.1.7.tar.gz", hash = "sha256:3c84b6283caca928595f08fd42e9a572f17ec3501dec508c3f2939d94bfbd9d2", size = 607537 } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/bd/1a875e0d592d447cbc02805fd3fe0f497714d6a2583f59d14fa9ebad96eb/huggingface_hub-0.36.0-py3-none-any.whl", hash = "sha256:7bcc9ad17d5b3f07b57c78e79d527102d08313caa278a641993acddcb894548d", size = 566094 }, + { url = "https://files.pythonhosted.org/packages/dd/4f/82e5ab009089a2c48472bf4248391fe4091cf0b9c3e951dbb8afe3b23d76/huggingface_hub-1.1.7-py3-none-any.whl", hash = "sha256:f3efa4779f4890e44c957bbbb0f197e6028887ad09f0cf95a21659fa7753605d", size = 516239 }, ] [[package]] @@ -768,11 +814,9 @@ wheels = [ [[package]] name = "mflux" -version = "0.13.0.dev0" +version = "0.13.0.dev1" source = { editable = "." } dependencies = [ - { name = "accelerate" }, - { name = "filelock" }, { name = "huggingface-hub" }, { name = "matplotlib" }, { name = "mlx" }, @@ -806,9 +850,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "accelerate", specifier = ">=0.31.0" }, - { name = "filelock", specifier = ">=3.18.0" }, - { name = "huggingface-hub", specifier = ">=0.24.5,<1.0" }, + { name = "huggingface-hub", specifier = ">=1.1.6,<2.0" }, { name = "matplotlib", specifier = ">=3.9.2,<4.0" }, { name = "matplotlib", marker = "extra == 'dev'", specifier = ">3.10,<4.0" }, { name = "mlx", specifier = ">=0.27.0,<0.31.0" }, @@ -833,7 +875,7 @@ requires-dist = [ { name = "torch", marker = "python_full_version < '3.13'", specifier = ">=2.3.1,<3.0" }, { name = "torch", marker = "python_full_version >= '3.13'", specifier = ">=2.8.0,<3.0" }, { name = "tqdm", specifier = ">=4.66.5,<5.0" }, - { name = "transformers", specifier = ">=4.57,<5.0" }, + { name = "transformers", specifier = ">=5.0.0rc0,<6.0" }, { name = "twine", specifier = ">=6.1.0,<7.0" }, ] @@ -1497,21 +1539,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, ] -[[package]] -name = "psutil" -version = "7.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051 }, - { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535 }, - { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004 }, - { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986 }, - { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544 }, - { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053 }, - { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885 }, -] - [[package]] name = "pycparser" version = "2.22" @@ -1947,6 +1974,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486 }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 }, +] + [[package]] name = "six" version = "1.17.0" @@ -2115,7 +2151,7 @@ wheels = [ [[package]] name = "transformers" -version = "4.57.3" +version = "5.0.0rc0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, @@ -2129,10 +2165,11 @@ dependencies = [ { name = "safetensors" }, { name = "tokenizers" }, { name = "tqdm" }, + { name = "typer-slim" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dd/70/d42a739e8dfde3d92bb2fff5819cbf331fe9657323221e79415cd5eb65ee/transformers-4.57.3.tar.gz", hash = "sha256:df4945029aaddd7c09eec5cad851f30662f8bd1746721b34cc031d70c65afebc", size = 10139680 } +sdist = { url = "https://files.pythonhosted.org/packages/94/29/835a5522b672c394b1c808b4f94a121b95fc8a8c58dc61404d69668f9d3e/transformers-5.0.0rc0.tar.gz", hash = "sha256:bb427000caa4a88943704f80448b2323ad8c6a2f4f13c1433e27d0a1f690c975", size = 8181669 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/6b/2f416568b3c4c91c96e5a365d164f8a4a4a88030aa8ab4644181fdadce97/transformers-4.57.3-py3-none-any.whl", hash = "sha256:c77d353a4851b1880191603d36acb313411d3577f6e2897814f333841f7003f4", size = 11993463 }, + { url = "https://files.pythonhosted.org/packages/8e/a9/fc60ee518b57fefca1551b15b36a5e934a101148327003fd18741c732b5b/transformers-5.0.0rc0-py3-none-any.whl", hash = "sha256:1935f8b396891c93b8520d951d4385da1b1b778914e1d79ed151ddbd32d83a22", size = 9815900 }, ] [[package]] @@ -2176,6 +2213,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/b6/74e927715a285743351233f33ea3c684528a0d374d2e43ff9ce9585b73fe/twine-6.1.0-py3-none-any.whl", hash = "sha256:a47f973caf122930bf0fbbf17f80b83bc1602c9ce393c7845f289a3001dc5384", size = 40791 }, ] +[[package]] +name = "typer-slim" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/45/81b94a52caed434b94da65729c03ad0fb7665fab0f7db9ee54c94e541403/typer_slim-0.20.0.tar.gz", hash = "sha256:9fc6607b3c6c20f5c33ea9590cbeb17848667c51feee27d9e314a579ab07d1a3", size = 106561 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/dd/5cbf31f402f1cc0ab087c94d4669cfa55bd1e818688b910631e131d74e75/typer_slim-0.20.0-py3-none-any.whl", hash = "sha256:f42a9b7571a12b97dddf364745d29f12221865acef7a2680065f9bb29c7dc89d", size = 47087 }, +] + [[package]] name = "typing-extensions" version = "4.14.1"