monster/Qwen-Image-Layered-MRP-MLX
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Return mlx arrays directly from tokenizer call

Browse Source
This commit is contained in:
filipstrand 2024-08-14 11:01:57 +02:00
parent 5023437e89
commit 37a6632259
2 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/flux_1_schnell/tokenizer/clip_tokenizer.py
View File

@ -9,13 +9,12 @@ class TokenizerCLIP:
self.tokenizer = tokenizer
def tokenize(self, prompt: str) -> mx.array:
text_input_ids = self.tokenizer(
return self.tokenizer(
[prompt],
padding="max_length",
max_length=TokenizerCLIP.MAX_TOKEN_LENGTH,
truncation=True,
return_length=False,
return_overflowing_tokens=False,
return_tensors="pt",
return_tensors="mlx",
).input_ids
return mx.array(text_input_ids.cpu().numpy())

5
src/flux_1_schnell/tokenizer/t5_tokenizer.py
View File

@ -9,13 +9,12 @@ class TokenizerT5:
self.tokenizer = tokenizer
def tokenize(self, prompt: str) -> mx.array:
text_input_ids = self.tokenizer(
return self.tokenizer(
[prompt],
padding="max_length",
max_length=TokenizerT5.MAX_TOKEN_LENGTH,
truncation=True,
return_length=False,
return_overflowing_tokens=False,
return_tensors="pt",
return_tensors="mlx",
).input_ids
return mx.array(text_input_ids.cpu().numpy())