Fix VLM error with transformers v.5

This commit is contained in:
filipstrand 2025-12-03 23:30:16 +01:00
parent ef5fce289a
commit 88a541f329
7 changed files with 144 additions and 115 deletions

View File

@ -30,6 +30,7 @@ class TokenizerLoader:
raw_tokenizer = TokenizerLoader._load_raw_tokenizer(
tokenizer_path=tokenizer_path,
tokenizer_class=definition.tokenizer_class,
chat_template=definition.chat_template,
)
return TokenizerLoader._create_tokenizer(
@ -102,6 +103,7 @@ class TokenizerLoader:
def _load_raw_tokenizer(
tokenizer_path: Path,
tokenizer_class: str,
chat_template: str | None = None,
):
if hasattr(transformers, tokenizer_class):
cls = getattr(transformers, tokenizer_class)
@ -111,12 +113,18 @@ class TokenizerLoader:
# Workaround for Qwen2Tokenizer bug in transformers 5.0.0rc0
# The tokenizer doesn't properly load vocab/merges from files, need to pass them directly
if tokenizer_class == "Qwen2Tokenizer":
return TokenizerLoader._load_qwen2_tokenizer_workaround(tokenizer_path, cls)
tokenizer = TokenizerLoader._load_qwen2_tokenizer_workaround(tokenizer_path, cls)
else:
tokenizer = cls.from_pretrained(
pretrained_model_name_or_path=str(tokenizer_path),
local_files_only=True,
)
return cls.from_pretrained(
pretrained_model_name_or_path=str(tokenizer_path),
local_files_only=True,
)
# Apply chat_template from definition if provided and not already set
if chat_template and not getattr(tokenizer, "chat_template", None):
tokenizer.chat_template = chat_template
return tokenizer
@staticmethod
def _load_qwen2_tokenizer_workaround(tokenizer_path: Path, cls):
@ -162,8 +170,9 @@ class TokenizerLoader:
if len(parts) == 2:
merges.append((parts[0], parts[1]))
# Load tokenizer config for special tokens
# Load tokenizer config for special tokens and chat template
config_kwargs = {}
chat_template = None
if config_file.exists():
with open(config_file, encoding="utf-8") as f:
config = json.load(f)
@ -184,7 +193,16 @@ class TokenizerLoader:
for k, v in added_tokens_decoder.items()
}
return cls(vocab=vocab, merges=merges, **config_kwargs)
# Extract chat_template if present
chat_template = config.get("chat_template")
tokenizer = cls(vocab=vocab, merges=merges, **config_kwargs)
# Set chat_template after initialization (not a constructor param)
if chat_template:
tokenizer.chat_template = chat_template
return tokenizer
@staticmethod
def _create_tokenizer(

View File

@ -57,3 +57,4 @@ class TokenizerDefinition:
add_special_tokens: bool = True
processor_class: type | None = None
image_token: str = "<|image_pad|>"
chat_template: str | None = None # Jinja2 template for apply_chat_template

View File

@ -5,6 +5,15 @@ from mflux.models.common.weights.loading.weight_definition import ComponentDefin
from mflux.models.fibo_vlm.tokenizer.qwen2vl_processor import Qwen2VLProcessor
from mflux.models.fibo_vlm.weights.fibo_vlm_weight_mapping import FIBOVLMWeightMapping
# Qwen2VL chat template (required for apply_chat_template)
QWEN2VL_CHAT_TEMPLATE = """{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
You are a helpful assistant.<|im_end|>
{% endif %}<|im_start|>{{ message['role'] }}
{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
{% endif %}""" # noqa: E501
class FIBOVLMWeightDefinition:
@staticmethod
@ -40,6 +49,7 @@ class FIBOVLMWeightDefinition:
max_length=1024,
fallback_subdirs=["tokenizer", "text_encoder"],
download_patterns=["vocab.json", "merges.txt", "tokenizer.json", "tokenizer_config.json"],
chat_template=QWEN2VL_CHAT_TEMPLATE,
),
]

View File

@ -11,35 +11,40 @@ OWL_PROMPT = """
"location": "center",
"relationship": "The owl is the sole subject, perched comfortably within its environment.",
"relative_size": "large within frame",
"shape_and_color": "Round head, large eyes, bulky body, predominantly brown and grey with silver accents.",
"texture": "Extremely soft, fluffy, and detailed feathers, giving a plush toy-like appearance.",
"appearance_details": "The eyes are wide, round, and have a glossy, reflective quality, suggesting innocence and curiosity. The beak is small and light-colored, almost blending with the feathers.",
"orientation": "upright, facing forward"
"shape_and_color": "Round head, large eyes, voluminous body shape. Predominantly brown and gray with silver accents.",
"texture": "extremely soft, fluffy, downy feathers",
"appearance_details": "Wide, dark pupils in large, light-colored eyes. Small, delicate beak. Visible ear tufts.",
"number_of_objects": 1,
"pose": "Sitting upright, facing forward.",
"expression": "curious and gentle",
"action": "looking directly at the viewer",
"gender": "unknown",
"orientation": "upright"
}
],
"background_setting": "A dark, blurred forest at night, with hints of tree trunks and foliage visible in the background. The darkness emphasizes the owl as the central focus.",
"background_setting": "A dense, dark forest at night. The trees are silhouetted against a dark sky, with subtle hints of moonlight filtering through the leaves. The overall impression is one of depth and nocturnal serenity.",
"lighting": {
"conditions": "moonlight",
"direction": "top-left, casting subtle highlights",
"shadows": "soft, diffused shadows, contributing to the depth without harshness"
"conditions": "night, moonlight",
"direction": "side-lit from the left, with some ambient light from above",
"shadows": "soft, elongated shadows cast by the owl and branches, adding depth"
},
"aesthetics": {
"composition": "centered, portrait composition",
"color_scheme": "cool blues and grays of the night sky and warm browns and grays of the owl, with silver highlights",
"mood_atmosphere": "whimsical, serene, enchanting, and slightly mysterious",
"composition": "centered framing, with the owl as the clear focal point",
"color_scheme": "cool blues and grays of moonlight contrasting with the warm browns and grays of the owl and forest",
"mood_atmosphere": "magical, serene, whimsical",
"aesthetic_score": "very high",
"preference_score": "very high"
},
"photographic_characteristics": {
"depth_of_field": "shallow",
"depth_of_field": "shallow, with the background softly blurred",
"focus": "sharp focus on the owl's face and eyes",
"camera_angle": "eye-level",
"lens_focal_length": "portrait lens (e.g., 50mm-85mm)"
"lens_focal_length": "medium portrait lens"
},
"style_medium": "digital illustration",
"text_render": [],
"context": "A whimsical character illustration, suitable for children's books, animated features, or as a charming decorative art piece.",
"artistic_style": "hyperrealistic, cute, illustrative"
"context": "This image is a charming illustration, suitable for a children's book, a whimsical art print, or a decorative piece for a nature-themed space.",
"artistic_style": "storybook, whimsical, detailed"
}
"""
@ -48,39 +53,44 @@ OWL_PROMPT_REFINED = """
"short_description": "A hyper-detailed, ultra-fluffy owl sitting in the trees at night, looking directly at the camera with wide, adorable, expressive eyes. Its feathers are soft and voluminous, catching the cool moonlight with subtle silver highlights. The owl's gaze is curious and full of charm, giving it a whimsical, storybook-like personality.",
"objects": [
{
"description": "An adorable, fluffy owl with large, expressive eyes and soft, voluminous feathers. Its plumage is white with subtle silver highlights from the moonlight.",
"description": "An adorable, fluffy owl with large, expressive eyes and soft, voluminous feathers. Its plumage is a mix of white and subtle silver highlights from the moonlight.",
"location": "center",
"relationship": "The owl is the sole subject, perched comfortably within its environment.",
"relative_size": "large within frame",
"shape_and_color": "Round head, large eyes, bulky body, predominantly white with silver accents.",
"texture": "Extremely soft, fluffy, and detailed feathers, giving a plush toy-like appearance.",
"appearance_details": "The eyes are wide, round, and have a glossy, reflective quality, suggesting innocence and curiosity. The beak is small and light-colored, almost blending with the feathers.",
"orientation": "upright, facing forward"
"shape_and_color": "Round head, large eyes, voluminous body shape. Predominantly white with silver accents.",
"texture": "extremely soft, fluffy, downy feathers",
"appearance_details": "Wide, dark pupils in large, light-colored eyes. Small, delicate beak. Visible ear tufts.",
"number_of_objects": 1,
"pose": "Sitting upright, facing forward.",
"expression": "curious and gentle",
"action": "looking directly at the viewer",
"gender": "unknown",
"orientation": "upright"
}
],
"background_setting": "A dark, blurred forest at night, with hints of tree trunks and foliage visible in the background. The darkness emphasizes the owl as the central focus.",
"background_setting": "A dense, dark forest at night. The trees are silhouetted against a dark sky, with subtle hints of moonlight filtering through the leaves. The overall impression is one of depth and nocturnal serenity.",
"lighting": {
"conditions": "moonlight",
"direction": "top-left, casting subtle highlights",
"shadows": "soft, diffused shadows, contributing to the depth without harshness"
"conditions": "night, moonlight",
"direction": "side-lit from the left, with some ambient light from above",
"shadows": "soft, elongated shadows cast by the owl and branches, adding depth"
},
"aesthetics": {
"composition": "centered, portrait composition",
"color_scheme": "cool blues and grays of the night sky and white of the owl, with silver accents",
"mood_atmosphere": "whimsical, serene, enchanting, and slightly mysterious",
"composition": "centered framing, with the owl as the clear focal point",
"color_scheme": "cool blues and grays of moonlight contrasting with the white of the owl and forest",
"mood_atmosphere": "magical, serene, whimsical",
"aesthetic_score": "very high",
"preference_score": "very high"
},
"photographic_characteristics": {
"depth_of_field": "shallow",
"depth_of_field": "shallow, with the background softly blurred",
"focus": "sharp focus on the owl's face and eyes",
"camera_angle": "eye-level",
"lens_focal_length": "portrait lens (e.g., 50mm-85mm)"
"lens_focal_length": "medium portrait lens"
},
"style_medium": "digital illustration",
"text_render": [],
"context": "A whimsical character illustration, suitable for children's books, animated features, or as a charming decorative art piece.",
"artistic_style": "hyperrealistic, cute, illustrative"
"context": "This image is a charming illustration, suitable for a children's book, a whimsical art print, or a decorative piece for a nature-themed space.",
"artistic_style": "storybook, whimsical, detailed"
}
"""

Binary file not shown.

Before

Width:  |  Height:  |  Size: 318 KiB

After

Width:  |  Height:  |  Size: 374 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 331 KiB

After

Width:  |  Height:  |  Size: 385 KiB

View File

@ -9,78 +9,67 @@ from tests.image_generation.test_generate_image_fibo import OWL_PROMPT, OWL_PROM
INSPIRE_PROMPT = """
{
"short_description": "A charming, stylized illustration of a young owl sitting on a mossy forest floor. The owl is facing forward with large, expressive eyes and fluffy ear tufts. It is surrounded by a softly blurred forest environment with tall trees and dappled moonlight. The overall impression is one of innocence and nocturnal wonder, rendered in a gentle, painterly style.",
"short_description": "A charming, stylized illustration of a young owl sitting on a mossy forest floor. The owl is facing forward with large, expressive eyes and fluffy ear tufts. The background depicts a dark, atmospheric forest with tall trees and dappled moonlight, creating a mystical and serene environment. The overall impression is one of innocence and enchantment, suitable for children's literature or fantasy art.",
"objects": [
{
"description": "A young, anthropomorphic owl with large, round eyes and prominent ear tufts. Its plumage is a soft, mottled grey and beige, with distinct feather patterns.",
"description": "A small, fluffy owl with large, round, dark eyes and prominent ear tufts. Its plumage is a soft, mottled grey and white, with subtle patterns suggesting feathers. It has a small, yellow beak.",
"location": "center",
"relationship": "The owl is the sole prominent subject, resting on the ground.",
"relative_size": "medium-to-large within frame",
"shape_and_color": "Round body, large circular eyes, conical beak. Predominantly grey and beige.",
"texture": "soft, feathery",
"appearance_details": "Large, dark pupils in its eyes, a small yellow beak, and delicate talons.",
"relationship": "The owl is the sole subject, positioned centrally.",
"relative_size": "medium within frame",
"shape_and_color": "Round body, large circular eyes, grey and white coloration.",
"texture": "soft, fluffy feathers",
"appearance_details": "Its eyes have a glossy sheen, and its ear tufts are particularly bushy.",
"number_of_objects": 1,
"pose": "Sitting upright, facing forward.",
"expression": "curious and gentle",
"action": "resting",
"pose": "Sitting upright, with its body facing forward and its head slightly tilted.",
"expression": "curious and innocent",
"action": "sitting and observing",
"gender": "neutral",
"skin_tone_and_texture": "N/A (owl)",
"orientation": "upright"
},
{
"description": "A patch of vibrant green moss covering the ground in the foreground and around the owl.",
"location": "bottom foreground and midground",
"relationship": "The owl is perched on this mossy ground.",
"relative_size": "medium",
"shape_and_color": "Irregular, organic shapes, vibrant green.",
"texture": "soft, velvety, slightly uneven",
"appearance_details": "Appears lush and damp, with small variations in shade.",
"location": "bottom foreground and surrounding the owl's feet",
"relationship": "The moss serves as the ground on which the owl is perched.",
"relative_size": "small",
"shape_and_color": "Irregular patches, vibrant green.",
"texture": "soft, velvety, organic",
"appearance_details": "Individual blades of grass and small twigs are visible within the moss.",
"orientation": "horizontal"
},
{
"description": "Several tall, slender tree trunks forming the background.",
"location": "background",
"relationship": "They create the forest environment behind the owl.",
"description": "Several tall, dark tree trunks forming the background.",
"location": "background, left, right, and top edges",
"relationship": "These trees create the forest environment behind the owl.",
"relative_size": "large",
"shape_and_color": "Vertical, cylindrical shapes. Dark brown and grey.",
"shape_and_color": "Vertical, dark brown and grey.",
"texture": "rough bark",
"appearance_details": "Some trunks are covered in moss. They are out of focus.",
"appearance_details": "Some trunks have patches of moss or lichen. The overall effect is dense and shadowy.",
"orientation": "vertical"
},
{
"description": "A few delicate, glowing orbs scattered in the background, resembling fireflies or distant lights.",
"location": "background, scattered",
"relationship": "They add a magical element to the forest scene.",
"relative_size": "small",
"shape_and_color": "Small, spherical, glowing yellow-white.",
"texture": "N/A",
"appearance_details": "Softly blurred, suggesting distance.",
"orientation": "N/A"
}
],
"background_setting": "A mystical forest at night, with tall, ancient trees, a soft undergrowth of moss, and a hint of moonlight filtering through the canopy. The atmosphere is serene and enchanting.",
"background_setting": "A dense, mystical forest at night. Tall trees with dark bark dominate the scene, with a hint of moonlight filtering through the canopy. The ground is covered in lush green moss and scattered leaves, with faint glowing orbs suggesting magical elements.",
"lighting": {
"conditions": "soft, ambient moonlight",
"direction": "diffused from above and behind",
"shadows": "soft, elongated shadows cast by the trees, minimal on the owl"
"conditions": "dim, atmospheric night lighting",
"direction": "diffused from above and behind the trees",
"shadows": "soft, elongated shadows cast by the trees, creating depth"
},
"aesthetics": {
"composition": "centered framing with the owl as the main focal point",
"color_scheme": "cool, muted tones of blues, greys, and greens, with warm accents from the owl's eyes and beak",
"mood_atmosphere": "magical, serene, innocent",
"composition": "centered composition with the owl as the focal point",
"color_scheme": "muted earth tones with vibrant green accents and deep blues/greys in the background",
"mood_atmosphere": "enchanting, serene, mystical",
"aesthetic_score": "very high",
"preference_score": "very high"
},
"photographic_characteristics": {
"depth_of_field": "shallow, with a strong bokeh effect in the background",
"depth_of_field": "shallow, with the background softly blurred",
"focus": "sharp focus on the owl",
"camera_angle": "eye-level",
"lens_focal_length": "standard lens (e.g., 50mm)"
},
"style_medium": "digital illustration",
"text_render": [],
"context": "This image is suitable for children's book illustrations, fantasy-themed art, or decorative prints.",
"artistic_style": "stylized, painterly, whimsical"
"context": "This image is suitable for a children's book illustration, a fantasy-themed greeting card, or concept art for a magical game.",
"artistic_style": "stylized, whimsical, detailed"
}
"""
@ -89,43 +78,44 @@ SKYSCRAPERS_INSPIRE_PROMPT = """
"short_description": "A dramatic, low-angle shot of several modern skyscrapers in black and white, emphasizing their towering height and geometric forms against a bright, overcast sky. The buildings feature repetitive window patterns and varying architectural details, creating a sense of urban grandeur and scale.",
"objects": [
{
"description": "A tall skyscraper with a facade of numerous rectangular windows, arranged in a grid pattern. The building has a slightly curved or angled top section.",
"description": "A tall skyscraper with a facade of numerous rectangular windows, arranged in a grid pattern. The building has a slightly curved or angled design, giving it a dynamic appearance.",
"location": "center-right foreground",
"relationship": "It is the most prominent building, dominating the right side of the frame and appearing to recede into the background.",
"relationship": "This is the most prominent building, dominating the right side of the frame and serving as a primary focal point.",
"relative_size": "large within frame",
"shape_and_color": "Rectangular with a complex, multi-faceted top; shades of gray and black.",
"texture": "Smooth concrete and glass, with visible window frames.",
"appearance_details": "The windows reflect the bright sky, appearing light gray or white. Some sections of the facade have decorative elements or protrusions.",
"shape_and_color": "Rectangular and angular forms, dark grey to black with bright white window reflections.",
"texture": "Smooth, reflective glass and concrete, with visible mullions.",
"appearance_details": "The windows appear to be uniformly spaced, and some sections of the building have protruding architectural elements.",
"orientation": "vertical"
},
{
"description": "A long, rectangular skyscraper with a facade of regularly spaced, horizontal windows.",
"location": "left midground",
"relationship": "It stands to the left of the central skyscraper, appearing slightly further back and narrower.",
"description": "A large, dark, flat roof or top section of a building, extending into the upper-left corner of the frame. It has a slightly textured surface.",
"location": "top-left foreground",
"relationship": "It frames the upper-left portion of the image, contrasting with the bright sky and the more distant buildings.",
"relative_size": "large within frame",
"shape_and_color": "Flat, angular, dark grey to black.",
"texture": "Rough, matte concrete or stone.",
"appearance_details": "The surface shows subtle imperfections and variations in tone.",
"orientation": "horizontal"
},
{
"description": "A tall, slender skyscraper with a facade composed of many small, rectangular windows. Its design appears more traditional with vertical emphasis.",
"location": "bottom-right midground",
"relationship": "It stands behind and to the right of the central skyscraper, adding depth to the urban landscape.",
"relative_size": "medium",
"shape_and_color": "Tall, slender rectangle; shades of gray and black.",
"texture": "Smooth concrete and glass.",
"appearance_details": "The horizontal lines of windows create a strong sense of rhythm. Its top is flat and extends horizontally.",
"shape_and_color": "Tall, slender rectangular form, dark grey with bright white window reflections.",
"texture": "Smooth glass and concrete.",
"appearance_details": "The windows are tightly packed, creating a uniform pattern.",
"orientation": "vertical"
},
{
"description": "A skyscraper with a facade featuring larger, rectangular windows that are not perfectly uniform, giving a slightly more textured appearance.",
"location": "center-left midground",
"relationship": "It is positioned behind the left midground skyscraper and to the left of the central skyscraper, partially obscured by the foreground elements.",
"relative_size": "medium",
"shape_and_color": "Tall, slender rectangle; shades of gray and black.",
"texture": "Smooth concrete and glass, with subtle variations in window size.",
"appearance_details": "Its upper section has a distinct, stepped design.",
"orientation": "vertical"
},
{
"description": "A building with a facade characterized by smaller, square-like windows, creating a more intricate pattern.",
"location": "bottom-right foreground",
"relationship": "It is partially visible at the bottom right, appearing to be closer to the viewer than the central skyscraper.",
"description": "A series of less distinct skyscrapers and urban structures, partially obscured by atmospheric haze or distance.",
"location": "midground and background",
"relationship": "These buildings form the distant urban backdrop, providing context for the foreground structures.",
"relative_size": "small",
"shape_and_color": "Rectangular base with a repeating pattern of small squares; shades of gray and black.",
"texture": "Smooth concrete and glass.",
"appearance_details": "The windows are arranged in a dense, grid-like pattern, almost appearing pixelated.",
"shape_and_color": "Various rectangular and indistinct shapes, dark grey.",
"texture": "Indistinct due to distance and haze.",
"appearance_details": "Their details are softened by the monochromatic atmosphere.",
"number_of_objects": 5,
"orientation": "vertical"
}
],
@ -136,22 +126,22 @@ SKYSCRAPERS_INSPIRE_PROMPT = """
"shadows": "soft, subtle shadows on the buildings, emphasizing their forms without harsh lines"
},
"aesthetics": {
"composition": "dynamic low-angle composition, with leading lines from the buildings converging towards the top-center, creating a sense of immense height and perspective.",
"color_scheme": "monochromatic (black and white) with a wide range of grays, emphasizing texture and form.",
"mood_atmosphere": "grand, imposing, architectural, and slightly mysterious.",
"composition": "dynamic and dramatic low-angle composition, with leading lines created by the converging architectural elements drawing the eye upwards.",
"color_scheme": "monochromatic (black and white) with a wide range of greys, emphasizing form and texture.",
"mood_atmosphere": "grand, imposing, modern, and slightly mysterious.",
"aesthetic_score": "very high",
"preference_score": "very high"
},
"photographic_characteristics": {
"depth_of_field": "deep",
"focus": "sharp focus on all visible skyscrapers, maintaining clarity across the architectural details.",
"camera_angle": "very low angle, looking upwards from ground level.",
"lens_focal_length": "wide-angle lens, to capture the full height and breadth of the buildings."
"focus": "sharp focus on the foreground and midground buildings, with slight softening towards the distant background.",
"camera_angle": "very low angle, looking upwards from street level.",
"lens_focal_length": "wide-angle"
},
"style_medium": "photograph",
"text_render": [],
"context": "An architectural photograph, likely intended for art prints, urban exploration blogs, or promotional material for city tourism, focusing on the abstract beauty of modern cityscapes.",
"artistic_style": "minimalist, abstract, architectural"
"context": "An architectural photograph, likely intended for art prints, urban exploration documentation, or a design magazine, focusing on the grandeur of modern cityscapes.",
"artistic_style": "monochromatic, architectural, dramatic"
}
"""