import mlx.core as mx from mlx import nn from tqdm import tqdm from mflux.callbacks.callbacks import Callbacks from mflux.config.config import Config from mflux.config.model_config import ModelConfig from mflux.config.runtime_config import RuntimeConfig from mflux.error.exceptions import StopImageGenerationException from mflux.flux.flux_initializer import FluxInitializer from mflux.flux_tools.fill.mask_util import MaskUtil from mflux.latent_creator.latent_creator import LatentCreator from mflux.models.text_encoder.clip_encoder.clip_encoder import CLIPEncoder from mflux.models.text_encoder.prompt_encoder import PromptEncoder from mflux.models.text_encoder.t5_encoder.t5_encoder import T5Encoder from mflux.models.transformer.transformer import Transformer from mflux.models.vae.vae import VAE from mflux.post_processing.array_util import ArrayUtil from mflux.post_processing.generated_image import GeneratedImage from mflux.post_processing.image_util import ImageUtil from mflux.weights.model_saver import ModelSaver class Flux1Fill(nn.Module): vae: VAE transformer: Transformer t5_text_encoder: T5Encoder clip_text_encoder: CLIPEncoder def __init__( self, model_config: ModelConfig, quantize: int | None = None, local_path: str | None = None, lora_paths: list[str] | None = None, lora_scales: list[float] | None = None, ): super().__init__() FluxInitializer.init( flux_model=self, model_config=model_config, quantize=quantize, local_path=local_path, lora_paths=lora_paths, lora_scales=lora_scales, ) def generate_image( self, seed: int, prompt: str, config: Config, ) -> GeneratedImage: # 0. Create a new runtime config based on the model type and input parameters config = RuntimeConfig(config, self.model_config) time_steps = tqdm(range(config.init_time_step, config.num_inference_steps)) # 1. Create the initial latents latents = LatentCreator.create( seed=seed, height=config.height, width=config.width, ) # 2. Encode the prompt prompt_embeds, pooled_prompt_embeds = PromptEncoder.encode_prompt( prompt=prompt, prompt_cache=self.prompt_cache, t5_tokenizer=self.t5_tokenizer, clip_tokenizer=self.clip_tokenizer, t5_text_encoder=self.t5_text_encoder, clip_text_encoder=self.clip_text_encoder, ) # 3. Create the static masked latents static_masked_latents = MaskUtil.create_masked_latents( vae=self.vae, config=config, latents=latents, img_path=config.image_path, mask_path=config.masked_image_path, ) # (Optional) Call subscribers for beginning of loop Callbacks.before_loop( seed=seed, prompt=prompt, latents=latents, config=config, ) for t in time_steps: try: # 4.t Concatenate the updated latents with the static masked latents hidden_states = mx.concatenate([latents, static_masked_latents], axis=-1) # 5.t Predict the noise noise = self.transformer( t=t, config=config, hidden_states=hidden_states, prompt_embeds=prompt_embeds, pooled_prompt_embeds=pooled_prompt_embeds, ) # 6.t Take one denoise step dt = config.sigmas[t + 1] - config.sigmas[t] latents += noise * dt # (Optional) Call subscribers in-loop Callbacks.in_loop( t=t, seed=seed, prompt=prompt, latents=latents, config=config, time_steps=time_steps, ) # (Optional) Evaluate to enable progress tracking mx.eval(latents) except KeyboardInterrupt: # noqa: PERF203 Callbacks.interruption( t=t, seed=seed, prompt=prompt, latents=latents, config=config, time_steps=time_steps, ) raise StopImageGenerationException(f"Stopping image generation at step {t + 1}/{len(time_steps)}") # (Optional) Call subscribers after loop Callbacks.after_loop( seed=seed, prompt=prompt, latents=latents, config=config, ) # 7. Decode the latent array and return the image latents = ArrayUtil.unpack_latents(latents=latents, height=config.height, width=config.width) decoded = self.vae.decode(latents) return ImageUtil.to_image( decoded_latents=decoded, config=config, seed=seed, prompt=prompt, quantization=self.bits, lora_paths=self.lora_paths, lora_scales=self.lora_scales, image_path=config.image_path, image_strength=config.image_strength, masked_image_path=config.masked_image_path, generation_time=time_steps.format_dict["elapsed"], ) def save_model(self, base_path: str) -> None: ModelSaver.save_model(self, self.bits, base_path)