diff --git a/.planning/codebase/ARCHITECTURE.md b/.planning/codebase/ARCHITECTURE.md index 83b85ae..fe7dae6 100644 --- a/.planning/codebase/ARCHITECTURE.md +++ b/.planning/codebase/ARCHITECTURE.md @@ -171,7 +171,7 @@ **Modifier Functions:** - Purpose: Compose enhancements as FnOnce closures chained via PresetBuilder::with_modifier() - Examples: real_esrgan_x4plus_anime_6_b(), sdxl_vae_fp16_fix(), taesd(), lcm_lora_sd_1_5() -- Pattern: FnOnce(ConfigsBuilder) -> Result +- Pattern: FnOnce(ConfigsBuilder) -> Result **Weight Type Subenum:** - Purpose: Model-specific quantization options (F32, F16, Q4_0, Q8_0, etc.) @@ -246,7 +246,7 @@ **Patterns:** - ConfigBuilder::build() returns Result with validation_fn(validate = "Self::validate") -- PresetBuilder::build() converts ApiError from downloads to ConfigBuilderError::ValidationError +- PresetBuilder::build() converts HFError from downloads to ConfigBuilderError::ValidationError - gen_img() returns Result<(), DiffusionError> with enum variants: Forward, StoreImages, Io, Upscaler - Null pointer check after generate_image() signals OOM/backend failure; returns Err(DiffusionError::Forward) @@ -261,7 +261,7 @@ **Authentication:** - HuggingFace token stored in thread-safe static OnceLock> -- Token optional; models marked as requiring access will fail download with ApiError if token missing +- Token optional; models marked as requiring access will fail download with HFError if token missing - CLI accepts --token flag to set before config building **File I/O:** diff --git a/Cargo.toml b/Cargo.toml index 677b602..2faba15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,21 +24,23 @@ documentation = "https://docs.rs/diffusion-rs" chrono.workspace = true derive_builder = "0.20.2" diffusion-rs-sys = { path = "sys", version = "0.1.20" } -hf-hub = { version = "0.4.2", default-features = false, features = ["ureq"] } -image = "0.25.5" -libc = "0.2.161" -little_exif = "0.6.21" -num_cpus = "1.16.0" +hf-hub = { version = "1.0.0", default-features = false, features = [ + "blocking", +] } +image = "0.25.10" +libc = "0.2.186" +little_exif = "0.6.23" +num_cpus = "1.17.0" strum.workspace = true -subenum = "1.1.3" -thiserror = "2.0.3" +subenum = "1.2.0" +thiserror = "2.0.19" walkdir = "2.5.0" [workspace.dependencies] -chrono = "0.4.42" -clap = { version = "4.5.53", features = ["default", "derive"] } +chrono = "0.4.45" +clap = { version = "4.6.2", features = ["default", "derive"] } execution-time = "0.3.1" -strum = { version = "0.27", features = ["derive"] } +strum = { version = "0.28.0", features = ["derive"] } [features] cuda = ["diffusion-rs-sys/cuda"] diff --git a/cli/src/main.rs b/cli/src/main.rs index ec77439..bea7d75 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -418,6 +418,7 @@ fn get_preset(args: &Args) -> Preset { PresetDiscriminants::LensTurbo => Preset::LensTurbo, PresetDiscriminants::BooguImage => Preset::BooguImage, PresetDiscriminants::BooguImageTurbo => Preset::BooguImageTurbo, + PresetDiscriminants::MiniT2I => Preset::MiniT2I, PresetDiscriminants::Krea2 => Preset::Krea2( args.weights .unwrap_or_else(|| Krea2Weight::default().into()) diff --git a/gui/lib/shared/models/preset_catalog.dart b/gui/lib/shared/models/preset_catalog.dart index 76edd4c..e2f1897 100644 --- a/gui/lib/shared/models/preset_catalog.dart +++ b/gui/lib/shared/models/preset_catalog.dart @@ -209,6 +209,7 @@ class PresetCatalog { height: 512, weight: 'Q3_K', ), + 'MiniT2I': PresetDefaults(steps: 100, width: 512, height: 512), }; /// Returns the default steps/width/height for [presetName]. diff --git a/src/api.rs b/src/api.rs index f8ece7e..e999081 100644 --- a/src/api.rs +++ b/src/api.rs @@ -12,6 +12,7 @@ use std::sync::mpsc::Sender; use chrono::Local; use derive_builder::Builder; +use diffusion_rs_sys::free_sd_images; use diffusion_rs_sys::free_upscaler_ctx; use diffusion_rs_sys::generate_image; use diffusion_rs_sys::new_upscaler_ctx; @@ -549,7 +550,7 @@ pub struct ModelConfig { /// Select the runtime backend used to execute model graphs #[builder(default = "(None, CLibString::default())", setter(custom))] - backend: (Option>, CLibString), + backend: (Option>>, CLibString), /// Select the backend used to allocate model parameters #[builder(default = "(None, CLibString::default())", setter(custom))] @@ -579,11 +580,19 @@ pub struct ModelConfig { #[builder(default = "false")] eager_load: bool, + /// Number of Qwen Image Layered layers; latent/output count is layers + 1 (default: 3) + #[builder(default = "3")] + qwen_image_layers: i32, + + /// Pick the diffusion/te/vae device placements automatically from the model size and the per-device memory budgets (max_vram; defaults to free memory minus a small margin). Overrides backend and params-backend; may split modules across GPUs (split-mode still selects layer or row) + #[builder(default = "false")] + auto_fit: bool, + #[builder(default = "None", private)] upscaler_ctx: Option<*mut upscaler_ctx_t>, #[builder(default = "None", private)] - diffusion_ctx: Option<(*mut sd_ctx_t, sd_ctx_params_t)>, + diffusion_ctx: Option<(*mut sd_ctx_t, sd_ctx_params_t, CLibString)>, } impl ModelConfigBuilder { @@ -710,10 +719,20 @@ impl ModelConfigBuilder { ) } - pub fn backend(&mut self, backend_map: HashMap) -> &mut Self { + pub fn backend(&mut self, backend_map: HashMap>) -> &mut Self { let backend_str = backend_map .iter() - .map(|(key, value)| format!("{}={}", key, value)) + .map(|(key, value)| { + format!( + "{}={}", + key, + value + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("&") + ) + }) .collect::>() .join(","); self.backend = Some((Some(backend_map), CLibString::from(backend_str))); @@ -813,12 +832,13 @@ impl ModelConfig { // This is required to support img2img after text2img generation // otherwise the context is cached and won't have a decode graph // leading to an assertion error in sdcpp - if let Some((sd_ctx, _)) = self.diffusion_ctx.as_ref() { + if let Some((sd_ctx, _, _)) = self.diffusion_ctx.as_ref() { sd_set_progress_callback(None, null_mut()); free_sd_ctx(*sd_ctx); self.diffusion_ctx = None; } if self.diffusion_ctx.is_none() { + let model_args = self.model_args(); let sd_ctx_params = sd_ctx_params_t { model_path: self.model.as_ptr(), llm_path: self.llm.as_ptr(), @@ -842,9 +862,6 @@ impl ModelConfig { diffusion_flash_attn: self.diffusion_flash_attention, flash_attn: self.flash_attention, diffusion_conv_direct: self.diffusion_conv_direct, - chroma_use_dit_mask: !self.chroma_disable_dit_mask, - chroma_use_t5_mask: self.chroma_enable_t5_mask, - chroma_t5_mask_pad: self.chroma_t5_mask_pad, vae_conv_direct: self.vae_conv_direct, prediction: self.prediction, force_sdxl_vae_conv_scale: self.force_sdxl_vae_conv_scale, @@ -852,9 +869,6 @@ impl ModelConfig { lora_apply_mode: self.lora_apply_mode, tensor_type_rules: null_mut(), sampler_rng_type: self.sampler_rng_type, - circular_x: self.circular || self.circular_x, - circular_y: self.circular || self.circular_y, - qwen_image_zero_cond_t: self.use_qwen_image_zero_cond_true, enable_mmap: self.enable_mmap, max_vram: self.max_vram.1.as_ptr(), backend: self.backend.1.as_ptr(), @@ -864,22 +878,33 @@ impl ModelConfig { vae_format: self.vae_format, stream_layers: self.stream_layers, rpc_servers: null(), + split_mode: null(), pulid_weights_path: self.pulid_weights_path.as_ptr(), eager_load: self.eager_load, + auto_fit: self.auto_fit, + model_args: model_args.as_ptr(), + motion_module_path: null(), }; let ctx = new_sd_ctx(&sd_ctx_params); - self.diffusion_ctx = Some((ctx, sd_ctx_params)) + self.diffusion_ctx = Some((ctx, sd_ctx_params, model_args)) } - self.diffusion_ctx.unwrap().0 + self.diffusion_ctx.as_ref().unwrap().0 } } + + fn model_args(&self) -> CLibString { + format!("chroma_use_dit_mask={}, chroma_use_t5_mask={}, chroma_t5_mask_pad={}, qwen_image_zero_cond_t={}", + !self.chroma_disable_dit_mask, self.chroma_enable_t5_mask, + self.chroma_t5_mask_pad, self.use_qwen_image_zero_cond_true + ).into() + } } impl Drop for ModelConfig { fn drop(&mut self) { //Cleanup CTX section unsafe { - if let Some((sd_ctx, _)) = self.diffusion_ctx { + if let Some((sd_ctx, _, _)) = self.diffusion_ctx { free_sd_ctx(sd_ctx); } @@ -952,7 +977,8 @@ impl From<&ModelConfig> for ModelConfigBuilder { .extra_sample_params(value.extra_sample_params.clone()) .backend(value.backend.0.clone().unwrap_or_default()) .params_backend(value.params_backend.0.clone().unwrap_or_default()) - .extra_tiling_args(value.extra_tiling_args.0.clone().unwrap_or_default()); + .extra_tiling_args(value.extra_tiling_args.0.clone().unwrap_or_default()) + .qwen_image_layers(value.qwen_image_layers); builder.lora_models_internal(value.lora_models.clone()); @@ -1095,6 +1121,10 @@ pub struct Config { #[builder(default = "false")] disable_auto_resize_ref_image: bool, + /// Automatically increase the indices of references images based on the order they are listed (starting with 1). + #[builder(default = "true")] + increase_ref_index: bool, + #[builder(default = "Self::cache_init()", private)] cache: (sd_cache_params_t, Option), } @@ -1250,6 +1280,7 @@ impl From<&Config> for ConfigBuilder { .skip_layer_end(value.skip_layer_end) .canny(value.canny) .disable_auto_resize_ref_image(value.disable_auto_resize_ref_image) + .increase_ref_index(value.increase_ref_index) .preview_output(value.preview_output.clone()) .preview_mode(value.preview_mode) .preview_noisy(value.preview_noisy) @@ -1329,15 +1360,21 @@ unsafe fn upscale( let upscale_factor = 4; // unused for RealESRGAN_x4plus_anime_6B.pth let mut current_image = data; for _ in 0..upscale_repeats { - let upscaled_image = - diffusion_rs_sys::upscale(upscaler_ctx, current_image, upscale_factor); - - if upscaled_image.data.is_null() { + let upscaled_image = null_mut(); + let mut upscale_count = 1; + + if !diffusion_rs_sys::upscale( + upscaler_ctx, + current_image, + upscale_factor, + upscaled_image, + &mut upscale_count, + ) { return Err(DiffusionError::Upscaler); } free(current_image.data as *mut c_void); - current_image = upscaled_image; + current_image = *(*(upscaled_image)); } Ok(current_image) } @@ -1598,6 +1635,22 @@ fn gen_img_maybe_progress( custom_sigmas_count: hires_sigmas_count, }; + let resize_before_vae = if config.disable_auto_resize_ref_image { + 0 + } else { + 1 + }; + let increaze_ref_index = if config.increase_ref_index { + ", ref_index_mode=increase" + } else { + "" + }; + let ref_image_args: CLibString = format!( + "resize_before_vae={}{}", + resize_before_vae, increaze_ref_index + ) + .into(); + let sd_img_gen_params = sd_img_gen_params_t { prompt: prompt.as_ptr(), negative_prompt: config.negative_prompt.as_ptr(), @@ -1605,7 +1658,6 @@ fn gen_img_maybe_progress( init_image, ref_images: ref_image_ptr, ref_images_count: num_ref_images as i32, - increase_ref_index: false, mask_image, width: config.width, height: config.height, @@ -1617,7 +1669,6 @@ fn gen_img_maybe_progress( control_strength: config.control_strength, pm_params, vae_tiling_params, - auto_resize_ref_image: config.disable_auto_resize_ref_image, cache, loras: loras.as_ptr(), lora_count: loras.len() as u32, @@ -1626,18 +1677,28 @@ fn gen_img_maybe_progress( id_embedding_path: model_config.pulid_id_embedding_path.as_ptr(), id_weight: 1.0, }, + qwen_image_layers: model_config.qwen_image_layers, + circular_x: model_config.circular_x, + circular_y: model_config.circular_y, + ref_image_args: ref_image_args.as_ptr(), }; let params_str = CString::from_raw(sd_img_gen_params_to_str(&sd_img_gen_params)) .into_string() .unwrap(); - - let slice = generate_image(sd_ctx, &sd_img_gen_params); + let mut images_out = null_mut(); + let mut images_out_count = 0; + let gen_result = generate_image( + sd_ctx, + &sd_img_gen_params, + &mut images_out, + &mut images_out_count, + ); let ret = { - if slice.is_null() { + if !gen_result || images_out.is_null() { return Err(DiffusionError::Forward); } - for (img, path) in slice::from_raw_parts(slice, config.batch_count as usize) + for (img, path) in slice::from_raw_parts(images_out, images_out_count as usize) .iter() .zip(files) { @@ -1655,7 +1716,7 @@ fn gen_img_maybe_progress( } Ok(()) }; - free(slice as *mut c_void); + free_sd_images(images_out, images_out_count); ret } } diff --git a/src/modifier.rs b/src/modifier.rs index 98b940c..ee21326 100644 --- a/src/modifier.rs +++ b/src/modifier.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use hf_hub::api::sync::ApiError; +use hf_hub::HFError; use strum::IntoEnumIterator; use crate::{ @@ -12,7 +12,7 @@ use crate::{ /// Add the upscaler pub fn real_esrgan_x4plus_anime_6_b( mut builder: ConfigsBuilder, -) -> Result { +) -> Result { let upscaler_path = download_file_hf_hub( "ximso/RealESRGAN_x4plus_anime_6B", "RealESRGAN_x4plus_anime_6B.pth", @@ -22,14 +22,14 @@ pub fn real_esrgan_x4plus_anime_6_b( } /// Apply to avoid black images with xl models -pub fn sdxl_vae_fp16_fix(mut builder: ConfigsBuilder) -> Result { +pub fn sdxl_vae_fp16_fix(mut builder: ConfigsBuilder) -> Result { let vae_path = download_file_hf_hub("madebyollin/sdxl-vae-fp16-fix", "sdxl.vae.safetensors")?; builder.1.vae(vae_path); Ok(builder) } /// Apply taesd autoencoder for faster decoding (SD v1/v2) -pub fn taesd(mut builder: ConfigsBuilder) -> Result { +pub fn taesd(mut builder: ConfigsBuilder) -> Result { let taesd_path = download_file_hf_hub("madebyollin/taesd", "diffusion_pytorch_model.safetensors")?; builder.1.taesd(taesd_path); @@ -37,7 +37,7 @@ pub fn taesd(mut builder: ConfigsBuilder) -> Result { } /// Apply taesd autoencoder for faster decoding (SDXL) -pub fn taesd_xl(mut builder: ConfigsBuilder) -> Result { +pub fn taesd_xl(mut builder: ConfigsBuilder) -> Result { let taesd_path = download_file_hf_hub("madebyollin/taesdxl", "diffusion_pytorch_model.safetensors")?; builder.1.taesd(taesd_path); @@ -45,7 +45,7 @@ pub fn taesd_xl(mut builder: ConfigsBuilder) -> Result } /// Apply taesd autoencoder for faster decoding (SD v1/v2) -pub fn hybrid_taesd(mut builder: ConfigsBuilder) -> Result { +pub fn hybrid_taesd(mut builder: ConfigsBuilder) -> Result { let taesd_path = download_file_hf_hub( "cqyan/hybrid-sd-tinyvae", "diffusion_pytorch_model.safetensors", @@ -55,7 +55,7 @@ pub fn hybrid_taesd(mut builder: ConfigsBuilder) -> Result taesd autoencoder for faster decoding (SDXL) -pub fn hybrid_taesd_xl(mut builder: ConfigsBuilder) -> Result { +pub fn hybrid_taesd_xl(mut builder: ConfigsBuilder) -> Result { let taesd_path = download_file_hf_hub( "cqyan/hybrid-sd-tinyvae-xl", "diffusion_pytorch_model.safetensors", @@ -66,7 +66,7 @@ pub fn hybrid_taesd_xl(mut builder: ConfigsBuilder) -> Result to reduce inference steps for SD v1 between 2-8 (default 8) /// cfg_scale 1. 8 steps. -pub fn lcm_lora_sd_1_5(mut builder: ConfigsBuilder) -> Result { +pub fn lcm_lora_sd_1_5(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub( "latent-consistency/lcm-lora-sdv1-5", "pytorch_lora_weights.safetensors", @@ -79,13 +79,13 @@ pub fn lcm_lora_sd_1_5(mut builder: ConfigsBuilder) -> Result to reduce inference steps for SD v1 between 2-8 (default 8) /// Enabled [SampleMethod::LCM_SAMPLE_METHOD]. cfg_scale 2. 8 steps. -pub fn lcm_lora_sdxl_base_1_0(mut builder: ConfigsBuilder) -> Result { +pub fn lcm_lora_sdxl_base_1_0(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub( "latent-consistency/lcm-lora-sdxl", "pytorch_lora_weights.safetensors", @@ -101,7 +101,7 @@ pub fn lcm_lora_sdxl_base_1_0(mut builder: ConfigsBuilder) -> Result Result pub fn lora_pixel_art_sdxl_base_1_0( mut builder: ConfigsBuilder, -) -> Result { +) -> Result { let lora_path = download_file_hf_hub("nerijs/pixel-art-xl", "pixel-art-xl.safetensors")?; builder.1.lora_models( @@ -125,7 +125,7 @@ pub fn lora_pixel_art_sdxl_base_1_0( } /// Apply -pub fn lora_pastelcomic_2_flux(mut builder: ConfigsBuilder) -> Result { +pub fn lora_pastelcomic_2_flux(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub("nerijs/pastelcomic-flux", "pastelcomic_v2.safetensors")?; builder.1.lora_models( @@ -140,7 +140,7 @@ pub fn lora_pastelcomic_2_flux(mut builder: ConfigsBuilder) -> Result -pub fn lora_ghibli_flux(mut builder: ConfigsBuilder) -> Result { +pub fn lora_ghibli_flux(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub( "strangerzonehf/Ghibli-Flux-Cartoon-LoRA", "Ghibili-Cartoon-Art.safetensors", @@ -158,7 +158,7 @@ pub fn lora_ghibli_flux(mut builder: ConfigsBuilder) -> Result -pub fn lora_midjourney_mix_2_flux(mut builder: ConfigsBuilder) -> Result { +pub fn lora_midjourney_mix_2_flux(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub( "strangerzonehf/Flux-Midjourney-Mix2-LoRA", "mjV6.safetensors", @@ -176,7 +176,7 @@ pub fn lora_midjourney_mix_2_flux(mut builder: ConfigsBuilder) -> Result -pub fn lora_retro_pixel_flux(mut builder: ConfigsBuilder) -> Result { +pub fn lora_retro_pixel_flux(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub( "prithivMLmods/Retro-Pixel-Flux-LoRA", "Retro-Pixel.safetensors", @@ -194,7 +194,7 @@ pub fn lora_retro_pixel_flux(mut builder: ConfigsBuilder) -> Result -pub fn lora_canopus_pixar_3d_flux(mut builder: ConfigsBuilder) -> Result { +pub fn lora_canopus_pixar_3d_flux(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub( "prithivMLmods/Canopus-Pixar-3D-Flux-LoRA", "Canopus-Pixar-3D-FluxDev-LoRA.safetensors", @@ -212,7 +212,7 @@ pub fn lora_canopus_pixar_3d_flux(mut builder: ConfigsBuilder) -> Result fp8_e4m3fn t5xxl text encoder to reduce memory usage -pub fn t5xxl_fp8_flux_1(mut builder: ConfigsBuilder) -> Result { +pub fn t5xxl_fp8_flux_1(mut builder: ConfigsBuilder) -> Result { let t5xxl_path = download_file_hf_hub( "comfyanonymous/flux_text_encoders", "t5xxl_fp8_e4m3fn.safetensors", @@ -224,7 +224,7 @@ pub fn t5xxl_fp8_flux_1(mut builder: ConfigsBuilder) -> Result /// Default for flux_1_dev/schnell -pub fn t5xxl_fp16_flux_1(mut builder: ConfigsBuilder) -> Result { +pub fn t5xxl_fp16_flux_1(mut builder: ConfigsBuilder) -> Result { let t5xxl_path = download_file_hf_hub( "comfyanonymous/flux_text_encoders", "t5xxl_fp16.safetensors", @@ -235,7 +235,7 @@ pub fn t5xxl_fp16_flux_1(mut builder: ConfigsBuilder) -> Result -pub fn t5xxl_q2_k_flux_1(mut builder: ConfigsBuilder) -> Result { +pub fn t5xxl_q2_k_flux_1(mut builder: ConfigsBuilder) -> Result { let t5xxl_path = download_file_hf_hub("Green-Sky/flux.1-schnell-GGUF", "t5xxl_q2_k.gguf")?; builder.1.t5xxl(t5xxl_path); @@ -243,7 +243,7 @@ pub fn t5xxl_q2_k_flux_1(mut builder: ConfigsBuilder) -> Result -pub fn t5xxl_q3_k_flux_1(mut builder: ConfigsBuilder) -> Result { +pub fn t5xxl_q3_k_flux_1(mut builder: ConfigsBuilder) -> Result { let t5xxl_path = download_file_hf_hub("Green-Sky/flux.1-schnell-GGUF", "t5xxl_q3_k.gguf")?; builder.1.t5xxl(t5xxl_path); @@ -252,7 +252,7 @@ pub fn t5xxl_q3_k_flux_1(mut builder: ConfigsBuilder) -> Result /// Default for flux_1_mini -pub fn t5xxl_q4_k_flux_1(mut builder: ConfigsBuilder) -> Result { +pub fn t5xxl_q4_k_flux_1(mut builder: ConfigsBuilder) -> Result { let t5xxl_path = download_file_hf_hub("Green-Sky/flux.1-schnell-GGUF", "t5xxl_q4_k.gguf")?; builder.1.t5xxl(t5xxl_path); @@ -260,7 +260,7 @@ pub fn t5xxl_q4_k_flux_1(mut builder: ConfigsBuilder) -> Result -pub fn t5xxl_q8_0_flux_1(mut builder: ConfigsBuilder) -> Result { +pub fn t5xxl_q8_0_flux_1(mut builder: ConfigsBuilder) -> Result { let t5xxl_path = download_file_hf_hub("Green-Sky/flux.1-schnell-GGUF", "t5xxl_q8_0.gguf")?; builder.1.t5xxl(t5xxl_path); @@ -268,7 +268,7 @@ pub fn t5xxl_q8_0_flux_1(mut builder: ConfigsBuilder) -> Result Result { +pub fn offload_params_to_cpu(mut builder: ConfigsBuilder) -> Result { let params: HashMap<_, _> = Module::iter() .map(|module| (module, BackendDevice::CPU)) .collect(); @@ -280,7 +280,7 @@ pub fn offload_params_to_cpu(mut builder: ConfigsBuilder) -> Result Result { +) -> Result { let params: HashMap<_, _> = Module::iter() .map(|module| (module, BackendDevice::DISK)) .collect(); @@ -291,7 +291,7 @@ pub fn lazily_load_params_from_disk( /// Apply to reduce inference steps for SD v1 between 2-8 (default 8) /// cfg_scale 1. 8 steps. -pub fn lcm_lora_ssd_1b(mut builder: ConfigsBuilder) -> Result { +pub fn lcm_lora_ssd_1b(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub( "kylielee505/mylcmlorassd", "pytorch_lora_weights.safetensors", @@ -304,42 +304,42 @@ pub fn lcm_lora_ssd_1b(mut builder: ConfigsBuilder) -> Result Result { +pub fn vae_tiling(mut builder: ConfigsBuilder) -> Result { builder.1.vae_tiling(true); Ok(builder) } /// Enable preview with [crate::api::PreviewType::PREVIEW_PROJ] -pub fn preview_proj(mut builder: ConfigsBuilder) -> Result { +pub fn preview_proj(mut builder: ConfigsBuilder) -> Result { builder.0.preview_mode(PreviewType::PREVIEW_PROJ); Ok(builder) } /// Enable preview with [crate::api::PreviewType::PREVIEW_TAE] -pub fn preview_tae(mut builder: ConfigsBuilder) -> Result { +pub fn preview_tae(mut builder: ConfigsBuilder) -> Result { builder.0.preview_mode(PreviewType::PREVIEW_TAE); Ok(builder) } /// Enable preview with [crate::api::PreviewType::PREVIEW_VAE] -pub fn preview_vae(mut builder: ConfigsBuilder) -> Result { +pub fn preview_vae(mut builder: ConfigsBuilder) -> Result { builder.0.preview_mode(PreviewType::PREVIEW_VAE); Ok(builder) } /// Enable flash attention -pub fn enable_flash_attention(mut builder: ConfigsBuilder) -> Result { +pub fn enable_flash_attention(mut builder: ConfigsBuilder) -> Result { builder.1.flash_attention(true); Ok(builder) } /// Apply to [crate::preset::Preset::SegmindVega] -pub fn lcm_lora_segmind_vega_rt(mut builder: ConfigsBuilder) -> Result { +pub fn lcm_lora_segmind_vega_rt(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub("segmind/Segmind-VegaRT", "pytorch_lora_weights.safetensors")?; builder.1.lora_models( @@ -350,12 +350,12 @@ pub fn lcm_lora_segmind_vega_rt(mut builder: ConfigsBuilder) -> Result -pub fn lora_anima_8_steps_turbo(mut builder: ConfigsBuilder) -> Result { +pub fn lora_anima_8_steps_turbo(mut builder: ConfigsBuilder) -> Result { let lora_path = download_file_hf_hub( "Einhorn/Anima-Preview_8_Step_Turbo_Lora", "Anima-Preview_Turbo_8step.safetensors", @@ -369,12 +369,12 @@ pub fn lora_anima_8_steps_turbo(mut builder: ConfigsBuilder) -> Result small decoder for faster decoding with a minor quality reduction -pub fn flux_2_small_decoder(mut builder: ConfigsBuilder) -> Result { +pub fn flux_2_small_decoder(mut builder: ConfigsBuilder) -> Result { let vae_path = download_file_hf_hub( "black-forest-labs/FLUX.2-small-decoder", "full_encoder_small_decoder.safetensors", @@ -385,7 +385,7 @@ pub fn flux_2_small_decoder(mut builder: ConfigsBuilder) -> Result(preset: Preset, prompt: &str, m: F) where - F: FnOnce(ConfigsBuilder) -> Result + 'static, + F: FnOnce(ConfigsBuilder) -> Result + 'static, { let (mut config, mut model_config) = PresetBuilder::default() .preset(preset) diff --git a/src/preset.rs b/src/preset.rs index 8197e34..85e1889 100644 --- a/src/preset.rs +++ b/src/preset.rs @@ -1,5 +1,5 @@ use derive_builder::Builder; -use hf_hub::api::sync::ApiError; +use hf_hub::HFError; use strum::{EnumDiscriminants, EnumString, VariantNames}; use subenum::subenum; @@ -10,11 +10,12 @@ use crate::{ dream_shaper_xl_2_1_turbo, ernie_image, ernie_image_turbo, flux_1_dev, flux_1_mini, flux_1_schnell, flux_2_dev, flux_2_klein_4b, flux_2_klein_9b, flux_2_klein_base_4b, flux_2_klein_base_9b, hi_dream_o1_image, hi_dream_o1_image_dev, juggernaut_xl_11, krea2, - krea2_turbo, lens, lens_turbo, long_cat_image, nitro_sd_realism, nitro_sd_vibrant, - ovis_image, qwen_image, sd_turbo, sdxl_base_1_0, sdxl_turbo_1_0, sdxs512_dream_shaper, - segmind_vega, ssd_1b, stable_diffusion_1_4, stable_diffusion_1_5, stable_diffusion_2_1, - stable_diffusion_3_5_large, stable_diffusion_3_5_large_turbo, stable_diffusion_3_5_medium, - stable_diffusion_3_medium, twinflow_z_image_turbo, z_image_turbo, + krea2_turbo, lens, lens_turbo, long_cat_image, mini_t2i, nitro_sd_realism, + nitro_sd_vibrant, ovis_image, qwen_image, sd_turbo, sdxl_base_1_0, sdxl_turbo_1_0, + sdxs512_dream_shaper, segmind_vega, ssd_1b, stable_diffusion_1_4, stable_diffusion_1_5, + stable_diffusion_2_1, stable_diffusion_3_5_large, stable_diffusion_3_5_large_turbo, + stable_diffusion_3_5_medium, stable_diffusion_3_medium, twinflow_z_image_turbo, + z_image_turbo, }, }; @@ -361,10 +362,12 @@ pub enum Preset { Krea2(Krea2Weight), /// Diffusion Flash attention enabled. 512x512. 4 steps. Offload params to CPU enabled. Krea2Turbo(Krea2Weight), + /// 512x512. 100 steps. cfg_scale 6.0. Enable [crate::api::SampleMethod::EULER_SAMPLE_METHOD] + MiniT2I, } impl Preset { - fn try_configs_builder(self) -> Result<(ConfigBuilder, ModelConfigBuilder), ApiError> { + fn try_configs_builder(self) -> Result<(ConfigBuilder, ModelConfigBuilder), HFError> { match self { Preset::StableDiffusion1_4 => stable_diffusion_1_4(), Preset::StableDiffusion1_5 => stable_diffusion_1_5(), @@ -411,6 +414,7 @@ impl Preset { Preset::BooguImageTurbo => boogu_image_turbo(), Preset::Krea2(sd_type_t) => krea2(sd_type_t), Preset::Krea2Turbo(sd_type_t) => krea2_turbo(sd_type_t), + Preset::MiniT2I => mini_t2i(), } } } @@ -422,7 +426,7 @@ pub type ConfigsBuilder = (ConfigBuilder, ModelConfigBuilder); pub type Configs = (Config, ModelConfig); /// Helper functions that modifies the [ConfigBuilder] See [crate::modifier] -type ModifierFunction = dyn FnOnce(ConfigsBuilder) -> Result; +type ModifierFunction = dyn FnOnce(ConfigsBuilder) -> Result; #[derive(Builder)] #[builder( @@ -443,7 +447,7 @@ impl PresetBuilder { /// Add modifier that will apply in sequence pub fn with_modifier(mut self, f: F) -> Self where - F: FnOnce(ConfigsBuilder) -> Result + 'static, + F: FnOnce(ConfigsBuilder) -> Result + 'static, { if self.modifiers.is_none() { self.modifiers = Some(Vec::new()); @@ -456,7 +460,7 @@ impl PresetBuilder { let preset = self.internal_build()?; let configs: ConfigsBuilder = preset .try_into() - .map_err(|err: ApiError| ConfigBuilderError::ValidationError(err.to_string()))?; + .map_err(|err: HFError| ConfigBuilderError::ValidationError(err.to_string()))?; let config = configs.0.build()?; let config_model = configs.1.build()?; @@ -465,7 +469,7 @@ impl PresetBuilder { } impl TryFrom for ConfigsBuilder { - type Error = ApiError; + type Error = HFError; fn try_from(value: PresetConfig) -> Result { let mut configs_builder = value.preset.try_configs_builder()?; diff --git a/src/preset_builder.rs b/src/preset_builder.rs index 4030f3d..982330f 100644 --- a/src/preset_builder.rs +++ b/src/preset_builder.rs @@ -16,11 +16,11 @@ use crate::{ }, }; use diffusion_rs_sys::scheduler_t; -use hf_hub::api::sync::ApiError; +use hf_hub::HFError; use crate::{api::ConfigBuilder, util::download_file_hf_hub}; -pub fn stable_diffusion_1_4() -> Result { +pub fn stable_diffusion_1_4() -> Result { let model_path = download_file_hf_hub("CompVis/stable-diffusion-v-1-4-original", "sd-v1-4.ckpt")?; @@ -31,7 +31,7 @@ pub fn stable_diffusion_1_4() -> Result { Ok((ConfigBuilder::default(), model_config)) } -pub fn stable_diffusion_1_5() -> Result { +pub fn stable_diffusion_1_5() -> Result { let model_path = download_file_hf_hub( "stablediffusiontutorials/stable-diffusion-v1.5", "v1-5-pruned-emaonly.safetensors", @@ -44,7 +44,7 @@ pub fn stable_diffusion_1_5() -> Result { Ok((ConfigBuilder::default(), model_config)) } -pub fn stable_diffusion_2_1() -> Result { +pub fn stable_diffusion_2_1() -> Result { let model_path = download_file_hf_hub( "stabilityai/stable-diffusion-2-1", "v2-1_768-nonema-pruned.safetensors", @@ -61,7 +61,7 @@ pub fn stable_diffusion_2_1() -> Result { Ok((config, model_config)) } -pub fn stable_diffusion_3_medium() -> Result { +pub fn stable_diffusion_3_medium() -> Result { let model_path = download_file_hf_hub( "stabilityai/stable-diffusion-3-medium", "sd3_medium_incl_clips_t5xxlfp16.safetensors", @@ -69,7 +69,7 @@ pub fn stable_diffusion_3_medium() -> Result { let mut config = ConfigBuilder::default(); - config.cfg_scale(4.5).steps(30).height(1024).width(1024); + config.cfg_scale(4.5_f32).steps(30).height(1024).width(1024); let mut model_config = ModelConfigBuilder::default(); @@ -78,7 +78,7 @@ pub fn stable_diffusion_3_medium() -> Result { Ok((config, model_config)) } -pub fn sdxl_base_1_0() -> Result { +pub fn sdxl_base_1_0() -> Result { let model_path = download_file_hf_hub( "stabilityai/stable-diffusion-xl-base-1.0", "sd_xl_base_1.0.safetensors", @@ -94,7 +94,7 @@ pub fn sdxl_base_1_0() -> Result { sdxl_vae_fp16_fix((config, model_config)) } -pub fn flux_1_dev(sd_type: Flux1Weight) -> Result { +pub fn flux_1_dev(sd_type: Flux1Weight) -> Result { let model_path = flux_1_model_weight("dev", sd_type)?; let mut builder = flux_1_dev_schnell("dev", 28)?; @@ -108,7 +108,7 @@ pub fn flux_1_dev(sd_type: Flux1Weight) -> Result { } } -pub fn flux_1_schnell(sd_type: Flux1Weight) -> Result { +pub fn flux_1_schnell(sd_type: Flux1Weight) -> Result { let model_path = flux_1_model_weight("schnell", sd_type)?; let mut builder = flux_1_dev_schnell("schnell", 4)?; @@ -122,7 +122,7 @@ pub fn flux_1_schnell(sd_type: Flux1Weight) -> Result } } -fn flux_1_model_weight(model: &str, sd_type: Flux1Weight) -> Result { +fn flux_1_model_weight(model: &str, sd_type: Flux1Weight) -> Result { let weight_type = match sd_type { Flux1Weight::Q3_K => "q3_k", Flux1Weight::Q2_K => "q2_k", @@ -136,7 +136,7 @@ fn flux_1_model_weight(model: &str, sd_type: Flux1Weight) -> Result Result { +fn flux_1_dev_schnell(vae_model: &str, steps: i32) -> Result { let vae_path = download_file_hf_hub( format!("black-forest-labs/FLUX.1-{vae_model}").as_str(), "ae.safetensors", @@ -151,7 +151,7 @@ fn flux_1_clip_vae( vae_path: PathBuf, clip_l_path: PathBuf, steps: i32, -) -> Result { +) -> Result { let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); @@ -159,42 +159,46 @@ fn flux_1_clip_vae( .vae(vae_path) .clip_l(clip_l_path) .vae_tiling(true); - config.cfg_scale(1.).steps(steps).height(1024).width(1024); + config + .cfg_scale(1.0_f32) + .steps(steps) + .height(1024) + .width(1024); Ok((config, model_config)) } -pub fn sd_turbo() -> Result { +pub fn sd_turbo() -> Result { let model_path = download_file_hf_hub("stabilityai/sd-turbo", "sd_turbo.safetensors")?; let mut config = ConfigsBuilder::default(); config.1.model(model_path); - config.0.guidance(0.).cfg_scale(1.).steps(4); + config.0.guidance(0.0_f32).cfg_scale(1.0_f32).steps(4); Ok(config) } -pub fn sdxl_turbo_1_0() -> Result { +pub fn sdxl_turbo_1_0() -> Result { let model_path = download_file_hf_hub("stabilityai/sdxl-turbo", "sd_xl_turbo_1.0_fp16.safetensors")?; let mut config = ConfigsBuilder::default(); config.1.model(model_path); - config.0.guidance(0.).cfg_scale(1.).steps(4); + config.0.guidance(0.0_f32).cfg_scale(1.0_f32).steps(4); sdxl_vae_fp16_fix(config) } -pub fn stable_diffusion_3_5_large() -> Result { +pub fn stable_diffusion_3_5_large() -> Result { stable_diffusion_3_5("large", "large", 28, 4.5) } -pub fn stable_diffusion_3_5_large_turbo() -> Result { +pub fn stable_diffusion_3_5_large_turbo() -> Result { stable_diffusion_3_5("large-turbo", "large_turbo", 4, 0.) } -pub fn stable_diffusion_3_5_medium() -> Result { +pub fn stable_diffusion_3_5_medium() -> Result { stable_diffusion_3_5("medium", "medium", 40, 4.5) } @@ -203,7 +207,7 @@ pub fn stable_diffusion_3_5( file_model: &str, steps: i32, cfg_scale: f32, -) -> Result { +) -> Result { let model_path = download_file_hf_hub( format!("stabilityai/stable-diffusion-3.5-{model}").as_str(), format!("sd3.5_{file_model}.safetensors").as_str(), @@ -242,7 +246,7 @@ pub fn stable_diffusion_3_5( Ok(config) } -pub fn juggernaut_xl_11() -> Result { +pub fn juggernaut_xl_11() -> Result { let model_path = download_file_hf_hub( "RunDiffusion/Juggernaut-XI-v11", "Juggernaut-XI-byRunDiffusion.safetensors", @@ -255,20 +259,20 @@ pub fn juggernaut_xl_11() -> Result { .0 .sampling_method(SampleMethod::DPM2_SAMPLE_METHOD) .steps(20) - .guidance(6.) + .guidance(6.0_f32) .height(1024) .width(1024); Ok(config) } -pub fn flux_1_mini(sd_type: Flux1MiniWeight) -> Result { +pub fn flux_1_mini(sd_type: Flux1MiniWeight) -> Result { let model_path = flux_1_mini_model_weight(sd_type)?; let vae_path = download_file_hf_hub("Green-Sky/flux.1-schnell-GGUF", "ae-f16.gguf")?; let clip_l_path = download_file_hf_hub("Green-Sky/flux.1-schnell-GGUF", "clip_l-q8_0.gguf")?; let mut builder = flux_1_clip_vae(vae_path, clip_l_path, 20)?; builder.1.diffusion_model(model_path); - builder.0.cfg_scale(1.); + builder.0.cfg_scale(1.0_f32); match sd_type { Flux1MiniWeight::F32 => t5xxl_fp16_flux_1(builder), Flux1MiniWeight::Q8_0 => t5xxl_q8_0_flux_1(builder), @@ -280,7 +284,7 @@ pub fn flux_1_mini(sd_type: Flux1MiniWeight) -> Result } } -fn flux_1_mini_model_weight(sd_type: Flux1MiniWeight) -> Result { +fn flux_1_mini_model_weight(sd_type: Flux1MiniWeight) -> Result { let (repo, file) = match sd_type { Flux1MiniWeight::F32 => ("TencentARC/flux-mini", "flux-mini.safetensors"), Flux1MiniWeight::BF16 => ("HyperX-Sentience/Flux-Mini-GGUF", "flux-mini-BF16.gguf"), @@ -293,7 +297,7 @@ fn flux_1_mini_model_weight(sd_type: Flux1MiniWeight) -> Result Result { +pub fn chroma(sd_type: ChromaWeight) -> Result { let model_path = chroma_model_weight(sd_type)?; let vae_path = download_file_hf_hub("black-forest-labs/FLUX.1-dev", "ae.safetensors")?; let mut config = ConfigBuilder::default(); @@ -304,7 +308,7 @@ pub fn chroma(sd_type: ChromaWeight) -> Result { .vae(vae_path) .vae_tiling(true); config - .cfg_scale(4.) + .cfg_scale(4.0_f32) .sampling_method(SampleMethod::EULER_SAMPLE_METHOD) .steps(20) .height(512) @@ -318,7 +322,7 @@ pub fn chroma(sd_type: ChromaWeight) -> Result { } } -fn chroma_model_weight(sd_type: ChromaWeight) -> Result { +fn chroma_model_weight(sd_type: ChromaWeight) -> Result { let (repo, file) = match sd_type { ChromaWeight::BF16 => ( "silveroxides/Chroma-GGUF", @@ -336,7 +340,7 @@ fn chroma_model_weight(sd_type: ChromaWeight) -> Result { download_file_hf_hub(repo, file) } -pub fn nitro_sd_realism(sd_type: NitroSDRealismWeight) -> Result { +pub fn nitro_sd_realism(sd_type: NitroSDRealismWeight) -> Result { let model_path = nitro_sd_realism_weight(sd_type)?; let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); @@ -345,11 +349,11 @@ pub fn nitro_sd_realism(sd_type: NitroSDRealismWeight) -> Result Result { +fn nitro_sd_realism_weight(sd_type: NitroSDRealismWeight) -> Result { let (repo, file) = match sd_type { NitroSDRealismWeight::F16 => ("mrfatso/NitroFusion-GGUF", "nitrosd-realism_f16.gguf"), NitroSDRealismWeight::Q2_K => ("mrfatso/NitroFusion-GGUF", "nitrosd-realism_q2_K.gguf"), @@ -362,7 +366,7 @@ fn nitro_sd_realism_weight(sd_type: NitroSDRealismWeight) -> Result Result { +pub fn nitro_sd_vibrant(sd_type: NitroSDVibrantWeight) -> Result { let model_path = nitro_sd_vibrant_weight(sd_type)?; let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); @@ -371,11 +375,11 @@ pub fn nitro_sd_vibrant(sd_type: NitroSDVibrantWeight) -> Result Result { +fn nitro_sd_vibrant_weight(sd_type: NitroSDVibrantWeight) -> Result { let (repo, file) = match sd_type { NitroSDVibrantWeight::F16 => ("mrfatso/NitroFusion-GGUF", "nitrosd-vibrant_f16.gguf"), NitroSDVibrantWeight::Q2_K => ("mrfatso/NitroFusion-GGUF", "nitrosd-vibrant_q2_K.gguf"), @@ -388,7 +392,7 @@ fn nitro_sd_vibrant_weight(sd_type: NitroSDVibrantWeight) -> Result Result { +pub fn diff_instruct_star(sd_type: DiffInstructStarWeight) -> Result { let model_path = diff_instruct_star_weight(sd_type)?; let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); @@ -397,11 +401,11 @@ pub fn diff_instruct_star(sd_type: DiffInstructStarWeight) -> Result Result { +fn diff_instruct_star_weight(sd_type: DiffInstructStarWeight) -> Result { let (repo, file) = match sd_type { DiffInstructStarWeight::F16 => ( "mrfatso/Diff-InstructStar-GGUF", @@ -435,19 +439,19 @@ fn diff_instruct_star_weight(sd_type: DiffInstructStarWeight) -> Result Result { +pub fn chroma_radiance(sd_type: ChromaRadianceWeight) -> Result { let model_path = chroma_radiance_weight(sd_type)?; let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); model_config.model(model_path); config - .cfg_scale(4.) + .cfg_scale(4.0_f32) .sampling_method(SampleMethod::EULER_SAMPLE_METHOD); t5xxl_fp16_flux_1((config, model_config)) } -fn chroma_radiance_weight(sd_type: ChromaRadianceWeight) -> Result { +fn chroma_radiance_weight(sd_type: ChromaRadianceWeight) -> Result { let (repo, file) = match sd_type { ChromaRadianceWeight::BF16 => ( "silveroxides/Chroma1-Radiance-GGUF", @@ -461,17 +465,17 @@ fn chroma_radiance_weight(sd_type: ChromaRadianceWeight) -> Result Result { +pub fn ssd_1b(sd_type: SSD1BWeight) -> Result { let model = ssd_1b_weight(sd_type)?; let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); model_config.model(model); - config.cfg_scale(9.).height(1024).width(1024); + config.cfg_scale(9.0_f32).height(1024).width(1024); Ok((config, model_config)) } -fn ssd_1b_weight(sd_type: SSD1BWeight) -> Result { +fn ssd_1b_weight(sd_type: SSD1BWeight) -> Result { let (repo, file) = match sd_type { SSD1BWeight::F16 => ("segmind/SSD-1B", "SSD-1B-A1111.safetensors"), SSD1BWeight::F8_E4M3 => ( @@ -482,7 +486,7 @@ fn ssd_1b_weight(sd_type: SSD1BWeight) -> Result { download_file_hf_hub(repo, file) } -pub fn flux_2_dev(sd_type: Flux2Weight) -> Result { +pub fn flux_2_dev(sd_type: Flux2Weight) -> Result { let (model, llm) = flux_2_dev_weight(sd_type)?; let vae = download_file_hf_hub( "black-forest-labs/FLUX.2-dev", @@ -498,13 +502,13 @@ pub fn flux_2_dev(sd_type: Flux2Weight) -> Result { .vae(vae) .vae_tiling(true); config - .cfg_scale(1.) + .cfg_scale(1.0_f32) .sampling_method(SampleMethod::EULER_SAMPLE_METHOD); offload_params_to_cpu((config, model_config)) } -fn flux_2_dev_weight(sd_type: Flux2Weight) -> Result<(PathBuf, PathBuf), ApiError> { +fn flux_2_dev_weight(sd_type: Flux2Weight) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { Flux2Weight::Q4_0 => ( ("city96/FLUX.2-dev-gguf", "flux2-dev-Q4_0.gguf"), @@ -589,7 +593,7 @@ fn flux_2_dev_weight(sd_type: Flux2Weight) -> Result<(PathBuf, PathBuf), ApiErro Ok((model_path, llm_path)) } -pub fn z_image_turbo(sd_type: ZImageTurboWeight) -> Result { +pub fn z_image_turbo(sd_type: ZImageTurboWeight) -> Result { let (model, llm) = z_image_turbo_weight(sd_type)?; let vae = download_file_hf_hub( "black-forest-labs/FLUX.1-schnell", @@ -604,12 +608,12 @@ pub fn z_image_turbo(sd_type: ZImageTurboWeight) -> Result Result<(PathBuf, PathBuf), ApiError> { +fn z_image_turbo_weight(sd_type: ZImageTurboWeight) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { ZImageTurboWeight::Q4_0 => ( ("leejet/Z-Image-Turbo-GGUF", "z_image_turbo-Q4_0.gguf"), @@ -676,7 +680,7 @@ fn z_image_turbo_weight(sd_type: ZImageTurboWeight) -> Result<(PathBuf, PathBuf) Ok((model_path, llm_path)) } -pub fn qwen_image(sd_type: QwenImageWeight) -> Result { +pub fn qwen_image(sd_type: QwenImageWeight) -> Result { let (model, llm) = qwen_image_weight(sd_type)?; let vae = download_file_hf_hub( "Comfy-Org/Qwen-Image_ComfyUI", @@ -691,17 +695,17 @@ pub fn qwen_image(sd_type: QwenImageWeight) -> Result .vae(vae) .flash_attention(true) .vae_tiling(true) - .flow_shift(3.0); + .flow_shift(3.0_f32); config .sampling_method(SampleMethod::EULER_SAMPLE_METHOD) - .cfg_scale(2.5) + .cfg_scale(2.5_f32) .height(1024) .width(1024); offload_params_to_cpu((config, model_config)) } -fn qwen_image_weight(sd_type: QwenImageWeight) -> Result<(PathBuf, PathBuf), ApiError> { +fn qwen_image_weight(sd_type: QwenImageWeight) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { QwenImageWeight::Q4_0 => ( ("QuantStack/Qwen-Image-GGUF", "Qwen_Image-Q4_0.gguf"), @@ -799,7 +803,7 @@ fn qwen_image_weight(sd_type: QwenImageWeight) -> Result<(PathBuf, PathBuf), Api Ok((model_path, llm_path)) } -pub fn ovis_image(sd_type: OvisImageWeight) -> Result { +pub fn ovis_image(sd_type: OvisImageWeight) -> Result { let (model, llm) = ovis_image_weight(sd_type)?; let vae = download_file_hf_hub( "black-forest-labs/FLUX.1-schnell", @@ -814,12 +818,12 @@ pub fn ovis_image(sd_type: OvisImageWeight) -> Result .vae(vae) .flash_attention(true) .vae_tiling(true); - config.steps(20).cfg_scale(5.).height(512).width(512); + config.steps(20).cfg_scale(5.0_f32).height(512).width(512); offload_params_to_cpu((config, model_config)) } -fn ovis_image_weight(sd_type: OvisImageWeight) -> Result<(PathBuf, PathBuf), ApiError> { +fn ovis_image_weight(sd_type: OvisImageWeight) -> Result<(PathBuf, PathBuf), HFError> { let model = match sd_type { OvisImageWeight::Q4_0 => ("leejet/Ovis-Image-7B-GGUF", "ovis_image-Q4_0.gguf"), OvisImageWeight::Q8_0 => ("leejet/Ovis-Image-7B-GGUF", "ovis_image-Q8_0.gguf"), @@ -836,7 +840,7 @@ fn ovis_image_weight(sd_type: OvisImageWeight) -> Result<(PathBuf, PathBuf), Api Ok((model_path, llm_path)) } -pub fn dream_shaper_xl_2_1_turbo() -> Result { +pub fn dream_shaper_xl_2_1_turbo() -> Result { let model_path = download_file_hf_hub( "Lykon/dreamshaper-xl-v2-turbo", "DreamShaperXL_Turbo_v2_1.safetensors", @@ -849,7 +853,7 @@ pub fn dream_shaper_xl_2_1_turbo() -> Result { .0 .sampling_method(SampleMethod::DPM2_SAMPLE_METHOD) .steps(6) - .guidance(2.) + .guidance(2.0_f32) .height(1024) .width(1024); @@ -858,7 +862,7 @@ pub fn dream_shaper_xl_2_1_turbo() -> Result { pub fn twinflow_z_image_turbo( sd_type: TwinFlowZImageTurboExpWeight, -) -> Result { +) -> Result { let (model, llm) = twinflow_z_image_turbo_weight(sd_type)?; let vae = download_file_hf_hub( "black-forest-labs/FLUX.1-schnell", @@ -876,7 +880,7 @@ pub fn twinflow_z_image_turbo( .scheduler(Scheduler::SMOOTHSTEP_SCHEDULER); config .steps(3) - .cfg_scale(1.) + .cfg_scale(1.0_f32) .height(1024) .width(512) .sampling_method(SampleMethod::DPM2_SAMPLE_METHOD); @@ -886,7 +890,7 @@ pub fn twinflow_z_image_turbo( fn twinflow_z_image_turbo_weight( sd_type: TwinFlowZImageTurboExpWeight, -) -> Result<(PathBuf, PathBuf), ApiError> { +) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { TwinFlowZImageTurboExpWeight::Q4_0 => ( ( @@ -954,7 +958,7 @@ fn twinflow_z_image_turbo_weight( Ok((model_path, llm_path)) } -pub fn sdxs512_dream_shaper(sd_type: SDXS512DreamShaperWeight) -> Result { +pub fn sdxs512_dream_shaper(sd_type: SDXS512DreamShaperWeight) -> Result { let model = match sd_type { SDXS512DreamShaperWeight::F16 => { download_file_hf_hub("akleine/sdxs-512", "sdxs.safetensors")? @@ -969,12 +973,12 @@ pub fn sdxs512_dream_shaper(sd_type: SDXS512DreamShaperWeight) -> Result Result { +pub fn flux_2_klein_4b(sd_type: Flux2Klein4BWeight) -> Result { let (model, llm) = flux_2_klein_4b_weight(sd_type)?; let vae = download_file_hf_hub( "black-forest-labs/FLUX.2-dev", @@ -989,12 +993,12 @@ pub fn flux_2_klein_4b(sd_type: Flux2Klein4BWeight) -> Result Result<(PathBuf, PathBuf), ApiError> { +fn flux_2_klein_4b_weight(sd_type: Flux2Klein4BWeight) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { Flux2Klein4BWeight::Q4_0 => ( ("leejet/FLUX.2-klein-4B-GGUF", "flux-2-klein-4b-Q4_0.gguf"), @@ -1017,7 +1021,7 @@ fn flux_2_klein_4b_weight(sd_type: Flux2Klein4BWeight) -> Result<(PathBuf, PathB Ok((model_path, llm_path)) } -pub fn flux_2_klein_base_4b(sd_type: Flux2KleinBase4BWeight) -> Result { +pub fn flux_2_klein_base_4b(sd_type: Flux2KleinBase4BWeight) -> Result { let (model, llm) = flux_2_klein_base_4b_weight(sd_type)?; let vae = download_file_hf_hub( "black-forest-labs/FLUX.2-dev", @@ -1032,14 +1036,14 @@ pub fn flux_2_klein_base_4b(sd_type: Flux2KleinBase4BWeight) -> Result Result<(PathBuf, PathBuf), ApiError> { +) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { Flux2KleinBase4BWeight::Q4_0 => ( ( @@ -1068,7 +1072,7 @@ fn flux_2_klein_base_4b_weight( Ok((model_path, llm_path)) } -pub fn flux_2_klein_9b(sd_type: Flux2Klein9BWeight) -> Result { +pub fn flux_2_klein_9b(sd_type: Flux2Klein9BWeight) -> Result { let (model, llm) = flux_2_klein_9b_weight(sd_type)?; let vae = download_file_hf_hub( "black-forest-labs/FLUX.2-dev", @@ -1083,12 +1087,12 @@ pub fn flux_2_klein_9b(sd_type: Flux2Klein9BWeight) -> Result Result<(PathBuf, PathBuf), ApiError> { +fn flux_2_klein_9b_weight(sd_type: Flux2Klein9BWeight) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { Flux2Klein9BWeight::Q4_0 => ( ("leejet/FLUX.2-klein-9B-GGUF", "flux-2-klein-9b-Q4_0.gguf"), @@ -1111,7 +1115,7 @@ fn flux_2_klein_9b_weight(sd_type: Flux2Klein9BWeight) -> Result<(PathBuf, PathB Ok((model_path, llm_path)) } -pub fn flux_2_klein_base_9b(sd_type: Flux2KleinBase9BWeight) -> Result { +pub fn flux_2_klein_base_9b(sd_type: Flux2KleinBase9BWeight) -> Result { let (model, llm) = flux_2_klein_base_9b_weight(sd_type)?; let vae = download_file_hf_hub( "black-forest-labs/FLUX.2-dev", @@ -1126,14 +1130,14 @@ pub fn flux_2_klein_base_9b(sd_type: Flux2KleinBase9BWeight) -> Result Result<(PathBuf, PathBuf), ApiError> { +) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { Flux2KleinBase9BWeight::Q4_0 => ( ( @@ -1155,18 +1159,18 @@ fn flux_2_klein_base_9b_weight( Ok((model_path, llm_path)) } -pub fn segmind_vega() -> Result { +pub fn segmind_vega() -> Result { let model = download_file_hf_hub("segmind/Segmind-Vega", "segmind-vega.safetensors")?; let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); model_config.model(model).vae_tiling(true); - config.guidance(9.).steps(25).height(1024).width(1024); + config.guidance(9.0_f32).steps(25).height(1024).width(1024); Ok((config, model_config)) } -pub fn anima(sd_type: AnimaWeight) -> Result { +pub fn anima(sd_type: AnimaWeight) -> Result { let (model, llm) = anima_weight(sd_type)?; let vae = download_file_hf_hub( "circlestone-labs/Anima", @@ -1180,12 +1184,12 @@ pub fn anima(sd_type: AnimaWeight) -> Result { .llm(llm) .vae(vae) .vae_tiling(true); - config.cfg_scale(4.).steps(30).height(1024).width(1024); + config.cfg_scale(4.0_f32).steps(30).height(1024).width(1024); Ok((config, model_config)) } -fn anima_weight(sd_type: AnimaWeight) -> Result<(PathBuf, PathBuf), ApiError> { +fn anima_weight(sd_type: AnimaWeight) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { AnimaWeight::Q4_K => ( ("Bedovyy/Anima-GGUF", "anima-preview-Q4_K_M.gguf"), @@ -1266,7 +1270,7 @@ fn anima_weight(sd_type: AnimaWeight) -> Result<(PathBuf, PathBuf), ApiError> { Ok((model_path, llm_path)) } -pub fn anima2(sd_type: Anima2Weight) -> Result { +pub fn anima2(sd_type: Anima2Weight) -> Result { let (model, llm) = anima2_weight(sd_type)?; let vae = download_file_hf_hub( "circlestone-labs/Anima", @@ -1280,12 +1284,12 @@ pub fn anima2(sd_type: Anima2Weight) -> Result { .llm(llm) .vae(vae) .vae_tiling(true); - config.cfg_scale(4.).steps(30).height(1024).width(1024); + config.cfg_scale(4.0_f32).steps(30).height(1024).width(1024); Ok((config, model_config)) } -fn anima2_weight(sd_type: Anima2Weight) -> Result<(PathBuf, PathBuf), ApiError> { +fn anima2_weight(sd_type: Anima2Weight) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { Anima2Weight::Q4_K => ( ("Bedovyy/Anima-GGUF", "anima-preview2-Q4_K_M.gguf"), @@ -1331,7 +1335,7 @@ fn anima2_weight(sd_type: Anima2Weight) -> Result<(PathBuf, PathBuf), ApiError> Ok((model_path, llm_path)) } -pub fn ernie_image(sd_type: ErnieImageWeight) -> Result { +pub fn ernie_image(sd_type: ErnieImageWeight) -> Result { let vae = ernie_image_vae()?; let llm = ernie_image_llm(sd_type)?; let model = ernie_image_weight(sd_type)?; @@ -1343,12 +1347,12 @@ pub fn ernie_image(sd_type: ErnieImageWeight) -> Result Result { +pub fn ernie_image_turbo(sd_type: ErnieImageWeight) -> Result { let vae = ernie_image_vae()?; let llm = ernie_image_llm(sd_type)?; let model = ernie_image_turbo_weight(sd_type)?; @@ -1360,12 +1364,12 @@ pub fn ernie_image_turbo(sd_type: ErnieImageWeight) -> Result Result { +fn ernie_image_weight(sd_type: ErnieImageWeight) -> Result { match sd_type { ErnieImageWeight::F16 => { download_file_hf_hub("unsloth/ERNIE-Image-GGUF", "ernie-image-F16.gguf") @@ -1406,7 +1410,7 @@ fn ernie_image_weight(sd_type: ErnieImageWeight) -> Result { } } -fn ernie_image_turbo_weight(sd_type: ErnieImageWeight) -> Result { +fn ernie_image_turbo_weight(sd_type: ErnieImageWeight) -> Result { match sd_type { ErnieImageWeight::F16 => download_file_hf_hub( "unsloth/ERNIE-Image-Turbo-GGUF", @@ -1459,11 +1463,11 @@ fn ernie_image_turbo_weight(sd_type: ErnieImageWeight) -> Result Result { +fn ernie_image_vae() -> Result { download_file_hf_hub("Comfy-Org/ERNIE-Image", "vae/flux2-vae.safetensors") } -fn ernie_image_llm(sd_type: ErnieImageWeight) -> Result { +fn ernie_image_llm(sd_type: ErnieImageWeight) -> Result { match sd_type { ErnieImageWeight::F16 => download_file_hf_hub( "unsloth/Ministral-3-3B-Instruct-2512-GGUF", @@ -1516,7 +1520,7 @@ fn ernie_image_llm(sd_type: ErnieImageWeight) -> Result { } } -pub fn hi_dream_o1_image_dev() -> Result { +pub fn hi_dream_o1_image_dev() -> Result { let model = download_file_hf_hub( "Comfy-Org/HiDream-O1-Image", "checkpoints/hidream_o1_image_dev_bf16.safetensors", @@ -1524,12 +1528,12 @@ pub fn hi_dream_o1_image_dev() -> Result { let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); model_config.model(model); - config.cfg_scale(1.0).steps(20).height(1024).width(1024); + config.cfg_scale(1.0_f32).steps(20).height(1024).width(1024); Ok((config, model_config)) } -pub fn hi_dream_o1_image() -> Result { +pub fn hi_dream_o1_image() -> Result { let model = download_file_hf_hub( "Comfy-Org/HiDream-O1-Image", "checkpoints/hidream_o1_image_bf16.safetensors", @@ -1537,12 +1541,12 @@ pub fn hi_dream_o1_image() -> Result { let mut config = ConfigBuilder::default(); let mut model_config = ModelConfigBuilder::default(); model_config.model(model); - config.cfg_scale(1.0).steps(20).height(1024).width(1024); + config.cfg_scale(1.0_f32).steps(20).height(1024).width(1024); Ok((config, model_config)) } -pub fn long_cat_image(sd_type: LongCatImageWeight) -> Result { +pub fn long_cat_image(sd_type: LongCatImageWeight) -> Result { let (model, llm) = long_cat_image_weight_llm(sd_type)?; let vae = download_file_hf_hub("black-forest-labs/FLUX.1-dev", "ae.safetensors")?; let mut config = ConfigBuilder::default(); @@ -1552,12 +1556,12 @@ pub fn long_cat_image(sd_type: LongCatImageWeight) -> Result Result Result<(PathBuf, PathBuf), ApiError> { +fn long_cat_image_weight_llm(sd_type: LongCatImageWeight) -> Result<(PathBuf, PathBuf), HFError> { let (model, llm) = match sd_type { LongCatImageWeight::Q4_0 => ( ( @@ -1673,7 +1677,7 @@ fn long_cat_image_weight_llm(sd_type: LongCatImageWeight) -> Result<(PathBuf, Pa Ok((model_path, llm_path)) } -pub fn lens_turbo() -> Result { +pub fn lens_turbo() -> Result { let vae = download_file_hf_hub("black-forest-labs/FLUX.2-dev", "ae.safetensors")?; let llm = download_file_hf_hub("unsloth/gpt-oss-20b-GGUF", "gpt-oss-20b-UD-Q4_K_XL.gguf")?; let model = download_file_hf_hub( @@ -1687,12 +1691,12 @@ pub fn lens_turbo() -> Result { .llm(llm) .vae(vae) .diffusion_flash_attention(true); - config.cfg_scale(1.).steps(4).height(512).width(512); + config.cfg_scale(1.0_f32).steps(4).height(512).width(512); Ok((config, model_config)) } -pub fn lens() -> Result { +pub fn lens() -> Result { let vae = download_file_hf_hub("black-forest-labs/FLUX.2-dev", "ae.safetensors")?; let llm = download_file_hf_hub("unsloth/gpt-oss-20b-GGUF", "gpt-oss-20b-UD-Q4_K_XL.gguf")?; let model = download_file_hf_hub("Comfy-Org/Lens", "diffusion_models/lens_bf16.safetensors")?; @@ -1703,12 +1707,12 @@ pub fn lens() -> Result { .llm(llm) .vae(vae) .diffusion_flash_attention(true); - config.cfg_scale(5.).height(512).width(512); + config.cfg_scale(5.0_f32).height(512).width(512); Ok((config, model_config)) } -pub fn boogu_image() -> Result { +pub fn boogu_image() -> Result { let model = download_file_hf_hub( "Comfy-Org/Boogu-Image", "diffusion_models/boogu_image_base_bf16.safetensors", @@ -1718,7 +1722,7 @@ pub fn boogu_image() -> Result { Ok((config, model_config)) } -pub fn boogu_image_turbo() -> Result { +pub fn boogu_image_turbo() -> Result { let model = download_file_hf_hub( "Comfy-Org/Boogu-Image", "diffusion_models/diffusion_models/diffusion_models/boogu_image_turbo_hotfix_bf16.safetensors", @@ -1729,7 +1733,7 @@ pub fn boogu_image_turbo() -> Result { Ok((config, model_config)) } -fn boogu_image_common() -> Result { +fn boogu_image_common() -> Result { let llm = download_file_hf_hub( "unsloth/Qwen3-VL-8B-Instruct-GGUF", "Qwen3-VL-8B-Instruct-Q4_K_M.gguf", @@ -1747,7 +1751,7 @@ fn boogu_image_common() -> Result { offload_params_to_cpu((config, model_config)) } -pub fn krea2(sd_type_t: Krea2Weight) -> Result { +pub fn krea2(sd_type_t: Krea2Weight) -> Result { let model = match sd_type_t { Krea2Weight::Q8_0 => { download_file_hf_hub("realrebelai/KREA-2_GGUFs", "BASE/Krea-2-Base-Q8_0.gguf") @@ -1770,7 +1774,7 @@ pub fn krea2(sd_type_t: Krea2Weight) -> Result { Ok((config, model_config)) } -pub fn krea2_turbo(sd_type_t: Krea2Weight) -> Result { +pub fn krea2_turbo(sd_type_t: Krea2Weight) -> Result { let model = match sd_type_t { Krea2Weight::Q8_0 => { download_file_hf_hub("realrebelai/KREA-2_GGUFs", "TURBO/Krea-2-Turbo-Q8_0.gguf") @@ -1794,7 +1798,7 @@ pub fn krea2_turbo(sd_type_t: Krea2Weight) -> Result { Ok((config, model_config)) } -fn krea2_common(sd_type_t: Krea2Weight) -> Result { +fn krea2_common(sd_type_t: Krea2Weight) -> Result { let llm = if sd_type_t == Krea2Weight::Q8_0 { download_file_hf_hub( "Qwen/Qwen3-VL-4B-Instruct-GGUF", @@ -1821,3 +1825,22 @@ fn krea2_common(sd_type_t: Krea2Weight) -> Result { offload_params_to_cpu((config, model_config)) } + +pub fn mini_t2i() -> Result { + let t5xxl = download_file_hf_hub("google/flan-t5-large", "model.safetensors")?; + let diffusion_model = download_file_hf_hub( + "MiniT2I/MiniT2I", + "minit2i-b-16/transformer/diffusion_pytorch_model.safetensors", + )?; + let mut config = ConfigBuilder::default(); + let mut model_config = ModelConfigBuilder::default(); + model_config.t5xxl(t5xxl).diffusion_model(diffusion_model); + config + .cfg_scale(6.0_f32) + .steps(100) + .height(512) + .width(512) + .sampling_method(SampleMethod::EULER_SAMPLE_METHOD); + + Ok((config, model_config)) +} diff --git a/src/util.rs b/src/util.rs index acdd7a7..3057704 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,9 +1,13 @@ use std::{ + env, path::PathBuf, sync::{OnceLock, RwLock}, }; -use hf_hub::api::sync::{ApiBuilder, ApiError}; +use hf_hub::{ + HFClientBuilder, HFError, + progress::{DownloadEvent, ProgressEvent, ProgressHandler}, +}; static TOKEN: OnceLock> = OnceLock::new(); @@ -15,11 +19,62 @@ pub fn set_hf_token(token: &str) { } /// Download file from huggingface hub -pub fn download_file_hf_hub(repo: &str, file: &str) -> Result { - let token = TOKEN.get().map(|token| token.read().unwrap().to_owned()); - let repo = ApiBuilder::new() - .with_token(token) - .build()? - .model(repo.to_string()); - repo.get(file) +pub fn download_file_hf_hub(repo: &str, file: &str) -> Result { + let (owner, repo) = repo.split_once("/").unwrap_or_default(); + let mut hf_client = + if let Some(token) = TOKEN.get().map(|token| token.read().unwrap().to_owned()) { + HFClientBuilder::new().token(token) + } else { + HFClientBuilder::new() + }; + if let Some(home_dir) = env::home_dir() { + let cache_dir = home_dir.join(".cache/huggingface"); + hf_client = hf_client.cache_dir(cache_dir); + } + hf_client + .build_sync()? + .model(owner, repo) + .download_file() + .filename(file) + .progress(PrintProgressHandler(repo.to_string(), file.to_string())) + .send() +} + +struct PrintProgressHandler(String, String); + +impl ProgressHandler for PrintProgressHandler { + fn on_progress(&self, event: &ProgressEvent) { + if let ProgressEvent::Download(dl) = event { + match dl { + DownloadEvent::Start { + total_files: _, + total_bytes, + } => { + println!( + "Starting download: {}/{}, {total_bytes} bytes", + self.0, self.1 + ); + } + DownloadEvent::Progress { files } => { + for f in files { + let pct = (f.bytes_completed * 100) + .checked_div(f.total_bytes) + .unwrap_or(0); + print!("\r"); + print!( + " {}: {pct}% ({}/{}) bytes", + format_args!("{}/{}", self.0, self.1), + f.bytes_completed, + f.total_bytes + ); + } + } + DownloadEvent::Complete => { + println!(); + println!("Download complete."); + } + _ => {} + } + } + } } diff --git a/sys/Cargo.toml b/sys/Cargo.toml index e00be01..c774480 100644 --- a/sys/Cargo.toml +++ b/sys/Cargo.toml @@ -78,6 +78,6 @@ vulkan = [] sycl = [] [build-dependencies] -cmake = "0.1.51" -bindgen = "0.71.1" +cmake = "0.1.58" +bindgen = "0.72.1" fs_extra = "1.3.0" diff --git a/sys/stable-diffusion.cpp b/sys/stable-diffusion.cpp index 8caa3f9..b290693 160000 --- a/sys/stable-diffusion.cpp +++ b/sys/stable-diffusion.cpp @@ -1 +1 @@ -Subproject commit 8caa3f908ae6d4a4bef531e73b9a969f266a3d1f +Subproject commit b2906939774dc73453467215c80390404d0a2701