From 6a3c6fd3232c834b5c622c0e4cf2e95cf106c324 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Thu, 6 Aug 2026 01:59:13 +0000 Subject: [PATCH 1/5] Enable Online FP8&NF4 on Minimax-H3 --- examples/minimax_h3/README.md | 70 +++++++++++++++++-- examples/minimax_h3/common.py | 48 ++++++++++++- .../minimax_h3_fl2va_bnb_nf4_h100.py | 55 +++++++++++++++ examples/minimax_h3/minimax_h3_fl2va_h100.py | 16 ++++- .../minimax_h3_fl2va_torchao_fp8_h100.py | 55 +++++++++++++++ examples/minimax_h3/minimax_h3_ref2va_h100.py | 5 ++ telefuser/models/minimax_h3_dit.py | 39 ++++++++++- telefuser/pipelines/minimax_h3/denoising.py | 15 ++++ 8 files changed, 296 insertions(+), 7 deletions(-) create mode 100644 examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py create mode 100644 examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py diff --git a/examples/minimax_h3/README.md b/examples/minimax_h3/README.md index eabdc2f4..4434474f 100644 --- a/examples/minimax_h3/README.md +++ b/examples/minimax_h3/README.md @@ -291,6 +291,68 @@ The Ulysses degree must divide 56 attention heads. Scripts must run from their g processes can spawn safely. H100 examples request packed FlashAttention 4 and fall back to packed PyTorch SDPA when FlashAttention 4 is unavailable. +## Online DiT Quantization + +MiniMax H3 supports two single-GPU online quantization backends for the DiT transformer Linear layers: + +| CLI value | Backend | Weight/activation path | +|---|---|---| +| torchao-fp8 | TorchAO | FP8 dynamic activation and FP8 weight when supported, otherwise TorchAO's FP8 weight-only path | +| bnb-nf4 | bitsandbytes | NF4 weight-only with BF16 compute | + +Both paths convert the 258 Linear layers in the main and token-refiner transformer blocks. The FP32 video/audio +patch projections, timestep embedding, output projections, text encoder, and VAEs retain their reference dtypes. +The BF16 DiT is loaded from the original shards, moved to CUDA after text encoding, quantized on first denoising use, +and then kept resident for the pipeline lifetime. This ordering avoids a simultaneous BF16 text encoder and DiT on +one GPU and avoids unsupported CPU transfers of quantized tensor subclasses. +TorchAO conversion has a transient memory peak near the BF16 footprint; use the full 80 GB device without colocated workloads. + +Use the dedicated TorchAO FP8 example: + +~~~bash +python examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py \ + --mode t2va \ + --duration 5 \ + --output outputs/minimax_h3_torchao_fp8.mp4 +~~~ + +Or the dedicated bitsandbytes NF4 example: + +~~~bash +python examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py \ + --mode t2va \ + --duration 5 \ + --output outputs/minimax_h3_bnb_nf4.mp4 +~~~ + +The standard FL2VA, Ref2VA, and JSON request CLIs also accept +--quantization with either torchao-fp8 or bnb-nf4. The Python loader accepts the same names: + +~~~python +from examples.minimax_h3.common import load_minimax_h3_pipeline + +pipeline = load_minimax_h3_pipeline( + "/path/to/MiniMaxAI_MiniMax-H3", + partition="FL2VA", + quantization="torchao-fp8", +) +~~~ + +Online quantization currently requires ulysses_degree=1, tp_degree=1, and FSDP disabled. Quantizing before TP/FSDP +would invalidate those wrappers' BF16 parameter-sharding contract, so unsupported combinations fail before checkpoint +loading. + +For matched BF16/FP8/NF4 profiling, use the validation benchmark. It writes the synchronized MP4 plus a JSON report +containing load time, end-to-end generation time, stage timings, and denoising allocator peaks: + +~~~bash +python tools/validation/benchmark_minimax_h3_quantization.py \ + --backend torchao-fp8 \ + --duration 5 \ + --steps 50 \ + --output outputs/minimax_h3_torchao_fp8_50step.mp4 +~~~ + For multi-GPU resident profiles, `WorkerTensorChannel` transports text conditioning, visual condition rows, and the final video latent directly between worker groups. CUDA intermediates therefore do not stage through the parent process or CPU. The pipeline reports media, text, condition VAE, denoising, video/audio decode, allocator peak, and @@ -314,10 +376,10 @@ The standard four-GPU profile already uses Ulysses2 x TP2 and therefore leaves F `load_minimax_h3_pipeline` directly to construct another supported combination; the product of Ulysses and TP degrees must be 1, 2, or 4. -Ring attention, CFG parallelism, pipeline parallelism, sparse attention, quantization, and `torch.compile` are not -enabled for H3. Video-VAE parallelism is spatial tiling over the existing TP process group, not parameter tensor -parallelism. The dedicated service manifests expose the pipeline without adding framework-level configuration fields -or changing the shared request schema. +Ring attention, CFG parallelism, pipeline parallelism, sparse attention, and `torch.compile` are not enabled for H3. +Video-VAE parallelism is spatial tiling over the existing TP process group, not parameter tensor parallelism. The +dedicated service manifests expose the pipeline without adding framework-level configuration fields or changing the +shared request schema. ## Four-GPU Regression diff --git a/examples/minimax_h3/common.py b/examples/minimax_h3/common.py index ac7270fa..bb8ca071 100644 --- a/examples/minimax_h3/common.py +++ b/examples/minimax_h3/common.py @@ -19,6 +19,9 @@ ModelRuntimeConfig, OffloadConfig, ParallelConfig, + QuantConfig, + QuantKernelBackend, + QuantType, WeightOffloadType, ) from telefuser.core.module_manager import ModuleManager @@ -127,6 +130,34 @@ def _checkpoint_shards(component: Path) -> list[str]: return shards +def minimax_h3_quant_config(quantization: str | QuantType | None) -> QuantConfig: + """Resolve a public MiniMax H3 online-quantization name to runtime config.""" + if quantization is None: + return QuantConfig() + if isinstance(quantization, str): + normalized = quantization.strip().lower().replace("_", "-") + names = { + "torchao-fp8": QuantType.TORCHAO_FP8, + "bnb-nf4": QuantType.BNB_NF4, + } + try: + quant_type = names[normalized] + except KeyError as exc: + raise ValueError("quantization must be 'torchao-fp8', 'bnb-nf4', or None") from exc + elif isinstance(quantization, QuantType): + quant_type = quantization + else: + raise TypeError("quantization must be a string, QuantType, or None") + + backends = { + QuantType.TORCHAO_FP8: QuantKernelBackend.TORCHAO, + QuantType.BNB_NF4: QuantKernelBackend.BITSANDBYTES, + } + if quant_type not in backends: + raise ValueError(f"MiniMax H3 does not support online quantization type {quant_type.name}") + return QuantConfig(enabled=True, quant_type=quant_type, kernel_backend=backends[quant_type]) + + def load_minimax_h3_pipeline( model_root: str | Path, *, @@ -141,6 +172,7 @@ def load_minimax_h3_pipeline( feature_cache_config: FeatureCacheConfig | None = None, adaln_cache_path: str | Path | None = None, online_adaln_cache: bool = False, + quantization: str | QuantType | None = None, ) -> MiniMaxH3Pipeline: if adaln_cache_path is not None and online_adaln_cache: raise ValueError("Choose either adaln_cache_path or online_adaln_cache, not both.") @@ -162,6 +194,11 @@ def load_minimax_h3_pipeline( raise ValueError("enable_fsdp requires multi-GPU sequence parallelism without tensor parallelism") if (adaln_cache_path is not None or online_adaln_cache) and resolved_enable_fsdp: raise ValueError("AdaLN cache modes do not yet support FSDP deployment.") + quant_config = minimax_h3_quant_config(quantization) + if quant_config.enabled and world_size != 1: + raise ValueError("MiniMax H3 online quantization currently requires a single-GPU profile") + if quant_config.enabled and resolved_enable_fsdp: + raise ValueError("MiniMax H3 online quantization cannot be combined with FSDP") if isinstance(attn_impl, str): try: attn_impl = AttnImplType[attn_impl] @@ -171,6 +208,8 @@ def load_minimax_h3_pipeline( if not component_root.is_dir(): raise FileNotFoundError(f"MiniMax H3 partition not found: {component_root}") runtime_device = torch.device(device) + if quant_config.enabled and runtime_device.type != "cuda": + raise ValueError("MiniMax H3 online quantization requires a CUDA device") use_resident_modules = world_size > 1 or resolved_enable_fsdp resident_offload = OffloadConfig( offload_type=( @@ -178,6 +217,11 @@ def load_minimax_h3_pipeline( ), pin_cpu_memory=False, ) + dit_offload = ( + OffloadConfig(offload_type=WeightOffloadType.NO_CPU_OFFLOAD, pin_cpu_memory=False) + if quant_config.enabled + else resident_offload + ) text_parallel = ( ParallelConfig( device_ids=list(range(resolved_encoder_tp)), @@ -203,9 +247,10 @@ def load_minimax_h3_pipeline( device_type=runtime_device.type, device_id=runtime_device.index or 0, torch_dtype=torch.bfloat16, - offload_config=resident_offload, + offload_config=dit_offload, attention_config=AttentionConfig.dense_attention(attn_impl), feature_cache_config=feature_cache_config or FeatureCacheConfig(), + quant_config=quant_config, parallel_config=ParallelConfig( device_ids=list(range(world_size)), sp_ulysses_degree=ulysses_degree, @@ -344,6 +389,7 @@ def save_generation(result: MiniMaxH3Generation, output_path: str | Path) -> Non "load_minimax_h3_pipeline", "load_minimax_h3_request", "minimax_h3_adaln_cache_timesteps", + "minimax_h3_quant_config", "partition_for_minimax_h3_request", "run_minimax_h3_request", "save_generation", diff --git a/examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py b/examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py new file mode 100644 index 00000000..2164a862 --- /dev/null +++ b/examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +"""MiniMax H3 FL2VA example with bitsandbytes NF4 online quantization.""" + +from __future__ import annotations + +from copy import deepcopy +from functools import wraps + +if __package__: + from . import minimax_h3_fl2va_h100 as base +else: + try: + from examples.minimax_h3 import minimax_h3_fl2va_h100 as base + except ModuleNotFoundError: + import minimax_h3_fl2va_h100 as base + +PPL_CONFIG = { + **base.PPL_CONFIG, + "name": "minimax_h3_fl2va_bnb_nf4_h100", + "quantization": "bnb-nf4", +} +PIPELINE_MANIFEST = deepcopy(base.PIPELINE_MANIFEST) +PIPELINE_MANIFEST["pipeline_name"] = PPL_CONFIG["name"] + + +@wraps(base.run) +def run(*args: object, **kwargs: object) -> base.MiniMaxH3Generation: + return base.run(*args, **kwargs) + + +@wraps(base.run_with_file) +def run_with_file(*args: object, **kwargs: object) -> dict[str, str]: + return base.run_with_file(*args, **kwargs) + + +def get_pipeline( + parallelism: int = 1, + model_root: str = PPL_CONFIG["model_root"], + **kwargs: object, +) -> base.MiniMaxH3Pipeline: + """Load the single-GPU bitsandbytes NF4 FL2VA pipeline.""" + return base.get_pipeline( + parallelism, + model_root, + quantization=PPL_CONFIG["quantization"], + **kwargs, + ) + + +def main() -> None: + base._main(PPL_CONFIG["quantization"]) + + +if __name__ == "__main__": + main() diff --git a/examples/minimax_h3/minimax_h3_fl2va_h100.py b/examples/minimax_h3/minimax_h3_fl2va_h100.py index 7521f334..3ba1dc87 100644 --- a/examples/minimax_h3/minimax_h3_fl2va_h100.py +++ b/examples/minimax_h3/minimax_h3_fl2va_h100.py @@ -38,6 +38,7 @@ "feature_cache_model_type": "MiniMax-H3-Base", "feature_cache_n_derivatives": 1, "feature_cache_taylor_threshold": 2, + "quantization": None, } @@ -88,6 +89,7 @@ def get_pipeline( feature_cache_model_type: str = PPL_CONFIG["feature_cache_model_type"], feature_cache_n_derivatives: int = PPL_CONFIG["feature_cache_n_derivatives"], feature_cache_taylor_threshold: int = PPL_CONFIG["feature_cache_taylor_threshold"], + quantization: str | None = PPL_CONFIG["quantization"], ) -> MiniMaxH3Pipeline: """Load the FL2VA checkpoint partition for one, two, or four GPUs.""" tp_degree = 2 if parallelism == 4 else 1 @@ -108,6 +110,7 @@ def get_pipeline( n_derivatives=feature_cache_n_derivatives, taylor_threshold=feature_cache_taylor_threshold, ), + quantization=quantization, ) @@ -241,7 +244,7 @@ def run_with_file( return {"output_path": str(Path(output_path))} -def main() -> None: +def _main(default_quantization: str | None = PPL_CONFIG["quantization"]) -> None: parser = argparse.ArgumentParser(description="Generate MiniMax H3 T2VA/FL2VA audio-video on H100 GPUs") parser.add_argument("--model-root", default=PPL_CONFIG["model_root"]) parser.add_argument("--mode", choices=("t2va", "first-frame", "last-frame", "first-last")) @@ -265,6 +268,12 @@ def main() -> None: parser.add_argument("--flow-shift", type=float, default=PPL_CONFIG["flow_shift"]) parser.add_argument("--audio-flow-shift", type=float, default=PPL_CONFIG["audio_flow_shift"]) parser.add_argument("--device", default=PPL_CONFIG["device"]) + parser.add_argument( + "--quantization", + choices=("torchao-fp8", "bnb-nf4"), + default=default_quantization, + help="Online DiT Linear quantization backend (single GPU only).", + ) parser.add_argument("--gpu-num", "--ulysses-degree", dest="gpu_num", type=int, choices=(1, 2, 4), default=1) parser.add_argument( "--attn-impl", @@ -319,6 +328,7 @@ def main() -> None: feature_cache_model_type=args.feature_cache_model_type, feature_cache_n_derivatives=args.feature_cache_n_derivatives, feature_cache_taylor_threshold=args.feature_cache_taylor_threshold, + quantization=args.quantization, ) try: result = run_with_file( @@ -339,5 +349,9 @@ def main() -> None: pipeline.stop() +def main() -> None: + _main() + + if __name__ == "__main__": main() diff --git a/examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py b/examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py new file mode 100644 index 00000000..1adc3cee --- /dev/null +++ b/examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +"""MiniMax H3 FL2VA example with TorchAO FP8 online quantization.""" + +from __future__ import annotations + +from copy import deepcopy +from functools import wraps + +if __package__: + from . import minimax_h3_fl2va_h100 as base +else: + try: + from examples.minimax_h3 import minimax_h3_fl2va_h100 as base + except ModuleNotFoundError: + import minimax_h3_fl2va_h100 as base + +PPL_CONFIG = { + **base.PPL_CONFIG, + "name": "minimax_h3_fl2va_torchao_fp8_h100", + "quantization": "torchao-fp8", +} +PIPELINE_MANIFEST = deepcopy(base.PIPELINE_MANIFEST) +PIPELINE_MANIFEST["pipeline_name"] = PPL_CONFIG["name"] + + +@wraps(base.run) +def run(*args: object, **kwargs: object) -> base.MiniMaxH3Generation: + return base.run(*args, **kwargs) + + +@wraps(base.run_with_file) +def run_with_file(*args: object, **kwargs: object) -> dict[str, str]: + return base.run_with_file(*args, **kwargs) + + +def get_pipeline( + parallelism: int = 1, + model_root: str = PPL_CONFIG["model_root"], + **kwargs: object, +) -> base.MiniMaxH3Pipeline: + """Load the single-GPU TorchAO FP8 FL2VA pipeline.""" + return base.get_pipeline( + parallelism, + model_root, + quantization=PPL_CONFIG["quantization"], + **kwargs, + ) + + +def main() -> None: + base._main(PPL_CONFIG["quantization"]) + + +if __name__ == "__main__": + main() diff --git a/examples/minimax_h3/minimax_h3_ref2va_h100.py b/examples/minimax_h3/minimax_h3_ref2va_h100.py index 0c2db8b2..0cadde30 100644 --- a/examples/minimax_h3/minimax_h3_ref2va_h100.py +++ b/examples/minimax_h3/minimax_h3_ref2va_h100.py @@ -36,6 +36,7 @@ "device": "cuda:0", "enable_fsdp": None, "online_adaln_cache": True, + "quantization": None, } PIPELINE_MANIFEST = build_pipeline_manifest( @@ -91,6 +92,7 @@ def get_pipeline( num_inference_steps: int = PPL_CONFIG["num_inference_steps"], enable_fsdp: bool | None = PPL_CONFIG["enable_fsdp"], online_adaln_cache: bool = PPL_CONFIG["online_adaln_cache"], + quantization: str | None = PPL_CONFIG["quantization"], ) -> MiniMaxH3Pipeline: """Load the Ref2VA checkpoint partition for one, two, or four GPUs.""" tp_degree = 2 if parallelism == 4 else 1 @@ -104,6 +106,7 @@ def get_pipeline( text_encoder_tp_degree=parallelism, enable_fsdp=enable_fsdp, online_adaln_cache=online_adaln_cache, + quantization=quantization, ) @@ -260,6 +263,7 @@ def main() -> None: parser.add_argument("--flow-shift", type=float, default=PPL_CONFIG["flow_shift"]) parser.add_argument("--audio-flow-shift", type=float, default=PPL_CONFIG["audio_flow_shift"]) parser.add_argument("--device", default=PPL_CONFIG["device"]) + parser.add_argument("--quantization", choices=("torchao-fp8", "bnb-nf4")) parser.add_argument("--gpu-num", "--ulysses-degree", dest="gpu_num", type=int, choices=(1, 2, 4), default=1) fsdp_group = parser.add_mutually_exclusive_group() fsdp_group.add_argument("--enable-fsdp", dest="enable_fsdp", action="store_true") @@ -299,6 +303,7 @@ def main() -> None: num_inference_steps=request_steps, enable_fsdp=args.enable_fsdp, online_adaln_cache=online_adaln_cache, + quantization=args.quantization, ) try: if args.request is not None: diff --git a/telefuser/models/minimax_h3_dit.py b/telefuser/models/minimax_h3_dit.py index 52b064d2..581d806c 100644 --- a/telefuser/models/minimax_h3_dit.py +++ b/telefuser/models/minimax_h3_dit.py @@ -16,7 +16,7 @@ import torch.nn as nn from telefuser.core.base_model import BaseModel -from telefuser.core.config import AttentionConfig, AttnImplType +from telefuser.core.config import AttentionConfig, AttnImplType, QuantConfig, QuantType from telefuser.distributed.collectives import all_gather_cat, all_reduce_sum_ from telefuser.distributed.device_mesh import ( get_tp_group, @@ -31,6 +31,7 @@ from telefuser.ops import RMSNorm, apply_qk_norm_rope_neox, indexed_gate, indexed_scale_shift, silu_and_mul_reuse_input from telefuser.ops.attention import attention from telefuser.ops.rotary import apply_rotary_emb_neox +from telefuser.utils.logging import logger MINIMAX_H3_ADALN_MODALITY_NUM = 3 MINIMAX_H3_FP32_PARAM_NAMES = frozenset( @@ -1209,6 +1210,42 @@ def enable_usp(self, device_mesh: Any | None = None) -> None: for block in self.blocks: block.attn.set_ulysses_group(group, communicator) + def enable_quant(self, quant_type: QuantConfig | str | torch.dtype) -> None: + """Apply supported online quantization to transformer Linear layers.""" + if not isinstance(quant_type, QuantConfig): + super().enable_quant(quant_type) + return + if not quant_type.enabled: + return + + include_names = quant_type.quantize_modules or ("blocks.",) + if quant_type.quant_type == QuantType.TORCHAO_FP8: + from telefuser.ops.torchao_fp8_linear import replace_linear_layers_with_torchao_fp8 + + replaced = replace_linear_layers_with_torchao_fp8( + self, + include_names=include_names, + exclude_names=quant_type.skip_modules, + ) + self.torchao_fp8_replaced_linear = replaced + elif quant_type.quant_type == QuantType.BNB_NF4: + from telefuser.ops.bnb_nf4_linear import replace_linear_layers_with_bnb_nf4 + + replaced = replace_linear_layers_with_bnb_nf4( + self, + compute_dtype=torch.bfloat16, + include_names=include_names, + exclude_names=quant_type.skip_modules, + ) + self.bnb_nf4_replaced_linear = replaced + else: + raise ValueError(f"MiniMax H3 does not support online quantization type {quant_type.quant_type.name}") + + if replaced == 0: + raise RuntimeError("MiniMax H3 online quantization did not select any Linear layers") + self.quant_type = quant_type.quant_type + logger.info(f"MiniMax H3 {quant_type.quant_type.name} converted {replaced} transformer Linear layers") + def enable_tp(self, device_mesh: Any | None = None) -> None: self.device_mesh = device_mesh if device_mesh is not None else self.device_mesh world_size = get_tp_world_size(self.device_mesh) diff --git a/telefuser/pipelines/minimax_h3/denoising.py b/telefuser/pipelines/minimax_h3/denoising.py index 9a45daf6..a6096a71 100644 --- a/telefuser/pipelines/minimax_h3/denoising.py +++ b/telefuser/pipelines/minimax_h3/denoising.py @@ -125,6 +125,20 @@ def __init__(self, module_manager: ModuleManager, model_runtime_config: ModelRun self.model_names = ["transformer"] self._request_serial = 0 + def _ensure_online_quantized(self) -> None: + quant_config = self.model_runtime_config.quant_config + if not quant_config.enabled: + return + if self.transformer.quant_type == quant_config.quant_type: + return + if self.transformer.quant_type is not None: + raise RuntimeError( + f"MiniMax H3 DiT is already quantized as {self.transformer.quant_type}, " + f"cannot apply {quant_config.quant_type}" + ) + self.transformer.enable_quant(quant_config) + current_platform.empty_cache() + def parallel_models(self) -> None: parallel_config = self.model_runtime_config.parallel_config unsupported = { @@ -222,6 +236,7 @@ def denoise( num_inference_steps: int, _transport_video: bool = False, ) -> MiniMaxH3DenoiseResult: + self._ensure_online_quantized() if isinstance(text, dict): text = MiniMaxH3TextCondition(**text) conditions = [ From 1a368c1f763b567ab9133743a2e8482a4500feb7 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Thu, 6 Aug 2026 02:20:58 +0000 Subject: [PATCH 2/5] fix(minimax-h3): align quantized example tests and docs --- PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md | 92 ++++++++++++++++++ .../minimax_h3_quantization_benchmark.svg | 46 +++++++++ tests/unit/models/test_minimax_h3_dit.py | 45 ++++++++- .../pipelines/minimax_h3/test_examples.py | 70 +++++++++++++- .../pipelines/minimax_h3/test_parallelism.py | 26 ++++- .../pipelines/minimax_h3/test_pipeline.py | 13 +++ .../service/test_example_service_parity.py | 2 + tests/unit/test_example_registry.py | 2 + .../benchmark_minimax_h3_quantization.py | 95 +++++++++++++++++++ 9 files changed, 388 insertions(+), 3 deletions(-) create mode 100644 PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md create mode 100644 docs/assets/minimax_h3_quantization_benchmark.svg create mode 100644 tools/validation/benchmark_minimax_h3_quantization.py diff --git a/PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md b/PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md new file mode 100644 index 00000000..6030d855 --- /dev/null +++ b/PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md @@ -0,0 +1,92 @@ +# MiniMax H3 Online Quantization Support + +## Summary + +This change adds single-GPU online DiT quantization for MiniMax H3 with two backends: + +- TorchAO FP8 dynamic activation and weight quantization (`torchao-fp8`) +- bitsandbytes NF4 weight-only quantization with BF16 compute (`bnb-nf4`) + +The quantized DiT is loaded from the original BF16 checkpoint, moved to CUDA after text encoding, converted on the +first denoising request, and kept resident for the remainder of the pipeline lifetime. The implementation converts +258 Linear layers across the main and token-refiner transformer blocks while preserving the reference dtype of the +FP32 projections, text encoder, and VAEs. + +## Motivation + +MiniMax H3's BF16 DiT profile requires most of an 80 GB H100. Online quantization provides a practical single-GPU +deployment option while retaining the existing checkpoint format, pipeline API, service contracts, and audio-video +output format. + +## Implementation + +- Added `MiniMaxH3DiT.enable_quant()` dispatch for TorchAO FP8 and BNB NF4. +- Added public `quantization` loading support and validation for CUDA, single-GPU execution, and FSDP exclusion. +- Added first-use quantization and allocator cache release in the denoising stage. +- Added dedicated H100 examples: + - `examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py` + - `examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py` +- Added `--quantization` to the existing FL2VA, Ref2VA, and JSON request examples. +- Added a reproducible benchmark at `tools/validation/benchmark_minimax_h3_quantization.py`. +- Added model, pipeline, loader, CLI, registry, and service-contract parity tests. +- Documented the lifecycle, backend behavior, constraints, and commands in `examples/minimax_h3/README.md`. + +## Benchmark Method + +The matched benchmark uses one NVIDIA H100 80 GB, MiniMax H3 FL2VA, 768p 16:9 output, five seconds, 50 inference +steps, seed `0`, and the prompt: + +> Steam rises from the ramen while the family talks in the background. + +The memory value is the runtime allocator's peak allocated bytes converted to decimal GB. The throughput value is +end-to-end `generation_seconds / 50`, so lower values are better. BF16 was measured on an H100 of the same model; +the final FP8 and NF4 runs were measured serially on an idle H100 to avoid unrelated GPU contention. + +![MiniMax H3 quantization benchmark](docs/assets/minimax_h3_quantization_benchmark.svg) + +| Precision | Backend | Peak allocated memory | Peak reserved memory | Generation time | s/step | Change vs BF16 | +|---|---|---:|---:|---:|---:|---:| +| BF16 | Reference | 71.66 GB | 75.38 GB | 406.33 s | 8.13 s/step | Baseline | +| FP8 | TorchAO | 43.49 GB | 59.38 GB | 361.29 s | 7.23 s/step | 11.1% faster, 39.3% less allocated memory | +| NF4 | bitsandbytes | 22.62 GB | 55.97 GB | 378.05 s | 7.56 s/step | 7.0% faster, 68.4% less allocated memory | + +FP8's core denoising time is 283.67 s, 4.3% below BF16. NF4's core denoising time is 298.13 s, effectively neutral +and 0.6% above BF16; its main benefit is memory reduction and avoiding the BF16 DiT offload footprint. + +## Generated Video Comparison + +The video file cells are intentionally blank for attaching or embedding the final review media. + +| Precision | Backend | Video file | Visual/audio observation | +|---|---|---|---| +| BF16 | Reference | | Reference generation for comparison | +| FP8 | TorchAO | | PSNR 21.52, SSIM 0.729, audio cosine 0.626 versus BF16; composition and lighting remain close | +| NF4 | bitsandbytes | | PSNR 14.45, SSIM 0.472, audio cosine 0.283 versus BF16; coherent scene, but composition and details diverge | + +All three finalized artifacts are H.264 1344x768 at 24 fps with AAC 32 kHz stereo audio and 5.175 seconds of media. + +## Validation + +- Unit tests were not rerun after syncing latest main, as requested; the migrated test changes are included for CI review. +- Python source compilation, Ruff linting, formatting, and whitespace checks pass on the rebased files. +- `ruff check` and `ruff format --check` pass for all changed Python files. +- `git diff --check` passes. +- The benchmark numbers and generated-media comparison above come from the completed H100 validation of the implementation path. + +Unit tests and full GPU generation were intentionally not rerun after the fork synchronization; no claim is made here about post-sync test execution. + +## Constraints And Follow-Up + +- Online quantization currently requires one CUDA device, `tp_degree=1`, `ulysses_degree=1`, and FSDP disabled. +- TorchAO's first conversion has a transient memory peak near the BF16 footprint; an otherwise idle 80 GB H100 is + recommended. +- PSNR, SSIM, and audio cosine compare fixed-seed trajectories against BF16. They are regression indicators, not an + absolute perceptual quality score. + +## Contribution Checklist + +- [x] Code follows the repository's ruff and formatting rules. +- [x] Tests were added for new quantization and lifecycle behavior. +- [x] Documentation and runnable examples were updated. +- [x] Prior implementation validation includes real H100 generation checks; post-sync UT was intentionally not rerun. +- [ ] Review video attachments: intentionally left blank in the comparison table above. diff --git a/docs/assets/minimax_h3_quantization_benchmark.svg b/docs/assets/minimax_h3_quantization_benchmark.svg new file mode 100644 index 00000000..743c6737 --- /dev/null +++ b/docs/assets/minimax_h3_quantization_benchmark.svg @@ -0,0 +1,46 @@ + + MiniMax H3 quantization memory and generation cost + Bars compare peak allocated memory in gigabytes. A red line compares end-to-end generation seconds per step. + + MiniMax H3 Online Quantization + Matched H100 benchmark: 768p, 5 seconds, 50 steps, seed 0 + + + + + + + + + + 020406080 + + + 6.57.07.58.08.5 + + + + + + + + + + 71.7 GB43.5 GB22.6 GB + BF16TorchAO FP8BNB NF4 + + + + + + + + 8.13 s/step7.23 s/step7.56 s/step + + + + Peak allocated memory (GB) + Generation time (s/step; lower is better) + + Left axis: memory. Right axis: end-to-end generation time divided by 50 steps. + diff --git a/tests/unit/models/test_minimax_h3_dit.py b/tests/unit/models/test_minimax_h3_dit.py index 5baa8c5a..dbc04de4 100644 --- a/tests/unit/models/test_minimax_h3_dit.py +++ b/tests/unit/models/test_minimax_h3_dit.py @@ -4,7 +4,7 @@ import pytest import torch -from telefuser.core.config import AttentionConfig, AttnImplType +from telefuser.core.config import AttentionConfig, AttnImplType, QuantConfig, QuantType from telefuser.models.minimax_h3_dit import ( MINIMAX_H3_FP32_BUFFER_NAMES, MINIMAX_H3_FP32_PARAM_NAMES, @@ -644,3 +644,46 @@ def gather_rank_copies(tensor: torch.Tensor, *, dim: int, **_: object) -> torch. torch.testing.assert_close(actual_video, expected_video) torch.testing.assert_close(actual_audio, expected_audio) + + +@pytest.mark.parametrize( + ("quant_type", "helper_path", "count_attribute"), + [ + ( + QuantType.TORCHAO_FP8, + "telefuser.ops.torchao_fp8_linear.replace_linear_layers_with_torchao_fp8", + "torchao_fp8_replaced_linear", + ), + ( + QuantType.BNB_NF4, + "telefuser.ops.bnb_nf4_linear.replace_linear_layers_with_bnb_nf4", + "bnb_nf4_replaced_linear", + ), + ], +) +def test_online_quantization_selects_only_transformer_blocks( + monkeypatch: pytest.MonkeyPatch, + quant_type: QuantType, + helper_path: str, + count_attribute: str, +) -> None: + model = MiniMaxH3DiT(_small_config()) + calls = [] + + def fake_replace(module: torch.nn.Module, **kwargs: object) -> int: + calls.append((module, kwargs)) + return 15 + + monkeypatch.setattr(helper_path, fake_replace) + model.enable_quant(QuantConfig(enabled=True, quant_type=quant_type)) + + assert calls[0][0] is model + assert calls[0][1]["include_names"] == ("blocks.",) + assert getattr(model, count_attribute) == 15 + assert model.quant_type == quant_type + + +def test_online_quantization_rejects_unsupported_type() -> None: + model = MiniMaxH3DiT(_small_config()) + with pytest.raises(ValueError, match="does not support"): + model.enable_quant(QuantConfig(enabled=True, quant_type=QuantType.INT8)) diff --git a/tests/unit/pipelines/minimax_h3/test_examples.py b/tests/unit/pipelines/minimax_h3/test_examples.py index 92963930..94f84feb 100644 --- a/tests/unit/pipelines/minimax_h3/test_examples.py +++ b/tests/unit/pipelines/minimax_h3/test_examples.py @@ -3,14 +3,18 @@ import pytest +from examples.minimax_h3 import minimax_h3_fl2va_bnb_nf4_h100 as bnb_nf4_example from examples.minimax_h3 import minimax_h3_fl2va_h100 as fl2va_example +from examples.minimax_h3 import minimax_h3_fl2va_torchao_fp8_h100 as torchao_fp8_example from examples.minimax_h3 import minimax_h3_ref2va_h100 as ref2va_example from examples.minimax_h3.common import ( MINIMAX_H3_DEFAULT_FL2VA_IMAGE, MINIMAX_H3_DEFAULT_REF2VA_AUDIO, MINIMAX_H3_DEFAULT_REF2VA_VIDEO, + load_minimax_h3_pipeline, load_minimax_h3_request, minimax_h3_adaln_cache_timesteps, + minimax_h3_quant_config, partition_for_minimax_h3_request, ) from examples.minimax_h3.minimax_h3_cache_calibrate import _apply_cache_profile @@ -20,7 +24,7 @@ default_ref2va_conditions, parse_ref2va_ordered_materials, ) -from telefuser.core.config import AttnImplType, FeatureCacheConfig +from telefuser.core.config import AttnImplType, FeatureCacheConfig, QuantKernelBackend, QuantType from telefuser.pipelines.minimax_h3.task_profiles import MINIMAX_H3_FINITE_ASPECT_RATIOS from telefuser.service.core.pipeline_contract import PipelineContract @@ -138,6 +142,7 @@ def fake_loader(model_root: str, **kwargs: object) -> object: n_derivatives=1, taylor_threshold=2, ), + "quantization": None, }, ) ] @@ -158,6 +163,69 @@ def test_cache_calibration_applies_validated_h3_profile(tmp_path: Path) -> None: assert params == {"K": 2, "retention_ratio": 0.2, "thresh": 0.03} +@pytest.mark.parametrize( + ("name", "quant_type", "backend"), + [ + ("torchao-fp8", QuantType.TORCHAO_FP8, QuantKernelBackend.TORCHAO), + ("torchao_fp8", QuantType.TORCHAO_FP8, QuantKernelBackend.TORCHAO), + ("bnb-nf4", QuantType.BNB_NF4, QuantKernelBackend.BITSANDBYTES), + ], +) +def test_quantization_names_resolve_to_runtime_config( + name: str, + quant_type: QuantType, + backend: QuantKernelBackend, +) -> None: + config = minimax_h3_quant_config(name) + assert config.enabled is True + assert config.quant_type == quant_type + assert config.kernel_backend == backend + + +def test_quantization_rejects_unsupported_parallel_and_cpu_profiles(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="single-GPU"): + load_minimax_h3_pipeline( + tmp_path, + partition="FL2VA", + ulysses_degree=2, + quantization="torchao-fp8", + ) + + (tmp_path / "FL2VA").mkdir() + with pytest.raises(ValueError, match="CUDA"): + load_minimax_h3_pipeline( + tmp_path, + partition="FL2VA", + device="cpu", + quantization="bnb-nf4", + ) + + +@pytest.mark.parametrize( + ("example", "quantization"), + [ + (torchao_fp8_example, "torchao-fp8"), + (bnb_nf4_example, "bnb-nf4"), + ], +) +def test_dedicated_quantized_examples_forward_fixed_backend( + monkeypatch: pytest.MonkeyPatch, + example: object, + quantization: str, +) -> None: + calls = [] + sentinel = object() + + def fake_get_pipeline(*args: object, **kwargs: object) -> object: + calls.append((args, kwargs)) + return sentinel + + monkeypatch.setattr(example.base, "get_pipeline", fake_get_pipeline) + assert example.get_pipeline(1, "/models/h3", device="cuda:1") is sentinel + assert calls == [((1, "/models/h3"), {"device": "cuda:1", "quantization": quantization})] + assert example.PIPELINE_MANIFEST["pipeline_name"] == example.PPL_CONFIG["name"] + + def test_fl2va_run_maps_standard_service_tasks_to_model_conditions() -> None: calls = [] marker = object() diff --git a/tests/unit/pipelines/minimax_h3/test_parallelism.py b/tests/unit/pipelines/minimax_h3/test_parallelism.py index 11794772..379366a0 100644 --- a/tests/unit/pipelines/minimax_h3/test_parallelism.py +++ b/tests/unit/pipelines/minimax_h3/test_parallelism.py @@ -3,7 +3,14 @@ import pytest import torch -from telefuser.core.config import ModelRuntimeConfig, OffloadConfig, ParallelConfig, WeightOffloadType +from telefuser.core.config import ( + ModelRuntimeConfig, + OffloadConfig, + ParallelConfig, + QuantConfig, + QuantType, + WeightOffloadType, +) from telefuser.pipelines.minimax_h3.denoising import ( MiniMaxH3DenoisingStage, _build_local_embedding_layout, @@ -116,6 +123,23 @@ def test_parallel_models_rejects_tp_with_fsdp() -> None: stage.parallel_models() +def test_online_quantization_is_applied_once_after_stage_onload() -> None: + stage, transformer = _stage(ParallelConfig()) + stage.model_runtime_config.quant_config = QuantConfig(enabled=True, quant_type=QuantType.TORCHAO_FP8) + transformer.quant_type = None + + def enable_quant(config: QuantConfig) -> None: + transformer.quant_type = config.quant_type + + transformer.enable_quant.side_effect = enable_quant + with patch("telefuser.pipelines.minimax_h3.denoising.current_platform.empty_cache") as empty_cache: + stage._ensure_online_quantized() + stage._ensure_online_quantized() + + transformer.enable_quant.assert_called_once_with(stage.model_runtime_config.quant_config) + empty_cache.assert_called_once_with() + + def test_text_encoder_direct_handoff_keeps_token_tags_on_cpu() -> None: manager = MagicMock() manager.fetch_module.return_value = MagicMock() diff --git a/tests/unit/pipelines/minimax_h3/test_pipeline.py b/tests/unit/pipelines/minimax_h3/test_pipeline.py index 5b64ec4f..9d65abb3 100644 --- a/tests/unit/pipelines/minimax_h3/test_pipeline.py +++ b/tests/unit/pipelines/minimax_h3/test_pipeline.py @@ -479,6 +479,19 @@ def init(self, manager, config) -> None: assert config.video_vae_config.parallel_config.tp_degree == 4 assert config.audio_vae_config.offload_config.offload_type is WeightOffloadType.NO_CPU_OFFLOAD + common.load_minimax_h3_pipeline( + tmp_path, + partition="Ref2VA", + device="cuda:0", + quantization="torchao-fp8", + ) + quantized_config = captured["config"] + assert quantized_config.dit_config.quant_config.enabled is True + assert quantized_config.dit_config.offload_config.offload_type is WeightOffloadType.NO_CPU_OFFLOAD + assert quantized_config.text_encoder_config.offload_config.offload_type is WeightOffloadType.MODEL_CPU_OFFLOAD + assert quantized_config.video_vae_config.offload_config.offload_type is WeightOffloadType.MODEL_CPU_OFFLOAD + assert quantized_config.audio_vae_config.offload_config.offload_type is WeightOffloadType.MODEL_CPU_OFFLOAD + def test_example_writer_preserves_complete_generated_audio( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/service/test_example_service_parity.py b/tests/unit/service/test_example_service_parity.py index 257f0fa6..7106b334 100644 --- a/tests/unit/service/test_example_service_parity.py +++ b/tests/unit/service/test_example_service_parity.py @@ -29,6 +29,8 @@ SERVICE_EXAMPLES = { "wan21_i2v_service": (Path("examples/wan_video/wan21_14b_image_to_video_480p_service.py"), "i2v", True), "minimax_h3_fl2va": (Path("examples/minimax_h3/minimax_h3_fl2va_h100.py"), "t2v", True), + "minimax_h3_fl2va_torchao_fp8": (Path("examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py"), "t2v", True), + "minimax_h3_fl2va_bnb_nf4": (Path("examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py"), "t2v", True), "minimax_h3_ref2va": (Path("examples/minimax_h3/minimax_h3_ref2va_h100.py"), "s2v", True), "wan22_i2v_distill": (Path("examples/wan_video/wan22_14b_image_to_video_distill_h100.py"), "i2v", True), "lingbot_video_dense": (Path("examples/lingbot_video/lingbot_video_dense_1_3b.py"), "t2i", True), diff --git a/tests/unit/test_example_registry.py b/tests/unit/test_example_registry.py index a56fb120..1925e870 100644 --- a/tests/unit/test_example_registry.py +++ b/tests/unit/test_example_registry.py @@ -15,6 +15,8 @@ "lingbot_video/lingbot_video_dense_1_3b.py", "lingbot_video/lingbot_video_moe_30b.py", "minimax_h3/minimax_h3_fl2va_h100.py", + "minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py", + "minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py", "minimax_h3/minimax_h3_ref2va_h100.py", } diff --git a/tools/validation/benchmark_minimax_h3_quantization.py b/tools/validation/benchmark_minimax_h3_quantization.py new file mode 100644 index 00000000..6eb89b46 --- /dev/null +++ b/tools/validation/benchmark_minimax_h3_quantization.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Benchmark MiniMax H3 BF16 and online-quantized single-GPU profiles.""" + +from __future__ import annotations + +import argparse +import json +import time +from importlib import metadata +from pathlib import Path + +from examples.minimax_h3.common import load_minimax_h3_pipeline, save_generation +from telefuser.core.config import AttnImplType + + +def _package_version(name: str) -> str | None: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return None + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-root", default="/hhb-data/aigc/model_zoo/MiniMaxAI_MiniMax-H3") + parser.add_argument("--backend", choices=("bf16", "torchao-fp8", "bnb-nf4"), required=True) + parser.add_argument("--prompt", default="Steam rises from the ramen while the family talks in the background.") + parser.add_argument("--duration", type=float, default=5.0) + parser.add_argument("--steps", type=int, default=50) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--aspect-ratio", default="16:9") + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--metrics-json", type=Path) + args = parser.parse_args() + + quantization = None if args.backend == "bf16" else args.backend + load_started = time.perf_counter() + pipeline = load_minimax_h3_pipeline( + args.model_root, + partition="FL2VA", + device=args.device, + num_inference_steps=args.steps, + attn_impl=AttnImplType.FLASH_ATTN_4, + quantization=quantization, + ) + load_seconds = time.perf_counter() - load_started + try: + generation_started = time.perf_counter() + result = pipeline( + task="t2va", + prompt=args.prompt, + conditions=[], + target={ + "short_edge": 768, + "aspect_ratio": args.aspect_ratio, + "duration_seconds": args.duration, + }, + seed=args.seed, + ) + generation_seconds = time.perf_counter() - generation_started + save_started = time.perf_counter() + save_generation(result, args.output) + save_seconds = time.perf_counter() - save_started + finally: + pipeline.stop() + + report = { + "backend": args.backend, + "model_root": str(Path(args.model_root)), + "output": str(args.output), + "prompt": args.prompt, + "duration_seconds": args.duration, + "num_inference_steps": args.steps, + "seed": args.seed, + "aspect_ratio": args.aspect_ratio, + "load_seconds": load_seconds, + "generation_seconds": generation_seconds, + "save_seconds": save_seconds, + "runtime_metrics": result.runtime_metrics, + "versions": { + "torch": _package_version("torch"), + "torchao": _package_version("torchao"), + "bitsandbytes": _package_version("bitsandbytes"), + "telefuser": _package_version("telefuser"), + }, + } + metrics_path = args.metrics_json or args.output.with_suffix(".metrics.json") + metrics_path.parent.mkdir(parents=True, exist_ok=True) + metrics_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() From 87cff9efc538b10f66429e31796d2e87c6536ad8 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Thu, 6 Aug 2026 02:23:14 +0000 Subject: [PATCH 3/5] chore: remove PR-only benchmark artifacts --- PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md | 92 ------------------- .../minimax_h3_quantization_benchmark.svg | 46 ---------- 2 files changed, 138 deletions(-) delete mode 100644 PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md delete mode 100644 docs/assets/minimax_h3_quantization_benchmark.svg diff --git a/PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md b/PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md deleted file mode 100644 index 6030d855..00000000 --- a/PR_DESCRIPTION_MINIMAX_H3_QUANTIZATION.md +++ /dev/null @@ -1,92 +0,0 @@ -# MiniMax H3 Online Quantization Support - -## Summary - -This change adds single-GPU online DiT quantization for MiniMax H3 with two backends: - -- TorchAO FP8 dynamic activation and weight quantization (`torchao-fp8`) -- bitsandbytes NF4 weight-only quantization with BF16 compute (`bnb-nf4`) - -The quantized DiT is loaded from the original BF16 checkpoint, moved to CUDA after text encoding, converted on the -first denoising request, and kept resident for the remainder of the pipeline lifetime. The implementation converts -258 Linear layers across the main and token-refiner transformer blocks while preserving the reference dtype of the -FP32 projections, text encoder, and VAEs. - -## Motivation - -MiniMax H3's BF16 DiT profile requires most of an 80 GB H100. Online quantization provides a practical single-GPU -deployment option while retaining the existing checkpoint format, pipeline API, service contracts, and audio-video -output format. - -## Implementation - -- Added `MiniMaxH3DiT.enable_quant()` dispatch for TorchAO FP8 and BNB NF4. -- Added public `quantization` loading support and validation for CUDA, single-GPU execution, and FSDP exclusion. -- Added first-use quantization and allocator cache release in the denoising stage. -- Added dedicated H100 examples: - - `examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py` - - `examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py` -- Added `--quantization` to the existing FL2VA, Ref2VA, and JSON request examples. -- Added a reproducible benchmark at `tools/validation/benchmark_minimax_h3_quantization.py`. -- Added model, pipeline, loader, CLI, registry, and service-contract parity tests. -- Documented the lifecycle, backend behavior, constraints, and commands in `examples/minimax_h3/README.md`. - -## Benchmark Method - -The matched benchmark uses one NVIDIA H100 80 GB, MiniMax H3 FL2VA, 768p 16:9 output, five seconds, 50 inference -steps, seed `0`, and the prompt: - -> Steam rises from the ramen while the family talks in the background. - -The memory value is the runtime allocator's peak allocated bytes converted to decimal GB. The throughput value is -end-to-end `generation_seconds / 50`, so lower values are better. BF16 was measured on an H100 of the same model; -the final FP8 and NF4 runs were measured serially on an idle H100 to avoid unrelated GPU contention. - -![MiniMax H3 quantization benchmark](docs/assets/minimax_h3_quantization_benchmark.svg) - -| Precision | Backend | Peak allocated memory | Peak reserved memory | Generation time | s/step | Change vs BF16 | -|---|---|---:|---:|---:|---:|---:| -| BF16 | Reference | 71.66 GB | 75.38 GB | 406.33 s | 8.13 s/step | Baseline | -| FP8 | TorchAO | 43.49 GB | 59.38 GB | 361.29 s | 7.23 s/step | 11.1% faster, 39.3% less allocated memory | -| NF4 | bitsandbytes | 22.62 GB | 55.97 GB | 378.05 s | 7.56 s/step | 7.0% faster, 68.4% less allocated memory | - -FP8's core denoising time is 283.67 s, 4.3% below BF16. NF4's core denoising time is 298.13 s, effectively neutral -and 0.6% above BF16; its main benefit is memory reduction and avoiding the BF16 DiT offload footprint. - -## Generated Video Comparison - -The video file cells are intentionally blank for attaching or embedding the final review media. - -| Precision | Backend | Video file | Visual/audio observation | -|---|---|---|---| -| BF16 | Reference | | Reference generation for comparison | -| FP8 | TorchAO | | PSNR 21.52, SSIM 0.729, audio cosine 0.626 versus BF16; composition and lighting remain close | -| NF4 | bitsandbytes | | PSNR 14.45, SSIM 0.472, audio cosine 0.283 versus BF16; coherent scene, but composition and details diverge | - -All three finalized artifacts are H.264 1344x768 at 24 fps with AAC 32 kHz stereo audio and 5.175 seconds of media. - -## Validation - -- Unit tests were not rerun after syncing latest main, as requested; the migrated test changes are included for CI review. -- Python source compilation, Ruff linting, formatting, and whitespace checks pass on the rebased files. -- `ruff check` and `ruff format --check` pass for all changed Python files. -- `git diff --check` passes. -- The benchmark numbers and generated-media comparison above come from the completed H100 validation of the implementation path. - -Unit tests and full GPU generation were intentionally not rerun after the fork synchronization; no claim is made here about post-sync test execution. - -## Constraints And Follow-Up - -- Online quantization currently requires one CUDA device, `tp_degree=1`, `ulysses_degree=1`, and FSDP disabled. -- TorchAO's first conversion has a transient memory peak near the BF16 footprint; an otherwise idle 80 GB H100 is - recommended. -- PSNR, SSIM, and audio cosine compare fixed-seed trajectories against BF16. They are regression indicators, not an - absolute perceptual quality score. - -## Contribution Checklist - -- [x] Code follows the repository's ruff and formatting rules. -- [x] Tests were added for new quantization and lifecycle behavior. -- [x] Documentation and runnable examples were updated. -- [x] Prior implementation validation includes real H100 generation checks; post-sync UT was intentionally not rerun. -- [ ] Review video attachments: intentionally left blank in the comparison table above. diff --git a/docs/assets/minimax_h3_quantization_benchmark.svg b/docs/assets/minimax_h3_quantization_benchmark.svg deleted file mode 100644 index 743c6737..00000000 --- a/docs/assets/minimax_h3_quantization_benchmark.svg +++ /dev/null @@ -1,46 +0,0 @@ - - MiniMax H3 quantization memory and generation cost - Bars compare peak allocated memory in gigabytes. A red line compares end-to-end generation seconds per step. - - MiniMax H3 Online Quantization - Matched H100 benchmark: 768p, 5 seconds, 50 steps, seed 0 - - - - - - - - - - 020406080 - - - 6.57.07.58.08.5 - - - - - - - - - - 71.7 GB43.5 GB22.6 GB - BF16TorchAO FP8BNB NF4 - - - - - - - - 8.13 s/step7.23 s/step7.56 s/step - - - - Peak allocated memory (GB) - Generation time (s/step; lower is better) - - Left axis: memory. Right axis: end-to-end generation time divided by 50 steps. - From 76f91dc0c435e51a9d07c3755bd814b7d47f4ffe Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Fri, 7 Aug 2026 06:54:35 +0000 Subject: [PATCH 4/5] Add MiniMax H3 tf-kernel FP8 support --- docs/en/quantization.md | 20 +++++++ examples/minimax_h3/README.md | 26 ++++++--- examples/minimax_h3/common.py | 4 +- examples/minimax_h3/minimax_h3_fl2va_h100.py | 2 +- .../minimax_h3_fl2va_tf_kernel_fp8_h100.py | 55 +++++++++++++++++++ examples/minimax_h3/minimax_h3_ref2va_h100.py | 2 +- telefuser/models/minimax_h3_dit.py | 25 ++++++++- telefuser/ops/fp8_gemm.py | 23 +++++++- tests/unit/models/test_minimax_h3_dit.py | 33 ++++++++++- tests/unit/ops/test_fp8_gemm.py | 3 +- .../pipelines/minimax_h3/test_examples.py | 3 + .../service/test_example_service_parity.py | 5 ++ tests/unit/test_example_registry.py | 1 + .../benchmark_minimax_h3_quantization.py | 2 +- 14 files changed, 188 insertions(+), 16 deletions(-) create mode 100644 examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py diff --git a/docs/en/quantization.md b/docs/en/quantization.md index b3e84c6d..4f292be3 100644 --- a/docs/en/quantization.md +++ b/docs/en/quantization.md @@ -79,6 +79,26 @@ TorchAO and PyTorch must be version-compatible. Check the [TorchAO release compatibility table](https://github.com/pytorch/ao/releases) instead of installing the newest TorchAO release blindly; an import warning or failure means that configuration has not been validated. +### tf-kernel FP8: online W8A8 + +MiniMax H3 can use TeleFuser's tf-kernel FP8 GEMM wrapper for online W8A8 inference directly from the original BF16 +checkpoint. Activations are quantized per token at each forward, weights are quantized per output channel and cached +on first use, and `tf_kernel.fp8_scaled_mm` produces BF16 output. This path requires a tf-kernel wheel built for the +runtime's PyTorch/CUDA ABI and GPU architecture; on H100, build the SM90 wheel from `tf-kernel/` with the Makefile. + +```python +quant_config = QuantConfig( + enabled=True, + quant_type=QuantType.FP8, + kernel_backend=QuantKernelBackend.TF_KERNEL, +) +``` + +For MiniMax H3, use `quantization="tf-kernel-fp8"` or +`examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py`. This backend is single-GPU only and keeps the FP8 +weights resident after first-use conversion. It is distinct from the scaled-FP8 checkpoint path below: the latter +expects weights and scales already serialized in the checkpoint. + ### bitsandbytes NF4: W4A16 NF4 uses a non-uniform 4-bit codebook designed for approximately normal weight distributions. TeleFuser replaces selected Linear layers with `bitsandbytes.nn.Linear4bit`, uses BF16 compute, and enables compressed quantization statistics. diff --git a/examples/minimax_h3/README.md b/examples/minimax_h3/README.md index 4434474f..13971481 100644 --- a/examples/minimax_h3/README.md +++ b/examples/minimax_h3/README.md @@ -293,19 +293,20 @@ FlashAttention 4 is unavailable. ## Online DiT Quantization -MiniMax H3 supports two single-GPU online quantization backends for the DiT transformer Linear layers: +MiniMax H3 supports three single-GPU online quantization backends for the DiT transformer Linear layers: | CLI value | Backend | Weight/activation path | |---|---|---| | torchao-fp8 | TorchAO | FP8 dynamic activation and FP8 weight when supported, otherwise TorchAO's FP8 weight-only path | +| tf-kernel-fp8 | TeleFuser tf-kernel | Per-token activation and per-output-channel weight FP8 (W8A8), BF16 output | | bnb-nf4 | bitsandbytes | NF4 weight-only with BF16 compute | -Both paths convert the 258 Linear layers in the main and token-refiner transformer blocks. The FP32 video/audio +All three paths convert the 258 Linear layers in the main and token-refiner transformer blocks. The FP32 video/audio patch projections, timestep embedding, output projections, text encoder, and VAEs retain their reference dtypes. The BF16 DiT is loaded from the original shards, moved to CUDA after text encoding, quantized on first denoising use, and then kept resident for the pipeline lifetime. This ordering avoids a simultaneous BF16 text encoder and DiT on one GPU and avoids unsupported CPU transfers of quantized tensor subclasses. -TorchAO conversion has a transient memory peak near the BF16 footprint; use the full 80 GB device without colocated workloads. +TorchAO and tf-kernel conversion have a transient memory peak near the BF16 footprint; use the full 80 GB device without colocated workloads. Use the dedicated TorchAO FP8 example: @@ -325,8 +326,17 @@ python examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py \ --output outputs/minimax_h3_bnb_nf4.mp4 ~~~ +Or the dedicated tf-kernel FP8 example: + +~~~bash +python examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py \ + --mode t2va \ + --duration 5 \ + --output outputs/minimax_h3_tf_kernel_fp8.mp4 +~~~ + The standard FL2VA, Ref2VA, and JSON request CLIs also accept ---quantization with either torchao-fp8 or bnb-nf4. The Python loader accepts the same names: +--quantization with torchao-fp8, tf-kernel-fp8, or bnb-nf4. The Python loader accepts the same names: ~~~python from examples.minimax_h3.common import load_minimax_h3_pipeline @@ -334,7 +344,7 @@ from examples.minimax_h3.common import load_minimax_h3_pipeline pipeline = load_minimax_h3_pipeline( "/path/to/MiniMaxAI_MiniMax-H3", partition="FL2VA", - quantization="torchao-fp8", + quantization="tf-kernel-fp8", ) ~~~ @@ -342,15 +352,15 @@ Online quantization currently requires ulysses_degree=1, tp_degree=1, and FSDP d would invalidate those wrappers' BF16 parameter-sharding contract, so unsupported combinations fail before checkpoint loading. -For matched BF16/FP8/NF4 profiling, use the validation benchmark. It writes the synchronized MP4 plus a JSON report +For matched BF16/TorchAO-FP8/tf-kernel-FP8/NF4 profiling, use the validation benchmark. It writes the synchronized MP4 plus a JSON report containing load time, end-to-end generation time, stage timings, and denoising allocator peaks: ~~~bash python tools/validation/benchmark_minimax_h3_quantization.py \ - --backend torchao-fp8 \ + --backend tf-kernel-fp8 \ --duration 5 \ --steps 50 \ - --output outputs/minimax_h3_torchao_fp8_50step.mp4 + --output outputs/minimax_h3_tf_kernel_fp8_50step.mp4 ~~~ For multi-GPU resident profiles, `WorkerTensorChannel` transports text conditioning, visual condition rows, and the diff --git a/examples/minimax_h3/common.py b/examples/minimax_h3/common.py index bb8ca071..60c043e3 100644 --- a/examples/minimax_h3/common.py +++ b/examples/minimax_h3/common.py @@ -139,11 +139,12 @@ def minimax_h3_quant_config(quantization: str | QuantType | None) -> QuantConfig names = { "torchao-fp8": QuantType.TORCHAO_FP8, "bnb-nf4": QuantType.BNB_NF4, + "tf-kernel-fp8": QuantType.FP8, } try: quant_type = names[normalized] except KeyError as exc: - raise ValueError("quantization must be 'torchao-fp8', 'bnb-nf4', or None") from exc + raise ValueError("quantization must be 'torchao-fp8', 'tf-kernel-fp8', 'bnb-nf4', or None") from exc elif isinstance(quantization, QuantType): quant_type = quantization else: @@ -152,6 +153,7 @@ def minimax_h3_quant_config(quantization: str | QuantType | None) -> QuantConfig backends = { QuantType.TORCHAO_FP8: QuantKernelBackend.TORCHAO, QuantType.BNB_NF4: QuantKernelBackend.BITSANDBYTES, + QuantType.FP8: QuantKernelBackend.TF_KERNEL, } if quant_type not in backends: raise ValueError(f"MiniMax H3 does not support online quantization type {quant_type.name}") diff --git a/examples/minimax_h3/minimax_h3_fl2va_h100.py b/examples/minimax_h3/minimax_h3_fl2va_h100.py index 3ba1dc87..57af404b 100644 --- a/examples/minimax_h3/minimax_h3_fl2va_h100.py +++ b/examples/minimax_h3/minimax_h3_fl2va_h100.py @@ -270,7 +270,7 @@ def _main(default_quantization: str | None = PPL_CONFIG["quantization"]) -> None parser.add_argument("--device", default=PPL_CONFIG["device"]) parser.add_argument( "--quantization", - choices=("torchao-fp8", "bnb-nf4"), + choices=("torchao-fp8", "tf-kernel-fp8", "bnb-nf4"), default=default_quantization, help="Online DiT Linear quantization backend (single GPU only).", ) diff --git a/examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py b/examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py new file mode 100644 index 00000000..df4d7d37 --- /dev/null +++ b/examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +"""MiniMax H3 FL2VA example with tf-kernel FP8 online quantization.""" + +from __future__ import annotations + +from copy import deepcopy +from functools import wraps + +if __package__: + from . import minimax_h3_fl2va_h100 as base +else: + try: + from examples.minimax_h3 import minimax_h3_fl2va_h100 as base + except ModuleNotFoundError: + import minimax_h3_fl2va_h100 as base + +PPL_CONFIG = { + **base.PPL_CONFIG, + "name": "minimax_h3_fl2va_tf_kernel_fp8_h100", + "quantization": "tf-kernel-fp8", +} +PIPELINE_MANIFEST = deepcopy(base.PIPELINE_MANIFEST) +PIPELINE_MANIFEST["pipeline_name"] = PPL_CONFIG["name"] + + +@wraps(base.run) +def run(*args: object, **kwargs: object) -> base.MiniMaxH3Generation: + return base.run(*args, **kwargs) + + +@wraps(base.run_with_file) +def run_with_file(*args: object, **kwargs: object) -> dict[str, str]: + return base.run_with_file(*args, **kwargs) + + +def get_pipeline( + parallelism: int = 1, + model_root: str = PPL_CONFIG["model_root"], + **kwargs: object, +) -> base.MiniMaxH3Pipeline: + """Load the single-GPU tf-kernel FP8 FL2VA pipeline.""" + return base.get_pipeline( + parallelism, + model_root, + quantization=PPL_CONFIG["quantization"], + **kwargs, + ) + + +def main() -> None: + base._main(PPL_CONFIG["quantization"]) + + +if __name__ == "__main__": + main() diff --git a/examples/minimax_h3/minimax_h3_ref2va_h100.py b/examples/minimax_h3/minimax_h3_ref2va_h100.py index 0cadde30..a7de40fe 100644 --- a/examples/minimax_h3/minimax_h3_ref2va_h100.py +++ b/examples/minimax_h3/minimax_h3_ref2va_h100.py @@ -263,7 +263,7 @@ def main() -> None: parser.add_argument("--flow-shift", type=float, default=PPL_CONFIG["flow_shift"]) parser.add_argument("--audio-flow-shift", type=float, default=PPL_CONFIG["audio_flow_shift"]) parser.add_argument("--device", default=PPL_CONFIG["device"]) - parser.add_argument("--quantization", choices=("torchao-fp8", "bnb-nf4")) + parser.add_argument("--quantization", choices=("torchao-fp8", "tf-kernel-fp8", "bnb-nf4")) parser.add_argument("--gpu-num", "--ulysses-degree", dest="gpu_num", type=int, choices=(1, 2, 4), default=1) fsdp_group = parser.add_mutually_exclusive_group() fsdp_group.add_argument("--enable-fsdp", dest="enable_fsdp", action="store_true") diff --git a/telefuser/models/minimax_h3_dit.py b/telefuser/models/minimax_h3_dit.py index 581d806c..98522b96 100644 --- a/telefuser/models/minimax_h3_dit.py +++ b/telefuser/models/minimax_h3_dit.py @@ -16,7 +16,7 @@ import torch.nn as nn from telefuser.core.base_model import BaseModel -from telefuser.core.config import AttentionConfig, AttnImplType, QuantConfig, QuantType +from telefuser.core.config import AttentionConfig, AttnImplType, QuantConfig, QuantKernelBackend, QuantType from telefuser.distributed.collectives import all_gather_cat, all_reduce_sum_ from telefuser.distributed.device_mesh import ( get_tp_group, @@ -1238,6 +1238,29 @@ def enable_quant(self, quant_type: QuantConfig | str | torch.dtype) -> None: exclude_names=quant_type.skip_modules, ) self.bnb_nf4_replaced_linear = replaced + elif quant_type.quant_type == QuantType.FP8: + if quant_type.kernel_backend not in (QuantKernelBackend.AUTO, QuantKernelBackend.TF_KERNEL): + raise ValueError( + "MiniMax H3 FP8 online quantization requires the tf-kernel backend; " + f"got {quant_type.kernel_backend.name}" + ) + from telefuser.ops.fp8_gemm import FP8GemmOptions, count_linear_layers, enable_fp8_gemm + + def module_filter(name: str, _module: nn.Module) -> bool: + return any(token in name for token in include_names) and not any( + token and token in name for token in quant_type.skip_modules + ) + + replaced = count_linear_layers(self, module_filter=module_filter) + enable_fp8_gemm( + self, + options=FP8GemmOptions( + fp16_weight_storage="keep" if quant_type.keep_fp16_weight else "discard", + materialize_fp8_on_wrap=True, + ), + module_filter=module_filter, + ) + self.tf_kernel_fp8_replaced_linear = replaced else: raise ValueError(f"MiniMax H3 does not support online quantization type {quant_type.quant_type.name}") diff --git a/telefuser/ops/fp8_gemm.py b/telefuser/ops/fp8_gemm.py index 92293843..0cd04e32 100644 --- a/telefuser/ops/fp8_gemm.py +++ b/telefuser/ops/fp8_gemm.py @@ -274,7 +274,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: bias = bias.to(dtype=out_dtype) x_shape = x_fp.shape - x_2d = x_fp.reshape(-1, x_shape[-1]) + x_2d = x_fp.reshape(-1, x_shape[-1]).contiguous() qinput = torch.empty_like(x_2d, dtype=torch.float8_e4m3fn) input_scale = torch.empty((x_2d.shape[0], 1), dtype=torch.float32, device=x_fp.device) self._tf_kernel.tf_per_token_quant_fp8(x_2d, qinput, input_scale) @@ -340,3 +340,24 @@ def _recurse(prefix: str, parent: nn.Module) -> None: _recurse("", model) return model + + +def count_linear_layers( + model: nn.Module, + *, + module_filter: Optional[Callable[[str, nn.Module], bool]] = None, +) -> int: + """Count the ``nn.Linear`` modules that ``enable_fp8_gemm`` would wrap.""" + + def _count(prefix: str, parent: nn.Module) -> int: + count = 0 + for child_name, child in parent.named_children(): + full_name = f"{prefix}.{child_name}" if prefix else child_name + if isinstance(child, nn.Linear): + if module_filter is None or module_filter(full_name, child): + count += 1 + else: + count += _count(full_name, child) + return count + + return _count("", model) diff --git a/tests/unit/models/test_minimax_h3_dit.py b/tests/unit/models/test_minimax_h3_dit.py index dbc04de4..35147de9 100644 --- a/tests/unit/models/test_minimax_h3_dit.py +++ b/tests/unit/models/test_minimax_h3_dit.py @@ -4,7 +4,7 @@ import pytest import torch -from telefuser.core.config import AttentionConfig, AttnImplType, QuantConfig, QuantType +from telefuser.core.config import AttentionConfig, AttnImplType, QuantConfig, QuantKernelBackend, QuantType from telefuser.models.minimax_h3_dit import ( MINIMAX_H3_FP32_BUFFER_NAMES, MINIMAX_H3_FP32_PARAM_NAMES, @@ -683,6 +683,37 @@ def fake_replace(module: torch.nn.Module, **kwargs: object) -> int: assert model.quant_type == quant_type +def test_tf_kernel_fp8_quantization_uses_filtered_linear_wrapper(monkeypatch: pytest.MonkeyPatch) -> None: + model = MiniMaxH3DiT(_small_config()) + calls: list[tuple[str, object]] = [] + + def fake_count(module: torch.nn.Module, **kwargs: object) -> int: + calls.append(("count", kwargs["module_filter"])) + return 15 + + def fake_enable(module: torch.nn.Module, **kwargs: object) -> torch.nn.Module: + calls.append(("enable", kwargs)) + return module + + monkeypatch.setattr("telefuser.ops.fp8_gemm.count_linear_layers", fake_count) + monkeypatch.setattr("telefuser.ops.fp8_gemm.enable_fp8_gemm", fake_enable) + + model.enable_quant( + QuantConfig( + enabled=True, + quant_type=QuantType.FP8, + kernel_backend=QuantKernelBackend.TF_KERNEL, + ) + ) + + assert calls[0][0] == "count" + assert calls[1][0] == "enable" + options = calls[1][1]["options"] + assert options.fp16_weight_storage == "discard" + assert getattr(model, "tf_kernel_fp8_replaced_linear") == 15 + assert model.quant_type == QuantType.FP8 + + def test_online_quantization_rejects_unsupported_type() -> None: model = MiniMaxH3DiT(_small_config()) with pytest.raises(ValueError, match="does not support"): diff --git a/tests/unit/ops/test_fp8_gemm.py b/tests/unit/ops/test_fp8_gemm.py index 4bf95a19..256fb3f8 100644 --- a/tests/unit/ops/test_fp8_gemm.py +++ b/tests/unit/ops/test_fp8_gemm.py @@ -37,7 +37,8 @@ def test_fp8_linear_keeps_cpu_fallback(monkeypatch: pytest.MonkeyPatch) -> None: def test_fp8_linear_tf_kernel_forward() -> None: torch.manual_seed(0) linear = nn.Linear(64, 128, device="cuda", dtype=torch.bfloat16) - inputs = torch.randn(2, 3, 64, device="cuda", dtype=torch.bfloat16) + inputs = torch.randn(2, 64, 3, device="cuda", dtype=torch.bfloat16).transpose(1, 2) + assert not inputs.is_contiguous() expected = linear(inputs) wrapped = fp8_gemm.FP8Linear( linear, diff --git a/tests/unit/pipelines/minimax_h3/test_examples.py b/tests/unit/pipelines/minimax_h3/test_examples.py index 94f84feb..38abe169 100644 --- a/tests/unit/pipelines/minimax_h3/test_examples.py +++ b/tests/unit/pipelines/minimax_h3/test_examples.py @@ -5,6 +5,7 @@ from examples.minimax_h3 import minimax_h3_fl2va_bnb_nf4_h100 as bnb_nf4_example from examples.minimax_h3 import minimax_h3_fl2va_h100 as fl2va_example +from examples.minimax_h3 import minimax_h3_fl2va_tf_kernel_fp8_h100 as tf_kernel_fp8_example from examples.minimax_h3 import minimax_h3_fl2va_torchao_fp8_h100 as torchao_fp8_example from examples.minimax_h3 import minimax_h3_ref2va_h100 as ref2va_example from examples.minimax_h3.common import ( @@ -168,6 +169,7 @@ def test_cache_calibration_applies_validated_h3_profile(tmp_path: Path) -> None: [ ("torchao-fp8", QuantType.TORCHAO_FP8, QuantKernelBackend.TORCHAO), ("torchao_fp8", QuantType.TORCHAO_FP8, QuantKernelBackend.TORCHAO), + ("tf-kernel-fp8", QuantType.FP8, QuantKernelBackend.TF_KERNEL), ("bnb-nf4", QuantType.BNB_NF4, QuantKernelBackend.BITSANDBYTES), ], ) @@ -205,6 +207,7 @@ def test_quantization_rejects_unsupported_parallel_and_cpu_profiles(tmp_path: Pa ("example", "quantization"), [ (torchao_fp8_example, "torchao-fp8"), + (tf_kernel_fp8_example, "tf-kernel-fp8"), (bnb_nf4_example, "bnb-nf4"), ], ) diff --git a/tests/unit/service/test_example_service_parity.py b/tests/unit/service/test_example_service_parity.py index 7106b334..a8b36ffc 100644 --- a/tests/unit/service/test_example_service_parity.py +++ b/tests/unit/service/test_example_service_parity.py @@ -30,6 +30,11 @@ "wan21_i2v_service": (Path("examples/wan_video/wan21_14b_image_to_video_480p_service.py"), "i2v", True), "minimax_h3_fl2va": (Path("examples/minimax_h3/minimax_h3_fl2va_h100.py"), "t2v", True), "minimax_h3_fl2va_torchao_fp8": (Path("examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py"), "t2v", True), + "minimax_h3_fl2va_tf_kernel_fp8": ( + Path("examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py"), + "t2v", + True, + ), "minimax_h3_fl2va_bnb_nf4": (Path("examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py"), "t2v", True), "minimax_h3_ref2va": (Path("examples/minimax_h3/minimax_h3_ref2va_h100.py"), "s2v", True), "wan22_i2v_distill": (Path("examples/wan_video/wan22_14b_image_to_video_distill_h100.py"), "i2v", True), diff --git a/tests/unit/test_example_registry.py b/tests/unit/test_example_registry.py index 1925e870..09a00d6a 100644 --- a/tests/unit/test_example_registry.py +++ b/tests/unit/test_example_registry.py @@ -16,6 +16,7 @@ "lingbot_video/lingbot_video_moe_30b.py", "minimax_h3/minimax_h3_fl2va_h100.py", "minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py", + "minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py", "minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py", "minimax_h3/minimax_h3_ref2va_h100.py", } diff --git a/tools/validation/benchmark_minimax_h3_quantization.py b/tools/validation/benchmark_minimax_h3_quantization.py index 6eb89b46..9ec5ea18 100644 --- a/tools/validation/benchmark_minimax_h3_quantization.py +++ b/tools/validation/benchmark_minimax_h3_quantization.py @@ -23,7 +23,7 @@ def _package_version(name: str) -> str | None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model-root", default="/hhb-data/aigc/model_zoo/MiniMaxAI_MiniMax-H3") - parser.add_argument("--backend", choices=("bf16", "torchao-fp8", "bnb-nf4"), required=True) + parser.add_argument("--backend", choices=("bf16", "torchao-fp8", "tf-kernel-fp8", "bnb-nf4"), required=True) parser.add_argument("--prompt", default="Steam rises from the ramen while the family talks in the background.") parser.add_argument("--duration", type=float, default=5.0) parser.add_argument("--steps", type=int, default=50) From 9bb9277372f662beb2be1bd56740c831fbe477a5 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Fri, 7 Aug 2026 08:07:40 +0000 Subject: [PATCH 5/5] Consolidate MiniMax H3 quantization examples --- docs/en/quantization.md | 4 +- examples/minimax_h3/README.md | 25 +++------ .../minimax_h3_fl2va_bnb_nf4_h100.py | 55 ------------------- .../minimax_h3_fl2va_tf_kernel_fp8_h100.py | 55 ------------------- .../minimax_h3_fl2va_torchao_fp8_h100.py | 55 ------------------- .../pipelines/minimax_h3/test_examples.py | 26 +++------ .../service/test_example_service_parity.py | 7 --- tests/unit/test_example_registry.py | 3 - 8 files changed, 20 insertions(+), 210 deletions(-) delete mode 100644 examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py delete mode 100644 examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py delete mode 100644 examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py diff --git a/docs/en/quantization.md b/docs/en/quantization.md index 4f292be3..0f56858e 100644 --- a/docs/en/quantization.md +++ b/docs/en/quantization.md @@ -94,8 +94,8 @@ quant_config = QuantConfig( ) ``` -For MiniMax H3, use `quantization="tf-kernel-fp8"` or -`examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py`. This backend is single-GPU only and keeps the FP8 +For MiniMax H3, use `quantization="tf-kernel-fp8"` with +`examples/minimax_h3/minimax_h3_fl2va_h100.py`. This backend is single-GPU only and keeps the FP8 weights resident after first-use conversion. It is distinct from the scaled-FP8 checkpoint path below: the latter expects weights and scales already serialized in the checkpoint. diff --git a/examples/minimax_h3/README.md b/examples/minimax_h3/README.md index 13971481..4ee99174 100644 --- a/examples/minimax_h3/README.md +++ b/examples/minimax_h3/README.md @@ -308,35 +308,28 @@ and then kept resident for the pipeline lifetime. This ordering avoids a simulta one GPU and avoids unsupported CPU transfers of quantized tensor subclasses. TorchAO and tf-kernel conversion have a transient memory peak near the BF16 footprint; use the full 80 GB device without colocated workloads. -Use the dedicated TorchAO FP8 example: +Use the single FL2VA example and choose the quantization backend with `--quantization`: ~~~bash -python examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py \ +python examples/minimax_h3/minimax_h3_fl2va_h100.py \ --mode t2va \ + --quantization torchao-fp8 \ --duration 5 \ --output outputs/minimax_h3_torchao_fp8.mp4 -~~~ - -Or the dedicated bitsandbytes NF4 example: - -~~~bash -python examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py \ +python examples/minimax_h3/minimax_h3_fl2va_h100.py \ --mode t2va \ + --quantization bnb-nf4 \ --duration 5 \ --output outputs/minimax_h3_bnb_nf4.mp4 -~~~ - -Or the dedicated tf-kernel FP8 example: - -~~~bash -python examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py \ +python examples/minimax_h3/minimax_h3_fl2va_h100.py \ --mode t2va \ + --quantization tf-kernel-fp8 \ --duration 5 \ --output outputs/minimax_h3_tf_kernel_fp8.mp4 ~~~ -The standard FL2VA, Ref2VA, and JSON request CLIs also accept ---quantization with torchao-fp8, tf-kernel-fp8, or bnb-nf4. The Python loader accepts the same names: +The FL2VA CLI accepts `--quantization` with `torchao-fp8`, `tf-kernel-fp8`, or `bnb-nf4`; omit it for BF16. The Python +loader accepts the same names: ~~~python from examples.minimax_h3.common import load_minimax_h3_pipeline diff --git a/examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py b/examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py deleted file mode 100644 index 2164a862..00000000 --- a/examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""MiniMax H3 FL2VA example with bitsandbytes NF4 online quantization.""" - -from __future__ import annotations - -from copy import deepcopy -from functools import wraps - -if __package__: - from . import minimax_h3_fl2va_h100 as base -else: - try: - from examples.minimax_h3 import minimax_h3_fl2va_h100 as base - except ModuleNotFoundError: - import minimax_h3_fl2va_h100 as base - -PPL_CONFIG = { - **base.PPL_CONFIG, - "name": "minimax_h3_fl2va_bnb_nf4_h100", - "quantization": "bnb-nf4", -} -PIPELINE_MANIFEST = deepcopy(base.PIPELINE_MANIFEST) -PIPELINE_MANIFEST["pipeline_name"] = PPL_CONFIG["name"] - - -@wraps(base.run) -def run(*args: object, **kwargs: object) -> base.MiniMaxH3Generation: - return base.run(*args, **kwargs) - - -@wraps(base.run_with_file) -def run_with_file(*args: object, **kwargs: object) -> dict[str, str]: - return base.run_with_file(*args, **kwargs) - - -def get_pipeline( - parallelism: int = 1, - model_root: str = PPL_CONFIG["model_root"], - **kwargs: object, -) -> base.MiniMaxH3Pipeline: - """Load the single-GPU bitsandbytes NF4 FL2VA pipeline.""" - return base.get_pipeline( - parallelism, - model_root, - quantization=PPL_CONFIG["quantization"], - **kwargs, - ) - - -def main() -> None: - base._main(PPL_CONFIG["quantization"]) - - -if __name__ == "__main__": - main() diff --git a/examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py b/examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py deleted file mode 100644 index df4d7d37..00000000 --- a/examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""MiniMax H3 FL2VA example with tf-kernel FP8 online quantization.""" - -from __future__ import annotations - -from copy import deepcopy -from functools import wraps - -if __package__: - from . import minimax_h3_fl2va_h100 as base -else: - try: - from examples.minimax_h3 import minimax_h3_fl2va_h100 as base - except ModuleNotFoundError: - import minimax_h3_fl2va_h100 as base - -PPL_CONFIG = { - **base.PPL_CONFIG, - "name": "minimax_h3_fl2va_tf_kernel_fp8_h100", - "quantization": "tf-kernel-fp8", -} -PIPELINE_MANIFEST = deepcopy(base.PIPELINE_MANIFEST) -PIPELINE_MANIFEST["pipeline_name"] = PPL_CONFIG["name"] - - -@wraps(base.run) -def run(*args: object, **kwargs: object) -> base.MiniMaxH3Generation: - return base.run(*args, **kwargs) - - -@wraps(base.run_with_file) -def run_with_file(*args: object, **kwargs: object) -> dict[str, str]: - return base.run_with_file(*args, **kwargs) - - -def get_pipeline( - parallelism: int = 1, - model_root: str = PPL_CONFIG["model_root"], - **kwargs: object, -) -> base.MiniMaxH3Pipeline: - """Load the single-GPU tf-kernel FP8 FL2VA pipeline.""" - return base.get_pipeline( - parallelism, - model_root, - quantization=PPL_CONFIG["quantization"], - **kwargs, - ) - - -def main() -> None: - base._main(PPL_CONFIG["quantization"]) - - -if __name__ == "__main__": - main() diff --git a/examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py b/examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py deleted file mode 100644 index 1adc3cee..00000000 --- a/examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""MiniMax H3 FL2VA example with TorchAO FP8 online quantization.""" - -from __future__ import annotations - -from copy import deepcopy -from functools import wraps - -if __package__: - from . import minimax_h3_fl2va_h100 as base -else: - try: - from examples.minimax_h3 import minimax_h3_fl2va_h100 as base - except ModuleNotFoundError: - import minimax_h3_fl2va_h100 as base - -PPL_CONFIG = { - **base.PPL_CONFIG, - "name": "minimax_h3_fl2va_torchao_fp8_h100", - "quantization": "torchao-fp8", -} -PIPELINE_MANIFEST = deepcopy(base.PIPELINE_MANIFEST) -PIPELINE_MANIFEST["pipeline_name"] = PPL_CONFIG["name"] - - -@wraps(base.run) -def run(*args: object, **kwargs: object) -> base.MiniMaxH3Generation: - return base.run(*args, **kwargs) - - -@wraps(base.run_with_file) -def run_with_file(*args: object, **kwargs: object) -> dict[str, str]: - return base.run_with_file(*args, **kwargs) - - -def get_pipeline( - parallelism: int = 1, - model_root: str = PPL_CONFIG["model_root"], - **kwargs: object, -) -> base.MiniMaxH3Pipeline: - """Load the single-GPU TorchAO FP8 FL2VA pipeline.""" - return base.get_pipeline( - parallelism, - model_root, - quantization=PPL_CONFIG["quantization"], - **kwargs, - ) - - -def main() -> None: - base._main(PPL_CONFIG["quantization"]) - - -if __name__ == "__main__": - main() diff --git a/tests/unit/pipelines/minimax_h3/test_examples.py b/tests/unit/pipelines/minimax_h3/test_examples.py index 38abe169..4655763f 100644 --- a/tests/unit/pipelines/minimax_h3/test_examples.py +++ b/tests/unit/pipelines/minimax_h3/test_examples.py @@ -3,10 +3,7 @@ import pytest -from examples.minimax_h3 import minimax_h3_fl2va_bnb_nf4_h100 as bnb_nf4_example from examples.minimax_h3 import minimax_h3_fl2va_h100 as fl2va_example -from examples.minimax_h3 import minimax_h3_fl2va_tf_kernel_fp8_h100 as tf_kernel_fp8_example -from examples.minimax_h3 import minimax_h3_fl2va_torchao_fp8_h100 as torchao_fp8_example from examples.minimax_h3 import minimax_h3_ref2va_h100 as ref2va_example from examples.minimax_h3.common import ( MINIMAX_H3_DEFAULT_FL2VA_IMAGE, @@ -203,17 +200,9 @@ def test_quantization_rejects_unsupported_parallel_and_cpu_profiles(tmp_path: Pa ) -@pytest.mark.parametrize( - ("example", "quantization"), - [ - (torchao_fp8_example, "torchao-fp8"), - (tf_kernel_fp8_example, "tf-kernel-fp8"), - (bnb_nf4_example, "bnb-nf4"), - ], -) -def test_dedicated_quantized_examples_forward_fixed_backend( +@pytest.mark.parametrize("quantization", ["torchao-fp8", "tf-kernel-fp8", "bnb-nf4"]) +def test_standard_example_forwards_selected_quantization( monkeypatch: pytest.MonkeyPatch, - example: object, quantization: str, ) -> None: calls = [] @@ -223,10 +212,13 @@ def fake_get_pipeline(*args: object, **kwargs: object) -> object: calls.append((args, kwargs)) return sentinel - monkeypatch.setattr(example.base, "get_pipeline", fake_get_pipeline) - assert example.get_pipeline(1, "/models/h3", device="cuda:1") is sentinel - assert calls == [((1, "/models/h3"), {"device": "cuda:1", "quantization": quantization})] - assert example.PIPELINE_MANIFEST["pipeline_name"] == example.PPL_CONFIG["name"] + monkeypatch.setattr(fl2va_example, "load_minimax_h3_pipeline", fake_get_pipeline) + assert fl2va_example.get_pipeline(1, "/models/h3", device="cuda:1", quantization=quantization) is sentinel + assert len(calls) == 1 + assert calls[0][0] == ("/models/h3",) + assert calls[0][1]["device"] == "cuda:1" + assert calls[0][1]["quantization"] == quantization + assert fl2va_example.PIPELINE_MANIFEST["pipeline_name"] == fl2va_example.PPL_CONFIG["name"] def test_fl2va_run_maps_standard_service_tasks_to_model_conditions() -> None: diff --git a/tests/unit/service/test_example_service_parity.py b/tests/unit/service/test_example_service_parity.py index a8b36ffc..257f0fa6 100644 --- a/tests/unit/service/test_example_service_parity.py +++ b/tests/unit/service/test_example_service_parity.py @@ -29,13 +29,6 @@ SERVICE_EXAMPLES = { "wan21_i2v_service": (Path("examples/wan_video/wan21_14b_image_to_video_480p_service.py"), "i2v", True), "minimax_h3_fl2va": (Path("examples/minimax_h3/minimax_h3_fl2va_h100.py"), "t2v", True), - "minimax_h3_fl2va_torchao_fp8": (Path("examples/minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py"), "t2v", True), - "minimax_h3_fl2va_tf_kernel_fp8": ( - Path("examples/minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py"), - "t2v", - True, - ), - "minimax_h3_fl2va_bnb_nf4": (Path("examples/minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py"), "t2v", True), "minimax_h3_ref2va": (Path("examples/minimax_h3/minimax_h3_ref2va_h100.py"), "s2v", True), "wan22_i2v_distill": (Path("examples/wan_video/wan22_14b_image_to_video_distill_h100.py"), "i2v", True), "lingbot_video_dense": (Path("examples/lingbot_video/lingbot_video_dense_1_3b.py"), "t2i", True), diff --git a/tests/unit/test_example_registry.py b/tests/unit/test_example_registry.py index 09a00d6a..a56fb120 100644 --- a/tests/unit/test_example_registry.py +++ b/tests/unit/test_example_registry.py @@ -15,9 +15,6 @@ "lingbot_video/lingbot_video_dense_1_3b.py", "lingbot_video/lingbot_video_moe_30b.py", "minimax_h3/minimax_h3_fl2va_h100.py", - "minimax_h3/minimax_h3_fl2va_torchao_fp8_h100.py", - "minimax_h3/minimax_h3_fl2va_tf_kernel_fp8_h100.py", - "minimax_h3/minimax_h3_fl2va_bnb_nf4_h100.py", "minimax_h3/minimax_h3_ref2va_h100.py", }