From cb9adda98002bf60065ece15b552bdfd170d4bbd Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 27 May 2026 23:30:50 -0700 Subject: [PATCH 1/2] Support full-context cache transpilation Add explicit cache context sizing for cached component transpilation so decoder graphs can reserve the model context length instead of deriving capacity from the capture prompt. Gemma4, Qwen, and LFM cached decode builders now share the same config-driven resolver while preserving their existing capture-derived fallback sizes when no model context field is available. For Gemma4, sliding-attention layers keep the configured sliding-window cache size while full-attention layers receive the requested full cache length. Add a retranspile path that reuses existing converted weights, wire the cache context option through the CLI and transpile entrypoint, and make lowering honor per-attention cache length metadata. Add token-file input support for exact-token live generation and a context-scaling benchmark tool that records CSV/JSONL summaries and plots. Cover the new CLI plumbing, retranspile behavior, context parsing, and per-layer cache metadata with focused tests. Signed-off-by: Noah Cylich --- cactus-engine/tests/chat.cpp | 36 +- python/cactus/cli/__init__.py | 6 + python/cactus/cli/convert.py | 2 + python/cactus/cli/model.py | 7 +- python/cactus/cli/run.py | 1 + python/cactus/transpile/hf_model.py | 6 + python/cactus/transpile/lower.py | 3 +- python/cactus/transpile/model_adapters.py | 124 ++++- python/cactus/transpile/optimize_graph.py | 29 ++ python/tests/test_bindings.py | 5 + python/tests/test_cli_run.py | 4 + python/tests/test_cli_transpile_defaults.py | 107 ++++- .../tests/test_optimize_gemma4_attention.py | 51 ++ .../tools/benchmark_gemma4_context_scaling.py | 445 ++++++++++++++++++ 14 files changed, 802 insertions(+), 24 deletions(-) create mode 100755 python/tests/transpile/tools/benchmark_gemma4_context_scaling.py diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index 594c75de2..b659121e0 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -285,7 +286,11 @@ bool file_exists(const std::string& path) { std::vector parse_token_ids(const std::string& text) { std::vector tokens; - std::stringstream ss(text); + std::string normalized = text; + for (char& ch : normalized) { + if (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n') ch = ','; + } + std::stringstream ss(normalized); std::string part; while (std::getline(ss, part, ',')) { size_t start = part.find_first_not_of(" \t\r\n"); @@ -302,6 +307,16 @@ std::vector parse_token_ids(const std::string& text) { return tokens; } +bool read_text_file(const std::string& path, std::string& text) { + std::ifstream in(path); + if (!in) return false; + std::ostringstream buffer; + buffer << in.rdbuf(); + if (!in.good() && !in.eof()) return false; + text = buffer.str(); + return true; +} + bool write_text_file(const std::string& path, const std::string& text) { std::ofstream out(path); if (!out) return false; @@ -364,7 +379,7 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--system ] [--image ] [--audio ]" - << " [--prompt ] [--input-ids ] [--max-new-tokens ]" + << " [--prompt ] [--input-ids ] [--input-ids-file ] [--max-new-tokens ]" << " [--result-json ] [--thinking]\n"; } @@ -382,6 +397,7 @@ int main(int argc, char** argv) { std::string current_audio; std::string initial_prompt; std::string input_ids; + std::string input_ids_file; std::string result_json; int max_new_tokens = kMaxTokens; bool thinking = false; @@ -398,6 +414,8 @@ int main(int argc, char** argv) { initial_prompt = argv[++i]; } else if (arg == "--input-ids" && i + 1 < argc) { input_ids = argv[++i]; + } else if (arg == "--input-ids-file" && i + 1 < argc) { + input_ids_file = expand_tilde(argv[++i]); } else if (arg == "--max-new-tokens" && i + 1 < argc) { max_new_tokens = std::max(0, std::atoi(argv[++i])); } else if (arg == "--result-json" && i + 1 < argc) { @@ -415,6 +433,20 @@ int main(int argc, char** argv) { std::cerr << "Audio file not found: " << current_audio << "\n"; return 1; } + if (!input_ids.empty() && !input_ids_file.empty()) { + std::cerr << "Use either --input-ids or --input-ids-file, not both\n"; + return 1; + } + if (!input_ids_file.empty()) { + if (!file_exists(input_ids_file)) { + std::cerr << "Input ids file not found: " << input_ids_file << "\n"; + return 1; + } + if (!read_text_file(input_ids_file, input_ids)) { + std::cerr << "Failed to read input ids file: " << input_ids_file << "\n"; + return 1; + } + } std::cout << "Loading model from " << model_path << "...\n"; cactus_model_t model = cactus_init(model_path.c_str(), nullptr, false); diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 562d7ab58..b552f6927 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -177,6 +177,8 @@ def create_parser(): help="Initial prompt to send immediately") run_parser.add_argument("--input-ids", default=None, help="Comma-separated token ids for transpiled causal-LM bundles") + run_parser.add_argument("--input-ids-file", default=None, + help="File containing token ids for transpiled causal-LM bundles") run_parser.add_argument("--max-new-tokens", type=int, default=None, help="Maximum tokens to generate for transpiled causal-LM bundles") run_parser.add_argument("--result-json", default=None, @@ -258,8 +260,12 @@ def create_parser(): help="Allow HF remote code during the transpile phase") convert_parser.add_argument("--local-files-only", action="store_true", help="Require HF model/processor files to already be local during transpile") + convert_parser.add_argument("--cache-context-length", default=None, + help="KV cache context length for cached decode graphs (default: model config)") convert_parser.add_argument("--reconvert", action="store_true", help="Force conversion from source") + convert_parser.add_argument("--retranspile", action="store_true", + help="Force transpilation while reusing existing converted weights") return parser diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 9a36145dc..7c8b3c09a 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -14,6 +14,7 @@ def cmd_convert(args): bits=args.bits or 4, token=args.token, reconvert=args.reconvert, + retranspile=args.retranspile, output_dir=output_dir, transpile=TranspileOptions( task=args.task or "auto", @@ -26,6 +27,7 @@ def cmd_convert(args): system_prompt=args.system_prompt, trust_remote_code=args.trust_remote_code, local_files_only=args.local_files_only, + cache_context_length=args.cache_context_length, ), ) return 0 diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index 7f97bbbfb..b2f1cc94f 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -200,10 +200,11 @@ class TranspileOptions: system_prompt: str | None = None trust_remote_code: bool = False local_files_only: bool = False + cache_context_length: str | int | None = None def ensure_bundle(model_id, *, bits=4, token=None, - reconvert=False, output_dir=None, transpile=None): + reconvert=False, retranspile=False, output_dir=None, transpile=None): """Return path to transpiled bundle, creating it if needed. """ from .download import get_weights_dir @@ -224,7 +225,7 @@ def ensure_bundle(model_id, *, bits=4, token=None, ) # Step 2: skip if already transpiled - if _has_transpiled_bundle(output_dir): + if _has_transpiled_bundle(output_dir) and not retranspile: return output_dir # Step 3: infer transpile spec from converted output @@ -312,6 +313,8 @@ def ensure_bundle(model_id, *, bits=4, token=None, extra_args.append("--trust-remote-code") if opts.local_files_only: extra_args.append("--local-files-only") + if opts.cache_context_length is not None: + extra_args.extend(["--cache-context-length", str(opts.cache_context_length)]) rc = run_transpile(model_id, extra_args=extra_args) if rc != 0: diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 165c8d2c6..d60bbacf1 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -44,6 +44,7 @@ def cmd_run(args): ("--image", args.image), ("--audio", args.audio), ("--input-ids", args.input_ids), + ("--input-ids-file", args.input_ids_file), ("--max-new-tokens", args.max_new_tokens), ("--result-json", args.result_json), ): diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index b0e4ee0ad..22a26ee4d 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -2870,6 +2870,11 @@ def main() -> int: "so run-transpiled can do greedy autoregressive decoding." ), ) + parser.add_argument( + "--cache-context-length", + default="auto", + help="KV cache context length for cached decode graphs. Use auto to read the model config.", + ) parser.add_argument( "--audio-file", default="", @@ -3136,6 +3141,7 @@ def main() -> int: weights_dir=weights_dir, inputs_metadata=prepared.metadata, components=requested_components, + cache_context_length=args.cache_context_length, ) use_component_pipeline = False if args.component_pipeline == "on": diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index b2f33db41..4cd5179c2 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -238,7 +238,8 @@ def _lower_attention_with_internal_kv_cache( ) if layer_key not in cache_states: default_cache_len = max(512, int(key_shape[1])) - max_cache_seq_len = int(ir.meta.get("max_cache_seq_len", default_cache_len) or default_cache_len) + requested_cache_len = node.meta.get("max_cache_seq_len", ir.meta.get("max_cache_seq_len", default_cache_len)) + max_cache_seq_len = max(default_cache_len, int(requested_cache_len or default_cache_len)) sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) window_size = int(node.attrs.get("window_size", 0) or 0) k_cache = g.kv_cache_state( diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index a31973b0e..f52fcde23 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -290,6 +290,97 @@ def _gemma4_text_config(multimodal_backbone: torch.nn.Module) -> object: return config +def _parse_cache_context_length(value: str | int | None) -> int | None: + if value is None: + return None + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"", "auto"}: + return None + parsed = int(normalized) + else: + parsed = int(value) + if parsed <= 0: + raise ValueError("--cache-context-length must be a positive integer or auto") + return parsed + + +def _config_int_value(config: object, key: str) -> int | None: + if config is None: + return None + if isinstance(config, dict): + raw_value = config.get(key) + else: + raw_value = getattr(config, key, None) + if raw_value is None: + return None + parsed = int(raw_value) + return parsed if parsed > 0 else None + + +def _config_candidates_for_context_length(model: torch.nn.Module) -> tuple[object, ...]: + candidates: list[object] = [] + seen: set[int] = set() + + def _add(candidate: object) -> None: + if candidate is None: + return + candidate_id = id(candidate) + if candidate_id in seen: + return + seen.add(candidate_id) + candidates.append(candidate) + + for owner in ( + model, + getattr(model, "model", None), + getattr(getattr(model, "model", None), "language_model", None), + ): + config = getattr(owner, "config", None) + _add(config) + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + _add(get_text_config()) + for attr in ("text_config", "llm_config", "language_config", "decoder_config"): + _add(getattr(config, attr, None)) + + return tuple(candidates) + + +def _cache_context_length( + model: torch.nn.Module, + *, + input_seq_len: int, + cache_context_length: str | int | None, + fallback_extra_tokens: int, +) -> int: + explicit_length = _parse_cache_context_length(cache_context_length) + if explicit_length is not None: + return explicit_length + + for candidate in _config_candidates_for_context_length(model): + for key in ("max_position_embeddings", "context_length", "model_max_length", "n_positions"): + value = _config_int_value(candidate, key) + if value is not None: + return value + return input_seq_len + int(fallback_extra_tokens) + + +def _max_cache_seq_len( + model: torch.nn.Module, + input_ids: torch.Tensor, + cache_context_length: str | int | None, + *, + fallback_extra_tokens: int, +) -> int: + return max(1024, _cache_context_length( + model, + input_seq_len=int(input_ids.shape[1]), + cache_context_length=cache_context_length, + fallback_extra_tokens=fallback_extra_tokens, + )) + + def _gemma4_special_token_ids(multimodal_backbone: torch.nn.Module) -> tuple[int, int, int]: config = getattr(multimodal_backbone, "config", None) text_config = _gemma4_text_config(multimodal_backbone) @@ -407,9 +498,9 @@ def _gemma4_text_attention_forward( query_states = apply_rotary_pos_emb(query_states, cos, sin, unsqueeze_dim=2) query_states = query_states.transpose(1, 2) - kv_shared_layer_index = getattr(attn, "kv_shared_layer_index", None) - if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_layer_index in shared_kv_states: - key_states, value_states = shared_kv_states[int(kv_shared_layer_index)] + kv_shared_key = getattr(attn, "layer_type", getattr(attn, "kv_shared_layer_index", None)) + if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_key in shared_kv_states: + key_states, value_states = shared_kv_states[kv_shared_key] key_states = key_states.to(query_states.device) value_states = value_states.to(query_states.device) else: @@ -424,10 +515,9 @@ def _gemma4_text_attention_forward( value_states = value_states.transpose(1, 2) layer_idx = getattr(attn, "layer_idx", None) - if layer_idx is not None and ( - bool(getattr(attn, "store_full_length_kv", False)) - or not bool(getattr(attn, "is_kv_shared_layer", False)) - ): + if bool(getattr(attn, "store_full_length_kv", False)): + shared_kv_states[kv_shared_key] = (key_states, value_states) + elif layer_idx is not None and not bool(getattr(attn, "is_kv_shared_layer", False)): shared_kv_states[int(layer_idx)] = (key_states, value_states) attention_interface = eager_attention_forward @@ -3152,6 +3242,7 @@ def _build_gemma4_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec]: pixel_values = named_tensors["pixel_values"] pixel_position_ids = named_tensors.get("pixel_position_ids") @@ -3370,7 +3461,7 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder"}, metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) - cache_seq_len = max(1024, int(input_ids.shape[1]) + 256) + cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=256) prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) if "decoder" in expanded_components: @@ -4086,6 +4177,7 @@ def _build_qwen_causal_lm_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None = None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: input_ids = named_tensors.get("input_ids") if input_ids is None: @@ -4122,7 +4214,7 @@ def _build_qwen_causal_lm_component_specs( if "decoder_step" in requested_set: step_input_ids = input_ids[:, :1] step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_step, @@ -4149,6 +4241,7 @@ def _build_qwen3_5_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: input_ids = named_tensors.get("input_ids") attention_mask = named_tensors.get("attention_mask") @@ -4233,6 +4326,7 @@ def _require(component: str) -> None: "task": "multimodal_causal_lm_logits", "adapter_family": "qwen3_5", } + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs: list[ComponentModuleSpec] = [] if "vision_encoder" in expanded_components: specs.append(ComponentModuleSpec( @@ -4293,7 +4387,6 @@ def _require(component: str) -> None: if "decoder_prefill_chunk" in expanded_components: if prefill_decoder_inputs is None: raise RuntimeError("Qwen3.5 decoder_prefill_chunk spec requires precomputed text chunk decoder inputs") - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", module=decoder_prefill_chunk, @@ -4317,7 +4410,6 @@ def _require(component: str) -> None: raise RuntimeError("Qwen3.5 decoder_media_step spec requires precomputed decoder inputs") step_inputs_embeds = decoder_inputs[0][:, :1, :] step_position_ids = decoder_inputs[2][:, :, :1] - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_media_step", module=decoder_media_step, @@ -4338,7 +4430,6 @@ def _require(component: str) -> None: if "decoder_step" in expanded_components: step_input_ids = input_ids[:, :1] step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_step, @@ -4756,6 +4847,7 @@ def _build_lfm2_vl_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec]: input_ids = named_tensors["input_ids"] attention_mask = named_tensors["attention_mask"] @@ -4857,6 +4949,7 @@ def _require(component: str) -> None: "task": "multimodal_causal_lm_logits", "adapter_family": "lfm2_vl", } + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs: list[ComponentModuleSpec] = [] if "vision_encoder" in expanded_components: specs.append(ComponentModuleSpec( @@ -4949,7 +5042,6 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values, spatial_shapes, pixel_attention_mask) decoder_inputs = lm_encoder(input_ids, attention_mask, image_features) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", module=decoder_last_token, @@ -4971,7 +5063,6 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values, spatial_shapes, pixel_attention_mask) decoder_inputs = lm_encoder(input_ids, attention_mask, image_features) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_last_token, @@ -4999,6 +5090,7 @@ def build_component_module_specs( weights_dir: str | None = None, inputs_metadata: dict[str, object] | None = None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: family = _family_key(model) if family == "qwen3_5" and task == "multimodal_causal_lm_logits": @@ -5007,6 +5099,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family in {"qwen3", "qwen3_5"} and task == "causal_lm_logits": return _build_qwen_causal_lm_component_specs( @@ -5014,6 +5107,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "gemma4" and task == "multimodal_causal_lm_logits": return _build_gemma4_multimodal_component_specs( @@ -5021,6 +5115,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "lfm2_vl" and task == "multimodal_causal_lm_logits": return _build_lfm2_vl_multimodal_component_specs( @@ -5028,6 +5123,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "parakeet_tdt" and task == "tdt_transcription": from cactus.transpile.tdt_runtime import build_parakeet_tdt_component_specs diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py index 430fd0e7d..cd14e1a09 100644 --- a/python/cactus/transpile/optimize_graph.py +++ b/python/cactus/transpile/optimize_graph.py @@ -701,6 +701,9 @@ def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bo if len(attention_nodes) != len(layer_types): return False + use_internal_cache = bool(graph.meta.get("use_internal_kv_cache", False)) + sliding_window = _graph_sliding_window(graph) + graph_cache_seq_len = _graph_max_cache_seq_len(graph) changed = False for layer_index, (node, layer_type) in enumerate(zip(attention_nodes, layer_types, strict=True)): if "attention_layer_type" not in node.meta: @@ -709,6 +712,16 @@ def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bo if "attention_layer_index" not in node.meta: node.meta["attention_layer_index"] = int(layer_index) changed = True + if use_internal_cache: + normalized_layer_type = str(layer_type).strip().lower() + node_cache_seq_len = None + if normalized_layer_type in {"sliding", "sliding_attention"} and sliding_window is not None and sliding_window > 0: + node_cache_seq_len = int(sliding_window) + elif normalized_layer_type in {"global", "full", "full_attention"} and graph_cache_seq_len is not None: + node_cache_seq_len = int(graph_cache_seq_len) + if node_cache_seq_len is not None and node.meta.get("max_cache_seq_len") != node_cache_seq_len: + node.meta["max_cache_seq_len"] = node_cache_seq_len + changed = True return changed @@ -1449,6 +1462,22 @@ def _graph_sliding_window(graph: IRGraph) -> int | None: return None +def _graph_max_cache_seq_len(graph: IRGraph) -> int | None: + max_cache_seq_len = _coerce_optional_int(graph.meta.get("max_cache_seq_len")) + if max_cache_seq_len is not None: + return max_cache_seq_len + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + max_cache_seq_len = _coerce_optional_int(provider_meta.get("max_cache_seq_len")) + if max_cache_seq_len is not None: + return max_cache_seq_len + return None + + def _coerce_string_tuple(value: object) -> tuple[str, ...]: if not isinstance(value, (tuple, list)): return () diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index 7ab6412b7..2adf8c513 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -244,6 +244,7 @@ def test_defaults(self): assert opts.system_prompt is None assert opts.trust_remote_code is False assert opts.local_files_only is False + assert opts.cache_context_length is None def test_custom_values(self): from cactus.cli.model import TranspileOptions @@ -252,11 +253,13 @@ def test_custom_values(self): prompt="Hello", max_new_tokens=256, trust_remote_code=True, + cache_context_length="131072", ) assert opts.task == "causal_lm_logits" assert opts.prompt == "Hello" assert opts.max_new_tokens == 256 assert opts.trust_remote_code is True + assert opts.cache_context_length == "131072" def test_frozen(self): from cactus.cli.model import TranspileOptions @@ -297,6 +300,7 @@ def test_run_command_chunked_bundle_flags(self): "--audio", "audio.wav", "--image", "image.png", "--input-ids", "1,2,3", + "--input-ids-file", "tokens.txt", "--max-new-tokens", "4", "--result-json", "result.json", ]) @@ -304,6 +308,7 @@ def test_run_command_chunked_bundle_flags(self): assert args.audio == "audio.wav" assert args.image == "image.png" assert args.input_ids == "1,2,3" + assert args.input_ids_file == "tokens.txt" assert args.max_new_tokens == 4 assert args.result_json == "result.json" diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py index 4c2b82c7c..ae29a74e6 100644 --- a/python/tests/test_cli_run.py +++ b/python/tests/test_cli_run.py @@ -19,8 +19,10 @@ def test_cmd_run_forwards_chunked_bundle_flags(monkeypatch, tmp_path: Path) -> N image_file = tmp_path / "image.png" audio_file = tmp_path / "audio.wav" result_json = tmp_path / "result.json" + input_ids_file = tmp_path / "tokens.txt" image_file.write_bytes(b"image") audio_file.write_bytes(b"audio") + input_ids_file.write_text("1 2 3", encoding="utf-8") calls = [] @@ -40,6 +42,7 @@ def fake_run(cmd): image=str(image_file), audio=str(audio_file), input_ids="1,2,3", + input_ids_file=str(input_ids_file), max_new_tokens=4, result_json=str(result_json), thinking=False, @@ -53,5 +56,6 @@ def fake_run(cmd): assert cmd[cmd.index("--image") + 1] == str(image_file) assert cmd[cmd.index("--audio") + 1] == str(audio_file) assert cmd[cmd.index("--input-ids") + 1] == "1,2,3" + assert cmd[cmd.index("--input-ids-file") + 1] == str(input_ids_file) assert cmd[cmd.index("--max-new-tokens") + 1] == "4" assert cmd[cmd.index("--result-json") + 1] == str(result_json) diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index 110becb7c..e12685f7c 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -1,14 +1,77 @@ from __future__ import annotations from pathlib import Path +from types import SimpleNamespace +import pytest from cactus import cli from cactus.cli import convert as convert_cli import cactus.cli.model as model_mod +from cactus.transpile.model_adapters import _cache_context_length + + +class _ConfigWithText: + def get_text_config(self): + return SimpleNamespace(max_position_embeddings=128000) + + +def test_cache_context_length_uses_explicit_value() -> None: + model = SimpleNamespace(config=SimpleNamespace(max_position_embeddings=40960)) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length="32768", + fallback_extra_tokens=512, + ) == 32768 + + +def test_cache_context_length_reads_top_level_config() -> None: + model = SimpleNamespace(config=SimpleNamespace(max_position_embeddings=40960)) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length=None, + fallback_extra_tokens=512, + ) == 40960 + + +def test_cache_context_length_reads_text_config() -> None: + model = SimpleNamespace(config=_ConfigWithText()) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length="auto", + fallback_extra_tokens=512, + ) == 128000 + + +def test_cache_context_length_falls_back_to_capture_size() -> None: + model = SimpleNamespace(config=SimpleNamespace()) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length=None, + fallback_extra_tokens=512, + ) == 2560 + + +def test_cache_context_length_rejects_non_positive_explicit_value() -> None: + with pytest.raises(ValueError): + _cache_context_length( + SimpleNamespace(config=SimpleNamespace()), + input_seq_len=2048, + cache_context_length="0", + fallback_extra_tokens=512, + ) def _write_gemma4_multimodal_config(output_dir: Path) -> None: output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "config.txt").write_text("model_type=gemma4\n", encoding="utf-8") (output_dir / "hf_config.json").write_text( ( '{"model_type":"gemma4",' @@ -21,8 +84,7 @@ def _write_gemma4_multimodal_config(output_dir: Path) -> None: def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[str]: - from cactus.cli.common import PROJECT_ROOT - assets_dir = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" + default_images, default_audio = model_mod._default_multimodal_assets() return [ "--weights-dir", str(model_dir), @@ -39,9 +101,9 @@ def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[s "--components", "vision_encoder,audio_encoder,lm_encoder,decoder", "--image-file", - str(assets_dir / "test_monkey.png"), + default_images[0], "--audio-file", - str(assets_dir / "test.wav"), + str(default_audio), "--trust-remote-code", ] @@ -139,6 +201,41 @@ def _fake_cq_main(command): assert transpile_calls[0]["extra_args"] == _gemma4_multimodal_extra_args(output_dir, output_dir) +def test_cmd_convert_retranspile_reuses_weights_and_forwards_cache_context(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + output_dir = tmp_path / "gemma4" + _write_gemma4_multimodal_config(output_dir) + (output_dir / "components").mkdir() + (output_dir / "components" / "manifest.json").write_text("{}", encoding="utf-8") + (output_dir / "raw_ir_decoder_step.json").write_text("{}", encoding="utf-8") + args = parser.parse_args([ + "convert", + "google/gemma-4-E2B-it", + str(output_dir), + "--retranspile", + "--cache-context-length", + "131072", + ]) + + def _unexpected_cq_main(command): + raise AssertionError(f"unexpected CQ conversion: {command}") + + import cactus.convert.cli as cq_cli + monkeypatch.setattr(cq_cli, "main", _unexpected_cq_main) + + transpile_calls: list[dict] = [] + _patch_transpile(monkeypatch, transpile_calls) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + assert not (output_dir / "components").exists() + assert not (output_dir / "raw_ir_decoder_step.json").exists() + assert len(transpile_calls) == 1 + extra_args = transpile_calls[0]["extra_args"] + assert extra_args[extra_args.index("--cache-context-length") + 1] == "131072" + + def test_cmd_convert_supplies_default_audio_for_parakeet(monkeypatch, tmp_path: Path) -> None: parser = cli.create_parser() output_dir = tmp_path / "parakeet" @@ -164,7 +261,7 @@ def _fake_cq_main(command): extra_args = transpile_calls[0]["extra_args"] assert extra_args[extra_args.index("--task") + 1] == "tdt_transcription" assert "--audio-file" in extra_args - assert extra_args[extra_args.index("--audio-file") + 1].endswith("cactus-engine/tests/assets/test.wav") + assert Path(extra_args[extra_args.index("--audio-file") + 1]).name == "test.wav" def test_cmd_convert_supplies_default_audio_for_whisper(monkeypatch, tmp_path: Path) -> None: diff --git a/python/tests/test_optimize_gemma4_attention.py b/python/tests/test_optimize_gemma4_attention.py index 843e3f65c..137e58c5c 100644 --- a/python/tests/test_optimize_gemma4_attention.py +++ b/python/tests/test_optimize_gemma4_attention.py @@ -178,6 +178,57 @@ def test_normalize_gemma4_attention_recovers_layer_hints_from_graph_meta() -> No assert graph.nodes["attn_1"].attrs["window_size"] == 800 +def test_normalize_gemma4_cached_attention_sets_per_layer_cache_lengths() -> None: + graph = IRGraph( + values={ + "query_0": IRValue(id="query_0", shape=(1, 8, 1, 256), dtype="fp16"), + "key_0": IRValue(id="key_0", shape=(1, 8, 1, 256), dtype="fp16"), + "value_0": IRValue(id="value_0", shape=(1, 8, 1, 256), dtype="fp16"), + "out_0": IRValue(id="out_0", shape=(1, 8, 1, 256), dtype="fp16", producer="attn_0"), + "query_1": IRValue(id="query_1", shape=(1, 8, 1, 256), dtype="fp16"), + "key_1": IRValue(id="key_1", shape=(1, 8, 1, 256), dtype="fp16"), + "value_1": IRValue(id="value_1", shape=(1, 8, 1, 256), dtype="fp16"), + "out_1": IRValue(id="out_1", shape=(1, 8, 1, 256), dtype="fp16", producer="attn_1"), + }, + nodes={ + "attn_0": IRNode( + id="attn_0", + op="attention", + inputs=["query_0", "key_0", "value_0"], + outputs=["out_0"], + attrs={"is_causal": False, "window_size": 0}, + ), + "attn_1": IRNode( + id="attn_1", + op="attention", + inputs=["query_1", "key_1", "value_1"], + outputs=["out_1"], + attrs={"is_causal": False, "window_size": 0}, + ), + }, + order=["attn_0", "attn_1"], + inputs=["query_0", "key_0", "value_0", "query_1", "key_1", "value_1"], + outputs=["out_1"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder_step", + "use_internal_kv_cache": True, + "sliding_window": 512, + "max_cache_seq_len": 131072, + "layer_types": ("sliding_attention", "full_attention"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn_0"].attrs["window_size"] == 512 + assert graph.nodes["attn_0"].meta["max_cache_seq_len"] == 512 + assert graph.nodes["attn_1"].attrs["window_size"] == 0 + assert graph.nodes["attn_1"].meta["max_cache_seq_len"] == 131072 + + def test_normalize_gemma4_attention_elides_logical_fp16_mask() -> None: graph = IRGraph( values={ diff --git a/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py b/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py new file mode 100755 index 000000000..9a4a90c53 --- /dev/null +++ b/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json +import shutil +import statistics +import subprocess +from datetime import datetime +from pathlib import Path +from typing import Any + + +LENGTHS = (64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768) +SWEEPS = ( + ("warmup", (64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768)), + ("measured_1", (2048, 4096, 1024, 8192, 512, 16384, 256, 32768, 128, 64)), + ("measured_2", (64, 128, 32768, 256, 16384, 512, 8192, 1024, 4096, 2048)), + ("measured_3", (4096, 1024, 8192, 512, 16384, 256, 32768, 128, 64, 2048)), +) +JSON_FIELDS = ( + "success", + "time_to_first_token_ms", + "total_time_ms", + "prefill_tps", + "prefill_compute_tps", + "prefill_prepare_tps", + "ttft_prompt_tps", + "cache_prime_ms", + "cache_prime_compute_ms", + "cache_state_copy_ms", + "cache_prime_tokens", + "first_decode_ms", + "first_token_from_prefill", + "prefill_padding_tokens", + "prefill_scalar_tail_tokens", + "decode_tps", + "prompt_tokens", + "completion_tokens", + "peak_ram_usage_mb", + "ram_usage_mb", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run Gemma4 E4B context-scaling decode benchmarks through cactus run.", + ) + parser.add_argument( + "--model", + default="weights/gemma-4-e4b-it", + help="Prepared Cactus bundle to benchmark.", + ) + parser.add_argument( + "--out-dir", + type=Path, + default=None, + help="Directory for token files, raw results, logs, JSONL, CSV, and summary.", + ) + parser.add_argument( + "--cactus-bin", + default="cactus", + help="Cactus CLI executable to invoke.", + ) + parser.add_argument( + "--token-id", + type=int, + default=106, + help="Token id repeated to build exact-length prompts.", + ) + parser.add_argument( + "--max-new-tokens", + type=int, + default=32, + help="Number of tokens to generate after each prefill.", + ) + parser.add_argument( + "--skip-build", + action="store_true", + help="Skip the required cactus build step when the caller already built this checkout.", + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + help="Record failed runs and continue instead of stopping at the first failure.", + ) + parser.add_argument( + "--no-plots", + action="store_true", + help="Do not generate PNG plots after the benchmark completes.", + ) + return parser.parse_args() + + +def default_out_dir() -> Path: + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + return Path("tmp") / "gemma4-context-scaling" / timestamp + + +def write_token_file(path: Path, *, token_id: int, count: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + chunk_size = 4096 + remaining = count + first = True + with path.open("w", encoding="utf-8") as handle: + while remaining > 0: + current = min(chunk_size, remaining) + text = ",".join([str(token_id)] * current) + if not first: + handle.write(",") + handle.write(text) + first = False + remaining -= current + + +def ensure_token_files(out_dir: Path, *, token_id: int) -> dict[int, Path]: + token_dir = out_dir / "tokens" + files: dict[int, Path] = {} + for length in LENGTHS: + path = token_dir / f"tokens_{length}.txt" + write_token_file(path, token_id=token_id, count=length) + files[length] = path + return files + + +def run_command(command: list[str], *, stdout_path: Path, stderr_path: Path) -> int: + with stdout_path.open("w", encoding="utf-8") as stdout, stderr_path.open("w", encoding="utf-8") as stderr: + proc = subprocess.run(command, stdout=stdout, stderr=stderr, text=True) + return int(proc.returncode) + + +def read_result_json(result_path: Path, stdout_path: Path) -> dict[str, Any]: + if result_path.exists(): + return json.loads(result_path.read_text(encoding="utf-8")) + for line in stdout_path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.startswith("{") and stripped.endswith("}"): + return json.loads(stripped) + raise RuntimeError(f"No result JSON found in {result_path} or {stdout_path}") + + +def decode_ms_per_token(payload: dict[str, Any]) -> float | None: + decode_tps = float(payload.get("decode_tps") or 0.0) + if decode_tps <= 0.0: + return None + return 1000.0 / decode_tps + + +def append_jsonl(path: Path, record: dict[str, Any]) -> None: + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + +def write_csv(path: Path, records: list[dict[str, Any]]) -> None: + fieldnames = [ + "phase", + "sweep_index", + "sweep_position", + "context_tokens", + "max_new_tokens", + "returncode", + "decode_ms_per_token", + *JSON_FIELDS, + "result_json", + "stdout_log", + "stderr_log", + ] + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for record in records: + writer.writerow({name: record.get(name) for name in fieldnames}) + + +def write_summary(path: Path, records: list[dict[str, Any]]) -> None: + measured = [record for record in records if str(record.get("phase", "")).startswith("measured")] + fieldnames = [ + "context_tokens", + "runs", + "decode_ms_per_token_mean", + "decode_ms_per_token_stdev", + "decode_tps_mean", + "decode_tps_stdev", + "total_time_ms_mean", + "total_time_ms_stdev", + "peak_ram_usage_mb_mean", + "peak_ram_usage_mb_stdev", + ] + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for length in LENGTHS: + rows = [record for record in measured if int(record.get("context_tokens", -1)) == length] + ms_values = [value for row in rows if (value := _optional_float(row.get("decode_ms_per_token"))) is not None] + tps_values = [value for row in rows if (value := _optional_float(row.get("decode_tps"))) is not None] + total_values = [value for row in rows if (value := _optional_float(row.get("total_time_ms"))) is not None] + ram_values = [value for row in rows if (value := _optional_float(row.get("peak_ram_usage_mb"))) is not None] + writer.writerow({ + "context_tokens": length, + "runs": len(rows), + "decode_ms_per_token_mean": statistics.mean(ms_values) if ms_values else None, + "decode_ms_per_token_stdev": statistics.stdev(ms_values) if len(ms_values) > 1 else 0.0 if ms_values else None, + "decode_tps_mean": statistics.mean(tps_values) if tps_values else None, + "decode_tps_stdev": statistics.stdev(tps_values) if len(tps_values) > 1 else 0.0 if tps_values else None, + "total_time_ms_mean": statistics.mean(total_values) if total_values else None, + "total_time_ms_stdev": statistics.stdev(total_values) if len(total_values) > 1 else 0.0 if total_values else None, + "peak_ram_usage_mb_mean": statistics.mean(ram_values) if ram_values else None, + "peak_ram_usage_mb_stdev": statistics.stdev(ram_values) if len(ram_values) > 1 else 0.0 if ram_values else None, + }) + + +def _float_column(rows: list[dict[str, str]], key: str, *, default: float = 0.0) -> list[float]: + values: list[float] = [] + for row in rows: + raw = row.get(key) + values.append(float(raw) if raw not in {None, ""} else default) + return values + + +def _optional_float(value: object) -> float | None: + if value in {None, ""}: + return None + return float(value) + + +def write_plots(summary_path: Path, out_dir: Path) -> list[Path]: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + with summary_path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + contexts = [int(row["context_tokens"]) for row in rows] + ms_mean = _float_column(rows, "decode_ms_per_token_mean") + ms_std = _float_column(rows, "decode_ms_per_token_stdev") + tps_mean = _float_column(rows, "decode_tps_mean") + tps_std = _float_column(rows, "decode_tps_stdev") + ram_mean = _float_column(rows, "peak_ram_usage_mb_mean") + ram_std = _float_column(rows, "peak_ram_usage_mb_stdev") + total_mean = [value / 1000.0 for value in _float_column(rows, "total_time_ms_mean")] + total_std = [value / 1000.0 for value in _float_column(rows, "total_time_ms_stdev")] + + plt.style.use("seaborn-v0_8-whitegrid") + error_style = { + "ecolor": "#1f1f1f", + "elinewidth": 1.8, + "capthick": 1.8, + "capsize": 6, + "markeredgecolor": "#1f1f1f", + "markeredgewidth": 0.9, + } + + def configure_x_axis(ax) -> None: + ax.set_xscale("log", base=2) + ax.set_xticks(contexts) + ax.set_xticklabels([str(context) for context in contexts], rotation=35, ha="right") + ax.grid(True, which="both", axis="y", alpha=0.35) + ax.grid(True, which="major", axis="x", alpha=0.18) + + def finish(fig, path: Path) -> Path: + fig.tight_layout() + fig.savefig(path, dpi=180) + plt.close(fig) + return path + + paths: list[Path] = [] + + fig, ax = plt.subplots(figsize=(9.5, 5.5)) + ax.errorbar(contexts, ms_mean, yerr=ms_std, marker="o", linewidth=2, color="#2f6f9f", **error_style) + configure_x_axis(ax) + ax.set_xlabel("Prefill context tokens") + ax.set_ylabel("Decode latency (ms/token)") + ax.set_title("Gemma 4 E4B IT decode latency vs context length") + paths.append(finish(fig, out_dir / "decode_ms_per_token.png")) + + fig, ax = plt.subplots(figsize=(9.5, 5.5)) + ax.errorbar(contexts, tps_mean, yerr=tps_std, marker="o", linewidth=2, color="#3d7d54", **error_style) + configure_x_axis(ax) + ax.set_xlabel("Prefill context tokens") + ax.set_ylabel("Decode throughput (tokens/sec)") + ax.set_title("Gemma 4 E4B IT decode throughput vs context length") + paths.append(finish(fig, out_dir / "decode_tps.png")) + + fig, ax1 = plt.subplots(figsize=(9.5, 5.5)) + line1 = ax1.errorbar( + contexts, + ms_mean, + yerr=ms_std, + marker="o", + linewidth=2, + color="#2f6f9f", + label="Decode ms/token", + **error_style, + ) + configure_x_axis(ax1) + ax1.set_xlabel("Prefill context tokens") + ax1.set_ylabel("Decode latency (ms/token)", color="#2f6f9f") + ax1.tick_params(axis="y", labelcolor="#2f6f9f") + ax2 = ax1.twinx() + line2 = ax2.errorbar( + contexts, + ram_mean, + yerr=ram_std, + marker="s", + linewidth=2, + color="#8a5a2b", + label="Peak RAM MB", + **error_style, + ) + ax2.set_ylabel("Peak RAM (MB)", color="#8a5a2b") + ax2.tick_params(axis="y", labelcolor="#8a5a2b") + ax1.set_title("Gemma 4 E4B IT decode latency and RAM vs context length") + ax1.legend([line1, line2], ["Decode ms/token", "Peak RAM MB"], loc="upper left") + paths.append(finish(fig, out_dir / "decode_latency_and_ram.png")) + + fig, ax = plt.subplots(figsize=(9.5, 5.5)) + ax.errorbar(contexts, total_mean, yerr=total_std, marker="o", linewidth=2, color="#7b4f9f", **error_style) + configure_x_axis(ax) + ax.set_yscale("log") + ax.set_xlabel("Prefill context tokens") + ax.set_ylabel("Mean total run time (seconds, log scale)") + ax.set_title("Gemma 4 E4B IT total run time vs context length") + paths.append(finish(fig, out_dir / "total_time_seconds.png")) + + baseline = ms_mean[0] + baseline_std = ms_std[0] + slowdown = [value / baseline for value in ms_mean] + slowdown_std = [] + for value, std in zip(ms_mean, ms_std): + if value <= 0.0 or baseline <= 0.0: + slowdown_std.append(0.0) + else: + relative_std = ((std / value) ** 2 + (baseline_std / baseline) ** 2) ** 0.5 + slowdown_std.append((value / baseline) * relative_std) + fig, ax = plt.subplots(figsize=(9.5, 5.5)) + ax.errorbar(contexts, slowdown, yerr=slowdown_std, marker="o", linewidth=2, color="#b24c4c", **error_style) + ax.axhline(1.0, color="#444444", linewidth=1, linestyle="--", alpha=0.7) + configure_x_axis(ax) + ax.set_xlabel("Prefill context tokens") + ax.set_ylabel("Decode slowdown vs 64-token baseline") + ax.set_title("Gemma 4 E4B IT decode slowdown vs initial speed") + for context, value in zip(contexts, slowdown): + ax.annotate(f"{value:.2f}x", (context, value), textcoords="offset points", xytext=(0, 8), ha="center", fontsize=8) + paths.append(finish(fig, out_dir / "decode_slowdown_vs_64.png")) + + return paths + + +def validate_environment(cactus_bin: str) -> None: + if shutil.which(cactus_bin) is None: + raise RuntimeError(f"Could not find Cactus CLI executable: {cactus_bin}") + + +def main() -> int: + args = parse_args() + validate_environment(args.cactus_bin) + + out_dir = args.out_dir or default_out_dir() + out_dir.mkdir(parents=True, exist_ok=True) + token_files = ensure_token_files(out_dir, token_id=args.token_id) + + metadata = { + "model": args.model, + "token_id": args.token_id, + "max_new_tokens": args.max_new_tokens, + "sweeps": [{"phase": phase, "lengths": list(lengths)} for phase, lengths in SWEEPS], + } + (out_dir / "metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + if not args.skip_build: + build_rc = run_command( + [args.cactus_bin, "build"], + stdout_path=out_dir / "build.stdout.log", + stderr_path=out_dir / "build.stderr.log", + ) + if build_rc != 0: + raise RuntimeError(f"cactus build failed; see {out_dir / 'build.stderr.log'}") + + records: list[dict[str, Any]] = [] + jsonl_path = out_dir / "runs.jsonl" + csv_path = out_dir / "runs.csv" + summary_path = out_dir / "summary.csv" + + for sweep_index, (phase, lengths) in enumerate(SWEEPS): + for sweep_position, length in enumerate(lengths): + prefix = f"{sweep_index:02d}_{phase}_{sweep_position:02d}_{length}" + result_path = out_dir / "results" / f"{prefix}.json" + stdout_path = out_dir / "logs" / f"{prefix}.stdout.log" + stderr_path = out_dir / "logs" / f"{prefix}.stderr.log" + result_path.parent.mkdir(parents=True, exist_ok=True) + stdout_path.parent.mkdir(parents=True, exist_ok=True) + + command = [ + args.cactus_bin, + "run", + args.model, + "--input-ids-file", + str(token_files[length]), + "--max-new-tokens", + str(args.max_new_tokens), + "--result-json", + str(result_path), + ] + print(f"[{phase} {sweep_position + 1}/{len(lengths)}] context={length}", flush=True) + returncode = run_command(command, stdout_path=stdout_path, stderr_path=stderr_path) + + if returncode == 0: + payload = read_result_json(result_path, stdout_path) + else: + payload = {"success": False} + + record = { + "phase": phase, + "sweep_index": sweep_index, + "sweep_position": sweep_position, + "context_tokens": length, + "max_new_tokens": args.max_new_tokens, + "returncode": returncode, + "decode_ms_per_token": decode_ms_per_token(payload), + "result_json": str(result_path), + "stdout_log": str(stdout_path), + "stderr_log": str(stderr_path), + } + for field in JSON_FIELDS: + record[field] = payload.get(field) + records.append(record) + append_jsonl(jsonl_path, record) + write_csv(csv_path, records) + write_summary(summary_path, records) + + if returncode != 0 and not args.continue_on_error: + raise RuntimeError(f"Benchmark run failed for context={length}; see {stderr_path}") + + print(f"wrote {jsonl_path}") + print(f"wrote {csv_path}") + print(f"wrote {summary_path}") + if not args.no_plots: + for path in write_plots(summary_path, out_dir): + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From e00859d653f917b7008c888e87aa3b4e112cd2a0 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 29 May 2026 09:51:01 -0700 Subject: [PATCH 2/2] removing benchmarking artifact Signed-off-by: Noah Cylich --- .../tools/benchmark_gemma4_context_scaling.py | 445 ------------------ 1 file changed, 445 deletions(-) delete mode 100755 python/tests/transpile/tools/benchmark_gemma4_context_scaling.py diff --git a/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py b/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py deleted file mode 100755 index 9a4a90c53..000000000 --- a/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py +++ /dev/null @@ -1,445 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import csv -import json -import shutil -import statistics -import subprocess -from datetime import datetime -from pathlib import Path -from typing import Any - - -LENGTHS = (64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768) -SWEEPS = ( - ("warmup", (64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768)), - ("measured_1", (2048, 4096, 1024, 8192, 512, 16384, 256, 32768, 128, 64)), - ("measured_2", (64, 128, 32768, 256, 16384, 512, 8192, 1024, 4096, 2048)), - ("measured_3", (4096, 1024, 8192, 512, 16384, 256, 32768, 128, 64, 2048)), -) -JSON_FIELDS = ( - "success", - "time_to_first_token_ms", - "total_time_ms", - "prefill_tps", - "prefill_compute_tps", - "prefill_prepare_tps", - "ttft_prompt_tps", - "cache_prime_ms", - "cache_prime_compute_ms", - "cache_state_copy_ms", - "cache_prime_tokens", - "first_decode_ms", - "first_token_from_prefill", - "prefill_padding_tokens", - "prefill_scalar_tail_tokens", - "decode_tps", - "prompt_tokens", - "completion_tokens", - "peak_ram_usage_mb", - "ram_usage_mb", -) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Run Gemma4 E4B context-scaling decode benchmarks through cactus run.", - ) - parser.add_argument( - "--model", - default="weights/gemma-4-e4b-it", - help="Prepared Cactus bundle to benchmark.", - ) - parser.add_argument( - "--out-dir", - type=Path, - default=None, - help="Directory for token files, raw results, logs, JSONL, CSV, and summary.", - ) - parser.add_argument( - "--cactus-bin", - default="cactus", - help="Cactus CLI executable to invoke.", - ) - parser.add_argument( - "--token-id", - type=int, - default=106, - help="Token id repeated to build exact-length prompts.", - ) - parser.add_argument( - "--max-new-tokens", - type=int, - default=32, - help="Number of tokens to generate after each prefill.", - ) - parser.add_argument( - "--skip-build", - action="store_true", - help="Skip the required cactus build step when the caller already built this checkout.", - ) - parser.add_argument( - "--continue-on-error", - action="store_true", - help="Record failed runs and continue instead of stopping at the first failure.", - ) - parser.add_argument( - "--no-plots", - action="store_true", - help="Do not generate PNG plots after the benchmark completes.", - ) - return parser.parse_args() - - -def default_out_dir() -> Path: - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") - return Path("tmp") / "gemma4-context-scaling" / timestamp - - -def write_token_file(path: Path, *, token_id: int, count: int) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - chunk_size = 4096 - remaining = count - first = True - with path.open("w", encoding="utf-8") as handle: - while remaining > 0: - current = min(chunk_size, remaining) - text = ",".join([str(token_id)] * current) - if not first: - handle.write(",") - handle.write(text) - first = False - remaining -= current - - -def ensure_token_files(out_dir: Path, *, token_id: int) -> dict[int, Path]: - token_dir = out_dir / "tokens" - files: dict[int, Path] = {} - for length in LENGTHS: - path = token_dir / f"tokens_{length}.txt" - write_token_file(path, token_id=token_id, count=length) - files[length] = path - return files - - -def run_command(command: list[str], *, stdout_path: Path, stderr_path: Path) -> int: - with stdout_path.open("w", encoding="utf-8") as stdout, stderr_path.open("w", encoding="utf-8") as stderr: - proc = subprocess.run(command, stdout=stdout, stderr=stderr, text=True) - return int(proc.returncode) - - -def read_result_json(result_path: Path, stdout_path: Path) -> dict[str, Any]: - if result_path.exists(): - return json.loads(result_path.read_text(encoding="utf-8")) - for line in stdout_path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if stripped.startswith("{") and stripped.endswith("}"): - return json.loads(stripped) - raise RuntimeError(f"No result JSON found in {result_path} or {stdout_path}") - - -def decode_ms_per_token(payload: dict[str, Any]) -> float | None: - decode_tps = float(payload.get("decode_tps") or 0.0) - if decode_tps <= 0.0: - return None - return 1000.0 / decode_tps - - -def append_jsonl(path: Path, record: dict[str, Any]) -> None: - with path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(record, sort_keys=True) + "\n") - - -def write_csv(path: Path, records: list[dict[str, Any]]) -> None: - fieldnames = [ - "phase", - "sweep_index", - "sweep_position", - "context_tokens", - "max_new_tokens", - "returncode", - "decode_ms_per_token", - *JSON_FIELDS, - "result_json", - "stdout_log", - "stderr_log", - ] - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - for record in records: - writer.writerow({name: record.get(name) for name in fieldnames}) - - -def write_summary(path: Path, records: list[dict[str, Any]]) -> None: - measured = [record for record in records if str(record.get("phase", "")).startswith("measured")] - fieldnames = [ - "context_tokens", - "runs", - "decode_ms_per_token_mean", - "decode_ms_per_token_stdev", - "decode_tps_mean", - "decode_tps_stdev", - "total_time_ms_mean", - "total_time_ms_stdev", - "peak_ram_usage_mb_mean", - "peak_ram_usage_mb_stdev", - ] - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - for length in LENGTHS: - rows = [record for record in measured if int(record.get("context_tokens", -1)) == length] - ms_values = [value for row in rows if (value := _optional_float(row.get("decode_ms_per_token"))) is not None] - tps_values = [value for row in rows if (value := _optional_float(row.get("decode_tps"))) is not None] - total_values = [value for row in rows if (value := _optional_float(row.get("total_time_ms"))) is not None] - ram_values = [value for row in rows if (value := _optional_float(row.get("peak_ram_usage_mb"))) is not None] - writer.writerow({ - "context_tokens": length, - "runs": len(rows), - "decode_ms_per_token_mean": statistics.mean(ms_values) if ms_values else None, - "decode_ms_per_token_stdev": statistics.stdev(ms_values) if len(ms_values) > 1 else 0.0 if ms_values else None, - "decode_tps_mean": statistics.mean(tps_values) if tps_values else None, - "decode_tps_stdev": statistics.stdev(tps_values) if len(tps_values) > 1 else 0.0 if tps_values else None, - "total_time_ms_mean": statistics.mean(total_values) if total_values else None, - "total_time_ms_stdev": statistics.stdev(total_values) if len(total_values) > 1 else 0.0 if total_values else None, - "peak_ram_usage_mb_mean": statistics.mean(ram_values) if ram_values else None, - "peak_ram_usage_mb_stdev": statistics.stdev(ram_values) if len(ram_values) > 1 else 0.0 if ram_values else None, - }) - - -def _float_column(rows: list[dict[str, str]], key: str, *, default: float = 0.0) -> list[float]: - values: list[float] = [] - for row in rows: - raw = row.get(key) - values.append(float(raw) if raw not in {None, ""} else default) - return values - - -def _optional_float(value: object) -> float | None: - if value in {None, ""}: - return None - return float(value) - - -def write_plots(summary_path: Path, out_dir: Path) -> list[Path]: - import matplotlib - matplotlib.use("Agg") - import matplotlib.pyplot as plt - - with summary_path.open(newline="", encoding="utf-8") as handle: - rows = list(csv.DictReader(handle)) - contexts = [int(row["context_tokens"]) for row in rows] - ms_mean = _float_column(rows, "decode_ms_per_token_mean") - ms_std = _float_column(rows, "decode_ms_per_token_stdev") - tps_mean = _float_column(rows, "decode_tps_mean") - tps_std = _float_column(rows, "decode_tps_stdev") - ram_mean = _float_column(rows, "peak_ram_usage_mb_mean") - ram_std = _float_column(rows, "peak_ram_usage_mb_stdev") - total_mean = [value / 1000.0 for value in _float_column(rows, "total_time_ms_mean")] - total_std = [value / 1000.0 for value in _float_column(rows, "total_time_ms_stdev")] - - plt.style.use("seaborn-v0_8-whitegrid") - error_style = { - "ecolor": "#1f1f1f", - "elinewidth": 1.8, - "capthick": 1.8, - "capsize": 6, - "markeredgecolor": "#1f1f1f", - "markeredgewidth": 0.9, - } - - def configure_x_axis(ax) -> None: - ax.set_xscale("log", base=2) - ax.set_xticks(contexts) - ax.set_xticklabels([str(context) for context in contexts], rotation=35, ha="right") - ax.grid(True, which="both", axis="y", alpha=0.35) - ax.grid(True, which="major", axis="x", alpha=0.18) - - def finish(fig, path: Path) -> Path: - fig.tight_layout() - fig.savefig(path, dpi=180) - plt.close(fig) - return path - - paths: list[Path] = [] - - fig, ax = plt.subplots(figsize=(9.5, 5.5)) - ax.errorbar(contexts, ms_mean, yerr=ms_std, marker="o", linewidth=2, color="#2f6f9f", **error_style) - configure_x_axis(ax) - ax.set_xlabel("Prefill context tokens") - ax.set_ylabel("Decode latency (ms/token)") - ax.set_title("Gemma 4 E4B IT decode latency vs context length") - paths.append(finish(fig, out_dir / "decode_ms_per_token.png")) - - fig, ax = plt.subplots(figsize=(9.5, 5.5)) - ax.errorbar(contexts, tps_mean, yerr=tps_std, marker="o", linewidth=2, color="#3d7d54", **error_style) - configure_x_axis(ax) - ax.set_xlabel("Prefill context tokens") - ax.set_ylabel("Decode throughput (tokens/sec)") - ax.set_title("Gemma 4 E4B IT decode throughput vs context length") - paths.append(finish(fig, out_dir / "decode_tps.png")) - - fig, ax1 = plt.subplots(figsize=(9.5, 5.5)) - line1 = ax1.errorbar( - contexts, - ms_mean, - yerr=ms_std, - marker="o", - linewidth=2, - color="#2f6f9f", - label="Decode ms/token", - **error_style, - ) - configure_x_axis(ax1) - ax1.set_xlabel("Prefill context tokens") - ax1.set_ylabel("Decode latency (ms/token)", color="#2f6f9f") - ax1.tick_params(axis="y", labelcolor="#2f6f9f") - ax2 = ax1.twinx() - line2 = ax2.errorbar( - contexts, - ram_mean, - yerr=ram_std, - marker="s", - linewidth=2, - color="#8a5a2b", - label="Peak RAM MB", - **error_style, - ) - ax2.set_ylabel("Peak RAM (MB)", color="#8a5a2b") - ax2.tick_params(axis="y", labelcolor="#8a5a2b") - ax1.set_title("Gemma 4 E4B IT decode latency and RAM vs context length") - ax1.legend([line1, line2], ["Decode ms/token", "Peak RAM MB"], loc="upper left") - paths.append(finish(fig, out_dir / "decode_latency_and_ram.png")) - - fig, ax = plt.subplots(figsize=(9.5, 5.5)) - ax.errorbar(contexts, total_mean, yerr=total_std, marker="o", linewidth=2, color="#7b4f9f", **error_style) - configure_x_axis(ax) - ax.set_yscale("log") - ax.set_xlabel("Prefill context tokens") - ax.set_ylabel("Mean total run time (seconds, log scale)") - ax.set_title("Gemma 4 E4B IT total run time vs context length") - paths.append(finish(fig, out_dir / "total_time_seconds.png")) - - baseline = ms_mean[0] - baseline_std = ms_std[0] - slowdown = [value / baseline for value in ms_mean] - slowdown_std = [] - for value, std in zip(ms_mean, ms_std): - if value <= 0.0 or baseline <= 0.0: - slowdown_std.append(0.0) - else: - relative_std = ((std / value) ** 2 + (baseline_std / baseline) ** 2) ** 0.5 - slowdown_std.append((value / baseline) * relative_std) - fig, ax = plt.subplots(figsize=(9.5, 5.5)) - ax.errorbar(contexts, slowdown, yerr=slowdown_std, marker="o", linewidth=2, color="#b24c4c", **error_style) - ax.axhline(1.0, color="#444444", linewidth=1, linestyle="--", alpha=0.7) - configure_x_axis(ax) - ax.set_xlabel("Prefill context tokens") - ax.set_ylabel("Decode slowdown vs 64-token baseline") - ax.set_title("Gemma 4 E4B IT decode slowdown vs initial speed") - for context, value in zip(contexts, slowdown): - ax.annotate(f"{value:.2f}x", (context, value), textcoords="offset points", xytext=(0, 8), ha="center", fontsize=8) - paths.append(finish(fig, out_dir / "decode_slowdown_vs_64.png")) - - return paths - - -def validate_environment(cactus_bin: str) -> None: - if shutil.which(cactus_bin) is None: - raise RuntimeError(f"Could not find Cactus CLI executable: {cactus_bin}") - - -def main() -> int: - args = parse_args() - validate_environment(args.cactus_bin) - - out_dir = args.out_dir or default_out_dir() - out_dir.mkdir(parents=True, exist_ok=True) - token_files = ensure_token_files(out_dir, token_id=args.token_id) - - metadata = { - "model": args.model, - "token_id": args.token_id, - "max_new_tokens": args.max_new_tokens, - "sweeps": [{"phase": phase, "lengths": list(lengths)} for phase, lengths in SWEEPS], - } - (out_dir / "metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - if not args.skip_build: - build_rc = run_command( - [args.cactus_bin, "build"], - stdout_path=out_dir / "build.stdout.log", - stderr_path=out_dir / "build.stderr.log", - ) - if build_rc != 0: - raise RuntimeError(f"cactus build failed; see {out_dir / 'build.stderr.log'}") - - records: list[dict[str, Any]] = [] - jsonl_path = out_dir / "runs.jsonl" - csv_path = out_dir / "runs.csv" - summary_path = out_dir / "summary.csv" - - for sweep_index, (phase, lengths) in enumerate(SWEEPS): - for sweep_position, length in enumerate(lengths): - prefix = f"{sweep_index:02d}_{phase}_{sweep_position:02d}_{length}" - result_path = out_dir / "results" / f"{prefix}.json" - stdout_path = out_dir / "logs" / f"{prefix}.stdout.log" - stderr_path = out_dir / "logs" / f"{prefix}.stderr.log" - result_path.parent.mkdir(parents=True, exist_ok=True) - stdout_path.parent.mkdir(parents=True, exist_ok=True) - - command = [ - args.cactus_bin, - "run", - args.model, - "--input-ids-file", - str(token_files[length]), - "--max-new-tokens", - str(args.max_new_tokens), - "--result-json", - str(result_path), - ] - print(f"[{phase} {sweep_position + 1}/{len(lengths)}] context={length}", flush=True) - returncode = run_command(command, stdout_path=stdout_path, stderr_path=stderr_path) - - if returncode == 0: - payload = read_result_json(result_path, stdout_path) - else: - payload = {"success": False} - - record = { - "phase": phase, - "sweep_index": sweep_index, - "sweep_position": sweep_position, - "context_tokens": length, - "max_new_tokens": args.max_new_tokens, - "returncode": returncode, - "decode_ms_per_token": decode_ms_per_token(payload), - "result_json": str(result_path), - "stdout_log": str(stdout_path), - "stderr_log": str(stderr_path), - } - for field in JSON_FIELDS: - record[field] = payload.get(field) - records.append(record) - append_jsonl(jsonl_path, record) - write_csv(csv_path, records) - write_summary(summary_path, records) - - if returncode != 0 and not args.continue_on_error: - raise RuntimeError(f"Benchmark run failed for context={length}; see {stderr_path}") - - print(f"wrote {jsonl_path}") - print(f"wrote {csv_path}") - print(f"wrote {summary_path}") - if not args.no_plots: - for path in write_plots(summary_path, out_dir): - print(f"wrote {path}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())