diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index 594c75de2..b659121e0 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -285,7 +286,11 @@ bool file_exists(const std::string& path) { std::vector parse_token_ids(const std::string& text) { std::vector tokens; - std::stringstream ss(text); + std::string normalized = text; + for (char& ch : normalized) { + if (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n') ch = ','; + } + std::stringstream ss(normalized); std::string part; while (std::getline(ss, part, ',')) { size_t start = part.find_first_not_of(" \t\r\n"); @@ -302,6 +307,16 @@ std::vector parse_token_ids(const std::string& text) { return tokens; } +bool read_text_file(const std::string& path, std::string& text) { + std::ifstream in(path); + if (!in) return false; + std::ostringstream buffer; + buffer << in.rdbuf(); + if (!in.good() && !in.eof()) return false; + text = buffer.str(); + return true; +} + bool write_text_file(const std::string& path, const std::string& text) { std::ofstream out(path); if (!out) return false; @@ -364,7 +379,7 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--system ] [--image ] [--audio ]" - << " [--prompt ] [--input-ids ] [--max-new-tokens ]" + << " [--prompt ] [--input-ids ] [--input-ids-file ] [--max-new-tokens ]" << " [--result-json ] [--thinking]\n"; } @@ -382,6 +397,7 @@ int main(int argc, char** argv) { std::string current_audio; std::string initial_prompt; std::string input_ids; + std::string input_ids_file; std::string result_json; int max_new_tokens = kMaxTokens; bool thinking = false; @@ -398,6 +414,8 @@ int main(int argc, char** argv) { initial_prompt = argv[++i]; } else if (arg == "--input-ids" && i + 1 < argc) { input_ids = argv[++i]; + } else if (arg == "--input-ids-file" && i + 1 < argc) { + input_ids_file = expand_tilde(argv[++i]); } else if (arg == "--max-new-tokens" && i + 1 < argc) { max_new_tokens = std::max(0, std::atoi(argv[++i])); } else if (arg == "--result-json" && i + 1 < argc) { @@ -415,6 +433,20 @@ int main(int argc, char** argv) { std::cerr << "Audio file not found: " << current_audio << "\n"; return 1; } + if (!input_ids.empty() && !input_ids_file.empty()) { + std::cerr << "Use either --input-ids or --input-ids-file, not both\n"; + return 1; + } + if (!input_ids_file.empty()) { + if (!file_exists(input_ids_file)) { + std::cerr << "Input ids file not found: " << input_ids_file << "\n"; + return 1; + } + if (!read_text_file(input_ids_file, input_ids)) { + std::cerr << "Failed to read input ids file: " << input_ids_file << "\n"; + return 1; + } + } std::cout << "Loading model from " << model_path << "...\n"; cactus_model_t model = cactus_init(model_path.c_str(), nullptr, false); diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 40b742411..2d818f376 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -187,6 +187,8 @@ def create_parser(): help="Initial prompt to send immediately") run_parser.add_argument("--input-ids", default=None, help="Comma-separated token ids for transpiled causal-LM bundles") + run_parser.add_argument("--input-ids-file", default=None, + help="File containing token ids for transpiled causal-LM bundles") run_parser.add_argument("--max-new-tokens", type=int, default=None, help="Maximum tokens to generate for transpiled causal-LM bundles") run_parser.add_argument("--result-json", default=None, @@ -277,8 +279,12 @@ def create_parser(): help="Allow HF remote code during the transpile phase") convert_parser.add_argument("--local-files-only", action="store_true", help="Require HF model/processor files to already be local during transpile") + convert_parser.add_argument("--cache-context-length", default=None, + help="KV cache context length for cached decode graphs (default: model config)") convert_parser.add_argument("--reconvert", action="store_true", help="Force conversion from source") + convert_parser.add_argument("--retranspile", action="store_true", + help="Force transpilation while reusing existing converted weights") convert_parser.add_argument("--npu", action="store_true", help="Also emit CoreML .mlpackage(s) for NPU (Apple Neural Engine) audio + vision encoders") convert_parser.add_argument("--npu-quantize", type=int, choices=[0, 4, 8], default=None, diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 2eaef3c40..87f488142 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -14,6 +14,7 @@ def cmd_convert(args): bits=args.bits or 4, token=args.token, reconvert=args.reconvert, + retranspile=args.retranspile, output_dir=output_dir, transpile=TranspileOptions( task=args.task or "auto", @@ -26,6 +27,7 @@ def cmd_convert(args): system_prompt=args.system_prompt, trust_remote_code=args.trust_remote_code, local_files_only=args.local_files_only, + cache_context_length=args.cache_context_length, npu=getattr(args, "npu", False), npu_quantize=getattr(args, "npu_quantize", None), npu_audio_quantize=getattr(args, "npu_audio_quantize", None), diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index f5e90ac94..4212cecb1 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -200,6 +200,7 @@ class TranspileOptions: system_prompt: str | None = None trust_remote_code: bool = False local_files_only: bool = False + cache_context_length: str | int | None = None npu: bool = False npu_quantize: int | None = None npu_audio_quantize: int | None = None @@ -207,7 +208,7 @@ class TranspileOptions: def ensure_bundle(model_id, *, bits=4, token=None, - reconvert=False, output_dir=None, transpile=None): + reconvert=False, retranspile=False, output_dir=None, transpile=None): """Return path to transpiled bundle, creating it if needed. """ from .download import get_weights_dir @@ -228,7 +229,7 @@ def ensure_bundle(model_id, *, bits=4, token=None, ) # Step 2: skip if already transpiled - if _has_transpiled_bundle(output_dir): + if _has_transpiled_bundle(output_dir) and not retranspile: return output_dir # Step 3: infer transpile spec from converted output @@ -316,6 +317,8 @@ def ensure_bundle(model_id, *, bits=4, token=None, extra_args.append("--trust-remote-code") if opts.local_files_only: extra_args.append("--local-files-only") + if opts.cache_context_length is not None: + extra_args.extend(["--cache-context-length", str(opts.cache_context_length)]) if opts.npu: extra_args.append("--npu") if opts.npu_quantize is not None: diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 165c8d2c6..d60bbacf1 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -44,6 +44,7 @@ def cmd_run(args): ("--image", args.image), ("--audio", args.audio), ("--input-ids", args.input_ids), + ("--input-ids-file", args.input_ids_file), ("--max-new-tokens", args.max_new_tokens), ("--result-json", args.result_json), ): diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index f9648e0fc..96d57fa98 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -2892,6 +2892,11 @@ def main() -> int: "so run-transpiled can do greedy autoregressive decoding." ), ) + parser.add_argument( + "--cache-context-length", + default="auto", + help="KV cache context length for cached decode graphs. Use auto to read the model config.", + ) parser.add_argument( "--audio-file", default="", @@ -3184,6 +3189,7 @@ def main() -> int: weights_dir=weights_dir, inputs_metadata=prepared.metadata, components=requested_components, + cache_context_length=args.cache_context_length, ) use_component_pipeline = False if args.component_pipeline == "on": diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index 5f2c3d1e2..e16af4f9b 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -238,7 +238,8 @@ def _lower_attention_with_internal_kv_cache( ) if layer_key not in cache_states: default_cache_len = max(512, int(key_shape[1])) - max_cache_seq_len = int(ir.meta.get("max_cache_seq_len", default_cache_len) or default_cache_len) + requested_cache_len = node.meta.get("max_cache_seq_len", ir.meta.get("max_cache_seq_len", default_cache_len)) + max_cache_seq_len = max(default_cache_len, int(requested_cache_len or default_cache_len)) sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) window_size = int(node.attrs.get("window_size", 0) or 0) k_cache = g.kv_cache_state( diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 47f8f73fb..9c7d4d9c3 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -290,6 +290,97 @@ def _gemma4_text_config(multimodal_backbone: torch.nn.Module) -> object: return config +def _parse_cache_context_length(value: str | int | None) -> int | None: + if value is None: + return None + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"", "auto"}: + return None + parsed = int(normalized) + else: + parsed = int(value) + if parsed <= 0: + raise ValueError("--cache-context-length must be a positive integer or auto") + return parsed + + +def _config_int_value(config: object, key: str) -> int | None: + if config is None: + return None + if isinstance(config, dict): + raw_value = config.get(key) + else: + raw_value = getattr(config, key, None) + if raw_value is None: + return None + parsed = int(raw_value) + return parsed if parsed > 0 else None + + +def _config_candidates_for_context_length(model: torch.nn.Module) -> tuple[object, ...]: + candidates: list[object] = [] + seen: set[int] = set() + + def _add(candidate: object) -> None: + if candidate is None: + return + candidate_id = id(candidate) + if candidate_id in seen: + return + seen.add(candidate_id) + candidates.append(candidate) + + for owner in ( + model, + getattr(model, "model", None), + getattr(getattr(model, "model", None), "language_model", None), + ): + config = getattr(owner, "config", None) + _add(config) + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + _add(get_text_config()) + for attr in ("text_config", "llm_config", "language_config", "decoder_config"): + _add(getattr(config, attr, None)) + + return tuple(candidates) + + +def _cache_context_length( + model: torch.nn.Module, + *, + input_seq_len: int, + cache_context_length: str | int | None, + fallback_extra_tokens: int, +) -> int: + explicit_length = _parse_cache_context_length(cache_context_length) + if explicit_length is not None: + return explicit_length + + for candidate in _config_candidates_for_context_length(model): + for key in ("max_position_embeddings", "context_length", "model_max_length", "n_positions"): + value = _config_int_value(candidate, key) + if value is not None: + return value + return input_seq_len + int(fallback_extra_tokens) + + +def _max_cache_seq_len( + model: torch.nn.Module, + input_ids: torch.Tensor, + cache_context_length: str | int | None, + *, + fallback_extra_tokens: int, +) -> int: + return max(1024, _cache_context_length( + model, + input_seq_len=int(input_ids.shape[1]), + cache_context_length=cache_context_length, + fallback_extra_tokens=fallback_extra_tokens, + )) + + def _gemma4_special_token_ids(multimodal_backbone: torch.nn.Module) -> tuple[int, int, int]: config = getattr(multimodal_backbone, "config", None) text_config = _gemma4_text_config(multimodal_backbone) @@ -407,9 +498,9 @@ def _gemma4_text_attention_forward( query_states = apply_rotary_pos_emb(query_states, cos, sin, unsqueeze_dim=2) query_states = query_states.transpose(1, 2) - kv_shared_layer_index = getattr(attn, "kv_shared_layer_index", None) - if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_layer_index in shared_kv_states: - key_states, value_states = shared_kv_states[int(kv_shared_layer_index)] + kv_shared_key = getattr(attn, "layer_type", getattr(attn, "kv_shared_layer_index", None)) + if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_key in shared_kv_states: + key_states, value_states = shared_kv_states[kv_shared_key] key_states = key_states.to(query_states.device) value_states = value_states.to(query_states.device) else: @@ -424,10 +515,9 @@ def _gemma4_text_attention_forward( value_states = value_states.transpose(1, 2) layer_idx = getattr(attn, "layer_idx", None) - if layer_idx is not None and ( - bool(getattr(attn, "store_full_length_kv", False)) - or not bool(getattr(attn, "is_kv_shared_layer", False)) - ): + if bool(getattr(attn, "store_full_length_kv", False)): + shared_kv_states[kv_shared_key] = (key_states, value_states) + elif layer_idx is not None and not bool(getattr(attn, "is_kv_shared_layer", False)): shared_kv_states[int(layer_idx)] = (key_states, value_states) attention_interface = eager_attention_forward @@ -3208,6 +3298,7 @@ def _build_gemma4_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec]: pixel_values = named_tensors["pixel_values"] pixel_position_ids = named_tensors.get("pixel_position_ids") @@ -3426,7 +3517,7 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder"}, metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) - cache_seq_len = max(1024, int(input_ids.shape[1]) + 256) + cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=256) prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) if "decoder" in expanded_components: @@ -4142,6 +4233,7 @@ def _build_qwen_causal_lm_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None = None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: input_ids = named_tensors.get("input_ids") if input_ids is None: @@ -4178,7 +4270,7 @@ def _build_qwen_causal_lm_component_specs( if "decoder_step" in requested_set: step_input_ids = input_ids[:, :1] step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_step, @@ -4205,6 +4297,7 @@ def _build_qwen3_5_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: input_ids = named_tensors.get("input_ids") attention_mask = named_tensors.get("attention_mask") @@ -4289,6 +4382,7 @@ def _require(component: str) -> None: "task": "multimodal_causal_lm_logits", "adapter_family": "qwen3_5", } + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs: list[ComponentModuleSpec] = [] if "vision_encoder" in expanded_components: specs.append(ComponentModuleSpec( @@ -4349,7 +4443,6 @@ def _require(component: str) -> None: if "decoder_prefill_chunk" in expanded_components: if prefill_decoder_inputs is None: raise RuntimeError("Qwen3.5 decoder_prefill_chunk spec requires precomputed text chunk decoder inputs") - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", module=decoder_prefill_chunk, @@ -4373,7 +4466,6 @@ def _require(component: str) -> None: raise RuntimeError("Qwen3.5 decoder_media_step spec requires precomputed decoder inputs") step_inputs_embeds = decoder_inputs[0][:, :1, :] step_position_ids = decoder_inputs[2][:, :, :1] - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_media_step", module=decoder_media_step, @@ -4394,7 +4486,6 @@ def _require(component: str) -> None: if "decoder_step" in expanded_components: step_input_ids = input_ids[:, :1] step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_step, @@ -4868,6 +4959,7 @@ def _build_lfm2_vl_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec]: input_ids = named_tensors["input_ids"] attention_mask = named_tensors["attention_mask"] @@ -4969,6 +5061,7 @@ def _require(component: str) -> None: "task": "multimodal_causal_lm_logits", "adapter_family": "lfm2_vl", } + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs: list[ComponentModuleSpec] = [] if "vision_encoder" in expanded_components: specs.append(ComponentModuleSpec( @@ -5061,7 +5154,6 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values, spatial_shapes, pixel_attention_mask) decoder_inputs = lm_encoder(input_ids, attention_mask, image_features) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", module=decoder_last_token, @@ -5083,7 +5175,6 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values, spatial_shapes, pixel_attention_mask) decoder_inputs = lm_encoder(input_ids, attention_mask, image_features) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_last_token, @@ -5111,6 +5202,7 @@ def build_component_module_specs( weights_dir: str | None = None, inputs_metadata: dict[str, object] | None = None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: family = _family_key(model) if family == "qwen3_5" and task == "multimodal_causal_lm_logits": @@ -5119,6 +5211,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family in {"qwen3", "qwen3_5"} and task == "causal_lm_logits": return _build_qwen_causal_lm_component_specs( @@ -5126,6 +5219,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "gemma4" and task == "multimodal_causal_lm_logits": return _build_gemma4_multimodal_component_specs( @@ -5133,6 +5227,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "lfm2_vl" and task == "multimodal_causal_lm_logits": return _build_lfm2_vl_multimodal_component_specs( @@ -5140,6 +5235,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "lfm2" and task == "causal_lm_logits": return _build_lfm2_causal_lm_component_specs( diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py index 430fd0e7d..cd14e1a09 100644 --- a/python/cactus/transpile/optimize_graph.py +++ b/python/cactus/transpile/optimize_graph.py @@ -701,6 +701,9 @@ def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bo if len(attention_nodes) != len(layer_types): return False + use_internal_cache = bool(graph.meta.get("use_internal_kv_cache", False)) + sliding_window = _graph_sliding_window(graph) + graph_cache_seq_len = _graph_max_cache_seq_len(graph) changed = False for layer_index, (node, layer_type) in enumerate(zip(attention_nodes, layer_types, strict=True)): if "attention_layer_type" not in node.meta: @@ -709,6 +712,16 @@ def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bo if "attention_layer_index" not in node.meta: node.meta["attention_layer_index"] = int(layer_index) changed = True + if use_internal_cache: + normalized_layer_type = str(layer_type).strip().lower() + node_cache_seq_len = None + if normalized_layer_type in {"sliding", "sliding_attention"} and sliding_window is not None and sliding_window > 0: + node_cache_seq_len = int(sliding_window) + elif normalized_layer_type in {"global", "full", "full_attention"} and graph_cache_seq_len is not None: + node_cache_seq_len = int(graph_cache_seq_len) + if node_cache_seq_len is not None and node.meta.get("max_cache_seq_len") != node_cache_seq_len: + node.meta["max_cache_seq_len"] = node_cache_seq_len + changed = True return changed @@ -1449,6 +1462,22 @@ def _graph_sliding_window(graph: IRGraph) -> int | None: return None +def _graph_max_cache_seq_len(graph: IRGraph) -> int | None: + max_cache_seq_len = _coerce_optional_int(graph.meta.get("max_cache_seq_len")) + if max_cache_seq_len is not None: + return max_cache_seq_len + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + max_cache_seq_len = _coerce_optional_int(provider_meta.get("max_cache_seq_len")) + if max_cache_seq_len is not None: + return max_cache_seq_len + return None + + def _coerce_string_tuple(value: object) -> tuple[str, ...]: if not isinstance(value, (tuple, list)): return () diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index 7ab6412b7..2adf8c513 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -244,6 +244,7 @@ def test_defaults(self): assert opts.system_prompt is None assert opts.trust_remote_code is False assert opts.local_files_only is False + assert opts.cache_context_length is None def test_custom_values(self): from cactus.cli.model import TranspileOptions @@ -252,11 +253,13 @@ def test_custom_values(self): prompt="Hello", max_new_tokens=256, trust_remote_code=True, + cache_context_length="131072", ) assert opts.task == "causal_lm_logits" assert opts.prompt == "Hello" assert opts.max_new_tokens == 256 assert opts.trust_remote_code is True + assert opts.cache_context_length == "131072" def test_frozen(self): from cactus.cli.model import TranspileOptions @@ -297,6 +300,7 @@ def test_run_command_chunked_bundle_flags(self): "--audio", "audio.wav", "--image", "image.png", "--input-ids", "1,2,3", + "--input-ids-file", "tokens.txt", "--max-new-tokens", "4", "--result-json", "result.json", ]) @@ -304,6 +308,7 @@ def test_run_command_chunked_bundle_flags(self): assert args.audio == "audio.wav" assert args.image == "image.png" assert args.input_ids == "1,2,3" + assert args.input_ids_file == "tokens.txt" assert args.max_new_tokens == 4 assert args.result_json == "result.json" diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py index 4c2b82c7c..ae29a74e6 100644 --- a/python/tests/test_cli_run.py +++ b/python/tests/test_cli_run.py @@ -19,8 +19,10 @@ def test_cmd_run_forwards_chunked_bundle_flags(monkeypatch, tmp_path: Path) -> N image_file = tmp_path / "image.png" audio_file = tmp_path / "audio.wav" result_json = tmp_path / "result.json" + input_ids_file = tmp_path / "tokens.txt" image_file.write_bytes(b"image") audio_file.write_bytes(b"audio") + input_ids_file.write_text("1 2 3", encoding="utf-8") calls = [] @@ -40,6 +42,7 @@ def fake_run(cmd): image=str(image_file), audio=str(audio_file), input_ids="1,2,3", + input_ids_file=str(input_ids_file), max_new_tokens=4, result_json=str(result_json), thinking=False, @@ -53,5 +56,6 @@ def fake_run(cmd): assert cmd[cmd.index("--image") + 1] == str(image_file) assert cmd[cmd.index("--audio") + 1] == str(audio_file) assert cmd[cmd.index("--input-ids") + 1] == "1,2,3" + assert cmd[cmd.index("--input-ids-file") + 1] == str(input_ids_file) assert cmd[cmd.index("--max-new-tokens") + 1] == "4" assert cmd[cmd.index("--result-json") + 1] == str(result_json) diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index 62ea0c99a..e12685f7c 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -1,14 +1,77 @@ from __future__ import annotations from pathlib import Path +from types import SimpleNamespace +import pytest from cactus import cli from cactus.cli import convert as convert_cli import cactus.cli.model as model_mod +from cactus.transpile.model_adapters import _cache_context_length + + +class _ConfigWithText: + def get_text_config(self): + return SimpleNamespace(max_position_embeddings=128000) + + +def test_cache_context_length_uses_explicit_value() -> None: + model = SimpleNamespace(config=SimpleNamespace(max_position_embeddings=40960)) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length="32768", + fallback_extra_tokens=512, + ) == 32768 + + +def test_cache_context_length_reads_top_level_config() -> None: + model = SimpleNamespace(config=SimpleNamespace(max_position_embeddings=40960)) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length=None, + fallback_extra_tokens=512, + ) == 40960 + + +def test_cache_context_length_reads_text_config() -> None: + model = SimpleNamespace(config=_ConfigWithText()) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length="auto", + fallback_extra_tokens=512, + ) == 128000 + + +def test_cache_context_length_falls_back_to_capture_size() -> None: + model = SimpleNamespace(config=SimpleNamespace()) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length=None, + fallback_extra_tokens=512, + ) == 2560 + + +def test_cache_context_length_rejects_non_positive_explicit_value() -> None: + with pytest.raises(ValueError): + _cache_context_length( + SimpleNamespace(config=SimpleNamespace()), + input_seq_len=2048, + cache_context_length="0", + fallback_extra_tokens=512, + ) def _write_gemma4_multimodal_config(output_dir: Path) -> None: output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "config.txt").write_text("model_type=gemma4\n", encoding="utf-8") (output_dir / "hf_config.json").write_text( ( '{"model_type":"gemma4",' @@ -21,7 +84,7 @@ def _write_gemma4_multimodal_config(output_dir: Path) -> None: def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[str]: - assets_dir = Path(model_mod.__file__).resolve().parent.parent / "assets" + default_images, default_audio = model_mod._default_multimodal_assets() return [ "--weights-dir", str(model_dir), @@ -38,9 +101,9 @@ def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[s "--components", "vision_encoder,audio_encoder,lm_encoder,decoder", "--image-file", - str(assets_dir / "test_monkey.png"), + default_images[0], "--audio-file", - str(assets_dir / "test.wav"), + str(default_audio), "--trust-remote-code", ] @@ -138,6 +201,41 @@ def _fake_cq_main(command): assert transpile_calls[0]["extra_args"] == _gemma4_multimodal_extra_args(output_dir, output_dir) +def test_cmd_convert_retranspile_reuses_weights_and_forwards_cache_context(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + output_dir = tmp_path / "gemma4" + _write_gemma4_multimodal_config(output_dir) + (output_dir / "components").mkdir() + (output_dir / "components" / "manifest.json").write_text("{}", encoding="utf-8") + (output_dir / "raw_ir_decoder_step.json").write_text("{}", encoding="utf-8") + args = parser.parse_args([ + "convert", + "google/gemma-4-E2B-it", + str(output_dir), + "--retranspile", + "--cache-context-length", + "131072", + ]) + + def _unexpected_cq_main(command): + raise AssertionError(f"unexpected CQ conversion: {command}") + + import cactus.convert.cli as cq_cli + monkeypatch.setattr(cq_cli, "main", _unexpected_cq_main) + + transpile_calls: list[dict] = [] + _patch_transpile(monkeypatch, transpile_calls) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + assert not (output_dir / "components").exists() + assert not (output_dir / "raw_ir_decoder_step.json").exists() + assert len(transpile_calls) == 1 + extra_args = transpile_calls[0]["extra_args"] + assert extra_args[extra_args.index("--cache-context-length") + 1] == "131072" + + def test_cmd_convert_supplies_default_audio_for_parakeet(monkeypatch, tmp_path: Path) -> None: parser = cli.create_parser() output_dir = tmp_path / "parakeet" @@ -163,7 +261,7 @@ def _fake_cq_main(command): extra_args = transpile_calls[0]["extra_args"] assert extra_args[extra_args.index("--task") + 1] == "tdt_transcription" assert "--audio-file" in extra_args - assert extra_args[extra_args.index("--audio-file") + 1].endswith("python/cactus/assets/test.wav") + assert Path(extra_args[extra_args.index("--audio-file") + 1]).name == "test.wav" def test_cmd_convert_supplies_default_audio_for_whisper(monkeypatch, tmp_path: Path) -> None: diff --git a/python/tests/test_optimize_gemma4_attention.py b/python/tests/test_optimize_gemma4_attention.py index 843e3f65c..137e58c5c 100644 --- a/python/tests/test_optimize_gemma4_attention.py +++ b/python/tests/test_optimize_gemma4_attention.py @@ -178,6 +178,57 @@ def test_normalize_gemma4_attention_recovers_layer_hints_from_graph_meta() -> No assert graph.nodes["attn_1"].attrs["window_size"] == 800 +def test_normalize_gemma4_cached_attention_sets_per_layer_cache_lengths() -> None: + graph = IRGraph( + values={ + "query_0": IRValue(id="query_0", shape=(1, 8, 1, 256), dtype="fp16"), + "key_0": IRValue(id="key_0", shape=(1, 8, 1, 256), dtype="fp16"), + "value_0": IRValue(id="value_0", shape=(1, 8, 1, 256), dtype="fp16"), + "out_0": IRValue(id="out_0", shape=(1, 8, 1, 256), dtype="fp16", producer="attn_0"), + "query_1": IRValue(id="query_1", shape=(1, 8, 1, 256), dtype="fp16"), + "key_1": IRValue(id="key_1", shape=(1, 8, 1, 256), dtype="fp16"), + "value_1": IRValue(id="value_1", shape=(1, 8, 1, 256), dtype="fp16"), + "out_1": IRValue(id="out_1", shape=(1, 8, 1, 256), dtype="fp16", producer="attn_1"), + }, + nodes={ + "attn_0": IRNode( + id="attn_0", + op="attention", + inputs=["query_0", "key_0", "value_0"], + outputs=["out_0"], + attrs={"is_causal": False, "window_size": 0}, + ), + "attn_1": IRNode( + id="attn_1", + op="attention", + inputs=["query_1", "key_1", "value_1"], + outputs=["out_1"], + attrs={"is_causal": False, "window_size": 0}, + ), + }, + order=["attn_0", "attn_1"], + inputs=["query_0", "key_0", "value_0", "query_1", "key_1", "value_1"], + outputs=["out_1"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder_step", + "use_internal_kv_cache": True, + "sliding_window": 512, + "max_cache_seq_len": 131072, + "layer_types": ("sliding_attention", "full_attention"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn_0"].attrs["window_size"] == 512 + assert graph.nodes["attn_0"].meta["max_cache_seq_len"] == 512 + assert graph.nodes["attn_1"].attrs["window_size"] == 0 + assert graph.nodes["attn_1"].meta["max_cache_seq_len"] == 131072 + + def test_normalize_gemma4_attention_elides_logical_fp16_mask() -> None: graph = IRGraph( values={