diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index b025cd0fe..8d6945109 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -292,6 +292,7 @@ CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, i CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_concat( cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, diff --git a/cactus-engine/src/graph_ffi.cpp b/cactus-engine/src/graph_ffi.cpp index 8e61d5fe0..bcb3d469f 100644 --- a/cactus-engine/src/graph_ffi.cpp +++ b/cactus-engine/src/graph_ffi.cpp @@ -515,6 +515,17 @@ int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus } } +int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_cumsum"); + try { + *out = static_cast(as_graph(graph)->graph.cumsum(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + int cactus_graph_concat(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, cactus_node_t* out) { if (!graph || !out) { last_error_message = "Invalid args to cactus_graph_concat"; diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index f43b715f3..1b552fa64 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -91,7 +91,7 @@ enum class OpType { ABS, POW, FLATTEN, VIEW, MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, BILINEAR_INTERPOLATION, - SUM, MEAN, VARIANCE, MIN, MAX, + SUM, MEAN, VARIANCE, MIN, MAX, CUMSUM, RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, @@ -497,6 +497,7 @@ class CactusGraph { size_t variance(size_t input, int axis); size_t min(size_t input, int axis); size_t max(size_t input, int axis); + size_t cumsum(size_t input, int axis); size_t softmax(size_t input, int axis = -1); size_t topk(size_t input, size_t k); diff --git a/cactus-graph/src/builder.cpp b/cactus-graph/src/builder.cpp index c0e2f92b7..952f29bb3 100644 --- a/cactus-graph/src/builder.cpp +++ b/cactus-graph/src/builder.cpp @@ -213,6 +213,23 @@ size_t CactusGraph::variance(size_t input, int axis) { return reduction_op(OpTyp size_t CactusGraph::min(size_t input, int axis) { return reduction_op(OpType::MIN, input, axis); } size_t CactusGraph::max(size_t input, int axis) { return reduction_op(OpType::MAX, input, axis); } +size_t CactusGraph::cumsum(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + if (input_buffer.shape.empty()) { + throw std::runtime_error("Cumsum requires at least one dimension"); + } + + int actual_axis = axis; + if (actual_axis < 0) { + actual_axis += static_cast(input_buffer.shape.size()); + } + if (actual_axis < 0 || static_cast(actual_axis) >= input_buffer.shape.size()) { + throw std::runtime_error("Invalid axis for cumsum operation"); + } + + return add_node(OpType::CUMSUM, {input}, input_buffer.shape, {.axis = actual_axis, .output_precision = input_buffer.precision}); +} + size_t CactusGraph::rms_norm(size_t input, size_t weight, float epsilon) { OpParams params{.epsilon = epsilon}; return add_node(OpType::RMS_NORM, {input, weight}, {}, params); diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index f2f7bc471..cc3e0c49c 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -118,6 +118,7 @@ static bool init_dispatch() { dispatch_flat[static_cast(OpType::VARIANCE)] = compute_reduce_node; dispatch_flat[static_cast(OpType::MIN)] = compute_reduce_node; dispatch_flat[static_cast(OpType::MAX)] = compute_reduce_node; + dispatch_flat[static_cast(OpType::CUMSUM)] = compute_reduce_node; dispatch_flat[static_cast(OpType::FLATTEN)] = compute_reshape_node; dispatch_flat[static_cast(OpType::VIEW)] = compute_reshape_node; dispatch_flat[static_cast(OpType::RESHAPE)] = compute_reshape_node; @@ -200,7 +201,7 @@ static const char* op_type_names[] = { "ABS", "POW", "FLATTEN", "VIEW", "MATMUL", "TRANSPOSE", "RESHAPE", "SLICE", "GATHER", "EMBEDDING", "BILINEAR_INTERPOLATION", - "SUM", "MEAN", "VARIANCE", "MIN", "MAX", + "SUM", "MEAN", "VARIANCE", "MIN", "MAX", "CUMSUM", "RMS_NORM", "ROPE", "ROPE_GPTJ", "SOFTMAX", "ATTENTION", "ATTENTION_INT8_HYBRID", "REL_POS_BIAS", "CONV1D_CAUSAL", "CONV1D_K3", "CONV1D_K7S3", "CONV1D", @@ -297,11 +298,20 @@ void CactusGraph::execute(const std::string& profile_file) { BufferPool& pool = buffer_pool_; const size_t n = nodes_.size(); + auto get_env_int = [](const char* name, int fallback) -> int { + const char* val = std::getenv(name); + return val ? std::atoi(val) : fallback; + }; + + bool trace_execution = get_env_int("CACTUS_TRACE_EXECUTE", 0) != 0; bool need_debug = !profile_file.empty(); if (!need_debug) { static const bool env_debug = check_debug_env(); need_debug = env_debug; } + if (trace_execution) { + need_debug = true; + } if (!need_debug) { for (size_t i = 0; i < n; ++i) { @@ -331,11 +341,6 @@ void CactusGraph::execute(const std::string& profile_file) { } } - auto get_env_int = [](const char* name, int fallback) -> int { - const char* val = std::getenv(name); - return val ? std::atoi(val) : fallback; - }; - auto get_env_str = [](const char* name) -> std::string { const char* val = std::getenv(name); return val ? std::string(val) : std::string(); @@ -390,11 +395,28 @@ void CactusGraph::execute(const std::string& profile_file) { for (size_t node_idx = 0; node_idx < n; ++node_idx) { auto& node = nodes_[node_idx]; - if (node->op_type != OpType::INPUT) { - node->output_buffer.allocate_from_pool(pool); + if (node->op_type == OpType::INPUT) { + continue; } - if (enable_profiling && node->op_type != OpType::INPUT) { + node->output_buffer.allocate_from_pool(pool); + + if (trace_execution) { + std::cerr << "[cactus:execute] begin idx=" << node_idx + << " id=" << node->id + << " op=" << get_op_name(node->op_type) + << " shape=["; + for (size_t dim_idx = 0; dim_idx < node->output_buffer.shape.size(); ++dim_idx) { + if (dim_idx > 0) std::cerr << ","; + std::cerr << node->output_buffer.shape[dim_idx]; + } + std::cerr << "]" << std::endl; + } + + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) { + dispatch_node(*node, nodes_, node_index_map_); + populated_node_ids_.insert(node->id); + } else if (enable_profiling) { auto start = std::chrono::high_resolution_clock::now(); dispatch_node(*node, nodes_, node_index_map_); if (node->op_type == OpType::PERSISTENT) { @@ -419,6 +441,13 @@ void CactusGraph::execute(const std::string& profile_file) { populated_node_ids_.insert(node->id); } } + + if (trace_execution) { + std::cerr << "[cactus:execute] done idx=" << node_idx + << " id=" << node->id + << " op=" << get_op_name(node->op_type) + << std::endl; + } } std::unique_ptr capture_file_stream; diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index bda707085..401603f52 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -220,6 +220,28 @@ void compute_reduce_node(GraphNode& node, const std::vector(axis) >= input_buffer.shape.size()) { + throw std::runtime_error("Cumsum axis out of range"); + } + + auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); + const __fp16* input = input_buffer.data_as<__fp16>(); + __fp16* output = node.output_buffer.data_as<__fp16>(); + + for (size_t outer = 0; outer < dims.outer; ++outer) { + for (size_t inner = 0; inner < dims.inner; ++inner) { + float running = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + size_t flat_index = ((outer * dims.axis_size) + axis_index) * dims.inner + inner; + running += static_cast(input[flat_index]); + output[flat_index] = static_cast<__fp16>(running); + } + } + } + return; + } + if (axis == -1) { switch (node.op_type) { case OpType::SUM: { diff --git a/cactus-graph/src/param_io.cpp b/cactus-graph/src/param_io.cpp index 599aa9203..91d8262a2 100644 --- a/cactus-graph/src/param_io.cpp +++ b/cactus-graph/src/param_io.cpp @@ -169,6 +169,7 @@ const Schema& op_schema(OpType op_type) { {OpType::VARIANCE, {{ParamField::Axis, FieldPersistence::Persistent}}}, {OpType::MIN, {{ParamField::Axis, FieldPersistence::Persistent}}}, {OpType::MAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::CUMSUM, {{ParamField::Axis, FieldPersistence::Persistent}}}, {OpType::INDEX, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::IndexValue, FieldPersistence::Persistent}}}, {OpType::CONCAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, {OpType::CAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, diff --git a/python/examples/compare_gemma4_multimodal.py b/python/examples/compare_gemma4_multimodal.py new file mode 100644 index 000000000..c176500b8 --- /dev/null +++ b/python/examples/compare_gemma4_multimodal.py @@ -0,0 +1,933 @@ +from __future__ import annotations + +import argparse +import copy +import gc +import json +import os +import sys +import time +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +EXAMPLES_DIR = Path(__file__).resolve().parent +PYTHON_ROOT = EXAMPLES_DIR.parent +PROJECT_ROOT = PYTHON_ROOT.parent + +sys.path.insert(0, str(PYTHON_ROOT)) +sys.path.insert(0, str(EXAMPLES_DIR)) + +from src.cactus import cactus_complete +from src.cactus import cactus_destroy +from src.cactus import cactus_init +from src.cactus import cactus_reset +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph +from transpile_hf_model import TranspileWrapper +from transpile_hf_model import _graph_to_dict +from transpile_hf_model import _load_transformers_bundle +from transpile_hf_model import _lower_preoptimized_ir +from transpile_hf_model import _parse_dtype +from transpile_hf_model import _prepare_gemma4_multimodal_inputs +from transpile_hf_model import _serialize_json_compatible +from transpile_hf_model import _validate_weights_dir +from transpile_hf_model import _write_json + + +_DEFAULT_STOP_SEQUENCES = ("", "", "", "<|im_end|>") + + +@dataclass +class NumericSummary: + mean: float + min: float + max: float + median: float + + +@dataclass +class HandwrittenRun: + wall_ms: float + total_time_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + total_tokens: int + response: str + response_json: dict[str, Any] + + +@dataclass +class TranspiledRun: + preprocess_ms: float + execute_ms: float + total_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + total_tokens: int + response: str + stop_reason: str + generated_token_ids: list[int] + input_shapes: dict[str, list[int]] + + +def _numeric_summary(values: list[float]) -> NumericSummary: + arr = np.asarray(values, dtype=np.float64) + return NumericSummary( + mean=float(np.mean(arr)), + min=float(np.min(arr)), + max=float(np.max(arr)), + median=float(np.median(arr)), + ) + + +def _roundtrip_jsonable(value: Any) -> Any: + return json.loads(json.dumps(_serialize_json_compatible(value))) + + +def _resolve_default_weights_dir(model_id: str) -> Path | None: + env_path = os.environ.get("CACTUS_TEST_GEMMA4_MODEL") + if env_path: + candidate = Path(env_path).resolve() + if (candidate / "config.txt").exists(): + return candidate + + candidates = ( + PROJECT_ROOT / "weights" / "gemma4_int4", + PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower(), + PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower().replace("_", "-"), + ) + for candidate in candidates: + if (candidate / "config.txt").exists(): + return candidate + return None + + +def _require_processor_tokenizer(processor_or_tokenizer: object) -> object: + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + if not hasattr(tokenizer_like, "decode") or not hasattr(tokenizer_like, "encode"): + raise RuntimeError( + "Gemma4 compare script requires a tokenizer-like object with encode/decode support." + ) + return tokenizer_like + + +def _build_messages_payload( + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + system_prompt: str = "", +) -> list[dict[str, object]]: + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system: + messages.append({"role": "system", "content": normalized_system}) + + user_message: dict[str, object] = { + "role": "user", + "content": prompt, + } + if image_files: + user_message["images"] = [str(Path(path).resolve()) for path in image_files] + if audio_file: + user_message["audio"] = [str(Path(audio_file).resolve())] + messages.append(user_message) + return messages + + +def _encode_stop_sequences(tokenizer: object, stop_sequences: tuple[str, ...]) -> list[list[int]]: + encoded: list[list[int]] = [] + encode = getattr(tokenizer, "encode", None) + if not callable(encode): + return encoded + for stop_sequence in stop_sequences: + try: + token_ids = list(encode(stop_sequence, add_special_tokens=False)) + except TypeError: + token_ids = list(encode(stop_sequence)) + if token_ids: + encoded.append([int(token_id) for token_id in token_ids]) + return encoded + + +def _has_suffix(token_ids: list[int], suffix: list[int]) -> bool: + if not suffix or len(token_ids) < len(suffix): + return False + return token_ids[-len(suffix) :] == suffix + + +def _trim_stop_suffix(token_ids: list[int], stop_sequences: list[list[int]]) -> bool: + for stop_sequence in stop_sequences: + if _has_suffix(token_ids, stop_sequence): + del token_ids[-len(stop_sequence) :] + return True + return False + + +def _decode_generated_text(tokenizer: object, token_ids: list[int], *, skip_special_tokens: bool) -> str: + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + raise RuntimeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") + try: + return str(decode(token_ids, skip_special_tokens=skip_special_tokens)) + except TypeError: + return str(decode(token_ids)) + + +def _summarize_handwritten_runs( + runs: list[HandwrittenRun], + *, + weights_dir: Path, +) -> dict[str, Any]: + if not runs: + raise ValueError("expected at least one handwritten run") + return { + "weights_dir": str(weights_dir), + "latest_response": runs[-1].response, + "total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), + "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), + "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], + } + + +def _summarize_transpiled_runs( + runs: list[TranspiledRun], + *, + compile_time_ms: float, + model_source: str, + weight_bindings: int, + prepared_input_shapes: dict[str, list[int]], +) -> dict[str, Any]: + if not runs: + raise ValueError("expected at least one transpiled run") + return { + "compile_time_ms": compile_time_ms, + "model_source": model_source, + "weight_bindings": weight_bindings, + "prepared_input_shapes": prepared_input_shapes, + "latest_response": runs[-1].response, + "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), + "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), + "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), + "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], + } + + +def _prepare_repeat_inputs( + processor: object, + *, + tokenizer: object, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str, + enable_thinking_if_supported: bool, + max_new_tokens: int, +) -> tuple[object, int, float]: + start = time.perf_counter() + prepared = _prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + use_gemma4_chat_template=True, + ) + pad_token_id = getattr(tokenizer, "pad_token_id", None) + if pad_token_id is None: + pad_token_id = 0 + prompt_tokens = 0 + for name, tensor in zip(prepared.names, prepared.tensors): + if name == "input_ids": + prompt_tokens = int(tensor.shape[1]) + break + if prompt_tokens <= 0: + raise RuntimeError("Gemma4 multimodal prepared inputs did not include a non-empty input_ids tensor") + target_tokens = prompt_tokens + int(max_new_tokens) + + input_tensors: list[torch.Tensor] = [] + for name, tensor in zip(prepared.names, prepared.tensors): + if name in {"input_ids", "attention_mask", "token_type_ids"}: + padded_shape = list(tensor.shape) + padded_shape[1] = target_tokens + if name == "input_ids": + padded = torch.full( + padded_shape, + int(pad_token_id), + dtype=tensor.dtype, + device=tensor.device, + ) + else: + padded = torch.zeros( + padded_shape, + dtype=tensor.dtype, + device=tensor.device, + ) + padded[:, : tensor.shape[1]] = tensor + input_tensors.append(padded) + else: + input_tensors.append(tensor) + metadata = dict(prepared.metadata) + metadata["prompt_token_count"] = prompt_tokens + metadata["target_token_count"] = int(target_tokens) + metadata["input_shapes"] = { + name: list(tensor.shape) + for name, tensor in zip(prepared.names, input_tensors) + } + prepared = type(prepared)( + names=prepared.names, + tensors=tuple(input_tensors), + metadata=metadata, + ) + preprocess_ms = (time.perf_counter() - start) * 1000.0 + return prepared, prompt_tokens, preprocess_ms + + +def _run_handwritten_once( + *, + model_handle: Any, + messages_json: str, + options_json: str, +) -> HandwrittenRun: + cactus_reset(model_handle) + streamed_token_ids: list[int] = [] + + def _collect(_text: str, token_id: int) -> None: + streamed_token_ids.append(int(token_id)) + + start = time.perf_counter() + response_json_text = cactus_complete( + model_handle, + messages_json, + options_json, + "[]", + _collect, + pcm_data=None, + ) + wall_ms = (time.perf_counter() - start) * 1000.0 + response_payload = json.loads(response_json_text) + return HandwrittenRun( + wall_ms=wall_ms, + total_time_ms=float(response_payload.get("total_time_ms", wall_ms)), + time_to_first_token_ms=float(response_payload.get("time_to_first_token_ms", 0.0)), + prefill_tps=float(response_payload.get("prefill_tps", 0.0)), + decode_tps=float(response_payload.get("decode_tps", 0.0)), + decode_tokens=len(streamed_token_ids), + total_tokens=int(response_payload.get("total_tokens", len(streamed_token_ids))), + response=str(response_payload.get("response", "")), + response_json=response_payload, + ) + + +def _run_transpiled_generation_once( + *, + tg, + processor: object, + tokenizer: object, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str, + enable_thinking_if_supported: bool, + max_new_tokens: int, + stop_sequences: tuple[str, ...], + expected_input_shapes: dict[str, list[int]] | None = None, + prepared_override: object | None = None, + progress_label: str = "transpiled", + progress_every_tokens: int = 8, +) -> TranspiledRun: + if prepared_override is not None: + prepared = type(prepared_override)( + names=prepared_override.names, + tensors=tuple(tensor.clone() for tensor in prepared_override.tensors), + metadata=dict(prepared_override.metadata), + ) + prompt_tokens = int(prepared.metadata.get("prompt_token_count", 0)) + preprocess_ms = 0.0 + else: + prepared, prompt_tokens, preprocess_ms = _prepare_repeat_inputs( + processor, + tokenizer=tokenizer, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + max_new_tokens=max_new_tokens, + ) + inputs_by_name = { + name: tensor.detach().cpu().numpy() + for name, tensor in zip(prepared.names, prepared.tensors) + } + if expected_input_shapes is not None: + for name, tensor in zip(prepared.names, prepared.tensors): + expected_shape = expected_input_shapes.get(name) + if expected_shape is None: + continue + actual_shape = list(tensor.shape) + if actual_shape != list(expected_shape): + raise RuntimeError( + f"runtime prepared input shape mismatch for {name}: " + f"compiled with {list(expected_shape)}, got {actual_shape}" + ) + if "input_ids" not in inputs_by_name: + raise RuntimeError("transpiled Gemma4 generation requires input_ids") + + target_tokens = int(prepared.metadata["target_token_count"]) + + eos_token_id = getattr(tokenizer, "eos_token_id", None) + + padded_inputs: dict[str, np.ndarray] = dict(inputs_by_name) + generated_ids: list[int] = [] + encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) + current_length = prompt_tokens + first_token_ms = 0.0 + execute_start = time.perf_counter() + stop_reason = "max_new_tokens" + progress_interval = max(1, int(progress_every_tokens)) + + for step_index in range(max_new_tokens): + runtime_inputs = [padded_inputs[name] for name in prepared.names] + if step_index == 0: + print("transpiled_step0_set_inputs_begin=true", flush=True) + tg.set_inputs(runtime_inputs) + if step_index == 0: + print("transpiled_step0_set_inputs_done=true", flush=True) + print("transpiled_step0_execute_begin=true", flush=True) + logits = tg.execute()[0].numpy() + if step_index == 0: + print("transpiled_step0_execute_done=true", flush=True) + if logits.ndim == 3: + token_position = current_length - 1 + if logits.shape[1] == 1: + token_position = 0 + next_token_id = int(np.argmax(logits[0, token_position])) + elif logits.ndim == 2: + next_token_id = int(np.argmax(logits[0])) + else: + raise RuntimeError(f"unexpected transpiled logits rank: {logits.ndim}") + generated_ids.append(next_token_id) + + if step_index == 0: + first_token_ms = preprocess_ms + (time.perf_counter() - execute_start) * 1000.0 + + if eos_token_id is not None and next_token_id == int(eos_token_id): + stop_reason = "eos_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + + if current_length >= target_tokens: + stop_reason = "context_limit" + break + + padded_inputs["input_ids"][0, current_length] = next_token_id + padded_inputs["attention_mask"][0, current_length] = 1 + padded_inputs["token_type_ids"][0, current_length] = 0 + if step_index == 0: + print("transpiled_step0_state_update_done=true", flush=True) + current_length += 1 + generated_count = len(generated_ids) + if generated_count % progress_interval == 0 or generated_count == max_new_tokens: + elapsed_ms = (time.perf_counter() - execute_start) * 1000.0 + print( + f"{progress_label}_decode_progress={generated_count}/{max_new_tokens} " + f"elapsed_ms={elapsed_ms:.3f}", + flush=True, + ) + + execute_ms = (time.perf_counter() - execute_start) * 1000.0 + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not response: + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() + decode_time_ms = max(0.0, execute_ms + preprocess_ms - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + prefill_tps = (prompt_tokens * 1000.0) / first_token_ms if first_token_ms > 0.0 else 0.0 + print( + f"{progress_label}_decode_done=true tokens={len(generated_ids)} " + f"stop_reason={stop_reason} execute_ms={execute_ms:.3f}", + flush=True, + ) + + return TranspiledRun( + preprocess_ms=preprocess_ms, + execute_ms=execute_ms, + total_ms=preprocess_ms + execute_ms, + time_to_first_token_ms=first_token_ms, + prefill_tps=prefill_tps, + decode_tps=decode_tps, + decode_tokens=len(generated_ids), + total_tokens=prompt_tokens + len(generated_ids), + response=response, + stop_reason=stop_reason, + generated_token_ids=generated_ids, + input_shapes={ + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + ) + + +def _print_summary(summary: dict[str, Any]) -> None: + print(f"model_id={summary['model_id']}") + print(f"prompt={summary['prompt']!r}") + if summary.get("image_files"): + print(f"image_files={summary['image_files']}") + if summary.get("audio_file"): + print(f"audio_file={summary['audio_file']}") + + handwritten = summary.get("handwritten") + if handwritten is not None: + hw_total = handwritten["total_time_ms"]["mean"] + hw_ttft = handwritten["time_to_first_token_ms"]["mean"] + hw_tokens = handwritten["decode_tokens"]["mean"] + hw_tps = hw_tokens * 1000.0 / max(hw_total, 1e-9) + print("handwritten:") + print(f" total_ms mean={hw_total:.3f}") + print(f" time_to_first_token_ms mean={hw_ttft:.3f}") + print(f" decode_tokens mean={hw_tokens:.3f}") + print(f" output_tokens_per_sec={hw_tps:.3f}") + print(f" latest_response={handwritten['latest_response']!r}") + + transpiled = summary.get("transpiled") + if transpiled is not None: + tr_total = transpiled["total_ms"]["mean"] + tr_ttft = transpiled["time_to_first_token_ms"]["mean"] + tr_tokens = transpiled["decode_tokens"]["mean"] + tr_tps = tr_tokens * 1000.0 / max(tr_total, 1e-9) + print("transpiled:") + print(f" compile_time_ms={transpiled['compile_time_ms']:.3f}") + print(f" total_ms mean={tr_total:.3f}") + print(f" time_to_first_token_ms mean={tr_ttft:.3f}") + print(f" decode_tokens mean={tr_tokens:.3f}") + print(f" output_tokens_per_sec={tr_tps:.3f}") + print(f" weight_bindings={transpiled['weight_bindings']}") + print(f" latest_response={transpiled['latest_response']!r}") + + comparison = summary.get("comparison") + if comparison is not None: + print("comparison:") + print( + " end_to_end_speedup_vs_handwritten=" + f"{comparison['end_to_end_speedup_vs_handwritten']:.6f}" + ) + print(f" responses_match={comparison['responses_match']}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Compare handwritten Cactus Gemma4 multimodal generation against the " + "transpiled Gemma4 multimodal graph on the same prompt, image, and audio." + ) + ) + parser.add_argument("--model-id", default="google/gemma-4-E2B-it") + parser.add_argument("--prompt", required=True) + parser.add_argument("--system-prompt", default="") + parser.add_argument("--image-file", action="append", default=[]) + parser.add_argument("--audio-file", default="") + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus weights directory. Required for handwritten comparison and reused for transpiled mmap bindings.", + ) + parser.add_argument("--artifact-dir", default="", help="Optional directory for summary and transpile artifacts.") + parser.add_argument("--graph-filename", default="graph.cactus") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=1) + parser.add_argument("--max-new-tokens", type=int, default=96) + parser.add_argument( + "--transpiled-progress-every-tokens", + type=int, + default=1, + help=( + "Emit transpiled decode progress every N generated tokens. " + "Use 1 for per-token visibility on long CPU-bound Gemma4 runs." + ), + ) + parser.add_argument("--torch-dtype", default="float16") + parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--skip-handwritten", action="store_true") + parser.add_argument("--skip-transpiled", action="store_true") + parser.add_argument("--enable-thinking", action="store_true") + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--top-p", type=float, default=1.0) + parser.add_argument("--top-k", type=int, default=1) + parser.add_argument("--stop-sequence", action="append", default=[]) + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + args = parser.parse_args() + + if args.skip_handwritten and args.skip_transpiled: + raise RuntimeError("cannot skip both handwritten and transpiled runs") + + image_files = tuple(str(Path(path).resolve()) for path in args.image_file if str(path).strip()) + audio_file = str(Path(args.audio_file).resolve()) if args.audio_file.strip() else None + if audio_file and not Path(audio_file).exists(): + raise FileNotFoundError(f"audio file does not exist: {audio_file}") + for image_file in image_files: + if not Path(image_file).exists(): + raise FileNotFoundError(f"image file does not exist: {image_file}") + + torch_dtype = _parse_dtype(args.torch_dtype) + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + + weights_dir_path = None + if args.weights_dir.strip(): + weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) + else: + weights_dir_path = _resolve_default_weights_dir(args.model_id) + + if not args.skip_handwritten and weights_dir_path is None: + raise RuntimeError( + "handwritten Gemma4 comparison requires a converted weights directory.\n" + "\n" + "Create one with something like:\n" + f" cactus convert {args.model_id} /path/to/gemma4_weights\n" + "\n" + "Then rerun with --weights-dir /path/to/gemma4_weights, or rerun with --skip-handwritten." + ) + + stop_sequences = tuple(args.stop_sequence) if args.stop_sequence else _DEFAULT_STOP_SEQUENCES + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + summary: dict[str, Any] = { + "model_id": args.model_id, + "prompt": args.prompt, + "system_prompt": args.system_prompt, + "image_files": list(image_files), + "audio_file": audio_file, + "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, + "warmup": args.warmup, + "repeats": args.repeats, + "max_new_tokens": args.max_new_tokens, + "transpiled_progress_every_tokens": int(args.transpiled_progress_every_tokens), + "torch_dtype": args.torch_dtype, + "trust_remote_code": bool(args.trust_remote_code), + "local_files_only": bool(args.local_files_only), + "enable_thinking": bool(args.enable_thinking), + "temperature": float(args.temperature), + "top_p": float(args.top_p), + "top_k": int(args.top_k), + "stop_sequences": list(stop_sequences), + } + + messages_payload = _build_messages_payload( + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + system_prompt=args.system_prompt, + ) + messages_json = json.dumps(messages_payload) + options_json = json.dumps( + { + "temperature": float(args.temperature), + "top_p": float(args.top_p), + "top_k": int(args.top_k), + "max_tokens": int(args.max_new_tokens), + "stop_sequences": list(stop_sequences), + "enable_thinking_if_supported": bool(args.enable_thinking), + "auto_handoff": False, + "telemetry_enabled": False, + } + ) + + transpiled_runs: list[TranspiledRun] = [] + if not args.skip_transpiled: + model_source, processor_or_tokenizer, model, _ = _load_transformers_bundle( + model_id=args.model_id, + task="multimodal_causal_lm_logits", + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + processor = processor_or_tokenizer + tokenizer = _require_processor_tokenizer(processor_or_tokenizer) + initial_prepared, _, initial_prepare_ms = _prepare_repeat_inputs( + processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + ) + canonical = canonicalize_model_interface( + model, + task="multimodal_causal_lm_logits", + input_names=initial_prepared.names, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + ) + if hasattr(canonical.module, "last_token_logits_only"): + canonical.module.last_token_logits_only = True + prime_static_features = getattr(canonical.module, "prime_static_multimodal_features", None) + if callable(prime_static_features): + prime_static_features(*initial_prepared.tensors) + wrapper = TranspileWrapper( + canonical.module, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + ).eval() + + compile_start = time.perf_counter() + print("transpile_capture_begin=true", flush=True) + prepare_cpu_float32_capture = getattr(canonical.module, "prepare_cpu_float32_capture", None) + restore_cpu_float32_capture = getattr(canonical.module, "restore_cpu_float32_capture", None) + if callable(prepare_cpu_float32_capture): + prepare_cpu_float32_capture() + try: + captured = capture_model(wrapper, initial_prepared.tensors) + finally: + if callable(restore_cpu_float32_capture): + restore_cpu_float32_capture() + print("transpile_capture_done=true", flush=True) + raw_ir_graph = copy.deepcopy(captured.ir_graph) + print("transpile_canonicalize_begin=true", flush=True) + canonicalize_exported_graph(captured.ir_graph) + print("transpile_canonicalize_done=true", flush=True) + print("transpile_optimize_begin=true", flush=True) + optimize_graph(captured.ir_graph, config=fusion_config) + print("transpile_optimize_done=true", flush=True) + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + print("transpile_lower_begin=true", flush=True) + tg = _lower_preoptimized_ir(captured.ir_graph) + print("transpile_lower_done=true", flush=True) + compile_time_ms = (time.perf_counter() - compile_start) * 1000.0 + + op_counts: dict[str, int] = {} + for node_id in optimized_ir_graph.order: + op = optimized_ir_graph.nodes[node_id].op + op_counts[op] = op_counts.get(op, 0) + 1 + weight_bindings = sum( + 1 + for value in optimized_ir_graph.values.values() + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) + ) + + summary["transpile_compile"] = { + "model_source": model_source, + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": weight_bindings, + "compile_time_ms": compile_time_ms, + "initial_prepare_ms": initial_prepare_ms, + "ops": op_counts, + } + prompt_token_count = int(initial_prepared.metadata.get("prompt_token_count", 0)) + target_token_count = int(initial_prepared.metadata.get("target_token_count", 0)) + total_transpiled_runs = int(args.warmup) + int(args.repeats) + print( + "transpiled_decode_plan=" + f"prompt_tokens:{prompt_token_count} " + f"target_tokens:{target_token_count} " + f"max_new_tokens:{int(args.max_new_tokens)} " + f"runs:{total_transpiled_runs} " + f"progress_every_tokens:{int(args.transpiled_progress_every_tokens)}", + flush=True, + ) + + if weights_dir_path is not None and weight_bindings == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir_path}\n" + "\n" + "The converted weights folder exists, but none of the captured constants matched weights_manifest.json.\n" + "Re-convert Gemma4 with the current converter before benchmarking." + ) + + if artifact_dir is not None: + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": "multimodal_causal_lm_logits", + "inputs": _serialize_json_compatible(initial_prepared.metadata), + "graph": _graph_to_dict(raw_ir_graph), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": "multimodal_causal_lm_logits", + "inputs": _serialize_json_compatible(initial_prepared.metadata), + "graph": _graph_to_dict(optimized_ir_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + if weight_bindings > 0: + skip_reason = ( + "skipped saved_graph because graph serialization does not yet preserve " + "mmap-bound weights/embeddings required by this transpiled Gemma4 graph" + ) + summary["transpile_compile"]["saved_graph_skipped_reason"] = skip_reason + print(f"note={skip_reason}") + else: + graph_path = artifact_dir / args.graph_filename + tg.graph.save(graph_path) + print(f"saved_graph={graph_path}") + + for _ in range(args.warmup): + warmup_index = _ + 1 + print(f"transpiled_warmup_begin={warmup_index}/{args.warmup}", flush=True) + _run_transpiled_generation_once( + tg=tg, + processor=processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + stop_sequences=stop_sequences, + expected_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + prepared_override=initial_prepared, + progress_label=f"transpiled_warmup{warmup_index}", + progress_every_tokens=args.transpiled_progress_every_tokens, + ) + for _ in range(args.repeats): + repeat_index = _ + 1 + print(f"transpiled_repeat_begin={repeat_index}/{args.repeats}", flush=True) + transpiled_runs.append( + _run_transpiled_generation_once( + tg=tg, + processor=processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + stop_sequences=stop_sequences, + expected_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + prepared_override=initial_prepared, + progress_label=f"transpiled_repeat{repeat_index}", + progress_every_tokens=args.transpiled_progress_every_tokens, + ) + ) + + summary["transpiled"] = _summarize_transpiled_runs( + transpiled_runs, + compile_time_ms=compile_time_ms, + model_source=model_source, + weight_bindings=weight_bindings, + prepared_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + ) + + # Keep the HF/export capture stack alive until the transpiled graph has + # completed execution. Gemma4 teardown before first execute has been + # unstable in practice, even though the lowered graph is already built. + del model + del wrapper + del canonical + del captured + gc.collect() + + handwritten_runs: list[HandwrittenRun] = [] + if not args.skip_handwritten and weights_dir_path is not None: + model_handle = cactus_init(str(weights_dir_path), None, False) + try: + for _ in range(args.warmup): + warmup_index = _ + 1 + print(f"handwritten_warmup_begin={warmup_index}/{args.warmup}", flush=True) + _run_handwritten_once( + model_handle=model_handle, + messages_json=messages_json, + options_json=options_json, + ) + for _ in range(args.repeats): + repeat_index = _ + 1 + print(f"handwritten_repeat_begin={repeat_index}/{args.repeats}", flush=True) + handwritten_runs.append( + _run_handwritten_once( + model_handle=model_handle, + messages_json=messages_json, + options_json=options_json, + ) + ) + finally: + cactus_destroy(model_handle) + summary["handwritten"] = _summarize_handwritten_runs( + handwritten_runs, + weights_dir=weights_dir_path, + ) + + if handwritten_runs and transpiled_runs: + handwritten_latest = handwritten_runs[-1].response.strip() + transpiled_latest = transpiled_runs[-1].response.strip() + summary["comparison"] = { + "end_to_end_speedup_vs_handwritten": ( + summary["handwritten"]["total_time_ms"]["mean"] + / max(summary["transpiled"]["total_ms"]["mean"], 1e-9) + ), + "responses_match": handwritten_latest == transpiled_latest, + "handwritten_latest_response": handwritten_latest, + "transpiled_latest_response": transpiled_latest, + } + + if artifact_dir is not None: + summary_path = artifact_dir / "summary.json" + summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") + print(f"saved_summary={summary_path}") + + _print_summary(summary) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/compare_parakeet_performance.py b/python/examples/compare_parakeet_performance.py new file mode 100644 index 000000000..621f1e1f5 --- /dev/null +++ b/python/examples/compare_parakeet_performance.py @@ -0,0 +1,793 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from collections import defaultdict +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from scipy.io import wavfile + +EXAMPLES_DIR = Path(__file__).resolve().parent +PYTHON_ROOT = EXAMPLES_DIR.parent +PROJECT_ROOT = PYTHON_ROOT.parent + +sys.path.insert(0, str(PYTHON_ROOT)) +sys.path.insert(0, str(EXAMPLES_DIR)) + +from src.cactus import cactus_destroy +from src.cactus import cactus_init +from src.cactus import cactus_transcribe +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph +from transpile_hf_model import TranspileWrapper +from transpile_hf_model import _ctc_greedy_decode_token_ids +from transpile_hf_model import _decode_token_ids +from transpile_hf_model import _infer_task_from_config +from transpile_hf_model import _load_optional_json +from transpile_hf_model import _load_optional_tokenizer +from transpile_hf_model import _load_transformers_bundle +from transpile_hf_model import _lower_preoptimized_ir +from transpile_hf_model import _parse_dtype +from transpile_hf_model import _prepare_audio_inputs +from transpile_hf_model import _validate_weights_dir + + +@dataclass +class NumericSummary: + mean: float + min: float + max: float + median: float + + +@dataclass +class ProfileEntry: + op: str + time_ms: float + shape: str + raw_line: str + + +@dataclass +class ProfileSection: + entries: list[ProfileEntry] + total_ms: float + + +@dataclass +class HandwrittenRun: + wall_ms: float + total_time_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + prefill_tokens: int + total_tokens: int + confidence: float + transcript: str + response_json: dict[str, Any] + + +@dataclass +class TranspiledRun: + preprocess_ms: float + execute_ms: float + decode_ms: float + total_ms: float + graph_only_ms: float + output_frames: int + output_vocab: int + decode_tokens: int + transcript: str | None + input_shapes: dict[str, list[int]] + + +def _numeric_summary(values: list[float]) -> NumericSummary: + arr = np.asarray(values, dtype=np.float64) + return NumericSummary( + mean=float(np.mean(arr)), + min=float(np.min(arr)), + max=float(np.max(arr)), + median=float(np.median(arr)), + ) + + +def _resolve_default_weights_dir(model_id: str) -> Path | None: + candidate = PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower() + if (candidate / "config.txt").exists(): + return candidate + return None + + +def _audio_duration_seconds(audio_file: str) -> float: + sample_rate, samples = wavfile.read(audio_file) + if sample_rate <= 0: + raise ValueError(f"invalid sample rate in {audio_file}: {sample_rate}") + return float(len(samples)) / float(sample_rate) + + +def _profile_env(profile_file: Path | None): + class _ProfileContext: + def __enter__(self_inner): + self_inner.old_profile = os.environ.get("CACTUS_PROFILE") + self_inner.old_profile_file = os.environ.get("CACTUS_PROFILE_FILE") + if profile_file is None: + os.environ.pop("CACTUS_PROFILE", None) + os.environ.pop("CACTUS_PROFILE_FILE", None) + return self_inner + profile_file.parent.mkdir(parents=True, exist_ok=True) + if profile_file.exists(): + profile_file.unlink() + os.environ.pop("CACTUS_PROFILE", None) + os.environ["CACTUS_PROFILE_FILE"] = str(profile_file) + return self_inner + + def __exit__(self_inner, exc_type, exc, tb): + if self_inner.old_profile is None: + os.environ.pop("CACTUS_PROFILE", None) + else: + os.environ["CACTUS_PROFILE"] = self_inner.old_profile + + if self_inner.old_profile_file is None: + os.environ.pop("CACTUS_PROFILE_FILE", None) + else: + os.environ["CACTUS_PROFILE_FILE"] = self_inner.old_profile_file + return False + + return _ProfileContext() + + +def _parse_profile_file(path: Path) -> list[ProfileSection]: + if not path.exists(): + return [] + + lines = path.read_text().splitlines() + sections: list[ProfileSection] = [] + idx = 0 + while idx < len(lines): + if lines[idx].strip() != "=== Graph Execution Profile ===": + idx += 1 + continue + + entries: list[ProfileEntry] = [] + total_ms = 0.0 + idx += 1 + while idx < len(lines): + line = lines[idx].rstrip() + stripped = line.strip() + if stripped.startswith("Total execution time:"): + try: + total_ms = float(stripped.split(":", 1)[1].strip().split()[0]) + except Exception: + total_ms = 0.0 + elif stripped == "================================": + break + elif ( + stripped + and not stripped.startswith("Operation") + and not set(stripped).issubset({"-"}) + and not stripped.startswith("Total execution time:") + ): + parts = stripped.split() + if len(parts) >= 3: + op = parts[0] + try: + time_ms = float(parts[1]) + except ValueError: + time_ms = 0.0 + shape = parts[2] if parts[2].startswith("[") else "" + entries.append(ProfileEntry(op=op, time_ms=time_ms, shape=shape, raw_line=stripped)) + idx += 1 + + sections.append(ProfileSection(entries=entries, total_ms=total_ms)) + idx += 1 + + return sections + + +def _summarize_profile_sections(sections: list[ProfileSection]) -> dict[str, Any]: + op_total_ms: dict[str, float] = defaultdict(float) + op_count: dict[str, int] = defaultdict(int) + all_entries: list[ProfileEntry] = [] + total_ms_values: list[float] = [] + + for section in sections: + total_ms_values.append(section.total_ms) + all_entries.extend(section.entries) + for entry in section.entries: + op_total_ms[entry.op] += entry.time_ms + op_count[entry.op] += 1 + + by_op = [] + for op, total_ms in sorted(op_total_ms.items(), key=lambda item: item[1], reverse=True): + count = op_count[op] + by_op.append( + { + "op": op, + "count": count, + "total_ms": round(total_ms, 6), + "mean_ms": round(total_ms / max(count, 1), 6), + } + ) + + slowest_nodes = [ + { + "op": entry.op, + "time_ms": round(entry.time_ms, 6), + "shape": entry.shape, + "raw_line": entry.raw_line, + } + for entry in sorted(all_entries, key=lambda item: item.time_ms, reverse=True)[:50] + ] + + return { + "runs_profiled": len(sections), + "graph_total_ms": asdict(_numeric_summary(total_ms_values)) if total_ms_values else None, + "entries_per_run": [len(section.entries) for section in sections], + "by_op": by_op, + "slowest_nodes": slowest_nodes, + } + + +def _run_handwritten_once( + *, + model_handle: Any, + audio_file: str, +) -> HandwrittenRun: + options_json = json.dumps( + { + "use_vad": False, + "max_tokens": 4096, + } + ) + start = time.perf_counter() + response = cactus_transcribe( + model_handle, + audio_file, + "", + options_json, + None, + None, + ) + end = time.perf_counter() + + payload = json.loads(response) + return HandwrittenRun( + wall_ms=(end - start) * 1000.0, + total_time_ms=float(payload.get("total_time_ms", 0.0)), + time_to_first_token_ms=float(payload.get("time_to_first_token_ms", 0.0)), + prefill_tps=float(payload.get("prefill_tps", 0.0)), + decode_tps=float(payload.get("decode_tps", 0.0)), + decode_tokens=int(payload.get("decode_tokens", 0)), + prefill_tokens=int(payload.get("prefill_tokens", 0)), + total_tokens=int(payload.get("total_tokens", 0)), + confidence=float(payload.get("confidence", 0.0)), + transcript=str(payload.get("response", "")), + response_json=payload, + ) + + +def _run_transpiled_once( + *, + tg: Any, + processor: object | None, + input_names: tuple[str, ...], + model_config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + torch_dtype: torch.dtype, + audio_file: str, + tokenizer: object | None, + blank_token_id: int | None, +) -> TranspiledRun: + start_pre = time.perf_counter() + prepared = _prepare_audio_inputs( + processor, + input_names=input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task="ctc_logits", + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + end_pre = time.perf_counter() + + input_arrays = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] + + start_exec = time.perf_counter() + tg.set_inputs(input_arrays) + outputs = tg.execute() + end_exec = time.perf_counter() + + start_decode = time.perf_counter() + logits = outputs[0].numpy().astype(np.float32) + token_ids = _ctc_greedy_decode_token_ids(logits, blank_token_id=blank_token_id) + transcript = _decode_token_ids(tokenizer, token_ids) if tokenizer is not None else None + end_decode = time.perf_counter() + + return TranspiledRun( + preprocess_ms=(end_pre - start_pre) * 1000.0, + execute_ms=(end_exec - start_exec) * 1000.0, + decode_ms=(end_decode - start_decode) * 1000.0, + total_ms=(end_decode - start_pre) * 1000.0, + graph_only_ms=(end_exec - start_exec) * 1000.0, + output_frames=int(logits.shape[1]), + output_vocab=int(logits.shape[2]), + decode_tokens=len(token_ids), + transcript=transcript, + input_shapes={ + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + ) + + +def _run_transpiled_graph_only_once( + *, + tg: Any, + cached_inputs: list[np.ndarray], +) -> float: + start = time.perf_counter() + tg.set_inputs(cached_inputs) + tg.execute() + end = time.perf_counter() + return (end - start) * 1000.0 + + +def _warmup(fn, count: int) -> None: + for _ in range(max(count, 0)): + fn() + + +def _roundtrip_jsonable(obj: Any) -> Any: + if isinstance(obj, dict): + return {str(key): _roundtrip_jsonable(value) for key, value in obj.items()} + if isinstance(obj, list): + return [_roundtrip_jsonable(value) for value in obj] + if isinstance(obj, tuple): + return [_roundtrip_jsonable(value) for value in obj] + if isinstance(obj, Path): + return str(obj) + if isinstance(obj, NumericSummary): + return asdict(obj) + if hasattr(obj, "__dataclass_fields__"): + return asdict(obj) + return obj + + +def _build_transpiled_graph( + *, + model_id: str, + audio_file: str, + weights_dir: str | None, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, + fusion_config: FusionConfig, +): + task = _infer_task_from_config(model_id) + if task != "ctc_logits": + raise RuntimeError(f"expected a CTC model, got task={task}") + + model_source, processor, model, model_config = _load_transformers_bundle( + model_id=model_id, + task=task, + torch_dtype=torch_dtype, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) + + preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") + if not preprocessor_config: + preprocessor_config = _load_optional_json(model_id, "preprocessor_config.json") + + tokenizer = _load_optional_tokenizer( + model_id=model_id, + model_source=model_source, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) + + canonical = canonicalize_model_interface(model, task=task) + prepared = _prepare_audio_inputs( + processor, + input_names=canonical.input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface(model, task=task, input_names=prepared.names) + + wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + + capture_started = time.perf_counter() + captured = capture_model(wrapper, prepared.tensors) + canonicalize_exported_graph(captured.ir_graph) + optimize_graph(captured.ir_graph, config=fusion_config) + tg = _lower_preoptimized_ir(captured.ir_graph) + capture_finished = time.perf_counter() + + cached_inputs = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] + blank_token_id = getattr(getattr(model, "config", None), "pad_token_id", None) + + return { + "tg": tg, + "model": model, + "processor": processor, + "model_config": model_config, + "preprocessor_config": preprocessor_config, + "tokenizer": tokenizer, + "blank_token_id": int(blank_token_id) if blank_token_id is not None else None, + "compile_time_ms": (capture_finished - capture_started) * 1000.0, + "cached_inputs": cached_inputs, + "input_names": tuple(prepared.names), + "prepared_input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + "model_source": model_source, + "weight_bindings": sum( + 1 + for value in captured.ir_graph.values.values() + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) + ), + } + + +def _benchmark_handwritten( + *, + weights_dir: Path, + audio_file: str, + warmup: int, + repeats: int, + profile_repeats: int, + artifact_dir: Path | None, +) -> dict[str, Any]: + model_handle = cactus_init(str(weights_dir), None, False) + try: + _warmup(lambda: _run_handwritten_once(model_handle=model_handle, audio_file=audio_file), warmup) + + runs: list[HandwrittenRun] = [] + for _ in range(max(repeats, 1)): + runs.append(_run_handwritten_once(model_handle=model_handle, audio_file=audio_file)) + + profile_path = ( + artifact_dir / "handwritten_profile.txt" + if artifact_dir is not None and profile_repeats > 0 + else None + ) + with _profile_env(profile_path): + for _ in range(max(profile_repeats, 0)): + _run_handwritten_once(model_handle=model_handle, audio_file=audio_file) + + profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] + profile_summary = _summarize_profile_sections(profile_sections) + + return { + "weights_dir": str(weights_dir), + "runs": [_roundtrip_jsonable(run) for run in runs], + "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), + "internal_total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "confidence": asdict(_numeric_summary([run.confidence for run in runs])), + "latest_transcript": runs[-1].transcript if runs else "", + "profile_file": str(profile_path) if profile_path is not None else None, + "profile_summary": profile_summary, + } + finally: + cactus_destroy(model_handle) + + +def _benchmark_transpiled( + *, + model_id: str, + audio_file: str, + weights_dir: str | None, + warmup: int, + repeats: int, + profile_repeats: int, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, + artifact_dir: Path | None, + fusion_config: FusionConfig, +) -> dict[str, Any]: + built = _build_transpiled_graph( + model_id=model_id, + audio_file=audio_file, + weights_dir=weights_dir, + torch_dtype=torch_dtype, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + fusion_config=fusion_config, + ) + + tg = built["tg"] + model = built["model"] + processor = built["processor"] + input_names = built["input_names"] + model_config = built["model_config"] + preprocessor_config = built["preprocessor_config"] + tokenizer = built["tokenizer"] + blank_token_id = built["blank_token_id"] + cached_inputs = built["cached_inputs"] + + _warmup( + lambda: _run_transpiled_once( + tg=tg, + processor=processor, + input_names=input_names, + model_config=model_config, + preprocessor_config=preprocessor_config, + model=model, + torch_dtype=torch_dtype, + audio_file=audio_file, + tokenizer=tokenizer, + blank_token_id=blank_token_id, + ), + warmup, + ) + _warmup(lambda: _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs), warmup) + + runs: list[TranspiledRun] = [] + graph_only_ms: list[float] = [] + for _ in range(max(repeats, 1)): + runs.append( + _run_transpiled_once( + tg=tg, + processor=processor, + input_names=input_names, + model_config=model_config, + preprocessor_config=preprocessor_config, + model=model, + torch_dtype=torch_dtype, + audio_file=audio_file, + tokenizer=tokenizer, + blank_token_id=blank_token_id, + ) + ) + graph_only_ms.append(_run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs)) + + profile_path = ( + artifact_dir / "transpiled_profile.txt" + if artifact_dir is not None and profile_repeats > 0 + else None + ) + with _profile_env(profile_path): + for _ in range(max(profile_repeats, 0)): + _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs) + + profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] + profile_summary = _summarize_profile_sections(profile_sections) + + return { + "model_source": built["model_source"], + "compile_time_ms": built["compile_time_ms"], + "weight_bindings": built["weight_bindings"], + "prepared_input_shapes": built["prepared_input_shapes"], + "runs": [_roundtrip_jsonable(run) for run in runs], + "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), + "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), + "decode_ms": asdict(_numeric_summary([run.decode_ms for run in runs])), + "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), + "graph_only_ms": asdict(_numeric_summary(graph_only_ms)), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "output_frames": asdict(_numeric_summary([float(run.output_frames) for run in runs])), + "latest_transcript": runs[-1].transcript if runs else "", + "profile_file": str(profile_path) if profile_path is not None else None, + "profile_summary": profile_summary, + } + + +def _print_summary( + *, + audio_duration_sec: float, + handwritten: dict[str, Any] | None, + transpiled: dict[str, Any] | None, +) -> None: + print() + print(f"audio_duration_sec={audio_duration_sec:.3f}") + + if handwritten is not None: + hw_total = handwritten["internal_total_time_ms"]["mean"] + hw_graph = None + profile_total = handwritten.get("profile_summary", {}).get("graph_total_ms") + if isinstance(profile_total, dict): + hw_graph = profile_total.get("mean") + hw_tok = handwritten["decode_tokens"]["mean"] + hw_rtf = hw_total / max(audio_duration_sec * 1000.0, 1e-9) + hw_xrt = (audio_duration_sec * 1000.0) / max(hw_total, 1e-9) + hw_tok_s = hw_tok * 1000.0 / max(hw_total, 1e-9) + print("handwritten:") + print(f" total_ms mean={hw_total:.3f}") + print(f" graph_profile_ms mean={hw_graph:.3f}" if hw_graph is not None else " graph_profile_ms unavailable") + print(f" decode_tokens mean={hw_tok:.3f}") + print(f" output_tokens_per_sec={hw_tok_s:.3f}") + print(f" real_time_factor={hw_rtf:.3f}") + print(f" audio_x_realtime={hw_xrt:.3f}") + + if transpiled is not None: + tr_total = transpiled["total_ms"]["mean"] + tr_graph = transpiled["graph_only_ms"]["mean"] + tr_tok = transpiled["decode_tokens"]["mean"] + tr_rtf = tr_total / max(audio_duration_sec * 1000.0, 1e-9) + tr_xrt = (audio_duration_sec * 1000.0) / max(tr_total, 1e-9) + tr_tok_s = tr_tok * 1000.0 / max(tr_total, 1e-9) + tr_graph_tok_s = tr_tok * 1000.0 / max(tr_graph, 1e-9) + print("transpiled:") + print(f" total_ms mean={tr_total:.3f}") + print(f" graph_only_ms mean={tr_graph:.3f}") + print(f" decode_tokens mean={tr_tok:.3f}") + print(f" output_tokens_per_sec={tr_tok_s:.3f}") + print(f" graph_only_tokens_per_sec={tr_graph_tok_s:.3f}") + print(f" real_time_factor={tr_rtf:.3f}") + print(f" audio_x_realtime={tr_xrt:.3f}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Compare handwritten Cactus Parakeet performance against the transpiled " + "Parakeet graph on the same WAV input, including per-op graph profiles." + ) + ) + parser.add_argument("--model-id", default="nvidia/parakeet-ctc-1.1b") + parser.add_argument("--audio-file", required=True) + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus weights directory. Reused for handwritten runtime and transpiled mmap bindings.", + ) + parser.add_argument("--artifact-dir", default="", help="Optional directory for JSON summaries and raw profiles.") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--profile-repeats", type=int, default=1) + parser.add_argument("--torch-dtype", default="float16") + parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--skip-handwritten", action="store_true") + parser.add_argument("--skip-transpiled", action="store_true") + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + args = parser.parse_args() + + audio_file = str(Path(args.audio_file).resolve()) + if not Path(audio_file).exists(): + raise FileNotFoundError(f"audio file does not exist: {audio_file}") + + torch_dtype = _parse_dtype(args.torch_dtype) + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + + weights_dir_path = None + if args.weights_dir.strip(): + weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) + else: + weights_dir_path = _resolve_default_weights_dir(args.model_id) + + if not args.skip_handwritten and weights_dir_path is None: + raise RuntimeError( + "handwritten comparison requires a converted weights directory.\n" + "\n" + f"Create one with:\n" + f" cactus convert {args.model_id} {PROJECT_ROOT / 'weights' / args.model_id.split('/')[-1].lower()}\n" + "\n" + "Or rerun with --skip-handwritten." + ) + + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + summary: dict[str, Any] = { + "model_id": args.model_id, + "audio_file": audio_file, + "audio_duration_sec": _audio_duration_seconds(audio_file), + "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, + "warmup": args.warmup, + "repeats": args.repeats, + "profile_repeats": args.profile_repeats, + "torch_dtype": args.torch_dtype, + "trust_remote_code": bool(args.trust_remote_code), + "local_files_only": bool(args.local_files_only), + } + + handwritten_summary = None + if not args.skip_handwritten and weights_dir_path is not None: + handwritten_summary = _benchmark_handwritten( + weights_dir=weights_dir_path, + audio_file=audio_file, + warmup=args.warmup, + repeats=args.repeats, + profile_repeats=args.profile_repeats, + artifact_dir=artifact_dir, + ) + summary["handwritten"] = handwritten_summary + + transpiled_summary = None + if not args.skip_transpiled: + transpiled_summary = _benchmark_transpiled( + model_id=args.model_id, + audio_file=audio_file, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + warmup=args.warmup, + repeats=args.repeats, + profile_repeats=args.profile_repeats, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + artifact_dir=artifact_dir, + fusion_config=fusion_config, + ) + summary["transpiled"] = transpiled_summary + + if handwritten_summary is not None and transpiled_summary is not None: + hw_total = handwritten_summary["internal_total_time_ms"]["mean"] + tr_total = transpiled_summary["total_ms"]["mean"] + hw_graph = handwritten_summary.get("profile_summary", {}).get("graph_total_ms") + tr_graph = transpiled_summary["graph_only_ms"]["mean"] + summary["comparison"] = { + "end_to_end_speedup_vs_handwritten": hw_total / max(tr_total, 1e-9), + "graph_only_speedup_vs_handwritten": ( + float(hw_graph["mean"]) / max(tr_graph, 1e-9) + if isinstance(hw_graph, dict) and "mean" in hw_graph + else None + ), + } + + if artifact_dir is not None: + summary_path = artifact_dir / "summary.json" + summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") + print(f"saved_summary={summary_path}") + if handwritten_summary is not None and handwritten_summary.get("profile_file"): + print(f"saved_handwritten_profile={handwritten_summary['profile_file']}") + if transpiled_summary is not None and transpiled_summary.get("profile_file"): + print(f"saved_transpiled_profile={transpiled_summary['profile_file']}") + + _print_summary( + audio_duration_sec=float(summary["audio_duration_sec"]), + handwritten=handwritten_summary, + transpiled=transpiled_summary, + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/inspect_gemma4_cleanup.py b/python/examples/inspect_gemma4_cleanup.py new file mode 100644 index 000000000..ec9e78d05 --- /dev/null +++ b/python/examples/inspect_gemma4_cleanup.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +import torch +from transformers import AutoModelForCausalLM + +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.model_adapters import canonicalize_model_interface + + +class Gemma4FullModelWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = canonicalize_model_interface(model, task="causal_lm_logits").module + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model(input_ids) + + +class Gemma4FirstBlockCheckpointWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module, checkpoint_name: str): + super().__init__() + adapter = canonicalize_model_interface(model, task="causal_lm_logits") + if adapter.family != "gemma4": + raise ValueError(f"expected gemma4 adapter, got family={adapter.family}") + if not hasattr(adapter.module, "debug_first_block"): + raise ValueError("Gemma4 adapter does not expose debug_first_block()") + self.adapter = adapter.module + self.checkpoint_name = checkpoint_name + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + checkpoints = self.adapter.debug_first_block(input_ids) + if self.checkpoint_name not in checkpoints: + available = ", ".join(sorted(checkpoints.keys())) + raise KeyError(f"unknown checkpoint {self.checkpoint_name!r}; available: {available}") + return checkpoints[self.checkpoint_name] + + +def format_ir_graph(graph: IRGraph, *, max_nodes: int | None = None) -> str: + lines: list[str] = [] + lines.append("IR Summary") + lines.append(f" inputs={graph.inputs}") + lines.append(f" outputs={graph.outputs}") + lines.append(f" nodes={len(graph.order)}") + if graph.meta: + lines.append(f" meta={graph.meta}") + lines.append("") + lines.append("IR Nodes") + + node_ids = graph.order if max_nodes is None else graph.order[:max_nodes] + for index, node_id in enumerate(node_ids): + node = graph.nodes[node_id] + lines.append(f"[{index}] {node.id} op={node.op}") + lines.append(f" inputs={node.inputs}") + lines.append(f" outputs={node.outputs}") + if node.attrs: + lines.append(f" attrs={node.attrs}") + if node.kind != "generic": + lines.append(f" kind={node.kind}") + if node.meta: + lines.append(f" meta={node.meta}") + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + lines.append( + f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" + ) + + if max_nodes is not None and len(graph.order) > max_nodes: + lines.append(f"... ({len(graph.order) - max_nodes} more nodes omitted)") + + if graph.constants: + lines.append("") + lines.append("IR Constants") + constant_ids = sorted(graph.constants.keys()) + preview_ids = constant_ids if max_nodes is None else constant_ids[: max(8, min(len(constant_ids), max_nodes))] + for value_id in preview_ids: + value = graph.constants[value_id] + value_meta = graph.values.get(value_id) + shape = None if value_meta is None else value_meta.shape + dtype = None if value_meta is None else value_meta.dtype + lines.append(f"- {value_id}: type={type(value).__name__} shape={shape} dtype={dtype}") + if len(preview_ids) < len(constant_ids): + lines.append(f"... ({len(constant_ids) - len(preview_ids)} more constants omitted)") + + return "\n".join(lines) + + +def format_unsupported_ops(graph: IRGraph) -> str: + counts = graph.meta.get("canonical_unsupported_op_counts", {}) + if not isinstance(counts, dict): + counts = {} + + examples: dict[str, list[str]] = {} + for node_id in graph.order: + node = graph.nodes[node_id] + print(f"node {node_id} op={node.op}") + if node.op not in counts: + continue + bucket = examples.setdefault(node.op, []) + if len(bucket) < 5: + bucket.append(node.id) + + total = sum(int(count) for count in counts.values()) + lines = ["Unsupported Ops", f" unique={len(counts)}", f" total_nodes={total}"] + if not counts: + lines.append(" none") + return "\n".join(lines) + + for op_name, count in counts.items(): + sample_nodes = ", ".join(examples.get(op_name, ())) + lines.append(f"- {op_name}: {count}") + if sample_nodes: + lines.append(f" examples={sample_nodes}") + return "\n".join(lines) + + +def resolve_local_model_path(model_id: str) -> str: + model_path = Path(model_id) + if model_path.exists(): + return str(model_path) + + cache_root = Path.home() / ".cache" / "huggingface" / "hub" / ("models--" + model_id.replace("/", "--")) + snapshots_dir = cache_root / "snapshots" + if not snapshots_dir.exists(): + raise FileNotFoundError( + f"could not find local snapshot for {model_id!r} under {snapshots_dir}" + ) + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + raise FileNotFoundError(f"no snapshots found for {model_id!r} under {snapshots_dir}") + return str(snapshots[-1]) + + +def load_model(model_id: str) -> torch.nn.Module: + token = os.environ.get("HF_TOKEN") + common_kwargs: dict[str, object] = { + "local_files_only": True, + } + if token: + common_kwargs["token"] = token + + local_model_path = resolve_local_model_path(model_id) + model = AutoModelForCausalLM.from_pretrained( + local_model_path, + torch_dtype=torch.float16, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model + + +def build_module( + model: torch.nn.Module, + *, + mode: str, + checkpoint: str, +) -> torch.nn.Module: + if mode == "full": + return Gemma4FullModelWrapper(model).eval() + if mode == "first_block": + return Gemma4FirstBlockCheckpointWrapper(model, checkpoint).eval() + raise ValueError(f"unknown mode: {mode}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture Gemma4, run canonicalize/cleanup.py, and report unsupported ops.", + ) + parser.add_argument( + "--model-id", + default=os.environ.get("CACTUS_GEMMA_HF_MODEL_ID", "google/gemma-4-E2B"), + help="Hugging Face model id. Uses local cache only.", + ) + parser.add_argument( + "--mode", + choices=("first_block", "full"), + default="first_block", + help="Capture a first-block checkpoint or the full Gemma4 adapter graph.", + ) + parser.add_argument( + "--checkpoint", + default="after_ffn_residual", + help="First-block checkpoint name when --mode=first_block.", + ) + parser.add_argument( + "--input-ids", + default="2,818,5279,529,7001,563", + help="Comma-separated integer token ids to capture.", + ) + parser.add_argument( + "--max-nodes", + type=int, + default=120, + help="Maximum IR nodes to print when --print-ir is enabled.", + ) + parser.add_argument( + "--print-ir", + action="store_true", + help="Also print the IR before and after canonical cleanup.", + ) + args = parser.parse_args() + + model = load_model(args.model_id) + module = build_module(model, mode=args.mode, checkpoint=args.checkpoint) + token_ids = [int(token.strip()) for token in args.input_ids.split(",") if token.strip()] + input_ids = torch.tensor([token_ids], dtype=torch.long) + + print(f"Model: {args.model_id}") + print(f"Mode: {args.mode}") + print(f"Checkpoint: {args.checkpoint if args.mode == 'first_block' else '(full model)'}") + print(f"Input IDs shape: {tuple(input_ids.shape)}") + print("") + + captured = capture_model(module, (input_ids,)) + + if args.print_ir: + print("=== IR Before Canonical Cleanup ===") + print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) + print("") + + canonicalize_exported_graph(captured.ir_graph) + + print("=== Unsupported Ops After Canonical Cleanup ===") + print(format_unsupported_ops(captured.ir_graph)) + + if args.print_ir: + print("") + print("=== IR After Canonical Cleanup ===") + print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) + + +if __name__ == "__main__": + main() diff --git a/python/examples/inspect_transpile_cleanup.py b/python/examples/inspect_transpile_cleanup.py new file mode 100644 index 000000000..366765f62 --- /dev/null +++ b/python/examples/inspect_transpile_cleanup.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import argparse + +import torch +import torch.nn as nn + +from src.transpile.capture_pytorch import capture_model +from src.transpile.capture_pytorch import dump_graph +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph + + +class Toy(nn.Module): + def forward(self, x, y): + z = x + y + z = z * 0.5 + z = torch.nn.functional.gelu(z) + z = torch.softmax(z, dim=-1) + return z + + +class AttentionBlockToy(nn.Module): + def __init__(self, hidden_size: int = 8): + super().__init__() + self.out_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=torch.float16) + + def forward(self, q, k, v, gate): + attn = torch.nn.functional.scaled_dot_product_attention(q, k, v, is_causal=True, scale=1.0) + attn = attn.transpose(1, 2).reshape(q.shape[0], q.shape[2], -1) + attn = attn * torch.sigmoid(gate) + attn = attn.to(self.out_proj.weight.dtype) + return self.out_proj(attn) + + +def build_case(name: str) -> tuple[nn.Module, tuple[torch.Tensor, ...]]: + if name == "toy": + model = Toy().eval() + args = ( + torch.randn(2, 4, dtype=torch.float16), + torch.randn(2, 4, dtype=torch.float16), + ) + return model, args + + if name == "attention_block": + model = AttentionBlockToy().eval() + args = ( + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 3, 8, dtype=torch.float16), + ) + return model, args + + raise ValueError(f"unknown case: {name}") + + +def format_ir_graph(graph: IRGraph) -> str: + lines: list[str] = [] + lines.append("IR Summary") + lines.append(f" inputs={graph.inputs}") + lines.append(f" outputs={graph.outputs}") + lines.append(f" nodes={len(graph.order)}") + if graph.meta: + lines.append(f" meta={graph.meta}") + lines.append("") + lines.append("IR Nodes") + + for index, node_id in enumerate(graph.order): + node = graph.nodes[node_id] + lines.append(f"[{index}] {node.id} op={node.op}") + lines.append(f" inputs={node.inputs}") + lines.append(f" outputs={node.outputs}") + if node.attrs: + lines.append(f" attrs={node.attrs}") + if node.kind != "generic": + lines.append(f" kind={node.kind}") + if node.meta: + lines.append(f" meta={node.meta}") + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + lines.append( + f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" + ) + + if graph.constants: + lines.append("") + lines.append("IR Constants") + for value_id, value in graph.constants.items(): + value_meta = graph.values.get(value_id) + shape = None if value_meta is None else value_meta.shape + dtype = None if value_meta is None else value_meta.dtype + typename = type(value).__name__ + lines.append(f"- {value_id}: type={typename} shape={shape} dtype={dtype}") + + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture a small PyTorch model, dump the exported graph, and print IR before/after cleanup.", + ) + parser.add_argument( + "--case", + choices=("toy", "attention_block"), + default="toy", + help="Which built-in model to inspect.", + ) + parser.add_argument( + "--no-meta", + action="store_true", + help="Hide exported graph metadata to keep the dump shorter.", + ) + args = parser.parse_args() + + model, example_args = build_case(args.case) + captured = capture_model(model, example_args) + + print(f"Case: {args.case}") + print(f"Model: {type(model).__name__}") + print("") + print("=== Exported Graph ===") + print(dump_graph(captured, include_meta=not args.no_meta)) + print("") + print("=== IR Before Cleanup ===") + print(format_ir_graph(captured.ir_graph)) + print("") + + canonicalize_exported_graph(captured.ir_graph) + + print("=== IR After Cleanup ===") + print(format_ir_graph(captured.ir_graph)) + + +if __name__ == "__main__": + main() diff --git a/python/examples/test_torch_capture.py b/python/examples/test_torch_capture.py new file mode 100644 index 000000000..c2ff748b4 --- /dev/null +++ b/python/examples/test_torch_capture.py @@ -0,0 +1,70 @@ +import torch +import torch.nn as nn + +from src.transpile.capture_pytorch import ( + capture_model_with_fallback, + dump_graph, + get_dtype, + get_shape, +) +from src.transpile.lower import transpile_captured + + +class TinyBlock(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(8, 16) + self.fc2 = nn.Linear(16, 4) + + def forward(self, x): + h = self.fc1(x) + h = torch.relu(h) + y = self.fc2(h) + return y + + +def main(): + model = TinyBlock().eval() + x = torch.randn(2, 8) + + captured = capture_model_with_fallback(model, args=(x,)) + + print("strict:", captured.strict) + print("graph_module type:", type(captured.graph_module).__name__) + print("state_dict keys:", list(captured.state_dict.keys())) + print() + + print("=== FX Graph ===") + print(captured.graph) + print() + + print("=== Node Walk ===") + for i, node in enumerate(captured.graph.nodes): + print(f"[{i}] op={node.op} name={node.name} target={node.target}") + print(f" args={node.args}") + print(f" kwargs={node.kwargs}") + print(f" shape={get_shape(node)} dtype={get_dtype(node)}") + print() + + print("=== Full Dump ===") + print(dump_graph(captured)) + print() + + print("=== Lower To Cactus ===") + try: + transpiled = transpile_captured(captured) + print("lowering succeeded") + print("lowered graph type:", type(transpiled.graph).__name__) + print("runtime inputs:", transpiled.runtime_inputs) + print("bound constants:", transpiled.bound_constants) + print("outputs:", transpiled.outputs) + print() + print("binding runtime input 0 from example tensor") + transpiled.set_input(0, x) + except NotImplementedError as exc: + print("lowering stopped on unimplemented case:") + print(f" {exc}") + + +if __name__ == "__main__": + main() diff --git a/python/examples/transpile_hf_causal_lm.py b/python/examples/transpile_hf_causal_lm.py new file mode 100644 index 000000000..05e70b8c9 --- /dev/null +++ b/python/examples/transpile_hf_causal_lm.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import runpy +import sys +from pathlib import Path + + +def main() -> int: + if "--task" not in sys.argv[1:]: + sys.argv[1:1] = ["--task", "causal_lm_logits"] + script_path = Path(__file__).with_name("transpile_hf_model.py") + runpy.run_path(str(script_path), run_name="__main__") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/transpile_hf_model.py b/python/examples/transpile_hf_model.py new file mode 100644 index 000000000..fda855056 --- /dev/null +++ b/python/examples/transpile_hf_model.py @@ -0,0 +1,2932 @@ +from __future__ import annotations + +import argparse +import builtins +import copy +import gc +import importlib.util +import json +import os +import re +import sys +from collections.abc import Mapping +from collections import Counter +from dataclasses import dataclass +from dataclasses import fields +from functools import lru_cache +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.transpile.runtime_compat import Graph +from src.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features +from src.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform +from src.transpile.audio_preprocess import prepare_cactus_audio_features +from src.transpile.audio_preprocess import prepare_native_gemma4_audio_features +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.component_partition import extract_component_subgraphs +from src.transpile.component_partition import summarize_ir_components +from src.transpile.component_pipeline import capture_component_spec +from src.transpile.component_pipeline import execute_component_pipeline +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import verify_ir +from src.transpile.lower import TranspiledGraph +from src.transpile.lower import _lower_constant_value +from src.transpile.lower import _lower_input_value +from src.transpile.lower import _lower_ir_node +from src.transpile.lower import _lookup_weight_binding +from src.transpile.model_adapters import build_component_module_specs +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph +from src.transpile.parakeet_tdt_local import greedy_decode_parakeet_tdt_token_ids +from src.transpile.parakeet_tdt_local import load_parakeet_tdt_local_model +from src.transpile.parakeet_tdt_local import prepare_parakeet_tdt_audio_features +from src.transpile.weight_compat import ensure_binding_compatible + +_TORCHVISION_COMPAT_LIBRARIES: list[object] = [] + + +@dataclass +class PreparedInputs: + names: tuple[str, ...] + tensors: tuple[torch.Tensor, ...] + metadata: dict[str, object] + + +def _transformers_supports_model_module(module_name: str) -> bool: + try: + return importlib.util.find_spec(module_name) is not None + except Exception: + return False + + +def _candidate_external_site_packages() -> list[Path]: + candidates: list[Path] = [] + major_minor = f"python{sys.version_info.major}.{sys.version_info.minor}" + pyenv_versions = Path.home() / ".pyenv" / "versions" + if pyenv_versions.exists(): + for version_dir in sorted(pyenv_versions.iterdir(), reverse=True): + site_packages = version_dir / "lib" / major_minor / "site-packages" + if site_packages.exists(): + candidates.append(site_packages) + return candidates + + +def _ensure_transformers_supports_model_type(model_type: str) -> str | None: + normalized = str(model_type or "").strip().lower() + target_module = { + "gemma4": "transformers.models.gemma4.modeling_gemma4", + }.get(normalized) + if not target_module: + return None + if _transformers_supports_model_module(target_module): + return None + + for site_packages in _candidate_external_site_packages(): + candidate = site_packages / Path(*target_module.split(".")) + if not candidate.with_suffix(".py").exists() and not candidate.is_dir(): + continue + sys.path.insert(0, str(site_packages)) + for module_name in list(sys.modules): + root_name = module_name.split(".", 1)[0] + if root_name in {"transformers", "huggingface_hub", "tokenizers"}: + del sys.modules[module_name] + if _transformers_supports_model_module(target_module): + return str(site_packages) + try: + sys.path.remove(str(site_packages)) + except ValueError: + pass + return None + + +def _resolve_local_snapshot(model_id_or_path: str) -> str | None: + explicit = Path(model_id_or_path) + if explicit.exists(): + return str(explicit) + + snapshots_dir = ( + Path.home() + / ".cache" + / "huggingface" + / "hub" + / ("models--" + model_id_or_path.replace("/", "--")) + / "snapshots" + ) + if not snapshots_dir.exists(): + return None + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + return None + return str(snapshots[-1]) + + +def _snapshot_has_model_weights(path: str | Path) -> bool: + root = Path(path) + if not root.exists() or not root.is_dir(): + return False + candidates = ( + "model.safetensors", + "model.safetensors.index.json", + "pytorch_model.bin", + "pytorch_model.bin.index.json", + ) + return any((root / name).exists() for name in candidates) + + +def _validate_weights_dir(weights_dir: str | None, *, model_id: str) -> Path | None: + if not weights_dir: + return None + + root = Path(weights_dir).resolve() + if not root.exists(): + raise RuntimeError( + f"weights_dir does not exist: {root}\n" + "\n" + f"Create the folder first with:\n" + f" cactus convert {model_id} {root}\n" + ) + + manifest_path = root / "weights_manifest.json" + if not manifest_path.exists(): + has_weight_files = any(root.glob("*.weights")) + if not has_weight_files: + raise RuntimeError( + f"weights_dir is missing weights_manifest.json: {manifest_path}\n" + "\n" + f"Re-convert with the current converter:\n" + f" cactus convert {model_id} {root}\n" + ) + print( + "note=weights_manifest_missing using filename-based weight binding fallback " + f"for {root}" + ) + + return root + + +def _serialize_json_compatible(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _serialize_json_compatible(inner) for key, inner in value.items()} + if isinstance(value, (list, tuple)): + return [_serialize_json_compatible(inner) for inner in value] + if isinstance(value, Path): + return str(value) + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.dtype): + return str(value) + if isinstance(value, torch.Tensor): + return { + "type": "torch.Tensor", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if isinstance(value, np.ndarray): + return { + "type": "numpy.ndarray", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if hasattr(value, "__dataclass_fields__"): + return { + field.name: _serialize_json_compatible(getattr(value, field.name)) + for field in fields(value) + } + try: + return repr(value) + except Exception: + return f"<{type(value).__module__}.{type(value).__name__}>" + + +def _slugify_model_artifact_name(value: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("._-") or "model" + + +def _default_artifact_dir_for_model(model_id: str) -> Path: + return (Path.cwd() / "transpiled" / _slugify_model_artifact_name(model_id)).resolve() + + +def _graph_to_dict(graph) -> dict[str, object]: + return { + "meta": _serialize_json_compatible(graph.meta), + "inputs": list(graph.inputs), + "outputs": list(graph.outputs), + "constants": { + value_id: _serialize_json_compatible(constant) + for value_id, constant in graph.constants.items() + }, + "values": { + value_id: _serialize_json_compatible(value) + for value_id, value in graph.values.items() + }, + "nodes": [ + _serialize_json_compatible(graph.nodes[node_id]) + for node_id in graph.order + ], + } + + +def _write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def _component_artifact_name(prefix: str, component: str) -> str: + return f"{prefix}_{component}.json" + + +def _component_graphs_to_payload(component_graphs: dict[str, IRGraph]) -> dict[str, object]: + return { + component: _graph_to_dict(graph) + for component, graph in component_graphs.items() + } + + +def _binding_entries_by_node_id( + bindings: list[dict[str, object]], +) -> dict[int, dict[str, object]]: + result: dict[int, dict[str, object]] = {} + for binding in bindings: + try: + result[int(binding["node_id"])] = binding + except Exception: + continue + return result + + +def _binding_entries_by_value_id( + bindings: list[dict[str, object]], +) -> dict[str, dict[str, object]]: + result: dict[str, dict[str, object]] = {} + for binding in bindings: + value_id = binding.get("value_id") + if isinstance(value_id, str) and value_id: + result[value_id] = binding + return result + + +def _write_graph_binding_manifest( + *, + artifact_dir: Path, + filename: str, + model_id: str, + model_source: str, + task: str, + family: str, + inputs_metadata: dict[str, object], + transpiled_graph: TranspiledGraph, +) -> Path: + manifest_path = artifact_dir / filename + _write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(inputs_metadata), + "runtime_input_node_ids": [int(tensor.id) for tensor in transpiled_graph.runtime_inputs], + "output_node_ids": [int(tensor.id) for tensor in transpiled_graph.outputs], + "bound_constant_bindings": _serialize_json_compatible(transpiled_graph.bound_constant_bindings), + }, + ) + return manifest_path + + +def _write_component_bundle( + *, + artifact_dir: Path, + model_id: str, + model_source: str, + task: str, + family: str, + inputs_metadata: dict[str, object], + raw_component_graphs: dict[str, IRGraph], + optimized_component_graphs: dict[str, IRGraph], + transpiled_component_graphs: dict[str, TranspiledGraph] | None = None, + component_io_signatures: dict[str, dict[str, tuple[str, ...]]] | None = None, + graph_filename: str = "graph.cactus", +) -> Path: + bundle_dir = artifact_dir / "components" + component_order = [ + component + for component in ( + "audio_encoder", + "vision_encoder", + "lm_encoder", + "decoder", + "unspecified", + ) + if component in raw_component_graphs or component in optimized_component_graphs + ] + extra_components = sorted( + component + for component in set(raw_component_graphs) | set(optimized_component_graphs) + if component not in component_order + ) + component_order.extend(extra_components) + + manifest_components: list[dict[str, object]] = [] + for component in component_order: + raw_graph = raw_component_graphs.get(component) + optimized_graph = optimized_component_graphs.get(component) + transpiled_graph = None if transpiled_component_graphs is None else transpiled_component_graphs.get(component) + component_dir = bundle_dir / component + raw_relpath = None + optimized_relpath = None + graph_relpath = None + if raw_graph is not None: + raw_relpath = Path(component) / "raw_ir.json" + _write_json( + bundle_dir / raw_relpath, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(inputs_metadata), + "graph": _graph_to_dict(raw_graph), + }, + ) + if optimized_graph is not None: + optimized_relpath = Path(component) / "optimized_ir.json" + _write_json( + bundle_dir / optimized_relpath, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(inputs_metadata), + "graph": _graph_to_dict(optimized_graph), + }, + ) + if transpiled_graph is not None: + graph_relpath = Path(component) / graph_filename + component_dir.mkdir(parents=True, exist_ok=True) + transpiled_graph.graph.save(bundle_dir / graph_relpath) + + materialized_constant_bindings: list[dict[str, object]] = [] + if transpiled_graph is not None: + existing_bindings = list(_serialize_json_compatible(transpiled_graph.bound_constant_bindings)) + existing_by_node_id = _binding_entries_by_node_id(existing_bindings) + existing_by_value_id = _binding_entries_by_value_id(existing_bindings) + constant_value_ids = dict(getattr(transpiled_graph, "bound_constant_value_ids", {})) + value_to_node_id = {value_id: node_id for node_id, value_id in constant_value_ids.items()} + constant_dir = component_dir / "bound_constants" + for constant_tensor in transpiled_graph.bound_constants: + node_id = int(constant_tensor.id) + if node_id in existing_by_node_id: + continue + constant_dir.mkdir(parents=True, exist_ok=True) + constant_filename = f"node_{node_id}.npy" + constant_relpath = Path(component) / "bound_constants" / constant_filename + constant_array = constant_tensor.numpy().copy() + np.save(bundle_dir / constant_relpath, constant_array, allow_pickle=False) + materialized_constant_bindings.append( + { + "node_id": node_id, + "value_id": str(constant_value_ids.get(node_id, f"materialized_constant_{node_id}")), + "path": str((bundle_dir / constant_relpath).relative_to(artifact_dir)), + "kind": "saved_constant", + "source_name": str(constant_value_ids.get(node_id, f"materialized_constant_{node_id}")), + "format": "npy", + "precision": int(constant_tensor.dtype), + } + ) + + graph_constants = (optimized_graph or raw_graph).constants if (optimized_graph or raw_graph) is not None else {} + for value_id, constant_value in graph_constants.items(): + if value_id in existing_by_value_id: + continue + if any(binding.get("value_id") == value_id for binding in materialized_constant_bindings): + continue + if isinstance(constant_value, torch.Tensor): + constant_array = constant_value.detach().cpu().numpy().copy() + elif isinstance(constant_value, np.ndarray): + constant_array = np.ascontiguousarray(constant_value) + else: + continue + constant_dir.mkdir(parents=True, exist_ok=True) + safe_value_id = re.sub(r"[^A-Za-z0-9._-]+", "_", value_id).strip("._-") or "constant" + constant_filename = f"{safe_value_id}.npy" + constant_relpath = Path(component) / "bound_constants" / constant_filename + np.save(bundle_dir / constant_relpath, constant_array, allow_pickle=False) + materialized_constant_bindings.append( + { + "node_id": int(value_to_node_id.get(value_id, -1)), + "value_id": str(value_id), + "path": str((bundle_dir / constant_relpath).relative_to(artifact_dir)), + "kind": "saved_constant", + "source_name": str(value_id), + "format": "npy", + "precision": int( + Graph.FP16 if constant_array.dtype == np.float16 + else Graph.FP32 if constant_array.dtype == np.float32 + else Graph.INT8 if constant_array.dtype == np.int8 + else Graph.INT8 + ), + } + ) + + graph_for_signature = optimized_graph or raw_graph + if graph_for_signature is None: + continue + manifest_components.append( + { + "component": component, + "directory": str(component_dir.relative_to(artifact_dir)), + "raw_ir": None if raw_relpath is None else str((bundle_dir / raw_relpath).relative_to(artifact_dir)), + "optimized_ir": None if optimized_relpath is None else str((bundle_dir / optimized_relpath).relative_to(artifact_dir)), + "graph": None if graph_relpath is None else str((bundle_dir / graph_relpath).relative_to(artifact_dir)), + "inputs": list(graph_for_signature.inputs), + "outputs": list(graph_for_signature.outputs), + "logical_inputs": list((component_io_signatures or {}).get(component, {}).get("input_keys", ())), + "logical_outputs": list((component_io_signatures or {}).get(component, {}).get("output_keys", ())), + "node_count": len(graph_for_signature.order), + "weight_binding_count": _count_weight_bindings(graph_for_signature), + "runtime_input_node_ids": [] if transpiled_graph is None else [int(tensor.id) for tensor in transpiled_graph.runtime_inputs], + "output_node_ids": [] if transpiled_graph is None else [int(tensor.id) for tensor in transpiled_graph.outputs], + "bound_constant_bindings": [] if transpiled_graph is None else ( + list(_serialize_json_compatible(transpiled_graph.bound_constant_bindings)) + + materialized_constant_bindings + ), + } + ) + + manifest_path = bundle_dir / "manifest.json" + _write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component_order": component_order, + "inputs": _serialize_json_compatible(inputs_metadata), + "components": manifest_components, + }, + ) + return manifest_path + + +def _named_tensor_store(prepared: PreparedInputs) -> dict[str, torch.Tensor]: + return { + name: tensor + for name, tensor in zip(prepared.names, prepared.tensors, strict=True) + } + + +def _aggregate_component_counts(component_graphs: dict[str, IRGraph]) -> dict[str, int]: + return { + component: len(graph.order) + for component, graph in sorted(component_graphs.items()) + } + + +def _aggregate_component_op_counts(component_graphs: dict[str, IRGraph]) -> Counter: + counter: Counter[str] = Counter() + for graph in component_graphs.values(): + counter.update(graph.nodes[node_id].op for node_id in graph.order) + return counter + + +def _count_component_weight_bindings(component_graphs: dict[str, IRGraph]) -> int: + return sum(_count_weight_bindings(graph) for graph in component_graphs.values()) + + +def _execute_gemma4_component_pipeline( + *, + component_graphs, + prepared: PreparedInputs, +) -> np.ndarray: + store, _ = execute_component_pipeline( + [component_graphs[name] for name in ("vision_encoder", "audio_encoder", "lm_encoder", "decoder")], + initial_store=_named_tensor_store(prepared), + ) + logits = store["logits"] + return np.asarray(logits, dtype=np.float32) + + +def _run_parakeet_tdt_component_decode( + *, + component_graphs, + model, + prepared: PreparedInputs, +) -> dict[str, object]: + store, _ = execute_component_pipeline( + [component_graphs["audio_encoder"]], + initial_store=_named_tensor_store(prepared), + ) + encoder_hidden_states = np.asarray(store["encoder_hidden_states"]) + batch_size = int(encoder_hidden_states.shape[0]) + if batch_size != 1: + raise ValueError("Parakeet TDT component decode currently expects batch size 1") + + hidden_dtype = prepared.tensors[0].dtype + initial_states = model.initial_decoder_state( + batch_size=batch_size, + device=torch.device("cpu"), + dtype=hidden_dtype, + ) + initial_state_arrays = tuple(state.detach().cpu().numpy() for state in initial_states) + decoder_component = component_graphs["decoder"] + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + input_store: dict[str, object] = { + "encoder_frame": np.ascontiguousarray(frame), + "token_ids": np.full((batch_size,), token_id, dtype=np.int64), + } + for index in range(model.config.predictor_num_layers): + input_store[f"state_h_{index}"] = np.ascontiguousarray(state_values[index * 2]) + input_store[f"state_c_{index}"] = np.ascontiguousarray(state_values[index * 2 + 1]) + + runtime_inputs = [input_store[key] for key in decoder_component.input_keys] + decoder_component.transpiled_graph.set_inputs(runtime_inputs) + outputs = decoder_component.transpiled_graph.execute() + logits = outputs[0].numpy().astype(np.float32, copy=False) + next_states = tuple(output.numpy() for output in outputs[1:]) + return logits, next_states + + emitted = greedy_decode_parakeet_tdt_token_ids( + config=model.config, + encoder_hidden_states=encoder_hidden_states, + initial_states=initial_state_arrays, + step=_step, + ) + + return { + "token_ids": emitted, + "transcript": model.decode_token_ids(emitted), + "encoder_hidden_shape": list(encoder_hidden_states.shape), + } + + +def _run_component_pipeline_transpile( + *, + args, + task: str, + family: str, + model_source: str, + model, + prepared: PreparedInputs, + component_specs, + fusion_config: FusionConfig, + weights_dir: str | None, + artifact_dir: Path | None, + processor_or_tokenizer, + canonical, +) -> int: + print(f"model_id={args.model_id}") + print(f"model_source={model_source}") + print(f"task={task}") + print(f"adapter_family={family}") + print("adapter_module=component_pipeline") + print(f"input_names={','.join(prepared.names)}") + for name, tensor in zip(prepared.names, prepared.tensors, strict=True): + print(f"input_{name}_shape={list(tensor.shape)}") + if weights_dir: + print(f"weights_dir={weights_dir}") + + print("capture_begin=true", flush=True) + captured_components = {} + for spec in component_specs: + print(f"capture_component_begin={spec.component}", flush=True) + captured_components[spec.component] = capture_component_spec(spec, fusion_config=fusion_config) + print(f"capture_component_done={spec.component}", flush=True) + print("capture_done=true", flush=True) + + raw_component_graphs = { + name: captured.raw_ir_graph + for name, captured in captured_components.items() + } + optimized_component_graphs = { + name: captured.optimized_ir_graph + for name, captured in captured_components.items() + } + transpiled_component_graphs = { + name: captured.transpiled_graph + for name, captured in captured_components.items() + } + component_io_signatures = { + name: { + "input_keys": tuple(captured.input_keys), + "output_keys": tuple(captured.output_keys), + } + for name, captured in captured_components.items() + } + + raw_ir_nodes = sum(len(graph.order) for graph in raw_component_graphs.values()) + optimized_ir_nodes = sum(len(graph.order) for graph in optimized_component_graphs.values()) + binding_count = _count_component_weight_bindings(optimized_component_graphs) + raw_component_counts = _aggregate_component_counts(raw_component_graphs) + optimized_component_counts = _aggregate_component_counts(optimized_component_graphs) + op_counts = _aggregate_component_op_counts(optimized_component_graphs) + + print(f"raw_ir_nodes={raw_ir_nodes}") + print(f"optimized_ir_nodes={optimized_ir_nodes}") + print(f"weight_bindings={binding_count}") + if raw_component_counts: + print( + "raw_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(raw_component_counts.items())) + ) + if optimized_component_counts: + print( + "optimized_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(optimized_component_counts.items())) + ) + print( + "ops=" + f"attention:{op_counts.get('attention', 0)} " + f"conv1d:{op_counts.get('conv1d', 0)} " + f"conv2d:{op_counts.get('conv2d', 0)} " + f"batch_norm:{op_counts.get('batch_norm', 0)} " + f"layer_norm:{op_counts.get('layer_norm', 0)} " + f"rms_norm:{op_counts.get('rms_norm', 0)} " + f"rope:{op_counts.get('rope', 0)} " + f"linear:{op_counts.get('linear', 0)}" + ) + + if weights_dir and binding_count == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir}\n" + "\n" + "The weights folder exists, but none of the component graphs matched entries in weights_manifest.json.\n" + "\n" + f"Recommended fix:\n" + f" cactus convert {args.model_id} {weights_dir}\n" + ) + + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "component_order": [spec.component for spec in component_specs], + "components": _component_graphs_to_payload(raw_component_graphs), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "component_order": [spec.component for spec in component_specs], + "components": _component_graphs_to_payload(optimized_component_graphs), + }, + ) + for component, component_graph in raw_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("raw_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + for component, component_graph in optimized_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("optimized_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + for component in raw_component_graphs: + print(f"saved_raw_component_ir_{component}={artifact_dir / _component_artifact_name('raw_ir', component)}") + for component in optimized_component_graphs: + print( + f"saved_optimized_component_ir_{component}=" + f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" + ) + component_manifest_path = _write_component_bundle( + artifact_dir=artifact_dir, + model_id=args.model_id, + model_source=model_source, + task=task, + family=family, + inputs_metadata=prepared.metadata, + raw_component_graphs=raw_component_graphs, + optimized_component_graphs=optimized_component_graphs, + transpiled_component_graphs=transpiled_component_graphs, + component_io_signatures=component_io_signatures, + graph_filename=args.graph_filename, + ) + print(f"saved_component_bundle_manifest={component_manifest_path}") + for component in transpiled_component_graphs: + print( + f"saved_component_graph_{component}=" + f"{artifact_dir / 'components' / component / args.graph_filename}" + ) + print("note=saved split component graphs; use the component bundle manifest to rebind mmap weights/embeddings when loading") + + if args.skip_execute: + return 0 + + if task == "multimodal_causal_lm_logits": + print("execute_begin=true") + transpiled_output = _execute_gemma4_component_pipeline( + component_graphs=captured_components, + prepared=prepared, + ) + print("execute_done=true") + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + if args.skip_reference_compare: + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(transpiled_output.shape), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "reference_compare_skipped": True, + } + print(f"output_shape={list(transpiled_output.shape)}") + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"transpiled_next_token_id={transpiled_next}") + result_payload["transpiled_next_token_id"] = transpiled_next + if hasattr(tokenizer_like, "decode"): + transpiled_token = tokenizer_like.decode([transpiled_next]) + print(f"transpiled_next_token={transpiled_token!r}") + result_payload["transpiled_next_token"] = transpiled_token + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + print("reference_begin=true") + with torch.no_grad(): + reference_output = canonical.module(*prepared.tensors).detach().float().cpu().numpy() + print("reference_done=true") + max_abs_diff = float(np.max(np.abs(reference_output - transpiled_output))) + mean_abs_diff = float(np.mean(np.abs(reference_output - transpiled_output))) + hf_next = int(np.argmax(reference_output[0, -1])) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"hf_next_token_id={hf_next}") + print(f"transpiled_next_token_id={transpiled_next}") + print(f"logits_max_abs_diff={max_abs_diff:.6f}") + print(f"logits_mean_abs_diff={mean_abs_diff:.6f}") + if hasattr(tokenizer_like, "decode"): + print(f"hf_next_token={tokenizer_like.decode([hf_next])!r}") + print(f"transpiled_next_token={tokenizer_like.decode([transpiled_next])!r}") + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(reference_output.shape), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "max_abs_diff": max_abs_diff, + "mean_abs_diff": mean_abs_diff, + "hf_next_token_id": hf_next, + "transpiled_next_token_id": transpiled_next, + } + if hasattr(tokenizer_like, "decode"): + result_payload["hf_next_token"] = tokenizer_like.decode([hf_next]) + result_payload["transpiled_next_token"] = tokenizer_like.decode([transpiled_next]) + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + if task == "tdt_transcription": + print("execute_begin=true") + transpiled_decode = _run_parakeet_tdt_component_decode( + component_graphs=captured_components, + model=model, + prepared=prepared, + ) + print("execute_done=true") + print(f"transpiled_transcript={transpiled_decode['transcript']!r}") + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "transpiled_token_ids": transpiled_decode["token_ids"], + "transpiled_transcript": transpiled_decode["transcript"], + "encoder_hidden_shape": transpiled_decode["encoder_hidden_shape"], + } + if args.skip_reference_compare: + result_payload["reference_compare_skipped"] = True + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + print("reference_begin=true") + reference_token_ids = model.greedy_decode_token_ids(prepared.tensors[0]) + reference_transcript = model.decode_token_ids(reference_token_ids) + print("reference_done=true") + print(f"reference_transcript={reference_transcript!r}") + print(f"transcript_match={reference_transcript == transpiled_decode['transcript']}") + result_payload["reference_token_ids"] = reference_token_ids + result_payload["reference_transcript"] = reference_transcript + result_payload["transcript_match"] = bool(reference_transcript == transpiled_decode["transcript"]) + result_payload["token_id_match"] = bool(reference_token_ids == transpiled_decode["token_ids"]) + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + raise NotImplementedError(f"component pipeline execution is not implemented for task={task}") + + +def _parse_dtype(name: str) -> torch.dtype: + normalized = name.strip().lower() + mapping = { + "float16": torch.float16, + "fp16": torch.float16, + "float32": torch.float32, + "fp32": torch.float32, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + } + if normalized not in mapping: + raise ValueError(f"unsupported torch dtype: {name}") + return mapping[normalized] + + +class TranspileWrapper(torch.nn.Module): + def __init__(self, adapter_module: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__() + self.adapter = adapter_module + self.weights_dir = weights_dir + + def forward(self, *bound_inputs: torch.Tensor) -> torch.Tensor: + return self.adapter(*bound_inputs) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.adapter, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta: dict[str, object] = {} + base_graph = metadata.get("graph", {}) + if isinstance(base_graph, dict): + graph_meta.update(base_graph) + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir + metadata["graph"] = graph_meta + return metadata + + +def _patch_transformers_torchvision_probe() -> str | None: + has_torchvision = importlib.util.find_spec("torchvision") is not None + has_lzma = importlib.util.find_spec("_lzma") is not None + + if not has_torchvision: + return None + + if has_lzma: + nms_patch_note = _patch_torchvision_missing_nms_op() + return nms_patch_note + + base_note: str | None = None + try: + import backports.lzma as backports_lzma # type: ignore + + sys.modules.setdefault("lzma", backports_lzma) + base_note = "using backports.lzma because this Python build is missing _lzma" + nms_patch_note = _patch_torchvision_missing_nms_op() + if nms_patch_note: + return f"{base_note}; {nms_patch_note}" + return base_note + except Exception: + pass + + class _InterpolationModeStub: + NEAREST = "nearest" + BILINEAR = "bilinear" + BICUBIC = "bicubic" + LANCZOS = "lanczos" + + class _TorchvisionFunctionalStub: + InterpolationMode = _InterpolationModeStub + + def __getattr__(self, name: str): + raise RuntimeError( + "torchvision functionality is unavailable because this Python build is missing _lzma; " + f"attempted to access torchvision.transforms.functional.{name}" + ) + + # Some Transformers multimodal image/video modules reference `F.InterpolationMode` + # or `tvF.InterpolationMode` in annotations even when torchvision probing is disabled. + # Installing a tiny builtins-level stub lets those modules import so the PIL fallback + # classes can be selected instead. + builtins.F = _TorchvisionFunctionalStub() + builtins.tvF = builtins.F + + import transformers.utils as tf_utils # type: ignore + import transformers.utils.import_utils as tf_import_utils # type: ignore + + @lru_cache + def _disabled() -> bool: + return False + + tf_import_utils.is_torchvision_available = _disabled + tf_import_utils.is_torchvision_v2_available = _disabled + tf_utils.is_torchvision_available = _disabled + tf_utils.is_torchvision_v2_available = _disabled + return "disabled torchvision import checks because this Python build is missing _lzma" + + +def _patch_torchvision_missing_nms_op() -> str | None: + """Keep mismatched torch/torchvision installs importable for processors. + + Some local environments have torchvision installed but built without the + custom operator registrations matching the active torch wheel. Transformers + imports torchvision just to access image interpolation enums, and that import + can fail before we ever touch model code because `torchvision::nms` is + missing. Defining the operator schema is enough for torchvision's fake/meta + registration path to complete; the transpiler never calls NMS. + """ + + try: + import torchvision # type: ignore # noqa: F401 + return None + except RuntimeError as exc: + if "operator torchvision::nms does not exist" not in str(exc): + return None + except Exception: + return None + + try: + import torchvision.extension as tv_extension # type: ignore + if bool(getattr(tv_extension, "_HAS_OPS", False)): + return None + except Exception: + pass + + try: + library = torch.library.Library("torchvision", "DEF") + library.define("nms(Tensor dets, Tensor scores, float iou_threshold) -> Tensor") + _TORCHVISION_COMPAT_LIBRARIES.append(library) + return "defined missing torchvision::nms operator for torchvision import compatibility" + except Exception: + return None + + +def _patch_torch_flex_attention_compat() -> str | None: + try: + import torch.nn.attention.flex_attention as flex_attention # type: ignore + except Exception: + return None + + if hasattr(flex_attention, "AuxRequest"): + return None + + class _AuxRequest: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + flex_attention.AuxRequest = _AuxRequest # type: ignore[attr-defined] + return "installed torch flex_attention AuxRequest compatibility stub" + + +def _tie_lfm2_vl_lm_head_if_needed(model: torch.nn.Module) -> str | None: + if str(getattr(getattr(model, "config", None), "model_type", "") or "").lower() != "lfm2_vl": + return None + lm_head = getattr(model, "lm_head", None) + model_root = getattr(model, "model", None) + language_model = getattr(model_root, "language_model", None) + embed_tokens = getattr(language_model, "embed_tokens", None) + if not isinstance(lm_head, torch.nn.Linear) or not isinstance(embed_tokens, torch.nn.Embedding): + return None + if tuple(lm_head.weight.shape) != tuple(embed_tokens.weight.shape): + return None + if lm_head.weight.data_ptr() == embed_tokens.weight.data_ptr(): + return None + lm_head.weight = embed_tokens.weight + return "tied LFM2-VL lm_head.weight to language_model.embed_tokens.weight" + + +def _load_local_torch_state_dict(model_source: str) -> dict[str, torch.Tensor] | None: + root = Path(model_source) + if not root.exists() or not root.is_dir(): + return None + + safetensors_path = root / "model.safetensors" + if safetensors_path.exists(): + try: + from safetensors.torch import load_file # type: ignore + + return load_file(str(safetensors_path)) + except Exception: + return None + + pytorch_path = root / "pytorch_model.bin" + if pytorch_path.exists(): + try: + loaded = torch.load(str(pytorch_path), map_location="cpu", weights_only=True) + except TypeError: + loaded = torch.load(str(pytorch_path), map_location="cpu") + return loaded if isinstance(loaded, dict) else None + + return None + + +def _remap_gemma4_checkpoint_state_dict(state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + remapped: dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + if "audio_tower" in new_key: + new_key = re.sub(r"subsample_conv_projection\.layer(\d+)\.", r"subsample_conv_projection.conv_\1.", new_key) + new_key = new_key.replace("audio_tower.layers.", "audio_tower.conformer.") + new_key = new_key.replace(".feed_forward1.", ".ffw_layer_start.") + new_key = new_key.replace(".feed_forward2.", ".ffw_layer_end.") + new_key = re.sub(r"\.self_attn\.(q_proj|k_proj|v_proj)\.", r".attention.attn.\1.", new_key) + new_key = new_key.replace(".self_attn.per_dim_scale", ".attention.attn.per_dim_scale") + new_key = new_key.replace(".self_attn.relative_k_proj.", ".attention.attn.relative_position_embedding.pos_proj.") + new_key = new_key.replace(".self_attn.post.", ".attention.post.") + new_key = new_key.replace(".norm_pre_attn.", ".attention.pre_attn_norm.") + new_key = new_key.replace(".norm_post_attn.", ".attention.post_norm.") + new_key = new_key.replace(".norm_out.", ".norm.") + new_key = new_key.replace(".linear.weight", ".weight") + remapped[new_key] = value + if new_key.endswith(".attention.attn.per_dim_scale"): + remapped[new_key.replace(".per_dim_scale", ".per_dim_key_scale")] = value + if "lm_head.weight" not in remapped: + tied_embedding = remapped.get("model.language_model.embed_tokens.weight") + if tied_embedding is not None: + remapped["lm_head.weight"] = tied_embedding + return remapped + + +def _repair_gemma4_checkpoint_weights(model: torch.nn.Module, model_source: str) -> dict[str, object]: + raw_state_dict = _load_local_torch_state_dict(model_source) + if raw_state_dict is None: + return {"applied": False, "reason": "no local checkpoint state_dict"} + + remapped_state_dict = _remap_gemma4_checkpoint_state_dict(raw_state_dict) + load_result = model.load_state_dict(remapped_state_dict, strict=False) + return { + "applied": True, + "missing_keys": list(load_result.missing_keys), + "unexpected_keys": list(load_result.unexpected_keys), + } + + +def _load_config_json(model_id_or_path: str) -> dict[str, object]: + local_snapshot = _resolve_local_snapshot(model_id_or_path) + config_source = Path(local_snapshot) / "config.json" if local_snapshot else None + if config_source is not None and config_source.exists(): + return json.loads(config_source.read_text()) + explicit = Path(model_id_or_path) / "config.json" + if explicit.exists(): + return json.loads(explicit.read_text()) + return {} + + +def _load_optional_json(model_id_or_path: str, filename: str) -> dict[str, object]: + local_snapshot = _resolve_local_snapshot(model_id_or_path) + candidate = Path(local_snapshot) / filename if local_snapshot else None + if candidate is not None and candidate.exists(): + return json.loads(candidate.read_text()) + explicit = Path(model_id_or_path) / filename + if explicit.exists(): + return json.loads(explicit.read_text()) + return {} + + +def _load_gemma4_tokenizer_fallback(source_candidates: list[str]) -> object | None: + try: + from transformers import PreTrainedTokenizerFast # type: ignore + except Exception: + return None + + for source in source_candidates: + root = Path(source) + if not root.exists() or not root.is_dir(): + continue + tokenizer_json = root / "tokenizer.json" + tokenizer_config_path = root / "tokenizer_config.json" + if not tokenizer_json.exists() or not tokenizer_config_path.exists(): + continue + + tokenizer_config = json.loads(tokenizer_config_path.read_text()) + tokenizer = PreTrainedTokenizerFast( + tokenizer_file=str(tokenizer_json), + bos_token=tokenizer_config.get("bos_token"), + eos_token=tokenizer_config.get("eos_token"), + unk_token=tokenizer_config.get("unk_token"), + pad_token=tokenizer_config.get("pad_token"), + mask_token=tokenizer_config.get("mask_token"), + padding_side=str(tokenizer_config.get("padding_side", "right")), + additional_special_tokens=list(tokenizer_config.get("extra_special_tokens", []) or []), + ) + model_max_length = tokenizer_config.get("model_max_length") + if isinstance(model_max_length, int): + tokenizer.model_max_length = model_max_length + + chat_template_path = root / "chat_template.jinja" + if chat_template_path.exists(): + tokenizer.chat_template = chat_template_path.read_text() + + for token_attr in ( + "image_token", + "audio_token", + "boi_token", + "eoi_token", + "boa_token", + "eoa_token", + ): + token_value = tokenizer_config.get(token_attr) + if isinstance(token_value, str): + setattr(tokenizer, token_attr, token_value) + setattr(tokenizer, f"{token_attr}_id", tokenizer.convert_tokens_to_ids(token_value)) + return tokenizer + + return None + + +def _load_gemma4_processor_fallback( + *, + source_candidates: list[str], + common_kwargs: dict[str, object], +) -> object | None: + try: + from transformers.models.gemma4.feature_extraction_gemma4 import Gemma4AudioFeatureExtractor # type: ignore + from transformers.models.gemma4.image_processing_gemma4 import Gemma4ImageProcessor # type: ignore + from transformers.models.gemma4.processing_gemma4 import Gemma4Processor # type: ignore + except Exception: + return None + + tokenizer = None + processor_config: dict[str, object] = {} + for source in source_candidates: + try: + try: + from transformers import AutoTokenizer # type: ignore + + tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) + except Exception: + tokenizer = _load_gemma4_tokenizer_fallback([source]) + processor_config = _load_optional_json(source, "processor_config.json") + if tokenizer is not None and processor_config: + break + except Exception: + continue + if tokenizer is None or not processor_config: + return None + + feature_config = dict(processor_config.get("feature_extractor", {}) or {}) + image_config = dict(processor_config.get("image_processor", {}) or {}) + feature_config.pop("feature_extractor_type", None) + image_config.pop("image_processor_type", None) + + feature_extractor = Gemma4AudioFeatureExtractor(**feature_config) + image_processor = Gemma4ImageProcessor(**image_config) + + processor_kwargs = {} + for key in ("image_seq_length", "audio_seq_length", "audio_ms_per_token"): + if key in processor_config: + processor_kwargs[key] = processor_config[key] + + return Gemma4Processor( + feature_extractor=feature_extractor, + image_processor=image_processor, + tokenizer=tokenizer, + **processor_kwargs, + ) + + +def _infer_task_from_config(model_id_or_path: str) -> str: + config = _load_config_json(model_id_or_path) + architectures = [str(value) for value in config.get("architectures", []) if isinstance(value, str)] + model_type = str(config.get("model_type", "") or "").lower() + decoding_cfg = config.get("decoding") + if isinstance(decoding_cfg, dict) and str(decoding_cfg.get("model_type", "") or "").lower() == "tdt": + return "tdt_transcription" + loss_cfg = config.get("loss") + if isinstance(loss_cfg, dict) and str(loss_cfg.get("loss_name", "") or "").lower() == "tdt": + return "tdt_transcription" + + if any("CausalLM" in value for value in architectures): + return "causal_lm_logits" + if any("CTC" in value for value in architectures): + return "ctc_logits" + if model_type == "whisper": + return "encoder_hidden_states" + if any("ConditionalGeneration" in value and "Whisper" in value for value in architectures): + return "encoder_hidden_states" + + lowered_id = model_id_or_path.lower() + if "parakeet-tdt" in lowered_id: + return "tdt_transcription" + if "whisper" in lowered_id: + return "encoder_hidden_states" + if "ctc" in lowered_id: + return "ctc_logits" + if any(token in lowered_id for token in ("qwen", "gemma", "llama", "mistral", "lfm")): + return "causal_lm_logits" + + raise RuntimeError( + f"Could not infer transpile task for {model_id_or_path}.\n" + "\n" + "Pass one explicitly with --task, for example:\n" + " --task causal_lm_logits\n" + " --task tdt_transcription\n" + " --task ctc_logits\n" + " --task encoder_hidden_states\n" + ) + + +def _resolve_audio_sample_rate(processor: object) -> int: + for attr_name in ("feature_extractor", "tokenizer"): + child = getattr(processor, attr_name, None) + sample_rate = getattr(child, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + sample_rate = getattr(processor, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + return 16000 + + +def _infer_fallback_audio_input_names(config: dict[str, object], task: str) -> tuple[str, ...]: + model_type = str(config.get("model_type", "") or "").lower() + if model_type == "whisper": + return ("input_features",) + if task == "encoder_hidden_states": + return ("input_features",) + if task == "tdt_transcription": + return ("input_features",) + audio_cfg = config.get("audio_config") + if isinstance(audio_cfg, dict) and any(key in audio_cfg for key in ("features", "input_feat_size", "num_mel_bins")): + return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) + encoder_cfg = config.get("encoder") + if isinstance(encoder_cfg, dict) and any(key in encoder_cfg for key in ("feat_in", "num_mel_bins")): + return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) + return ("input_values", "attention_mask") if task == "ctc_logits" else ("input_values",) + + +def _resolve_encoder_module(model: torch.nn.Module) -> torch.nn.Module | None: + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder = get_encoder() + if isinstance(encoder, torch.nn.Module): + return encoder + encoder = getattr(model, "encoder", None) + if isinstance(encoder, torch.nn.Module): + return encoder + model_attr = getattr(model, "model", None) + if model_attr is not None: + encoder = getattr(model_attr, "encoder", None) + if isinstance(encoder, torch.nn.Module): + return encoder + return None + + +def _infer_expected_input_feature_frames(model: torch.nn.Module) -> int | None: + config = getattr(model, "config", None) + max_source_positions = getattr(config, "max_source_positions", None) + if not isinstance(max_source_positions, int) or max_source_positions <= 0: + return None + + encoder = _resolve_encoder_module(model) + if encoder is None: + return None + + stride_product = 1 + found_conv = False + for child in encoder.children(): + if isinstance(child, torch.nn.Conv1d): + stride = child.stride[0] if isinstance(child.stride, tuple) else child.stride + stride_product *= int(stride) + found_conv = True + + if not found_conv: + return None + return int(max_source_positions) * stride_product + + +def _prepare_fallback_audio_inputs( + *, + input_names: tuple[str, ...], + config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + task: str, + audio_file: str, + torch_dtype: torch.dtype, +) -> PreparedInputs: + if not input_names: + input_names = _infer_fallback_audio_input_names(config, task) + preprocessor_root = config.get("preprocessor") + preprocessor_root = preprocessor_root if isinstance(preprocessor_root, dict) else {} + encoder_root = config.get("encoder") + encoder_root = encoder_root if isinstance(encoder_root, dict) else {} + + sample_rate = int( + preprocessor_config.get( + "sampling_rate", + preprocessor_root.get("sample_rate", config.get("sampling_rate", 16000)), + ) + or 16000 + ) + num_mels = int( + preprocessor_config.get( + "feature_size", + preprocessor_root.get( + "features", + encoder_root.get("feat_in", config.get("num_mel_bins", config.get("feature_size", 80))), + ), + ) + or 80 + ) + encoder_cfg = config.get("encoder_config") + if isinstance(encoder_cfg, dict): + num_mels = int(encoder_cfg.get("num_mel_bins", encoder_cfg.get("feat_in", num_mels)) or num_mels) + raw_hop_length = preprocessor_config.get( + "hop_length", + preprocessor_root.get("window_stride", config.get("hop_length", 160)), + ) + if raw_hop_length is None: + raw_hop_length = 160 + if isinstance(raw_hop_length, float) and raw_hop_length > 0.0 and raw_hop_length < 1.0: + hop_length = max(1, int(round(float(sample_rate) * raw_hop_length))) + else: + hop_length = int(raw_hop_length) + + n_fft = int(preprocessor_config.get("n_fft", config.get("n_fft", preprocessor_root.get("n_fft", 400))) or 400) + raw_frame_length = preprocessor_config.get( + "win_length", + preprocessor_config.get( + "frame_length", + preprocessor_root.get("window_size", preprocessor_root.get("n_window_size", config.get("frame_length", n_fft))), + ), + ) + if raw_frame_length is None: + raw_frame_length = n_fft + if isinstance(raw_frame_length, float) and raw_frame_length > 0.0 and raw_frame_length < 1.0: + frame_length = max(1, int(round(float(sample_rate) * raw_frame_length))) + else: + frame_length = int(raw_frame_length) + preemphasis = preprocessor_config.get("preemphasis", preprocessor_root.get("preemph")) + if preemphasis is not None: + preemphasis = float(preemphasis) + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + + tensors: list[torch.Tensor] = [] + if input_names and input_names[0] == "input_features": + model_type = str(config.get("model_type", "") or "").lower() + is_parakeet_like = ( + task == "tdt_transcription" + or "parakeet" in model_type + or "parakeet" in type(model).__module__.lower() + ) + if is_parakeet_like: + expected_frames = _infer_expected_input_feature_frames(model) + parakeet_features, _ = prepare_parakeet_tdt_audio_features( + audio_file, + expected_frames=expected_frames, + expected_mels=num_mels, + torch_dtype=torch_dtype, + ) + tensors.append(parakeet_features) + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "native_parakeet_audio_preprocessor": True, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + is_whisper_like = ( + str(config.get("model_type", "") or "").lower() == "whisper" + or "transformers.models.whisper." in type(model).__module__ + ) + if is_whisper_like: + expected_frames = _infer_expected_input_feature_frames(model) or 3000 + try: + whisper_features, active_frames = prepare_cactus_audio_features( + audio_file, + model_type="whisper", + expected_frames=expected_frames, + expected_mels=num_mels, + torch_dtype=torch_dtype, + layout="mels_frames", + ) + tensors.append(whisper_features) + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "native_cactus_audio_preprocessor": True, + "active_feature_frames": active_frames, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + except Exception: + pass + features, feature_length = _generic_log_mel_features( + waveform, + sample_rate=sample_rate, + num_mels=num_mels, + n_fft=n_fft, + hop_length=hop_length, + frame_length=frame_length, + preemphasis=preemphasis, + ) + expected_frames = _infer_expected_input_feature_frames(model) + attention_mask: np.ndarray | None = None + target_frames = feature_length + if isinstance(expected_frames, int) and expected_frames > 0: + target_frames = expected_frames + + active_frames = min(feature_length, target_frames) + features = features[:active_frames, :] + if target_frames > active_frames: + pad_width = target_frames - active_frames + features = np.pad(features, ((0, pad_width), (0, 0)), mode="constant") + attention_mask = np.zeros((target_frames,), dtype=np.bool_) + attention_mask[:active_frames] = True + + if is_whisper_like: + features = np.ascontiguousarray(features.T) + + tensors.append(torch.from_numpy(features).unsqueeze(0).to(dtype=torch_dtype)) + if attention_mask is not None and len(input_names) > 1 and input_names[1] == "attention_mask": + tensors.append(torch.from_numpy(attention_mask).unsqueeze(0)) + else: + input_values = torch.from_numpy(waveform).unsqueeze(0).to(dtype=torch_dtype) + tensors.append(input_values) + if len(input_names) > 1 and input_names[1] == "attention_mask": + tensors.append(torch.ones_like(input_values, dtype=torch.float32)) + + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + + +def _prepare_audio_inputs( + processor: object | None, + *, + input_names: tuple[str, ...], + config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + task: str, + audio_file: str, + torch_dtype: torch.dtype, +) -> PreparedInputs: + if processor is None: + return _prepare_fallback_audio_inputs( + input_names=input_names, + config=config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + + model_type = str(config.get("model_type", "") or "").lower() + if model_type == "whisper" or "transformers.models.whisper." in type(model).__module__: + return _prepare_fallback_audio_inputs( + input_names=input_names, + config=config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + + sample_rate = _resolve_audio_sample_rate(processor) + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + batch = processor( + waveform, + sampling_rate=sample_rate, + return_tensors="pt", + ) + + preferred_keys = tuple(input_names) + tuple( + key for key in ("input_features", "input_values", "attention_mask") if key not in input_names + ) + tensor_keys = [key for key, value in batch.items() if isinstance(value, torch.Tensor)] + ordered_keys = [key for key in preferred_keys if key in tensor_keys] + ordered_keys.extend(key for key in tensor_keys if key not in ordered_keys) + if not ordered_keys: + raise RuntimeError(f"processor did not return tensor inputs for audio file: {audio_file}") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": False, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + }, + ) + + +def _tokenize_text_prompt(tokenizer: object, prompt: str) -> torch.Tensor: + apply_chat_template = getattr(tokenizer, "apply_chat_template", None) + if callable(apply_chat_template): + try: + encoded = apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + ) + if isinstance(encoded, torch.Tensor) and encoded.ndim == 2: + return encoded.to(dtype=torch.long) + if isinstance(encoded, Mapping): + input_ids = encoded.get("input_ids") + if isinstance(input_ids, torch.Tensor) and input_ids.ndim == 2: + return input_ids.to(dtype=torch.long) + except Exception: + pass + encoded = tokenizer(prompt, return_tensors="pt") + return encoded["input_ids"].to(dtype=torch.long) + + +def _prepare_text_inputs(tokenizer: object, *, prompt: str, input_ids_text: str | None) -> PreparedInputs: + if input_ids_text: + token_ids = [int(part.strip()) for part in input_ids_text.split(",") if part.strip()] + if not token_ids: + raise ValueError("--input-ids was provided but no ids were parsed") + input_ids = torch.tensor([token_ids], dtype=torch.long) + else: + input_ids = _tokenize_text_prompt(tokenizer, prompt) + return PreparedInputs( + names=("input_ids",), + tensors=(input_ids,), + metadata={ + "prompt": prompt, + "input_ids": input_ids.tolist(), + }, + ) + + +_GEMMA4_MULTIMODAL_INPUT_ORDER = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", +) + + +def _normalize_multimodal_prompt( + prompt: str, + *, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, +) -> str: + normalized = prompt.strip() + prefixes: list[str] = [] + + if image_token and num_images > 0: + image_count = normalized.count(image_token) + if image_count < num_images: + prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) + if audio_token and num_audio_segments > 0: + audio_count = normalized.count(audio_token) + if audio_count < num_audio_segments: + prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) + + if prefixes: + prefix = "\n".join(part for part in prefixes if part) + if normalized: + return f"{prefix}\n{normalized}" + return prefix + return normalized + + +def _build_gemma4_chat_prompt( + *, + prompt: str, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> str: + result = "" + normalized_system = system_prompt.strip() + if enable_thinking_if_supported or normalized_system: + result += "<|turn>system\n" + if enable_thinking_if_supported: + result += "<|think|>" + result += normalized_system + result += "\n" + + result += "<|turn>user\n" + if image_token and num_images > 0: + for _ in range(num_images): + # Gemma4Processor expands the raw placeholder into the native + # <|image> ... soft-token region. Supplying wrappers here + # double-wraps the prompt and shifts the multimodal prefill. + result += f"\n\n{image_token}\n\n" + result += prompt + if audio_token and num_audio_segments > 0: + result += "".join( + audio_token + for _ in range(num_audio_segments) + ) + result += "\n" + result += "<|turn>model\n" + return result + + +def _gemma4_split_cactus_newline_token_merges(batch: object) -> None: + """Match native Cactus Gemma4 tokenization for newline runs. + + The HF fast tokenizer merges ``\n\n`` and ``\n\n\n`` into dedicated tokens + (108 and 109). Cactus' converted BPE tokenizer emits repeated newline token + 107 instead. Gemma4's multimodal prefill is sensitive enough that this one + token of drift changes the first generated token, so normalize only this + Gemma4-specific quirk after the processor has expanded image/audio regions. + """ + + input_ids = batch.get("input_ids") if hasattr(batch, "get") else None + if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: + return + + expansions = { + 108: (107, 107), + 109: (107, 107, 107), + } + if not any(int(token) in expansions for token in input_ids.reshape(-1).tolist()): + return + + lengths: list[list[int]] = [] + max_len = 0 + for row in input_ids.detach().cpu().tolist(): + row_lengths = [len(expansions.get(int(token), (int(token),))) for token in row] + lengths.append(row_lengths) + max_len = max(max_len, sum(row_lengths)) + + for key in ("input_ids", "attention_mask", "token_type_ids"): + value = batch.get(key) if hasattr(batch, "get") else None + if not isinstance(value, torch.Tensor) or value.ndim != 2: + continue + pad_value = 0 + expanded = torch.full( + (value.shape[0], max_len), + pad_value, + dtype=value.dtype, + device=value.device, + ) + for row_idx, row in enumerate(value.detach().cpu().tolist()): + out: list[int] = [] + for token_idx, item in enumerate(row): + if key == "input_ids": + out.extend(expansions.get(int(item), (int(item),))) + else: + out.extend([int(item)] * lengths[row_idx][token_idx]) + expanded[row_idx, : len(out)] = torch.tensor(out, dtype=value.dtype, device=value.device) + batch[key] = expanded + + +def _build_gemma4_chat_messages( + *, + prompt: str, + num_images: int, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> list[dict[str, object]]: + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system or enable_thinking_if_supported: + system_text = normalized_system + if enable_thinking_if_supported: + system_text = f"<|think|>{system_text}" + messages.append({"role": "system", "content": system_text}) + + user_content: list[dict[str, str]] = [] + for _ in range(num_images): + user_content.append({"type": "image"}) + user_content.append({"type": "text", "text": prompt.strip()}) + for _ in range(num_audio_segments): + user_content.append({"type": "audio"}) + messages.append({"role": "user", "content": user_content}) + return messages + + +def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: + if not image_files: + return [] + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for --image-file: {exc}") from exc + + images: list[object] = [] + for image_file in image_files: + path = Path(image_file).resolve() + if not path.exists(): + raise RuntimeError(f"image_file does not exist: {path}") + with Image.open(path) as image: + images.append(image.convert("RGB").copy()) + return images + + +def _get_processor_image_attr(processor: object, name: str, default: object) -> object: + image_processor = getattr(processor, "image_processor", None) + if image_processor is not None and hasattr(image_processor, name): + return getattr(image_processor, name) + if isinstance(image_processor, dict) and name in image_processor: + return image_processor[name] + return default + + +def _prepare_gemma4_native_image_tensors( + processor: object, + image_files: tuple[str, ...], +) -> tuple[torch.Tensor, torch.Tensor] | None: + if not image_files: + return None + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for Gemma4 native image preprocessing: {exc}") from exc + + patch_size = int(_get_processor_image_attr(processor, "patch_size", 16)) + pooling_kernel_size = int(_get_processor_image_attr(processor, "pooling_kernel_size", 3)) + max_soft_tokens = int(_get_processor_image_attr(processor, "max_soft_tokens", 280)) + rescale_factor = float(_get_processor_image_attr(processor, "rescale_factor", 1.0 / 255.0)) + max_patches = max_soft_tokens * pooling_kernel_size * pooling_kernel_size + side_multiple = pooling_kernel_size * patch_size + patch_dim = 3 * patch_size * patch_size + if patch_size <= 0 or pooling_kernel_size <= 0 or max_patches <= 0: + return None + + try: + resample_bilinear = Image.Resampling.BILINEAR + except AttributeError: # pragma: no cover + resample_bilinear = Image.BILINEAR + + pixel_batches: list[np.ndarray] = [] + position_batches: list[np.ndarray] = [] + for image_file in image_files: + path = Path(image_file).resolve() + with Image.open(path) as image: + rgb = image.convert("RGB") + width, height = rgb.size + target_pixels = float(max_patches * patch_size * patch_size) + factor = float(np.sqrt(target_pixels / max(1.0, float(width * height)))) + target_h = int(np.floor(factor * height / side_multiple)) * side_multiple + target_w = int(np.floor(factor * width / side_multiple)) * side_multiple + if target_h == 0: + target_h = side_multiple + if target_w == 0: + target_w = side_multiple + if (target_w, target_h) != rgb.size: + rgb = rgb.resize((target_w, target_h), resample=resample_bilinear) + array = np.asarray(rgb, dtype=np.float32) * rescale_factor + + patch_h = target_h // patch_size + patch_w = target_w // patch_size + num_patches = patch_h * patch_w + if num_patches > max_patches: + raise RuntimeError( + f"Gemma4 native image preprocessing produced {num_patches} patches, " + f"but max_patches={max_patches}" + ) + chw = np.transpose(array, (2, 0, 1)) + patches = ( + chw.reshape(3, patch_h, patch_size, patch_w, patch_size) + .transpose(1, 3, 2, 4, 0) + .reshape(num_patches, patch_dim) + ) + + padded_patches = np.zeros((max_patches, patch_dim), dtype=np.float32) + padded_patches[:num_patches] = patches + positions = np.full((max_patches, 2), -1, dtype=np.int64) + valid_positions = np.zeros((num_patches, 2), dtype=np.int64) + for patch_y in range(patch_h): + row_start = patch_y * patch_w + valid_positions[row_start: row_start + patch_w, 0] = np.arange(patch_w, dtype=np.int64) + valid_positions[row_start: row_start + patch_w, 1] = patch_y + positions[:num_patches] = valid_positions + + pixel_batches.append(padded_patches) + position_batches.append(positions) + + return ( + torch.from_numpy(np.stack(pixel_batches, axis=0)), + torch.from_numpy(np.stack(position_batches, axis=0)), + ) + + +def _prepare_gemma4_multimodal_inputs( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, + use_gemma4_chat_template: bool = False, +) -> PreparedInputs: + if processor is None: + raise RuntimeError("multimodal Gemma4 transpile requires an AutoProcessor with image and audio support") + + images = _load_image_inputs(image_files) + audio_waveforms: list[np.ndarray] = [] + sample_rate: int | None = None + if audio_file: + sample_rate = _resolve_audio_sample_rate(processor) + audio_waveforms.append(_load_audio_waveform(audio_file, target_sample_rate=sample_rate)) + + image_token = getattr(processor, "image_token", None) + audio_token = getattr(processor, "audio_token", None) + processor_prompt = _normalize_multimodal_prompt( + prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + ) + normalized_prompt = processor_prompt + if use_gemma4_chat_template: + normalized_prompt = prompt.strip() + # Match the native Cactus Gemma4 prompt builder instead of the HF chat + # template. The HF template omits the begin/end multimodal wrapper tokens, + # which shifts the multimodal prefill by four tokens for image+audio prompts. + processor_prompt = _build_gemma4_chat_prompt( + prompt=normalized_prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + + batch = processor( + text=processor_prompt, + images=images or None, + audio=audio_waveforms or None, + return_tensors="pt", + ) + if use_gemma4_chat_template: + _gemma4_split_cactus_newline_token_merges(batch) + native_image_tensors = _prepare_gemma4_native_image_tensors(processor, image_files) + if native_image_tensors is not None: + batch["pixel_values"], batch["pixel_position_ids"] = native_image_tensors + if audio_file and isinstance(batch.get("input_features"), torch.Tensor): + feature_tensor = batch["input_features"] + expected_mels = int(feature_tensor.shape[-1]) + try: + native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( + audio_file, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + except Exception as exc: + print(f"note=falling back to processor gemma4 audio features: {exc}") + batch["input_features"] = feature_tensor.to(dtype=torch_dtype) + fallback_mask = batch.get("input_features_mask") + if isinstance(fallback_mask, torch.Tensor): + batch["input_features_mask"] = fallback_mask.to(dtype=torch.bool) + native_audio_frames = int(fallback_mask.to(dtype=torch.int32).sum().item()) + else: + native_audio_frames = int(feature_tensor.shape[1]) + batch["input_features_mask"] = torch.ones( + (int(feature_tensor.shape[0]), native_audio_frames), + dtype=torch.bool, + ) + batch["native_audio_frames"] = native_audio_frames + else: + batch["input_features"] = native_audio + batch["input_features_mask"] = native_audio_mask + batch["native_audio_frames"] = native_audio_frames + + ordered_keys = [ + key + for key in _GEMMA4_MULTIMODAL_INPUT_ORDER + if isinstance(batch.get(key), torch.Tensor) + ] + if not ordered_keys: + raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + metadata: dict[str, object] = { + "prompt": normalized_prompt, + "processor_prompt": processor_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + } + if audio_file: + metadata["audio_file"] = str(Path(audio_file).resolve()) + if sample_rate is not None: + metadata["sample_rate"] = sample_rate + native_audio_frames = batch.get("native_audio_frames") + if isinstance(native_audio_frames, int): + metadata["native_audio_frames"] = native_audio_frames + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata=metadata, + ) + + +def _load_model_source(model_id: str, *, local_files_only: bool) -> str: + local_snapshot = _resolve_local_snapshot(model_id) + if local_snapshot and _snapshot_has_model_weights(local_snapshot): + return local_snapshot + if local_snapshot and local_files_only: + raise RuntimeError( + f"Found local snapshot for {model_id}, but it is incomplete and has no model weights:\n" + f" {local_snapshot}\n" + "\n" + "Re-run without --local-files-only to let transformers download the missing weights." + ) + return model_id + + +def _load_transformers_bundle( + *, + model_id: str, + task: str, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, +): + config = _load_config_json(model_id) + config_model_type = str(config.get("model_type", "") or "").lower() + external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) + patch_note = _patch_transformers_torchvision_probe() + if patch_note: + print(f"note={patch_note}") + flex_patch_note = _patch_torch_flex_attention_compat() + if flex_patch_note: + print(f"note={flex_patch_note}") + if external_transformers_site_packages: + print(f"note=using external transformers install for {config_model_type}: {external_transformers_site_packages}") + + try: + from transformers import AutoFeatureExtractor # type: ignore + from transformers import AutoModel # type: ignore + from transformers import AutoModelForCTC # type: ignore + from transformers import AutoModelForCausalLM # type: ignore + from transformers import AutoModelForImageTextToText # type: ignore + from transformers import AutoModelForSeq2SeqLM # type: ignore + from transformers import AutoModelForSpeechSeq2Seq # type: ignore + from transformers import AutoProcessor # type: ignore + from transformers import AutoTokenizer # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"transformers is not available: {exc}") from exc + + model_source = _load_model_source(model_id, local_files_only=local_files_only) + source_candidates = [] + for candidate in (model_source, model_id): + if candidate not in source_candidates: + source_candidates.append(candidate) + common_kwargs: dict[str, object] = { + "trust_remote_code": trust_remote_code, + "local_files_only": local_files_only, + } + if token: + common_kwargs["token"] = token + + if task == "tdt_transcription": + model = load_parakeet_tdt_local_model(model_source, torch_dtype=torch_dtype).eval() + return model_source, None, model, config + + if task == "causal_lm_logits": + tokenizer = None + tokenizer_errors: list[str] = [] + for source in source_candidates: + try: + tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + tokenizer_errors.append(f"{source}: {exc}") + if tokenizer is None: + raise RuntimeError( + f"Could not load tokenizer for {model_id}:\n" + + "\n".join(tokenizer_errors) + ) + if config_model_type == "lfm2_vl": + model = AutoModelForImageTextToText.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + tie_note = _tie_lfm2_vl_lm_head_if_needed(model) + if tie_note: + print(f"note={tie_note}") + else: + model = AutoModelForCausalLM.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model_source, tokenizer, model, config + if task == "multimodal_causal_lm_logits": + processor = None + processor_errors: list[str] = [] + for source in source_candidates: + try: + processor = AutoProcessor.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + processor_errors.append(f"{source}: {exc}") + if processor is None and config_model_type == "gemma4": + processor = _load_gemma4_processor_fallback( + source_candidates=source_candidates, + common_kwargs=common_kwargs, + ) + if processor is not None: + print("note=using manual gemma4 processor fallback") + if processor is None: + processor_config_hint = "" + if config_model_type == "gemma4": + processor_config_hint = ( + "\n" + "Gemma4 multimodal transpile needs a processor bundle, not just tokenizer/model weights.\n" + "Your local snapshot may be missing files such as `processor_config.json` or modality-specific\n" + "preprocessor configs. Use an official Gemma4 snapshot that includes the processor, or let\n" + "transformers download one by re-running without `--local-files-only`.\n" + ) + raise RuntimeError( + f"Could not load processor for {model_id}:\n" + + "\n".join(processor_errors) + + processor_config_hint + ) + + model = AutoModelForCausalLM.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + if config_model_type == "gemma4": + repair_result = _repair_gemma4_checkpoint_weights(model, model_source) + if repair_result.get("applied"): + missing = repair_result.get("missing_keys", []) + unexpected = repair_result.get("unexpected_keys", []) + print( + "note=applied gemma4 checkpoint key remap" + f" missing_after={len(missing)} unexpected_after={len(unexpected)}" + ) + return model_source, processor, model, config + + processor = None + processor_errors: list[str] = [] + missing_optional_audio_dep: str | None = None + for source in source_candidates: + for loader in (AutoProcessor, AutoFeatureExtractor): + try: + processor = loader.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + processor_errors.append(f"{loader.__name__}@{source}: {exc}") + if isinstance(exc, ImportError) and "requires the librosa library" in str(exc): + missing_optional_audio_dep = "librosa" + break + if processor is not None: + break + if missing_optional_audio_dep is not None: + break + if processor is None: + if missing_optional_audio_dep == "librosa": + print("note=falling back to built-in audio preprocessing because the HF feature extractor requires librosa") + else: + print("note=falling back to built-in audio preprocessing because no HF processor/feature extractor was available") + + if task == "ctc_logits": + model_loaders = (AutoModelForCTC, AutoModel) + elif task == "encoder_hidden_states": + model_loaders = (AutoModelForSpeechSeq2Seq, AutoModelForSeq2SeqLM, AutoModel) + else: + raise NotImplementedError(f"unsupported generic HF task: {task}") + + load_errors: list[str] = [] + for loader in model_loaders: + try: + model = loader.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model_source, processor, model, config + except Exception as exc: + load_errors.append(f"{loader.__name__}: {exc}") + + raise RuntimeError( + f"Could not load model for task={task} from {model_source}.\n" + "\n".join(load_errors) + ) + + +def _load_optional_tokenizer( + *, + model_id: str, + model_source: str, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, +): + try: + from transformers import AutoTokenizer # type: ignore + except Exception: + return None + + source_candidates = [] + for candidate in (model_source, model_id): + if candidate not in source_candidates: + source_candidates.append(candidate) + + common_kwargs: dict[str, object] = { + "trust_remote_code": trust_remote_code, + "local_files_only": local_files_only, + } + if token: + common_kwargs["token"] = token + + for source in source_candidates: + try: + return AutoTokenizer.from_pretrained(source, **common_kwargs) + except Exception: + continue + return None + + +def _ctc_greedy_decode_token_ids(logits: np.ndarray, *, blank_token_id: int | None) -> list[int]: + if logits.ndim != 3 or logits.shape[0] < 1: + raise ValueError(f"expected CTC logits with shape [batch, time, vocab], got {list(logits.shape)}") + + raw_ids = np.argmax(logits[0], axis=-1).tolist() + collapsed: list[int] = [] + previous: int | None = None + for token_id in raw_ids: + if token_id != previous: + collapsed.append(int(token_id)) + previous = int(token_id) + + if blank_token_id is None: + return collapsed + return [token_id for token_id in collapsed if int(token_id) != int(blank_token_id)] + + +def _decode_token_ids(tokenizer: object, token_ids: list[int]) -> str: + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + raise TypeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") + try: + return str(decode(token_ids, skip_special_tokens=True)) + except TypeError: + return str(decode(token_ids)) + + +def _count_weight_bindings(ir_graph: IRGraph) -> int: + count = 0 + for value in ir_graph.values.values(): + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str): + count += 1 + return count + + +def _lower_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + graph = Graph() + graph._transpile_materialized_constants = [] # type: ignore[attr-defined] + env: dict[str, Any] = {} + runtime_inputs = [] + bound_constants = [] + bound_constant_bindings = [] + bound_constant_value_ids: dict[int, str] = {} + + for value_id in ir.inputs: + value = ir.values[value_id] + tensor = _lower_input_value(graph, value) + env[value_id] = tensor + runtime_inputs.append(tensor) + + for value_id, const in ir.constants.items(): + value = ir.values[value_id] + binding = _lookup_weight_binding(value) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(graph, value, const, binding=binding) + env[value_id] = lowered_const + if hasattr(lowered_const, "g") and hasattr(lowered_const, "id"): + bound_constants.append(lowered_const) + bound_constant_value_ids[int(lowered_const.id)] = str(value_id) + if binding is not None: + bound_constant_bindings.append( + { + "node_id": int(lowered_const.id), + "value_id": str(value_id), + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + ) + + for node_id in ir.order: + node = ir.nodes[node_id] + outputs = _lower_ir_node(graph, node, env, ir) + if len(outputs) != len(node.outputs): + raise ValueError( + f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" + ) + for output_id, tensor in zip(node.outputs, outputs): + env[output_id] = tensor + + outputs = [env[value_id] for value_id in ir.outputs] + seen_bound_constant_ids = {int(tensor.id) for tensor in bound_constants} + for tensor in getattr(graph, "_transpile_materialized_constants", []): + tensor_id = int(tensor.id) + if tensor_id in seen_bound_constant_ids: + continue + bound_constants.append(tensor) + seen_bound_constant_ids.add(tensor_id) + return TranspiledGraph( + graph=graph, + runtime_inputs=runtime_inputs, + bound_constants=bound_constants, + bound_constant_bindings=bound_constant_bindings, + bound_constant_value_ids=bound_constant_value_ids, + outputs=outputs, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Load a Hugging Face model, canonicalize it into a generic transpile task, " + "capture it with the Cactus transpiler, lower it to a Cactus Graph, and " + "optionally save artifacts or run the lowered graph." + ) + ) + parser.add_argument( + "--model-id", + required=True, + help="Hugging Face model id or local snapshot path.", + ) + parser.add_argument( + "--task", + default="auto", + choices=( + "auto", + "causal_lm_logits", + "multimodal_causal_lm_logits", + "ctc_logits", + "encoder_hidden_states", + "tdt_transcription", + ), + help="Transpile task. Use auto to infer from config/model id.", + ) + parser.add_argument( + "--prompt", + default="The capital of France is", + help="Prompt used for causal_lm_logits or multimodal_causal_lm_logits when --input-ids is not set.", + ) + parser.add_argument( + "--system-prompt", + default="", + help="Optional system prompt used for Gemma4 multimodal chat-style prompt construction.", + ) + parser.add_argument( + "--enable-thinking", + action="store_true", + help="Enable model-specific thinking markers when the multimodal prompt format supports them.", + ) + parser.add_argument( + "--input-ids", + default="", + help="Optional comma-separated token ids for causal_lm_logits.", + ) + parser.add_argument( + "--audio-file", + default="", + help="Path to a WAV file for audio or multimodal tasks.", + ) + parser.add_argument( + "--image-file", + action="append", + default=[], + help="Path to an image file for multimodal tasks. Repeat to pass multiple images.", + ) + parser.add_argument( + "--torch-dtype", + default="float16", + help="Torch dtype for model loading: float16, float32, or bfloat16.", + ) + parser.add_argument( + "--token", + default=os.environ.get("HF_TOKEN"), + help="Optional Hugging Face token. Defaults to HF_TOKEN.", + ) + parser.add_argument( + "--trust-remote-code", + action="store_true", + help="Pass trust_remote_code=True to transformers loaders.", + ) + parser.add_argument( + "--local-files-only", + action="store_true", + help="Require the model/processor to already exist locally.", + ) + parser.add_argument( + "--weights-dir", + default="", + help="Optional converted Cactus weights directory for mmap weight binding.", + ) + parser.add_argument( + "--artifact-dir", + default="", + help="Directory where raw_ir.json, optimized_ir.json, graph.cactus, component graphs, and result.json are saved. Defaults to ./transpiled//.", + ) + parser.add_argument( + "--graph-filename", + default="graph.cactus", + help="Filename to use for Graph.save() inside --artifact-dir.", + ) + parser.add_argument( + "--component-pipeline", + default="auto", + choices=("auto", "on", "off"), + help="Use a split component pipeline when the model adapter supports it.", + ) + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + parser.add_argument( + "--skip-execute", + action="store_true", + help="Stop after lowering instead of running the transpiled graph.", + ) + parser.add_argument( + "--skip-reference-compare", + action="store_true", + help="Run the transpiled graph but skip the follow-up PyTorch reference pass.", + ) + args = parser.parse_args() + + image_files = tuple(str(path) for path in args.image_file if str(path).strip()) + if args.task == "auto": + inferred_task = _infer_task_from_config(args.model_id) + config_for_auto = _load_config_json(args.model_id) + model_type_for_auto = str(config_for_auto.get("model_type", "") or "").lower() + has_multimodal_config = ( + model_type_for_auto == "gemma4" + and ( + isinstance(config_for_auto.get("vision_config"), dict) + or isinstance(config_for_auto.get("audio_config"), dict) + ) + ) + if image_files or (args.audio_file and has_multimodal_config and inferred_task == "causal_lm_logits"): + task = "multimodal_causal_lm_logits" + else: + task = inferred_task + else: + task = args.task + torch_dtype = _parse_dtype(args.torch_dtype) + validated_weights_dir = _validate_weights_dir(args.weights_dir.strip() or None, model_id=args.model_id) + weights_dir = str(validated_weights_dir) if validated_weights_dir is not None else None + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else _default_artifact_dir_for_model(args.model_id) + + model_source, processor_or_tokenizer, model, model_config = _load_transformers_bundle( + model_id=args.model_id, + task=task, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") + if not preprocessor_config: + preprocessor_config = _load_optional_json(args.model_id, "preprocessor_config.json") + auxiliary_tokenizer = None + if task == "ctc_logits": + auxiliary_tokenizer = _load_optional_tokenizer( + model_id=args.model_id, + model_source=model_source, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + + canonical = None + if task == "causal_lm_logits": + prepared = _prepare_text_inputs( + processor_or_tokenizer, + prompt=args.prompt, + input_ids_text=args.input_ids.strip() or None, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + weights_dir=weights_dir, + ) + elif task == "multimodal_causal_lm_logits": + prepared = _prepare_gemma4_multimodal_inputs( + processor_or_tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=args.audio_file.strip() or None, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + use_gemma4_chat_template=True, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + weights_dir=weights_dir, + ) + prime_static_features = getattr(canonical.module, "prime_static_multimodal_features", None) + if callable(prime_static_features): + prime_static_features(*prepared.tensors) + elif task == "tdt_transcription": + if not args.audio_file: + raise RuntimeError(f"--audio-file is required for task={task}") + prepared = _prepare_audio_inputs( + processor_or_tokenizer, + input_names=("input_features",), + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=args.audio_file, + torch_dtype=torch_dtype, + ) + else: + if not args.audio_file: + raise RuntimeError(f"--audio-file is required for task={task}") + canonical = canonicalize_model_interface(model, task=task) + prepared = _prepare_audio_inputs( + processor_or_tokenizer, + input_names=canonical.input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=args.audio_file, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + ) + component_specs = build_component_module_specs( + model, + task=task, + named_tensors=_named_tensor_store(prepared), + weights_dir=weights_dir, + ) + use_component_pipeline = False + if args.component_pipeline == "on": + if component_specs is None: + raise RuntimeError( + f"--component-pipeline=on was requested, but {type(model).__name__} does not expose component specs for task={task}" + ) + use_component_pipeline = True + elif args.component_pipeline == "auto" and component_specs is not None: + use_component_pipeline = True + + if use_component_pipeline: + family = canonical.family if canonical is not None else getattr(model, "family", type(model).__name__) + return _run_component_pipeline_transpile( + args=args, + task=task, + family=str(family), + model_source=model_source, + model=model, + prepared=prepared, + component_specs=component_specs, + fusion_config=FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ), + weights_dir=weights_dir, + artifact_dir=artifact_dir, + processor_or_tokenizer=processor_or_tokenizer, + canonical=canonical, + ) + + if canonical is None: + raise RuntimeError(f"task={task} requires a component pipeline but none was selected") + wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + + print(f"model_id={args.model_id}") + print(f"model_source={model_source}") + print(f"task={task}") + print(f"adapter_family={canonical.family}") + print(f"adapter_module={type(canonical.module).__name__}") + print(f"input_names={','.join(prepared.names)}") + for name, tensor in zip(prepared.names, prepared.tensors): + print(f"input_{name}_shape={list(tensor.shape)}") + if weights_dir: + print(f"weights_dir={weights_dir}") + + print("capture_begin=true", flush=True) + prepare_cpu_float32_capture = getattr(canonical.module, "prepare_cpu_float32_capture", None) + restore_cpu_float32_capture = getattr(canonical.module, "restore_cpu_float32_capture", None) + if callable(prepare_cpu_float32_capture): + prepare_cpu_float32_capture() + try: + captured = capture_model(wrapper, prepared.tensors) + finally: + if callable(restore_cpu_float32_capture): + restore_cpu_float32_capture() + print("capture_done=true", flush=True) + raw_ir_graph = copy.deepcopy(captured.ir_graph) + + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + print("canonicalize_begin=true", flush=True) + canonicalize_exported_graph(captured.ir_graph) + print("canonicalize_done=true", flush=True) + print("optimize_begin=true", flush=True) + optimize_graph(captured.ir_graph, config=fusion_config) + print("optimize_done=true", flush=True) + print("lower_begin=true", flush=True) + tg = _lower_preoptimized_ir(captured.ir_graph) + print("lower_done=true", flush=True) + + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + binding_count = _count_weight_bindings(optimized_ir_graph) + op_counts = Counter(optimized_ir_graph.nodes[node_id].op for node_id in optimized_ir_graph.order) + raw_component_counts = summarize_ir_components(raw_ir_graph) + optimized_component_counts = summarize_ir_components(optimized_ir_graph) + raw_component_graphs = extract_component_subgraphs(raw_ir_graph) + optimized_component_graphs = extract_component_subgraphs(optimized_ir_graph) + transpiled_component_graphs: dict[str, TranspiledGraph] | None = None + + print(f"raw_ir_nodes={len(raw_ir_graph.order)}") + print(f"optimized_ir_nodes={len(optimized_ir_graph.order)}") + print(f"weight_bindings={binding_count}") + if raw_component_counts: + print( + "raw_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(raw_component_counts.items())) + ) + if optimized_component_counts: + print( + "optimized_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(optimized_component_counts.items())) + ) + print( + "ops=" + f"attention:{op_counts.get('attention', 0)} " + f"conv1d:{op_counts.get('conv1d', 0)} " + f"conv2d:{op_counts.get('conv2d', 0)} " + f"batch_norm:{op_counts.get('batch_norm', 0)} " + f"layer_norm:{op_counts.get('layer_norm', 0)} " + f"rms_norm:{op_counts.get('rms_norm', 0)} " + f"rope:{op_counts.get('rope', 0)} " + f"linear:{op_counts.get('linear', 0)}" + ) + + if weights_dir and binding_count == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir}\n" + "\n" + "The weights folder exists, but none of the captured constants matched entries in weights_manifest.json.\n" + "\n" + f"Recommended fix:\n" + f" cactus convert {args.model_id} {weights_dir}\n" + ) + + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + transpiled_component_graphs = { + component: _lower_preoptimized_ir(copy.deepcopy(component_graph)) + for component, component_graph in optimized_component_graphs.items() + } + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(raw_ir_graph), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(optimized_ir_graph), + }, + ) + for component, component_graph in raw_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("raw_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + for component, component_graph in optimized_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("optimized_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + for component in raw_component_graphs: + print(f"saved_raw_component_ir_{component}={artifact_dir / _component_artifact_name('raw_ir', component)}") + for component in optimized_component_graphs: + print( + f"saved_optimized_component_ir_{component}=" + f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" + ) + component_manifest_path = _write_component_bundle( + artifact_dir=artifact_dir, + model_id=args.model_id, + model_source=model_source, + task=task, + family=canonical.family, + inputs_metadata=prepared.metadata, + raw_component_graphs=raw_component_graphs, + optimized_component_graphs=optimized_component_graphs, + transpiled_component_graphs=transpiled_component_graphs, + graph_filename=args.graph_filename, + ) + print(f"saved_component_bundle_manifest={component_manifest_path}") + for component in optimized_component_graphs: + print( + f"saved_component_graph_{component}=" + f"{artifact_dir / 'components' / component / args.graph_filename}" + ) + graph_path = artifact_dir / args.graph_filename + tg.graph.save(graph_path) + print(f"saved_graph={graph_path}") + graph_binding_manifest_path = _write_graph_binding_manifest( + artifact_dir=artifact_dir, + filename="graph_bindings.json", + model_id=args.model_id, + model_source=model_source, + task=task, + family=canonical.family, + inputs_metadata=prepared.metadata, + transpiled_graph=tg, + ) + print(f"saved_graph_bindings={graph_binding_manifest_path}") + if binding_count > 0: + print( + "note=saved graph structure without embedded mmap bindings; use the saved component/full-graph manifests " + "to rebind weights/embeddings from the weights folder when loading" + ) + + if args.skip_execute: + return 0 + + tg.set_inputs([tensor.cpu().numpy() for tensor in prepared.tensors]) + print("execute_begin=true") + transpiled_output = tg.execute()[0].numpy().astype(np.float32) + print("execute_done=true") + + if args.skip_reference_compare: + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(transpiled_output.shape), + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": binding_count, + "reference_compare_skipped": True, + } + print(f"output_shape={list(transpiled_output.shape)}") + if task in {"causal_lm_logits", "multimodal_causal_lm_logits"}: + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"transpiled_next_token_id={transpiled_next}") + result_payload["transpiled_next_token_id"] = transpiled_next + if hasattr(tokenizer_like, "decode"): + transpiled_token = tokenizer_like.decode([transpiled_next]) + print(f"transpiled_next_token={transpiled_token!r}") + result_payload["transpiled_next_token"] = transpiled_token + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + del model + del wrapper + gc.collect() + return 0 + + print("reference_begin=true") + with torch.no_grad(): + reference_output = wrapper(*prepared.tensors).detach().float().cpu().numpy() + print("reference_done=true") + + max_abs_diff = float(np.max(np.abs(reference_output - transpiled_output))) + mean_abs_diff = float(np.mean(np.abs(reference_output - transpiled_output))) + result_payload: dict[str, object] = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(reference_output.shape), + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": binding_count, + "max_abs_diff": max_abs_diff, + "mean_abs_diff": mean_abs_diff, + } + + if task == "causal_lm_logits": + tokenizer = processor_or_tokenizer + hf_next = int(np.argmax(reference_output[0, -1])) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"hf_next_token_id={hf_next}") + print(f"transpiled_next_token_id={transpiled_next}") + print(f"logits_max_abs_diff={max_abs_diff:.6f}") + print(f"logits_mean_abs_diff={mean_abs_diff:.6f}") + print(f"hf_next_token={tokenizer.decode([hf_next])!r}") + print(f"transpiled_next_token={tokenizer.decode([transpiled_next])!r}") + result_payload.update( + { + "hf_next_token_id": hf_next, + "transpiled_next_token_id": transpiled_next, + "hf_next_token": tokenizer.decode([hf_next]), + "transpiled_next_token": tokenizer.decode([transpiled_next]), + } + ) + elif task == "ctc_logits": + blank_token_id = getattr(model.config, "pad_token_id", None) + if blank_token_id is None and auxiliary_tokenizer is not None: + blank_token_id = getattr(auxiliary_tokenizer, "pad_token_id", None) + print(f"output_shape={list(reference_output.shape)}") + print(f"output_max_abs_diff={max_abs_diff:.6f}") + print(f"output_mean_abs_diff={mean_abs_diff:.6f}") + if auxiliary_tokenizer is not None: + hf_token_ids = _ctc_greedy_decode_token_ids(reference_output, blank_token_id=blank_token_id) + transpiled_token_ids = _ctc_greedy_decode_token_ids(transpiled_output, blank_token_id=blank_token_id) + hf_transcript = _decode_token_ids(auxiliary_tokenizer, hf_token_ids) + transpiled_transcript = _decode_token_ids(auxiliary_tokenizer, transpiled_token_ids) + print(f"hf_transcript={hf_transcript!r}") + print(f"transpiled_transcript={transpiled_transcript!r}") + result_payload.update( + { + "blank_token_id": None if blank_token_id is None else int(blank_token_id), + "hf_transcript_token_ids": hf_token_ids, + "transpiled_transcript_token_ids": transpiled_token_ids, + "hf_transcript": hf_transcript, + "transpiled_transcript": transpiled_transcript, + } + ) + else: + print(f"output_shape={list(reference_output.shape)}") + print(f"output_max_abs_diff={max_abs_diff:.6f}") + print(f"output_mean_abs_diff={mean_abs_diff:.6f}") + + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/src/cactus.py b/python/src/cactus.py index 15479c0cd..eac3f6a71 100644 --- a/python/src/cactus.py +++ b/python/src/cactus.py @@ -1,12 +1,16 @@ """Cactus Python FFI bindings.""" import ctypes +import os import platform from pathlib import Path TokenCallback = ctypes.CFUNCTYPE(None, ctypes.c_char_p, ctypes.c_uint32, ctypes.c_void_p) _DIR = Path(__file__).parent.parent.parent -if platform.system() == "Darwin": +_OVERRIDE_LIB_PATH = os.environ.get("CACTUS_LIB_PATH") +if _OVERRIDE_LIB_PATH: + _LIB_PATH = Path(_OVERRIDE_LIB_PATH).expanduser().resolve() +elif platform.system() == "Darwin": _LIB_PATH = _DIR / "cactus" / "build" / "libcactus.dylib" else: _LIB_PATH = _DIR / "cactus" / "build" / "libcactus.so" @@ -19,6 +23,16 @@ _lib = ctypes.CDLL(str(_LIB_PATH)) + +def _bind_optional(name, argtypes, restype): + try: + fn = getattr(_lib, name) + except AttributeError: + return None + fn.argtypes = argtypes + fn.restype = restype + return fn + cactus_graph_t = ctypes.c_void_p cactus_node_t = ctypes.c_uint64 @@ -88,11 +102,22 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_divide.restype = ctypes.c_int +_bind_optional( + "cactus_graph_not_equal", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_precision_cast.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_precision_cast.restype = ctypes.c_int +_bind_optional( + "cactus_graph_quantize_activations", + [cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) + _lib.cactus_graph_scalar_add.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] @@ -109,6 +134,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_scalar_divide.restype = ctypes.c_int +_bind_optional( + "cactus_graph_scalar_floor_divide", + [cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_scalar_not_equal", + [cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_scalar_exp.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] @@ -129,6 +164,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_scalar_log.restype = ctypes.c_int +_bind_optional( + "cactus_graph_masked_select_prefix", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_masked_scatter", + [cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_abs.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) @@ -154,6 +199,11 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_reshape.restype = ctypes.c_int +_bind_optional( + "cactus_graph_expand", + [cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_transpose.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] @@ -182,11 +232,11 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_graph_cat.restype = ctypes.c_int _lib.cactus_graph_matmul.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, ctypes.c_int32, ctypes.POINTER(cactus_node_t) + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, ctypes.c_int32, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_matmul.restype = ctypes.c_int _lib.cactus_graph_gather.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_gather.restype = ctypes.c_int _lib.cactus_graph_embedding_from_tensor.argtypes = [ @@ -209,6 +259,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_bilinear_interpolation.restype = ctypes.c_int +_bind_optional( + "cactus_graph_set_grouped_scales", + [cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_void_p], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_set_interleaved", + [cactus_graph_t, cactus_node_t, ctypes.c_bool, ctypes.c_size_t], + ctypes.c_int, +) _lib.cactus_graph_release_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] _lib.cactus_graph_release_weight_pages.restype = ctypes.c_int _lib.cactus_graph_prefetch_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] @@ -236,6 +296,10 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_max.restype = ctypes.c_int +_lib.cactus_graph_cumsum.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_cumsum.restype = ctypes.c_int _lib.cactus_graph_relu.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) @@ -351,6 +415,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_conv2d_pointwise_1x1.restype = ctypes.c_int +_bind_optional( + "cactus_graph_conv2d_k3s1p1", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) +_lib.cactus_graph_conv2d.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, + ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv2d.restype = ctypes.c_int _lib.cactus_graph_lstm_cell.argtypes = [ cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] @@ -443,11 +517,14 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_transcribe.restype = ctypes.c_int -_lib.cactus_detect_language.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_detect_language.restype = ctypes.c_int +_bind_optional( + "cactus_detect_language", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) _lib.cactus_embed.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_float), @@ -467,24 +544,44 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_audio_embed.restype = ctypes.c_int -_lib.cactus_vad.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_vad.restype = ctypes.c_int - -_lib.cactus_diarize.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_diarize.restype = ctypes.c_int - -_lib.cactus_embed_speaker.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, - ctypes.POINTER(ctypes.c_float), ctypes.c_size_t -] -_lib.cactus_embed_speaker.restype = ctypes.c_int +try: + _lib.cactus_preprocess_audio_features.argtypes = [ + ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, + ctypes.POINTER(ctypes.c_size_t), ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t) + ] + _lib.cactus_preprocess_audio_features.restype = ctypes.c_int +except AttributeError: + pass + +_bind_optional( + "cactus_vad", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) + +_bind_optional( + "cactus_diarize", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) + +_bind_optional( + "cactus_embed_speaker", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, + ], + ctypes.c_int, +) _lib.cactus_reset.argtypes = [ctypes.c_void_p] _lib.cactus_reset.restype = None @@ -507,6 +604,20 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_tokenize.restype = ctypes.c_int +_bind_optional( + "cactus_decode_tokens", + [ + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_uint32), + ctypes.c_size_t, + ctypes.c_float, + ctypes.c_float, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_uint32), + ], + ctypes.c_int, +) + _lib.cactus_score_window.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), @@ -525,19 +636,22 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_rag_query.restype = ctypes.c_int -_lib.cactus_stream_transcribe_start.argtypes = [ctypes.c_void_p, ctypes.c_char_p] -_lib.cactus_stream_transcribe_start.restype = ctypes.c_void_p +_bind_optional("cactus_stream_transcribe_start", [ctypes.c_void_p, ctypes.c_char_p], ctypes.c_void_p) -_lib.cactus_stream_transcribe_process.argtypes = [ - ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, - ctypes.c_char_p, ctypes.c_size_t -] -_lib.cactus_stream_transcribe_process.restype = ctypes.c_int +_bind_optional( + "cactus_stream_transcribe_process", + [ + ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ctypes.c_char_p, ctypes.c_size_t, + ], + ctypes.c_int, +) -_lib.cactus_stream_transcribe_stop.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t -] -_lib.cactus_stream_transcribe_stop.restype = ctypes.c_int +_bind_optional( + "cactus_stream_transcribe_stop", + [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t], + ctypes.c_int, +) _lib.cactus_index_init.argtypes = [ctypes.c_char_p, ctypes.c_size_t] _lib.cactus_index_init.restype = ctypes.c_void_p @@ -710,8 +824,6 @@ def cactus_prefill(model, messages_json, options_json, tools_json, pcm_data=None ) if rc < 0: raise RuntimeError(_err("Prefill failed")) - return buf.value.decode("utf-8", errors="ignore") - def cactus_detect_language(model, audio_path, options_json, pcm_data): """Detects the spoken language in audio. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) @@ -785,6 +897,29 @@ def cactus_audio_embed(model, audio_path): return list(buf[:dim.value]) +def cactus_preprocess_audio_features(audio_path, model_type, mel_bins, capacity): + """Returns Cactus-native audio frontend features as (values, mel_bins, frames).""" + if not hasattr(_lib, "cactus_preprocess_audio_features"): + raise RuntimeError("cactus_preprocess_audio_features is unavailable; rebuild with cactus build --python") + buf = (ctypes.c_float * int(capacity))() + feature_count = ctypes.c_size_t() + out_mels = ctypes.c_size_t() + out_frames = ctypes.c_size_t() + rc = _lib.cactus_preprocess_audio_features( + _enc(audio_path), + _enc(model_type), + ctypes.c_size_t(int(mel_bins)), + buf, + ctypes.sizeof(buf), + ctypes.byref(feature_count), + ctypes.byref(out_mels), + ctypes.byref(out_frames), + ) + if rc < 0: + raise RuntimeError(_err("Audio feature preprocessing failed")) + return list(buf[:feature_count.value]), int(out_mels.value), int(out_frames.value) + + def cactus_vad(model, audio_path, options_json, pcm_data): """Runs voice activity detection. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) @@ -858,6 +993,26 @@ def cactus_tokenize(model, text): return list(arr[:n.value]) +def cactus_decode_tokens(model, tokens, temperature=0.0, top_p=1.0, top_k=1): + """Returns the next token ID from native Cactus for exact raw token IDs.""" + if not tokens: + raise ValueError("tokens must be non-empty") + arr = (ctypes.c_uint32 * len(tokens))(*[int(token) for token in tokens]) + out = ctypes.c_uint32(0) + rc = _lib.cactus_decode_tokens( + model, + arr, + len(tokens), + ctypes.c_float(float(temperature)), + ctypes.c_float(float(top_p)), + ctypes.c_size_t(int(top_k)), + ctypes.byref(out), + ) + if rc < 0: + raise RuntimeError(_err("Raw token decode failed")) + return int(out.value) + + def cactus_score_window(model, tokens, start, end, context): """Scores a window of tokens. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) diff --git a/python/src/cli.py b/python/src/cli.py index 0d24b50cd..b8e5572e1 100644 --- a/python/src/cli.py +++ b/python/src/cli.py @@ -13,11 +13,10 @@ def _looks_like_project_root(path: Path) -> bool: - return ( - (path / "python" / "src" / "cli.py").exists() - and (path / "cactus").exists() - and (path / "tests").exists() - ) + has_cli = (path / "python" / "src" / "cli.py").exists() + has_engine = (path / "cactus").exists() or (path / "cactus-engine").exists() + has_tests = (path / "tests").exists() or (path / "python" / "tests").exists() + return has_cli and has_engine and has_tests def _resolve_project_root() -> Path: @@ -47,6 +46,152 @@ def _resolve_project_root() -> Path: DEFAULT_MODEL_ID = "google/gemma-4-E2B-it" DEFAULT_TEST_MODEL_ID = "google/gemma-4-E2B-it" WEIGHTS_VARIANT_CHOICES = ["auto", "apple", "standard"] +MODEL_ID_ALIASES = { + "gemma4": DEFAULT_MODEL_ID, + "gemma4-e2b": DEFAULT_MODEL_ID, + "parakeet": "nvidia/parakeet-tdt-0.6b-v3", + "parakeet-tdt": "nvidia/parakeet-tdt-0.6b-v3", + "whisper": "openai/whisper-small", + "qwen": "Qwen/Qwen3-1.7B", + "lfm": "LiquidAI/LFM2-350M", +} + + +def _python_runtime_library_path() -> Path: + suffix = ".dylib" if platform.system() == "Darwin" else ".so" + return PROJECT_ROOT / "cactus" / "build" / f"libcactus{suffix}" + + +def _static_cactus_library_path() -> Path: + return PROJECT_ROOT / "cactus" / "build" / "libcactus.a" + + +def _build_static_cactus_library() -> Path: + build_script = PROJECT_ROOT / "cactus" / "build.sh" + if not build_script.exists(): + raise RuntimeError( + "The Cactus build script is missing.\n" + f"Expected: {build_script}" + ) + + build = subprocess.run([str(build_script)], cwd=PROJECT_ROOT / "cactus") + if build.returncode != 0: + raise RuntimeError("Failed to build the Cactus static runtime") + + static_library_path = _static_cactus_library_path() + if not static_library_path.exists(): + raise RuntimeError( + "The Cactus build completed, but the static library was not produced.\n" + f"Expected: {static_library_path}" + ) + return static_library_path + + +def _public_cactus_api_symbols(static_library_path: Path) -> list[str]: + if platform.system() == "Darwin": + command = ["nm", "-gU", str(static_library_path)] + else: + command = ["nm", "-g", "--defined-only", str(static_library_path)] + + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError( + "Failed to inspect the Cactus static runtime symbols.\n" + f"Command: {' '.join(command)}\n" + f"{result.stderr.strip()}" + ) + + symbols: list[str] = [] + for line in result.stdout.splitlines(): + parts = line.split() + if not parts: + continue + symbol = parts[-1].strip() + normalized = symbol[1:] if symbol.startswith("_") else symbol + if not normalized.startswith("cactus_"): + continue + if symbol not in symbols: + symbols.append(symbol) + if not symbols: + raise RuntimeError(f"Could not find any public cactus_* symbols in {static_library_path}") + return symbols + + +def _link_python_runtime_library(*, static_library_path: Path, library_path: Path) -> None: + build_dir = library_path.parent + build_dir.mkdir(parents=True, exist_ok=True) + if library_path.exists(): + library_path.unlink() + + exported_symbols = _public_cactus_api_symbols(static_library_path) + system = platform.system() + if system == "Darwin": + compiler = shutil.which("clang++") or shutil.which("c++") + if not compiler: + raise RuntimeError("Failed to find a C++ compiler for linking libcactus.dylib") + + command = [ + compiler, + "-dynamiclib", + "-o", + str(library_path), + *[f"-Wl,-u,{symbol}" for symbol in exported_symbols], + str(static_library_path), + "-Wl,-install_name,@rpath/libcactus.dylib", + "-lcurl", + "-framework", + "Accelerate", + "-framework", + "CoreML", + "-framework", + "Foundation", + "-framework", + "Security", + "-framework", + "SystemConfiguration", + "-framework", + "CFNetwork", + ] + else: + compiler = shutil.which("g++") or shutil.which("c++") + if not compiler: + raise RuntimeError("Failed to find a C++ compiler for linking libcactus.so") + + command = [ + compiler, + "-shared", + "-o", + str(library_path), + *[f"-Wl,--undefined={symbol}" for symbol in exported_symbols], + str(static_library_path), + "-lcurl", + "-pthread", + "-ldl", + "-lm", + ] + + result = subprocess.run(command, cwd=build_dir) + if result.returncode != 0 or not library_path.exists(): + raise RuntimeError( + "Failed to link the Cactus shared runtime.\n" + f"Expected: {library_path}" + ) + + +def _ensure_python_runtime_library() -> Path: + library_path = _python_runtime_library_path() + if library_path.exists(): + return library_path + + print_color(YELLOW, "Building Cactus shared runtime for transpiler...") + static_library_path = _build_static_cactus_library() + _link_python_runtime_library(static_library_path=static_library_path, library_path=library_path) + if not library_path.exists(): + raise RuntimeError( + "The Cactus build completed, but the shared library was not produced.\n" + f"Expected: {library_path}" + ) + return library_path # dont fail if models.json is missing or malformed - just have an empty registry and rely on HF download_args try: @@ -67,6 +212,11 @@ def print_color(color, message): print(f"{color}{message}{NC}") +def resolve_model_id_alias(model_id: str) -> str: + normalized = (model_id or "").strip() + return MODEL_ID_ALIASES.get(normalized.lower(), normalized) + + from .downloads import get_model_dir_name, get_weights_dir, download_from_hf as _download_from_hf_impl from .cq_download import ( combo_label, @@ -1031,25 +1181,10 @@ def cmd_build_python(args): print(" Ubuntu: sudo apt-get install cmake") return 1 - cactus_dir = PROJECT_ROOT / "cactus" - build_script = cactus_dir / "build.sh" - if not build_script.exists(): - print_color(RED, f"Error: build.sh not found at {build_script}") - return 1 - - result = run_command(str(build_script), cwd=cactus_dir, check=False) - if result.returncode != 0: - print_color(RED, "Build failed") - return 1 - - if platform.system() == "Darwin": - lib_name = "libcactus.dylib" - else: - lib_name = "libcactus.so" - - lib_path = cactus_dir / "build" / lib_name - if not lib_path.exists(): - print_color(RED, f"Shared library not found at {lib_path}") + try: + lib_path = _ensure_python_runtime_library() + except Exception as exc: + print_color(RED, f"Error: {exc}") return 1 print_color(GREEN, "Python build complete!") @@ -1078,8 +1213,89 @@ def prompt_for_api_key(config): return api_key +def _resolve_transpiled_manifest(path_value): + candidate = Path(path_value).expanduser().resolve() + if not candidate.exists(): + return None + if candidate.is_file() and candidate.name == "manifest.json": + return candidate + manifest = candidate / "components" / "manifest.json" + if manifest.exists(): + return manifest + manifest = candidate / "manifest.json" + if manifest.exists(): + return manifest + return None + + +def cmd_run_transpiled(args): + """Run a saved transpiled component bundle.""" + transpile_lib = _ensure_python_runtime_library() + os.environ["CACTUS_LIB_PATH"] = str(transpile_lib) + from .transpile.component_bundle_runtime import run_transpiled_bundle + + bundle_dir = getattr(args, "bundle_dir", None) or getattr(args, "model_id", None) + image_values: list[str] = [] + for attr_name in ("image_file", "image_files"): + value = getattr(args, attr_name, None) + if isinstance(value, str) and value.strip(): + image_values.append(value.strip()) + elif isinstance(value, (list, tuple)): + image_values.extend(str(item).strip() for item in value if str(item).strip()) + image_arg = getattr(args, "image", None) + if isinstance(image_arg, str) and image_arg.strip(): + image_values.append(image_arg.strip()) + result = run_transpiled_bundle( + bundle_dir, + audio_file=getattr(args, "audio_file", None) or getattr(args, "audio", None), + image_files=tuple(image_values), + prompt=getattr(args, "prompt", None), + input_ids=getattr(args, "input_ids", None), + weights_dir=getattr(args, "weights_dir", None), + system_prompt=getattr(args, "system", None), + enable_thinking=bool(getattr(args, "thinking", False)), + ) + print(json.dumps(result, indent=2, sort_keys=True)) + if getattr(args, "result_json", None): + result_path = Path(args.result_json).expanduser().resolve() + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print_color(GREEN, f"Saved result to {result_path}") + return 0 + + +def cmd_transpile(args): + """Invoke the saved-model transpiler entrypoint.""" + script_path = PROJECT_ROOT / "python" / "examples" / "transpile_hf_model.py" + if not script_path.exists(): + print_color(RED, f"Error: transpiler script not found at {script_path}") + return 1 + + transpile_lib = _ensure_python_runtime_library() + model_id = resolve_model_id_alias(args.model_id) + extra_args = list(getattr(args, "extra_args", []) or []) + command = [sys.executable, str(script_path), "--model-id", model_id] + if "--weights-dir" not in extra_args: + default_weights_dir = get_weights_dir(model_id) + if default_weights_dir.exists(): + command.extend(["--weights-dir", str(default_weights_dir)]) + if not getattr(args, "execute_after_transpile", False) and "--skip-execute" not in extra_args: + command.append("--skip-execute") + command.extend(extra_args) + env = os.environ.copy() + env["CACTUS_LIB_PATH"] = str(transpile_lib) + result = subprocess.run(command, cwd=PROJECT_ROOT, env=env) + return result.returncode + + def cmd_run(args): """Download model if needed and start interactive chat.""" + model_id_or_path = getattr(args, "model_id", DEFAULT_MODEL_ID) + manifest_path = _resolve_transpiled_manifest(model_id_or_path) + if manifest_path is not None: + args.bundle_dir = str(manifest_path.parent.parent if manifest_path.parent.name == "components" else manifest_path.parent) + return cmd_run_transpiled(args) + from .config_utils import CactusConfig config = CactusConfig() @@ -1088,7 +1304,7 @@ def cmd_run(args): if api_key: os.environ["CACTUS_CLOUD_KEY"] = api_key - model_id = args.model_id + model_id = resolve_model_id_alias(args.model_id) if getattr(args, 'no_cloud_tele', False): os.environ["CACTUS_NO_CLOUD_TELE"] = "1" @@ -2154,13 +2370,51 @@ def create_parser(): help='Path to image file for VLM inference (attached to first message)') run_parser.add_argument('--audio', help='Path to audio file (WAV) for audio chat (attached to first message)') + run_parser.add_argument('--file', dest='audio_file', default=None, + help='Audio file for transpiled bundles when model_id points to a transpiled folder') + run_parser.add_argument('--weights-dir', default=None, + help='Converted weights directory for transpiled bundles with bound weights') run_parser.add_argument('--system', help='System prompt to prepend to all messages') run_parser.add_argument('--prompt', help='Initial prompt to send immediately') + run_parser.add_argument('--input-ids', default=None, + help='Comma-separated token ids for transpiled causal-LM bundles') + run_parser.add_argument('--result-json', default=None, + help='Optional path to save transpiled bundle results as JSON') run_parser.add_argument('--thinking', action='store_true', help='Enable thinking/reasoning for models that support it') + transpile_parser = subparsers.add_parser('transpile', help='Transpile a HuggingFace model into Cactus component graphs') + transpile_parser.add_argument('model_id', help='HuggingFace model ID to transpile (aliases: gemma4, parakeet, whisper, qwen, lfm)') + transpile_parser.add_argument( + '--execute-after-transpile', + action='store_true', + help='Also run the transpiled graph immediately. By default `cactus transpile` saves the bundle and skips execution.', + ) + + run_transpiled_parser = subparsers.add_parser('run-transpiled', help='Run a saved transpiled component bundle') + run_transpiled_parser.add_argument('bundle_dir', + help='Bundle root directory or manifest.json path from transpile_hf_model.py') + run_transpiled_parser.add_argument('--file', dest='audio_file', default=None, + help='Audio file for transcription bundles such as parakeet-tdt') + run_transpiled_parser.add_argument('--image', default=None, + help='Image file for multimodal transpiled bundles') + run_transpiled_parser.add_argument('--image-file', action='append', default=[], + help='Repeatable image input for multimodal transpiled bundles') + run_transpiled_parser.add_argument('--weights-dir', default=None, + help='Optional converted weights directory for bound-weight bundles') + run_transpiled_parser.add_argument('--prompt', default=None, + help='Prompt for causal-LM bundles') + run_transpiled_parser.add_argument('--input-ids', default=None, + help='Comma-separated token ids for causal-LM bundles') + run_transpiled_parser.add_argument('--result-json', default=None, + help='Optional path to save the result payload as JSON') + run_transpiled_parser.add_argument('--system', default=None, + help='Optional system prompt for multimodal transpiled bundles') + run_transpiled_parser.add_argument('--thinking', action='store_true', + help='Enable thinking/reasoning markers when supported by the transpiled bundle') + transcribe_parser = subparsers.add_parser('transcribe', help='Download ASR model and run transcription') transcribe_parser.add_argument('model_id', nargs='?', default=DEFAULT_ASR_MODEL_ID, help=f'HuggingFace model ID (default: {DEFAULT_ASR_MODEL_ID})') @@ -2256,6 +2510,9 @@ def preprocess_eval_args(parser, argv): if getattr(args, 'command', None) == 'eval': setattr(args, 'extra_args', unknown) return args + if getattr(args, 'command', None) == 'transpile': + setattr(args, 'extra_args', unknown) + return args if unknown: parser.error(f"unrecognized arguments: {' '.join(unknown)}") @@ -2278,8 +2535,12 @@ def main(): sys.exit(cmd_build(args)) elif args.command == 'run': sys.exit(cmd_run(args)) + elif args.command == 'transpile': + sys.exit(cmd_transpile(args)) elif args.command == 'transcribe': sys.exit(cmd_transcribe(args)) + elif args.command == 'run-transpiled': + sys.exit(cmd_run_transpiled(args)) elif args.command == 'test': sys.exit(cmd_test(args)) elif args.command == 'eval': diff --git a/python/src/converter.py b/python/src/converter.py index 30720b120..e03e027a8 100644 --- a/python/src/converter.py +++ b/python/src/converter.py @@ -1,3 +1,4 @@ +import json import os import re from pathlib import Path @@ -74,10 +75,23 @@ def _gemma_tower_output_name(hf_key, strip_prefix, add_prefix): return add_prefix + name + ext -def convert_hf_model_weights(model, output_dir, precision='FP16', args=None): +def _gemma_weight_kind(source_name: str) -> str: + if source_name in { + 'model.language_model.embed_tokens_per_layer.weight', + 'model.embed_vision.embedding.weight', + 'model.embed_audio.embedding.weight', + }: + return "embedding" + return "weight" + + +def convert_hf_model_weights(model, output_dir, precision='INT8', args=None): """Convert HuggingFace model weights to Cactus binary format.""" import gc quantization_stats = create_quantization_stats() + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + weights_manifest: dict[str, dict[str, object]] = {} state_dict = model.state_dict() root_config = model.config @@ -138,6 +152,33 @@ def convert_hf_model_weights(model, output_dir, precision='FP16', args=None): print(" Normalized gemma4 audio tower key naming for conversion") state_dict = remapped_state_dict + def save_and_record( + tensor, + output_path, + tensor_precision, + *, + transpose=False, + source_name: str | None = None, + source_names: list[str] | tuple[str, ...] | None = None, + kind: str = "weight", + ): + save_tensor_with_header( + tensor, + output_path, + tensor_precision, + transpose=transpose, + stats_tracker=quantization_stats, + args=args, + model_type=detected_model_type, + ) + names: list[str] = [] + if source_name is not None: + names.append(source_name) + if source_names: + names.extend(str(name) for name in source_names) + for name in names: + weights_manifest[str(name)] = {"filename": Path(output_path).name, "kind": kind} + model_config = extract_base_config(config, root_config) model_config['tie_word_embeddings'] = tie_word_embeddings model_config['model_type'] = detected_model_type @@ -279,7 +320,14 @@ class _Args: for name in EMBED_NAMES: if name in state_dict: embedding_tensor = state_dict[name] - save_tensor_with_header(embedding_tensor, output_dir / "token_embeddings.weights", precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + embedding_tensor, + output_dir / "token_embeddings.weights", + precision, + transpose=False, + source_name=name, + kind="embedding", + ) saved_tensor_full_names.add(name) embedding_found = True break @@ -287,7 +335,14 @@ class _Args: if model_type_str == 'nomic_bert': if 'embeddings.word_embeddings.weight' in state_dict: fused_embedding_tensor = state_dict['embeddings.word_embeddings.weight'] + state_dict.get('embeddings.token_type_embeddings.weight', torch.zeros([1])) - save_tensor_with_header(fused_embedding_tensor, output_dir / "token_embeddings.weights", precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + fused_embedding_tensor, + output_dir / "token_embeddings.weights", + precision, + transpose=False, + source_names=['embeddings.word_embeddings.weight', 'embeddings.token_type_embeddings.weight'], + kind="embedding", + ) saved_tensor_full_names.add('embeddings.word_embeddings.weight') if 'embeddings.token_type_embeddings.weight' in state_dict: saved_tensor_full_names.add('embeddings.token_type_embeddings.weight') @@ -296,7 +351,7 @@ class _Args: elif model_type_str == 'whisper': for name, save_name in WHISPER_GLOBAL_WEIGHTS: if name in state_dict: - save_tensor_with_header(state_dict[name], output_dir / save_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(state_dict[name], output_dir / save_name, precision, transpose=False, source_name=name) saved_tensor_full_names.add(name) embedding_found = True model_config['num_mel_bins'] = int(cfg_get(config, 'num_mel_bins', 80)) @@ -309,8 +364,8 @@ class _Args: tensor = state_dict[name] if name == 'model.encoder.conv2.weight': tensor = tensor.permute(1, 2, 0).contiguous() - - save_tensor_with_header(tensor, output_dir / save_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + + save_and_record(tensor, output_dir / save_name, precision, transpose=False, source_name=name) saved_tensor_full_names.add(name) embedding_found = True model_config['dec_hidden_act'] = config.decoder_hidden_act @@ -330,7 +385,7 @@ class _Args: embedding_norm_names = {'emb_ln.weight': 'embedding_layernorm.weight', 'emb_ln.bias': 'embedding_layernorm.bias'} for name, file_name in embedding_norm_names.items(): if name in state_dict: - save_tensor_with_header(state_dict[name], output_dir / file_name, precision, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(state_dict[name], output_dir / file_name, precision, source_name=name) saved_tensor_full_names.add(name) if tie_word_embeddings: @@ -342,14 +397,14 @@ class _Args: for name in OUTPUT_NAMES: if name in state_dict: tensor = state_dict[name] - save_tensor_with_header(tensor, output_dir / "output_weight.weights", precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / "output_weight.weights", precision, transpose=False, source_name=name) saved_tensor_full_names.add(name) break for name in OUTPUT_NORM_NAMES: if name in state_dict: tensor = state_dict[name] - save_tensor_with_header(tensor, output_dir / "output_norm.weights", precision, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / "output_norm.weights", precision, source_name=name) saved_tensor_full_names.add(name) break @@ -361,10 +416,7 @@ class _Args: ] for key, out_name in mtp_global_mappings: if key in state_dict: - save_tensor_with_header( - state_dict[key], output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type - ) + save_and_record(state_dict[key], output_dir / out_name, precision, transpose=False, source_name=key) saved_tensor_full_names.add(key) mtp_layer_mappings = [ @@ -382,10 +434,7 @@ class _Args: ] for key, out_name in mtp_layer_mappings: if key in state_dict: - save_tensor_with_header( - state_dict[key], output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type - ) + save_and_record(state_dict[key], output_dir / out_name, precision, transpose=False, source_name=key) saved_tensor_full_names.add(key) if is_vlm: @@ -472,26 +521,52 @@ class _Args: for name, save_name in GEMMA4_GLOBAL_WEIGHTS: if name in state_dict: - save_tensor_with_header(state_dict[name], output_dir / save_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + state_dict[name], + output_dir / save_name, + precision, + transpose=False, + source_name=name, + kind=_gemma_weight_kind(name), + ) saved_tensor_full_names.add(name) text_hidden = int(model_config.get('hidden_dim', 0)) assert text_hidden > 0, "Hidden dim must be specified in config for gemma4 model" proj_norm = np.ones(text_hidden, dtype=np.float32) - save_tensor_with_header(proj_norm, output_dir / 'embed_vision_post_proj_norm.weights', 'FP16', - stats_tracker=quantization_stats, model_type=detected_model_type) + save_and_record( + proj_norm, + output_dir / 'embed_vision_post_proj_norm.weights', + 'FP16', + source_names=( + 'model.embed_vision.post_projection_norm.weight', + 'model.embed_vision.embedding_post_projection_norm.weight', + ), + ) for hf_key in sorted(state_dict.keys()): if hf_key.startswith(GEMMA4_VISION_TOWER_PREFIX) and hf_key not in saved_tensor_full_names: out_name = _gemma_tower_output_name(hf_key, GEMMA4_VISION_TOWER_PREFIX, 'vision_') - save_tensor_with_header(state_dict[hf_key], output_dir / out_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + state_dict[hf_key], + output_dir / out_name, + precision, + transpose=False, + source_name=hf_key, + ) saved_tensor_full_names.add(hf_key) del state_dict[hf_key] for hf_key in sorted(state_dict.keys()): if hf_key.startswith(GEMMA4_AUDIO_TOWER_PREFIX) and hf_key not in saved_tensor_full_names: out_name = _gemma_tower_output_name(hf_key, GEMMA4_AUDIO_TOWER_PREFIX, 'audio_') - save_tensor_with_header(state_dict[hf_key], output_dir / out_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + state_dict[hf_key], + output_dir / out_name, + precision, + transpose=False, + source_name=hf_key, + ) saved_tensor_full_names.add(hf_key) del state_dict[hf_key] @@ -536,9 +611,12 @@ class _Args: is_hwio_pw = tensor.shape[1] == 1 and tensor.shape[2] == 1 and tensor.shape[3] > 1 if is_hwio_k3 or is_hwio_pw: tensor = tensor.permute(0, 3, 1, 2).contiguous() - save_tensor_with_header( - tensor, output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + tensor, + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) @@ -594,9 +672,12 @@ class _Args: continue missing_tensors.append((i, out_name, [key])) continue - save_tensor_with_header( - state_dict[key], output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + state_dict[key], + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) tracked_key = layer_prefix + 'conv.norm.num_batches_tracked' @@ -638,9 +719,12 @@ class _Args: is_hwio_pw = tensor.shape[1] == 1 and tensor.shape[2] == 1 and tensor.shape[3] > 1 if is_hwio_k3 or is_hwio_pw: tensor = tensor.permute(0, 3, 1, 2).contiguous() - save_tensor_with_header( - tensor, output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + tensor, + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) @@ -660,6 +744,7 @@ class _Args: state_dict[key], output_dir / out_name, "FP16", transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type ) + weights_manifest[key] = {"filename": Path(out_name).name, "kind": "weight"} saved_tensor_full_names.add(key) for i in range(predictor_layers): @@ -673,9 +758,12 @@ class _Args: if key is None: missing_tensors.append((-1, out_name, candidate_keys)) continue - save_tensor_with_header( - state_dict[key], output_dir / out_name, "FP16", transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + state_dict[key], + output_dir / out_name, + "FP16", + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) @@ -692,9 +780,12 @@ class _Args: if key is None: missing_tensors.append((-1, out_name, candidate_keys)) continue - save_tensor_with_header( - state_dict[key], output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + state_dict[key], + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) @@ -845,9 +936,12 @@ class _Args: elif out_suffix == 'conv_depthwise.weights': if hasattr(tensor, 'shape') and len(tensor.shape) == 3 and tensor.shape[1] == 9 and tensor.shape[2] == 1: tensor = tensor.permute(0, 2, 1).contiguous() - save_tensor_with_header( - tensor, output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + tensor, + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) tracked_keys = [ @@ -892,7 +986,7 @@ class _Args: channel_output_name = output_name.replace('{channel}', str(channel_idx)) tensor = state_dict[full_name] - save_tensor_with_header(tensor, output_dir / channel_output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / channel_output_name, tensor_precision, transpose=should_transpose, source_name=full_name) saved_tensor_full_names.add(full_name) matched_any_channel = True @@ -923,8 +1017,8 @@ class _Args: if 'encoder' in layer_prefix: save_name_prefix = "encoder_" + save_name_prefix - save_tensor_with_header(w_gate, output_dir / (save_name_prefix + "ffn_gate.weights"), precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) - save_tensor_with_header(w_up, output_dir / (save_name_prefix + "ffn_up.weights"), precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(w_gate, output_dir / (save_name_prefix + "ffn_gate.weights"), precision, transpose=should_transpose, source_name=full_name) + save_and_record(w_up, output_dir / (save_name_prefix + "ffn_up.weights"), precision, transpose=should_transpose, source_name=full_name) if b is not None: b_up = b[:half_dim] @@ -947,10 +1041,7 @@ class _Args: v_dim = int(model_config.get('linear_v_proj_dim', 0)) row_dim = int(tensor.shape[0]) - save_tensor_with_header( - tensor, output_dir / output_name, tensor_precision, transpose=should_transpose, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type - ) + save_and_record(tensor, output_dir / output_name, tensor_precision, transpose=should_transpose, source_name=full_name) if q_dim > 0 and k_dim > 0 and v_dim > 0 and row_dim == (q_dim + k_dim + v_dim): q_weight = tensor[:q_dim, :] @@ -983,7 +1074,7 @@ class _Args: raise ValueError(f"Invalid tensor shape: {tensor.shape}") for j, ch in enumerate(['q', 'k', 'v']): channel_output_name = output_name.replace('{channel}', ch) - save_tensor_with_header(tensor[j], output_dir / channel_output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor[j], output_dir / channel_output_name, tensor_precision, transpose=should_transpose, source_name=full_name) saved_tensor_full_names.add(full_name) found = True break @@ -995,18 +1086,18 @@ class _Args: for expert_idx in range(num_experts): expert_tensor = tensor[expert_idx] expert_output_name = output_name.replace('{channel}', str(expert_idx)) - save_tensor_with_header(expert_tensor, output_dir / expert_output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(expert_tensor, output_dir / expert_output_name, tensor_precision, transpose=should_transpose, source_name=full_name) saved_tensor_full_names.add(full_name) found = True break if model_type_str == 'whisper': temp = layer_prefix[:layer_prefix.find('.')] + "." + output_name - save_tensor_with_header(tensor, output_dir / temp, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / temp, tensor_precision, transpose=should_transpose, source_name=full_name) elif model_type_str == 'moonshine' and 'encoder' in layer_prefix: enc_output_name = "encoder_" + output_name - save_tensor_with_header(tensor, output_dir / enc_output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / enc_output_name, tensor_precision, transpose=should_transpose, source_name=full_name) else: - save_tensor_with_header(tensor, output_dir / output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / output_name, tensor_precision, transpose=should_transpose, source_name=full_name) saved_tensor_full_names.add(full_name) found = True break @@ -1039,8 +1130,8 @@ class _Args: if 'encoder' in layer_prefix: save_name_prefix = "encoder_" + save_name_prefix - save_tensor_with_header(w_gate, output_dir / (save_name_prefix + "ffn_gate.weights"), precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) - save_tensor_with_header(w_up, output_dir / (save_name_prefix + "ffn_up.weights"), precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(w_gate, output_dir / (save_name_prefix + "ffn_gate.weights"), precision, transpose=should_transpose, source_name=w_name) + save_and_record(w_up, output_dir / (save_name_prefix + "ffn_up.weights"), precision, transpose=should_transpose, source_name=w_name) if b is not None: b_up = b[:half_dim] @@ -1063,9 +1154,9 @@ class _Args: k_weight = combined_weight[:, hidden_size:2*hidden_size] v_weight = combined_weight[:, 2*hidden_size:] - save_tensor_with_header(q_weight, output_dir / f'layer_{i}_attn_q.weights', precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) - save_tensor_with_header(k_weight, output_dir / f'layer_{i}_attn_k.weights', precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) - save_tensor_with_header(v_weight, output_dir / f'layer_{i}_attn_v.weights', precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(q_weight, output_dir / f'layer_{i}_attn_q.weights', precision, transpose=False, source_name=attn_name) + save_and_record(k_weight, output_dir / f'layer_{i}_attn_k.weights', precision, transpose=False, source_name=attn_name) + save_and_record(v_weight, output_dir / f'layer_{i}_attn_v.weights', precision, transpose=False, source_name=attn_name) saved_tensor_full_names.add(attn_name) found = True @@ -1092,6 +1183,10 @@ class _Args: fh.write(f"layer={layer_idx}, output={output_name}, patterns=[{pattern_list}]\n") print(f"Warning: {len(missing_tensors)} tensors were not exported. See {missing_report.name} for details.") + manifest_path = output_dir / "weights_manifest.json" + with open(manifest_path, "w") as fh: + json.dump(dict(sorted(weights_manifest.items())), fh, indent=2, sort_keys=True) + print_quantization_summary(quantization_stats, args) if detected_model_type in ['whisper', 'moonshine', 'parakeet', 'parakeet_tdt']: @@ -1340,7 +1435,7 @@ def _build_needle_config(model_cfg, params, precision): hidden_dim = int(model_cfg["d_model"]) heads = int(model_cfg["num_heads"]) decoder_layers = int(model_cfg["num_decoder_layers"]) - compute_precision = "FP16" + compute_precision = "FP16" if precision in ("INT4", "INT8") else precision return { "model_type": "needle", "precision": compute_precision, diff --git a/python/src/graph.py b/python/src/graph.py index 52f706c6e..b6a257963 100644 --- a/python/src/graph.py +++ b/python/src/graph.py @@ -1,7 +1,8 @@ import ctypes +from os import wait import numpy as np -from .cactus import _lib, cactus_node_t, cactus_tensor_info_t +from .cactus import _err, _lib, cactus_node_t, cactus_tensor_info_t class Graph: @@ -12,6 +13,7 @@ class Graph: CQ2 = 4 CQ3 = 5 CQ4 = 6 + INT4 = CQ1 CPU = 0 NPU = 1 ACT_SILU = 0 @@ -52,7 +54,7 @@ def input(self, shape, dtype=FP16): out = cactus_node_t() rc = _lib.cactus_graph_input(self.h, arr, len(shape), int(dtype), ctypes.byref(out)) if rc != 0: - raise RuntimeError("graph_input failed") + raise RuntimeError(_err("graph_input failed")) return self._tensor_from_node(out.value) def set_input(self, tensor, data, dtype=None): @@ -62,6 +64,25 @@ def set_input(self, tensor, data, dtype=None): raise ValueError("tensor belongs to a different graph") target_dtype = int(tensor.dtype if dtype is None else dtype) arr = self._coerce_input_array(data, target_dtype) + info = self._get_output_info(tensor.id) + expected_shape = tuple(int(x) for x in info["shape"]) + expected_num_elements = int(info["num_elements"]) + expected_byte_size = int(info["byte_size"]) + if tuple(int(x) for x in arr.shape) != expected_shape: + raise ValueError( + "graph input shape mismatch for node " + f"{tensor.id}: expected {expected_shape}, got {tuple(int(x) for x in arr.shape)}" + ) + if int(arr.size) != expected_num_elements: + raise ValueError( + "graph input element-count mismatch for node " + f"{tensor.id}: expected {expected_num_elements}, got {int(arr.size)}" + ) + if int(arr.nbytes) != expected_byte_size: + raise ValueError( + "graph input byte-size mismatch for node " + f"{tensor.id}: expected {expected_byte_size}, got {int(arr.nbytes)}" + ) rc = _lib.cactus_graph_set_input( self.h, cactus_node_t(tensor.id), @@ -69,7 +90,7 @@ def set_input(self, tensor, data, dtype=None): target_dtype, ) if rc != 0: - raise RuntimeError("graph_set_input failed") + raise RuntimeError(_err("graph_set_input failed")) def set_external_input(self, tensor, data_ptr, dtype=None): if not isinstance(tensor, Tensor): @@ -85,7 +106,7 @@ def set_external_input(self, tensor, data_ptr, dtype=None): target_dtype, ) if rc != 0: - raise RuntimeError("graph_set_external_input failed") + raise RuntimeError(_err("graph_set_external_input failed")) def hard_reset(self): rc = _lib.cactus_graph_hard_reset(self.h) @@ -95,7 +116,7 @@ def hard_reset(self): def execute(self): rc = _lib.cactus_graph_execute(self.h) if rc != 0: - raise RuntimeError("graph_execute failed") + raise RuntimeError(_err("graph_execute failed")) def add(self, a, b): return self._binary("cactus_graph_add", a, b) @@ -112,6 +133,9 @@ def multiply(self, a, b): def divide(self, a, b): return self._binary("cactus_graph_divide", a, b) + def not_equal(self, a, b): + return self._binary("cactus_graph_not_equal", a, b) + def abs(self, x): x = self._ensure_tensor(x) out = cactus_node_t() @@ -136,6 +160,13 @@ def precision_cast(self, x, dtype): raise RuntimeError("graph_precision_cast failed") return self._tensor_from_node(out.value) + def quantize_activations(self, x): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_quantize_activations(self.h, cactus_node_t(x.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_quantize_activations failed") + return self._tensor_from_node(out.value) def _scalar(self, fn_name, x, value=None): x = self._ensure_tensor(x) @@ -161,6 +192,12 @@ def scalar_multiply(self, x, value): def scalar_divide(self, x, value): return self._scalar("cactus_graph_scalar_divide", x, value) + def scalar_floor_divide(self, x, value): + return self._scalar("cactus_graph_scalar_floor_divide", x, value) + + def scalar_not_equal(self, x, value): + return self._scalar("cactus_graph_scalar_not_equal", x, value) + def scalar_exp(self, x): return self._scalar("cactus_graph_scalar_exp", x) @@ -176,6 +213,25 @@ def scalar_sin(self, x): def scalar_log(self, x): return self._scalar("cactus_graph_scalar_log", x) + def masked_select_prefix(self, x, mask): + return self._binary("cactus_graph_masked_select_prefix", x, mask) + + def masked_scatter(self, x, mask, source): + x = self._ensure_tensor(x) + mask = self._ensure_tensor(mask) + source = self._ensure_tensor(source) + out = cactus_node_t() + rc = _lib.cactus_graph_masked_scatter( + self.h, + cactus_node_t(x.id), + cactus_node_t(mask.id), + cactus_node_t(source.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_masked_scatter failed")) + return self._tensor_from_node(out.value) + def view(self, x, shape): x = self._ensure_tensor(x) shape = tuple(int(v) for v in shape) @@ -196,6 +252,16 @@ def reshape(self, x, shape): raise RuntimeError("graph_reshape failed") return self._tensor_from_node(out.value) + def expand(self, x, shape): + x = self._ensure_tensor(x) + shape = tuple(int(v) for v in shape) + arr = (ctypes.c_size_t * len(shape))(*shape) + out = cactus_node_t() + rc = _lib.cactus_graph_expand(self.h, cactus_node_t(x.id), arr, len(shape), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_expand failed") + return self._tensor_from_node(out.value) + def flatten(self, x, start_dim=0, end_dim=-1): x = self._ensure_tensor(x) out = cactus_node_t() @@ -269,27 +335,35 @@ def permute(self, x, permutation, backend=CPU): raise RuntimeError("graph_transpose_n failed") return self._tensor_from_node(out.value) - def matmul(self, a, b, pretransposed_rhs=False, backend=CPU): + def matmul(self, a, b, pretransposed_rhs=False, backend=CPU, output_dtype=None): a = self._ensure_tensor(a) b = self._ensure_tensor(b) out = cactus_node_t() + target_dtype = int(a.dtype if output_dtype is None else output_dtype) rc = _lib.cactus_graph_matmul( self.h, cactus_node_t(a.id), cactus_node_t(b.id), ctypes.c_bool(bool(pretransposed_rhs)), ctypes.c_int32(int(backend)), + ctypes.c_int32(target_dtype), ctypes.byref(out), ) if rc != 0: - raise RuntimeError("graph_matmul failed") + raise RuntimeError(_err("graph_matmul failed")) return self._tensor_from_node(out.value) - def gather(self, tensor, indices): + def gather(self, tensor, indices, axis=0): tensor = self._ensure_tensor(tensor) indices = self._ensure_tensor(indices) out = cactus_node_t() - rc = _lib.cactus_graph_gather(self.h, cactus_node_t(tensor.id), cactus_node_t(indices.id), ctypes.byref(out)) + rc = _lib.cactus_graph_gather( + self.h, + cactus_node_t(tensor.id), + cactus_node_t(indices.id), + ctypes.c_int32(int(axis)), + ctypes.byref(out), + ) if rc != 0: raise RuntimeError("graph_gather failed") return self._tensor_from_node(out.value) @@ -337,6 +411,27 @@ def bilinear_interpolation(self, pos_embeds, dst_height, dst_width): raise RuntimeError("graph_bilinear_interpolation failed") return self._tensor_from_node(out.value) + def set_grouped_scales(self, tensor, group_size, num_groups, scales): + tensor = self._ensure_tensor(tensor) + arr = np.ascontiguousarray(scales, dtype=np.float16) + rc = _lib.cactus_graph_set_grouped_scales( + self.h, + cactus_node_t(tensor.id), + ctypes.c_size_t(int(group_size)), + ctypes.c_size_t(int(num_groups)), + arr.ctypes.data_as(ctypes.c_void_p), + ) + if rc != 0: + raise RuntimeError("graph_set_grouped_scales failed") + + def set_interleaved(self, tensor, interleaved=True, original_n=0): + tensor = self._ensure_tensor(tensor) + rc = _lib.cactus_graph_set_interleaved( + self.h, cactus_node_t(tensor.id), ctypes.c_bool(bool(interleaved)), ctypes.c_size_t(int(original_n)) + ) + if rc != 0: + raise RuntimeError("graph_set_interleaved failed") + def release_weight_pages(self, tensor): tensor = self._ensure_tensor(tensor) rc = _lib.cactus_graph_release_weight_pages(self.h, cactus_node_t(tensor.id)) @@ -373,18 +468,13 @@ def cat(self, tensors, axis=0): tensors = [self._ensure_tensor(t) for t in tensors] if not tensors: raise ValueError("cat requires at least one tensor") - ids = (cactus_node_t * len(tensors))(*(cactus_node_t(t.id) for t in tensors)) - out = cactus_node_t() - rc = _lib.cactus_graph_cat( - self.h, - ids, - ctypes.c_size_t(len(tensors)), - ctypes.c_int32(int(axis)), - ctypes.byref(out), - ) - if rc != 0: - raise RuntimeError("graph_cat failed") - return self._tensor_from_node(out.value) + if len(tensors) == 1: + return tensors[0] + + result = tensors[0] + for tensor in tensors[1:]: + result = self.concat(result, tensor, axis=axis) + return result def groupnorm(self, x, weight, bias, num_groups, eps=1e-5): x = self._ensure_tensor(x) @@ -524,6 +614,9 @@ def min(self, x, axis): def max(self, x, axis): return self._reduce("cactus_graph_max", x, axis) + + def cumsum(self, x, axis): + return self._reduce("cactus_graph_cumsum", x, axis) def softmax(self, x, axis=-1): x = self._ensure_tensor(x) @@ -719,6 +812,21 @@ def conv2d_depthwise_k3s2p1(self, x, weight, bias=None): def conv2d_pointwise_1x1(self, x, weight, bias=None): return self._conv_with_optional_bias("cactus_graph_conv2d_pointwise_1x1", x, weight, bias) + def conv2d_k3s1p1(self, x, weight, bias=None): + return self._conv_with_optional_bias("cactus_graph_conv2d_k3s1p1", x, weight, bias) + + def conv2d(self, x, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + return self._conv_with_optional_bias( + "cactus_graph_conv2d", + x, + weight, + bias, + ctypes.c_size_t(int(stride)), + ctypes.c_size_t(int(padding)), + ctypes.c_size_t(int(dilation)), + ctypes.c_size_t(int(groups)), + ) + def _conv_with_optional_bias(self, fn_name, x, weight, bias=None, *extra): x = self._ensure_tensor(x) weight = self._ensure_tensor(weight) @@ -934,6 +1042,8 @@ def _coerce_input_array(self, data, precision): arr = np.ascontiguousarray(arr, dtype=np.float16) elif precision == self.FP32: arr = np.ascontiguousarray(arr, dtype=np.float32) + elif precision in (self.INT4, self.CQ2, self.CQ3, self.CQ4): + arr = np.ascontiguousarray(arr, dtype=np.uint8) else: raise RuntimeError("unsupported precision") return arr @@ -958,6 +1068,11 @@ def __mul__(self, other): def __truediv__(self, other): return self.g.divide(self, other) + def __ne__(self, other): + if isinstance(other, Tensor): + return self.g.not_equal(self, other) + return self.g.scalar_not_equal(self, other) + def abs(self): return self.g.abs(self) @@ -967,6 +1082,8 @@ def pow(self, exponent): def precision_cast(self, dtype): return self.g.precision_cast(self, dtype) + def quantize_activations(self): + return self.g.quantize_activations(self) def scalar_add(self, value): return self.g.scalar_add(self, value) @@ -980,6 +1097,12 @@ def scalar_multiply(self, value): def scalar_divide(self, value): return self.g.scalar_divide(self, value) + def scalar_floor_divide(self, value): + return self.g.scalar_floor_divide(self, value) + + def scalar_not_equal(self, value): + return self.g.scalar_not_equal(self, value) + def scalar_exp(self): return self.g.scalar_exp(self) @@ -1019,6 +1142,9 @@ def view(self, shape): def reshape(self, shape): return self.g.reshape(self, shape) + def expand(self, shape): + return self.g.expand(self, shape) + def flatten(self, start_dim=0, end_dim=-1): return self.g.flatten(self, start_dim=start_dim, end_dim=end_dim) @@ -1067,8 +1193,14 @@ def softmax(self, axis=-1): def glu(self, axis=-1): return self.g.glu(self, axis=axis) - def matmul(self, other, pretransposed_rhs=False, backend=Graph.CPU): - return self.g.matmul(self, other, pretransposed_rhs=pretransposed_rhs, backend=backend) + def matmul(self, other, pretransposed_rhs=False, backend=Graph.CPU, output_dtype=None): + return self.g.matmul( + self, + other, + pretransposed_rhs=pretransposed_rhs, + backend=backend, + output_dtype=output_dtype, + ) def sum(self, axis): return self.g.sum(self, axis) @@ -1085,6 +1217,9 @@ def min(self, axis): def max(self, axis): return self.g.max(self, axis) + def cumsum(self, axis): + return self.g.cumsum(self, axis) + def numpy(self): info = cactus_tensor_info_t() rc = _lib.cactus_graph_get_output_info(self.g.h, cactus_node_t(self.id), ctypes.byref(info)) @@ -1107,6 +1242,9 @@ def numpy(self): arr = np.ctypeslib.as_array((ctypes.c_float * num_elements).from_address(out_ptr.value)) elif precision == Graph.INT8: arr = np.ctypeslib.as_array((ctypes.c_int8 * num_elements).from_address(out_ptr.value)) + elif precision in (Graph.INT4, Graph.CQ2, Graph.CQ3, Graph.CQ4): + arr = np.ctypeslib.as_array((ctypes.c_uint8 * int(info.byte_size)).from_address(out_ptr.value)) + return arr.copy() else: raise RuntimeError("unsupported precision") diff --git a/python/src/tensor_io.py b/python/src/tensor_io.py index 25512269a..726903482 100644 --- a/python/src/tensor_io.py +++ b/python/src/tensor_io.py @@ -2,6 +2,7 @@ import struct from pathlib import Path from typing import Optional, Dict, Any, List +from .weight_patterns import EMBED_NAMES try: import torch @@ -9,11 +10,16 @@ torch = None +GROUP_SIZE = 32 +INTERLEAVE_BLOCK = 4 + CACTUS_MAGIC = b'CACT' CACTUS_ALIGNMENT = 32 +FLAG_HAS_SCALES = 1 << 0 FLAG_PAGE_ALIGNED = 1 << 1 FLAG_TRANSPOSED = 1 << 2 +FLAG_INTERLEAVED = 1 << 3 def align_offset(offset: int, alignment: int) -> int: @@ -85,6 +91,25 @@ def interleave_scales(scales: np.ndarray, block_size: int = INTERLEAVE_BLOCK) -> return scales.reshape(-1), original_N +def pack_int4_pairs(data: np.ndarray) -> np.ndarray: + """Pack INT4 values (stored as int8) into bytes using planar layout. + + Input: array of int8 values in range [-8, 7], length must be a multiple of 32 + Output: array of uint8 with half the length + + Packing format (planar, groups of 32): + For each group of 32 values, the first 16 are stored in the low nibbles + and the next 16 in the high nibbles of 16 consecutive bytes. + Nibbles are stored in two's complement form (value & 0x0F). + """ + assert len(data) % 32 == 0, "Data length must be a multiple of 32 for INT4 planar packing" + + groups = data.reshape(-1, 32) + low = (groups[:, :16].astype(np.int8).view(np.uint8) & 0x0F).astype(np.uint8) + high = ((groups[:, 16:].astype(np.int8).view(np.uint8) & 0x0F).astype(np.uint8)) << 4 + + return (low | high).astype(np.uint8).reshape(-1) + def fold_bn_into_conv(conv_w, bn_weight, bn_bias, bn_mean, bn_var, eps=1e-5): gamma = bn_weight.float().numpy() @@ -101,13 +126,18 @@ def fold_bn_into_conv(conv_w, bn_weight, bn_bias, bn_mean, bn_var, eps=1e-5): return new_w, new_b -def save_tensor_with_header(tensor, output_path, precision='FP16', transpose=False, stats_tracker=None, args=None, model_type=None): - """Save a tensor to binary format with header metadata. +def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=False, stats_tracker=None, args=None, model_type=None): + """Save a tensor to binary format with header metadata and group-wise quantization. + + For 2D tensors with INT8/INT4 precision: + - INT4 is packed (2 values per byte) for 50% storage savings, unpacked to INT8 at load time + - Weights are interleaved in blocks of 4 rows for SIMD efficiency + - Layout: [N/4, K/4, 4, 4] enables vdotq_laneq_s32 for efficient GEMV/GEMM Args: tensor: The tensor to save (PyTorch or NumPy) output_path: Path to save the tensor - precision: Storage precision ('FP16') + precision: Quantization precision ('INT4', 'INT8', 'FP16') transpose: Whether to transpose 2D tensors stats_tracker: Optional dict to track quantization statistics args: Optional args object with additional settings @@ -141,6 +171,21 @@ def save_tensor_with_header(tensor, output_path, precision='FP16', transpose=Fal elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE + if precision in ('INT8', 'INT4'): + filename = output_path.name + if any(x in filename for x in ['norm', 'bias', 'vision', 'position_embeddings', 'embed_positions']): + precision = 'FP16' + elif model_type == 'gemma4' and ( + filename.startswith('audio_') or + filename in ('embed_audio_proj.weights', 'embed_audio_embedding.weights') + ): + precision = 'FP16' + elif precision == 'INT4' and ( + any(x in filename for x in EMBED_NAMES) + or filename in ('token_embeddings.weights', 'embed_tokens_per_layer.weights') + ): + precision = 'INT8' + shape = list(data.shape) if transpose and len(shape) == 2: data = data.T @@ -155,6 +200,253 @@ def save_tensor_with_header(tensor, output_path, precision='FP16', transpose=Fal data = data.transpose(0, 2, 1) shape = [shape[0], shape[2], shape[1]] + if precision == 'INT8' and len(shape) == 2: + N, K = shape + original_N = N + + if K % GROUP_SIZE != 0: + pad_k = GROUP_SIZE - (K % GROUP_SIZE) + data = np.pad(data, ((0, 0), (0, pad_k)), mode='constant', constant_values=0) + K = data.shape[1] + + num_groups = K // GROUP_SIZE + + data_grouped = data.reshape(N, num_groups, GROUP_SIZE) + group_abs_max = np.max(np.abs(data_grouped), axis=2) + scales = (group_abs_max / 127.0).astype(np.float32) + scales = np.maximum(scales, 1e-10) + + quantized = np.clip( + np.round(data_grouped / scales[:, :, np.newaxis]), + -128, 127 + ).astype(np.int8) + quantized_2d = quantized.reshape(N, K) + + dequantized = (quantized.astype(np.float32) * scales[:, :, np.newaxis]).reshape(N, K) + mse_error = np.mean((data[:original_N, :] - dequantized[:original_N, :]) ** 2) + snr_db = 10 * np.log10(np.var(data[:original_N, :]) / mse_error) if mse_error > 0 else float('inf') + data_flat = data[:original_N, :].flatten() + dequant_flat = dequantized[:original_N, :].flatten() + cos_sim = np.dot(data_flat, dequant_flat) / (np.linalg.norm(data_flat) * np.linalg.norm(dequant_flat) + 1e-10) + del data_flat, dequantized + + quantized_interleaved, _ = interleave_weights(quantized_2d, INTERLEAVE_BLOCK) + scales_interleaved, _ = interleave_scales(scales, INTERLEAVE_BLOCK) + scales_fp16 = scales_interleaved.astype(np.float16) + + N_padded = ((N + INTERLEAVE_BLOCK - 1) // INTERLEAVE_BLOCK) * INTERLEAVE_BLOCK + + if stats_tracker: + stats_tracker['int8_tensors'] += 1 + stats_tracker['quantized_parameters'] += original_N * K + stats_tracker['mse_values'].append(mse_error) + stats_tracker['snr_values'].append(snr_db) + stats_tracker['cos_sim_values'].append(cos_sim) + stats_tracker['total_tensors'] += 1 + stats_tracker['total_parameters'] += original_N * K + + del data + + with open(output_path, 'wb') as f: + ndim = 2 + data_bytes = quantized_interleaved.size + scales_bytes = scales_fp16.size * 2 + flags = FLAG_HAS_SCALES | FLAG_INTERLEAVED + if transpose: + flags |= FLAG_TRANSPOSED + + f.write(CACTUS_MAGIC) # 4 bytes + f.write(struct.pack(' 0 else float('inf') + data_flat = data[:original_N, :].flatten() + dequant_flat = dequantized[:original_N, :].flatten() + cos_sim = np.dot(data_flat, dequant_flat) / (np.linalg.norm(data_flat) * np.linalg.norm(dequant_flat) + 1e-10) + del data_flat, dequantized + + quantized_interleaved, _ = interleave_weights(quantized_2d, INTERLEAVE_BLOCK) + scales_interleaved, _ = interleave_scales(scales, INTERLEAVE_BLOCK) + scales_fp16 = scales_interleaved.astype(np.float16) + + packed_data = pack_int4_pairs(quantized_interleaved) + + N_padded = ((N + INTERLEAVE_BLOCK - 1) // INTERLEAVE_BLOCK) * INTERLEAVE_BLOCK + + if stats_tracker: + stats_tracker['int4_tensors'] += 1 + stats_tracker['quantized_parameters'] += original_N * K + stats_tracker['mse_values'].append(mse_error) + stats_tracker['snr_values'].append(snr_db) + stats_tracker['cos_sim_values'].append(cos_sim) + stats_tracker['total_tensors'] += 1 + stats_tracker['total_parameters'] += original_N * K + + del data + + with open(output_path, 'wb') as f: + ndim = 2 + data_bytes = packed_data.size + scales_bytes = scales_fp16.size * 2 + flags = FLAG_HAS_SCALES | FLAG_INTERLEAVED + if transpose: + flags |= FLAG_TRANSPOSED + + f.write(CACTUS_MAGIC) # 4 bytes + f.write(struct.pack(' 0 else float('inf') + cos_sim = np.dot(data, dequantized) / (np.linalg.norm(data) * np.linalg.norm(dequantized) + 1e-10) + del dequantized + + scales_fp16 = scales.flatten().astype(np.float16) + + if stats_tracker: + stats_tracker['int8_tensors'] += 1 + stats_tracker['quantized_parameters'] += data.size + stats_tracker['mse_values'].append(mse_error) + stats_tracker['snr_values'].append(snr_db) + stats_tracker['cos_sim_values'].append(cos_sim) + stats_tracker['total_tensors'] += 1 + stats_tracker['total_parameters'] += data.size + + with open(output_path, 'wb') as f: + ndim = len(shape) + data_bytes = quantized_flat.size + scales_bytes = scales_fp16.size * 2 + flags = FLAG_HAS_SCALES + if transpose: + flags |= FLAG_TRANSPOSED + + f.write(CACTUS_MAGIC) + f.write(struct.pack(' 0: - print(f"\nConversion Summary: {fp16_count} FP16 tensors, {params} total parameters") + quantized_count = int8_count + int4_count + + if quantized_count > 0: + mse_values = np.array(quantization_stats['mse_values']) + snr_values = np.array(quantization_stats['snr_values']) + cos_sim_values = np.array(quantization_stats['cos_sim_values']) + + print("\nQuantization Summary:") + print(f"MSE - Mean: {np.mean(mse_values):.2e}, Max: {np.max(mse_values):.2e}, Median: {np.median(mse_values):.2e}, Min: {np.min(mse_values):.2e}") + print(f"SNR - Mean: {np.mean(snr_values):.1f}dB, Max: {np.max(snr_values):.1f}dB, Median: {np.median(snr_values):.1f}dB, Min: {np.min(snr_values):.1f}dB") + print(f"CosSim - Mean: {np.mean(cos_sim_values):.6f}, Max: {np.max(cos_sim_values):.6f}, Median: {np.median(cos_sim_values):.6f}, Min: {np.min(cos_sim_values):.6f}") + print(f"Processed {int8_count} INT8 tensors, {int4_count} INT4 tensors, {fp16_count} FP16 tensors") diff --git a/python/src/transpile/README.md b/python/src/transpile/README.md new file mode 100644 index 000000000..ce57bb68d --- /dev/null +++ b/python/src/transpile/README.md @@ -0,0 +1,1073 @@ +# Cactus Transpiler + +This folder contains the Python side of the Cactus transpiler: the code that takes +a Hugging Face or PyTorch model, captures it with `torch.export`, converts that +export into Cactus' intermediate representation (`IRGraph`), canonicalizes and +optimizes the IR, lowers it into a runtime `Graph`, and optionally saves or runs +the result. + +The transpiler has two main output shapes: + +- A single lowered graph for simple tasks such as text logits or encoder-only audio. +- A component bundle for models that are easier to split into staged graphs, most + notably Gemma4 multimodal and Parakeet TDT. + +This README is a code-first walkthrough of the actual flow in the repo today. + +## Related Entry Points + +These files are outside `python/src/transpile`, but they are the public top-level +entry points for the transpiler: + +- `python/src/cli.py` +- `python/examples/transpile_hf_model.py` + +`cactus transpile` is a thin CLI wrapper around `python/examples/transpile_hf_model.py`. +The example script is the real top-level transpile program. + +## CLI Entry Flow + +### `cactus transpile` + +`python/src/cli.py` resolves the transpiler script, ensures the Python runtime +library is available, and forwards most arguments to the example script: + +```python +def cmd_transpile(args): + script_path = PROJECT_ROOT / "python" / "examples" / "transpile_hf_model.py" + transpile_lib = _ensure_python_runtime_library() + model_id = resolve_model_id_alias(args.model_id) + extra_args = list(getattr(args, "extra_args", []) or []) + command = [sys.executable, str(script_path), "--model-id", model_id] + if "--weights-dir" not in extra_args: + default_weights_dir = get_weights_dir(model_id) + if default_weights_dir.exists(): + command.extend(["--weights-dir", str(default_weights_dir)]) + if not getattr(args, "execute_after_transpile", False) and "--skip-execute" not in extra_args: + command.append("--skip-execute") + command.extend(extra_args) +``` + +Two important details: + +- `cactus transpile` saves artifacts and skips execution by default. +- The CLI only has an explicit `--execute-after-transpile` flag; most other + transpile options are forwarded as unknown args to `transpile_hf_model.py`. + +### `cactus run-transpiled` + +Saved bundles are executed through `cmd_run_transpiled()` in `python/src/cli.py`: + +```python +def cmd_run_transpiled(args): + transpile_lib = _ensure_python_runtime_library() + os.environ["CACTUS_LIB_PATH"] = str(transpile_lib) + from .transpile.component_bundle_runtime import run_transpiled_bundle + + result = run_transpiled_bundle( + bundle_dir, + audio_file=getattr(args, "audio_file", None) or getattr(args, "audio", None), + image_files=tuple(image_values), + prompt=getattr(args, "prompt", None), + input_ids=getattr(args, "input_ids", None), + weights_dir=getattr(args, "weights_dir", None), + system_prompt=getattr(args, "system", None), + enable_thinking=bool(getattr(args, "thinking", False)), + ) +``` + +`cactus run ` also auto-detects transpiled bundles by looking for a +`manifest.json` and dispatches to the same runtime path. + +## End-To-End Timeline + +## 1. Parse Args, Infer Task, Load Model Bundle + +The real transpile program lives in `python/examples/transpile_hf_model.py`. +Its `main()` function decides what kind of model/task it is dealing with, validates +optional converted weights, and loads the HF model plus tokenizer/processor: + +```python +if args.task == "auto": + inferred_task = _infer_task_from_config(args.model_id) + ... + if image_files or (args.audio_file and has_multimodal_config and inferred_task == "causal_lm_logits"): + task = "multimodal_causal_lm_logits" + else: + task = inferred_task +else: + task = args.task + +validated_weights_dir = _validate_weights_dir(args.weights_dir.strip() or None, model_id=args.model_id) +weights_dir = str(validated_weights_dir) if validated_weights_dir is not None else None + +model_source, processor_or_tokenizer, model, model_config = _load_transformers_bundle( + model_id=args.model_id, + task=task, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, +) +``` + +### What decides the task? + +`_infer_task_from_config()` looks at: + +- `config.json` architectures +- model type +- known special cases like Parakeet TDT and Whisper +- model id substrings as a last fallback + +Today the main task shapes are: + +- `causal_lm_logits` +- `multimodal_causal_lm_logits` +- `ctc_logits` +- `encoder_hidden_states` +- `tdt_transcription` + +## 2. Prepare Example Inputs + +Before capture, the transpiler builds one concrete example input batch. That +example batch drives both `torch.export` capture and later optional execution. + +The high-level branches are: + +- `_prepare_text_inputs()` for text-only causal LM capture +- `_prepare_audio_inputs()` for speech/audio tasks +- `_prepare_gemma4_multimodal_inputs()` for Gemma4 multimodal + +Example: + +```python +if task == "causal_lm_logits": + prepared = _prepare_text_inputs(...) +elif task == "multimodal_causal_lm_logits": + prepared = _prepare_gemma4_multimodal_inputs(...) +elif task == "tdt_transcription": + prepared = _prepare_audio_inputs(...) +else: + prepared = _prepare_audio_inputs(...) +``` + +The `PreparedInputs` object carries: + +- `names`: logical input names like `input_ids` or `input_features` +- `tensors`: the actual example tensors +- `metadata`: prompt/audio/image/sample-rate info that later gets written into + manifests + +## 3. Canonicalize The Model Interface + +The transpiler does not try to capture every model's native HF forward signature +as-is. It first wraps the model in an adapter that exposes a smaller, more stable +task-specific interface. + +That happens in `model_adapters.py`: + +```python +def canonicalize_model_interface(model, task="causal_lm_logits", *, input_names=None, weights_dir=None): + family = _family_key(model) + ... + if task == "causal_lm_logits": + ... + elif task == "multimodal_causal_lm_logits": + adapter_factory = lambda inner_model: Gemma4MultimodalCausalLMLogitsAdapter( + inner_model, + input_names=resolved_input_names, + weights_dir=weights_dir, + ) + elif task == "ctc_logits": + adapter_factory = lambda inner_model: CTCLogitsAdapter(...) + elif task == "encoder_hidden_states": + adapter_factory = lambda inner_model: EncoderHiddenStatesAdapter(...) +``` + +This step is where family-specific knowledge lives: + +- Gemma/Gemma3/Gemma4/Qwen causal LM wrappers +- Gemma4 multimodal wrappers +- generic CTC and encoder wrappers + +The adapter is also where graph metadata and import hints get attached. + +## 4. Decide Between Monolithic Capture And Component Capture + +After input prep, the script asks `build_component_module_specs()` whether the +current model supports a split component pipeline: + +```python +component_specs = build_component_module_specs( + model, + task=task, + named_tensors=_named_tensor_store(prepared), + weights_dir=weights_dir, +) +``` + +Today that returns component specs for: + +- Gemma4 multimodal: `vision_encoder`, `audio_encoder`, `lm_encoder`, `decoder` +- Parakeet TDT: `audio_encoder`, `decoder` + +If component specs exist and `--component-pipeline` is `auto` or `on`, the script +takes the component path. Otherwise it captures a single wrapped model graph. + +## 5A. Component Pipeline Path + +The component path is driven by `_run_component_pipeline_transpile()` in +`python/examples/transpile_hf_model.py`: + +```python +for spec in component_specs: + captured_components[spec.component] = capture_component_spec(spec, fusion_config=fusion_config) +``` + +Each component spec is captured by `component_pipeline.capture_component_spec()`: + +```python +captured = capture_model(wrapped, spec.example_inputs) +raw_ir_graph = copy.deepcopy(captured.ir_graph) +optimized_ir_graph = copy.deepcopy(captured.ir_graph) +canonicalize_exported_graph(optimized_ir_graph) +optimize_graph(optimized_ir_graph, config=fusion_config) +transpiled_graph = transpile_preoptimized_ir(copy.deepcopy(optimized_ir_graph)) +``` + +So each component goes through the same capture/import/canonicalize/optimize/lower +pipeline independently. + +For Gemma4 multimodal, `model_adapters.py` builds the specs by first computing +example image/audio features, then generating example inputs for the downstream +`lm_encoder` and `decoder` stages. + +## 5B. Monolithic Capture Path + +The non-component path wraps the canonical adapter in `TranspileWrapper` and runs +the full graph through capture: + +```python +wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + +captured = capture_model(wrapper, prepared.tensors) +raw_ir_graph = copy.deepcopy(captured.ir_graph) + +canonicalize_exported_graph(captured.ir_graph) +optimize_graph(captured.ir_graph, config=fusion_config) +tg = _lower_preoptimized_ir(captured.ir_graph) +``` + +`TranspileWrapper` mainly exists to pass `weights_dir` into the graph metadata: + +```python +def get_transpile_metadata(self) -> dict[str, object]: + ... + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir +``` + +## 6. Capture With `torch.export` + +`capture_pytorch.py` owns the `torch.export` boundary: + +```python +normalized_kwargs = _inject_export_safe_kwargs(model, _normalize_kwargs(kwargs)) +... +transpile_metadata = _collect_transpile_metadata(model, example_args, example_kwargs) + +ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) +... +ir_graph = import_captured_to_ir(raw_captured, strict=strict) +``` + +Important behavior here: + +- HF-style `use_cache=False` and `return_dict=False` are injected automatically + when the model signature supports them. +- Metadata providers are collected from any module that defines + `get_transpile_metadata()` or `transpile_metadata()`. +- The captured export is immediately imported into the repo's own IR. + +The capture output is a `CapturedModel` dataclass holding: + +- the exported program +- the FX graph +- the lifted state dict +- the imported `IRGraph` +- the example inputs +- the collected metadata + +## 7. Import Exported FX Into `IRGraph` + +`import_ir.py` converts the exported FX/Export graph into the repo's intermediate +representation: + +```python +ir = IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[], constants={}, meta=dict(graph_meta)) +ctx = ImportContext(strict=strict, transpile_metadata=transpile_metadata) +weights_dir = resolve_transpile_weights_dir(ir.meta) + +for node in captured.graph.nodes: + if node.op == "placeholder": + ... + if node.op == "get_attr": + ... + if node.op == "call_function": + import_call_function(...) + if node.op == "output": + import_output(ir, node, ctx) + +apply_import_semantics(ir) +annotate_ir_components(ir) +verify_ir(ir) +``` + +### What gets imported? + +- Real graph inputs become `IRGraph.inputs` +- lifted parameters/buffers/constants become `IRGraph.constants` +- FX `call_function` nodes become `IRNode`s via `importers.py` + +### How weight binding metadata enters the IR + +When a captured constant corresponds to a known converted weight file, +`import_captured_to_ir()` resolves that mapping early: + +```python +binding = resolve_weight_binding(weights_dir=weights_dir, source_name=target) +... +ir.values[value_id_str].meta.update( + { + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } +) +``` + +That metadata is what later lets lowering use `g.mmap_weights()` or +`g.mmap_embeddings()` instead of embedding the constant payload into the graph. + +## 8. Importers: FX Target To Canonical IR Node + +`importers.py` is the large target-dispatch layer. It normalizes FX/PyTorch targets +to canonical op names and creates `IRNode`s with stable attributes. + +The dispatch point is: + +```python +def import_call_function(ir, node, ctx, *, shape, dtype, torch_op): + op = normalize_target(torch_op) + import_hint = ctx.lookup_import_hint(node, torch_op=torch_op, op_name=op) + importer = OP_IMPORTERS.get(op) + if importer is None: + import_opaque_call_function(...) + else: + importer(...) + _apply_import_hint(ir, node, import_hint) +``` + +Some notable importer behaviors: + +- `normalize.py` reduces many `aten.*` spellings into one canonical name. +- Unknown ops are imported as `kind="opaque"` IR nodes rather than being silently + dropped. +- Import hints can inject attrs/meta into matching nodes after import. +- Some nested exported subgraphs such as `wrap_with_set_grad_enabled` are inlined + explicitly in `import_ir.py`. + +## 9. Import-Time Semantic Rewrites + +Immediately after raw import, `import_semantics.py` performs a first semantic pass: + +```python +def apply_import_semantics(graph: IRGraph) -> IRGraph: + _rewrite_explicit_attention(graph) + _tag_explicit_linear(graph) + _rewrite_early_rope(graph) + rebuild_graph(graph) + verify_ir(graph) + return graph +``` + +This step is intentionally early. It rewrites export patterns into higher-level +meaning before the main optimizer sees them. + +Examples: + +- `scaled_dot_product_attention` becomes semantic `attention` +- explicit linears get tagged +- RoPE patterns can become semantic `rope` before later fusion passes + +## 10. Canonical Cleanup + +`canonicalize/cleanup.py` turns imported IR into a stricter canonical form: + +```python +def canonicalize_exported_graph(graph: IRGraph, *, max_passes: int = 8) -> IRGraph: + for _ in range(max_passes): + for node_id in list(graph.order): + _canonicalize_node(graph, node) + rebuild_graph(graph) + + for node_id in list(graph.order): + _simplify_node(graph, node) + rebuild_graph(graph) + + if _legalize_precisions(graph): + rebuild_graph(graph) + + dce(graph) +``` + +What this pass is responsible for: + +- renaming ops into canonical spellings +- turning shape-only ops into a stable `view` +- rewriting `transpose` and `movedim` into `permute` +- constant-folding trivial expressions +- materializing `ones`, `zeros`, and `arange` constants +- removing no-op casts/views/slices +- inserting FP16 legalizing casts where the runtime requires them +- dead-code elimination + +It also records unsupported canonical ops in `graph.meta["canonical_unsupported_op_counts"]`. + +## 11. Fusion And Graph Optimization + +`optimize_graph.py` runs the semantic fusion passes: + +```python +def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: + canonicalize_exported_graph(graph) + ... + if config.enable_rms_norm: + fuse_rms_norm(graph) + fuse_rms_norm_scale_multiply(graph) + if config.enable_rope: + fuse_rope(graph) + if config.enable_attention: + fuse_attention(graph) + ... + normalize_gemma4_decoder_attention_semantics(graph) + ... + annotate_gold_patterns(graph) + _prune_unused_inputs(graph) +``` + +This is where recognizable subgraphs collapse into semantic or fused nodes such as: + +- `rms_norm` +- `rope` +- `attention` +- `attention_block` +- `self_attention_block` +- `conv_module` +- `lstm_cell` +- `gated_deltanet_prefill` +- `gated_deltanet_decode` + +### Gemma4-specific optimization logic + +Gemma4 gets extra handling in this pass: + +- import hints and graph metadata carry layer types and sliding window info +- exported masks may be elided and rewritten into `window_size` +- some decoder attention behavior is normalized specifically for the saved runtime + +## 12. Lower IR Into A Cactus `Graph` + +`lower.py` is the last compiler stage. It maps `IRValue`s and `IRNode`s into the +runtime `Graph` object from `src.graph`. + +The high-level structure is: + +```python +def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + g = Graph() + env = {} + + for value_id in ir.inputs: + tensor = _lower_input_value(g, ir.values[value_id]) + env[value_id] = tensor + + for value_id, const in ir.constants.items(): + binding = _lookup_weight_binding(value) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(g, value, const, binding=binding) + env[value_id] = lowered_const + + for node_id in ir.order: + outputs = _lower_ir_node(g, ir.nodes[node_id], env, ir) +``` + +### Constant lowering + +This is the critical constant binding behavior: + +```python +def _lower_constant_value(g, value, const, *, binding=None): + if binding is not None: + if binding.kind == "embedding": + return g.mmap_embeddings(binding.path) + return g.mmap_weights(binding.path) + + if tensor_value.numel() == 1: + return tensor_value.item() + + return _materialize_constant_tensor(g, tensor_value) +``` + +So constants can become: + +- `g.mmap_weights(path)` +- `g.mmap_embeddings(path)` +- an inline scalar +- a materialized tensor node baked into the graph + +### Where weight compatibility is repaired + +`weight_compat.py` sits between IR metadata and actual lowering. It can rewrite a +binding to a more executable companion file, for example: + +- convert INT8 embeddings to a cached CQ embedding format +- materialize CQ4 companion files for legacy packed INT4 tensors + +### What lowering implements + +`_lower_ir_node()` is the actual op dispatcher. It covers: + +- elementwise math and comparisons +- views, reshape, expand, permute, transpose +- matmul and linear +- embedding and gather/index paths +- reductions and softmax +- conv1d/conv2d +- norms +- fused attention-family nodes +- fused LSTM and Conformer-style blocks + +If an IR op survives all prior passes and still has no lowering, `lower.py` raises: + +```python +raise NotImplementedError(f"unsupported IR op in lowering: {op}") +``` + +### Gemma4 decoder attention fallback + +One especially important special case is that Gemma4 decoder attention can bypass +the runtime attention kernel and be lowered manually as matmul + mask + softmax: + +```python +def _should_lower_gemma4_decoder_attention_without_kernel(ir, node): + family = str(ir.meta.get("adapter_family") or ir.meta.get("family") or "").strip().lower() + component = str(ir.meta.get("component", "") or "").strip().lower() + if family != "gemma4" or component != "decoder": + return False + return node.op in {"attention", "scaled_dot_product_attention"} +``` + +That path exists because the transpiled Gemma4 decoder has some runtime/kernel +compatibility constraints that are easier to control in lowered graph code. + +## 13. Save Artifacts + +After lowering, `transpile_hf_model.py` writes the IR and bundle artifacts. + +For the monolithic path it saves: + +- `raw_ir.json` +- `optimized_ir.json` +- `graph.cactus` +- `graph_bindings.json` +- component subgraphs under `components/` + +For the component path it saves: + +- `raw_ir.json` +- `optimized_ir.json` +- per-component IRs +- `components/manifest.json` +- one lowered graph per component + +The component bundle writer is `_write_component_bundle()`: + +```python +manifest_path = bundle_dir / "manifest.json" +_write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component_order": component_order, + "inputs": _serialize_json_compatible(inputs_metadata), + "components": manifest_components, + }, +) +``` + +Why both a `.cactus` graph and a binding manifest? + +Because `Graph.save()` stores the graph structure, but the file-backed weight +bindings need to be reattached later. The manifests carry: + +- logical input/output names +- runtime node ids +- paths to bound weights or saved `.npy` constants + +## 14. Optional Execution And Reference Compare + +If execution is enabled, the example script will run the lowered graph immediately. + +By task: + +- text LM: run graph, report next token +- Gemma4 multimodal: run pipeline, report next token +- Parakeet TDT: run component encoder + custom decode loop, report transcript +- generic audio: run encoder graph and report output shapes + +Many paths also compare the transpiled output against a PyTorch reference forward +unless `--skip-reference-compare` is set. + +## 15. Run A Saved Bundle Later + +`component_bundle_runtime.py` is the runtime-side loader for saved bundles. + +The public entry is: + +```python +def run_transpiled_bundle(bundle_dir_or_manifest, *, audio_file=None, image_files=(), prompt=None, input_ids=None, weights_dir=None, ...): + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + component_graphs, manifest = load_saved_component_graphs( + bundle_dir_or_manifest, + weights_dir=resolved_weights_dir, + ) + ... +``` + +There are two ways a saved component can be reloaded: + +1. Load a serialized `.cactus` graph and rebind its constants. +2. Rebuild from saved raw/optimized IR JSON and re-run lowering. + +That second path is handled by `_load_component_graph_from_ir()`: + +```python +ir_graph = _deserialize_saved_ir_graph(...) +if use_raw_ir: + canonicalize_exported_graph(ir_graph) + optimize_graph(ir_graph) +else: + if family == "gemma4" and component == "decoder": + optimize_graph(ir_graph) +transpiled = transpile_preoptimized_ir(ir_graph) +``` + +So saved IR artifacts are not just for inspection; they are also a fallback loading +format that can be re-lowered on demand. + +### Bound constant rebinding at runtime + +The runtime loader reattaches `.weights` and `.npy` files with +`graph.set_external_input(...)`: + +```python +tensor = graph._tensor_from_node(node_id) +graph.set_external_input(tensor, int(tensor_file.data.ctypes.data), dtype=tensor_file.precision) +... +if tensor_file.scales is not None and tensor_file.group_size > 0: + _lib.cactus_graph_set_grouped_scales(...) +``` + +That is why the binding manifests are essential. + +## Major Situations And How They Are Handled + +### Hugging Face models that normally return caches or `ModelOutput` + +`capture_pytorch.py` forces export-safe kwargs: + +- `use_cache=False` +- `return_dict=False` + +This keeps the capture boundary tensor-only. + +### Weights directory exists but has no manifest + +`transpile_hf_model.py` allows a filename-based fallback: + +```python +if not manifest_path.exists(): + has_weight_files = any(root.glob("*.weights")) + if not has_weight_files: + raise RuntimeError(...) + print("note=weights_manifest_missing using filename-based weight binding fallback ...") +``` + +### Weights directory resolves, but no bindings match + +The transpile script fails hard and suggests re-running conversion: + +- component path: if component graphs resolve `0` bindings +- monolithic path: if full optimized IR resolves `0` bindings + +### Local snapshot exists but is incomplete + +`_load_model_source()` will reject incomplete local snapshots when +`--local-files-only` is set. + +### Gemma4 transformers support is missing locally + +`transpile_hf_model.py` and `gemma4_runtime.py` search alternate site-packages and +patch import compatibility for: + +- missing `transformers.models.gemma4` +- missing `_lzma` +- torchvision import probes +- missing `torchvision::nms` +- missing `flex_attention.AuxRequest` + +### Gemma4 local checkpoint key mismatches + +`_repair_gemma4_checkpoint_weights()` remaps local checkpoint keys before loading +the model. + +### Gemma4 multimodal prompt drift + +The transpiler normalizes prompt and tokenization details to match native Cactus: + +- inserts missing image/audio placeholder tokens +- builds a Cactus-style chat prompt when requested +- expands HF newline-merge tokens `108` and `109` back into repeated newline token + `107` +- can replace processor-generated image/audio tensors with native-like versions + +### Parakeet TDT does not run as a single generic forward + +`parakeet_tdt_local.py` provides: + +- a local PyTorch reconstruction of the model +- audio preprocessing +- a greedy TDT decode loop +- component specs for an encoder graph and a recurrent decoder-step graph + +### Some runtime symbols are not present in the current shared library + +`runtime_compat.py` monkey-patches the Python `Graph` wrapper so the transpiler can +still build graphs against the current runtime. It provides compatibility behavior +for items like: + +- generic `matmul` +- `gather(axis=0)` +- limited `conv2d` +- `not_equal` +- scalar math legalization + +### Opaque import is allowed, but execution may still fail later + +If `importers.py` does not know a target, it creates an opaque IR node instead of +dropping it. That preserves information, but lowering still needs a real lowering +case. Opaque nodes are therefore a sign that later lowering may stop with +`unsupported IR op`. + +## Artifact Layout + +A typical transpile artifact directory looks like this: + +```text +transpiled// + raw_ir.json + optimized_ir.json + graph.cactus + graph_bindings.json + result.json + components/ + manifest.json + vision_encoder/ + graph.cactus + bound_constants/ + ... + audio_encoder/ + graph.cactus + lm_encoder/ + graph.cactus + decoder/ + graph.cactus +``` + +Notes: + +- `graph.cactus` is the monolithic lowered graph, when one exists. +- `components/manifest.json` is the portable bundle manifest. +- `graph_bindings.json` and per-component `bound_constant_bindings` tell the loader + how to reattach mmap-backed weights or saved `.npy` constants. + +## File-By-File Map + +### Core transpiler files + +| File | Responsibility | +| --- | --- | +| `audio_preprocess.py` | WAV loading, resampling, generic log-mel extraction, Cactus audio frontend wrappers, Parakeet-native features, Gemma4-native audio features. | +| `capture_pytorch.py` | `torch.export` capture wrapper, export-safe kwargs injection, metadata collection, shape propagation, and `CapturedModel` creation. | +| `component_bundle_runtime.py` | Loader and executor for saved transpiled bundles; reloads `.cactus` graphs or re-lowers saved IR, then runs task-specific bundle executors. | +| `component_partition.py` | Heuristic component labeling of IR nodes/values and subgraph extraction into `audio_encoder` / `vision_encoder` / `lm_encoder` / `decoder`. | +| `component_pipeline.py` | Capture/optimize/lower loop for one component spec, plus runtime execution of a staged pipeline of components. | +| `gemma4_runtime.py` | Shared Gemma4 runtime helpers used by bundle execution: processor compatibility, prompt normalization, native-like multimodal preprocessing. | +| `graph_ir.py` | Definitions for `IRValue`, `IRNode`, `IRGraph`, deep-copy behavior, and IR consistency verification. | +| `import_ir.py` | Top-level importer from exported FX graph into `IRGraph`; resolves lifted constants, weight binding metadata, and inlined subgraphs. | +| `import_semantics.py` | Early semantic rewrites done immediately after import, especially attention and RoPE recognition. | +| `importers.py` | Large target-to-IR importer registry; converts normalized PyTorch ops into canonical `IRNode`s. | +| `lower.py` | Final lowering from optimized IR into runtime `Graph`, including op lowering, constant binding, fused op handling, and execution wrapper. | +| `model_adapters.py` | Family- and task-specific forward adapters, Gemma4 multimodal pipeline adapters, task canonicalization, and component spec builders. | +| `model_patterns.py` | Catalog of "gold patterns" used to annotate recognizable transformer structures after optimization. | +| `normalize.py` | Target-name normalization (`aten.*` to canonical names) and dtype-name normalization into IR spellings. | +| `ops.py` | Canonical op registry and alias table for IR ops, including backend op names and attr schemas. | +| `optimize_graph.py` | Main fusion/optimization pass driver, pattern fusion, Gemma4 attention normalization, and gold-pattern annotation. | +| `parakeet_tdt_local.py` | Local Parakeet TDT model reconstruction, config loader, decoder loop, and component-spec generation. | +| `runtime_compat.py` | Runtime wrapper patches that make the current Python `Graph` API usable for transpiled graphs even when some C symbols are absent. | +| `weight_binding.py` | Resolves IR constants to converted `.weights` files by manifest or filename heuristics; also resolves default weights dirs. | +| `weight_compat.py` | Makes bound weights executable with the current runtime by materializing compatibility companion files when needed. | + +### Canonicalization subdirectory + +| File | Responsibility | +| --- | --- | +| `canonicalize/cleanup.py` | Multi-pass canonical cleanup, constant folding, no-op removal, attr normalization, and precision legalization. | +| `canonicalize/utils.py` | Shared graph rebuild, DCE, node removal, dtype helpers, and constant materialization utilities. | + +### Fusion subdirectory + +| File | Responsibility | +| --- | --- | +| `fusion/__init__.py` | Re-exports the fusion matchers used by `optimize_graph.py`. | +| `fusion/common.py` | Shared fusion helper types and graph traversal helpers. | +| `fusion/linear.py` | Matcher for linear projection patterns. | +| `fusion/rms_norm.py` | Matcher for RMSNorm computational subgraphs. | +| `fusion/rope.py` | Matcher for rotary embedding patterns. | +| `fusion/attention.py` | Matchers for attention, attention blocks, self-attention blocks, and GQA-style projection patterns. | +| `fusion/conv.py` | Matcher for Conformer-like convolution modules. | +| `fusion/lstm.py` | Matcher for LSTM cell subgraphs. | +| `fusion/mlp.py` | Matcher for gated MLP branches such as GELU/SiLU gate-up-down patterns. | +| `fusion/rel_pos_bias.py` | Matcher for relative-position-bias subgraphs. | +| `fusion/deltanet.py` | Matcher for fused gated DeltaNet patterns. | + +## CLI Usage + +## 1. Basic Transpile + +The simplest public command is: + +```bash +cactus transpile +``` + +By default this: + +- resolves the runtime library +- launches `python/examples/transpile_hf_model.py` +- saves artifacts under `./transpiled//` +- adds `--skip-execute`, so it does not run the graph unless you ask + +To also execute right after transpiling: + +```bash +cactus transpile --execute-after-transpile +``` + +## 2. Forwarded Transpile Options + +Most useful transpile options are parsed by `python/examples/transpile_hf_model.py` +and simply forwarded through `cactus transpile`. + +Common ones: + +- `--task` +- `--prompt` +- `--input-ids` +- `--audio-file` +- `--image-file` +- `--weights-dir` +- `--artifact-dir` +- `--component-pipeline` +- `--skip-reference-compare` +- `--graph-filename` +- `--no-fuse-rms-norm` +- `--no-fuse-rope` +- `--no-fuse-attention` + +Examples: + +### Text causal LM + +```bash +cactus transpile \ + --prompt "The capital of France is" \ + --artifact-dir ./transpiled/text_demo +``` + +### Text causal LM with explicit converted weights + +```bash +cactus transpile \ + --weights-dir /path/to/converted_weights \ + --prompt "Write a haiku about compilers" +``` + +### Gemma4 multimodal + +```bash +cactus transpile \ + --task multimodal_causal_lm_logits \ + --image-file /path/to/image.jpg \ + --audio-file /path/to/audio.wav \ + --prompt "Describe what is happening in the image and audio." \ + --weights-dir /path/to/converted_weights +``` + +### Whisper-style encoder capture + +```bash +cactus transpile \ + --audio-file /path/to/sample.wav \ + --task encoder_hidden_states +``` + +### Parakeet TDT + +```bash +cactus transpile \ + --audio-file /path/to/sample.wav \ + --task tdt_transcription \ + --weights-dir /path/to/converted_weights +``` + +If you want the full direct script help: + +```bash +python python/examples/transpile_hf_model.py --help +``` + +## 3. Run A Saved Transpiled Bundle + +Use the bundle root directory or a `manifest.json` path: + +```bash +cactus run-transpiled /path/to/transpiled/model +``` + +For direct bundle execution, audio input is passed with `--file`. The `--audio` +flag belongs to `cactus run`, which may forward into the same runtime after +bundle auto-detection. + +### Text causal LM bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --prompt "The capital of France is" +``` + +Or pass token ids directly: + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --input-ids 2,13,42 +``` + +### Multimodal Gemma4 bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --image-file /path/to/image.jpg \ + --file /path/to/audio.wav \ + --prompt "What do you observe?" \ + --system "Answer concisely." +``` + +### Audio encoder / transcription bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --file /path/to/sample.wav +``` + +### Bundles with bound weights + +If the bundle expects converted mmap weights, pass the weights directory when the +saved paths are not directly reachable: + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --weights-dir /path/to/converted_weights \ + --prompt "Hello" +``` + +### Save result payload + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --prompt "Hello" \ + --result-json ./result.json +``` + +## 4. `cactus run` Also Works + +`cactus run` detects transpiled bundles automatically: + +```bash +cactus run /path/to/transpiled/model --prompt "Hello" +``` + +If you are passing media through `cactus run`, use `--image` and `--audio` +rather than `run-transpiled`'s direct `--file` audio flag. + +Internally, `cli.py` checks whether the path contains a transpiled manifest and +redirects to `cmd_run_transpiled()`. + +## Current Runtime Limitations + +As of the current code: + +- Saved bundle execution is implemented for `causal_lm_logits`, + `multimodal_causal_lm_logits`, `encoder_hidden_states`, and + `tdt_transcription`. +- `ctc_logits` can be transpiled, but `run_transpiled_bundle()` does not currently + implement a saved-bundle executor for that task. +- The component pipeline is model-family-specific; it is not a universal split-IR + path. + +## Useful Environment Variables + +- `CACTUS_TRANSPILER_WEIGHTS_DIR`: + fallback weights directory used by `weight_binding.py` +- `CACTUS_TRANSPILER_WEIGHTS_DIR_`: + family-specific weights directory override +- `CACTUS_TRANSPILER_DEBUG_MMAP=1`: + log constant binding and fallback behavior during lowering +- `CACTUS_GEMMA4_CAPTURE_FP32=1`: + temporarily promote some Gemma4 text modules to FP32 during capture on CPU + +## Practical Summary + +If you only want the high-level lifecycle, it is: + +1. `cactus transpile` launches `python/examples/transpile_hf_model.py`. +2. The script infers a task, loads the HF model, and creates one example input batch. +3. `model_adapters.py` wraps the model into a smaller canonical task interface. +4. `capture_pytorch.py` runs `torch.export`. +5. `import_ir.py` + `importers.py` convert the export to `IRGraph`. +6. `import_semantics.py` adds early semantic rewrites. +7. `canonicalize/cleanup.py` normalizes and simplifies the IR. +8. `optimize_graph.py` fuses patterns into higher-level semantic ops. +9. `lower.py` builds a runtime `Graph`, mmaps bound weights when possible, and + materializes everything else. +10. The script writes `raw_ir.json`, `optimized_ir.json`, `graph.cactus`, and + `components/manifest.json`. +11. `cactus run-transpiled` later reloads those artifacts with + `component_bundle_runtime.py`, rebinds external tensors, and executes the + task-specific bundle runner. diff --git a/python/src/transpile/audio_preprocess.py b/python/src/transpile/audio_preprocess.py new file mode 100644 index 000000000..59d71862b --- /dev/null +++ b/python/src/transpile/audio_preprocess.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +import numpy as np +import torch +from scipy.io import wavfile +from scipy.signal import resample_poly + + +_PARAKEET_SAMPLE_RATE = 16000 +_PARAKEET_NUM_MELS = 128 +_PARAKEET_N_FFT = 512 +_PARAKEET_FRAME_LENGTH = 400 +_PARAKEET_HOP_LENGTH = 160 +_PARAKEET_PREEMPHASIS = 0.97 +_PARAKEET_LOG_FLOOR = np.float32(2**-24) + + +def normalize_audio_samples(samples: np.ndarray) -> np.ndarray: + if samples.dtype == np.uint8: + return ((samples.astype(np.float32) - 128.0) / 128.0).astype(np.float32) + if np.issubdtype(samples.dtype, np.integer): + info = np.iinfo(samples.dtype) + scale = float(max(abs(info.min), info.max)) + if scale <= 0: + scale = 1.0 + return (samples.astype(np.float32) / scale).astype(np.float32) + return samples.astype(np.float32) + + +def load_audio_waveform(audio_file: str | Path, *, target_sample_rate: int) -> np.ndarray: + sample_rate, waveform = wavfile.read(str(audio_file)) + if waveform.ndim > 1: + waveform = waveform.mean(axis=1) + waveform = normalize_audio_samples(np.asarray(waveform)) + if int(sample_rate) != int(target_sample_rate): + gcd = int(np.gcd(int(sample_rate), int(target_sample_rate))) + waveform = resample_poly( + waveform, + int(target_sample_rate) // gcd, + int(sample_rate) // gcd, + ).astype(np.float32) + return waveform.astype(np.float32) + + +def _hz_to_mel(hz: np.ndarray) -> np.ndarray: + return 2595.0 * np.log10(1.0 + hz / 700.0) + + +def _mel_to_hz(mel: np.ndarray) -> np.ndarray: + return 700.0 * (np.power(10.0, mel / 2595.0) - 1.0) + + +@lru_cache(maxsize=16) +def _mel_filter_bank(num_mels: int, sample_rate: int, n_fft: int) -> np.ndarray: + try: + from transformers.audio_utils import mel_filter_bank as transformers_mel_filter_bank # type: ignore + + return transformers_mel_filter_bank( + num_frequency_bins=n_fft // 2 + 1, + num_mel_filters=num_mels, + min_frequency=0.0, + max_frequency=sample_rate / 2.0, + sampling_rate=sample_rate, + norm="slaney", + mel_scale="slaney", + ).astype(np.float32) + except Exception: + pass + + num_freq_bins = n_fft // 2 + 1 + mel_points = np.linspace( + _hz_to_mel(np.array([0.0], dtype=np.float32))[0], + _hz_to_mel(np.array([sample_rate / 2.0], dtype=np.float32))[0], + num_mels + 2, + ) + hz_points = _mel_to_hz(mel_points) + bins = np.floor((n_fft + 1) * hz_points / sample_rate).astype(np.int64) + + filters = np.zeros((num_mels, num_freq_bins), dtype=np.float32) + for index in range(1, num_mels + 1): + left = max(0, min(num_freq_bins - 1, int(bins[index - 1]))) + center = max(left + 1, min(num_freq_bins - 1, int(bins[index]))) + right = max(center + 1, min(num_freq_bins, int(bins[index + 1]))) + for freq in range(left, center): + filters[index - 1, freq] = float(freq - left) / float(max(center - left, 1)) + for freq in range(center, right): + filters[index - 1, freq] = float(right - freq) / float(max(right - center, 1)) + return filters + + +def generic_log_mel_features( + waveform: np.ndarray, + *, + sample_rate: int, + num_mels: int, + n_fft: int, + hop_length: int, + frame_length: int, + preemphasis: float | None = None, + mel_floor: np.float32 = _PARAKEET_LOG_FLOOR, + normalize_active_frames_only: bool = True, +) -> tuple[np.ndarray, int]: + waveform_tensor = torch.from_numpy(waveform).to(torch.float32).unsqueeze(0) + if preemphasis is not None and abs(preemphasis) > 0.0: + waveform_tensor = torch.cat( + [ + waveform_tensor[:, :1], + waveform_tensor[:, 1:] - float(preemphasis) * waveform_tensor[:, :-1], + ], + dim=1, + ) + + window = torch.hann_window(frame_length, periodic=False, dtype=torch.float32) + stft = torch.stft( + waveform_tensor, + n_fft=n_fft, + hop_length=hop_length, + win_length=frame_length, + window=window, + return_complex=True, + center=True, + pad_mode="constant", + ) + magnitudes = torch.view_as_real(stft) + magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1)) + power = magnitudes.pow(2) + + feature_length = max(1, int(waveform.shape[0] // hop_length)) + mel_filters = torch.from_numpy(_mel_filter_bank(num_mels, sample_rate, n_fft)).to(torch.float32) + mel_spec = power.transpose(1, 2) @ mel_filters + mel_spec = torch.log(torch.clamp(mel_spec, min=float(mel_floor))) + + if normalize_active_frames_only: + attention_mask = ( + torch.arange(mel_spec.shape[1], dtype=torch.long, device=mel_spec.device).unsqueeze(0) + < feature_length + ) + mask = attention_mask.unsqueeze(-1).to(torch.float32) + masked = mel_spec * mask + mean = masked.sum(dim=1) / float(feature_length) + centered = masked - mean.unsqueeze(1) + denom = float(max(1, feature_length - 1)) + variance = (centered.pow(2) * mask).sum(dim=1) / denom + mel_spec = (mel_spec - mean.unsqueeze(1)) / torch.sqrt(variance.unsqueeze(1) + np.float32(1e-5)) + mel_spec = mel_spec * mask + else: + # Native Parakeet normalizes across the full centered-STFT frame count, + # then trims to waveform_samples // hop_length. Keep that order for parity. + frame_count = int(mel_spec.shape[1]) + mean = mel_spec.mean(dim=1) + centered = mel_spec - mean.unsqueeze(1) + denom = float(max(1, frame_count - 1)) + variance = centered.pow(2).sum(dim=1) / denom + mel_spec = centered / torch.sqrt(variance.unsqueeze(1) + np.float32(1e-5)) + return mel_spec[0].cpu().numpy().astype(np.float32), feature_length + + +def prepare_cactus_audio_features( + audio_file: str | Path, + *, + model_type: str, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, + layout: str = "frames_mels", +) -> tuple[torch.Tensor, int]: + from src.cactus import cactus_preprocess_audio_features + + capacity_frames = int(expected_frames) if isinstance(expected_frames, int) and expected_frames > 0 else 0 + if capacity_frames <= 0: + lowered_model_type = model_type.lower() + if "whisper" in lowered_model_type: + capacity_frames = 3000 + else: + sample_rate, waveform = wavfile.read(str(audio_file)) + samples = int(waveform.shape[0]) if hasattr(waveform, "shape") else len(waveform) + if sample_rate <= 0: + sample_rate = _PARAKEET_SAMPLE_RATE + capacity_frames = max( + 1, + int(np.ceil(samples * (_PARAKEET_SAMPLE_RATE / float(sample_rate)) / _PARAKEET_HOP_LENGTH)) + 8, + ) + if "whisper" in model_type.lower(): + capacity_frames = max(capacity_frames, 3000) + capacity = max(1, capacity_frames * int(expected_mels)) + values, mel_bins, frames = cactus_preprocess_audio_features( + str(Path(audio_file).expanduser().resolve()), + model_type, + int(expected_mels), + capacity, + ) + if int(mel_bins) != int(expected_mels): + raise ValueError(f"Cactus audio frontend returned {mel_bins} mel bins, expected {expected_mels}") + feature_array = np.asarray(values, dtype=np.float32).reshape(int(mel_bins), int(frames)) + active_frames = int(frames) + if isinstance(expected_frames, int) and expected_frames > 0: + active_frames = min(active_frames, expected_frames) + if layout == "frames_mels": + feature_array = feature_array[:, :active_frames].T + if isinstance(expected_frames, int) and expected_frames > active_frames: + feature_array = np.pad(feature_array, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + elif layout == "mels_frames": + feature_array = feature_array[:, :active_frames] + if isinstance(expected_frames, int) and expected_frames > active_frames: + feature_array = np.pad(feature_array, ((0, 0), (0, expected_frames - active_frames)), mode="constant") + else: + raise ValueError(f"unsupported Cactus audio feature layout: {layout!r}") + tensor = torch.from_numpy(np.ascontiguousarray(feature_array)).unsqueeze(0).to(dtype=torch_dtype) + return tensor, active_frames + + +def prepare_native_parakeet_audio_features( + audio_file: str | Path, + *, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, +) -> tuple[torch.Tensor, int]: + try: + return prepare_cactus_audio_features( + audio_file, + model_type="parakeet", + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + except Exception: + pass + + waveform = load_audio_waveform(audio_file, target_sample_rate=_PARAKEET_SAMPLE_RATE) + features, feature_length = generic_log_mel_features( + waveform, + sample_rate=_PARAKEET_SAMPLE_RATE, + num_mels=expected_mels or _PARAKEET_NUM_MELS, + n_fft=_PARAKEET_N_FFT, + hop_length=_PARAKEET_HOP_LENGTH, + frame_length=_PARAKEET_FRAME_LENGTH, + preemphasis=_PARAKEET_PREEMPHASIS, + mel_floor=_PARAKEET_LOG_FLOOR, + normalize_active_frames_only=False, + ) + active_frames = feature_length + if isinstance(expected_frames, int) and expected_frames > 0: + active_frames = min(active_frames, expected_frames) + features = features[:active_frames, :] + if features.shape[1] != expected_mels: + raise ValueError( + f"feature mel dimension mismatch: expected {expected_mels}, got {features.shape[1]}" + ) + if isinstance(expected_frames, int) and expected_frames > active_frames: + features = np.pad(features, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + tensor = torch.from_numpy(np.ascontiguousarray(features)).unsqueeze(0).to(dtype=torch_dtype) + return tensor, active_frames + + +def prepare_native_gemma4_audio_features( + audio_file: str | Path, + *, + expected_mels: int, + torch_dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor, int]: + waveform = load_audio_waveform(audio_file, target_sample_rate=_PARAKEET_SAMPLE_RATE) + # Native Gemma4 pads to the nearest 320 samples, adds 160 samples of + # semicausal left padding, and then runs a 321-point analysis frame. + estimated_frames = int(np.ceil((len(waveform) + 640) / 160.0)) + 8 + capacity = max(1, estimated_frames * int(expected_mels)) + values, mel_bins, frames = prepare_cactus_audio_features_raw( + audio_file, + model_type="gemma4", + mel_bins=int(expected_mels), + capacity=capacity, + ) + if int(mel_bins) != int(expected_mels): + raise ValueError(f"Cactus Gemma4 audio frontend returned {mel_bins} mel bins, expected {expected_mels}") + features = np.asarray(values, dtype=np.float32).reshape(int(frames), int(mel_bins)) + feature_tensor = torch.from_numpy(np.ascontiguousarray(features)).unsqueeze(0).to(dtype=torch_dtype) + mask = torch.ones((1, int(frames)), dtype=torch.bool) + return feature_tensor, mask, int(frames) + + +def prepare_cactus_audio_features_raw( + audio_file: str | Path, + *, + model_type: str, + mel_bins: int, + capacity: int, +) -> tuple[list[float], int, int]: + from src.cactus import cactus_preprocess_audio_features + + return cactus_preprocess_audio_features( + str(Path(audio_file).expanduser().resolve()), + model_type, + int(mel_bins), + int(capacity), + ) diff --git a/python/src/transpile/canonicalize/cleanup.py b/python/src/transpile/canonicalize/cleanup.py new file mode 100644 index 000000000..fe801cb84 --- /dev/null +++ b/python/src/transpile/canonicalize/cleanup.py @@ -0,0 +1,657 @@ +from __future__ import annotations + +from collections import Counter + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.ops import canonicalize_op +from src.transpile.ops import get_op +from src.transpile.ops import has_op +from src.transpile.canonicalize.utils import bypass_unary_node +from src.transpile.canonicalize.utils import dce +from src.transpile.canonicalize.utils import ir_dtype_to_torch +from src.transpile.canonicalize.utils import materialize_constant_output +from src.transpile.canonicalize.utils import normalize_dim +from src.transpile.canonicalize.utils import normalize_dtype_name +from src.transpile.canonicalize.utils import numel +from src.transpile.canonicalize.utils import rebuild_graph + + +COMPILER_SUPPORTED_OPS = { + "arange", + "ones", +} + +CANONICAL_UNSUPPORTED_RENAMES = { + "aten.__and__.Tensor": "logical_and", + "aten.__or__.Tensor": "logical_or", + "aten.cumsum.default": "cumsum", + "aten.eq.Scalar": "equal", + "aten.eq.Tensor": "equal", + "aten.gt.Tensor": "greater", + "aten.index.Tensor": "advanced_index", + "aten.le.Tensor": "less_equal", +} + +FP32_SUPPORTED_ALL_INPUT_OPS = { + "add", + "subtract", + "multiply", + "divide", + "precision_cast", + "view", + "permute", + "transpose", + "slice", + "index", + "advanced_index", + "gather", + "matmul", + "linear", + "addmm", + "layer_norm", + "batch_norm", + "glu", + "sum", + "mean", + "variance", + "min", + "max", + "scalar_add", + "scalar_subtract", + "scalar_subtract_reverse", + "scalar_multiply", + "scalar_divide", + "scalar_floor_divide", + "scalar_not_equal", + "scalar_equal", + "abs", + "negate", + "pow", + "scalar_exp", + "scalar_sqrt", + "scalar_log", + "scalar_cos", + "scalar_sin", +} + +FP32_SUPPORTED_INPUT_INDICES = { + "embedding": {1}, +} + +FP16_ONLY_OUTPUT_OPS = { + "scalar_divide_reverse", + "not_equal", + "equal", + "greater", + "less_equal", + "logical_and", + "logical_or", + "logical_not", + "one_hot", + "scalar_not_equal", + "scalar_equal", + "cat", + "softmax", + "rms_norm", + "embedding", + "group_norm", + "lstm_cell", + "relu", + "silu", + "gelu", + "gelu_erf", + "sigmoid", + "tanh", + "attention", + "attention_block", + "conv_module", + "rope", +} + + +def canonicalize_exported_graph(graph: IRGraph, *, max_passes: int = 8) -> IRGraph: + verify_ir(graph) + + for _ in range(max_passes): + changed = False + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if _canonicalize_node(graph, node): + changed = True + + rebuild_graph(graph) + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if _simplify_node(graph, node): + changed = True + + rebuild_graph(graph) + + if _legalize_precisions(graph): + changed = True + rebuild_graph(graph) + + dce(graph) + + if not changed: + break + + unsupported_counts = summarize_unsupported_ops(graph) + graph.meta = dict(graph.meta) + graph.meta["canonical_unsupported_ops"] = tuple(sorted(unsupported_counts)) + graph.meta["canonical_unsupported_op_counts"] = unsupported_counts + verify_ir(graph) + return graph + + +def summarize_unsupported_ops(graph: IRGraph) -> dict[str, int]: + counts: Counter[str] = Counter() + for node_id in graph.order: + node = graph.nodes[node_id] + if not _is_supported_without_backend_kernel(node.op): + counts[node.op] += 1 + return dict(sorted(counts.items())) + + +def _canonicalize_node(graph: IRGraph, node: IRNode) -> bool: + original_op = node.op + + if original_op in {"aten.new_ones.default", "aten.ones.default"}: + return _materialize_new_ones_constant(graph, node) + + if original_op in {"aten.new_zeros.default", "aten.zeros.default", "new_zeros"}: + return _materialize_new_zeros_constant(graph, node) + + if original_op == "arange": + return _materialize_arange_constant(graph, node) + + if original_op in {"identity", "contiguous"}: + return bypass_unary_node(graph, node) + + if original_op in CANONICAL_UNSUPPORTED_RENAMES: + node.op = CANONICAL_UNSUPPORTED_RENAMES[original_op] + return True + + if original_op == "multiply_inplace": + node.op = "multiply" + return True + + if original_op == "view" and "shape" not in node.attrs: + return _rewrite_view_from_output_shape(graph, node) + + if original_op in {"reshape", "flatten", "unsqueeze", "squeeze"}: + return _rewrite_view_from_output_shape(graph, node) + + if original_op == "expand": + return _rewrite_expand_if_view_equivalent(graph, node) + + if original_op == "transpose": + return _rewrite_transpose_as_permute(graph, node) + + if original_op == "movedim": + return _rewrite_movedim_as_permute(graph, node) + + if original_op == "type_as": + return _rewrite_type_as(graph, node) + + if original_op == "concat": + node.op = "cat" + if "dim" in node.attrs and "axis" not in node.attrs: + node.attrs = {**node.attrs, "axis": int(node.attrs["dim"])} + return True + + if original_op == "scaled_dot_product_attention": + node.op = "attention" + node.attrs = { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", False)), + "window_size": int(node.attrs.get("window_size", 0)), + **({"mask": node.attrs["mask"]} if "mask" in node.attrs else {}), + **({"additive_mask": bool(node.attrs.get("additive_mask", False))} if "additive_mask" in node.attrs else {}), + } + return True + + canonical_op = canonicalize_op(original_op) + if canonical_op != original_op: + node.op = canonical_op + _normalize_attr_spellings(node) + if has_op(node.op): + _filter_attrs_to_schema(node) + return True + + if has_op(node.op): + _normalize_attr_spellings(node) + _filter_attrs_to_schema(node) + return True + + return False + + +def _simplify_node(graph: IRGraph, node: IRNode) -> bool: + if _fold_constant_node(graph, node): + return True + + if node.op == "precision_cast": + return _remove_noop_precision_cast(graph, node) + + if node.op == "view": + if _remove_noop_view(graph, node): + return True + return _collapse_view_chain(graph, node) + + if node.op == "slice": + return _remove_noop_slice(graph, node) + + if node.op == "scalar_add": + return _remove_noop_scalar(graph, node, neutral=0.0) + + if node.op == "scalar_subtract": + return _remove_noop_scalar(graph, node, neutral=0.0) + + if node.op == "scalar_multiply": + return _remove_noop_scalar(graph, node, neutral=1.0) + + if node.op == "scalar_divide": + return _remove_noop_scalar(graph, node, neutral=1.0) + + return False + + +def _fold_constant_node(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + + output_value = graph.values.get(node.outputs[0]) + if output_value is None: + return False + + if node.op in FP16_ONLY_OUTPUT_OPS: + output_value.dtype = "fp16" + node.meta["dtype"] = "fp16" + + output_torch_dtype = ir_dtype_to_torch(output_value.dtype) if output_value.dtype is not None else None + + constant_inputs = [graph.constants.get(value_id) for value_id in node.inputs] + if node.op == "view" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + if output_value.shape is None: + return False + return materialize_constant_output( + graph, + node, + constant_inputs[0].reshape(tuple(int(dim) for dim in output_value.shape)).to(dtype=output_torch_dtype), + ) + + if node.op == "expand" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + if output_value.shape is None: + return False + return materialize_constant_output( + graph, + node, + constant_inputs[0].expand(tuple(int(dim) for dim in output_value.shape)).clone().to(dtype=output_torch_dtype), + ) + + if node.op == "precision_cast" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + dtype = normalize_dtype_name(node.attrs.get("dtype")) + if dtype is None: + return False + return materialize_constant_output(graph, node, constant_inputs[0].to(dtype=ir_dtype_to_torch(dtype))) + + if node.op in {"cos", "scalar_cos"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + return materialize_constant_output(graph, node, torch.cos(constant_inputs[0])) + + if node.op in {"sin", "scalar_sin"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + return materialize_constant_output(graph, node, torch.sin(constant_inputs[0])) + + if node.op in {"scalar_add", "scalar_subtract", "scalar_multiply", "scalar_divide"}: + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + scalar = float(node.attrs["value"]) + if node.op == "scalar_add": + result = constant_inputs[0] + scalar + elif node.op == "scalar_subtract": + result = constant_inputs[0] - scalar + elif node.op == "scalar_multiply": + result = constant_inputs[0] * scalar + else: + result = constant_inputs[0] / scalar + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op in {"scalar_not_equal", "scalar_equal"}: + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + scalar = float(node.attrs["value"]) + if node.op == "scalar_not_equal": + result = constant_inputs[0] != scalar + else: + result = constant_inputs[0] == scalar + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op in {"not_equal", "equal", "greater", "less_equal", "logical_and", "logical_or"}: + if len(constant_inputs) != 2 or not all(isinstance(value, torch.Tensor) for value in constant_inputs): + return False + lhs, rhs = constant_inputs + if node.op == "not_equal": + result = lhs != rhs + elif node.op == "equal": + result = lhs == rhs + elif node.op == "greater": + result = lhs > rhs + elif node.op == "less_equal": + result = lhs <= rhs + elif node.op == "logical_and": + result = torch.logical_and(lhs != 0, rhs != 0) + else: + result = torch.logical_or(lhs != 0, rhs != 0) + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op == "logical_not": + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + result = torch.logical_not(constant_inputs[0] != 0) + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + return False + + +def _legalize_precisions(graph: IRGraph) -> bool: + changed = False + counter = 0 + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + if node.op in FP32_SUPPORTED_ALL_INPUT_OPS: + continue + + inserted: list[IRNode] = [] + allowed_indices = FP32_SUPPORTED_INPUT_INDICES.get(node.op, set()) + for input_index, value_id in enumerate(list(node.inputs)): + if input_index in allowed_indices: + continue + value = graph.values.get(value_id) + if value is None or normalize_dtype_name(value.dtype) != "fp32": + continue + + counter += 1 + cast_node_id = _unique_ir_id( + f"{node.id}_legalize_fp16_{counter}", + graph.nodes.keys(), + graph.order, + ) + cast_value_id = _unique_ir_id( + f"{value_id}_legalize_fp16_{counter}", + graph.values.keys(), + graph.constants.keys(), + ) + cast_node = IRNode( + id=cast_node_id, + op="precision_cast", + inputs=[value_id], + outputs=[cast_value_id], + attrs={"dtype": "fp16"}, + meta={"inserted_by": "canonicalize_exported_graph"}, + ) + graph.values[cast_value_id] = IRValue( + id=cast_value_id, + shape=value.shape, + dtype="fp16", + producer=cast_node_id, + users=[], + ) + node.inputs[input_index] = cast_value_id + inserted.append(cast_node) + changed = True + + if inserted: + insert_at = graph.order.index(node_id) + for offset, cast_node in enumerate(inserted): + graph.nodes[cast_node.id] = cast_node + graph.order.insert(insert_at + offset, cast_node.id) + + if node.op in FP16_ONLY_OUTPUT_OPS: + for output_id in node.outputs: + output_value = graph.values.get(output_id) + if output_value is not None: + output_value.dtype = "fp16" + node.meta["dtype"] = "fp16" + + return changed + + +def _unique_ir_id(base: str, *existing_groups) -> str: + existing: set[str] = set() + for group in existing_groups: + existing.update(str(item) for item in group) + if base not in existing: + return base + suffix = 1 + while True: + candidate = f"{base}_{suffix}" + if candidate not in existing: + return candidate + suffix += 1 + + +def _rewrite_view_from_output_shape(graph: IRGraph, node: IRNode) -> bool: + if not node.outputs: + return False + output_value = graph.values.get(node.outputs[0]) + if output_value is None or output_value.shape is None: + return False + node.op = "view" + node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} + return True + + +def _rewrite_expand_if_view_equivalent(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs or not node.outputs: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if numel(input_value.shape) != numel(output_value.shape): + node.meta["canonicalization_blocked"] = "expand_requires_broadcast" + return False + node.op = "view" + node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} + return True + + +def _rewrite_transpose_as_permute(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.shape is None: + return False + rank = len(input_value.shape) + dim0 = normalize_dim(int(node.attrs["dim0"]), rank) + dim1 = normalize_dim(int(node.attrs["dim1"]), rank) + permutation = list(range(rank)) + permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] + node.op = "permute" + node.attrs = {"permutation": tuple(permutation)} + return True + + +def _rewrite_movedim_as_permute(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.shape is None: + return False + rank = len(input_value.shape) + source = [normalize_dim(int(dim), rank) for dim in node.attrs["source"]] + destination = [normalize_dim(int(dim), rank) for dim in node.attrs["destination"]] + if len(source) != len(destination): + return False + + remaining = [dim for dim in range(rank) if dim not in source] + permutation = list(remaining) + for dst, src in sorted(zip(destination, source)): + permutation.insert(dst, src) + + node.op = "permute" + node.attrs = {"permutation": tuple(permutation)} + return True + + +def _rewrite_type_as(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + target_dtype = None + if node.outputs: + output_value = graph.values.get(node.outputs[0]) + if output_value is not None: + target_dtype = output_value.dtype + if target_dtype is None and len(node.inputs) > 1: + target_value = graph.values.get(node.inputs[1]) + if target_value is not None: + target_dtype = target_value.dtype + if target_dtype is None: + return False + node.op = "precision_cast" + node.inputs = [node.inputs[0]] + node.attrs = {"dtype": normalize_dtype_name(target_dtype)} + return True + + +def _normalize_attr_spellings(node: IRNode) -> None: + attrs = dict(node.attrs) + if "dim" in attrs and "axis" not in attrs and node.op in {"softmax", "sum", "mean", "variance", "min", "max", "cat", "slice", "index"}: + attrs["axis"] = attrs.pop("dim") + if node.op == "permute" and "dims" in attrs and "permutation" not in attrs: + attrs["permutation"] = tuple(int(dim) for dim in attrs.pop("dims")) + if node.op == "slice": + attrs.setdefault("step", 1) + if node.op == "precision_cast" and "dtype" in attrs: + attrs["dtype"] = normalize_dtype_name(attrs["dtype"]) + node.attrs = attrs + + +def _filter_attrs_to_schema(node: IRNode) -> None: + schema = get_op(node.op) + if not schema.attrs: + node.attrs = {} + return + node.attrs = {name: node.attrs[name] for name in schema.attrs if name in node.attrs} + + +def _remove_noop_precision_cast(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + target_dtype = normalize_dtype_name(node.attrs.get("dtype")) + input_dtype = None if input_value is None else normalize_dtype_name(input_value.dtype) + output_dtype = None if output_value is None else normalize_dtype_name(output_value.dtype) + if target_dtype is None or target_dtype == input_dtype or (input_dtype is not None and output_dtype == input_dtype): + return bypass_unary_node(graph, node) + return False + + +def _remove_noop_view(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if tuple(input_value.shape) == tuple(output_value.shape): + return bypass_unary_node(graph, node) + return False + + +def _collapse_view_chain(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.producer is None: + return False + producer = graph.nodes.get(input_value.producer) + if producer is None or producer.op != "view" or len(producer.inputs) != 1: + return False + node.inputs = [producer.inputs[0]] + return True + + +def _remove_noop_slice(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if tuple(input_value.shape) == tuple(output_value.shape): + return bypass_unary_node(graph, node) + return False + + +def _remove_noop_scalar(graph: IRGraph, node: IRNode, *, neutral: float) -> bool: + value = node.attrs.get("value") + if value is None: + return False + if float(value) != float(neutral): + return False + return bypass_unary_node(graph, node) + + +def _materialize_new_ones_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None or output_value.dtype is None: + return False + torch_dtype = ir_dtype_to_torch(output_value.dtype) + const = torch.ones(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _materialize_new_zeros_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None or output_value.dtype is None: + return False + torch_dtype = ir_dtype_to_torch(output_value.dtype) + const = torch.zeros(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _materialize_arange_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + start = int(node.attrs.get("start", 0)) + end = node.attrs.get("end") + if end is None: + return False + step = int(node.attrs.get("step", 1)) + dtype = normalize_dtype_name(node.attrs.get("dtype", "fp32")) + torch_dtype = ir_dtype_to_torch(dtype) + const = torch.arange(start, int(end), step=step, dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _is_supported_without_backend_kernel(op_name: str) -> bool: + return has_op(op_name) or op_name in COMPILER_SUPPORTED_OPS diff --git a/python/src/transpile/canonicalize/utils.py b/python/src/transpile/canonicalize/utils.py new file mode 100644 index 000000000..aa2bb7297 --- /dev/null +++ b/python/src/transpile/canonicalize/utils.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +from typing import Any + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir + + +def normalize_dtype_name(dtype: Any) -> str | None: + if dtype is None: + return None + if not isinstance(dtype, str): + return str(dtype) + if dtype.startswith("torch."): + torch_name = dtype.split(".", 1)[1] + mapping = { + "bool": "bool", + "int8": "int8", + "int16": "int16", + "int32": "int32", + "int64": "int64", + "float16": "fp16", + "bfloat16": "bf16", + "float32": "fp32", + "float64": "fp64", + } + return mapping.get(torch_name, dtype) + return dtype + + +def ir_dtype_to_torch(dtype: str) -> torch.dtype: + normalized = normalize_dtype_name(dtype) + if normalized == "bool": + return torch.bool + if normalized == "int8": + return torch.int8 + if normalized == "int16": + return torch.int16 + if normalized == "int32": + return torch.int32 + if normalized == "int64": + return torch.int64 + if normalized == "fp16": + return torch.float16 + if normalized == "bf16": + return torch.bfloat16 + if normalized == "fp32": + return torch.float32 + if normalized == "fp64": + return torch.float64 + raise NotImplementedError(f"unsupported IR dtype for constant materialization: {dtype}") + + +def normalize_dim(dim: int, rank: int) -> int: + if dim < 0: + dim += rank + return dim + + +def numel(shape: tuple[int, ...]) -> int: + total = 1 + for dim in shape: + total *= int(dim) + return total + + +def refresh_value_users(graph: IRGraph) -> None: + for value in graph.values.values(): + value.users = [] + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + value = graph.values.get(input_id) + if value is not None: + value.users.append(node_id) + + +def rebuild_graph(graph: IRGraph) -> None: + new_values: dict[str, IRValue] = {} + + for value_id in graph.inputs: + old = graph.values.get(value_id, IRValue(id=value_id)) + new_values[value_id] = IRValue( + id=value_id, + shape=old.shape, + dtype=old.dtype, + producer=None, + users=[], + meta=dict(old.meta), + ) + + for value_id in graph.constants.keys(): + old = graph.values.get(value_id, IRValue(id=value_id)) + new_values[value_id] = IRValue( + id=value_id, + shape=old.shape, + dtype=old.dtype, + producer=None, + users=[], + meta=dict(old.meta), + ) + + for node_id in graph.order: + node = graph.nodes[node_id] + for output_id in node.outputs: + old = graph.values.get(output_id, IRValue(id=output_id)) + new_values[output_id] = IRValue( + id=output_id, + shape=old.shape, + dtype=old.dtype, + producer=node_id, + users=[], + meta=dict(old.meta), + ) + + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + if input_id not in new_values: + old = graph.values.get(input_id, IRValue(id=input_id)) + new_values[input_id] = IRValue( + id=input_id, + shape=old.shape, + dtype=old.dtype, + producer=old.producer, + users=[], + meta=dict(old.meta), + ) + new_values[input_id].users.append(node_id) + + for output_id in graph.outputs: + if output_id not in new_values: + old = graph.values.get(output_id, IRValue(id=output_id)) + new_values[output_id] = IRValue( + id=output_id, + shape=old.shape, + dtype=old.dtype, + producer=old.producer, + users=[], + meta=dict(old.meta), + ) + + graph.values = new_values + verify_ir(graph) + + +def dce(graph: IRGraph) -> None: + live_values = set(graph.outputs) + live_nodes: set[str] = set() + + changed = True + while changed: + changed = False + for node_id in reversed(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if not any(output in live_values for output in node.outputs): + continue + if node_id not in live_nodes: + live_nodes.add(node_id) + changed = True + for input_id in node.inputs: + if input_id not in live_values: + live_values.add(input_id) + changed = True + + graph.order = [node_id for node_id in graph.order if node_id in live_nodes] + graph.nodes = {node_id: node for node_id, node in graph.nodes.items() if node_id in live_nodes} + + live_value_ids = set(graph.inputs) | set(graph.outputs) + for node in graph.nodes.values(): + live_value_ids.update(node.inputs) + live_value_ids.update(node.outputs) + + graph.values = {value_id: value for value_id, value in graph.values.items() if value_id in live_value_ids} + graph.constants = {value_id: value for value_id, value in graph.constants.items() if value_id in live_value_ids} + rebuild_graph(graph) + + +def remove_node(graph: IRGraph, node_id: str) -> None: + if node_id in graph.nodes: + del graph.nodes[node_id] + graph.order = [existing_id for existing_id in graph.order if existing_id != node_id] + + +def drop_value_if_unused(graph: IRGraph, value_id: str) -> None: + if value_id in graph.inputs or value_id in graph.outputs or value_id in graph.constants: + return + value = graph.values.get(value_id) + if value is None: + return + if value.producer is None and not value.users: + del graph.values[value_id] + + +def bypass_unary_node(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_id = node.inputs[0] + output_id = node.outputs[0] + + for other in graph.nodes.values(): + other.inputs = [input_id if value_id == output_id else value_id for value_id in other.inputs] + graph.outputs = [input_id if value_id == output_id else value_id for value_id in graph.outputs] + + remove_node(graph, node.id) + if output_id in graph.values: + del graph.values[output_id] + drop_value_if_unused(graph, input_id) + return True + + +def _is_gemma4_graph(graph: IRGraph) -> bool: + return str(graph.meta.get("adapter_family") or graph.meta.get("family") or "").lower() == "gemma4" + + +def _weight_binding_meta(value: IRValue | None) -> dict[str, object] | None: + if value is None: + return None + path = value.meta.get("path") + kind = value.meta.get("kind") + source_name = value.meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + return dict(value.meta) + return None + + +def _gemma4_materialized_binding_meta(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + if not _is_gemma4_graph(graph) or len(node.inputs) != 1: + return None + if node.op == "precision_cast": + pass + elif node.op == "scalar_add" and float(node.attrs.get("value", 0.0)) in (0.0, 1.0): + pass + else: + return None + + value_id = node.inputs[0] + for _ in range(4): + value = graph.values.get(value_id) + binding_meta = _weight_binding_meta(value) + if binding_meta is not None: + binding_meta["materialized_from_value_id"] = value_id + binding_meta["materialized_by_op"] = node.op + return binding_meta + producer_id = value.producer if value is not None else None + producer_node = graph.nodes.get(producer_id) if producer_id is not None else None + if producer_node is None or producer_node.op not in {"precision_cast", "type_as"}: + break + if len(producer_node.inputs) != 1: + break + value_id = producer_node.inputs[0] + return None + + +def materialize_constant_output(graph: IRGraph, node: IRNode, value: Any) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None: + return False + binding_meta = _gemma4_materialized_binding_meta(graph, node) + if binding_meta is not None: + output_value.meta.update(binding_meta) + graph.constants[output_id] = value + output_value.producer = None + remove_node(graph, node.id) + return True diff --git a/python/src/transpile/capture/import_semantics.py b/python/src/transpile/capture/import_semantics.py deleted file mode 100644 index b97666f2b..000000000 --- a/python/src/transpile/capture/import_semantics.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -import torch - -from src.transpile.capture.graph_ir import IRGraph -from src.transpile.capture.graph_ir import IRNode -from src.transpile.capture.graph_ir import verify_ir - -def apply_import_semantics(graph: IRGraph) -> IRGraph: - return graph diff --git a/python/src/transpile/capture/weight_binding.py b/python/src/transpile/capture/weight_binding.py deleted file mode 100644 index 29c07dc81..000000000 --- a/python/src/transpile/capture/weight_binding.py +++ /dev/null @@ -1,185 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import json -import os -from pathlib import Path -import re - - -@dataclass(frozen=True) -class WeightBinding: - path: str - kind: str # "weight" | "embedding" - source_name: str - - -_PROJECT_ROOT = Path(__file__).resolve().parents[3] - - -def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: - candidates: list[str] = [] - raw = model_name_or_path.strip() - if not raw: - return candidates - - def _add(name: str) -> None: - name = name.strip().lower() - if name and name not in candidates: - candidates.append(name) - - _add(raw.split("/")[-1]) - - path = Path(raw) - parts = path.parts - for part in parts: - if part.startswith("models--"): - _add(part[len("models--"):].split("--")[-1]) - break - - return candidates - - -def _default_weights_dir_for_model_name(model_name_or_path: str) -> str | None: - if not model_name_or_path: - return None - for model_dir_name in _candidate_model_dir_names(model_name_or_path): - candidate = _PROJECT_ROOT / "weights" / model_dir_name - if candidate.exists(): - return str(candidate) - return None - - -def _normalized_source_candidates(source_name: str) -> list[str]: - candidates: list[str] = [] - - def _add(name: str) -> None: - if name and name not in candidates: - candidates.append(name) - - raw = source_name.strip() - _add(raw) - - for prefix in ("p_", "b_", "c_"): - if raw.startswith(prefix): - _add(raw[len(prefix):]) - - stripped = candidates[-1] - - if stripped.startswith("module.backbone."): - tail = stripped[len("module.backbone."):] - _add(f"model.{tail}") - _add(f"model.language_model.{tail}") - if tail.startswith("layers."): - _add(f"model.{tail}") - _add(f"model.language_model.{tail}") - - layer_match = re.match( - r"^(?:module_)?backbone_layers_slice_none__\d+__none____modules__(\d+)___(.+)$", - stripped, - ) - if layer_match: - layer_index = int(layer_match.group(1)) - tail = layer_match.group(2) - tail_map = { - "self_attn_q_proj_weight": "self_attn.q_proj.weight", - "self_attn_k_proj_weight": "self_attn.k_proj.weight", - "self_attn_v_proj_weight": "self_attn.v_proj.weight", - "self_attn_o_proj_weight": "self_attn.o_proj.weight", - "self_attn_q_norm_weight": "self_attn.q_norm.weight", - "self_attn_k_norm_weight": "self_attn.k_norm.weight", - "mlp_gate_proj_weight": "mlp.gate_proj.weight", - "mlp_up_proj_weight": "mlp.up_proj.weight", - "mlp_down_proj_weight": "mlp.down_proj.weight", - "input_layernorm_weight": "input_layernorm.weight", - "post_attention_layernorm_weight": "post_attention_layernorm.weight", - "linear_attn_in_proj_qkv_weight": "linear_attn.in_proj_qkv.weight", - "linear_attn_conv1d_weight": "linear_attn.conv1d.weight", - "linear_attn_norm_weight": "linear_attn.norm.weight", - "linear_attn_dt_bias": "linear_attn.dt_bias", - "linear_attn_A_log": "linear_attn.A_log", - } - dotted_tail = tail_map.get(tail) - if dotted_tail is not None: - _add(f"model.layers.{layer_index}.{dotted_tail}") - _add(f"model.language_model.layers.{layer_index}.{dotted_tail}") - - embed_map = { - "module_backbone_embed_tokens_weight": [ - "model.embed_tokens.weight", - "model.language_model.embed_tokens.weight", - ], - "module_backbone_embed_tokens_per_layer_weight": [ - "model.embed_tokens_per_layer.weight", - "model.language_model.embed_tokens_per_layer.weight", - ], - "module_backbone_per_layer_model_projection_weight": [ - "model.per_layer_model_projection.weight", - "model.language_model.per_layer_model_projection.weight", - ], - "module_backbone_norm_weight": [ - "model.norm.weight", - "model.language_model.norm.weight", - ], - "module_model_lm_head_weight": [ - "lm_head.weight", - ], - } - for key, mapped in embed_map.items(): - if stripped == key: - for item in mapped: - _add(item) - - return candidates - - -def resolve_transpile_weights_dir(graph_meta: dict[str, object]) -> str | None: - explicit = graph_meta.get("weights_dir") - if isinstance(explicit, str) and explicit: - return explicit - - family = str(graph_meta.get("adapter_family", "")).upper() - family_env = f"CACTUS_TRANSPILER_WEIGHTS_DIR_{family}" - if family and family_env in os.environ and os.environ[family_env]: - return os.environ[family_env] - - generic = os.environ.get("CACTUS_TRANSPILER_WEIGHTS_DIR") - if generic: - return generic - - model_name_or_path = graph_meta.get("model_name_or_path") - if isinstance(model_name_or_path, str) and model_name_or_path: - default_dir = _default_weights_dir_for_model_name(model_name_or_path) - if default_dir: - return default_dir - return None - - -def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> WeightBinding | None: - if not weights_dir: - return None - root = Path(weights_dir) - if not root.exists(): - return None - manifest_path = root / "weights_manifest.json" - if not manifest_path.exists(): - return None - try: - manifest = json.loads(manifest_path.read_text()) - except Exception: - return None - if not isinstance(manifest, dict): - return None - for candidate_name in _normalized_source_candidates(source_name): - entry = manifest.get(candidate_name) - if not isinstance(entry, dict): - continue - filename = entry.get("filename") - kind = entry.get("kind", "weight") - if not isinstance(filename, str) or not isinstance(kind, str): - continue - candidate = root / filename - if not candidate.exists(): - continue - return WeightBinding(path=str(candidate), kind=kind, source_name=candidate_name) - return None diff --git a/python/src/transpile/capture/capture_pytorch.py b/python/src/transpile/capture_pytorch.py similarity index 85% rename from python/src/transpile/capture/capture_pytorch.py rename to python/src/transpile/capture_pytorch.py index 43448936f..e21f1c712 100644 --- a/python/src/transpile/capture/capture_pytorch.py +++ b/python/src/transpile/capture_pytorch.py @@ -5,8 +5,9 @@ import torch from torch.export import export +from torch.fx.passes.shape_prop import ShapeProp -from src.transpile.capture.graph_ir import IRGraph +from src.transpile.graph_ir import IRGraph class CapturePhaseError(RuntimeError): @@ -168,6 +169,19 @@ def _collect_transpile_metadata(model: torch.nn.Module, args: tuple[Any, ...], k "import_hints": import_hints, } + +def _prepare_import_graph_module(ep: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + graph_module = ep.graph_module + if kwargs: + return graph_module + + try: + graph_module = ep.module() + ShapeProp(graph_module).propagate(*args) + except Exception: + return ep.graph_module + return graph_module + def capture_model(model, args, kwargs=None, *, strict=True) -> CapturedModel: if not isinstance(model, torch.nn.Module): raise TypeError("model must be a torch.nn.Module") @@ -190,12 +204,14 @@ def capture_model(model, args, kwargs=None, *, strict=True) -> CapturedModel: f"torch.export failed for model={type(model).__name__} strict={strict}: {exc}", cause=exc, ) from exc - from src.transpile.capture.import_ir import import_captured_to_ir + from src.transpile.import_ir import import_captured_to_ir + + import_graph_module = _prepare_import_graph_module(ep, example_args, example_kwargs) raw_captured = CapturedModel( exported_program=ep, - graph_module=ep.graph_module, - graph=ep.graph, + graph_module=import_graph_module, + graph=import_graph_module.graph, state_dict=dict(ep.state_dict), ir_graph=IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[]), source_module=model, @@ -244,14 +260,40 @@ def resolve_attr(root: Any, target: str) -> Any: def format_target(node: Any) -> str: return str(node.target) + +def _try_materialize_int_tuple(values: Iterable[Any]) -> tuple[int, ...] | None: + materialized: list[int] = [] + try: + for value in values: + materialized.append(int(value)) + except Exception: + return None + return tuple(materialized) + + def get_tensor_metadata(node: Any) -> TensorMetadata | None: meta = getattr(node, "meta", {}) or {} + tensor_meta = meta.get("tensor_meta") + if tensor_meta is not None: + shape = tuple(getattr(tensor_meta, "shape", ())) + dtype = getattr(tensor_meta, "dtype", None) + stride_value = getattr(tensor_meta, "stride", None) + stride = _try_materialize_int_tuple(stride_value) if isinstance(stride_value, Iterable) else None + device = getattr(tensor_meta, "device", None) + if shape or dtype is not None or stride is not None or device is not None: + return TensorMetadata( + shape=shape, + dtype=dtype, + stride=stride, + device=device, + ) + value = meta.get("val") if value is None: return None if isinstance(value, torch.Tensor): - stride = tuple(int(s) for s in value.stride()) if value.layout == torch.strided else None + stride = _try_materialize_int_tuple(value.stride()) if value.layout == torch.strided else None return TensorMetadata( shape=tuple(value.shape), dtype=value.dtype, diff --git a/python/src/transpile/component_bundle_runtime.py b/python/src/transpile/component_bundle_runtime.py new file mode 100644 index 000000000..b7a9b2062 --- /dev/null +++ b/python/src/transpile/component_bundle_runtime.py @@ -0,0 +1,1192 @@ +from __future__ import annotations + +from dataclasses import dataclass +import ctypes +import json +from collections.abc import Mapping +from pathlib import Path +import re +import struct +import time +from typing import Any + +import numpy as np +import torch + +from src.tensor_io import CACTUS_MAGIC +from src.tensor_io import FLAG_INTERLEAVED +from src.tensor_io import align_offset +from src.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features +from src.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform +from src.transpile.audio_preprocess import prepare_cactus_audio_features +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.gemma4_runtime import ensure_transformers_supports_gemma4 +from src.transpile.gemma4_runtime import patch_torch_flex_attention_compat +from src.transpile.gemma4_runtime import patch_transformers_torchvision_probe +from src.transpile.gemma4_runtime import prepare_gemma4_multimodal_inputs +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.lower import transpile_preoptimized_ir +from src.transpile.optimize_graph import optimize_graph +from src.transpile.parakeet_tdt_local import greedy_decode_parakeet_tdt_token_ids +from src.transpile.parakeet_tdt_local import load_parakeet_tdt_config +from src.transpile.parakeet_tdt_local import prepare_parakeet_tdt_audio_features +from src.transpile.runtime_compat import _lib +from src.transpile.runtime_compat import cactus_node_t +from src.transpile.runtime_compat import Graph +from src.transpile.runtime_compat import Tensor + + +_HEADER_SIZE = 84 +_PRECISION_TO_DTYPE = { + Graph.INT8: np.int8, + Graph.FP16: np.float16, + Graph.FP32: np.float32, + Graph.INT4: np.uint8, + getattr(Graph, "CQ2", 4): np.uint8, + getattr(Graph, "CQ3", 5): np.uint8, + getattr(Graph, "CQ4", 6): np.uint8, +} + + +@dataclass +class LoadedTensorFile: + path: Path + precision: int + shape: tuple[int, ...] + data: np.memmap + scales: np.memmap | None + group_size: int + num_groups: int + is_interleaved: bool + original_n: int + + +@dataclass +class LoadedComponentGraph: + component: str + graph: Graph + runtime_inputs: list[Tensor] + outputs: list[Tensor] + bound_constant_bindings: list[dict[str, object]] + bound_tensor_files: list[object] + + def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: + if index < 0 or index >= len(self.runtime_inputs): + raise IndexError( + f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" + ) + self.graph.set_input(self.runtime_inputs[index], data, dtype=dtype) + + def set_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> None: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" + ) + for index, value in enumerate(inputs): + self.set_input(index, value) + + def execute(self) -> list[Tensor]: + self.graph.execute() + return self.outputs + + +def load_component_bundle_manifest(bundle_dir_or_manifest: str | Path) -> tuple[Path, dict[str, object]]: + path = Path(bundle_dir_or_manifest).expanduser().resolve() + if path.is_dir(): + candidate = path / "manifest.json" if path.name == "components" else path / "components" / "manifest.json" + if not candidate.exists(): + candidate = path / "manifest.json" + path = candidate + if not path.exists(): + raise FileNotFoundError(f"component bundle manifest not found: {path}") + manifest = json.loads(path.read_text()) + bundle_root = path.parent.parent if path.parent.name == "components" else path.parent + return bundle_root, manifest + + +def load_saved_component_graph( + *, + bundle_root: str | Path, + component_entry: dict[str, object], + weights_dir: str | Path | None = None, +) -> LoadedComponentGraph: + root = Path(bundle_root).expanduser().resolve() + raw_ir_relpath = component_entry.get("raw_ir") + optimized_ir_relpath = component_entry.get("optimized_ir") + has_raw_ir = isinstance(raw_ir_relpath, str) and bool(raw_ir_relpath) + has_optimized_ir = isinstance(optimized_ir_relpath, str) and bool(optimized_ir_relpath) + if has_raw_ir or has_optimized_ir: + return _load_component_graph_from_ir( + bundle_root=root, + component_entry=component_entry, + weights_dir=weights_dir, + ) + + graph_relpath = component_entry.get("graph") + if not isinstance(graph_relpath, str) or not graph_relpath: + raise ValueError(f"component entry is missing graph path: {component_entry}") + + graph_path = (root / graph_relpath).resolve() + graph = Graph.load(graph_path) + + runtime_inputs = [ + graph._tensor_from_node(int(node_id)) + for node_id in component_entry.get("runtime_input_node_ids", []) + ] + outputs = [ + graph._tensor_from_node(int(node_id)) + for node_id in component_entry.get("output_node_ids", []) + ] + bound_constant_bindings = list(component_entry.get("bound_constant_bindings") or []) + bound_tensor_files = _rebind_bound_constants( + graph=graph, + bundle_root=root, + bindings=bound_constant_bindings, + weights_dir=weights_dir, + ) + return LoadedComponentGraph( + component=str(component_entry.get("component", "unknown")), + graph=graph, + runtime_inputs=runtime_inputs, + outputs=outputs, + bound_constant_bindings=bound_constant_bindings, + bound_tensor_files=bound_tensor_files, + ) + + +def _load_component_graph_from_ir( + *, + bundle_root: Path, + component_entry: dict[str, object], + weights_dir: str | Path | None, +) -> LoadedComponentGraph: + raw_ir_relpath = component_entry.get("raw_ir") + optimized_ir_relpath = component_entry.get("optimized_ir") + has_raw_ir = isinstance(raw_ir_relpath, str) and bool(raw_ir_relpath) + has_optimized_ir = isinstance(optimized_ir_relpath, str) and bool(optimized_ir_relpath) + use_raw_ir = has_raw_ir and not has_optimized_ir + ir_relpath = str(raw_ir_relpath if use_raw_ir else optimized_ir_relpath) + ir_path = (bundle_root / ir_relpath).resolve() + payload = json.loads(ir_path.read_text()) + graph_payload = payload.get("graph") + if not isinstance(graph_payload, dict): + raise ValueError(f"saved IR payload is missing graph data: {ir_path}") + + ir_graph = _deserialize_saved_ir_graph( + graph_payload=graph_payload, + component_entry=component_entry, + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + if use_raw_ir: + canonicalize_exported_graph(ir_graph) + optimize_graph(ir_graph) + else: + family = str(ir_graph.meta.get("adapter_family") or ir_graph.meta.get("family") or "").strip().lower() + component = str(ir_graph.meta.get("component", "") or component_entry.get("component", "") or "").strip().lower() + if family == "gemma4" and component == "decoder": + optimize_graph(ir_graph) + transpiled = transpile_preoptimized_ir(ir_graph) + return LoadedComponentGraph( + component=str(component_entry.get("component", "unknown")), + graph=transpiled.graph, + runtime_inputs=list(transpiled.runtime_inputs), + outputs=list(transpiled.outputs), + bound_constant_bindings=list(component_entry.get("bound_constant_bindings") or []), + bound_tensor_files=[], + ) + + +def load_saved_component_graphs( + bundle_dir_or_manifest: str | Path, + *, + weights_dir: str | Path | None = None, +) -> tuple[dict[str, LoadedComponentGraph], dict[str, object]]: + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + loaded: dict[str, LoadedComponentGraph] = {} + for component_entry in manifest.get("components", []): + if not isinstance(component_entry, dict): + continue + component_name = str(component_entry.get("component", "")).strip() + if not component_name: + continue + loaded[component_name] = load_saved_component_graph( + bundle_root=bundle_root, + component_entry=component_entry, + weights_dir=weights_dir, + ) + return loaded, manifest + + +def run_transpiled_bundle( + bundle_dir_or_manifest: str | Path, + *, + audio_file: str | Path | None = None, + image_files: tuple[str, ...] = (), + prompt: str | None = None, + input_ids: str | list[int] | tuple[int, ...] | None = None, + weights_dir: str | Path | None = None, + torch_dtype: torch.dtype = torch.float16, + system_prompt: str | None = None, + enable_thinking: bool = False, +) -> dict[str, object]: + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + resolved_weights_dir = _default_weights_dir_for_manifest(manifest, explicit=weights_dir) + component_graphs, manifest = load_saved_component_graphs( + bundle_dir_or_manifest, + weights_dir=resolved_weights_dir, + ) + family = str(manifest.get("family", "") or "") + task = str(manifest.get("task", "") or "") + if family == "parakeet_tdt" and task == "tdt_transcription": + if audio_file is None: + raise ValueError("audio_file is required for Parakeet TDT component bundles") + return _run_parakeet_tdt_bundle( + component_graphs=component_graphs, + manifest=manifest, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + if task == "multimodal_causal_lm_logits": + return _run_multimodal_causal_lm_bundle( + component_graphs=component_graphs, + manifest=manifest, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking=enable_thinking, + ) + if task == "causal_lm_logits": + return _run_causal_lm_logits_bundle( + component_graphs=component_graphs, + manifest=manifest, + prompt=prompt, + input_ids=input_ids, + ) + if task == "encoder_hidden_states": + if audio_file is None: + inputs_meta = manifest.get("inputs") + if isinstance(inputs_meta, dict): + stored_audio = inputs_meta.get("audio_file") + if isinstance(stored_audio, str) and stored_audio: + audio_file = stored_audio + if audio_file is None: + raise ValueError("audio_file is required for encoder_hidden_states bundles") + return _run_encoder_hidden_states_bundle( + component_graphs=component_graphs, + manifest=manifest, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + raise NotImplementedError( + f"saved transpiled bundle execution is not implemented for family={family!r} task={task!r}" + ) + + +def _default_weights_dir_for_manifest( + manifest: Mapping[str, object], + *, + explicit: str | Path | None, +) -> str | Path | None: + if explicit is not None: + return explicit + model_id = str(manifest.get("model_id", "") or "").strip() + if not model_id: + return None + try: + from src.downloads import get_weights_dir + + candidate = get_weights_dir(model_id) + except Exception: + return None + return candidate if candidate.exists() else None + + +def _deserialize_saved_ir_graph( + *, + graph_payload: Mapping[str, object], + component_entry: Mapping[str, object], + bundle_root: Path, + weights_dir: str | Path | None, +) -> IRGraph: + values_payload = graph_payload.get("values") + nodes_payload = graph_payload.get("nodes") + if not isinstance(values_payload, dict) or not isinstance(nodes_payload, list): + raise ValueError("saved optimized IR graph is missing values or nodes payload") + + values: dict[str, IRValue] = {} + for value_id, raw_value in values_payload.items(): + if not isinstance(value_id, str) or not isinstance(raw_value, dict): + continue + shape = raw_value.get("shape") + values[value_id] = IRValue( + id=str(raw_value.get("id", value_id)), + shape=None if shape is None else tuple(int(dim) for dim in shape), + dtype=None if raw_value.get("dtype") is None else str(raw_value.get("dtype")), + producer=None if raw_value.get("producer") is None else str(raw_value.get("producer")), + users=[str(item) for item in raw_value.get("users", [])], + meta=dict(raw_value.get("meta") or {}), + ) + + nodes: dict[str, IRNode] = {} + order: list[str] = [] + for raw_node in nodes_payload: + if not isinstance(raw_node, dict): + continue + node_id = str(raw_node["id"]) + node = IRNode( + id=node_id, + op=str(raw_node["op"]), + inputs=[str(item) for item in raw_node.get("inputs", [])], + outputs=[str(item) for item in raw_node.get("outputs", [])], + attrs=dict(raw_node.get("attrs") or {}), + meta=dict(raw_node.get("meta") or {}), + kind=str(raw_node.get("kind", "generic")), + ) + nodes[node_id] = node + order.append(node_id) + + constants_payload = graph_payload.get("constants") + if not isinstance(constants_payload, dict): + raise ValueError("saved optimized IR graph is missing constants payload") + + constant_bindings_by_value_id = { + str(binding.get("value_id")): binding + for binding in component_entry.get("bound_constant_bindings", []) + if isinstance(binding, dict) and isinstance(binding.get("value_id"), str) + } + + constants: dict[str, object] = {} + for value_id, serialized in constants_payload.items(): + if not isinstance(value_id, str): + continue + value = values.get(value_id) + if value is None: + continue + constants[value_id] = _deserialize_saved_ir_constant( + value=value, + serialized=serialized, + binding=constant_bindings_by_value_id.get(value_id), + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + + return IRGraph( + values=values, + nodes=nodes, + order=order, + inputs=[str(item) for item in graph_payload.get("inputs", [])], + outputs=[str(item) for item in graph_payload.get("outputs", [])], + constants=constants, + meta=dict(graph_payload.get("meta") or {}), + ) + + +def _deserialize_saved_ir_constant( + *, + value: IRValue, + serialized: object, + binding: Mapping[str, object] | None, + bundle_root: Path, + weights_dir: str | Path | None, +) -> object: + def _torch_dtype_from_name(name: str) -> torch.dtype | None: + return { + "torch.bool": torch.bool, + "torch.uint8": torch.uint8, + "torch.int8": torch.int8, + "torch.int16": torch.int16, + "torch.int32": torch.int32, + "torch.int64": torch.int64, + "torch.float16": torch.float16, + "torch.float32": torch.float32, + "torch.float64": torch.float64, + }.get(name) + + meta = value.meta if isinstance(value.meta, dict) else {} + if isinstance(meta.get("path"), str) and isinstance(meta.get("kind"), str): + # The lowerer ignores the constant payload when a weight binding is present + # in IRValue metadata and re-attaches the mmap-backed tensor directly. + return 0 + + if isinstance(binding, Mapping): + binding_format = str(binding.get("format", "tensor_io") or "tensor_io") + if binding_format == "npy": + tensor_path = _resolve_bound_tensor_path( + str(binding["path"]), + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + array = np.load(tensor_path, mmap_mode="r") + return torch.from_numpy(np.array(array, copy=True)) + + if isinstance(serialized, (str, int, float, bool)) or serialized is None: + return serialized + + if isinstance(serialized, list): + return serialized + + if isinstance(serialized, dict): + value_type = str(serialized.get("type", "")) + if value_type in {"torch.Tensor", "numpy.ndarray"}: + shape = tuple(int(dim) for dim in serialized.get("shape", []) or []) + if not shape: + dtype = _torch_dtype_from_name(str(serialized.get("dtype", ""))) or torch.float32 + zero_scalar = False if dtype is torch.bool else 0 + return torch.tensor(zero_scalar, dtype=dtype) + raise ValueError( + "saved optimized IR is missing a materialized constant payload for " + f"{value.id} with shape={shape}; expected a bound_constants entry" + ) + return dict(serialized) + + return serialized + + +def execute_loaded_component_pipeline( + components: list[LoadedComponentGraph], + *, + initial_store: dict[str, Any], +) -> tuple[dict[str, np.ndarray], dict[str, list[np.ndarray]]]: + store: dict[str, np.ndarray] = {} + for key, value in initial_store.items(): + store[key] = _to_numpy(value) + + outputs_by_component: dict[str, list[np.ndarray]] = {} + for component in components: + runtime_inputs = [] + input_names = component_input_names(component) + for input_name in input_names: + if input_name not in store: + raise KeyError( + f"component {component.component} is missing pipeline input {input_name!r}" + ) + runtime_inputs.append(store[input_name]) + component.set_inputs(runtime_inputs) + raw_outputs = component.execute() + numpy_outputs = [output.numpy().copy() for output in raw_outputs] + output_names = component_output_names(component) + if len(numpy_outputs) != len(output_names): + raise ValueError( + f"component {component.component} produced {len(numpy_outputs)} outputs, " + f"expected {len(output_names)}" + ) + for output_name, value in zip(output_names, numpy_outputs, strict=True): + store[output_name] = value + outputs_by_component[component.component] = numpy_outputs + return store, outputs_by_component + + +def component_input_names(component: LoadedComponentGraph) -> tuple[str, ...]: + return tuple(str(value) for value in getattr(component, "_input_names", ())) + + +def component_output_names(component: LoadedComponentGraph) -> tuple[str, ...]: + return tuple(str(value) for value in getattr(component, "_output_names", ())) + + +def _run_parakeet_tdt_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + audio_file: str | Path, + torch_dtype: torch.dtype, +) -> dict[str, object]: + if "audio_encoder" not in component_graphs or "decoder" not in component_graphs: + raise ValueError("Parakeet TDT component bundle must include audio_encoder and decoder graphs") + + inputs_meta = manifest.get("inputs") or {} + input_shapes = inputs_meta.get("input_shapes") if isinstance(inputs_meta, dict) else {} + if not isinstance(input_shapes, dict): + input_shapes = {} + expected_shape = input_shapes.get("input_features") + if not (isinstance(expected_shape, list) and len(expected_shape) == 3): + raise ValueError("Parakeet TDT bundle manifest is missing inputs.input_shapes.input_features") + + model_source = str(manifest.get("model_source", "") or "") + config = load_parakeet_tdt_config(model_source) + preprocess_start = time.perf_counter() + input_features, active_frames = prepare_parakeet_tdt_audio_features( + audio_file=audio_file, + expected_frames=int(expected_shape[1]), + expected_mels=int(expected_shape[2]), + torch_dtype=torch_dtype, + ) + preprocess_end = time.perf_counter() + + _attach_component_io_names(manifest, component_graphs) + encoder_start = time.perf_counter() + store, _ = execute_loaded_component_pipeline( + [component_graphs["audio_encoder"]], + initial_store={"input_features": input_features}, + ) + encoder_end = time.perf_counter() + encoder_hidden_states = np.asarray(store["encoder_hidden_states"]) + batch_size = int(encoder_hidden_states.shape[0]) + if batch_size != 1: + raise ValueError("Parakeet TDT saved bundle runtime currently expects batch size 1") + + state_dtype = np.float16 if torch_dtype == torch.float16 else np.float32 + initial_states: list[np.ndarray] = [] + for _ in range(config.predictor_num_layers): + state_shape = (batch_size, config.predictor_hidden_dim) + initial_states.append(np.zeros(state_shape, dtype=state_dtype)) + initial_states.append(np.zeros(state_shape, dtype=state_dtype)) + + decoder_component = component_graphs["decoder"] + decoder_steps = 0 + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + nonlocal decoder_steps + decoder_steps += 1 + input_store: dict[str, object] = { + "encoder_frame": np.ascontiguousarray(frame), + "token_ids": np.full((batch_size,), token_id, dtype=np.int64), + } + for index in range(config.predictor_num_layers): + input_store[f"state_h_{index}"] = np.ascontiguousarray(state_values[index * 2]) + input_store[f"state_c_{index}"] = np.ascontiguousarray(state_values[index * 2 + 1]) + + runtime_inputs = [input_store[name] for name in component_input_names(decoder_component)] + decoder_component.set_inputs(runtime_inputs) + outputs = decoder_component.execute() + logits = outputs[0].numpy().astype(np.float32, copy=False) + next_states = tuple(output.numpy() for output in outputs[1:]) + return logits, next_states + + decoder_start = time.perf_counter() + emitted = greedy_decode_parakeet_tdt_token_ids( + config=config, + encoder_hidden_states=encoder_hidden_states, + initial_states=tuple(initial_states), + step=_step, + ) + decoder_end = time.perf_counter() + total_end = decoder_end + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "audio_file": str(Path(audio_file).expanduser().resolve()), + "preprocess_ms": (preprocess_end - preprocess_start) * 1000.0, + "encoder_ms": (encoder_end - encoder_start) * 1000.0, + "decoder_ms": (decoder_end - decoder_start) * 1000.0, + "total_ms": (total_end - preprocess_start) * 1000.0, + "decoder_steps": decoder_steps, + "active_feature_frames": active_frames, + "token_ids": emitted, + "transcript": _decode_parakeet_tdt_token_ids(config.vocabulary, emitted), + "encoder_hidden_shape": list(encoder_hidden_states.shape), + "component_order": list(manifest.get("component_order", [])), + } + + +def _run_multimodal_causal_lm_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + prompt: str | None, + image_files: tuple[str, ...], + audio_file: str | Path | None, + torch_dtype: torch.dtype, + system_prompt: str | None, + enable_thinking: bool, +) -> dict[str, object]: + required_components = ("vision_encoder", "audio_encoder", "lm_encoder", "decoder") + missing = [name for name in required_components if name not in component_graphs] + if missing: + raise ValueError( + "multimodal causal LM bundle requires components " + f"{required_components!r}, missing {missing!r}" + ) + + inputs_meta = manifest.get("inputs") if isinstance(manifest.get("inputs"), dict) else {} + resolved_prompt = prompt if prompt is not None else str(inputs_meta.get("prompt", "") or "") + if not resolved_prompt: + raise ValueError("provide --prompt for multimodal causal-LM bundles") + + resolved_image_files: tuple[str, ...] + if image_files: + resolved_image_files = tuple(str(Path(path).expanduser().resolve()) for path in image_files) + else: + stored_images = inputs_meta.get("image_files") + if isinstance(stored_images, list): + resolved_image_files = tuple(str(Path(path).expanduser().resolve()) for path in stored_images if isinstance(path, str) and path) + else: + resolved_image_files = () + if not resolved_image_files: + raise ValueError("provide --image or --image-file for multimodal causal-LM bundles") + + resolved_audio: str | None = None + if audio_file is not None: + resolved_audio = str(Path(audio_file).expanduser().resolve()) + else: + stored_audio = inputs_meta.get("audio_file") + if isinstance(stored_audio, str) and stored_audio: + resolved_audio = str(Path(stored_audio).expanduser().resolve()) + + external_transformers_site_packages = ensure_transformers_supports_gemma4() + if external_transformers_site_packages: + print( + "note=using external transformers install for gemma4 runtime: " + f"{external_transformers_site_packages}" + ) + patch_transformers_torchvision_probe() + patch_torch_flex_attention_compat() + + from transformers import AutoProcessor + + model_source = str(manifest.get("model_source", "") or manifest.get("model_id", "") or "") + if not model_source: + raise ValueError("bundle manifest is missing model_source/model_id for multimodal preprocessing") + + processor = AutoProcessor.from_pretrained( + model_source, + local_files_only=Path(model_source).exists(), + trust_remote_code=True, + ) + prepared = prepare_gemma4_multimodal_inputs( + processor, + prompt=resolved_prompt, + image_files=resolved_image_files, + audio_file=resolved_audio, + torch_dtype=torch_dtype, + system_prompt=system_prompt or "", + enable_thinking_if_supported=enable_thinking, + use_gemma4_chat_template=True, + ) + + _attach_component_io_names(manifest, component_graphs) + prepared_store = { + name: tensor.detach().cpu().numpy() + for name, tensor in zip(prepared.names, prepared.tensors, strict=True) + } + start = time.perf_counter() + store, _ = execute_loaded_component_pipeline( + [component_graphs[name] for name in required_components], + initial_store=prepared_store, + ) + end = time.perf_counter() + logits = np.asarray(store["logits"], dtype=np.float32) + if logits.ndim != 3: + raise RuntimeError(f"expected logits with shape [batch, seq, vocab], got {list(logits.shape)}") + + next_token_id = int(np.argmax(logits[0, -1])) + tokenizer = getattr(processor, "tokenizer", processor) + next_token = None + if hasattr(tokenizer, "decode"): + next_token = tokenizer.decode([next_token_id]) + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "component_order": list(manifest.get("component_order", [])), + "prompt": resolved_prompt, + "image_files": list(resolved_image_files), + "audio_file": resolved_audio, + "input_shapes": { + name: list(np.asarray(value).shape) + for name, value in prepared_store.items() + }, + "output_shape": list(logits.shape), + "total_ms": (end - start) * 1000.0, + "next_token_id": next_token_id, + "next_token": next_token, + } + + +def _run_causal_lm_logits_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + prompt: str | None, + input_ids: str | list[int] | tuple[int, ...] | None, +) -> dict[str, object]: + if "decoder" not in component_graphs: + raise ValueError("causal LM component bundle must include a decoder graph") + + token_ids, tokenizer = _resolve_causal_lm_input_ids( + manifest=manifest, + prompt=prompt, + input_ids=input_ids, + ) + if not token_ids: + raise ValueError("causal LM bundle input token ids are empty") + + _attach_component_io_names(manifest, component_graphs) + decoder = component_graphs["decoder"] + runtime_inputs = component_input_names(decoder) + if runtime_inputs and runtime_inputs != ("input_ids",): + raise ValueError( + "causal LM bundle runner currently expects decoder logical input ('input_ids',), " + f"got {runtime_inputs!r}" + ) + + input_array = np.asarray([token_ids], dtype=np.int64) + start = time.perf_counter() + decoder.set_inputs([input_array]) + outputs = decoder.execute() + end = time.perf_counter() + if not outputs: + raise RuntimeError("causal LM decoder graph produced no outputs") + + logits = outputs[0].numpy() + if logits.ndim != 3: + raise RuntimeError(f"expected logits with shape [batch, seq, vocab], got {list(logits.shape)}") + next_token_id = int(np.argmax(logits[0, -1])) + decoded = None + if tokenizer is None: + try: + tokenizer = _load_bundle_tokenizer(manifest) + except Exception: + tokenizer = None + if tokenizer is not None and hasattr(tokenizer, "decode"): + decoded = tokenizer.decode([next_token_id]) + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "component_order": list(manifest.get("component_order", [])), + "input_ids": token_ids, + "input_shape": list(input_array.shape), + "output_shape": list(logits.shape), + "decoder_ms": (end - start) * 1000.0, + "total_ms": (end - start) * 1000.0, + "next_token_id": next_token_id, + "next_token": decoded, + } + + +def _run_encoder_hidden_states_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + audio_file: str | Path, + torch_dtype: torch.dtype, +) -> dict[str, object]: + component_name = "audio_encoder" if "audio_encoder" in component_graphs else "encoder" + if component_name not in component_graphs: + raise ValueError("encoder_hidden_states bundle must include an audio_encoder or encoder graph") + + inputs_meta = manifest.get("inputs") or {} + input_shapes = inputs_meta.get("input_shapes") if isinstance(inputs_meta, dict) else {} + if not isinstance(input_shapes, dict): + input_shapes = {} + expected_shape = input_shapes.get("input_features") + if not (isinstance(expected_shape, list) and len(expected_shape) == 3): + raise ValueError("encoder bundle manifest is missing inputs.input_shapes.input_features") + + preprocess_start = time.perf_counter() + input_features, active_frames = _prepare_generic_audio_encoder_features( + audio_file=audio_file, + manifest=manifest, + expected_shape=expected_shape, + torch_dtype=torch_dtype, + ) + preprocess_end = time.perf_counter() + + _attach_component_io_names(manifest, component_graphs) + encoder = component_graphs[component_name] + encoder_start = time.perf_counter() + encoder.set_inputs([input_features]) + outputs = encoder.execute() + encoder_end = time.perf_counter() + if not outputs: + raise RuntimeError("encoder graph produced no outputs") + hidden = outputs[0].numpy() + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "audio_file": str(Path(audio_file).expanduser().resolve()), + "component_order": list(manifest.get("component_order", [])), + "active_feature_frames": active_frames, + "input_shape": list(input_features.shape), + "encoder_hidden_shape": list(hidden.shape), + "preprocess_ms": (preprocess_end - preprocess_start) * 1000.0, + "encoder_ms": (encoder_end - encoder_start) * 1000.0, + "total_ms": (encoder_end - preprocess_start) * 1000.0, + } + + +def _prepare_generic_audio_encoder_features( + *, + audio_file: str | Path, + manifest: dict[str, object], + expected_shape: list[object], + torch_dtype: torch.dtype, +) -> tuple[np.ndarray, int]: + family = str(manifest.get("family", "") or "") + inputs_meta = manifest.get("inputs") if isinstance(manifest.get("inputs"), dict) else {} + sample_rate = int(inputs_meta.get("sample_rate", 16000) if isinstance(inputs_meta, dict) else 16000) + batch = int(expected_shape[0]) + if batch != 1: + raise ValueError("saved audio encoder bundle runtime currently expects batch size 1") + + if family == "whisper": + expected_mels = int(expected_shape[1]) + expected_frames = int(expected_shape[2]) + try: + features, active_frames = prepare_cactus_audio_features( + audio_file, + model_type="whisper", + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + layout="mels_frames", + ) + return np.ascontiguousarray(features.detach().cpu().numpy()), active_frames + except Exception: + pass + else: + expected_frames = int(expected_shape[1]) + expected_mels = int(expected_shape[2]) + + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + features, feature_length = _generic_log_mel_features( + waveform, + sample_rate=sample_rate, + num_mels=expected_mels, + n_fft=400, + hop_length=160, + frame_length=400, + preemphasis=None, + ) + active_frames = min(feature_length, expected_frames) + features = features[:active_frames, :] + if expected_frames > active_frames: + features = np.pad(features, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + if family == "whisper": + features = np.ascontiguousarray(features.T) + features = np.ascontiguousarray(features, dtype=np.float16 if torch_dtype == torch.float16 else np.float32) + return np.expand_dims(features, axis=0), active_frames + + +def _resolve_causal_lm_input_ids( + *, + manifest: dict[str, object], + prompt: str | None, + input_ids: str | list[int] | tuple[int, ...] | None, +) -> tuple[list[int], object | None]: + if input_ids is not None: + return _parse_input_ids(input_ids), None + + if prompt is None: + inputs_meta = manifest.get("inputs") + if isinstance(inputs_meta, dict): + stored_ids = inputs_meta.get("input_ids") + parsed = _parse_nested_manifest_input_ids(stored_ids) + if parsed: + return parsed, None + stored_prompt = inputs_meta.get("prompt") + if isinstance(stored_prompt, str) and stored_prompt: + prompt = stored_prompt + if not prompt: + raise ValueError("provide --input-ids or --prompt for causal LM component bundles") + + tokenizer = _load_bundle_tokenizer(manifest) + token_ids = _tokenize_bundle_prompt(tokenizer, prompt) + return token_ids, tokenizer + + +def _parse_input_ids(input_ids: str | list[int] | tuple[int, ...]) -> list[int]: + if isinstance(input_ids, str): + parsed = [int(part.strip()) for part in input_ids.split(",") if part.strip()] + else: + parsed = [int(value) for value in input_ids] + if not parsed: + raise ValueError("input_ids was provided but no token ids were parsed") + return parsed + + +def _parse_nested_manifest_input_ids(value: object) -> list[int] | None: + if isinstance(value, list) and value: + if all(isinstance(item, int) for item in value): + return [int(item) for item in value] + first = value[0] + if isinstance(first, list) and all(isinstance(item, int) for item in first): + return [int(item) for item in first] + return None + + +def _patch_missing_lzma_backport() -> str | None: + try: + import importlib.util + import sys + + if importlib.util.find_spec("_lzma") is not None: + return None + if importlib.util.find_spec("backports.lzma") is None: + return None + import backports.lzma as backports_lzma # type: ignore + + sys.modules.setdefault("lzma", backports_lzma) + return "using backports.lzma because this Python build is missing _lzma" + except Exception: + return None + + +def _load_bundle_tokenizer(manifest: dict[str, object]): + _patch_missing_lzma_backport() + try: + from transformers import AutoTokenizer # type: ignore + except Exception as exc: + raise RuntimeError(f"transformers is required to tokenize --prompt: {exc}") from exc + + model_source = str(manifest.get("model_source", "") or manifest.get("model_id", "") or "") + if not model_source: + raise ValueError("bundle manifest is missing model_source/model_id; provide --input-ids instead") + return AutoTokenizer.from_pretrained( + model_source, + local_files_only=True, + trust_remote_code=True, + ) + + +def _tokenize_bundle_prompt(tokenizer: object, prompt: str) -> list[int]: + apply_chat_template = getattr(tokenizer, "apply_chat_template", None) + if callable(apply_chat_template): + try: + encoded = apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=True, + add_generation_prompt=True, + return_dict=True, + ) + ids = encoded["input_ids"] if isinstance(encoded, Mapping) else encoded + if ids and isinstance(ids[0], list): + ids = ids[0] + return [int(value) for value in ids] + except Exception: + pass + + encoded = tokenizer(prompt, return_tensors=None) # type: ignore[operator] + ids = encoded["input_ids"] if isinstance(encoded, Mapping) else encoded + if ids and isinstance(ids[0], list): + ids = ids[0] + return [int(value) for value in ids] + + +def _attach_component_io_names( + manifest: dict[str, object], + component_graphs: dict[str, LoadedComponentGraph], +) -> None: + family = str(manifest.get("family", "") or "") + task = str(manifest.get("task", "") or "") + config = None + if family == "parakeet_tdt" and task == "tdt_transcription": + config = load_parakeet_tdt_config(str(manifest.get("model_source", "") or "")) + + for component_entry in manifest.get("components", []): + if not isinstance(component_entry, dict): + continue + name = str(component_entry.get("component", "")).strip() + if not name or name not in component_graphs: + continue + component = component_graphs[name] + logical_inputs = tuple(str(value) for value in component_entry.get("logical_inputs", [])) + logical_outputs = tuple(str(value) for value in component_entry.get("logical_outputs", [])) + if not logical_inputs or not logical_outputs: + logical_inputs, logical_outputs = _infer_legacy_component_io_names( + family=family, + task=task, + component_name=name, + config=config, + ) + component._input_names = logical_inputs + component._output_names = logical_outputs + + +def _infer_legacy_component_io_names( + *, + family: str, + task: str, + component_name: str, + config, +) -> tuple[tuple[str, ...], tuple[str, ...]]: + if task in {"causal_lm_logits", "multimodal_causal_lm_logits"} and component_name == "decoder": + return ("input_ids",), ("logits",) + if task == "encoder_hidden_states" and component_name in {"audio_encoder", "encoder"}: + return ("input_features",), ("encoder_hidden_states",) + if family == "parakeet_tdt" and task == "tdt_transcription": + if component_name == "audio_encoder": + return ("input_features",), ("encoder_hidden_states",) + if component_name == "decoder": + predictor_layers = 0 if config is None else int(config.predictor_num_layers) + state_inputs: list[str] = [] + state_outputs: list[str] = [] + for index in range(predictor_layers): + state_inputs.extend((f"state_h_{index}", f"state_c_{index}")) + state_outputs.extend((f"state_h_{index}", f"state_c_{index}")) + return ( + ("encoder_frame", "token_ids", *state_inputs), + ("step_logits", *state_outputs), + ) + raise ValueError( + f"component bundle manifest is missing logical IO names for family={family!r} task={task!r} component={component_name!r}" + ) + + +def _rebind_bound_constants( + *, + graph: Graph, + bundle_root: Path, + bindings: list[dict[str, object]], + weights_dir: str | Path | None, +) -> list[object]: + loaded: list[object] = [] + for binding in bindings: + node_id = int(binding["node_id"]) + raw_path = str(binding["path"]) + tensor_path = _resolve_bound_tensor_path( + raw_path, + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + tensor = graph._tensor_from_node(node_id) + binding_format = str(binding.get("format", "tensor_io") or "tensor_io") + if binding_format == "npy": + constant_array = np.load(tensor_path, mmap_mode="r") + precision = int(binding.get("precision", tensor.dtype)) + graph.set_external_input(tensor, int(constant_array.ctypes.data), dtype=precision) + loaded.append(constant_array) + continue + + tensor_file = _open_cactus_tensor_file(tensor_path) + graph.set_external_input(tensor, int(tensor_file.data.ctypes.data), dtype=tensor_file.precision) + if tensor_file.scales is not None and tensor_file.group_size > 0: + rc = _lib.cactus_graph_set_grouped_scales( + graph.h, + cactus_node_t(tensor.id), + int(tensor_file.group_size), + int(tensor_file.num_groups), + tensor_file.scales.ctypes.data_as(ctypes.c_void_p), + ) + if rc != 0: + raise RuntimeError("graph_set_grouped_scales failed") + if tensor_file.is_interleaved: + graph.set_interleaved(tensor, True, tensor_file.original_n) + loaded.append(tensor_file) + return loaded + + +def _resolve_bound_tensor_path( + raw_path: str, + *, + bundle_root: Path, + weights_dir: str | Path | None, +) -> Path: + candidates: list[Path] = [] + explicit = Path(raw_path).expanduser() + if explicit.is_absolute(): + candidates.append(explicit) + else: + candidates.append((bundle_root / explicit).resolve()) + candidates.append(explicit.resolve()) + + if weights_dir is not None: + weights_root = Path(weights_dir).expanduser().resolve() + raw_parts = Path(raw_path).parts + for index in range(len(raw_parts)): + candidates.append(weights_root.joinpath(*raw_parts[index:])) + candidates.append(weights_root / Path(raw_path).name) + + for candidate in candidates: + if candidate.exists(): + return candidate.resolve() + raise FileNotFoundError( + f"could not resolve bound tensor file {raw_path!r} from bundle_root={bundle_root}" + + ("" if weights_dir is None else f" weights_dir={Path(weights_dir).expanduser().resolve()}") + ) + + +def _open_cactus_tensor_file(path: str | Path) -> LoadedTensorFile: + tensor_path = Path(path).expanduser().resolve() + with tensor_path.open("rb") as handle: + header = handle.read(_HEADER_SIZE) + if len(header) < _HEADER_SIZE: + raise RuntimeError(f"tensor file is too small for a Cactus header: {tensor_path}") + + magic = header[:4] + if magic != CACTUS_MAGIC: + raise RuntimeError(f"tensor file is missing the CACT header: {tensor_path}") + + flags = struct.unpack_from(" 0) + precision = int(struct.unpack_from(" 0 else 0 + data_offset = ( + align_offset(scales_offset + scales_bytes, alignment) + if scales_bytes > 0 + else aligned_header + ) + + data_element_count = byte_size // np.dtype(dtype).itemsize + data = np.memmap(tensor_path, mode="r", dtype=dtype, offset=data_offset, shape=(data_element_count,)) + scales = None + if scales_bytes > 0: + scales = np.memmap( + tensor_path, + mode="r", + dtype=np.float16, + offset=scales_offset, + shape=(scales_bytes // np.dtype(np.float16).itemsize,), + ) + return LoadedTensorFile( + path=tensor_path, + precision=precision, + shape=shape, + data=data, + scales=scales, + group_size=group_size, + num_groups=num_groups, + is_interleaved=bool(flags & FLAG_INTERLEAVED), + original_n=original_n, + ) + + +def _decode_parakeet_tdt_token_ids(vocabulary: tuple[str, ...], token_ids: list[int]) -> str: + pieces: list[str] = [] + for token_id in token_ids: + if token_id < 0 or token_id >= len(vocabulary): + continue + piece = vocabulary[token_id] + if piece.startswith("<|") and piece.endswith("|>"): + continue + pieces.append(piece) + text = "".join(pieces).replace("▁", " ") + return re.sub(r"\s+", " ", text).strip() + + +def _to_numpy(value: Any) -> np.ndarray: + if isinstance(value, np.ndarray): + return np.ascontiguousarray(value) + if isinstance(value, torch.Tensor): + return np.ascontiguousarray(value.detach().cpu().numpy()) + raise TypeError(f"unsupported runtime value type: {type(value).__name__}") diff --git a/python/src/transpile/component_partition.py b/python/src/transpile/component_partition.py new file mode 100644 index 000000000..baf4c738e --- /dev/null +++ b/python/src/transpile/component_partition.py @@ -0,0 +1,414 @@ +from __future__ import annotations + +import copy +from collections import Counter + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir + + +COMPONENT_AUDIO_ENCODER = "audio_encoder" +COMPONENT_VISION_ENCODER = "vision_encoder" +COMPONENT_LM_ENCODER = "lm_encoder" +COMPONENT_DECODER = "decoder" +COMPONENT_UNSPECIFIED = "unspecified" + +COMPONENT_ORDER = ( + COMPONENT_AUDIO_ENCODER, + COMPONENT_VISION_ENCODER, + COMPONENT_LM_ENCODER, + COMPONENT_DECODER, +) + + +def _normalized_name_candidates( + *, + value_id: str | None = None, + source_name: str | None = None, + meta: dict[str, object] | None = None, +) -> tuple[str, ...]: + names: list[str] = [] + + def _add(value: object) -> None: + if not isinstance(value, str): + return + normalized = value.strip().lower() + if normalized and normalized not in names: + names.append(normalized) + + if value_id is not None: + _add(value_id) + if source_name is not None: + _add(source_name) + if isinstance(meta, dict): + _add(meta.get("torch_name")) + for key in ("module_paths", "import_hint_providers"): + raw_paths = meta.get(key) + if isinstance(raw_paths, (tuple, list)): + for path in raw_paths: + _add(path) + return tuple(names) + + +def _contains_any(name: str, needles: tuple[str, ...]) -> bool: + return any(needle in name for needle in needles) + + +def classify_component_name_candidates( + candidates: tuple[str, ...], + *, + family: str, + task: str, +) -> str | None: + if not candidates: + return None + + for name in candidates: + if _contains_any( + name, + ( + "pixel_values", + "image_features", + "vision_tower", + "vision_model", + "vision_encoder", + "vision_backbone", + "embed_vision", + "image_tower", + "visual_encoder", + "visual.", + ), + ): + return COMPONENT_VISION_ENCODER + + for name in candidates: + if _contains_any( + name, + ( + "input_features", + "audio_features", + "speech_features", + "mel_features", + "audio_tower", + "audio_encoder", + "speech_encoder", + "speech_model.encoder", + "acoustic_encoder", + "feature_encoder", + "encoder.pre_encode", + "encoder.subsampling", + "encoder.layers", + "encoder.conformer", + "conformer.", + "subsample_conv_projection", + ), + ): + return COMPONENT_AUDIO_ENCODER + + for name in candidates: + if _contains_any( + name, + ( + "lm_head", + "transformer.h", + "decoder.layers", + "decoder.layer", + "decoder.block", + "language_model.layers", + "language_model.model.layers", + "model.language_model.layers", + "text_model.layers", + "model.layers", + "backbone.layers", + "adapter_backbone_layers_slice", + "decoder.prediction", + "dec_rnn", + "joint.", + "jointnet", + "ctc_head", + "classifier", + "output_projection", + "output_layer", + ), + ): + return COMPONENT_DECODER + + for name in candidates: + if task == "multimodal_causal_lm_logits" and _contains_any( + name, + ( + "input_ids", + "inputs_embeds", + "text_embeds", + "position_ids", + "get_input_embeddings", + "embed_tokens", + "embed_text", + "embed_tokens_per_layer", + "per_layer_model_projection", + "per_layer_input", + "placeholder", + "multimodal_projector", + "multi_modal_projector", + "connector", + "merge", + "masked_scatter", + "create_causal_mask", + "create_masks_for_generate", + "token_type", + "attention_mask", + "pixel_position", + "multimodal_backbone", + "adapter_model_model_language_model_embed_tokens", + "adapter_backbone_embed_tokens_per_layer", + "adapter_backbone_per_layer_model_projection", + ), + ): + return COMPONENT_LM_ENCODER + + for name in candidates: + if task in {"ctc_logits", "encoder_hidden_states"} and name.startswith("encoder."): + return COMPONENT_AUDIO_ENCODER + + if family == "gemma4" and task == "multimodal_causal_lm_logits": + for name in candidates: + if _contains_any( + name, + ( + "input_ids", + "token_type_ids", + "attention_mask", + "input_features_mask", + ), + ): + return COMPONENT_LM_ENCODER + + if family == "parakeet_tdt": + for name in candidates: + if _contains_any( + name, + ( + "decoder.prediction", + "dec_rnn", + "joint.", + "jointnet", + "predictor_", + "tdt_joint", + ), + ): + return COMPONENT_DECODER + if name.startswith("encoder."): + return COMPONENT_AUDIO_ENCODER + + return None + + +def classify_value_component(value_id: str, value: IRValue, *, family: str, task: str) -> str | None: + return classify_component_name_candidates( + _normalized_name_candidates( + value_id=value_id, + source_name=value.meta.get("source_name") if isinstance(value.meta, dict) else None, + meta=value.meta, + ), + family=family, + task=task, + ) + + +def classify_node_component(node: IRNode, *, family: str, task: str) -> str | None: + return classify_component_name_candidates( + _normalized_name_candidates(meta=node.meta), + family=family, + task=task, + ) + + +def _task_default_component(*, family: str, task: str) -> str: + if task == "multimodal_causal_lm_logits": + return COMPONENT_DECODER + if task == "causal_lm_logits": + return COMPONENT_DECODER + if task in {"ctc_logits", "encoder_hidden_states"}: + return COMPONENT_AUDIO_ENCODER + if family == "parakeet_tdt": + return COMPONENT_AUDIO_ENCODER + return COMPONENT_UNSPECIFIED + + +def _rebuild_users(graph: IRGraph) -> None: + for value in graph.values.values(): + value.users = [] + for value_id in graph.inputs: + graph.values[value_id].producer = None + for value_id in graph.constants: + graph.values[value_id].producer = None + for node_id in graph.order: + node = graph.nodes[node_id] + for output_id in node.outputs: + graph.values[output_id].producer = node_id + for input_id in node.inputs: + graph.values[input_id].users.append(node_id) + + +def annotate_ir_components(graph: IRGraph) -> None: + family = str(graph.meta.get("adapter_family", "") or "").lower() + task = str(graph.meta.get("task", "") or "").lower() + default_component = _task_default_component(family=family, task=task) + + for value_id, value in graph.values.items(): + component = classify_value_component(value_id, value, family=family, task=task) + if component is not None: + value.meta["component"] = component + + for node_id in graph.order: + node = graph.nodes[node_id] + component = classify_node_component(node, family=family, task=task) + if component is not None: + node.meta["component"] = component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = component + + for _ in range(8): + changed = False + for node_id in graph.order: + node = graph.nodes[node_id] + component = node.meta.get("component") + if component not in COMPONENT_ORDER: + input_components = { + graph.values[input_id].meta.get("component") + for input_id in node.inputs + if graph.values[input_id].meta.get("component") in COMPONENT_ORDER + } + if len(input_components) == 1: + component = next(iter(input_components)) + node.meta["component"] = component + changed = True + component = node.meta.get("component") + if component in COMPONENT_ORDER: + for output_id in node.outputs: + if graph.values[output_id].meta.get("component") != component: + graph.values[output_id].meta["component"] = component + changed = True + + for node_id in reversed(graph.order): + node = graph.nodes[node_id] + component = node.meta.get("component") + if component in COMPONENT_ORDER: + continue + output_components: set[str] = set() + for output_id in node.outputs: + value = graph.values[output_id] + for user_id in value.users: + user_component = graph.nodes[user_id].meta.get("component") + if user_component in COMPONENT_ORDER: + output_components.add(user_component) + if len(output_components) == 1: + component = next(iter(output_components)) + node.meta["component"] = component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = component + changed = True + + if not changed: + break + + for value in graph.values.values(): + component = value.meta.get("component") + if component not in COMPONENT_ORDER and default_component != COMPONENT_UNSPECIFIED: + value.meta["component"] = default_component + + for node_id in graph.order: + node = graph.nodes[node_id] + component = node.meta.get("component") + if component not in COMPONENT_ORDER and default_component != COMPONENT_UNSPECIFIED: + node.meta["component"] = default_component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = default_component + + counts = Counter( + str(graph.nodes[node_id].meta.get("component", COMPONENT_UNSPECIFIED)) + for node_id in graph.order + ) + graph.meta["component_counts"] = dict(sorted(counts.items())) + + +def summarize_ir_components(graph: IRGraph) -> dict[str, int]: + annotate_ir_components(graph) + counts = graph.meta.get("component_counts", {}) + if isinstance(counts, dict): + return {str(key): int(value) for key, value in counts.items()} + return {} + + +def _clone_value_for_subgraph(value: IRValue, *, as_input: bool) -> IRValue: + cloned = copy.deepcopy(value) + if as_input: + cloned.producer = None + cloned.users = [] + return cloned + + +def extract_component_subgraphs(graph: IRGraph) -> dict[str, IRGraph]: + annotate_ir_components(graph) + component_graphs: dict[str, IRGraph] = {} + + for component in COMPONENT_ORDER: + node_ids = [ + node_id + for node_id in graph.order + if graph.nodes[node_id].meta.get("component") == component + ] + if not node_ids: + continue + + node_id_set = set(node_ids) + inputs: list[str] = [] + outputs: list[str] = [] + constants: dict[str, object] = {} + values: dict[str, IRValue] = {} + nodes: dict[str, IRNode] = {} + + for node_id in node_ids: + node = copy.deepcopy(graph.nodes[node_id]) + nodes[node_id] = node + for input_id in node.inputs: + if input_id in graph.constants: + constants[input_id] = graph.constants[input_id] + if input_id not in values: + values[input_id] = _clone_value_for_subgraph(graph.values[input_id], as_input=True) + continue + producer = graph.values[input_id].producer + if producer not in node_id_set and input_id not in inputs: + inputs.append(input_id) + values[input_id] = _clone_value_for_subgraph(graph.values[input_id], as_input=True) + for output_id in node.outputs: + if output_id not in values: + values[output_id] = _clone_value_for_subgraph(graph.values[output_id], as_input=False) + + for node_id in node_ids: + node = graph.nodes[node_id] + for output_id in node.outputs: + value = graph.values[output_id] + if output_id in graph.outputs or any(user_id not in node_id_set for user_id in value.users): + if output_id not in outputs: + outputs.append(output_id) + + component_graph = IRGraph( + values=values, + nodes=nodes, + order=list(node_ids), + inputs=inputs, + outputs=outputs, + constants=constants, + meta={ + **copy.deepcopy(graph.meta), + "component": component, + }, + ) + _rebuild_users(component_graph) + verify_ir(component_graph) + component_graphs[component] = component_graph + + return component_graphs diff --git a/python/src/transpile/component_pipeline.py b/python/src/transpile/component_pipeline.py new file mode 100644 index 000000000..e9e7c5568 --- /dev/null +++ b/python/src/transpile/component_pipeline.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import copy +from dataclasses import dataclass +from dataclasses import field +import re +from typing import Any + +import numpy as np +import torch + +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.lower import TranspiledGraph +from src.transpile.lower import transpile_preoptimized_ir +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph + + +@dataclass +class ComponentModuleSpec: + component: str + module: torch.nn.Module + example_inputs: tuple[torch.Tensor, ...] + input_keys: tuple[str, ...] + output_keys: tuple[str, ...] + graph_meta: dict[str, object] = field(default_factory=dict) + metadata: dict[str, object] = field(default_factory=dict) + + +@dataclass +class CapturedComponent: + component: str + input_keys: tuple[str, ...] + output_keys: tuple[str, ...] + raw_ir_graph: IRGraph + optimized_ir_graph: IRGraph + transpiled_graph: TranspiledGraph + metadata: dict[str, object] = field(default_factory=dict) + + +_GRAPH_ARG_INPUT_RE = re.compile(r"^v_args_(\d+)$") + + +def _align_input_keys_to_graph_inputs( + declared_input_keys: tuple[str, ...], + graph_inputs: tuple[str, ...] | list[str], +) -> tuple[str, ...]: + aligned: list[str] = [] + for value_id in graph_inputs: + match = _GRAPH_ARG_INPUT_RE.match(str(value_id)) + if match is None: + return declared_input_keys + input_index = int(match.group(1)) + if input_index < 0 or input_index >= len(declared_input_keys): + return declared_input_keys + aligned.append(declared_input_keys[input_index]) + return tuple(aligned) + + +def _call_optional_capture_hook(spec: ComponentModuleSpec, attr_name: str) -> None: + hook = getattr(spec.module, attr_name, None) + if not callable(hook): + return + try: + hook( + component=spec.component, + example_inputs=spec.example_inputs, + input_keys=spec.input_keys, + output_keys=spec.output_keys, + ) + except TypeError: + hook() + + +def capture_component_spec( + spec: ComponentModuleSpec, + *, + fusion_config: FusionConfig | None = None, +) -> CapturedComponent: + wrapped = _ComponentCaptureWrapper(spec.module, graph_meta=spec.graph_meta).eval() + capture_error: Exception | None = None + _call_optional_capture_hook(spec, "prepare_for_capture") + try: + captured = capture_model(wrapped, spec.example_inputs) + except Exception as exc: + capture_error = exc + raise + finally: + try: + _call_optional_capture_hook(spec, "restore_after_capture") + except Exception: + if capture_error is None: + raise + raw_ir_graph = copy.deepcopy(captured.ir_graph) + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + canonicalize_exported_graph(optimized_ir_graph) + optimize_graph(optimized_ir_graph, config=fusion_config) + transpiled_graph = transpile_preoptimized_ir(copy.deepcopy(optimized_ir_graph)) + if len(spec.output_keys) != len(transpiled_graph.outputs): + raise ValueError( + f"component {spec.component} declared {len(spec.output_keys)} output keys but lowered " + f"{len(transpiled_graph.outputs)} graph outputs" + ) + aligned_input_keys = _align_input_keys_to_graph_inputs(spec.input_keys, optimized_ir_graph.inputs) + return CapturedComponent( + component=spec.component, + input_keys=aligned_input_keys, + output_keys=spec.output_keys, + raw_ir_graph=raw_ir_graph, + optimized_ir_graph=optimized_ir_graph, + transpiled_graph=transpiled_graph, + metadata=dict(spec.metadata), + ) + + +def execute_component_pipeline( + components: list[CapturedComponent], + *, + initial_store: dict[str, Any], +) -> tuple[dict[str, np.ndarray], dict[str, list[np.ndarray]]]: + store: dict[str, np.ndarray] = {} + for key, value in initial_store.items(): + store[key] = _to_numpy(value) + + outputs_by_component: dict[str, list[np.ndarray]] = {} + for component in components: + runtime_inputs = [] + for input_key in component.input_keys: + if input_key not in store: + raise KeyError( + f"component {component.component} is missing pipeline input {input_key!r}" + ) + runtime_inputs.append(store[input_key]) + component.transpiled_graph.set_inputs(runtime_inputs) + raw_outputs = component.transpiled_graph.execute() + numpy_outputs = [output.numpy().copy() for output in raw_outputs] + if len(numpy_outputs) != len(component.output_keys): + raise ValueError( + f"component {component.component} produced {len(numpy_outputs)} outputs, " + f"expected {len(component.output_keys)}" + ) + for output_key, value in zip(component.output_keys, numpy_outputs, strict=True): + store[output_key] = value + outputs_by_component[component.component] = numpy_outputs + + return store, outputs_by_component + + +def initial_store_from_named_tensors( + keys: tuple[str, ...] | list[str], + tensors: tuple[torch.Tensor, ...] | list[torch.Tensor], +) -> dict[str, np.ndarray]: + if len(keys) != len(tensors): + raise ValueError(f"expected {len(keys)} tensors, got {len(tensors)}") + return { + str(key): _to_numpy(tensor) + for key, tensor in zip(keys, tensors, strict=True) + } + + +def _to_numpy(value: Any) -> np.ndarray: + if isinstance(value, np.ndarray): + return np.ascontiguousarray(value) + if isinstance(value, torch.Tensor): + return np.ascontiguousarray(value.detach().cpu().numpy()) + raise TypeError(f"unsupported pipeline value type: {type(value).__name__}") + + +class _ComponentCaptureWrapper(torch.nn.Module): + def __init__(self, module: torch.nn.Module, *, graph_meta: dict[str, object]): + super().__init__() + self.module = module + self.graph_meta = dict(graph_meta) + + def forward(self, *args: torch.Tensor) -> Any: + return self.module(*args) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.module, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta = {} + existing = metadata.get("graph", {}) + if isinstance(existing, dict): + graph_meta.update(existing) + graph_meta.update(self.graph_meta) + metadata["graph"] = graph_meta + return metadata diff --git a/python/src/transpile/fusion/__init__.py b/python/src/transpile/fusion/__init__.py new file mode 100644 index 000000000..da61cf10e --- /dev/null +++ b/python/src/transpile/fusion/__init__.py @@ -0,0 +1,47 @@ +from src.transpile.fusion.conv import ConvModuleMatch +from src.transpile.fusion.conv import match_conv_module +from src.transpile.fusion.attention import AttentionBlockMatch +from src.transpile.fusion.attention import AttentionMatch +from src.transpile.fusion.attention import SelfAttentionBlockMatch +from src.transpile.fusion.attention import match_attention +from src.transpile.fusion.attention import match_attention_block +from src.transpile.fusion.attention import match_self_attention_block +from src.transpile.fusion.deltanet import GatedDeltaNetMatch +from src.transpile.fusion.deltanet import match_gated_deltanet +from src.transpile.fusion.linear import LinearMatch +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.lstm import LSTMCellMatch +from src.transpile.fusion.lstm import match_lstm_cell +from src.transpile.fusion.mlp import GatedMLPMatch +from src.transpile.fusion.mlp import match_gated_mlp +from src.transpile.fusion.rel_pos_bias import RelPosBiasMatch +from src.transpile.fusion.rel_pos_bias import match_rel_pos_bias +from src.transpile.fusion.rms_norm import RMSNormMatch +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.fusion.rope import RoPEMatch +from src.transpile.fusion.rope import match_rope + +__all__ = [ + "ConvModuleMatch", + "AttentionBlockMatch", + "AttentionMatch", + "SelfAttentionBlockMatch", + "GatedDeltaNetMatch", + "GatedMLPMatch", + "LinearMatch", + "LSTMCellMatch", + "RelPosBiasMatch", + "RMSNormMatch", + "RoPEMatch", + "match_attention", + "match_attention_block", + "match_self_attention_block", + "match_conv_module", + "match_gated_deltanet", + "match_gated_mlp", + "match_linear", + "match_lstm_cell", + "match_rel_pos_bias", + "match_rms_norm", + "match_rope", +] diff --git a/python/src/transpile/fusion/attention.py b/python/src/transpile/fusion/attention.py new file mode 100644 index 000000000..88017bca8 --- /dev/null +++ b/python/src/transpile/fusion/attention.py @@ -0,0 +1,550 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import collect_node_ids +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.fusion.rope import match_rope + + +@dataclass(frozen=True) +class AttentionMatch: + query_value_id: str + key_value_id: str + value_value_id: str + source_input_value_ids: tuple[str, str, str] + weight_value_ids: tuple[str | None, str | None, str | None] + has_rope: bool + has_qk_norm: bool + has_gqa_repeat: bool + is_causal: bool + scale: float + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class AttentionBlockMatch: + attention_node_id: str + query_value_id: str + key_value_id: str + value_value_id: str + mask_value_id: str | None + gate_value_id: str | None + output_projection_weight_value_id: str + output_projection_bias_value_id: str | None + attention_output_shape: tuple[int, ...] + is_causal: bool + additive_mask: bool + scale: float + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class SelfAttentionBlockMatch: + attention_node_id: str + hidden_value_id: str + query_weight_value_id: str + query_projection_bias_value_id: str | None + query_add_value_id: str | None + rel_query_add_value_id: str | None + key_weight_value_id: str + key_projection_bias_value_id: str | None + value_weight_value_id: str + value_projection_bias_value_id: str | None + mask_value_id: str | None + relative_key_input_value_id: str | None + relative_key_weight_value_id: str | None + relative_key_projection_bias_value_id: str | None + gate_value_id: str | None + output_projection_weight_value_id: str + output_projection_bias_value_id: str | None + query_shape: tuple[int, ...] + key_shape: tuple[int, ...] + value_shape: tuple[int, ...] + relative_key_shape: tuple[int, ...] | None + attention_output_shape: tuple[int, ...] + is_causal: bool + additive_mask: bool + scale: float + rel_pos_scale: float | None + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _ProjectedAttentionInputMatch: + hidden_value_id: str + weight_value_id: str + projection_bias_value_id: str | None + projected_shape: tuple[int, ...] + add_value_id: str | None + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _RelativeKeyProjectionMatch: + input_value_id: str + weight_value_id: str + projection_bias_value_id: str | None + projected_shape: tuple[int, ...] + node_ids: tuple[str, ...] + + +def match_attention(graph: IRGraph, node: IRNode) -> AttentionMatch | None: + if node.op not in {"attention", "scaled_dot_product_attention"} or len(node.inputs) < 3: + return None + + q_info = _extract_attention_input(graph, node.inputs[0], role="q") + k_info = _extract_attention_input(graph, node.inputs[1], role="k") + v_info = _extract_attention_input(graph, node.inputs[2], role="v") + if q_info is None or k_info is None or v_info is None: + return None + + has_rope = bool(q_info["has_rope"] and k_info["has_rope"]) + has_qk_norm = bool(q_info["has_rms_norm"] and k_info["has_rms_norm"]) + has_gqa = bool(k_info["has_gqa_repeat"] or v_info["has_gqa_repeat"] or node.attrs.get("enable_gqa", False)) + node_ids = {node.id, *q_info["node_ids"], *k_info["node_ids"], *v_info["node_ids"]} + + return AttentionMatch( + query_value_id=node.inputs[0], + key_value_id=node.inputs[1], + value_value_id=node.inputs[2], + source_input_value_ids=(q_info["source_input"], k_info["source_input"], v_info["source_input"]), + weight_value_ids=(q_info.get("weight_value_id"), k_info.get("weight_value_id"), v_info.get("weight_value_id")), + has_rope=has_rope, + has_qk_norm=has_qk_norm, + has_gqa_repeat=has_gqa, + is_causal=bool(node.attrs.get("is_causal", False)), + scale=float(node.attrs.get("scale", 0.0)), + window_size=int(node.attrs.get("window_size", 0)), + node_ids=tuple(sorted(node_ids)), + ) + + +def match_self_attention_block(graph: IRGraph, node: IRNode) -> SelfAttentionBlockMatch | None: + projection = match_linear(graph, node) + if projection is None: + return None + + output_path = _extract_attention_output_path(graph, projection.input_value_id) + if output_path is None: + return None + attention_node = graph.nodes.get(output_path["attention_node_id"]) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: + return None + + query_match = _extract_projected_attention_input(graph, attention_node.inputs[0], role="q") + key_match = _extract_projected_attention_input(graph, attention_node.inputs[1], role="k") + value_match = _extract_projected_attention_input(graph, attention_node.inputs[2], role="v") + if query_match is None or key_match is None or value_match is None: + return None + + if not ( + query_match.hidden_value_id == key_match.hidden_value_id == value_match.hidden_value_id + ): + return None + + mask_value_id = attention_node.inputs[3] if len(attention_node.inputs) > 3 else None + rel_query_add_value_id: str | None = None + relative_key_input_value_id: str | None = None + relative_key_weight_value_id: str | None = None + relative_key_projection_bias_value_id: str | None = None + relative_key_shape: tuple[int, ...] | None = None + rel_pos_scale: float | None = None + rel_pos_node_ids: tuple[str, ...] = () + + if mask_value_id is not None: + rel_pos_node = producer(graph, strip_passthrough(graph, mask_value_id)) + if rel_pos_node is not None and rel_pos_node.op == "rel_pos_bias" and len(rel_pos_node.inputs) == 2: + rel_query_match = _extract_projected_attention_input(graph, rel_pos_node.inputs[0], role="q") + relative_key_match = _extract_relative_key_projection(graph, rel_pos_node.inputs[1]) + if rel_query_match is None or relative_key_match is None: + return None + if not ( + rel_query_match.hidden_value_id == query_match.hidden_value_id + and rel_query_match.weight_value_id == query_match.weight_value_id + and rel_query_match.projection_bias_value_id == query_match.projection_bias_value_id + and rel_query_match.projected_shape == query_match.projected_shape + ): + return None + rel_query_add_value_id = rel_query_match.add_value_id + relative_key_input_value_id = relative_key_match.input_value_id + relative_key_weight_value_id = relative_key_match.weight_value_id + relative_key_projection_bias_value_id = relative_key_match.projection_bias_value_id + relative_key_shape = relative_key_match.projected_shape + rel_pos_scale = float(rel_pos_node.attrs.get("scale", 1.0)) + rel_pos_node_ids = collect_node_ids( + rel_pos_node.id, + rel_pos_node.meta.get("rel_pos_bias_nodes", ()), + rel_query_match.node_ids, + relative_key_match.node_ids, + ) + mask_value_id = None + + return SelfAttentionBlockMatch( + attention_node_id=attention_node.id, + hidden_value_id=query_match.hidden_value_id, + query_weight_value_id=query_match.weight_value_id, + query_projection_bias_value_id=query_match.projection_bias_value_id, + query_add_value_id=query_match.add_value_id, + rel_query_add_value_id=rel_query_add_value_id, + key_weight_value_id=key_match.weight_value_id, + key_projection_bias_value_id=key_match.projection_bias_value_id, + value_weight_value_id=value_match.weight_value_id, + value_projection_bias_value_id=value_match.projection_bias_value_id, + mask_value_id=mask_value_id, + relative_key_input_value_id=relative_key_input_value_id, + relative_key_weight_value_id=relative_key_weight_value_id, + relative_key_projection_bias_value_id=relative_key_projection_bias_value_id, + gate_value_id=output_path["gate_value_id"], + output_projection_weight_value_id=projection.weight_value_id, + output_projection_bias_value_id=projection.bias_value_id, + query_shape=query_match.projected_shape, + key_shape=key_match.projected_shape, + value_shape=value_match.projected_shape, + relative_key_shape=relative_key_shape, + attention_output_shape=output_path["attention_output_shape"], + is_causal=bool(attention_node.attrs.get("is_causal", True)), + additive_mask=bool(attention_node.attrs.get("additive_mask", False)), + scale=float(attention_node.attrs.get("scale", 0.0)), + rel_pos_scale=rel_pos_scale, + window_size=int(attention_node.attrs.get("window_size", 0)), + node_ids=collect_node_ids( + attention_node.id, + projection.node_ids, + output_path["node_ids"], + query_match.node_ids, + key_match.node_ids, + value_match.node_ids, + rel_pos_node_ids, + ), + ) + + +def match_attention_block(graph: IRGraph, node: IRNode) -> AttentionBlockMatch | None: + projection = match_linear(graph, node) + if projection is None: + return None + + output_path = _extract_attention_output_path(graph, projection.input_value_id) + if output_path is None: + return None + attention_node = graph.nodes.get(output_path["attention_node_id"]) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: + return None + + return AttentionBlockMatch( + attention_node_id=attention_node.id, + query_value_id=attention_node.inputs[0], + key_value_id=attention_node.inputs[1], + value_value_id=attention_node.inputs[2], + mask_value_id=attention_node.inputs[3] if len(attention_node.inputs) > 3 else None, + gate_value_id=output_path["gate_value_id"], + output_projection_weight_value_id=projection.weight_value_id, + output_projection_bias_value_id=projection.bias_value_id, + attention_output_shape=output_path["attention_output_shape"], + is_causal=bool(attention_node.attrs.get("is_causal", True)), + additive_mask=bool(attention_node.attrs.get("additive_mask", False)), + scale=float(attention_node.attrs.get("scale", 0.0)), + window_size=int(attention_node.attrs.get("window_size", 0)), + node_ids=tuple(sorted({attention_node.id, *projection.node_ids, *output_path["node_ids"]})), + ) + + +def _extract_attention_output_path(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + gate_value_id: str | None = None + + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + break + + node = producer(graph, current) + if node is not None and node.op == "multiply" and len(node.inputs) == 2: + for attn_candidate, gate_candidate in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + attn_match = _extract_attention_output_path(graph, attn_candidate) + if attn_match is None: + continue + attn_match["gate_value_id"] = gate_candidate + attn_match["node_ids"] = tuple(sorted({node.id, *attn_match["node_ids"]})) + return attn_match + + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + attention_output_shape = tuple(int(v) for v in reshape_node.attrs.get("shape", ())) + current = reshape_node.inputs[0] + transpose_node = producer(graph, current) + if transpose_node is None: + return None + + if transpose_node.op == "transpose": + if int(transpose_node.attrs.get("dim0", -1)) != 1 or int(transpose_node.attrs.get("dim1", -1)) != 2: + return None + current = transpose_node.inputs[0] + elif transpose_node.op == "permute": + permutation = tuple(int(v) for v in transpose_node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + current = transpose_node.inputs[0] + else: + return None + + attention_node = producer(graph, current) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"}: + return None + + return { + "attention_node_id": attention_node.id, + "gate_value_id": gate_value_id, + "attention_output_shape": attention_output_shape, + "node_ids": tuple(sorted({reshape_node.id, transpose_node.id})), + } + + +def _extract_projected_attention_input(graph: IRGraph, value_id: str, *, role: str) -> _ProjectedAttentionInputMatch | None: + current = strip_passthrough(graph, value_id) + add_node = producer(graph, current) + if add_node is not None and add_node.op == "add" and len(add_node.inputs) == 2: + for projected_candidate, add_candidate in ((add_node.inputs[0], add_node.inputs[1]), (add_node.inputs[1], add_node.inputs[0])): + projected = _extract_transposed_projection(graph, projected_candidate) + if projected is None: + continue + if not _looks_like_attention_addend(graph, add_candidate, projected.projected_shape): + continue + return _ProjectedAttentionInputMatch( + hidden_value_id=projected.hidden_value_id, + weight_value_id=projected.weight_value_id, + projection_bias_value_id=projected.projection_bias_value_id, + projected_shape=projected.projected_shape, + add_value_id=strip_passthrough(graph, add_candidate), + node_ids=collect_node_ids(add_node.id, projected.node_ids), + ) + + if role == "q": + return None + + return _extract_transposed_projection(graph, current) + + +def _extract_transposed_projection(graph: IRGraph, value_id: str) -> _ProjectedAttentionInputMatch | None: + current = strip_passthrough(graph, value_id) + layout_node = producer(graph, current) + if layout_node is None or len(layout_node.inputs) != 1: + return None + + if layout_node.op == "permute": + permutation = tuple(int(v) for v in layout_node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + elif layout_node.op == "transpose": + if int(layout_node.attrs.get("dim0", -1)) != 1 or int(layout_node.attrs.get("dim1", -1)) != 2: + return None + else: + return None + + projected_value_id = strip_passthrough(graph, layout_node.inputs[0]) + projected_value = graph.values.get(projected_value_id) + if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: + return None + + reshape_node = producer(graph, projected_value_id) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + linear_node = producer(graph, reshape_node.inputs[0]) + if linear_node is None: + return None + linear = match_linear(graph, linear_node) + if linear is None: + return None + + return _ProjectedAttentionInputMatch( + hidden_value_id=linear.input_value_id, + weight_value_id=linear.weight_value_id, + projection_bias_value_id=linear.bias_value_id, + projected_shape=tuple(int(v) for v in projected_value.shape), + add_value_id=None, + node_ids=collect_node_ids(layout_node.id, reshape_node.id, linear.node_ids), + ) + + +def _extract_relative_key_projection(graph: IRGraph, value_id: str) -> _RelativeKeyProjectionMatch | None: + current = strip_passthrough(graph, value_id) + projected_value = graph.values.get(current) + if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: + return None + + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + linear_node = producer(graph, reshape_node.inputs[0]) + if linear_node is None: + return None + linear = match_linear(graph, linear_node) + if linear is None: + return None + + return _RelativeKeyProjectionMatch( + input_value_id=linear.input_value_id, + weight_value_id=linear.weight_value_id, + projection_bias_value_id=linear.bias_value_id, + projected_shape=tuple(int(v) for v in projected_value.shape), + node_ids=collect_node_ids(reshape_node.id, linear.node_ids), + ) + + +def _looks_like_attention_addend(graph: IRGraph, value_id: str, target_shape: tuple[int, ...]) -> bool: + value = graph.values.get(strip_passthrough(graph, value_id)) + if value is None or value.shape is None: + return False + + shape = tuple(int(v) for v in value.shape) + if len(shape) != 4 or len(target_shape) != 4: + return False + + batch, seq_len, heads, head_dim = target_shape + if shape[0] not in (1, batch) or shape[3] != head_dim: + return False + + # Allow either BHSD-style tensors like [B, H, 1, D] or BSHD-style tensors like [B, 1, H, D]. + if shape[1] == heads and shape[2] in (1, seq_len): + return True + if shape[1] in (1, seq_len) and shape[2] == heads: + return True + return False + + +def _extract_attention_input(graph: IRGraph, value_id: str, *, role: str) -> dict[str, object] | None: + node_ids: set[str] = set() + current = strip_passthrough(graph, value_id) + has_gqa_repeat = False + + while True: + node = producer(graph, current) + if node is None: + return { + "source_input": current, + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + node_ids.add(node.id) + + if node.op in {"reshape", "view", "transpose", "permute"}: + current = node.inputs[0] + continue + + rope = match_rope(graph, current) + if rope is not None: + rope_info = _extract_attention_input(graph, rope.input_value_id, role=role) + if rope_info is None: + return None + rope_info["has_rope"] = True + rope_info["node_ids"] = tuple(sorted(set(rope_info["node_ids"]) | node_ids | set(rope.node_ids))) + rope_info["has_gqa_repeat"] = bool(rope_info["has_gqa_repeat"] or has_gqa_repeat) + return rope_info + + rms = match_rms_norm(graph, node) + if rms is not None: + if role == "v": + return None + rms_info = _extract_attention_input(graph, rms.input_value_id, role=role) + if rms_info is None: + return None + rms_info["has_rms_norm"] = True + rms_info["node_ids"] = tuple(sorted(set(rms_info["node_ids"]) | node_ids | set(rms.node_ids))) + rms_info["has_gqa_repeat"] = bool(rms_info["has_gqa_repeat"] or has_gqa_repeat) + return rms_info + + if _looks_like_gqa_repeat(graph, current): + has_gqa_repeat = True + current = _unwrap_gqa_repeat(graph, current) + if current is None: + return None + continue + + linear_node = producer(graph, current) + if linear_node is None: + return { + "source_input": strip_passthrough(graph, current), + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + linear = match_linear(graph, linear_node) + if linear is None: + return { + "source_input": strip_passthrough(graph, current), + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + return { + "source_input": strip_passthrough(graph, linear.input_value_id), + "weight_value_id": linear.weight_value_id, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(set(node_ids) | set(linear.node_ids))), + } + + +def _looks_like_gqa_repeat(graph: IRGraph, value_id: str) -> bool: + return _unwrap_gqa_repeat(graph, value_id) is not None + + +def _unwrap_gqa_repeat(graph: IRGraph, value_id: str) -> str | None: + current = strip_passthrough(graph, value_id) + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"}: + return None + current = reshape_node.inputs[0] + expand_node = producer(graph, current) + if expand_node is None or expand_node.op != "expand": + return None + current = expand_node.inputs[0] + while True: + node = producer(graph, current) + if node is None: + return None + if node.op == "slice": + current = node.inputs[0] + continue + if node.op != "unsqueeze": + return None + if int(node.attrs.get("dim", -999)) != 2: + return None + break + base_value_id = strip_passthrough(graph, node.inputs[0]) + base_shape = graph.values.get(base_value_id).shape if graph.values.get(base_value_id) is not None else None + target_shape = tuple(int(v) for v in expand_node.attrs.get("shape", ())) + if base_shape is None or len(base_shape) != 4 or len(target_shape) != 5: + return None + expected = (int(base_shape[0]), int(base_shape[1]), int(target_shape[2]), int(base_shape[2]), int(base_shape[3])) + if expected != target_shape: + return None + return base_value_id diff --git a/python/src/transpile/fusion/common.py b/python/src/transpile/fusion/common.py new file mode 100644 index 000000000..828783b7e --- /dev/null +++ b/python/src/transpile/fusion/common.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class FusionMatch: + kind: str + anchor_node_id: str + node_ids: tuple[str, ...] + value_ids: tuple[str, ...] + attrs: dict[str, object] + + +def producer(graph: IRGraph, value_id: str) -> IRNode | None: + value = graph.values.get(value_id) + if value is None or value.producer is None: + return None + return graph.nodes.get(value.producer) + + +def strip_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def collect_node_ids(*parts: Any) -> tuple[str, ...]: + node_ids: set[str] = set() + for part in parts: + if part is None: + continue + if isinstance(part, str): + node_ids.add(part) + continue + if isinstance(part, IRNode): + node_ids.add(part.id) + continue + if isinstance(part, (list, tuple, set)): + for item in part: + if isinstance(item, str): + node_ids.add(item) + elif isinstance(item, IRNode): + node_ids.add(item.id) + return tuple(sorted(node_ids)) diff --git a/python/src/transpile/fusion/conv.py b/python/src/transpile/fusion/conv.py new file mode 100644 index 000000000..1f75d6e79 --- /dev/null +++ b/python/src/transpile/fusion/conv.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import collect_node_ids +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class ConvModuleMatch: + input_value_id: str + pointwise1_weight_value_id: str + pointwise1_bias_value_id: str | None + depthwise_weight_value_id: str + depthwise_bias_value_id: str | None + batch_norm_weight_value_id: str + batch_norm_bias_value_id: str + batch_norm_running_mean_value_id: str + batch_norm_running_var_value_id: str + pointwise2_weight_value_id: str + pointwise2_bias_value_id: str | None + eps: float + depthwise_kernel_size: int + depthwise_padding: int + node_ids: tuple[str, ...] + + +def match_conv_module(graph: IRGraph, node: IRNode) -> ConvModuleMatch | None: + if node.op != "permute" or len(node.inputs) != 1 or len(node.outputs) != 1: + return None + if tuple(node.attrs.get("permutation", ())) != (0, 2, 1): + return None + + pointwise2 = producer(graph, strip_passthrough(graph, node.inputs[0])) + if not _is_pointwise_conv1d(pointwise2): + return None + + silu = producer(graph, strip_passthrough(graph, pointwise2.inputs[0])) + if silu is None or silu.op != "silu" or len(silu.inputs) != 1: + return None + + batch_norm = producer(graph, strip_passthrough(graph, silu.inputs[0])) + if batch_norm is None or batch_norm.op != "batch_norm" or len(batch_norm.inputs) != 5: + return None + + depthwise = producer(graph, strip_passthrough(graph, batch_norm.inputs[0])) + if not _is_same_depthwise_conv1d(depthwise): + return None + + glu = producer(graph, strip_passthrough(graph, depthwise.inputs[0])) + if glu is None or glu.op != "glu" or len(glu.inputs) != 1: + return None + if int(glu.attrs.get("axis", -1)) not in {1, -2}: + return None + + pointwise1 = producer(graph, strip_passthrough(graph, glu.inputs[0])) + if not _is_pointwise_conv1d(pointwise1): + return None + + input_permute = producer(graph, strip_passthrough(graph, pointwise1.inputs[0])) + if input_permute is None or input_permute.op != "permute" or len(input_permute.inputs) != 1: + return None + if tuple(input_permute.attrs.get("permutation", ())) != (0, 2, 1): + return None + + depthwise_weight_shape = graph.values.get(depthwise.inputs[1]).shape if depthwise.inputs[1] in graph.values else None + if depthwise_weight_shape is None or len(depthwise_weight_shape) != 3: + return None + depthwise_kernel_size = int(depthwise_weight_shape[2]) + + return ConvModuleMatch( + input_value_id=strip_passthrough(graph, input_permute.inputs[0]), + pointwise1_weight_value_id=pointwise1.inputs[1], + pointwise1_bias_value_id=pointwise1.inputs[2] if len(pointwise1.inputs) > 2 else None, + depthwise_weight_value_id=depthwise.inputs[1], + depthwise_bias_value_id=depthwise.inputs[2] if len(depthwise.inputs) > 2 else None, + batch_norm_weight_value_id=batch_norm.inputs[1], + batch_norm_bias_value_id=batch_norm.inputs[2], + batch_norm_running_mean_value_id=batch_norm.inputs[3], + batch_norm_running_var_value_id=batch_norm.inputs[4], + pointwise2_weight_value_id=pointwise2.inputs[1], + pointwise2_bias_value_id=pointwise2.inputs[2] if len(pointwise2.inputs) > 2 else None, + eps=float(batch_norm.attrs.get("eps", 1e-5)), + depthwise_kernel_size=depthwise_kernel_size, + depthwise_padding=int(depthwise.attrs.get("padding", 0)), + node_ids=collect_node_ids( + node, + pointwise2, + silu, + batch_norm, + depthwise, + glu, + pointwise1, + input_permute, + ), + ) + + +def _is_pointwise_conv1d(node: IRNode | None) -> bool: + return ( + node is not None + and node.op == "conv1d" + and len(node.inputs) >= 2 + and int(node.attrs.get("stride", 1)) == 1 + and int(node.attrs.get("padding", 0)) == 0 + and int(node.attrs.get("dilation", 1)) == 1 + and int(node.attrs.get("groups", 1)) == 1 + ) + + +def _is_same_depthwise_conv1d(node: IRNode | None) -> bool: + if node is None or node.op != "conv1d" or len(node.inputs) < 2: + return False + groups = int(node.attrs.get("groups", 1)) + padding = int(node.attrs.get("padding", 0)) + stride = int(node.attrs.get("stride", 1)) + dilation = int(node.attrs.get("dilation", 1)) + if stride != 1 or dilation != 1: + return False + return groups > 1 and padding >= 0 diff --git a/python/src/transpile/fusion/deltanet.py b/python/src/transpile/fusion/deltanet.py new file mode 100644 index 000000000..cef656a5d --- /dev/null +++ b/python/src/transpile/fusion/deltanet.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class GatedDeltaNetMatch: + output_value_id: str + normalized_input_value_id: str + qkv_weight_value_id: str + a_weight_value_id: str + b_weight_value_id: str + norm_weight_value_id: str + z_weight_value_id: str | None + dt_bias_value_id: str | None + a_log_value_id: str | None + conv_weight_value_id: str | None + num_k_heads: int + num_v_heads: int + key_dim: int + value_dim: int + eps: float + chunk_size: int + mode: str + node_ids: tuple[str, ...] + + +def match_gated_deltanet(graph: IRGraph, node: IRNode) -> GatedDeltaNetMatch | None: + if node.op not in {"view", "reshape"} or len(node.outputs) != 1: + return None + output_value = graph.values.get(node.outputs[0]) + if output_value is None or output_value.shape is None or len(output_value.shape) != 3: + return None + + body_value_id = strip_passthrough(graph, node.inputs[0]) + body_node = producer(graph, body_value_id) + if body_node is None: + return None + + z_weight_value_id: str | None = None + rms_match = None + normalized_input_value_id: str | None = None + node_ids: set[str] = {node.id} + + if body_node.op == "multiply" and len(body_node.inputs) == 2: + for rms_value_id, z_value_id in ((body_node.inputs[0], body_node.inputs[1]), (body_node.inputs[1], body_node.inputs[0])): + rms_match = _match_rms_from_value(graph, rms_value_id) + if rms_match is None: + continue + z_weight_value_id, normalized_input_value_id, z_node_ids = _extract_z_branch(graph, z_value_id) + if normalized_input_value_id is None: + continue + node_ids.update(z_node_ids) + node_ids.add(body_node.id) + break + if rms_match is None or normalized_input_value_id is None: + return None + else: + rms_match = _match_rms_from_value(graph, body_value_id) + if rms_match is None: + return None + + y2d_value_id = rms_match.input_value_id + if rms_match.weight_value_id is None: + return None + norm_weight_value_id = rms_match.weight_value_id + eps = float(rms_match.eps) + node_ids.update(rms_match.node_ids) + + if normalized_input_value_id is None: + normalized_input_value_id = _infer_normalized_input_from_y_branch(graph, y2d_value_id) + if normalized_input_value_id is None: + return None + + qkv_info = _find_qkv_linear(graph, normalized_input_value_id) + if qkv_info is None: + return None + qkv_weight_value_id, conv_weight_value_id, num_k_heads, key_dim, num_v_heads, value_dim, qkv_node_ids = qkv_info + node_ids.update(qkv_node_ids) + + a_info = _find_a_linear(graph, normalized_input_value_id) + b_info = _find_b_linear(graph, normalized_input_value_id) + if a_info is None or b_info is None: + return None + a_weight_value_id, dt_bias_value_id, a_log_value_id, a_node_ids = a_info + b_weight_value_id, b_node_ids = b_info + node_ids.update(a_node_ids) + node_ids.update(b_node_ids) + + mode, chunk_size = _classify_mode(graph, y2d_value_id, match_node_ids=node_ids) + + return GatedDeltaNetMatch( + output_value_id=node.outputs[0], + normalized_input_value_id=normalized_input_value_id, + qkv_weight_value_id=qkv_weight_value_id, + a_weight_value_id=a_weight_value_id, + b_weight_value_id=b_weight_value_id, + norm_weight_value_id=norm_weight_value_id, + z_weight_value_id=z_weight_value_id, + dt_bias_value_id=dt_bias_value_id, + a_log_value_id=a_log_value_id, + conv_weight_value_id=conv_weight_value_id, + num_k_heads=num_k_heads, + num_v_heads=num_v_heads, + key_dim=key_dim, + value_dim=value_dim, + eps=eps, + chunk_size=chunk_size, + mode=mode, + node_ids=tuple(sorted(node_ids)), + ) + + +def _match_rms_from_value(graph: IRGraph, value_id: str): + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + return None + return match_rms_norm(graph, node) + + +def _extract_z_branch(graph: IRGraph, value_id: str) -> tuple[str | None, str | None, tuple[str, ...]]: + current = strip_passthrough(graph, value_id) + silu_node = producer(graph, current) + if silu_node is None or silu_node.op != "silu" or len(silu_node.inputs) != 1: + return None, None, () + linear_value_id = strip_passthrough(graph, silu_node.inputs[0]) + linear_node = producer(graph, linear_value_id) + while linear_node is not None and linear_node.op in {"view", "reshape"} and len(linear_node.inputs) == 1: + linear_value_id = strip_passthrough(graph, linear_node.inputs[0]) + linear_node = producer(graph, linear_value_id) + if linear_node is None: + return None, None, () + linear_match = match_linear(graph, linear_node) + if linear_match is None: + return None, None, () + return ( + linear_match.weight_value_id, + linear_match.input_value_id, + tuple(sorted({silu_node.id, *linear_match.node_ids})), + ) + + +def _infer_normalized_input_from_y_branch(graph: IRGraph, value_id: str) -> str | None: + current = strip_passthrough(graph, value_id) + node = producer(graph, current) + visited: set[str] = set() + while node is not None and node.id not in visited: + visited.add(node.id) + for input_value_id in node.inputs: + candidate = producer(graph, strip_passthrough(graph, input_value_id)) + if candidate is None: + continue + linear_match = match_linear(graph, candidate) + if linear_match is not None: + return linear_match.input_value_id + if len(node.inputs) != 1: + return None + node = producer(graph, strip_passthrough(graph, node.inputs[0])) + return None + + +def _find_qkv_linear( + graph: IRGraph, + normalized_input_value_id: str, +) -> tuple[str, str | None, int, int, int, int, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + split_node = _find_descendant_op(graph, linear_match.output_value_id, {"split_with_sizes"}, max_depth=8) + if split_node is None: + continue + split_sizes = tuple(int(v) for v in split_node.attrs.get("sizes", ())) + if len(split_sizes) != 3: + continue + + conv_node = _find_descendant_op(graph, linear_match.output_value_id, {"conv1d"}, max_depth=6) + conv_weight_value_id = conv_node.inputs[1] if conv_node is not None and len(conv_node.inputs) >= 2 else None + + q_shape = None + v_shape = None + node_ids = {node.id, split_node.id} + if conv_node is not None: + node_ids.add(conv_node.id) + + split_value = graph.values.get(split_node.outputs[0]) + if split_value is None: + continue + for getitem_node_id in split_value.users: + getitem_node = graph.nodes.get(getitem_node_id) + if getitem_node is None or getitem_node.op != "getitem" or not getitem_node.outputs: + continue + shape = _find_rank4_shape(graph, getitem_node.outputs[0], max_depth=5) + if shape is None: + continue + branch_has_sum = _find_descendant_op(graph, getitem_node.outputs[0], {"sum"}, max_depth=8) is not None + node_ids.add(getitem_node.id) + if branch_has_sum and q_shape is None: + q_shape = shape + elif not branch_has_sum and v_shape is None: + v_shape = shape + + if q_shape is None or v_shape is None: + continue + + num_k_heads = int(q_shape[2]) + key_dim = int(q_shape[3]) + num_v_heads = int(v_shape[2]) + value_dim = int(v_shape[3]) + if num_k_heads <= 0 or key_dim <= 0 or num_v_heads <= 0 or value_dim <= 0: + continue + + return ( + linear_match.weight_value_id, + conv_weight_value_id, + num_k_heads, + key_dim, + num_v_heads, + value_dim, + tuple(sorted(node_ids | set(linear_match.node_ids))), + ) + return None + + +def _find_a_linear( + graph: IRGraph, + normalized_input_value_id: str, +) -> tuple[str, str | None, str | None, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + softplus_node = _find_descendant_op(graph, linear_match.output_value_id, {"softplus"}, max_depth=6) + if softplus_node is None: + continue + dt_bias_value_id = None + softplus_input_node = producer(graph, strip_passthrough(graph, softplus_node.inputs[0])) + if softplus_input_node is not None and softplus_input_node.op == "add": + for input_value_id in softplus_input_node.inputs: + value = graph.values.get(input_value_id) + if value is not None and value.producer is None and input_value_id in graph.constants: + dt_bias_value_id = input_value_id + break + a_log_value_id = None + multiply_node = _find_descendant_op(graph, softplus_node.outputs[0], {"multiply"}, max_depth=6) + if multiply_node is not None: + for input_value_id in multiply_node.inputs: + a_log_value_id = _extract_exp_constant_input(graph, input_value_id) + if a_log_value_id is not None: + break + node_ids = {softplus_node.id, *linear_match.node_ids} + if softplus_input_node is not None: + node_ids.add(softplus_input_node.id) + if multiply_node is not None: + node_ids.add(multiply_node.id) + return linear_match.weight_value_id, dt_bias_value_id, a_log_value_id, tuple(sorted(node_ids)) + return None + + +def _find_b_linear(graph: IRGraph, normalized_input_value_id: str) -> tuple[str, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + sigmoid_node = _find_descendant_op(graph, linear_match.output_value_id, {"sigmoid"}, max_depth=4) + if sigmoid_node is None: + continue + return linear_match.weight_value_id, tuple(sorted({sigmoid_node.id, *linear_match.node_ids})) + return None + + +def _extract_exp_constant_input(graph: IRGraph, value_id: str) -> str | None: + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + value = graph.values.get(value_id) + if value is not None and value.producer is None and value_id in graph.constants: + return value_id + return None + if node.op == "scalar_multiply": + return _extract_exp_constant_input(graph, node.inputs[0]) + if node.op == "scalar_exp": + source = strip_passthrough(graph, node.inputs[0]) + value = graph.values.get(source) + if value is not None and value.producer is None and source in graph.constants: + return source + return None + + +def _find_rank4_shape(graph: IRGraph, value_id: str, *, max_depth: int) -> tuple[int, int, int, int] | None: + frontier = [(value_id, 0)] + seen: set[str] = set() + while frontier: + current, depth = frontier.pop(0) + if current in seen or depth > max_depth: + continue + seen.add(current) + value = graph.values.get(current) + if value is not None and value.shape is not None and len(value.shape) == 4: + shape = tuple(int(dim) for dim in value.shape) + return shape # type: ignore[return-value] + for user_id in graph.values.get(current, None).users if graph.values.get(current, None) is not None else (): + user = graph.nodes.get(user_id) + if user is None or not user.outputs: + continue + frontier.extend((output_id, depth + 1) for output_id in user.outputs) + return None + + +def _find_descendant_op(graph: IRGraph, value_id: str, targets: set[str], *, max_depth: int) -> IRNode | None: + frontier = [(value_id, 0)] + seen: set[str] = set() + while frontier: + current, depth = frontier.pop(0) + if current in seen or depth > max_depth: + continue + seen.add(current) + value = graph.values.get(current) + if value is None: + continue + for user_id in value.users: + user = graph.nodes.get(user_id) + if user is None or not user.outputs: + continue + if user.op in targets: + return user + frontier.extend((output_id, depth + 1) for output_id in user.outputs) + return None + + +def _classify_mode(graph: IRGraph, y2d_value_id: str, *, match_node_ids: set[str]) -> tuple[str, int]: + del y2d_value_id + chunk_size = _infer_prefill_chunk_size(graph, match_node_ids) + # Current exported graphs are uncached/full-sequence captures, so fuse the + # linear-attention block to the prefill kernel by default. Decode should be + # emitted only once the capture path models cache/state explicitly. + return "prefill", chunk_size + + +def _infer_prefill_chunk_size(graph: IRGraph, match_node_ids: set[str]) -> int: + for node_id in match_node_ids: + node = graph.nodes.get(node_id) + if node is None or node.op != "aten.triu.default" or not node.inputs: + continue + ones_value = graph.values.get(node.inputs[0]) + if ones_value is not None and ones_value.shape and len(ones_value.shape) >= 1: + return int(ones_value.shape[-1]) + return 64 diff --git a/python/src/transpile/fusion/linear.py b/python/src/transpile/fusion/linear.py new file mode 100644 index 000000000..df5346730 --- /dev/null +++ b/python/src/transpile/fusion/linear.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class LinearMatch: + input_value_id: str + weight_value_id: str + bias_value_id: str | None + output_value_id: str + node_ids: tuple[str, ...] + kind: str + + +def match_linear(graph: IRGraph, node: IRNode) -> LinearMatch | None: + if node.op == "linear" and len(node.inputs) >= 2 and len(node.outputs) == 1: + return LinearMatch( + input_value_id=strip_passthrough(graph, node.inputs[0]), + weight_value_id=node.inputs[1], + bias_value_id=node.inputs[2] if bool(node.attrs.get("has_bias", False)) and len(node.inputs) > 2 else None, + output_value_id=node.outputs[0], + node_ids=(node.id,), + kind="linear", + ) + + if node.op == "addmm" and len(node.inputs) == 3 and len(node.outputs) == 1: + return LinearMatch( + input_value_id=strip_passthrough(graph, node.inputs[1]), + weight_value_id=node.inputs[2], + bias_value_id=node.inputs[0], + output_value_id=node.outputs[0], + node_ids=(node.id,), + kind="addmm", + ) + + if node.op != "add" or len(node.inputs) != 2 or len(node.outputs) != 1: + return None + + for matmul_value_id, bias_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + matmul_node = producer(graph, strip_passthrough(graph, matmul_value_id)) + if matmul_node is None or matmul_node.op != "matmul" or len(matmul_node.inputs) != 2: + continue + return LinearMatch( + input_value_id=strip_passthrough(graph, matmul_node.inputs[0]), + weight_value_id=matmul_node.inputs[1], + bias_value_id=bias_value_id, + output_value_id=node.outputs[0], + node_ids=tuple(sorted({node.id, matmul_node.id})), + kind="matmul_add", + ) + + return None diff --git a/python/src/transpile/fusion/lstm.py b/python/src/transpile/fusion/lstm.py new file mode 100644 index 000000000..f342071c4 --- /dev/null +++ b/python/src/transpile/fusion/lstm.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import collect_node_ids +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class LSTMCellMatch: + anchor_node_id: str + node_ids: tuple[str, ...] + x_value_id: str + h_prev_value_id: str + c_prev_value_id: str + weight_ih_value_id: str + weight_hh_value_id: str + bias_ih_value_id: str + bias_hh_value_id: str | None + h_output_value_id: str + c_output_value_id: str + + +def match_lstm_cell(graph: IRGraph, node: IRNode) -> LSTMCellMatch | None: + if node.op != "multiply" or len(node.inputs) != 2 or len(node.outputs) != 1: + return None + + for output_gate_value_id, c_tanh_value_id in ( + (node.inputs[0], node.inputs[1]), + (node.inputs[1], node.inputs[0]), + ): + output_gate = _extract_gate_activation(graph, output_gate_value_id, expected_activation="sigmoid") + if output_gate is None or output_gate["index"] != 3: + continue + + c_tanh_node = producer(graph, strip_passthrough(graph, c_tanh_value_id)) + if c_tanh_node is None or c_tanh_node.op != "tanh" or len(c_tanh_node.inputs) != 1: + continue + + c_next_value_id = strip_passthrough(graph, c_tanh_node.inputs[0]) + c_next_node = producer(graph, c_next_value_id) + if c_next_node is None or c_next_node.op != "add" or len(c_next_node.inputs) != 2 or len(c_next_node.outputs) != 1: + continue + + for forget_term_value_id, input_term_value_id in ( + (c_next_node.inputs[0], c_next_node.inputs[1]), + (c_next_node.inputs[1], c_next_node.inputs[0]), + ): + forget_term = _extract_forget_term(graph, forget_term_value_id) + input_term = _extract_input_term(graph, input_term_value_id) + if forget_term is None or input_term is None: + continue + + input_gate = input_term["input_gate"] + cell_gate = input_term["cell_gate"] + forget_gate = forget_term["forget_gate"] + chunk_node_id = output_gate["chunk_node_id"] + if ( + input_gate["chunk_node_id"] != chunk_node_id + or forget_gate["chunk_node_id"] != chunk_node_id + or cell_gate["chunk_node_id"] != chunk_node_id + ): + continue + if input_gate["index"] != 0 or forget_gate["index"] != 1 or cell_gate["index"] != 2: + continue + + chunk_node = graph.nodes.get(chunk_node_id) + if chunk_node is None or len(chunk_node.inputs) != 1: + continue + + gates = _extract_lstm_gate_sum(graph, chunk_node.inputs[0]) + if gates is None: + continue + + return LSTMCellMatch( + anchor_node_id=node.id, + node_ids=collect_node_ids( + node, + c_tanh_node, + c_next_node, + output_gate["node_ids"], + forget_gate["node_ids"], + input_gate["node_ids"], + cell_gate["node_ids"], + forget_term["node_ids"], + input_term["node_ids"], + chunk_node, + gates["node_ids"], + ), + x_value_id=gates["x_value_id"], + h_prev_value_id=gates["h_prev_value_id"], + c_prev_value_id=forget_term["c_prev_value_id"], + weight_ih_value_id=gates["weight_ih_value_id"], + weight_hh_value_id=gates["weight_hh_value_id"], + bias_ih_value_id=gates["bias_value_ids"][0], + bias_hh_value_id=gates["bias_value_ids"][1] if len(gates["bias_value_ids"]) > 1 else None, + h_output_value_id=node.outputs[0], + c_output_value_id=c_next_node.outputs[0], + ) + + return None + + +def _extract_gate_activation( + graph: IRGraph, + value_id: str, + *, + expected_activation: str, +) -> dict[str, object] | None: + activation_node = producer(graph, strip_passthrough(graph, value_id)) + if activation_node is None or activation_node.op != expected_activation or len(activation_node.inputs) != 1: + return None + + getitem_node = producer(graph, strip_passthrough(graph, activation_node.inputs[0])) + if getitem_node is None or getitem_node.op != "getitem" or len(getitem_node.inputs) != 1: + return None + + chunk_node = producer(graph, strip_passthrough(graph, getitem_node.inputs[0])) + if chunk_node is None or chunk_node.op != "chunk": + return None + if int(chunk_node.attrs.get("chunks", 0)) != 4: + return None + + return { + "chunk_node_id": chunk_node.id, + "index": int(getitem_node.attrs.get("index", -1)), + "preactivation_value_id": getitem_node.inputs[0], + "node_ids": (activation_node.id, getitem_node.id), + } + + +def _extract_forget_term(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul_node = producer(graph, strip_passthrough(graph, value_id)) + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for gate_value_id, c_prev_value_id in ( + (mul_node.inputs[0], mul_node.inputs[1]), + (mul_node.inputs[1], mul_node.inputs[0]), + ): + forget_gate = _extract_gate_activation(graph, gate_value_id, expected_activation="sigmoid") + if forget_gate is None: + continue + return { + "forget_gate": forget_gate, + "c_prev_value_id": strip_passthrough(graph, c_prev_value_id), + "node_ids": (mul_node.id,), + } + return None + + +def _extract_input_term(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul_node = producer(graph, strip_passthrough(graph, value_id)) + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for input_gate_value_id, cell_gate_value_id in ( + (mul_node.inputs[0], mul_node.inputs[1]), + (mul_node.inputs[1], mul_node.inputs[0]), + ): + input_gate = _extract_gate_activation(graph, input_gate_value_id, expected_activation="sigmoid") + if input_gate is None: + continue + cell_tanh_node = producer(graph, strip_passthrough(graph, cell_gate_value_id)) + if cell_tanh_node is None or cell_tanh_node.op != "tanh" or len(cell_tanh_node.inputs) != 1: + continue + cell_gate = _extract_gate_activation(graph, cell_gate_value_id, expected_activation="tanh") + if cell_gate is None: + continue + return { + "input_gate": input_gate, + "cell_gate": cell_gate, + "node_ids": (mul_node.id, cell_tanh_node.id), + } + return None + + +def _extract_lstm_gate_sum(graph: IRGraph, value_id: str) -> dict[str, object] | None: + terms, node_ids = _flatten_add_terms(graph, strip_passthrough(graph, value_id)) + linear_nodes: list[IRNode] = [] + bias_value_ids: list[str] = [] + for term_value_id in terms: + term_value_id = strip_passthrough(graph, term_value_id) + term_node = producer(graph, term_value_id) + if term_node is not None and term_node.op == "linear" and len(term_node.inputs) >= 2: + linear_nodes.append(term_node) + continue + if term_value_id in graph.constants: + bias_value_ids.append(term_value_id) + continue + return None + + if len(linear_nodes) != 2 or len(bias_value_ids) not in {1, 2}: + return None + + first_linear, second_linear = linear_nodes + return { + "x_value_id": first_linear.inputs[0], + "weight_ih_value_id": first_linear.inputs[1], + "h_prev_value_id": second_linear.inputs[0], + "weight_hh_value_id": second_linear.inputs[1], + "bias_value_ids": tuple(bias_value_ids), + "node_ids": (*node_ids, first_linear.id, second_linear.id), + } + + +def _flatten_add_terms(graph: IRGraph, value_id: str) -> tuple[list[str], tuple[str, ...]]: + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return [value_id], () + left_terms, left_nodes = _flatten_add_terms(graph, node.inputs[0]) + right_terms, right_nodes = _flatten_add_terms(graph, node.inputs[1]) + return [*left_terms, *right_terms], (*left_nodes, *right_nodes, node.id) diff --git a/python/src/transpile/fusion/mlp.py b/python/src/transpile/fusion/mlp.py new file mode 100644 index 000000000..1c3e0ab34 --- /dev/null +++ b/python/src/transpile/fusion/mlp.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear + + +@dataclass(frozen=True) +class GatedMLPMatch: + input_value_id: str + gate_weight_value_id: str + up_weight_value_id: str + down_weight_value_id: str + activation: str + bias_value_ids: tuple[str | None, str | None, str | None] + node_ids: tuple[str, ...] + + +def match_gated_mlp(graph: IRGraph, node: IRNode) -> GatedMLPMatch | None: + down = match_linear(graph, node) + if down is None: + return None + + mul_node = graph.nodes.get(graph.values[down.input_value_id].producer) if graph.values.get(down.input_value_id) is not None else None + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for activated_value_id, up_value_id in ((mul_node.inputs[0], mul_node.inputs[1]), (mul_node.inputs[1], mul_node.inputs[0])): + activated_node = graph.nodes.get(graph.values[strip_passthrough(graph, activated_value_id)].producer) if graph.values.get(strip_passthrough(graph, activated_value_id)) is not None else None + if activated_node is None or activated_node.op not in {"gelu", "silu"}: + continue + + gate = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer]) if graph.values.get(strip_passthrough(graph, activated_node.inputs[0])) is not None and graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer in graph.nodes else None + up = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, up_value_id)].producer]) if graph.values.get(strip_passthrough(graph, up_value_id)) is not None and graph.values[strip_passthrough(graph, up_value_id)].producer in graph.nodes else None + if gate is None or up is None: + continue + if strip_passthrough(graph, gate.input_value_id) != strip_passthrough(graph, up.input_value_id): + continue + + return GatedMLPMatch( + input_value_id=strip_passthrough(graph, gate.input_value_id), + gate_weight_value_id=gate.weight_value_id, + up_weight_value_id=up.weight_value_id, + down_weight_value_id=down.weight_value_id, + activation=activated_node.op, + bias_value_ids=(gate.bias_value_id, up.bias_value_id, down.bias_value_id), + node_ids=tuple(sorted({*gate.node_ids, activated_node.id, *up.node_ids, mul_node.id, *down.node_ids})), + ) + + return None diff --git a/python/src/transpile/fusion/rel_pos_bias.py b/python/src/transpile/fusion/rel_pos_bias.py new file mode 100644 index 000000000..bcb623ffd --- /dev/null +++ b/python/src/transpile/fusion/rel_pos_bias.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class RelPosBiasMatch: + query_value_id: str + relative_key_value_id: str + scale: float + node_ids: tuple[str, ...] + + +def match_rel_pos_bias(graph: IRGraph, node: IRNode) -> RelPosBiasMatch | None: + if node.op != "scalar_multiply" or len(node.inputs) != 1: + return None + + gather_match = _match_gather_rel_pos_bias(graph, node) + if gather_match is not None: + return gather_match + + final_slice = producer(graph, strip_passthrough(graph, node.inputs[0])) + if final_slice is None or final_slice.op != "slice": + return None + if int(final_slice.attrs.get("axis", -1)) != 3 or int(final_slice.attrs.get("start", -1)) != 0: + return None + + reshape_after_shift = producer(graph, strip_passthrough(graph, final_slice.inputs[0])) + if reshape_after_shift is None or reshape_after_shift.op not in {"reshape", "view"} or len(reshape_after_shift.inputs) != 1: + return None + + shift_slice = producer(graph, strip_passthrough(graph, reshape_after_shift.inputs[0])) + if shift_slice is None or shift_slice.op != "slice": + return None + if int(shift_slice.attrs.get("axis", -1)) != 2 or int(shift_slice.attrs.get("start", -1)) != 1: + return None + + reshape_for_shift = producer(graph, strip_passthrough(graph, shift_slice.inputs[0])) + if reshape_for_shift is None or reshape_for_shift.op not in {"reshape", "view"} or len(reshape_for_shift.inputs) != 1: + return None + + pad = producer(graph, strip_passthrough(graph, reshape_for_shift.inputs[0])) + if pad is None or pad.op != "pad" or len(pad.inputs) != 1: + return None + if tuple(int(v) for v in pad.attrs.get("pads", ())) != (1, 0): + return None + if str(pad.attrs.get("mode", "constant")) != "constant": + return None + if float(pad.attrs.get("value", 0.0)) != 0.0: + return None + + matmul = producer(graph, strip_passthrough(graph, pad.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + + rhs_permute = producer(graph, strip_passthrough(graph, matmul.inputs[1])) + if rhs_permute is None or rhs_permute.op != "permute" or len(rhs_permute.inputs) != 1: + return None + if tuple(int(v) for v in rhs_permute.attrs.get("permutation", ())) != (0, 2, 3, 1): + return None + + query_value_id = strip_passthrough(graph, matmul.inputs[0]) + relative_key_value_id = strip_passthrough(graph, rhs_permute.inputs[0]) + + query_shape = graph.values.get(query_value_id).shape if query_value_id in graph.values else None + relative_key_shape = graph.values.get(relative_key_value_id).shape if relative_key_value_id in graph.values else None + output_shape = graph.values.get(node.outputs[0]).shape if node.outputs and node.outputs[0] in graph.values else None + if query_shape is None or relative_key_shape is None or output_shape is None: + return None + if len(query_shape) != 4 or len(relative_key_shape) != 4 or len(output_shape) != 4: + return None + + batch, heads, seq_len, head_dim = (int(v) for v in query_shape) + rel_batch, rel_len, rel_heads, rel_dim = (int(v) for v in relative_key_shape) + out_batch, out_heads, out_t, out_s = (int(v) for v in output_shape) + if rel_batch not in {1, batch}: + return None + if rel_heads != heads or rel_dim != head_dim: + return None + if rel_len < 2 * seq_len - 1: + return None + if (out_batch, out_heads, out_t, out_s) != (batch, heads, seq_len, seq_len): + return None + + return RelPosBiasMatch( + query_value_id=query_value_id, + relative_key_value_id=relative_key_value_id, + scale=float(node.attrs.get("value", 1.0)), + node_ids=tuple( + sorted( + { + node.id, + final_slice.id, + reshape_after_shift.id, + shift_slice.id, + reshape_for_shift.id, + pad.id, + matmul.id, + rhs_permute.id, + } + ) + ), + ) + + +def _match_gather_rel_pos_bias(graph: IRGraph, node: IRNode) -> RelPosBiasMatch | None: + gather = producer(graph, strip_passthrough(graph, node.inputs[0])) + if gather is None or gather.op != "gather" or len(gather.inputs) < 2: + return None + if int(gather.attrs.get("axis", -1)) not in {-1, 3}: + return None + + matmul = producer(graph, strip_passthrough(graph, gather.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + + query_heads = producer(graph, strip_passthrough(graph, matmul.inputs[0])) + rhs_transpose = producer(graph, strip_passthrough(graph, matmul.inputs[1])) + if query_heads is None or query_heads.op != "permute" or len(query_heads.inputs) != 1: + return None + if rhs_transpose is None or rhs_transpose.op != "permute" or len(rhs_transpose.inputs) != 1: + return None + if tuple(int(v) for v in query_heads.attrs.get("permutation", ())) != (0, 2, 1, 3): + return None + if tuple(int(v) for v in rhs_transpose.attrs.get("permutation", ())) != (0, 1, 3, 2): + return None + + relative_key_heads = producer(graph, strip_passthrough(graph, rhs_transpose.inputs[0])) + if relative_key_heads is None or relative_key_heads.op != "permute" or len(relative_key_heads.inputs) != 1: + return None + if tuple(int(v) for v in relative_key_heads.attrs.get("permutation", ())) != (0, 2, 1, 3): + return None + + query_value_id = strip_passthrough(graph, query_heads.inputs[0]) + relative_key_value_id = strip_passthrough(graph, relative_key_heads.inputs[0]) + output_value_id = node.outputs[0] if node.outputs else "" + + query_shape = graph.values.get(query_value_id).shape if query_value_id in graph.values else None + relative_key_shape = graph.values.get(relative_key_value_id).shape if relative_key_value_id in graph.values else None + output_shape = graph.values.get(output_value_id).shape if output_value_id in graph.values else None + if query_shape is None or relative_key_shape is None or output_shape is None: + return None + if len(query_shape) != 4 or len(relative_key_shape) != 4 or len(output_shape) != 4: + return None + + batch, seq_len, heads, head_dim = (int(v) for v in query_shape) + rel_batch, rel_len, rel_heads, rel_dim = (int(v) for v in relative_key_shape) + out_batch, out_heads, out_t, out_s = (int(v) for v in output_shape) + if rel_batch not in {1, batch}: + return None + if rel_heads != heads or rel_dim != head_dim: + return None + if rel_len < 2 * seq_len - 1: + return None + if (out_batch, out_heads, out_t, out_s) != (batch, heads, seq_len, seq_len): + return None + + return RelPosBiasMatch( + query_value_id=query_value_id, + relative_key_value_id=relative_key_value_id, + scale=float(node.attrs.get("value", 1.0)), + node_ids=tuple( + sorted( + { + node.id, + gather.id, + matmul.id, + query_heads.id, + rhs_transpose.id, + relative_key_heads.id, + } + ) + ), + ) diff --git a/python/src/transpile/fusion/rms_norm.py b/python/src/transpile/fusion/rms_norm.py new file mode 100644 index 000000000..b906aad96 --- /dev/null +++ b/python/src/transpile/fusion/rms_norm.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class RMSNormMatch: + input_value_id: str + weight_value_id: str | None + weight_offset: float + eps: float + node_ids: tuple[str, ...] + + +def match_rms_norm(graph: IRGraph, node: IRNode) -> RMSNormMatch | None: + anchor_value_id = node.inputs[0] if node.op == "type_as" and node.inputs else node.outputs[0] + anchor_value_id = strip_passthrough(graph, anchor_value_id) + anchor_node = producer(graph, anchor_value_id) + if anchor_node is None or anchor_node.op != "multiply" or len(anchor_node.inputs) != 2: + return None + + for normed_value_id, scale_value_id in ( + (anchor_node.inputs[0], anchor_node.inputs[1]), + (anchor_node.inputs[1], anchor_node.inputs[0]), + ): + scale = _extract_scale(graph, scale_value_id) + norm = _extract_normed_value(graph, normed_value_id) + if scale is None or norm is None: + continue + if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): + continue + return RMSNormMatch( + input_value_id=norm["input_value_id"], + weight_value_id=scale["weight_value_id"], + weight_offset=float(scale["offset"]), + eps=float(norm["eps"]), + node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"], *scale["node_ids"]})), + ) + + norm = _extract_normed_value(graph, anchor_node.outputs[0]) + if norm is None: + return None + if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): + return None + return RMSNormMatch( + input_value_id=norm["input_value_id"], + weight_value_id=None, + weight_offset=0.0, + eps=float(norm["eps"]), + node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"]})), + ) + + +def _extract_scale(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + offset = 0.0 + node_ids: set[str] = set() + producer_node = producer(graph, current) + if producer_node is not None and producer_node.op == "scalar_add": + add_value = float(producer_node.attrs.get("value", 0.0)) + if add_value in (0.0, 1.0): + offset = add_value + node_ids.add(producer_node.id) + current = producer_node.inputs[0] + current = strip_passthrough(graph, current) + if current not in graph.constants or not isinstance(graph.constants[current], torch.Tensor): + return None + return {"weight_value_id": current, "offset": offset, "node_ids": tuple(sorted(node_ids))} + + +def _extract_normed_value(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul = producer(graph, strip_passthrough(graph, value_id)) + if mul is None or mul.op != "multiply" or len(mul.inputs) != 2: + return None + for x_candidate, rsqrt_candidate in ((mul.inputs[0], mul.inputs[1]), (mul.inputs[1], mul.inputs[0])): + rsqrt = _extract_rsqrt_chain(graph, rsqrt_candidate) + if rsqrt is None: + continue + if strip_passthrough(graph, x_candidate) != strip_passthrough(graph, rsqrt["pow_input_value_id"]): + continue + return { + "input_value_id": strip_passthrough(graph, x_candidate), + "pow_input_value_id": rsqrt["pow_input_value_id"], + "eps": rsqrt["eps"], + "node_ids": tuple(sorted({mul.id, *rsqrt["node_ids"]})), + } + return None + + +def _extract_rsqrt_chain(graph: IRGraph, value_id: str) -> dict[str, object] | None: + pow_node = producer(graph, strip_passthrough(graph, value_id)) + if pow_node is None or pow_node.op != "pow" or float(pow_node.attrs.get("exponent", 0.0)) != -0.5: + return None + add_node = producer(graph, pow_node.inputs[0]) + if add_node is None or add_node.op != "scalar_add": + return None + mean_node = producer(graph, add_node.inputs[0]) + if mean_node is None or mean_node.op != "mean" or not bool(mean_node.attrs.get("keepdim", False)): + return None + pow2_node = producer(graph, mean_node.inputs[0]) + if pow2_node is None or pow2_node.op != "pow" or float(pow2_node.attrs.get("exponent", 0.0)) != 2.0: + return None + return { + "pow_input_value_id": pow2_node.inputs[0], + "eps": float(add_node.attrs.get("value", 0.0)), + "node_ids": (pow_node.id, add_node.id, mean_node.id, pow2_node.id), + } diff --git a/python/src/transpile/fusion/rope.py b/python/src/transpile/fusion/rope.py new file mode 100644 index 000000000..592ee2933 --- /dev/null +++ b/python/src/transpile/fusion/rope.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_layout_passthrough +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class RoPEMatch: + input_value_id: str + theta: float + position_offset: int + partial: bool + node_ids: tuple[str, ...] + + +def match_rope(graph: IRGraph, value_id: str) -> RoPEMatch | None: + desc = _extract_rope_descriptor(graph, value_id) + if desc is None: + return None + return RoPEMatch( + input_value_id=strip_passthrough(graph, desc["input_value_id"]), + theta=float(desc["theta"]), + position_offset=int(desc["position_offset"]), + partial=bool(desc.get("partial_rope", False)), + node_ids=tuple(sorted(set(desc.get("node_ids", ())))), + ) + + +def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = strip_layout_passthrough(graph, value_id) + node = producer(graph, current) + if node is None: + return None + if node.op == "rope": + return { + "input_value_id": node.inputs[0], + "theta": float(node.attrs.get("theta", 0.0)), + "position_offset": int(node.attrs.get("position_offset", 0)), + "partial_rope": False, + "node_ids": (node.id,), + } + classic = _extract_classic_rope_descriptor(graph, current) + if classic is not None: + classic["partial_rope"] = False + return classic + if node.op == "cat" and len(node.inputs) == 2: + for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) + if rope_branch is None: + continue + passthrough_node = producer(graph, strip_layout_passthrough(graph, passthrough_value_id)) + if passthrough_node is None or passthrough_node.op != "slice": + continue + if strip_passthrough(graph, passthrough_node.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): + continue + rope_branch["partial_rope"] = True + rope_branch["node_ids"] = tuple(sorted({node.id, passthrough_node.id, *rope_branch["node_ids"]})) + return rope_branch + return None + + +def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node = producer(graph, strip_layout_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return None + for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + direct_mult = producer(graph, direct_mult_id) + rotated_mult = producer(graph, rotated_mult_id) + if direct_mult is None or direct_mult.op != "multiply" or rotated_mult is None or rotated_mult.op != "multiply": + continue + direct_match = _extract_direct_rope_branch(graph, direct_mult) + rotated_match = _extract_rotated_rope_branch(graph, rotated_mult) + if direct_match is None or rotated_match is None: + continue + input_value_id = strip_passthrough(graph, direct_match["input_value_id"]) + if input_value_id != strip_passthrough(graph, rotated_match["input_value_id"]): + continue + cos_info = _extract_rope_trig(graph, direct_match["trig_value_id"], expected="scalar_cos") + sin_info = _extract_rope_trig(graph, rotated_match["trig_value_id"], expected="scalar_sin") + if cos_info is None or sin_info is None: + continue + if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: + continue + if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): + continue + return { + "input_value_id": input_value_id, + "theta": cos_info["theta"], + "position_offset": cos_info["position_offset"], + "node_ids": tuple(sorted({node.id, *direct_match["node_ids"], *rotated_match["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), + } + return None + + +def _extract_direct_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + if _unwrap_trig_value(graph, trig_value_id, expected="scalar_cos") is not None: + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotated_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + input_value_id = _extract_rotate_half_source(graph, rotated_value_id) + if input_value_id is None: + continue + if _unwrap_trig_value(graph, trig_value_id, expected="scalar_sin") is None: + continue + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + neg_node = producer(graph, cat_node.inputs[0]) + right_slice = producer(graph, cat_node.inputs[1]) + if neg_node is None or neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: + return None + left_slice = producer(graph, neg_node.inputs[0]) + if left_slice is None or right_slice is None or left_slice.op != "slice" or right_slice.op != "slice": + return None + if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): + return None + return left_slice.inputs[0] + + +def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: + trig_info = _unwrap_trig_value(graph, value_id, expected=expected) + if trig_info is None: + return None + trig_node = trig_info["trig_node"] + cat_node = producer(graph, trig_node.inputs[0]) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: + return None + angle_info = _extract_rope_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + matmul_node = angle_info["matmul_node"] + inv_freq_const_id = None + arange_node = None + for input_id in matmul_node.inputs: + if inv_freq_const_id is None: + inv_freq_const_id = _find_constant_ancestor(graph, input_id) + if arange_node is None: + arange_node = _find_arange_ancestor(graph, input_id) + if inv_freq_const_id is None or arange_node is None: + return None + theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) + if theta is None: + return None + return { + "theta": theta, + "position_offset": int(arange_node.attrs.get("start", 0)), + "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), + } + + +def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"unsqueeze", "precision_cast", "reshape", "view", "expand"}: + current = node.inputs[0] + continue + if node.op == "type_as": + current = node.inputs[0] + continue + if node.op == "scalar_multiply": + current = node.inputs[0] + continue + if node.op != expected: + return None + return {"trig_node": node} + + +def _extract_rope_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + while True: + current = strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "permute": + matmul_node = producer(graph, node.inputs[0]) + if matmul_node is None or matmul_node.op != "matmul": + return None + return {"matmul_node": matmul_node} + if node.op == "matmul": + return {"matmul_node": node} + if node.op in {"index", "slice"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + return None + + +def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_layout_passthrough(graph, current) + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return current + node = producer(graph, current) + if node is None or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "arange": + return node + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return None + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _infer_rope_theta(value: Any) -> float | None: + if not isinstance(value, torch.Tensor): + return None + flat = value.detach().cpu().float().reshape(-1) + if flat.numel() < 2: + return None + second = float(flat[1].item()) + if second <= 0.0: + return None + inv_count = flat.numel() + return float((1.0 / second) ** inv_count) diff --git a/python/src/transpile/gemma4_runtime.py b/python/src/transpile/gemma4_runtime.py new file mode 100644 index 000000000..e848f1e9d --- /dev/null +++ b/python/src/transpile/gemma4_runtime.py @@ -0,0 +1,526 @@ +from __future__ import annotations + +import builtins +import importlib.util +import sys +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +import numpy as np +import torch + +from src.transpile.audio_preprocess import prepare_native_gemma4_audio_features + + +_TORCHVISION_COMPAT_LIBRARIES: list[object] = [] + + +@dataclass +class PreparedInputs: + names: tuple[str, ...] + tensors: tuple[torch.Tensor, ...] + metadata: dict[str, object] + + +def _transformers_supports_model_module(module_name: str) -> bool: + try: + return importlib.util.find_spec(module_name) is not None + except Exception: + return False + + +def _candidate_external_site_packages() -> list[Path]: + candidates: list[Path] = [] + major_minor = f"python{sys.version_info.major}.{sys.version_info.minor}" + pyenv_versions = Path.home() / ".pyenv" / "versions" + if not pyenv_versions.exists(): + return candidates + for version_dir in sorted(pyenv_versions.iterdir(), reverse=True): + site_packages = version_dir / "lib" / major_minor / "site-packages" + if site_packages.exists(): + candidates.append(site_packages) + return candidates + + +def ensure_transformers_supports_gemma4() -> str | None: + target_module = "transformers.models.gemma4.modeling_gemma4" + if _transformers_supports_model_module(target_module): + return None + + for site_packages in _candidate_external_site_packages(): + candidate = site_packages / Path(*target_module.split(".")) + if not candidate.with_suffix(".py").exists() and not candidate.is_dir(): + continue + sys.path.insert(0, str(site_packages)) + for module_name in list(sys.modules): + root_name = module_name.split(".", 1)[0] + if root_name in {"transformers", "huggingface_hub", "tokenizers"}: + del sys.modules[module_name] + if _transformers_supports_model_module(target_module): + return str(site_packages) + try: + sys.path.remove(str(site_packages)) + except ValueError: + pass + return None + + +def _patch_torchvision_missing_nms_op() -> str | None: + """Keep mismatched torch/torchvision installs importable for processors.""" + + try: + import torchvision # type: ignore # noqa: F401 + return None + except RuntimeError as exc: + if "operator torchvision::nms does not exist" not in str(exc): + return None + except Exception: + return None + + try: + import torchvision.extension as tv_extension # type: ignore + if bool(getattr(tv_extension, "_HAS_OPS", False)): + return None + except Exception: + pass + + try: + library = torch.library.Library("torchvision", "DEF") + library.define("nms(Tensor dets, Tensor scores, float iou_threshold) -> Tensor") + _TORCHVISION_COMPAT_LIBRARIES.append(library) + return "defined missing torchvision::nms operator for torchvision import compatibility" + except Exception: + return None + + +def patch_transformers_torchvision_probe() -> str | None: + has_torchvision = importlib.util.find_spec("torchvision") is not None + has_lzma = importlib.util.find_spec("_lzma") is not None + + if not has_torchvision: + return None + + if has_lzma: + nms_patch_note = _patch_torchvision_missing_nms_op() + return nms_patch_note + + base_note: str | None = None + try: + import backports.lzma as backports_lzma # type: ignore + + sys.modules.setdefault("lzma", backports_lzma) + base_note = "using backports.lzma because this Python build is missing _lzma" + nms_patch_note = _patch_torchvision_missing_nms_op() + if nms_patch_note: + return f"{base_note}; {nms_patch_note}" + return base_note + except Exception: + pass + + class _InterpolationModeStub: + NEAREST = "nearest" + BILINEAR = "bilinear" + BICUBIC = "bicubic" + LANCZOS = "lanczos" + + class _TorchvisionFunctionalStub: + InterpolationMode = _InterpolationModeStub + + def __getattr__(self, name: str): + raise RuntimeError( + "torchvision functionality is unavailable because this Python build is missing _lzma; " + f"attempted to access torchvision.transforms.functional.{name}" + ) + + builtins.F = _TorchvisionFunctionalStub() + builtins.tvF = builtins.F + + import transformers.utils as tf_utils # type: ignore + import transformers.utils.import_utils as tf_import_utils # type: ignore + + @lru_cache + def _disabled() -> bool: + return False + + tf_import_utils.is_torchvision_available = _disabled + tf_import_utils.is_torchvision_v2_available = _disabled + tf_utils.is_torchvision_available = _disabled + tf_utils.is_torchvision_v2_available = _disabled + return "disabled torchvision import checks because this Python build is missing _lzma" + + +def patch_torch_flex_attention_compat() -> str | None: + try: + import torch.nn.attention.flex_attention as flex_attention # type: ignore + except Exception: + return None + + if hasattr(flex_attention, "AuxRequest"): + return None + + class _AuxRequest: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + flex_attention.AuxRequest = _AuxRequest # type: ignore[attr-defined] + return "installed torch flex_attention AuxRequest compatibility stub" + + +def _resolve_audio_sample_rate(processor: object) -> int: + for attr_name in ("feature_extractor", "tokenizer"): + child = getattr(processor, attr_name, None) + sample_rate = getattr(child, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + sample_rate = getattr(processor, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + return 16000 + + +_GEMMA4_MULTIMODAL_INPUT_ORDER = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", +) + + +def _normalize_multimodal_prompt( + prompt: str, + *, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, +) -> str: + normalized = prompt.strip() + prefixes: list[str] = [] + + if image_token and num_images > 0: + image_count = normalized.count(image_token) + if image_count < num_images: + prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) + if audio_token and num_audio_segments > 0: + audio_count = normalized.count(audio_token) + if audio_count < num_audio_segments: + prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) + + if prefixes: + prefix = "\n".join(part for part in prefixes if part) + if normalized: + return f"{prefix}\n{normalized}" + return prefix + return normalized + + +def _build_gemma4_chat_prompt( + *, + prompt: str, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> str: + result = "" + normalized_system = system_prompt.strip() + if enable_thinking_if_supported or normalized_system: + result += "<|turn>system\n" + if enable_thinking_if_supported: + result += "<|think|>" + result += normalized_system + result += "\n" + + result += "<|turn>user\n" + if image_token and num_images > 0: + for _ in range(num_images): + result += f"\n\n{image_token}\n\n" + result += prompt + if audio_token and num_audio_segments > 0: + result += "".join(audio_token for _ in range(num_audio_segments)) + result += "\n" + result += "<|turn>model\n" + return result + + +def _gemma4_split_cactus_newline_token_merges(batch: object) -> None: + input_ids = batch.get("input_ids") if hasattr(batch, "get") else None + if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: + return + + expansions = { + 108: (107, 107), + 109: (107, 107, 107), + } + if not any(int(token) in expansions for token in input_ids.reshape(-1).tolist()): + return + + lengths: list[list[int]] = [] + max_len = 0 + for row in input_ids.detach().cpu().tolist(): + row_lengths = [len(expansions.get(int(token), (int(token),))) for token in row] + lengths.append(row_lengths) + max_len = max(max_len, sum(row_lengths)) + + for key in ("input_ids", "attention_mask", "token_type_ids"): + value = batch.get(key) if hasattr(batch, "get") else None + if not isinstance(value, torch.Tensor) or value.ndim != 2: + continue + expanded = torch.full( + (value.shape[0], max_len), + 0, + dtype=value.dtype, + device=value.device, + ) + for row_idx, row in enumerate(value.detach().cpu().tolist()): + out: list[int] = [] + for token_idx, item in enumerate(row): + if key == "input_ids": + out.extend(expansions.get(int(item), (int(item),))) + else: + out.extend([int(item)] * lengths[row_idx][token_idx]) + expanded[row_idx, : len(out)] = torch.tensor(out, dtype=value.dtype, device=value.device) + batch[key] = expanded + + +def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: + if not image_files: + return [] + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for --image-file: {exc}") from exc + + images: list[object] = [] + for image_file in image_files: + path = Path(image_file).resolve() + if not path.exists(): + raise RuntimeError(f"image_file does not exist: {path}") + with Image.open(path) as image: + images.append(image.convert("RGB").copy()) + return images + + +def _get_processor_image_attr(processor: object, name: str, default: object) -> object: + image_processor = getattr(processor, "image_processor", None) + if image_processor is not None and hasattr(image_processor, name): + return getattr(image_processor, name) + if isinstance(image_processor, dict) and name in image_processor: + return image_processor[name] + return default + + +def _prepare_gemma4_native_image_tensors( + processor: object, + image_files: tuple[str, ...], +) -> tuple[torch.Tensor, torch.Tensor] | None: + if not image_files: + return None + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for Gemma4 native image preprocessing: {exc}") from exc + + patch_size = int(_get_processor_image_attr(processor, "patch_size", 16)) + pooling_kernel_size = int(_get_processor_image_attr(processor, "pooling_kernel_size", 3)) + max_soft_tokens = int(_get_processor_image_attr(processor, "max_soft_tokens", 280)) + rescale_factor = float(_get_processor_image_attr(processor, "rescale_factor", 1.0 / 255.0)) + max_patches = max_soft_tokens * pooling_kernel_size * pooling_kernel_size + side_multiple = pooling_kernel_size * patch_size + patch_dim = 3 * patch_size * patch_size + if patch_size <= 0 or pooling_kernel_size <= 0 or max_patches <= 0: + return None + + try: + resample_bilinear = Image.Resampling.BILINEAR + except AttributeError: # pragma: no cover + resample_bilinear = Image.BILINEAR + + pixel_batches: list[np.ndarray] = [] + position_batches: list[np.ndarray] = [] + for image_file in image_files: + path = Path(image_file).resolve() + with Image.open(path) as image: + rgb = image.convert("RGB") + width, height = rgb.size + target_pixels = float(max_patches * patch_size * patch_size) + factor = float(np.sqrt(target_pixels / max(1.0, float(width * height)))) + target_h = int(np.floor(factor * height / side_multiple)) * side_multiple + target_w = int(np.floor(factor * width / side_multiple)) * side_multiple + if target_h == 0: + target_h = side_multiple + if target_w == 0: + target_w = side_multiple + if (target_w, target_h) != rgb.size: + rgb = rgb.resize((target_w, target_h), resample=resample_bilinear) + array = np.asarray(rgb, dtype=np.float32) * rescale_factor + + patch_h = target_h // patch_size + patch_w = target_w // patch_size + num_patches = patch_h * patch_w + if num_patches > max_patches: + raise RuntimeError( + f"Gemma4 native image preprocessing produced {num_patches} patches, " + f"but max_patches={max_patches}" + ) + chw = np.transpose(array, (2, 0, 1)) + patches = ( + chw.reshape(3, patch_h, patch_size, patch_w, patch_size) + .transpose(1, 3, 2, 4, 0) + .reshape(num_patches, patch_dim) + ) + + padded_patches = np.zeros((max_patches, patch_dim), dtype=np.float32) + padded_patches[:num_patches] = patches + positions = np.full((max_patches, 2), -1, dtype=np.int64) + valid_positions = np.zeros((num_patches, 2), dtype=np.int64) + for patch_y in range(patch_h): + row_start = patch_y * patch_w + valid_positions[row_start : row_start + patch_w, 0] = np.arange(patch_w, dtype=np.int64) + valid_positions[row_start : row_start + patch_w, 1] = patch_y + positions[:num_patches] = valid_positions + + pixel_batches.append(padded_patches) + position_batches.append(positions) + + return ( + torch.from_numpy(np.stack(pixel_batches, axis=0)), + torch.from_numpy(np.stack(position_batches, axis=0)), + ) + + +def prepare_gemma4_multimodal_inputs( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, + use_gemma4_chat_template: bool = False, +) -> PreparedInputs: + if processor is None: + raise RuntimeError("multimodal Gemma4 transpile requires an AutoProcessor with image and audio support") + + images = _load_image_inputs(image_files) + audio_waveforms: list[np.ndarray] = [] + sample_rate: int | None = None + if audio_file: + sample_rate = _resolve_audio_sample_rate(processor) + from src.transpile.audio_preprocess import load_audio_waveform + + audio_waveforms.append(load_audio_waveform(audio_file, target_sample_rate=sample_rate)) + + image_token = getattr(processor, "image_token", None) + audio_token = getattr(processor, "audio_token", None) + processor_prompt = _normalize_multimodal_prompt( + prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + ) + normalized_prompt = processor_prompt + if use_gemma4_chat_template: + normalized_prompt = prompt.strip() + processor_prompt = _build_gemma4_chat_prompt( + prompt=normalized_prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + + batch = processor( + text=processor_prompt, + images=images or None, + audio=audio_waveforms or None, + return_tensors="pt", + ) + if use_gemma4_chat_template: + _gemma4_split_cactus_newline_token_merges(batch) + native_image_tensors = _prepare_gemma4_native_image_tensors(processor, image_files) + if native_image_tensors is not None: + batch["pixel_values"], batch["pixel_position_ids"] = native_image_tensors + if audio_file and isinstance(batch.get("input_features"), torch.Tensor): + feature_tensor = batch["input_features"] + expected_mels = int(feature_tensor.shape[-1]) + try: + native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( + audio_file, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + except Exception as exc: + print(f"note=falling back to processor gemma4 audio features: {exc}") + batch["input_features"] = feature_tensor.to(dtype=torch_dtype) + fallback_mask = batch.get("input_features_mask") + if isinstance(fallback_mask, torch.Tensor): + batch["input_features_mask"] = fallback_mask.to(dtype=torch.bool) + native_audio_frames = int(fallback_mask.to(dtype=torch.int32).sum().item()) + else: + native_audio_frames = int(feature_tensor.shape[1]) + batch["input_features_mask"] = torch.ones( + (int(feature_tensor.shape[0]), native_audio_frames), + dtype=torch.bool, + ) + batch["native_audio_frames"] = native_audio_frames + else: + batch["input_features"] = native_audio + batch["input_features_mask"] = native_audio_mask + batch["native_audio_frames"] = native_audio_frames + + ordered_keys = [ + key + for key in _GEMMA4_MULTIMODAL_INPUT_ORDER + if isinstance(batch.get(key), torch.Tensor) + ] + if not ordered_keys: + raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + metadata: dict[str, object] = { + "prompt": normalized_prompt, + "processor_prompt": processor_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + } + if audio_file: + metadata["audio_file"] = str(Path(audio_file).resolve()) + if sample_rate is not None: + metadata["sample_rate"] = sample_rate + native_audio_frames = batch.get("native_audio_frames") + if isinstance(native_audio_frames, int): + metadata["native_audio_frames"] = native_audio_frames + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata=metadata, + ) + + +_patch_transformers_torchvision_probe = patch_transformers_torchvision_probe +_patch_torch_flex_attention_compat = patch_torch_flex_attention_compat +_prepare_gemma4_multimodal_inputs = prepare_gemma4_multimodal_inputs diff --git a/python/src/transpile/capture/graph_ir.py b/python/src/transpile/graph_ir.py similarity index 65% rename from python/src/transpile/capture/graph_ir.py rename to python/src/transpile/graph_ir.py index 56d9dcd18..632585c57 100644 --- a/python/src/transpile/capture/graph_ir.py +++ b/python/src/transpile/graph_ir.py @@ -1,6 +1,32 @@ +import copy from dataclasses import dataclass from dataclasses import field +import torch + + +def _clone_ir_payload(value, memo): + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.Tensor): + # IR constants may be real tensors or export-time tensor wrappers. + # They are treated as immutable graph payloads, so preserve them by + # reference instead of triggering Tensor.__deepcopy__. + return value + if isinstance(value, tuple): + return tuple(_clone_ir_payload(item, memo) for item in value) + if isinstance(value, list): + return [_clone_ir_payload(item, memo) for item in value] + if isinstance(value, dict): + return { + copy.deepcopy(key, memo): _clone_ir_payload(inner, memo) + for key, inner in value.items() + } + try: + return copy.deepcopy(value, memo) + except Exception: + return value + @dataclass class IRValue: @@ -11,6 +37,21 @@ class IRValue: users: list[str] = field(default_factory=list) meta: dict[str, object] = field(default_factory=dict) + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRValue( + id=self.id, + shape=_clone_ir_payload(self.shape, memo), + dtype=self.dtype, + producer=self.producer, + users=list(self.users), + meta=_clone_ir_payload(self.meta, memo), + ) + memo[id(self)] = cloned + return cloned + @dataclass class IRNode: id: str @@ -21,6 +62,22 @@ class IRNode: meta: dict[str, object] = field(default_factory=dict) kind: str = "generic" + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRNode( + id=self.id, + op=self.op, + inputs=list(self.inputs), + outputs=list(self.outputs), + attrs=_clone_ir_payload(self.attrs, memo), + meta=_clone_ir_payload(self.meta, memo), + kind=self.kind, + ) + memo[id(self)] = cloned + return cloned + @dataclass class IRGraph: values: dict[str, IRValue] @@ -31,6 +88,25 @@ class IRGraph: constants: dict[str, object] = field(default_factory=dict) meta: dict[str, object] = field(default_factory=dict) + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRGraph( + values={value_id: copy.deepcopy(value, memo) for value_id, value in self.values.items()}, + nodes={node_id: copy.deepcopy(node, memo) for node_id, node in self.nodes.items()}, + order=list(self.order), + inputs=list(self.inputs), + outputs=list(self.outputs), + constants={ + value_id: _clone_ir_payload(constant, memo) + for value_id, constant in self.constants.items() + }, + meta=_clone_ir_payload(self.meta, memo), + ) + memo[id(self)] = cloned + return cloned + def add_node(self, node: IRNode) -> None: if node.id in self.nodes: raise ValueError(f"duplicate IR node id: {node.id}") diff --git a/python/src/transpile/capture/import_ir.py b/python/src/transpile/import_ir.py similarity index 90% rename from python/src/transpile/capture/import_ir.py rename to python/src/transpile/import_ir.py index d1d02ea9e..291debdef 100644 --- a/python/src/transpile/capture/import_ir.py +++ b/python/src/transpile/import_ir.py @@ -7,23 +7,24 @@ from torch.export.graph_signature import ConstantArgument from torch.export.graph_signature import InputKind -from src.transpile.capture.capture_pytorch import format_target -from src.transpile.capture.capture_pytorch import get_dtype -from src.transpile.capture.capture_pytorch import get_shape -from src.transpile.capture.capture_pytorch import resolve_attr -from src.transpile.capture.graph_ir import IRGraph -from src.transpile.capture.graph_ir import verify_ir -from src.transpile.capture.importers import extract_literals -from src.transpile.capture.importers import ImportContext -from src.transpile.capture.importers import import_call_function -from src.transpile.capture.importers import import_get_attr -from src.transpile.capture.importers import import_output -from src.transpile.capture.importers import import_placeholder -from src.transpile.capture.importers import value_id -from src.transpile.capture.normalize import dtype_to_ir -from src.transpile.capture.import_semantics import apply_import_semantics -from src.transpile.capture.weight_binding import resolve_transpile_weights_dir -from src.transpile.capture.weight_binding import resolve_weight_binding +from src.transpile.capture_pytorch import format_target +from src.transpile.capture_pytorch import get_dtype +from src.transpile.capture_pytorch import get_shape +from src.transpile.capture_pytorch import resolve_attr +from src.transpile.component_partition import annotate_ir_components +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import verify_ir +from src.transpile.importers import extract_literals +from src.transpile.importers import ImportContext +from src.transpile.importers import import_call_function +from src.transpile.importers import import_get_attr +from src.transpile.importers import import_output +from src.transpile.importers import import_placeholder +from src.transpile.importers import value_id +from src.transpile.normalize import dtype_to_ir +from src.transpile.import_semantics import apply_import_semantics +from src.transpile.weight_binding import resolve_transpile_weights_dir +from src.transpile.weight_binding import resolve_weight_binding def import_captured_to_ir(captured: Any, *, strict: bool = True) -> IRGraph: @@ -253,6 +254,9 @@ def _inline_wrap_with_set_grad_enabled(node: Any, shape: Any, dtype: Any) -> Non import_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) continue + if node.op == "call_module" and str(getattr(node, "target", "")) == "_guards_fn" and not node.users: + continue + if node.op == "output": import_output(ir, node, ctx) continue @@ -262,5 +266,6 @@ def _inline_wrap_with_set_grad_enabled(node: Any, shape: Any, dtype: Any) -> Non ctx.pop_node() apply_import_semantics(ir) + annotate_ir_components(ir) verify_ir(ir) return ir diff --git a/python/src/transpile/import_semantics.py b/python/src/transpile/import_semantics.py new file mode 100644 index 000000000..9b9e86a32 --- /dev/null +++ b/python/src/transpile/import_semantics.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.canonicalize.utils import rebuild_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import verify_ir + + +@dataclass(frozen=True) +class EarlyRoPEMatch: + anchor_node_id: str + input_value_id: str + theta: float + position_offset: int + partial: bool + node_ids: tuple[str, ...] + + +def apply_import_semantics(graph: IRGraph) -> IRGraph: + _rewrite_explicit_attention(graph) + _tag_explicit_linear(graph) + _rewrite_early_rope(graph) + rebuild_graph(graph) + verify_ir(graph) + return graph + + +def _rewrite_explicit_attention(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "scaled_dot_product_attention": + continue + additive_mask = bool(node.attrs.get("additive_mask", False)) + if not additive_mask and len(node.inputs) > 3: + mask_value = graph.values.get(node.inputs[3]) + if mask_value is not None and mask_value.dtype not in (None, "bool"): + additive_mask = True + node.op = "attention" + node.kind = "semantic" + node.meta["semantic_source"] = "import" + node.attrs = { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", False)), + "window_size": int(node.attrs.get("window_size", 0)), + **({"enable_gqa": bool(node.attrs.get("enable_gqa", False))} if "enable_gqa" in node.attrs else {}), + **({"additive_mask": True} if additive_mask else {}), + } + + +def _tag_explicit_linear(graph: IRGraph) -> None: + for node in graph.nodes.values(): + if node.op not in {"linear", "addmm"}: + continue + node.meta.setdefault("semantic_source", "import") + + +def _rewrite_early_rope(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or not node.outputs: + continue + match = match_early_rope(graph, node.outputs[0]) + if match is None or match.anchor_node_id != node.id: + continue + if match.partial: + node.meta["semantic_hint"] = "partial_rope" + node.meta["rope_theta_hint"] = float(match.theta) + node.meta["rope_position_offset_hint"] = int(match.position_offset) + continue + node.op = "rope" + node.inputs = [match.input_value_id] + node.attrs = { + "theta": float(match.theta), + "position_offset": int(match.position_offset), + } + node.kind = "semantic" + node.meta["semantic_source"] = "import" + + +def match_early_rope(graph: IRGraph, value_id: str) -> EarlyRoPEMatch | None: + desc = _extract_rope_descriptor(graph, value_id) + if desc is None: + return None + return EarlyRoPEMatch( + anchor_node_id=str(desc["anchor_node_id"]), + input_value_id=str(desc["input_value_id"]), + theta=float(desc["theta"]), + position_offset=int(desc["position_offset"]), + partial=bool(desc.get("partial_rope", False)), + node_ids=tuple(sorted(set(desc.get("node_ids", ())))), + ) + + +def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = _strip_layout_passthrough(graph, value_id) + node = producer(graph, current) + if node is None: + return None + if node.op == "rope": + return { + "anchor_node_id": node.id, + "input_value_id": node.inputs[0], + "theta": float(node.attrs.get("theta", 0.0)), + "position_offset": int(node.attrs.get("position_offset", 0)), + "partial_rope": False, + "node_ids": (node.id,), + } + classic = _extract_classic_rope_descriptor(graph, current) + if classic is not None: + classic["partial_rope"] = False + return classic + if node.op == "cat" and len(node.inputs) == 2: + for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) + if rope_branch is None: + continue + passthrough = producer(graph, _strip_layout_passthrough(graph, passthrough_value_id)) + if passthrough is None or passthrough.op != "slice": + continue + if strip_passthrough(graph, passthrough.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): + continue + rope_branch["anchor_node_id"] = node.id + rope_branch["partial_rope"] = True + rope_branch["node_ids"] = tuple(sorted({node.id, passthrough.id, *rope_branch["node_ids"]})) + return rope_branch + return None + + +def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node = producer(graph, _strip_layout_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return None + for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + direct_mult = producer(graph, direct_mult_id) + rotated_mult = producer(graph, rotated_mult_id) + if direct_mult is None or rotated_mult is None: + continue + if direct_mult.op != "multiply" or rotated_mult.op != "multiply": + continue + direct = _extract_direct_branch(graph, direct_mult) + rotated = _extract_rotated_branch(graph, rotated_mult) + if direct is None or rotated is None: + continue + input_value_id = strip_passthrough(graph, direct["input_value_id"]) + if input_value_id != strip_passthrough(graph, rotated["input_value_id"]): + continue + cos_info = _extract_rope_trig(graph, direct["trig_value_id"], expected=("cos", "scalar_cos")) + sin_info = _extract_rope_trig(graph, rotated["trig_value_id"], expected=("sin", "scalar_sin")) + if cos_info is None or sin_info is None: + continue + if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: + continue + if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): + continue + return { + "anchor_node_id": node.id, + "input_value_id": input_value_id, + "theta": float(cos_info["theta"]), + "position_offset": int(cos_info["position_offset"]), + "node_ids": tuple(sorted({node.id, *direct["node_ids"], *rotated["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), + } + return None + + +def _extract_direct_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + if _unwrap_trig_value(graph, trig_value_id, expected=("cos", "scalar_cos")) is not None: + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotated_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + input_value_id = _extract_rotate_half_source(graph, rotated_value_id) + if input_value_id is None: + continue + if _unwrap_trig_value(graph, trig_value_id, expected=("sin", "scalar_sin")) is None: + continue + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + neg_node = producer(graph, cat_node.inputs[0]) + right_slice = producer(graph, cat_node.inputs[1]) + if neg_node is None or right_slice is None: + return None + if neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: + return None + left_slice = producer(graph, neg_node.inputs[0]) + if left_slice is None or left_slice.op != "slice" or right_slice.op != "slice": + return None + if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): + return None + return left_slice.inputs[0] + + +def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: + trig = _unwrap_trig_value(graph, value_id, expected=expected) + if trig is None: + return None + trig_node = trig["trig_node"] + cat_node = producer(graph, trig_node.inputs[0]) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: + return None + angle_info = _extract_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + matmul_node = angle_info["matmul_node"] + inv_freq_const_id = None + arange_node = None + for input_id in matmul_node.inputs: + if inv_freq_const_id is None: + inv_freq_const_id = _find_constant_ancestor(graph, input_id) + if arange_node is None: + arange_node = _find_arange_ancestor(graph, input_id) + if inv_freq_const_id is None or arange_node is None: + return None + theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) + if theta is None: + return None + return { + "theta": theta, + "position_offset": int(arange_node.attrs.get("start", 0)), + "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), + } + + +def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"unsqueeze", "reshape", "view", "expand", "precision_cast"}: + current = node.inputs[0] + continue + if node.op == "type_as": + current = node.inputs[0] + continue + if node.op == "scalar_multiply": + current = node.inputs[0] + continue + if node.op not in expected: + return None + return {"trig_node": node} + + +def _extract_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + while True: + current = _strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op in {"transpose", "permute"}: + matmul_node = producer(graph, node.inputs[0]) + if matmul_node is None or matmul_node.op != "matmul": + return None + return {"matmul_node": matmul_node} + if node.op == "matmul": + return {"matmul_node": node} + if node.op in {"index", "slice"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + return None + + +def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = _strip_layout_passthrough(graph, current) + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return current + node = producer(graph, current) + if node is None or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = _strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "arange": + return node + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return None + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def _infer_rope_theta(value: Any) -> float | None: + if not isinstance(value, torch.Tensor): + return None + flat = value.detach().cpu().float().reshape(-1) + if flat.numel() < 2: + return None + second = float(flat[1].item()) + if second <= 0.0: + return None + return float((1.0 / second) ** flat.numel()) diff --git a/python/src/transpile/capture/importers.py b/python/src/transpile/importers.py similarity index 64% rename from python/src/transpile/capture/importers.py rename to python/src/transpile/importers.py index 4b3eccd9d..79486201c 100644 --- a/python/src/transpile/capture/importers.py +++ b/python/src/transpile/importers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import math from dataclasses import dataclass from dataclasses import field from types import SimpleNamespace @@ -7,11 +8,11 @@ import torch -from src.transpile.capture.graph_ir import IRGraph -from src.transpile.capture.graph_ir import IRNode -from src.transpile.capture.graph_ir import IRValue -from src.transpile.capture.normalize import dtype_to_ir -from src.transpile.capture.normalize import normalize_target +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.normalize import dtype_to_ir +from src.transpile.normalize import normalize_target class UnsupportedImportError(NotImplementedError): @@ -159,7 +160,13 @@ def extract_input_ids(arg: Any, ctx: ImportContext) -> list[str]: def extract_literals(value: Any) -> Any: if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return extract_literals(meta_value) return None + if isinstance(value, torch.Size): + return tuple(extract_literals(v) for v in value) if isinstance(value, (tuple, list)): return type(value)(extract_literals(v) for v in value) if isinstance(value, dict): @@ -213,13 +220,115 @@ def _extract_numeric_literal(value: Any) -> float | int | None: return None +def _extract_static_int(value: Any) -> int | None: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return _extract_static_int(meta_value) + return None + + literal = extract_literals(value) + if isinstance(literal, bool): + return int(literal) + if isinstance(literal, int): + return int(literal) + if isinstance(literal, float) and float(literal).is_integer(): + return int(literal) + + try: + return int(value) + except Exception: + return None + + +def _extract_int_sequence_literal(value: Any) -> tuple[int, ...] | None: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return _extract_int_sequence_literal(meta_value) + return None + + if isinstance(value, torch.Size): + value = tuple(value) + + if isinstance(value, (tuple, list)): + result: list[int] = [] + for item in value: + item_value = _extract_static_int(item) + if item_value is None: + return None + result.append(item_value) + return tuple(result) + + scalar_value = _extract_static_int(value) + if scalar_value is not None: + return (scalar_value,) + return None + + +def _try_materialize_shape(shape: tuple[Any, ...] | None) -> tuple[int, ...] | None: + if shape is None: + return None + result: list[int] = [] + for dim in shape: + dim_value = _extract_static_int(dim) + if dim_value is None: + return None + result.append(dim_value) + return tuple(result) + + +def _extract_fx_tensor_shape(value: Any) -> tuple[Any, ...] | None: + if not is_fx_node(value): + return None + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if isinstance(meta_value, torch.Tensor): + return tuple(meta_value.shape) + tensor_meta = meta.get("tensor_meta") + tensor_shape = getattr(tensor_meta, "shape", None) + if tensor_shape is not None: + return tuple(tensor_shape) + return None + + def _base_meta(shape: tuple[int, ...] | None, dtype: str | None, torch_op: str, node: Any) -> dict[str, object]: - return { + meta = { "shape": shape, "dtype": dtype, "torch_op": torch_op, "torch_name": node.name, } + raw_meta = getattr(node, "meta", {}) or {} + raw_stack = raw_meta.get("nn_module_stack") + if isinstance(raw_stack, dict): + module_paths: list[str] = [] + for entry in raw_stack.values(): + if not isinstance(entry, (tuple, list)) or not entry: + continue + module_path = entry[0] + if isinstance(module_path, str) and module_path and module_path not in module_paths: + module_paths.append(module_path) + if module_paths: + meta["module_paths"] = tuple(module_paths) + return meta + + +def _extract_fx_torch_dtype(node: Any) -> torch.dtype | None: + if not is_fx_node(node): + return None + meta = getattr(node, "meta", {}) or {} + value = meta.get("val") + dtype = getattr(value, "dtype", None) + if isinstance(dtype, torch.dtype): + return dtype + tensor_meta = meta.get("tensor_meta") + dtype = getattr(tensor_meta, "dtype", None) + if isinstance(dtype, torch.dtype): + return dtype + return None def _import_scalar_binary( @@ -418,7 +527,7 @@ def import_multiply(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[ def import_multiply_inplace(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply_inplace") + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") def import_divide(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: @@ -463,6 +572,113 @@ def import_not_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op="equal", + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if lhs_is_node and rhs_literal is not None: + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op="scalar_equal", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_binary(op_name: str): + def _importer( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, + ) -> None: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + return _importer + + +def import_where( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + condition = node.args[0] + true_value = node.args[1] + false_value = node.args[2] + + inputs = [value_id(condition, ctx)] + attrs: dict[str, object] = {} + + if is_fx_node(true_value): + inputs.append(value_id(true_value, ctx)) + attrs["true_is_scalar"] = False + else: + literal = _extract_numeric_literal(true_value) + if literal is None: + ctx.fail(f"unsupported true branch for {torch_op}: {node.args!r}") + attrs["true_is_scalar"] = True + attrs["true_value"] = float(literal) + + if is_fx_node(false_value): + inputs.append(value_id(false_value, ctx)) + attrs["false_is_scalar"] = False + else: + literal = _extract_numeric_literal(false_value) + if literal is None: + ctx.fail(f"unsupported false branch for {torch_op}: {node.args!r}") + attrs["false_is_scalar"] = True + attrs["false_value"] = float(literal) + + ir_node = IRNode( + id=node_id(node), + op="where", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_negate(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: ir_node = IRNode( id=node_id(node), @@ -544,7 +760,9 @@ def import_identity(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[ def import_reshape(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_shape = tuple(int(v) for v in node.args[1]) + target_shape = _extract_int_sequence_literal(node.args[1]) + if target_shape is None: + ctx.fail(f"unsupported reshape shape for {torch_op}: {node.args!r}") ir_node = IRNode( id=node_id(node), op="reshape", @@ -571,8 +789,9 @@ def import_flatten(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i def import_unsqueeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: dim = int(node.args[1]) attrs: dict[str, object] = {"dim": dim} - if shape is not None: - attrs["shape"] = tuple(int(v) for v in shape) + materialized_shape = _try_materialize_shape(shape) + if materialized_shape is not None: + attrs["shape"] = materialized_shape ir_node = IRNode( id=node_id(node), op="unsqueeze", @@ -584,8 +803,71 @@ def import_unsqueeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_squeeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + dim = int(node.args[1]) if len(node.args) > 1 and node.args[1] is not None else 0 + attrs: dict[str, object] = {"dim": dim} + materialized_shape = _try_materialize_shape(shape) + if materialized_shape is not None: + attrs["shape"] = materialized_shape + ir_node = IRNode( + id=node_id(node), + op="squeeze", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_masked_scatter( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + ir_node = IRNode( + id=node_id(node), + op="masked_scatter", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_masked_fill( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + value = _extract_numeric_literal(node.args[2]) + if value is None: + ctx.fail(f"unsupported masked_fill value for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="masked_fill", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": float(value)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_expand(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_shape = tuple(int(v) for v in node.args[1]) + target_shape = _extract_int_sequence_literal(node.args[1]) + if target_shape is None and shape is not None: + target_shape = _extract_int_sequence_literal(shape) + if target_shape is None: + ctx.fail(f"unsupported expand shape for {torch_op}: {node.args!r}") ir_node = IRNode( id=node_id(node), op="expand", @@ -597,6 +879,55 @@ def import_expand(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[in register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_one_hot(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + num_classes = _extract_static_int(node.args[1]) if len(node.args) > 1 else None + if (num_classes is None or num_classes <= 0) and shape is not None and shape: + num_classes = int(shape[-1]) + if num_classes is None or num_classes <= 0: + ctx.fail(f"unsupported one_hot num_classes for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="one_hot", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"num_classes": int(num_classes)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_tril(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + diagonal = int(extract_literals(node.args[1])) if len(node.args) > 1 and extract_literals(node.args[1]) is not None else 0 + ir_node = IRNode( + id=node_id(node), + op="tril", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"diagonal": diagonal}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_unfold(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 4: + ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") + dimension = extract_literals(node.args[1]) + size = extract_literals(node.args[2]) + step = extract_literals(node.args[3]) + if not isinstance(dimension, int) or not isinstance(size, int) or not isinstance(step, int): + ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="unfold", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"dimension": dimension, "size": size, "step": step}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_transpose(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: attrs = {"dim0": 0, "dim1": 1} if torch_op == "aten.t.default" else {"dim0": int(node.args[1]), "dim1": int(node.args[2])} ir_node = IRNode( @@ -611,7 +942,9 @@ def import_transpose(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple def import_permute(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - permutation = tuple(int(v) for v in node.args[1]) + permutation = _extract_int_sequence_literal(node.args[1]) + if permutation is None: + ctx.fail(f"unsupported permute dims for {torch_op}: {node.args!r}") ir_node = IRNode( id=node_id(node), op="permute", @@ -623,6 +956,39 @@ def import_permute(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_numpy_t(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + input_shape = _extract_fx_tensor_shape(node.args[0]) + rank_source = input_shape if input_shape is not None else shape + if rank_source is None: + ctx.fail(f"unsupported numpy_T rank inference for {torch_op}: {node.args!r}") + permutation = tuple(range(len(rank_source) - 1, -1, -1)) + ir_node = IRNode( + id=node_id(node), + op="permute", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"permutation": permutation}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_movedim(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + source = _extract_int_sequence_literal(node.args[1]) + destination = _extract_int_sequence_literal(node.args[2]) + if source is None or destination is None: + ctx.fail(f"unsupported movedim dims for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="movedim", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"source": source, "destination": destination}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_matmul(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: ir_node = IRNode( id=node_id(node), @@ -675,6 +1041,25 @@ def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, . return _importer +def import_clamp(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + min_value = extract_literals(node.args[1]) if len(node.args) > 1 else None + max_value = extract_literals(node.args[2]) if len(node.args) > 2 else None + attrs: dict[str, object] = {} + if isinstance(min_value, (int, float)): + attrs["min"] = float(min_value) + if isinstance(max_value, (int, float)): + attrs["max"] = float(max_value) + ir_node = IRNode( + id=node_id(node), + op="clamp", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_pow_rsqrt(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: ir_node = IRNode( id=node_id(node), @@ -687,6 +1072,43 @@ def import_pow_rsqrt(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_reciprocal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="pow", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"exponent": -1.0}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_floor_divide( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported floor_divide signature for {torch_op}: {node.args!r}") + divisor = _extract_numeric_literal(node.args[1]) + if divisor is None: + ctx.fail(f"unsupported floor_divide divisor for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="scalar_floor_divide", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": float(divisor)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_softmax(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: ir_node = IRNode( id=node_id(node), @@ -698,6 +1120,22 @@ def import_softmax(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i ) register_node(ir, ir_node, shape=shape, dtype=dtype) + +def import_glu(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + axis = -1 + if len(node.args) > 1 and extract_literals(node.args[1]) is not None: + axis = int(node.args[1]) + ir_node = IRNode( + id=node_id(node), + op="glu", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_scaled_dot_product_attention( ir: IRGraph, node: Any, @@ -715,6 +1153,9 @@ def import_scaled_dot_product_attention( attrs["mask"] = mask_literal elif is_fx_node(node.args[3]): inputs.append(value_id(node.args[3], ctx)) + mask_dtype = _extract_fx_torch_dtype(node.args[3]) + if mask_dtype is not None and mask_dtype != torch.bool: + attrs["additive_mask"] = True if len(node.args) > 4 and extract_literals(node.args[4]) is not None: attrs["dropout_p"] = float(node.args[4]) if len(node.args) > 5 and extract_literals(node.args[5]) is not None: @@ -775,8 +1216,8 @@ def import_cat(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, def import_split_with_sizes(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: if len(node.args) < 2: ctx.fail(f"unsupported split_with_sizes signature for {torch_op}: {node.args!r}") - sizes = extract_literals(node.args[1]) - if not isinstance(sizes, (tuple, list)) or not all(isinstance(v, int) for v in sizes): + sizes = _extract_int_sequence_literal(node.args[1]) + if sizes is None: ctx.fail(f"unsupported split sizes for {torch_op}: {node.args!r}") axis = -1 if len(node.args) > 2 and extract_literals(node.args[2]) is not None: @@ -786,7 +1227,7 @@ def import_split_with_sizes(ir: IRGraph, node: Any, ctx: ImportContext, *, shape op="split_with_sizes", inputs=[value_id(node.args[0], ctx)], outputs=[value_id(node, ctx)], - attrs={"sizes": tuple(int(v) for v in sizes), "axis": axis}, + attrs={"sizes": sizes, "axis": axis}, meta=_base_meta(shape, dtype, torch_op, node), ) register_node(ir, ir_node, shape=shape, dtype=dtype) @@ -813,9 +1254,44 @@ def import_chunk(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int def import_pow(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx)] + base_literal = _extract_numeric_literal(node.args[0]) if len(node.args) > 0 else None attrs: dict[str, object] = {} exponent_literal = extract_literals(node.args[1]) if len(node.args) > 1 else None + + if base_literal is not None and isinstance(exponent_literal, (int, float)): + add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) + ir.constants[value_id(node, ctx)] = base_literal**exponent_literal + return + + if base_literal is not None: + if not (len(node.args) > 1 and is_fx_node(node.args[1])): + ctx.fail(f"unsupported pow signature for {torch_op}: scalar base with non-literal exponent {node.args!r}") + if float(base_literal) <= 0.0: + ctx.fail(f"unsupported pow signature for {torch_op}: non-positive scalar base {base_literal!r}") + + exponent_value_id = value_id(node.args[1], ctx) + mul_output_id = f"{value_id(node, ctx)}__pow_scalar_base_log_mul" + mul_node = IRNode( + id=f"{node_id(node)}__pow_scalar_base_log_mul", + op="scalar_multiply", + inputs=[exponent_value_id], + outputs=[mul_output_id], + attrs={"value": float(math.log(float(base_literal)))}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, mul_node, shape=shape, dtype=dtype) + + exp_node = IRNode( + id=node_id(node), + op="scalar_exp", + inputs=[mul_output_id], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, exp_node, shape=shape, dtype=dtype) + return + + inputs = [value_id(node.args[0], ctx)] if exponent_literal is None and len(node.args) > 1 and is_fx_node(node.args[1]): inputs.append(value_id(node.args[1], ctx)) else: @@ -873,16 +1349,34 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, prepend_value = None append_value = None + prepend_node = None + append_node = None if len(inputs) > 3: - prepend_value = extract_literals(inputs[3]) + if is_fx_node(inputs[3]): + prepend_node = inputs[3] + else: + prepend_value = extract_literals(inputs[3]) if len(inputs) > 4: - append_value = extract_literals(inputs[4]) + if is_fx_node(inputs[4]): + append_node = inputs[4] + else: + append_value = extract_literals(inputs[4]) if "prepend" in getattr(node, "kwargs", {}): - prepend_value = extract_literals(node.kwargs["prepend"]) + prepend_arg = node.kwargs["prepend"] + if is_fx_node(prepend_arg): + prepend_node = prepend_arg + prepend_value = None + else: + prepend_value = extract_literals(prepend_arg) if "append" in getattr(node, "kwargs", {}): - append_value = extract_literals(node.kwargs["append"]) + append_arg = node.kwargs["append"] + if is_fx_node(append_arg): + append_node = append_arg + append_value = None + else: + append_value = extract_literals(append_arg) if prepend_value is not None or append_value is not None: - ctx.fail(f"unsupported diff signature for {torch_op}: prepend/append are not supported") + ctx.fail(f"unsupported diff signature for {torch_op}: literal prepend/append are not supported") source_value_id = value_id(source, ctx) source_shape = None @@ -902,16 +1396,75 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, if dim_extent < 1: ctx.fail(f"unsupported diff input extent for {torch_op}: dim size {dim_extent}") - left_shape = list(source_shape) - left_shape[normalized_dim] = max(0, dim_extent - 1) + concat_inputs = [source_value_id] + concat_shapes = [tuple(int(v) for v in source_shape)] + + def _append_concat_input(arg: Any, label: str) -> None: + nonlocal concat_inputs, concat_shapes + if arg is None: + return + arg_value_id = value_id(arg, ctx) + arg_value = ir.values.get(arg_value_id) + arg_shape = arg_value.shape if arg_value is not None else None + if arg_shape is None: + ctx.fail(f"unsupported diff import for {torch_op}: missing {label} shape") + arg_shape = tuple(int(v) for v in arg_shape) + if len(arg_shape) != rank: + ctx.fail( + f"unsupported diff import for {torch_op}: {label} rank {len(arg_shape)} does not match source rank {rank}" + ) + for axis, (arg_dim, src_dim) in enumerate(zip(arg_shape, source_shape, strict=True)): + if axis == normalized_dim: + continue + if int(arg_dim) != int(src_dim): + ctx.fail( + f"unsupported diff import for {torch_op}: {label} shape {arg_shape} " + f"is incompatible with source shape {source_shape}" + ) + concat_inputs.append(arg_value_id) + concat_shapes.append(arg_shape) + + if prepend_node is not None: + concat_inputs = [] + concat_shapes = [] + _append_concat_input(prepend_node, "prepend") + concat_inputs.append(source_value_id) + concat_shapes.append(tuple(int(v) for v in source_shape)) + if append_node is not None: + _append_concat_input(append_node, "append") + elif append_node is not None: + _append_concat_input(append_node, "append") + + augmented_shape = list(source_shape) + augmented_shape[normalized_dim] = sum(int(shape_item[normalized_dim]) for shape_item in concat_shapes) + augmented_shape_tuple = tuple(int(v) for v in augmented_shape) + diff_shape = list(augmented_shape) + diff_shape[normalized_dim] = max(0, int(augmented_shape[normalized_dim]) - 1) + diff_shape_tuple = tuple(int(v) for v in diff_shape) + + diff_source_value_id = source_value_id + if len(concat_inputs) > 1: + concat_output_id = f"{value_id(node, ctx)}__diff_source" + concat_node = IRNode( + id=f"{node_id(node)}__diff_source", + op="cat", + inputs=concat_inputs, + outputs=[concat_output_id], + attrs={"axis": normalized_dim}, + meta=_base_meta(augmented_shape_tuple, dtype, torch_op, node), + ) + register_node(ir, concat_node, shape=augmented_shape_tuple, dtype=dtype) + diff_source_value_id = concat_output_id + + left_shape = list(diff_shape_tuple) left_shape_tuple = tuple(int(v) for v in left_shape) left_node = IRNode( id=f"{node_id(node)}__diff_left", op="slice", - inputs=[source_value_id], + inputs=[diff_source_value_id], outputs=[f"{value_id(node, ctx)}__diff_left"], - attrs={"axis": normalized_dim, "start": 0, "end": dim_extent - 1, "step": 1}, + attrs={"axis": normalized_dim, "start": 0, "end": int(augmented_shape[normalized_dim]) - 1, "step": 1}, meta=_base_meta(left_shape_tuple, dtype, torch_op, node), ) register_node(ir, left_node, shape=left_shape_tuple, dtype=dtype) @@ -919,9 +1472,9 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, right_node = IRNode( id=f"{node_id(node)}__diff_right", op="slice", - inputs=[source_value_id], + inputs=[diff_source_value_id], outputs=[f"{value_id(node, ctx)}__diff_right"], - attrs={"axis": normalized_dim, "start": 1, "end": dim_extent, "step": 1}, + attrs={"axis": normalized_dim, "start": 1, "end": int(augmented_shape[normalized_dim]), "step": 1}, meta=_base_meta(left_shape_tuple, dtype, torch_op, node), ) register_node(ir, right_node, shape=left_shape_tuple, dtype=dtype) @@ -931,9 +1484,9 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, op="subtract", inputs=[right_node.outputs[0], left_node.outputs[0]], outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), + meta=_base_meta(diff_shape_tuple, dtype, torch_op, node), ) - register_node(ir, diff_node, shape=shape, dtype=dtype) + register_node(ir, diff_node, shape=diff_shape_tuple, dtype=dtype) def import_index(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: @@ -1023,6 +1576,30 @@ def import_conv1d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[in register_node(ir, ir_node, shape=shape, dtype=dtype) +def import_conv2d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported conv2d signature for {torch_op}: {node.args!r}") + + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + if len(node.args) > 2 and is_fx_node(node.args[2]): + inputs.append(value_id(node.args[2], ctx)) + + stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 + padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 + dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 + groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 + + ir_node = IRNode( + id=node_id(node), + op="conv2d", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + def import_pad(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: if len(node.args) < 2: ctx.fail(f"unsupported pad signature for {torch_op}: {node.args!r}") @@ -1058,12 +1635,8 @@ def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, if not node.args: ctx.fail(f"unsupported ones signature for {torch_op}: {node.args!r}") - shape_literal = extract_literals(node.args[0]) - if isinstance(shape_literal, int): - output_shape = (int(shape_literal),) - elif isinstance(shape_literal, (tuple, list)) and all(isinstance(v, int) for v in shape_literal): - output_shape = tuple(int(v) for v in shape_literal) - else: + output_shape = _extract_int_sequence_literal(node.args[0]) + if output_shape is None: ctx.fail(f"unsupported ones shape for {torch_op}: {node.args!r}") node_dtype = dtype @@ -1086,8 +1659,39 @@ def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, def import_layer_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx), value_id(node.args[3], ctx)] - attrs = {"normalized_shape": tuple(int(v) for v in node.args[1]), "eps": float(node.args[4])} + if not node.args: + ctx.fail(f"unsupported layer_norm signature for {torch_op}: {node.args!r}") + + normalized_shape_value = None + if len(node.args) > 1: + normalized_shape_value = _extract_int_sequence_literal(node.args[1]) + if normalized_shape_value is None: + normalized_shape_value = _extract_int_sequence_literal(getattr(node, "kwargs", {}).get("normalized_shape")) + if normalized_shape_value is None: + ctx.fail(f"unsupported normalized_shape for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + + eps_value = None + if len(node.args) > 4: + eps_value = _extract_numeric_literal(node.args[4]) + if eps_value is None: + eps_value = _extract_numeric_literal(getattr(node, "kwargs", {}).get("eps")) + if eps_value is None: + eps_value = 1e-5 + + inputs = [value_id(node.args[0], ctx)] + weight = node.args[2] if len(node.args) > 2 else getattr(node, "kwargs", {}).get("weight") + bias = node.args[3] if len(node.args) > 3 else getattr(node, "kwargs", {}).get("bias") + if not is_fx_node(weight): + ctx.fail(f"unsupported layer_norm weight form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + inputs.append(value_id(weight, ctx)) + if bias is None: + pass + elif is_fx_node(bias): + inputs.append(value_id(bias, ctx)) + else: + ctx.fail(f"unsupported layer_norm bias form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + + attrs = {"normalized_shape": tuple(int(v) for v in normalized_shape_value), "eps": float(eps_value)} ir_node = IRNode( id=node_id(node), op="layer_norm", @@ -1185,19 +1789,35 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i "type_as": import_type_as, "negate": import_negate, "abs": import_unary("abs"), + "clamp": import_clamp, "not_equal": import_not_equal, + "equal": import_equal, + "logical_and": import_binary("logical_and"), + "logical_or": import_binary("logical_or"), + "logical_not": import_unary("logical_not"), + "where": import_where, + "masked_scatter": import_masked_scatter, + "masked_fill": import_masked_fill, "cos": import_unary("cos"), "sin": import_unary("sin"), + "floor_divide": import_floor_divide, "scalar_exp": import_unary("scalar_exp"), "scalar_sqrt": import_unary("scalar_sqrt"), + "reciprocal": import_reciprocal, "scalar_log": import_unary("scalar_log"), "rsqrt": import_pow_rsqrt, "reshape": import_reshape, "flatten": import_flatten, "unsqueeze": import_unsqueeze, + "squeeze": import_squeeze, "expand": import_expand, + "one_hot": import_one_hot, + "tril": import_tril, + "unfold": import_unfold, + "numpy_T": import_numpy_t, "transpose": import_transpose, "permute": import_permute, + "movedim": import_movedim, "matmul": import_matmul, "linear": import_linear, "addmm": import_addmm, @@ -1206,6 +1826,7 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i "gelu": import_unary("gelu"), "gelu_erf": import_unary("gelu_erf"), "sigmoid": import_unary("sigmoid"), + "glu": import_glu, "softplus": import_unary("softplus"), "tanh": import_unary("tanh"), "softmax": import_softmax, @@ -1227,6 +1848,7 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i "gather": import_gather, "embedding": import_embedding, "conv1d": import_conv1d, + "conv2d": import_conv2d, "layer_norm": import_layer_norm, "rms_norm": import_rms_norm, "group_norm": import_group_norm, diff --git a/python/src/transpile/lower.py b/python/src/transpile/lower.py new file mode 100644 index 000000000..eb46f27c8 --- /dev/null +++ b/python/src/transpile/lower.py @@ -0,0 +1,2680 @@ +from __future__ import annotations + +from dataclasses import dataclass +import math +import os +from typing import Any + +import numpy as np +import torch + +from src.transpile.runtime_compat import Graph +from src.transpile.runtime_compat import Tensor +from src.transpile.capture_pytorch import CapturedModel +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.optimize_graph import optimize_graph +from src.transpile.weight_compat import ensure_binding_compatible +from src.transpile.weight_binding import WeightBinding + + +@dataclass +class TranspiledGraph: + graph: Graph + runtime_inputs: list[Tensor] + bound_constants: list[Tensor] + bound_constant_bindings: list[dict[str, object]] + bound_constant_value_ids: dict[int, str] + outputs: list[Tensor] + + def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: + if index < 0 or index >= len(self.runtime_inputs): + raise IndexError( + f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" + ) + self.graph.set_input(self.runtime_inputs[index], data, dtype=dtype) + + def set_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> None: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" + ) + for index, value in enumerate(inputs): + self.set_input(index, value) + + def execute(self) -> list[Tensor]: + self.graph.execute() + return self.outputs + + +@dataclass +class BroadcastAlias: + tensor: Tensor + logical_shape: tuple[int, ...] + kind: str + + +def _is_quantized_runtime_dtype(dtype: int) -> bool: + return int(dtype) in { + int(Graph.INT8), + int(getattr(Graph, "CQ1", 3)), + int(getattr(Graph, "CQ2", 4)), + int(getattr(Graph, "CQ3", 5)), + int(getattr(Graph, "CQ4", 6)), + } + + +def transpile_captured(captured: CapturedModel) -> TranspiledGraph: + return transpile_ir(captured.ir_graph) + + +def transpile_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + canonicalize_exported_graph(ir) + optimize_graph(ir) + return transpile_preoptimized_ir(ir) + + +def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + g = Graph() + g._transpile_materialized_constants = [] # type: ignore[attr-defined] + env: dict[str, Any] = {} + runtime_inputs: list[Tensor] = [] + bound_constants: list[Tensor] = [] + bound_constant_bindings: list[dict[str, object]] = [] + bound_constant_value_ids: dict[int, str] = {} + + for value_id in ir.inputs: + value = ir.values[value_id] + tensor = _lower_input_value(g, value) + env[value_id] = tensor + runtime_inputs.append(tensor) + + for value_id, const in ir.constants.items(): + value = ir.values[value_id] + binding = _lookup_weight_binding(value) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(g, value, const, binding=binding) + env[value_id] = lowered_const + if isinstance(lowered_const, Tensor): + bound_constants.append(lowered_const) + bound_constant_value_ids[int(lowered_const.id)] = str(value_id) + if binding is not None: + bound_constant_bindings.append( + { + "node_id": int(lowered_const.id), + "value_id": str(value_id), + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + ) + + for node_id in ir.order: + node = ir.nodes[node_id] + outputs = _lower_ir_node(g, node, env, ir) + if len(outputs) != len(node.outputs): + raise ValueError( + f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" + ) + for output_id, tensor in zip(node.outputs, outputs): + env[output_id] = tensor + + outputs = [env[value_id] for value_id in ir.outputs] + seen_bound_constant_ids = {int(tensor.id) for tensor in bound_constants} + for tensor in getattr(g, "_transpile_materialized_constants", []): + tensor_id = int(tensor.id) + if tensor_id in seen_bound_constant_ids: + continue + bound_constants.append(tensor) + seen_bound_constant_ids.add(tensor_id) + return TranspiledGraph( + graph=g, + runtime_inputs=runtime_inputs, + bound_constants=bound_constants, + bound_constant_bindings=bound_constant_bindings, + bound_constant_value_ids=bound_constant_value_ids, + outputs=outputs, + ) + + +def _lower_input_value(g: Graph, value: IRValue) -> Tensor: + if value.shape is None or value.dtype is None: + raise ValueError(f"IR input missing shape or dtype: {value.id}") + return g.input(shape=value.shape, dtype=_map_ir_dtype(value.dtype)) + +def _lower_constant_value( + g: Graph, + value: IRValue, + const: Any, + *, + binding: WeightBinding | None = None, +) -> Any: + if binding is None: + binding = _lookup_weight_binding(value) + if binding is not None: + _debug_mmap_binding(value.id, binding) + if binding.kind == "embedding": + return g.mmap_embeddings(binding.path) + return g.mmap_weights(binding.path) + + _debug_constant_fallback(value, const) + + if isinstance(const, torch.nn.Parameter): + const = const.detach() + if isinstance(const, torch.Tensor): + tensor_value = const.detach().cpu() + else: + raise NotImplementedError( + f"unsupported IR constant type for {value.id}: {type(const).__name__}" + ) + + if tensor_value.numel() == 1: + return tensor_value.item() + + return _materialize_constant_tensor(g, tensor_value) + + +def _lookup_weight_binding(value: IRValue) -> WeightBinding | None: + meta = getattr(value, "meta", None) + if isinstance(meta, dict): + path = meta.get("path") + kind = meta.get("kind") + source_name = meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + return WeightBinding(path=path, kind=kind, source_name=source_name) + return None + + +def _debug_mmap_binding(value_id: str, binding: WeightBinding) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + print( + "[transpile:mmap] " + f"value={value_id} " + f"kind={binding.kind} " + f"source={binding.source_name} " + f"path={binding.path}" + ) + + +def _debug_constant_fallback(value: IRValue, const: Any) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + const_type = type(const).__name__ + shape = None + dtype = None + if isinstance(const, torch.Tensor): + shape = tuple(const.shape) + dtype = str(const.dtype) + source_name = None + if isinstance(value.meta, dict): + source_name = value.meta.get("source_name") + print( + "[transpile:fallback] " + f"value={value.id} " + f"source={source_name} " + f"const_type={const_type} " + f"shape={shape} " + f"dtype={dtype}" + ) + + +def _debug_embedding_lowering(node: IRNode, embedding_tensor: Tensor, indices_tensor: Tensor) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + print( + "[transpile:embedding] " + f"node={node.id} " + f"embedding_id={embedding_tensor.id} " + f"embedding_shape={tuple(embedding_tensor.shape)} " + f"embedding_dtype={embedding_tensor.dtype} " + f"indices_id={indices_tensor.id} " + f"indices_shape={tuple(indices_tensor.shape)} " + f"indices_dtype={indices_tensor.dtype}" + ) + + +def _matmul_with_quantized_rhs_legalization( + g: Graph, + lhs: Tensor, + rhs: Tensor, + *, + pretransposed_rhs: bool = False, + output_dtype: int | None = None, +) -> Tensor: + if _is_quantized_runtime_dtype(rhs.dtype) and lhs.dtype == Graph.FP32: + lhs = g.precision_cast(lhs, Graph.FP16) + return g.matmul(lhs, rhs, pretransposed_rhs=pretransposed_rhs, output_dtype=output_dtype) + + +def _matmul_output_dtype( + lhs: Tensor, + rhs: Tensor, + *, + output_dtype: int | None, +) -> int | None: + if output_dtype != Graph.FP16: + return output_dtype + if _is_quantized_runtime_dtype(rhs.dtype): + return output_dtype + if lhs.dtype == Graph.FP32 or rhs.dtype == Graph.FP32: + return Graph.FP32 + return output_dtype + + +def _trim_padded_last_dim(g: Graph, tensor: Tensor, expected_shape: tuple[int, ...] | None) -> Tensor: + if expected_shape is None: + return tensor + actual_shape = tuple(int(dim) for dim in tensor.shape) + target_shape = tuple(int(dim) for dim in expected_shape) + if actual_shape == target_shape: + return tensor + if ( + len(actual_shape) == len(target_shape) + and len(actual_shape) >= 1 + and actual_shape[:-1] == target_shape[:-1] + and actual_shape[-1] > target_shape[-1] + ): + return g.slice(tensor, len(actual_shape) - 1, 0, target_shape[-1]) + return tensor + + +def _legalize_for_transpose(g: Graph, x: Tensor) -> Tensor: + return x + + +def _reshape_attention_output_for_linear( + g: Graph, + attn_out: Tensor, + target_shape: tuple[int, ...], +) -> Tensor: + actual_shape = tuple(int(dim) for dim in attn_out.shape) + target_shape = tuple(int(dim) for dim in target_shape) + if len(actual_shape) == 4 and len(target_shape) == 3 and actual_shape[0] == target_shape[0]: + batch, dim1, dim2, dim3 = actual_shape + _, target_seq, target_hidden = target_shape + if dim1 == target_seq and dim2 * dim3 == target_hidden: + return g.reshape(attn_out, target_shape) + if dim2 == target_seq and dim1 * dim3 == target_hidden: + return g.reshape(g.permute(attn_out, (0, 2, 1, 3)), target_shape) + return g.reshape(attn_out, target_shape) + + +def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> list[Any]: + op = node.op + + if op == "arange": + start = int(node.attrs.get("start", 0)) + end = int(node.attrs["end"]) + step = int(node.attrs.get("step", 1)) + dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) + tensor_value = torch.arange(start, end, step=step, dtype=dtype) + return [_materialize_constant_tensor(g, tensor_value)] + + if op == "add": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "add")] + + if op == "add_clipped": + lhs, rhs = _legalize_elementwise_binary_inputs(g, _tensor(env, node.inputs[0]), _tensor(env, node.inputs[1])) + return [g.add_clipped(lhs, rhs)] + + if op == "subtract": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "subtract")] + + if op == "multiply": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] + + if op == "multiply_inplace": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] + + if op == "divide": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "divide")] + + if op == "not_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "not_equal")] + + if op == "equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "equal")] + + if op == "greater": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "greater")] + + if op == "less_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "less_equal")] + + if op == "logical_and": + lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + return [_lower_binary_op(g, lhs, rhs, "multiply")] + + if op == "logical_or": + lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + return [_lower_compare_op(g, _lower_binary_op(g, lhs, rhs, "add"), 0.0, "not_equal")] + + if op == "logical_not": + return [_lower_compare_op(g, env[node.inputs[0]], 0.0, "equal")] + + if op == "where": + return [_lower_where_op(g, node, env)] + + if op == "scalar_add": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_add(x, float(node.attrs["value"]))] + + if op == "scalar_subtract": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_subtract(x, float(node.attrs["value"]))] + + if op == "scalar_subtract_reverse": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_add(g.scalar_multiply(x, -1.0), float(node.attrs["value"]))] + + if op == "scalar_multiply": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_multiply(x, float(node.attrs["value"]))] + + if op == "scalar_divide": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_divide(x, float(node.attrs["value"]))] + + if op == "scalar_floor_divide": + source = _tensor(env, node.inputs[0]) + x = _ensure_scalar_math_tensor(g, source) + y = g.scalar_floor_divide(x, float(node.attrs["value"])) + output_dtype = ir.values[node.outputs[0]].dtype + if output_dtype is None: + return [y] + target_dtype = _map_ir_dtype(output_dtype) + if y.dtype == target_dtype: + return [y] + return [g.precision_cast(y, target_dtype)] + + if op == "scalar_not_equal": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_not_equal(x, float(node.attrs["value"]))] + + if op == "scalar_equal": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + not_equal = g.scalar_not_equal(x, float(node.attrs["value"])) + return [g.scalar_not_equal(not_equal, 1.0)] + + if op == "scalar_divide_reverse": + raise NotImplementedError("scalar_divide_reverse is not directly supported by Cactus graph ops") + + if op == "precision_cast": + target_dtype = node.attrs.get("dtype") + if target_dtype is None: + return [_tensor(env, node.inputs[0])] + return [g.precision_cast(_tensor(env, node.inputs[0]), _map_ir_or_torch_dtype(target_dtype))] + + if op == "type_as": + source = _tensor(env, node.inputs[0]) + target = _tensor(env, node.inputs[1]) + return [g.precision_cast(source, target.dtype)] + + if op == "abs": + return [g.abs(_tensor(env, node.inputs[0]))] + + if op == "clamp": + x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) + min_value = node.attrs.get("min") + max_value = node.attrs.get("max") + if min_value is not None and math.isfinite(float(min_value)): + x = g.scalar_add(g.relu(g.scalar_subtract(x, float(min_value))), float(min_value)) + if max_value is not None and math.isfinite(float(max_value)): + upper_delta = g.relu(g.scalar_add(g.scalar_multiply(x, -1.0), float(max_value))) + x = g.scalar_add(g.scalar_multiply(upper_delta, -1.0), float(max_value)) + return [x] + + if op == "negate": + return [g.scalar_multiply(_tensor(env, node.inputs[0]), -1.0)] + + if op == "pow": + exponent = node.attrs.get("exponent") + if exponent is None: + raise NotImplementedError("tensor-tensor pow is not supported by Cactus graph ops") + return [g.pow(_tensor(env, node.inputs[0]), float(exponent))] + + if op == "scalar_exp": + return [g.scalar_exp(_tensor(env, node.inputs[0]))] + + if op == "scalar_sqrt": + return [g.scalar_sqrt(_tensor(env, node.inputs[0]))] + + if op == "scalar_log": + return [g.scalar_log(_tensor(env, node.inputs[0]))] + + if op == "scalar_cos": + return [g.scalar_cos(_tensor(env, node.inputs[0]))] + + if op == "scalar_sin": + return [g.scalar_sin(_tensor(env, node.inputs[0]))] + + if op in {"reshape", "view"}: + source = env[node.inputs[0]] + if isinstance(source, BroadcastAlias): + input_shape = tuple(source.logical_shape) + else: + input_shape = tuple(_tensor(env, node.inputs[0]).shape) + target_shape = _resolve_reshape_shape(input_shape, tuple(node.attrs["shape"])) + if isinstance(source, BroadcastAlias): + if source.kind != "gqa_repeat_kv": + raise NotImplementedError(f"unsupported broadcast alias kind in reshape: {source.kind}") + base_shape = tuple(source.tensor.shape) + if len(base_shape) != 4: + raise NotImplementedError(f"gqa_repeat_kv alias requires 4D base tensor, got {base_shape}") + if len(target_shape) != 4: + raise NotImplementedError(f"gqa_repeat_kv reshape requires 4D target shape, got {target_shape}") + batch, kv_heads, seq_len, head_dim = base_shape + if target_shape[0] != batch or target_shape[2] != seq_len or target_shape[3] != head_dim: + raise NotImplementedError( + f"gqa_repeat_kv reshape mismatch: base {base_shape}, target {target_shape}" + ) + if target_shape[1] % max(kv_heads, 1) != 0: + raise NotImplementedError( + f"gqa_repeat_kv target head count must be a multiple of kv heads: {base_shape} -> {target_shape}" + ) + return [BroadcastAlias(source.tensor, target_shape, source.kind)] + return [g.reshape(_tensor(env, node.inputs[0]), target_shape)] + + if op == "flatten": + return [ + g.flatten( + _tensor(env, node.inputs[0]), + start_dim=int(node.attrs["start_dim"]), + end_dim=int(node.attrs["end_dim"]), + ) + ] + + if op == "unsqueeze": + x = _tensor(env, node.inputs[0]) + target_shape = node.attrs.get("shape") + if target_shape is None: + dim = _normalize_dim(int(node.attrs["dim"]), len(x.shape) + 1) + shape_list = list(x.shape) + shape_list.insert(dim, 1) + target_shape = tuple(shape_list) + return [g.reshape(x, tuple(target_shape))] + + if op == "expand": + matched_base = _match_gqa_expand_alias(ir, node) + if matched_base is not None: + return [BroadcastAlias(_tensor(env, matched_base), tuple(node.attrs["shape"]), "gqa_repeat_kv")] + x = _tensor(env, node.inputs[0]) + target_shape = _resolve_expand_shape(tuple(x.shape), tuple(node.attrs["shape"])) + if target_shape == tuple(x.shape): + return [x] + return [g.expand(x, target_shape)] + + if op == "one_hot": + indices = _tensor(env, node.inputs[0]) + num_classes = int(node.attrs["num_classes"]) + eye = torch.eye(num_classes, dtype=torch.float16) + embedding = _materialize_constant_tensor(g, eye) + return [g.embedding_from_tensor(embedding, indices)] + + if op == "transpose": + x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + dim0 = _normalize_dim(int(node.attrs["dim0"]), len(x.shape)) + dim1 = _normalize_dim(int(node.attrs["dim1"]), len(x.shape)) + rank = len(x.shape) + permutation = list(range(rank)) + permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] + if rank == 2 and permutation == [1, 0]: + return [g.transpose(x)] + return [g.permute(x, permutation)] + + if op == "permute": + x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + permutation = tuple(_normalize_dim(int(dim), len(x.shape)) for dim in node.attrs["permutation"]) + if len(permutation) == 2 and permutation == (1, 0): + return [g.transpose(x)] + return [g.permute(x, permutation)] + + if op == "matmul": + lhs = _tensor(env, node.inputs[0]) + rhs = _tensor(env, node.inputs[1]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = ( + _map_ir_dtype(output_value.dtype) + if output_value is not None and output_value.dtype is not None + else None + ) + matmul_output_dtype = _matmul_output_dtype(lhs, rhs, output_dtype=output_dtype) + if len(lhs.shape) == 2 and len(rhs.shape) == 2: + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + legalized = _legalize_matmul_inputs(g, lhs, rhs, node) + if legalized is None: + static_batched = _lower_static_batched_matmul(g, lhs, rhs, output_dtype=matmul_output_dtype) + if static_batched is not None: + if output_dtype is not None and static_batched.dtype != output_dtype: + static_batched = g.precision_cast(static_batched, output_dtype) + return [static_batched] + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + lhs_2d, rhs_2d, output_shape = legalized + out = _matmul_with_quantized_rhs_legalization(g, lhs_2d, rhs_2d, output_dtype=matmul_output_dtype) + out = g.reshape(out, output_shape) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "linear": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else x.dtype + reshape_back: tuple[int, ...] | None = None + if len(x.shape) > 2: + x = _flatten_to_2d_for_linear(g, x) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"linear missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + matmul_output_dtype = _matmul_output_dtype(x, weight, output_dtype=output_dtype) + if node.attrs.get("has_bias"): + bias = _tensor(env, node.inputs[2]) + if ( + x.dtype == Graph.FP16 + and weight.dtype == Graph.FP16 + and output_dtype == Graph.FP16 + ): + matmul_output_dtype = Graph.FP32 + elif bias.dtype == Graph.FP32 and not _is_quantized_runtime_dtype(weight.dtype): + matmul_output_dtype = Graph.FP32 + out = _matmul_with_quantized_rhs_legalization( + g, + x, + weight, + pretransposed_rhs=True, + output_dtype=matmul_output_dtype, + ) + out = _trim_padded_last_dim(g, out, output_value.shape if output_value is not None else None) + if node.attrs.get("has_bias"): + bias = _ensure_tensor_dtype(g, bias, matmul_output_dtype) + out = g.add(out, bias) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "addmm": + bias = _tensor(env, node.inputs[0]) + lhs = _tensor(env, node.inputs[1]) + rhs = _tensor(env, node.inputs[2]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else lhs.dtype + matmul_output_dtype = _matmul_output_dtype(lhs, rhs, output_dtype=output_dtype) + if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 and bias.dtype == Graph.FP16 and output_dtype == Graph.FP16: + matmul_output_dtype = Graph.FP32 + elif bias.dtype == Graph.FP32 and not _is_quantized_runtime_dtype(rhs.dtype): + matmul_output_dtype = Graph.FP32 + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + out = _trim_padded_last_dim(g, out, output_value.shape if output_value is not None else None) + bias = _ensure_tensor_dtype(g, bias, matmul_output_dtype) + out = g.add(bias, out) + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "relu": + return [g.relu(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] + + if op == "silu": + return [g.silu(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] + + if op == "gelu": + return [g.gelu(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] + + if op == "gelu_erf": + return [g.gelu_erf(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] + + if op == "sigmoid": + return [g.sigmoid(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] + + if op == "glu": + return [g.glu(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] + + if op == "softplus": + x = _tensor(env, node.inputs[0]) + return [_lower_softplus(g, x)] + + if op == "tanh": + return [g.tanh(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] + + if op == "softmax": + return [g.softmax(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] + + if op == "lstm_cell": + x = _tensor(env, node.inputs[0]) + h_prev = _tensor(env, node.inputs[1]) + c_prev = _tensor(env, node.inputs[2]) + weight_ih = _tensor(env, node.inputs[3]) + weight_hh = _tensor(env, node.inputs[4]) + bias_ih = _tensor(env, node.inputs[5]) + bias_hh = _tensor(env, node.inputs[6]) + + lstm_out = g.lstm_cell( + _ensure_fp16_tensor(g, x), + _ensure_fp16_tensor(g, h_prev), + _ensure_fp16_tensor(g, c_prev), + _ensure_fp16_tensor(g, weight_ih), + _ensure_fp16_tensor(g, weight_hh), + _ensure_fp16_tensor(g, bias_ih), + _ensure_fp16_tensor(g, bias_hh), + ) + h_new = g.slice(lstm_out, axis=2, start=0, length=1) + c_new = g.slice(lstm_out, axis=2, start=1, length=1) + h_shape = ir.values[node.outputs[0]].shape + c_shape = ir.values[node.outputs[1]].shape + if h_shape is not None: + h_new = g.reshape(h_new, tuple(int(dim) for dim in h_shape)) + if c_shape is not None: + c_new = g.reshape(c_new, tuple(int(dim) for dim in c_shape)) + return [h_new, c_new] + + if op in {"scaled_dot_product_attention", "attention"}: + if _should_lower_gemma4_decoder_attention_without_kernel(ir, node): + return [_lower_gemma4_decoder_attention_without_kernel(g, ir, env, node)] + mask = node.attrs.get("mask") + mask_tensor: Tensor | None = None + additive_mask = bool(node.attrs.get("additive_mask", False)) + if len(node.inputs) > 3: + mask_tensor = _tensor(env, node.inputs[3]) + elif mask is not None: + raise NotImplementedError(f"{op} with literal mask is not supported yet") + if op == "scaled_dot_product_attention": + dropout_p = float(node.attrs.get("dropout_p", 0.0)) + if dropout_p != 0.0: + raise NotImplementedError("scaled_dot_product_attention with dropout is not supported yet") + + # PyTorch SDPA exports tensors as [batch, heads, seq, dim], while Cactus attention + # expects [batch, seq, heads, dim]. Convert into Cactus layout, call attention, then + # convert back so downstream exported ops still see PyTorch layout. + query = g.permute(_attention_tensor(env, node.inputs[0]), (0, 2, 1, 3)) + key = g.permute(_attention_tensor(env, node.inputs[1]), (0, 2, 1, 3)) + value = g.permute(_attention_tensor(env, node.inputs[2]), (0, 2, 1, 3)) + if mask_tensor is not None: + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=additive_mask, + ) + return [g.permute(out, (0, 2, 1, 3))] + + if op == "attention_block": + has_mask = bool(node.attrs.get("has_mask", False)) + has_gate = bool(node.attrs.get("has_gate", False)) + has_bias = bool(node.attrs.get("has_bias", False)) + input_index = 0 + query = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + key = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + value = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + mask_tensor: Tensor | None = None + if has_mask: + mask_tensor = _tensor(env, node.inputs[input_index]) + input_index += 1 + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + + attn_out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=bool(node.attrs.get("additive_mask", False)), + ) + flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) + flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () + if flat_shape: + attn_out = _reshape_attention_output_for_linear(g, attn_out, flat_shape) + + if has_gate: + gate = _tensor(env, node.inputs[input_index]) + input_index += 1 + attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) + attn_out = g.multiply(attn_out, gate) + + weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + reshape_back: tuple[int, ...] | None = None + linear_input = attn_out + if len(linear_input.shape) > 2: + linear_input = _flatten_to_2d_for_linear(g, linear_input) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"attention_block missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) + if has_bias: + out = g.add(out, _tensor(env, node.inputs[input_index])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "self_attention_block": + has_mask = bool(node.attrs.get("has_mask", False)) + has_gate = bool(node.attrs.get("has_gate", False)) + has_bias = bool(node.attrs.get("has_bias", False)) + has_query_projection_bias = bool(node.attrs.get("has_query_projection_bias", False)) + has_query_add = bool(node.attrs.get("has_query_add", False)) + has_rel_query_add = bool(node.attrs.get("has_rel_query_add", False)) + has_key_projection_bias = bool(node.attrs.get("has_key_projection_bias", False)) + has_value_projection_bias = bool(node.attrs.get("has_value_projection_bias", False)) + has_rel_pos_bias = bool(node.attrs.get("has_rel_pos_bias", False)) + has_relative_key_projection_bias = bool(node.attrs.get("has_relative_key_projection_bias", False)) + + input_index = 0 + hidden = _tensor(env, node.inputs[input_index]) + input_index += 1 + + query_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + query_projection_bias = None + if has_query_projection_bias: + query_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + query_add = None + if has_query_add: + query_add = _tensor(env, node.inputs[input_index]) + input_index += 1 + rel_query_add = None + if has_rel_query_add: + rel_query_add = _tensor(env, node.inputs[input_index]) + input_index += 1 + + key_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + key_projection_bias = None + if has_key_projection_bias: + key_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + value_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + value_projection_bias = None + if has_value_projection_bias: + value_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + mask_tensor: Tensor | None = None + if has_mask: + mask_tensor = _tensor(env, node.inputs[input_index]) + input_index += 1 + + relative_key_input = None + relative_key_weight = None + relative_key_projection_bias = None + if has_rel_pos_bias: + relative_key_input = _tensor(env, node.inputs[input_index]) + input_index += 1 + relative_key_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + if has_relative_key_projection_bias: + relative_key_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + gate = None + if has_gate: + gate = _tensor(env, node.inputs[input_index]) + input_index += 1 + + output_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + output_bias = None + if has_bias: + output_bias = _tensor(env, node.inputs[input_index]) + + query_shape = tuple(int(v) for v in node.attrs.get("query_shape", ())) + key_shape = tuple(int(v) for v in node.attrs.get("key_shape", ())) + value_shape = tuple(int(v) for v in node.attrs.get("value_shape", ())) + relative_key_shape = tuple(int(v) for v in node.attrs.get("relative_key_shape", ())) + + query_base = _lower_projected_attention_tensor(g, hidden, query_weight, query_projection_bias, query_shape) + query = query_base + if query_add is not None: + query_add = _normalize_attention_add_tensor(g, query_add, query_shape) + query, query_add = _legalize_elementwise_binary_inputs(g, query, query_add) + query = g.add(query, query_add) + + key = _lower_projected_attention_tensor(g, hidden, key_weight, key_projection_bias, key_shape) + value = _lower_projected_attention_tensor(g, hidden, value_weight, value_projection_bias, value_shape) + + if has_rel_pos_bias: + if relative_key_input is None or relative_key_weight is None or not relative_key_shape: + raise NotImplementedError(f"self_attention_block missing relative position bias inputs for node {node.id}") + rel_query = query_base + if rel_query_add is not None: + rel_query_add = _normalize_attention_add_tensor(g, rel_query_add, query_shape) + rel_query, rel_query_add = _legalize_elementwise_binary_inputs(g, rel_query, rel_query_add) + rel_query = g.add(rel_query, rel_query_add) + relative_key = _lower_projected_attention_tensor( + g, + relative_key_input, + relative_key_weight, + relative_key_projection_bias, + relative_key_shape, + ) + rel_mask = g.rel_pos_bias(rel_query, relative_key, float(node.attrs.get("rel_pos_scale", 1.0))) + if mask_tensor is None: + mask_tensor = rel_mask + else: + mask_tensor, rel_mask = _legalize_elementwise_binary_inputs(g, mask_tensor, rel_mask) + mask_tensor = g.add(mask_tensor, rel_mask) + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + elif mask_tensor is not None: + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + if mask_tensor is not None: + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + + attn_out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=bool(node.attrs.get("additive_mask", False)), + ) + flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) + flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () + if flat_shape: + attn_out = _reshape_attention_output_for_linear(g, attn_out, flat_shape) + + if gate is not None: + attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) + attn_out = g.multiply(attn_out, gate) + + linear_input = attn_out + reshape_back: tuple[int, ...] | None = None + if len(linear_input.shape) > 2: + linear_input = _flatten_to_2d_for_linear(g, linear_input) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"self_attention_block missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, linear_input, output_weight, pretransposed_rhs=True) + if output_bias is not None: + out = g.add(out, output_bias) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op in ("sum", "mean", "variance", "min", "max"): + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + axes = _normalize_reduction_axes(node.attrs.get("axis"), len(x.shape)) + keepdim = bool(node.attrs.get("keepdim", False)) + out = _lower_reduction(g, op, x, axes=axes) + if keepdim: + new_shape = list(x.shape) + for reduced_axis in axes: + new_shape[reduced_axis] = 1 + out = g.reshape(out, tuple(new_shape)) + return [out] + + if op == "cumsum": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + raw_axis = node.attrs.get("axis") + if not isinstance(raw_axis, int): + args = node.attrs.get("args", ()) + if isinstance(args, (list, tuple)) and len(args) > 1 and isinstance(args[1], int): + raw_axis = int(args[1]) + else: + raw_axis = -1 + axis = _normalize_dim(int(raw_axis), len(x.shape)) + return [g.cumsum(x, axis)] + + if op == "cat": + tensors = [_tensor(env, value_id) for value_id in node.inputs] + output_dtype: int | None = None + if node.outputs: + output_value = ir.values.get(node.outputs[0]) + if output_value is not None and output_value.dtype is not None: + output_dtype = _map_ir_or_torch_dtype(output_value.dtype) + return [_cat_with_legalized_dtype(g, tensors, axis=int(node.attrs.get("axis", 0)), dtype=output_dtype)] + + if op in {"stack", "aten.stack.default"}: + tensors = [_tensor(env, value_id) for value_id in node.inputs] + if not tensors: + raise NotImplementedError("stack requires at least one input tensor") + + raw_axis = node.attrs.get("axis") + if not isinstance(raw_axis, int): + args = node.attrs.get("args", ()) + if isinstance(args, (list, tuple)) and len(args) > 1 and isinstance(args[1], int): + raw_axis = int(args[1]) + else: + raw_axis = 0 + + base_shape = tuple(int(dim) for dim in tensors[0].shape) + axis = _normalize_dim(int(raw_axis), len(base_shape) + 1) + reshaped: list[Tensor] = [] + for tensor in tensors: + if tuple(int(dim) for dim in tensor.shape) != base_shape: + raise NotImplementedError( + f"stack requires equal input shapes, got {[tuple(int(dim) for dim in t.shape) for t in tensors]}" + ) + expanded_shape = list(base_shape) + expanded_shape.insert(axis, 1) + reshaped.append(g.reshape(tensor, tuple(expanded_shape))) + return [g.cat(reshaped, axis=axis)] + + if op == "slice": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs["axis"]), len(x.shape)) + start = int(node.attrs["start"]) + end = int(node.attrs["end"]) + step = int(node.attrs.get("step", 1)) + dim_size = x.shape[axis] + if step == 0: + raise NotImplementedError("slice with step == 0 is invalid") + if step != 1: + indices = list(range(*slice(start, end, step).indices(dim_size))) + if axis == 0 or x.dtype == Graph.FP16: + return [_lower_static_strided_slice_via_gather(g, x, axis=axis, indices=indices)] + if not indices: + return [g.slice(x, axis=axis, start=0, length=0)] + expanded_shape = list(int(dim) for dim in x.shape) + expanded_shape[axis] = 1 + pieces: list[Tensor] = [] + for idx in indices: + piece = g.index(x, index_value=idx, axis=axis) + piece = g.reshape(piece, tuple(expanded_shape)) + pieces.append(piece) + if len(pieces) == 1: + return [pieces[0]] + return [g.cat(pieces, axis=axis)] + start = _normalize_index(start, dim_size) + end = _normalize_slice_end(end, dim_size) + length = max(0, end - start) + return [g.slice(x, axis=axis, start=start, length=length)] + + if op == "split_with_sizes": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", -1)), len(x.shape)) + sizes = tuple(int(v) for v in node.attrs["sizes"]) + start = 0 + outputs: list[Tensor] = [] + for size in sizes: + outputs.append(g.slice(x, axis=axis, start=start, length=int(size))) + start += int(size) + return [outputs] + + if op == "chunk": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) + chunks = int(node.attrs["chunks"]) + if chunks <= 0: + raise NotImplementedError(f"chunk requires a positive chunk count, got {chunks}") + dim_size = int(x.shape[axis]) + if dim_size <= 0: + return [[g.slice(x, axis=axis, start=0, length=0)]] + chunk_size = (dim_size + chunks - 1) // chunks + outputs: list[Tensor] = [] + start = 0 + while start < dim_size: + length = min(chunk_size, dim_size - start) + outputs.append(g.slice(x, axis=axis, start=start, length=length)) + start += length + return [outputs] + + if op == "ones": + shape = tuple(int(v) for v in node.attrs.get("shape", ())) + if not shape: + raise NotImplementedError(f"ones requires a static shape, got {shape}") + torch_dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) + return [_materialize_constant_tensor(g, torch.ones(shape, dtype=torch_dtype))] + + if op == "tril": + x = _tensor(env, node.inputs[0]) + shape = tuple(int(dim) for dim in x.shape) + if len(shape) < 2: + raise NotImplementedError(f"tril expects rank >= 2, got shape {shape}") + diagonal = int(node.attrs.get("diagonal", 0)) + mask_dtype = _torch_dtype_for_graph_dtype(x.dtype) + mask = torch.ones(shape, dtype=mask_dtype).tril(diagonal=diagonal) + mask_tensor = _materialize_constant_tensor(g, mask) + return [_lower_binary_op(g, x, mask_tensor, "multiply")] + + if op == "unfold": + x = _tensor(env, node.inputs[0]) + rank = len(x.shape) + axis = _normalize_dim(int(node.attrs["dimension"]), rank) + size = int(node.attrs["size"]) + step = int(node.attrs["step"]) + if size <= 0: + raise NotImplementedError(f"unfold requires a positive size, got {size}") + if step <= 0: + raise NotImplementedError(f"unfold requires a positive step, got {step}") + dim_size = int(x.shape[axis]) + if size > dim_size: + raise NotImplementedError( + f"unfold size {size} exceeds dimension size {dim_size} for axis {axis}" + ) + + window_starts = list(range(0, dim_size - size + 1, step)) + if not window_starts: + raise NotImplementedError( + f"unfold produced no windows for axis={axis}, size={size}, step={step}, dim_size={dim_size}" + ) + + base_shape = [int(dim) for dim in x.shape] + trailing_shape = [base_shape[idx] for idx in range(rank) if idx != axis] + [size] + expanded_shape = tuple(trailing_shape[:axis] + [1] + trailing_shape[axis:]) + permutation = tuple(idx for idx in range(rank) if idx != axis) + (axis,) + + windows: list[Tensor] = [] + for start in window_starts: + piece = g.slice(x, axis=axis, start=start, length=size) + if axis != rank - 1: + piece = g.permute(piece, permutation=permutation) + piece = g.reshape(piece, expanded_shape) + windows.append(piece) + if len(windows) == 1: + return [windows[0]] + return [g.cat(windows, axis=axis)] + + if op == "pad": + x = _tensor(env, node.inputs[0]) + mode = str(node.attrs.get("mode", "constant")) + if mode != "constant": + raise NotImplementedError(f"pad mode is unsupported: {mode}") + pads = tuple(int(v) for v in node.attrs.get("pads", ())) + if len(pads) % 2 != 0: + raise NotImplementedError(f"pad expects an even-length pads tuple, got {pads}") + value = float(node.attrs.get("value", 0.0)) + + current = x + current_shape = list(int(dim) for dim in x.shape) + pad_dims = len(pads) // 2 + if pad_dims > len(current_shape): + raise NotImplementedError( + f"pad rank mismatch: pads={pads} for input shape {tuple(current_shape)}" + ) + + for pad_index in range(pad_dims): + before = pads[2 * pad_index] + after = pads[2 * pad_index + 1] + axis = len(current_shape) - 1 - pad_index + if before < 0 or after < 0: + raise NotImplementedError(f"negative pad is unsupported: {pads}") + pieces = [] + torch_dtype = _torch_dtype_for_graph_dtype(current.dtype) + if before > 0: + left_shape = list(current_shape) + left_shape[axis] = before + left = _materialize_constant_tensor( + g, + torch.full(tuple(left_shape), value, dtype=torch_dtype), + ) + pieces.append(left) + pieces.append(current) + if after > 0: + right_shape = list(current_shape) + right_shape[axis] = after + right = _materialize_constant_tensor( + g, + torch.full(tuple(right_shape), value, dtype=torch_dtype), + ) + pieces.append(right) + if len(pieces) > 1: + current = g.cat(pieces, axis=axis) + current_shape[axis] += before + after + return [current] + + if op == "index": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) + index_value = _normalize_index(int(node.attrs["index_value"]), x.shape[axis]) + return [g.index(x, index_value=index_value, axis=axis)] + + if op == "gather": + return [ + g.gather( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + axis=int(node.attrs.get("axis", 0)), + ) + ] + + if op == "embedding": + embedding_tensor = _tensor(env, node.inputs[0]) + indices_tensor = _tensor(env, node.inputs[1]) + _debug_embedding_lowering(node, embedding_tensor, indices_tensor) + return [g.embedding_from_tensor(embedding_tensor, indices_tensor)] + + if op == "conv_module": + input_index = 0 + x_nlc = _tensor(env, node.inputs[input_index]) + input_index += 1 + + pointwise1_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + pointwise1_bias = None + if bool(node.attrs.get("has_pointwise1_bias", False)): + pointwise1_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + depthwise_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + depthwise_bias = None + if bool(node.attrs.get("has_depthwise_bias", False)): + depthwise_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + batch_norm_weight = _tensor(env, node.inputs[input_index]) + batch_norm_bias = _tensor(env, node.inputs[input_index + 1]) + batch_norm_running_mean = _tensor(env, node.inputs[input_index + 2]) + batch_norm_running_var = _tensor(env, node.inputs[input_index + 3]) + input_index += 4 + + pointwise2_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + pointwise2_bias = None + if bool(node.attrs.get("has_pointwise2_bias", False)): + pointwise2_bias = _tensor(env, node.inputs[input_index]) + + if len(x_nlc.shape) != 3: + raise NotImplementedError(f"conv_module expects rank-3 NLC input, got {x_nlc.shape}") + kernel_size = int(node.attrs.get("depthwise_kernel_size", 0)) + padding = int(node.attrs.get("depthwise_padding", 0)) + if len(depthwise_weight.shape) != 3 or kernel_size != 9 or padding != 4: + raise NotImplementedError( + f"conv_module currently supports same depthwise kernel-9 lowering, got " + f"kernel_size={kernel_size} padding={padding} weight_shape={depthwise_weight.shape}" + ) + + current = g.conv1d_pointwise(x_nlc, pointwise1_weight, bias=pointwise1_bias) + current = g.glu(current, axis=-1) + current = g.conv1d_same_depthwise_k9(current, depthwise_weight, bias=depthwise_bias) + current = g.batch_norm( + current, + batch_norm_weight, + batch_norm_bias, + batch_norm_running_mean, + batch_norm_running_var, + axis=len(current.shape) - 1, + eps=float(node.attrs.get("eps", 1e-5)), + ) + current = g.silu(current) + current = g.conv1d_pointwise(current, pointwise2_weight, bias=pointwise2_bias) + return [current] + + if op == "conv1d": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + stride = int(node.attrs.get("stride", 1)) + padding = int(node.attrs.get("padding", 0)) + dilation = int(node.attrs.get("dilation", 1)) + groups = int(node.attrs.get("groups", 1)) + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and weight.shape[2] == 1 + and stride == 1 + and padding == 0 + and dilation == 1 + and groups == 1 + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_pointwise(x_nlc, weight, bias=bias) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and weight.shape[2] == 9 + and stride == 1 + and padding == 4 + and dilation == 1 + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_same_depthwise_k9(x_nlc, weight, bias=bias) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and stride == 1 + and padding == dilation * max(weight.shape[2] - 1, 0) + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_causal(x_nlc, weight, kernel_size=weight.shape[2], dilation=dilation) + if bias is not None: + bias_reshaped = g.reshape(bias, (1, 1, int(weight.shape[0]))) + out_nlc, bias_reshaped = _legalize_elementwise_binary_inputs(g, out_nlc, bias_reshaped) + out_nlc = g.add(out_nlc, bias_reshaped) + return [g.permute(out_nlc, (0, 2, 1))] + + if dilation != 1: + raise NotImplementedError(f"conv1d with dilation != 1 is unsupported by generic lowering: {dilation}") + if padding != 0: + if len(x.shape) != 3: + raise NotImplementedError(f"generic conv1d padding expects rank-3 input, got {x.shape}") + batch_size, channels, _ = (int(dim) for dim in x.shape) + pad_dtype = _torch_dtype_for_graph_dtype(x.dtype) + pieces: list[Tensor] = [] + if padding > 0: + left = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, padding), dtype=pad_dtype), + ) + pieces.append(left) + pieces.append(x) + if padding > 0: + right = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, padding), dtype=pad_dtype), + ) + pieces.append(right) + x = g.cat(pieces, axis=2) + if groups != 1: + if len(x.shape) != 3 or len(weight.shape) != 3: + raise NotImplementedError( + f"grouped conv1d lowering expects rank-3 input and weight, got {x.shape} and {weight.shape}" + ) + input_channels = int(x.shape[1]) + if input_channels % groups != 0: + raise NotImplementedError( + f"grouped conv1d requires input channels divisible by groups, got C_in={input_channels}, groups={groups}" + ) + output_channels = int(weight.shape[0]) + if output_channels % groups != 0: + raise NotImplementedError( + f"grouped conv1d requires output channels divisible by groups, got C_out={output_channels}, groups={groups}" + ) + input_channels_per_group = input_channels // groups + output_channels_per_group = output_channels // groups + if int(weight.shape[1]) != input_channels_per_group: + raise NotImplementedError( + "grouped conv1d weight has incompatible per-group input channels: " + f"weight C_in/group={int(weight.shape[1])}, expected {input_channels_per_group}" + ) + outputs: list[Tensor] = [] + for group_index in range(groups): + x_group = g.slice( + x, + axis=1, + start=group_index * input_channels_per_group, + length=input_channels_per_group, + ) + weight_group = g.slice( + weight, + axis=0, + start=group_index * output_channels_per_group, + length=output_channels_per_group, + ) + bias_group = None + if bias is not None: + bias_group = g.slice( + bias, + axis=0, + start=group_index * output_channels_per_group, + length=output_channels_per_group, + ) + outputs.append(g.conv1d(x_group, weight_group, bias=bias_group, stride=stride)) + if len(outputs) == 1: + return [outputs[0]] + return [g.cat(outputs, axis=1)] + return [g.conv1d(x, weight, bias=bias, stride=stride)] + + if op == "conv2d": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + stride = int(node.attrs.get("stride", 1)) + padding = int(node.attrs.get("padding", 0)) + dilation = int(node.attrs.get("dilation", 1)) + groups = int(node.attrs.get("groups", 1)) + + if len(x.shape) != 4 or len(weight.shape) != 4: + raise NotImplementedError(f"conv2d lowering expects rank-4 tensors, got {x.shape} and {weight.shape}") + if dilation != 1: + raise NotImplementedError(f"conv2d with dilation != 1 is unsupported: {dilation}") + + if stride == 2 and padding == 0 and groups == 1 and weight.shape[2:] == (3, 3): + batch_size, channels, height, width = (int(dim) for dim in x.shape) + pad_dtype = _torch_dtype_for_graph_dtype(x.dtype) + top = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, 1, width), dtype=pad_dtype), + ) + bottom = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, 1, width), dtype=pad_dtype), + ) + x = g.cat([top, x, bottom], axis=2) + left = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, height + 2, 1), dtype=pad_dtype), + ) + right = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, height + 2, 1), dtype=pad_dtype), + ) + x = g.cat([left, x, right], axis=3) + y = g.conv2d_k3s2p1(x, weight, bias=bias) + output_height = ((height - 3) // 2) + 1 + output_width = ((width - 3) // 2) + 1 + y = g.slice(y, axis=2, start=1, length=output_height) + y = g.slice(y, axis=3, start=1, length=output_width) + return [y] + + if stride == 2 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): + return [g.conv2d_k3s2p1(x, weight, bias=bias)] + + if ( + stride == 2 + and padding == 1 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and weight.shape[2:] == (3, 3) + ): + return [g.conv2d_depthwise_k3s2p1(x, weight, bias=bias)] + + if ( + stride == 1 + and padding == 0 + and groups == 1 + and weight.shape[2:] == (1, 1) + ): + return [g.conv2d_pointwise_1x1(x, weight, bias=bias)] + + if stride == 1 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): + return [g.conv2d_k3s1p1(x, weight, bias=bias)] + + return [g.conv2d(x, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups)] + + if op == "layer_norm": + x = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[0])) + weight = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[1])) + bias = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[2])) if len(node.inputs) > 2 else None + return [g.layer_norm(x, weight, bias=bias, eps=float(node.attrs["eps"]))] + + if op == "rms_norm": + x = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[0])) + weight = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[1])) + reshape_back: tuple[int, ...] | None = None + if len(x.shape) > 2: + reshape_back = tuple(int(dim) for dim in x.shape) + x = _flatten_to_2d_for_linear(g, x) + out = g.rms_norm(x, weight, eps=float(node.attrs["eps"])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "rope": + rope_input = _tensor(env, node.inputs[0]) + if _rope_input_is_bhsd(ir, node.inputs[0]): + rope_input = g.permute(rope_input, (0, 2, 1, 3)) + rope_out = g.rope( + rope_input, + float(node.attrs["theta"]), + position_offset=int(node.attrs.get("position_offset", 0)), + ) + return [g.permute(rope_out, (0, 2, 1, 3))] + return [ + g.rope( + rope_input, + float(node.attrs["theta"]), + position_offset=int(node.attrs.get("position_offset", 0)), + ) + ] + + if op == "rel_pos_bias": + query = _attention_tensor(env, node.inputs[0]) + if len(query.shape) != 4: + raise NotImplementedError(f"rel_pos_bias expects rank-4 query input, got {query.shape}") + relative_key = _tensor(env, node.inputs[1]) + if len(relative_key.shape) != 4: + raise NotImplementedError(f"rel_pos_bias expects rank-4 relative_key input, got {relative_key.shape}") + if int(query.shape[2]) == int(relative_key.shape[2]) and int(query.shape[3]) == int(relative_key.shape[3]): + pass + elif int(query.shape[1]) == int(relative_key.shape[2]) and int(query.shape[3]) == int(relative_key.shape[3]): + query = g.permute(query, (0, 2, 1, 3)) + else: + raise NotImplementedError( + "rel_pos_bias query/relative_key layout mismatch: " + f"query={query.shape}, relative_key={relative_key.shape}" + ) + return [g.rel_pos_bias(query, relative_key, float(node.attrs.get("scale", 1.0)))] + + if op in {"gated_deltanet_prefill", "gated_deltanet_decode"}: + x = _tensor(env, node.inputs[0]) + qkv_weight = _tensor(env, node.inputs[1]) + a_weight = _tensor(env, node.inputs[2]) + b_weight = _tensor(env, node.inputs[3]) + norm_weight = _tensor(env, node.inputs[4]) + + input_index = 5 + z_weight = None + if bool(node.attrs.get("has_z", False)): + z_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + dt_bias = None + if bool(node.attrs.get("has_dt_bias", False)): + dt_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + a_log = None + if bool(node.attrs.get("has_a_log", False)): + a_log = _tensor(env, node.inputs[input_index]) + input_index += 1 + conv_weight = None + if bool(node.attrs.get("has_conv", False)): + conv_weight = _tensor(env, node.inputs[input_index]) + + if len(x.shape) != 3: + raise NotImplementedError(f"{op} currently expects rank-3 normalized input, got {x.shape}") + + batch_size, seq_len, hidden_dim = (int(dim) for dim in x.shape) + if batch_size != 1: + raise NotImplementedError(f"{op} currently supports batch size 1, got {x.shape}") + + num_k_heads = int(node.attrs["num_k_heads"]) + num_v_heads = int(node.attrs["num_v_heads"]) + key_dim = int(node.attrs["key_dim"]) + value_dim = int(node.attrs["value_dim"]) + eps = float(node.attrs.get("eps", 1e-6)) + chunk_size = int(node.attrs.get("chunk_size", 64)) + + mixed_qkv = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + qkv_weight, + pretransposed_rhs=True, + ) + mixed_qkv_dim = int(qkv_weight.shape[0]) + mixed_qkv = g.reshape(mixed_qkv, (batch_size, seq_len, mixed_qkv_dim)) + if conv_weight is not None: + kernel_size = int(conv_weight.shape[2]) + mixed_qkv = g.conv1d_causal(mixed_qkv, conv_weight, kernel_size=kernel_size, dilation=1) + mixed_qkv = g.silu(mixed_qkv) + + q_proj_dim = num_k_heads * key_dim + v_proj_dim = num_v_heads * value_dim + k_proj_dim = num_k_heads * key_dim + q_proj = g.slice(mixed_qkv, axis=2, start=0, length=q_proj_dim) + k_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim, length=k_proj_dim) + v_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim + k_proj_dim, length=v_proj_dim) + + q_4d = g.reshape(q_proj, (batch_size, seq_len, num_k_heads, key_dim)) + k_4d = g.reshape(k_proj, (batch_size, seq_len, num_k_heads, key_dim)) + v_4d = g.reshape(v_proj, (batch_size, seq_len, num_v_heads, value_dim)) + + q_norm = g.sum(g.multiply(q_4d, q_4d), axis=3) + q_norm = g.scalar_sqrt(g.scalar_add(q_norm, eps)) + q_norm = g.reshape(q_norm, (batch_size, seq_len, num_k_heads, 1)) + q_4d = g.divide(q_4d, q_norm) + + k_norm = g.sum(g.multiply(k_4d, k_4d), axis=3) + k_norm = g.scalar_sqrt(g.scalar_add(k_norm, eps)) + k_norm = g.reshape(k_norm, (batch_size, seq_len, num_k_heads, 1)) + k_4d = g.divide(k_4d, k_norm) + + a_logits = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + a_weight, + pretransposed_rhs=True, + ) + a_logits = g.reshape(a_logits, (batch_size, seq_len, int(a_weight.shape[0]))) + b_logits = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + b_weight, + pretransposed_rhs=True, + ) + b_logits = g.reshape(b_logits, (batch_size, seq_len, int(b_weight.shape[0]))) + + if dt_bias is not None: + dt_bias_2d = g.reshape(dt_bias, (1, int(dt_bias.shape[0]))) + a_logits, dt_bias_2d = _legalize_elementwise_binary_inputs(g, a_logits, dt_bias_2d) + a_logits = g.add(a_logits, dt_bias_2d) + a_softplus = _lower_softplus(g, a_logits) + + if a_log is not None: + a_log_2d = g.reshape(a_log, (1, int(a_log.shape[0]))) + neg_exp_a = g.scalar_multiply(g.scalar_exp(a_log_2d), -1.0) + neg_exp_a, a_softplus = _legalize_elementwise_binary_inputs(g, neg_exp_a, a_softplus) + gate_log = g.multiply(neg_exp_a, a_softplus) + else: + gate_log = g.scalar_multiply(a_softplus, -1.0) + beta = g.sigmoid(b_logits) + + initial_state = _materialize_constant_tensor( + g, + torch.zeros((batch_size, key_dim, num_v_heads, value_dim), dtype=torch.float16), + ) + + if op == "gated_deltanet_decode": + deltanet_out = g.gated_deltanet_decode(q_4d, k_4d, v_4d, gate_log, beta, initial_state, 0.0) + else: + deltanet_out = g.gated_deltanet_prefill(q_4d, k_4d, v_4d, gate_log, beta, initial_state, chunk_size, 0.0) + + y_4d = g.slice(deltanet_out, axis=1, start=0, length=seq_len) + y_2d = g.reshape(y_4d, (seq_len * num_v_heads, value_dim)) + + if z_weight is not None: + z_proj = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + z_weight, + pretransposed_rhs=True, + ) + z_proj = g.reshape(z_proj, (seq_len * num_v_heads, value_dim)) + y_2d = g.multiply( + g.rms_norm( + _ensure_runtime_norm_tensor(g, y_2d), + _ensure_runtime_norm_tensor(g, norm_weight), + eps=eps, + ), + g.silu(_ensure_runtime_activation_tensor(g, z_proj)), + ) + + return [g.reshape(y_2d, (batch_size, seq_len, num_v_heads * value_dim))] + + if op == "group_norm": + return [ + g.group_norm( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + _tensor(env, node.inputs[2]), + num_groups=int(node.attrs["num_groups"]), + eps=float(node.attrs["eps"]), + ) + ] + + if op == "batch_norm": + axis = int(node.attrs.get("axis", 1)) + return [ + g.batch_norm( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + _tensor(env, node.inputs[2]), + _tensor(env, node.inputs[3]), + _tensor(env, node.inputs[4]), + axis=axis, + eps=float(node.attrs["eps"]), + ) + ] + + if op == "identity": + return [env[node.inputs[0]]] + + if op == "contiguous": + return [_tensor(env, node.inputs[0])] + + if op == "advanced_index": + lowered = _try_lower_advanced_index(g, node, env, ir) + if lowered is not None: + return [lowered] + raise NotImplementedError(f"unsupported advanced_index pattern for node {node.id}") + + if op in {"masked_scatter", "aten.masked_scatter.default"}: + base = _tensor(env, node.inputs[0]) + mask = _tensor(env, node.inputs[1]) + source = _tensor(env, node.inputs[2]) + if source.dtype != base.dtype: + source = _ensure_tensor_dtype(g, source, base.dtype) + return [g.masked_scatter(base, mask, source)] + + if op == "masked_fill": + base = _tensor(env, node.inputs[0]) + mask = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + inverse_mask = g.scalar_add(g.scalar_multiply(mask, -1.0), 1.0) + fill_value = _clamp_scalar_for_dtype(float(node.attrs["value"]), base.dtype) + fill_term = g.scalar_multiply(_ensure_tensor_dtype(g, mask, base.dtype), fill_value) + base_term = _lower_binary_op(g, base, _ensure_tensor_dtype(g, inverse_mask, base.dtype), "multiply") + return [_lower_binary_op(g, fill_term, base_term, "add")] + + if op == "getitem": + source = env[node.inputs[0]] + index = int(node.attrs["index"]) + if isinstance(source, (tuple, list)): + return [source[index]] + if isinstance(source, Tensor): + normalized_index = _normalize_index(index, int(source.shape[0])) + return [g.index(source, normalized_index, axis=0)] + raise NotImplementedError(f"getitem source is not tuple/list for node {node.id}") + + raise NotImplementedError(f"unsupported IR op in lowering: {op}") + + +def _tensor(env: dict[str, Any], value_id: str) -> Tensor: + try: + value = env[value_id] + except KeyError as exc: + raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc + if not isinstance(value, Tensor): + raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") + return value + + +def _attention_tensor(env: dict[str, Any], value_id: str) -> Tensor: + try: + value = env[value_id] + except KeyError as exc: + raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc + if isinstance(value, BroadcastAlias): + if value.kind != "gqa_repeat_kv": + raise TypeError(f"unsupported broadcast alias for attention input {value_id}: {value.kind}") + return value.tensor + if not isinstance(value, Tensor): + raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") + return value + + +def _resolve_attention_scale(node: IRNode, query: Tensor) -> float: + scale = float(node.attrs.get("scale", 0.0) or 0.0) + if scale > 0.0: + return scale + if len(query.shape) >= 1 and int(query.shape[-1]) > 0: + return float(int(query.shape[-1]) ** -0.5) + return 1.0 + + +def _normalize_attention_mask_for_cactus(g: Graph, mask: Tensor, query: Tensor) -> Tensor: + """Convert broadcastable HF masks into the native Cactus attention contract.""" + + mask_shape = tuple(int(dim) for dim in mask.shape) + query_shape = tuple(int(dim) for dim in query.shape) + if len(query_shape) != 4: + return mask + + batch_size, seq_len, num_heads, _ = query_shape + if len(mask_shape) == 2 and batch_size == 1 and mask_shape[0] == seq_len: + return g.reshape(mask, (1, mask_shape[0], mask_shape[1])) + + if len(mask_shape) == 4: + if ( + mask_shape[0] == batch_size + and mask_shape[1] == 1 + and mask_shape[2] == seq_len + ): + return g.reshape(mask, (mask_shape[0], mask_shape[2], mask_shape[3])) + if ( + mask_shape[0] == batch_size + and mask_shape[1] == num_heads + and mask_shape[2] == seq_len + ): + return mask + + return mask + + +def _lower_projected_attention_tensor( + g: Graph, + hidden: Tensor, + weight: Tensor, + bias: Tensor | None, + target_shape: tuple[int, ...], +) -> Tensor: + if len(target_shape) != 4: + raise NotImplementedError(f"projected attention tensor expects rank-4 target shape, got {target_shape}") + + linear_input = _flatten_to_2d_for_linear(g, hidden) + out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) + if bias is not None: + out = g.add(out, bias) + return g.reshape(out, target_shape) + + +def _normalize_attention_add_tensor(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: + if tuple(tensor.shape) == target_shape: + return tensor + + if len(tensor.shape) != 4 or len(target_shape) != 4: + raise NotImplementedError( + f"unsupported attention add tensor shape {tuple(tensor.shape)} for target {target_shape}" + ) + + batch, seq_len, heads, head_dim = target_shape + shape = tuple(int(v) for v in tensor.shape) + + if shape[0] in (1, batch) and shape[1] == heads and shape[2] in (1, seq_len) and shape[3] == head_dim: + return g.permute(tensor, (0, 2, 1, 3)) + if shape[0] in (1, batch) and shape[1] in (1, seq_len) and shape[2] == heads and shape[3] == head_dim: + return tensor + + raise NotImplementedError( + f"unsupported attention add tensor shape {shape} for target {target_shape}" + ) + + +def _rope_input_is_bhsd(ir: IRGraph, value_id: str) -> bool: + value = ir.values.get(value_id) + if value is None or value.producer is None: + return False + node = ir.nodes.get(value.producer) + if node is None: + return False + if node.op == "permute": + permutation = tuple(int(dim) for dim in node.attrs.get("permutation", ())) + return permutation == (0, 2, 1, 3) + if node.op == "transpose": + return int(node.attrs.get("dim0", -1)) == 1 and int(node.attrs.get("dim1", -1)) == 2 + return False + + +def _lower_softplus(g: Graph, x: Tensor) -> Tensor: + # Stable fp16 softplus: relu(x) + log(1 + exp(-abs(x))). + abs_x = g.abs(x) + neg_abs_x = g.scalar_multiply(abs_x, -1.0) + exp_term = g.scalar_exp(neg_abs_x) + log_term = g.scalar_log(g.scalar_add(exp_term, 1.0)) + return g.add(g.relu(x), log_term) + + +def _normalize_dim(dim: int, rank: int) -> int: + if dim < 0: + dim += rank + return dim + + +def _normalize_reduction_axes(axis: Any, rank: int) -> tuple[int, ...]: + if axis is None: + return tuple(range(rank)) + if isinstance(axis, int): + return (_normalize_dim(axis, rank),) + if isinstance(axis, (list, tuple)): + normalized: list[int] = [] + seen: set[int] = set() + for raw_axis in axis: + if not isinstance(raw_axis, int): + raise NotImplementedError("reduction axes must be integers") + reduced_axis = _normalize_dim(raw_axis, rank) + if reduced_axis in seen: + continue + seen.add(reduced_axis) + normalized.append(reduced_axis) + if normalized: + return tuple(sorted(normalized)) + raise NotImplementedError("reduction axes must be an int, a tuple/list of ints, or None") + + +def _lower_reduction(g: Graph, op: str, x: Tensor, *, axes: tuple[int, ...]) -> Tensor: + if not axes: + return x + + fn = getattr(g, op) + if len(axes) == 1: + return fn(x, axes[0]) + + if op == "variance": + flattened = g.flatten(x, start_dim=axes[0], end_dim=axes[-1]) + collapsed_axes = set(axes) + expected_axes = set(range(axes[0], axes[-1] + 1)) + if collapsed_axes != expected_axes: + raise NotImplementedError("variance currently requires contiguous multi-axis reductions") + return fn(flattened, axes[0]) + + reduced = x + for axis in sorted(axes, reverse=True): + reduced = fn(reduced, axis) + return reduced + + +def _normalize_index(index: int, dim_size: int) -> int: + if index < 0: + index += dim_size + return index + + +def _static_shape(shape: Any) -> tuple[int, ...] | None: + if not isinstance(shape, tuple): + return None + dims: list[int] = [] + for dim in shape: + try: + dims.append(int(dim)) + except Exception: + return None + return tuple(dims) + + +def _constant_tensor_from_ir(ir: IRGraph, value_id: str) -> torch.Tensor | None: + const = ir.constants.get(value_id) + if isinstance(const, torch.nn.Parameter): + const = const.detach() + if isinstance(const, torch.Tensor): + return const.detach().cpu() + return None + + +def _try_lower_identity_broadcast_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + if len(node.inputs) != 3: + return None + + source = _tensor(env, node.inputs[0]) + source_shape = tuple(int(dim) for dim in source.shape) + if len(source_shape) != 2 or int(source_shape[0]) != 1: + return None + + output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + if output_shape is None: + return None + + batch_index = _constant_tensor_from_ir(ir, node.inputs[1]) + token_index = _constant_tensor_from_ir(ir, node.inputs[2]) + if batch_index is None or token_index is None: + return None + + try: + batch_index = torch.broadcast_to(batch_index.to(torch.int64), output_shape) + token_index = torch.broadcast_to(token_index.to(torch.int64), output_shape) + except RuntimeError: + return None + + if torch.count_nonzero(batch_index).item() != 0: + return None + + flat_index = token_index.reshape(-1) + expected = torch.arange(int(source_shape[1]), dtype=torch.int64) + if flat_index.numel() != expected.numel() or not torch.equal(flat_index, expected): + return None + + return g.reshape(source, output_shape) + + +def _try_lower_prefix_mask_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + if len(node.inputs) != 2: + return None + + source = _tensor(env, node.inputs[0]) + mask = _tensor(env, node.inputs[1]) + source_shape = tuple(int(dim) for dim in source.shape) + mask_shape = tuple(int(dim) for dim in mask.shape) + if not mask_shape or len(mask_shape) > len(source_shape): + return None + if source_shape[: len(mask_shape)] != mask_shape: + return None + + output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + if output_shape: + prefix_elems = 1 + for dim in mask_shape: + prefix_elems *= int(dim) + trailing_shape = source_shape[len(mask_shape) :] + flattened_shape = (prefix_elems, *trailing_shape) + flattened = source if source_shape == flattened_shape else g.reshape(source, flattened_shape) + expected_prefix = int(output_shape[0]) + if 0 <= expected_prefix <= prefix_elems: + selected = g.slice(flattened, axis=0, start=0, length=expected_prefix) + if tuple(int(dim) for dim in selected.shape) != tuple(int(dim) for dim in output_shape): + selected = g.reshape(selected, output_shape) + return selected + + selected = g.masked_select_prefix(source, mask) + if output_shape is None or not output_shape: + return selected + if len(output_shape) != len(selected.shape): + return selected + expected_prefix = int(output_shape[0]) + if expected_prefix < int(selected.shape[0]): + selected = g.slice(selected, axis=0, start=0, length=expected_prefix) + return selected + + +def _try_lower_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + lowered = _try_lower_identity_broadcast_advanced_index(g, node, env, ir) + if lowered is not None: + return lowered + return _try_lower_prefix_mask_advanced_index(g, node, env, ir) + + +def _normalize_slice_end(end: int, dim_size: int) -> int: + if end < 0: + end += dim_size + return max(0, min(end, dim_size)) + + +def _legalize_elementwise_binary_inputs(g: Graph, lhs: Tensor, rhs: Tensor) -> tuple[Tensor, Tensor]: + if lhs.shape == rhs.shape: + return lhs, rhs + + lhs_rank = len(lhs.shape) + rhs_rank = len(rhs.shape) + + if lhs_rank > rhs_rank: + rhs = _reshape_for_trailing_broadcast(g, rhs, lhs.shape) + elif rhs_rank > lhs_rank: + lhs = _reshape_for_trailing_broadcast(g, lhs, rhs.shape) + + if lhs.shape == rhs.shape: + return lhs, rhs + + return lhs, rhs + + +def _is_scalar_like(value: Any) -> bool: + return isinstance(value, (int, float, bool)) + + +def _lower_binary_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: + if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): + lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) + target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 + lhs = _ensure_tensor_dtype(g, lhs, target_dtype) + rhs = _ensure_tensor_dtype(g, rhs, target_dtype) + if op == "add": + return g.add(lhs, rhs) + if op == "subtract": + return g.subtract(lhs, rhs) + if op == "multiply": + return g.multiply(lhs, rhs) + if op == "divide": + return g.divide(lhs, rhs) + raise NotImplementedError(f"unsupported binary op: {op}") + + if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): + lhs_value = _ensure_scalar_math_tensor(g, lhs_value) + scalar = float(rhs_value) + if op == "add": + return g.scalar_add(lhs_value, scalar) + if op == "subtract": + return g.scalar_subtract(lhs_value, scalar) + if op == "multiply": + return g.scalar_multiply(lhs_value, scalar) + if op == "divide": + return g.scalar_divide(lhs_value, scalar) + raise NotImplementedError(f"unsupported binary op: {op}") + + if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): + rhs_value = _ensure_scalar_math_tensor(g, rhs_value) + scalar = float(lhs_value) + if op == "add": + return g.scalar_add(rhs_value, scalar) + if op == "subtract": + return g.scalar_add(g.scalar_multiply(rhs_value, -1.0), scalar) + if op == "multiply": + return g.scalar_multiply(rhs_value, scalar) + if op == "divide": + raise NotImplementedError("scalar/tensor divide is not directly supported by Cactus graph ops") + raise NotImplementedError(f"unsupported binary op: {op}") + + raise TypeError( + f"unsupported lowered operand types for {op}: " + f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" + ) + + +def _lower_compare_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: + if op == "equal": + not_equal = _lower_compare_op(g, lhs_value, rhs_value, "not_equal") + return g.scalar_not_equal(not_equal, 1.0) + + if op == "greater": + delta = _lower_binary_op(g, lhs_value, rhs_value, "subtract") + delta = _ensure_fp16_tensor(g, delta) + return g.scalar_not_equal(g.relu(delta), 0.0) + + if op == "less_equal": + greater = _lower_compare_op(g, lhs_value, rhs_value, "greater") + return g.scalar_not_equal(greater, 1.0) + + if op != "not_equal": + raise NotImplementedError(f"unsupported compare op: {op}") + + if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): + lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) + target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 + lhs = _ensure_tensor_dtype(g, lhs, target_dtype) + rhs = _ensure_tensor_dtype(g, rhs, target_dtype) + return g.not_equal(lhs, rhs) + + if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): + lhs_value = _ensure_tensor_dtype( + g, + lhs_value, + Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + return g.scalar_not_equal(lhs_value, float(rhs_value)) + + if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): + rhs_value = _ensure_tensor_dtype( + g, + rhs_value, + Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + return g.scalar_not_equal(rhs_value, float(lhs_value)) + + raise TypeError( + f"unsupported lowered operand types for {op}: " + f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" + ) + + +def _reshape_for_trailing_broadcast(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: + tensor_shape = tuple(tensor.shape) + target_rank = len(target_shape) + tensor_rank = len(tensor_shape) + + if tensor_rank > target_rank: + return tensor + + padded_shape = (1,) * (target_rank - tensor_rank) + tensor_shape + + # Only legalize cases that are valid trailing broadcasts, e.g.: + # (H,) -> (1, H), (1, 1, H), etc. + for src_dim, tgt_dim in zip(padded_shape, target_shape): + if src_dim != 1 and src_dim != tgt_dim: + return tensor + + if padded_shape == tensor_shape: + return tensor + + return g.reshape(tensor, padded_shape) + + +def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_runtime_activation_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + if os.environ.get("CACTUS_TRANSPILER_DEBUG_ACTIVATION_CASTS") == "1": + print( + "[transpile:activation-cast] " + f"from_dtype={int(tensor.dtype)} " + f"to_dtype={int(Graph.FP16)} " + f"node_id={int(tensor.id)}" + ) + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_runtime_norm_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + if os.environ.get("CACTUS_TRANSPILER_DEBUG_NORM_CASTS") == "1": + print( + "[transpile:norm-cast] " + f"from_dtype={int(tensor.dtype)} " + f"to_dtype={int(Graph.FP16)} " + f"node_id={int(tensor.id)}" + ) + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_scalar_math_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_tensor_dtype(g: Graph, tensor: Tensor, dtype: int) -> Tensor: + if tensor.dtype == dtype: + return tensor + return g.precision_cast(tensor, dtype) + + +def _cat_with_legalized_dtype(g: Graph, tensors: list[Tensor], *, axis: int, dtype: int | None) -> Tensor: + if not tensors: + raise NotImplementedError("cat requires at least one tensor") + if dtype is None: + dtypes = {tensor.dtype for tensor in tensors} + dtype = tensors[0].dtype if len(dtypes) == 1 else Graph.FP16 + legalized = [_ensure_tensor_dtype(g, tensor, dtype) for tensor in tensors] + return g.cat(legalized, axis=axis) + + +def _invert_permutation(permutation: tuple[int, ...]) -> tuple[int, ...]: + inverse = [0] * len(permutation) + for index, source_axis in enumerate(permutation): + inverse[int(source_axis)] = index + return tuple(inverse) + + +def _lower_static_strided_slice_via_gather( + g: Graph, + x: Tensor, + *, + axis: int, + indices: list[int], +) -> Tensor: + if not indices: + return g.slice(x, axis=axis, start=0, length=0) + + indices_tensor = _materialize_constant_tensor(g, torch.tensor(indices, dtype=torch.float32)) + if axis == 0: + return g.gather(x, indices_tensor) + + permutation = (axis, *[dim for dim in range(len(x.shape)) if dim != axis]) + transposed = g.permute(x, permutation) + gathered = g.gather(transposed, indices_tensor) + inverse_permutation = _invert_permutation(permutation) + return g.permute(gathered, inverse_permutation) + + +def _lower_where_branch_term(g: Graph, branch_value: Any, mask: Tensor, *, dtype: int | None) -> Tensor: + if dtype is not None: + mask = _ensure_tensor_dtype(g, mask, dtype) + if isinstance(branch_value, Tensor): + branch_mask = mask if dtype is None else _ensure_tensor_dtype(g, mask, branch_value.dtype) + return _lower_binary_op(g, branch_value, branch_mask, "multiply") + return g.scalar_multiply(mask, _clamp_scalar_for_dtype(float(branch_value), mask.dtype)) + + +def _clamp_scalar_for_dtype(value: float, dtype: int | None) -> float: + if dtype != Graph.FP16: + return value + if math.isnan(value): + return value + if value == math.inf: + return 65504.0 + if value == -math.inf: + return -65504.0 + return max(-65504.0, min(65504.0, value)) + + +def _lower_where_op(g: Graph, node: IRNode, env: dict[str, Any]) -> Tensor: + condition = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + false_mask = g.scalar_add(g.scalar_multiply(condition, -1.0), 1.0) + + input_index = 1 + if bool(node.attrs.get("true_is_scalar", False)): + true_value: Any = float(node.attrs["true_value"]) + else: + true_value = env[node.inputs[input_index]] + input_index += 1 + + if bool(node.attrs.get("false_is_scalar", False)): + false_value: Any = float(node.attrs["false_value"]) + else: + false_value = env[node.inputs[input_index]] + + result_dtype: int | None = None + if isinstance(true_value, Tensor): + result_dtype = true_value.dtype + elif isinstance(false_value, Tensor): + result_dtype = false_value.dtype + + true_term = _lower_where_branch_term(g, true_value, condition, dtype=result_dtype) + false_term = _lower_where_branch_term(g, false_value, false_mask, dtype=result_dtype) + return _lower_binary_op(g, true_term, false_term, "add") + + +def _flatten_to_2d_for_linear(g: Graph, tensor: Tensor) -> Tensor: + shape = tuple(tensor.shape) + if len(shape) <= 2: + return tensor + leading = 1 + for dim in shape[:-1]: + leading *= int(dim) + return g.view(tensor, (leading, int(shape[-1]))) + + +def _broadcast_batch_shape(lhs_batch: tuple[int, ...], rhs_batch: tuple[int, ...]) -> tuple[int, ...] | None: + rank = max(len(lhs_batch), len(rhs_batch)) + lhs_padded = (1,) * (rank - len(lhs_batch)) + lhs_batch + rhs_padded = (1,) * (rank - len(rhs_batch)) + rhs_batch + result: list[int] = [] + for lhs_dim, rhs_dim in zip(lhs_padded, rhs_padded): + if lhs_dim == rhs_dim: + result.append(int(lhs_dim)) + elif lhs_dim == 1: + result.append(int(rhs_dim)) + elif rhs_dim == 1: + result.append(int(lhs_dim)) + else: + return None + return tuple(result) + + +def _project_broadcast_coord(coord: tuple[int, ...], operand_batch: tuple[int, ...]) -> tuple[int, ...]: + if not operand_batch: + return () + offset = len(coord) - len(operand_batch) + projected: list[int] = [] + for axis, dim in enumerate(operand_batch): + value = coord[offset + axis] + projected.append(0 if int(dim) == 1 else int(value)) + return tuple(projected) + + +def _index_batch_prefix(g: Graph, tensor: Tensor, coord: tuple[int, ...]) -> Tensor: + for index_value in coord: + tensor = g.index(tensor, index_value, axis=0) + return tensor + + +def _lower_static_batched_matmul( + g: Graph, + lhs: Tensor, + rhs: Tensor, + *, + output_dtype: int | None = None, +) -> Tensor | None: + lhs_shape = tuple(int(dim) for dim in lhs.shape) + rhs_shape = tuple(int(dim) for dim in rhs.shape) + if len(lhs_shape) < 3 or len(rhs_shape) < 3: + return None + if lhs_shape[-1] != rhs_shape[-2]: + return None + + lhs_batch = lhs_shape[:-2] + rhs_batch = rhs_shape[:-2] + batch_shape = _broadcast_batch_shape(lhs_batch, rhs_batch) + if batch_shape is None: + return None + + output_matrix_shape = (int(lhs_shape[-2]), int(rhs_shape[-1])) + rank = len(batch_shape) + + def _build(dim: int, coord_prefix: tuple[int, ...]) -> Tensor: + if dim == rank: + lhs_coord = _project_broadcast_coord(coord_prefix, lhs_batch) + rhs_coord = _project_broadcast_coord(coord_prefix, rhs_batch) + lhs_slice = _index_batch_prefix(g, lhs, lhs_coord) + rhs_slice = _index_batch_prefix(g, rhs, rhs_coord) + out = _matmul_with_quantized_rhs_legalization( + g, + lhs_slice, + rhs_slice, + output_dtype=output_dtype, + ) + return g.reshape(out, (1,) * rank + output_matrix_shape) + + pieces = [_build(dim + 1, coord_prefix + (index_value,)) for index_value in range(int(batch_shape[dim]))] + if len(pieces) == 1: + return pieces[0] + return g.cat(pieces, axis=dim) + + return _build(0, ()) + + +def _should_lower_gemma4_decoder_attention_without_kernel(ir: IRGraph, node: IRNode) -> bool: + family = str(ir.meta.get("adapter_family") or ir.meta.get("family") or "").strip().lower() + component = str(ir.meta.get("component", "") or "").strip().lower() + if family != "gemma4" or component != "decoder": + return False + return node.op in {"attention", "scaled_dot_product_attention"} + + +def _lower_gemma4_decoder_attention_without_kernel( + g: Graph, + ir: IRGraph, + env: dict[str, Any], + node: IRNode, +) -> Tensor: + query = _attention_tensor(env, node.inputs[0]) + key = _attention_tensor(env, node.inputs[1]) + value = _attention_tensor(env, node.inputs[2]) + + key_transposed = g.permute(key, (0, 1, 3, 2)) + scores = _lower_static_batched_matmul(g, query, key_transposed, output_dtype=Graph.FP16) + if scores is None: + raise NotImplementedError("Gemma4 decoder attention requires static batched matmul support") + + scale = float(node.attrs.get("scale", 1.0)) + if scale != 1.0: + scores = g.scalar_multiply(_ensure_scalar_math_tensor(g, scores), scale) + + additive_mask: Tensor | None = None + if len(node.inputs) > 3: + mask_tensor = _ensure_fp16_tensor(g, _tensor(env, node.inputs[3])) + if bool(node.attrs.get("additive_mask", False)): + additive_mask = mask_tensor + else: + additive_mask = _lower_binary_op(g, mask_tensor, 1.0, "subtract") + additive_mask = g.scalar_multiply(_ensure_scalar_math_tensor(g, additive_mask), 1.0e4) + else: + query_shape = tuple(int(dim) for dim in query.shape) + key_shape = tuple(int(dim) for dim in key.shape) + if len(query_shape) >= 4 and len(key_shape) >= 4: + query_seq = int(query_shape[-2]) + key_seq = int(key_shape[-2]) + additive_mask = _materialize_gemma4_attention_mask( + g, + query_seq=query_seq, + key_seq=key_seq, + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + dtype=torch.float16, + ) + + if additive_mask is not None: + scores = _lower_binary_op(g, scores, additive_mask, "add") + + probs = g.softmax(scores, axis=-1) + probs = _ensure_tensor_dtype(g, probs, value.dtype) + out = _lower_static_batched_matmul(g, probs, value, output_dtype=value.dtype) + if out is None: + raise NotImplementedError("Gemma4 decoder attention output requires static batched matmul support") + + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else out.dtype + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return out + + +def _materialize_gemma4_attention_mask( + g: Graph, + *, + query_seq: int, + key_seq: int, + is_causal: bool, + window_size: int, + dtype: torch.dtype, +) -> Tensor | None: + if query_seq <= 0 or key_seq <= 0: + return None + if not is_causal and window_size <= 0: + return None + + q_index = np.arange(query_seq, dtype=np.int32)[:, None] + k_index = np.arange(key_seq, dtype=np.int32)[None, :] + allowed = np.ones((query_seq, key_seq), dtype=np.bool_) + + if is_causal: + allowed &= k_index <= q_index + if window_size > 0 and window_size < key_seq: + allowed &= k_index >= (q_index - (window_size - 1)) + + mask = np.where(allowed, 0.0, -1.0e4).astype(np.float16) + tensor = torch.from_numpy(mask.reshape(1, 1, query_seq, key_seq)).to(dtype=dtype) + return _materialize_constant_tensor(g, tensor) + + +def _legalize_matmul_inputs( + g: Graph, + lhs: Tensor, + rhs: Tensor, + node: IRNode, +) -> tuple[Tensor, Tensor, tuple[int, ...]] | None: + lhs_shape = tuple(lhs.shape) + rhs_shape = tuple(rhs.shape) + if len(lhs_shape) <= 2 and len(rhs_shape) <= 2: + return None + + if len(lhs_shape) > 2 and len(rhs_shape) == 2 and lhs_shape[-1] == rhs_shape[0]: + leading = 1 + for dim in lhs_shape[:-1]: + leading *= int(dim) + lhs_2d = g.reshape(lhs, (leading, int(lhs_shape[-1]))) + output_shape = lhs_shape[:-1] + (int(rhs_shape[1]),) + return lhs_2d, rhs, output_shape + + # Cactus matmul is 2D-only. Legalize the narrow case where both operands have + # only singleton leading dims, e.g. rotary helper matmuls: + # (1, M, K) @ (1, K, N) -> reshape to (M, K) @ (K, N) -> reshape back. + if any(dim != 1 for dim in lhs_shape[:-2]): + return None + if any(dim != 1 for dim in rhs_shape[:-2]): + return None + + lhs_2d_shape = lhs_shape[-2:] + rhs_2d_shape = rhs_shape[-2:] + if lhs_2d_shape[-1] != rhs_2d_shape[0]: + return None + + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + output_shape = lhs_shape[:-2] + (lhs_2d_shape[0], rhs_2d_shape[1]) + output_shape = tuple(int(v) for v in output_shape) + + lhs_2d = g.reshape(lhs, lhs_2d_shape) + rhs_2d = g.reshape(rhs, rhs_2d_shape) + return lhs_2d, rhs_2d, output_shape + + +def _resolve_expand_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: + if len(requested_shape) < len(input_shape): + raise NotImplementedError(f"expand cannot reduce rank: {input_shape} -> {requested_shape}") + + padded_input = (1,) * (len(requested_shape) - len(input_shape)) + input_shape + resolved: list[int] = [] + for in_dim, req_dim in zip(padded_input, requested_shape): + if req_dim == -1: + resolved.append(in_dim) + continue + if req_dim < -1: + raise NotImplementedError(f"invalid expand dimension: {req_dim}") + resolved.append(int(req_dim)) + return tuple(resolved) + + +def _resolve_reshape_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: + resolved = [int(v) for v in requested_shape] + unknown_indices = [idx for idx, dim in enumerate(resolved) if dim == -1] + if len(unknown_indices) > 1: + raise NotImplementedError(f"reshape with multiple inferred dimensions is unsupported: {requested_shape}") + if not unknown_indices: + return tuple(resolved) + + input_elements = 1 + for dim in input_shape: + input_elements *= int(dim) + + known_elements = 1 + for dim in resolved: + if dim != -1: + known_elements *= int(dim) + + if known_elements == 0 or input_elements % known_elements != 0: + raise NotImplementedError(f"cannot infer reshape target {requested_shape} from input shape {input_shape}") + + resolved[unknown_indices[0]] = input_elements // known_elements + return tuple(resolved) + + +def _match_gqa_expand_alias(ir: IRGraph, expand_node: IRNode) -> str | None: + target_shape = tuple(int(v) for v in expand_node.attrs["shape"]) + if len(target_shape) != 5: + return None + + current_value_id = expand_node.inputs[0] + base_value_id: str | None = None + + while True: + producer_id = ir.values[current_value_id].producer + if producer_id is None: + return None + producer = ir.nodes[producer_id] + if producer.op == "slice": + axis = int(producer.attrs.get("axis", -1)) + input_shape = ir.values[producer.inputs[0]].shape + if input_shape is None: + return None + normalized_axis = _normalize_dim(axis, len(input_shape)) + start = int(producer.attrs.get("start", 0)) + end = int(producer.attrs.get("end", input_shape[normalized_axis])) + if start != 0 or end < input_shape[normalized_axis]: + return None + current_value_id = producer.inputs[0] + continue + if producer.op == "unsqueeze": + base_value_id = producer.inputs[0] + base_shape = ir.values[base_value_id].shape + if base_shape is None: + return None + unsqueezed_dim = _normalize_dim(int(producer.attrs.get("dim", 0)), len(base_shape) + 1) + if unsqueezed_dim != 2: + return None + break + if producer.op in {"reshape", "view"}: + base_value_id = producer.inputs[0] + base_shape = ir.values[base_value_id].shape + view_shape = ir.values[current_value_id].shape + if base_shape is None or view_shape is None: + return None + if len(base_shape) != 4 or len(view_shape) != 5: + return None + if tuple(int(v) for v in view_shape) != ( + int(base_shape[0]), + int(base_shape[1]), + 1, + int(base_shape[2]), + int(base_shape[3]), + ): + return None + break + return None + + if base_value_id is None: + return None + base_shape = ir.values[base_value_id].shape + if base_shape is None or len(base_shape) != 4: + return None + + expected_target = ( + int(base_shape[0]), + int(base_shape[1]), + int(target_shape[2]), + int(base_shape[2]), + int(base_shape[3]), + ) + if target_shape != expected_target: + return None + if int(target_shape[2]) <= 1: + return None + + return base_value_id + + +def _map_ir_dtype(dtype: str) -> int: + if dtype == "bf16": + return Graph.FP32 + if dtype == "fp16": + return Graph.FP16 + if dtype in ("fp32", "fp64"): + return Graph.FP32 + if dtype == "int8": + return Graph.INT8 + if dtype in ("int32", "int64", "bool"): + return Graph.FP32 + + raise NotImplementedError(f"unsupported IR dtype: {dtype}") + + +def _map_ir_or_torch_dtype(dtype: Any) -> int: + if dtype is None: + raise NotImplementedError("missing dtype for precision_cast") + if isinstance(dtype, str): + if dtype.startswith("torch."): + return _map_torch_dtype(getattr(torch, dtype.split(".", 1)[1])) + return _map_ir_dtype(dtype) + return _map_torch_dtype(dtype) + + +def _map_torch_dtype(dtype: Any) -> int: + if dtype == torch.bfloat16: + return Graph.FP32 + if dtype == torch.float16: + return Graph.FP16 + if dtype == torch.float32: + return Graph.FP32 + if dtype == torch.float64: + return Graph.FP32 + if dtype == torch.int8: + return Graph.INT8 + if dtype in (torch.int16, torch.int32, torch.int64, torch.bool): + return Graph.FP32 + raise NotImplementedError(f"unsupported torch dtype: {dtype}") + + +def _materialize_constant_tensor(g: Graph, tensor_value: torch.Tensor) -> Tensor: + graph_dtype = _map_torch_dtype(tensor_value.dtype) + materialized = tensor_value.detach().cpu() + if graph_dtype == Graph.FP32 and materialized.dtype not in (torch.float32, torch.float64): + materialized = materialized.to(torch.float32) + elif graph_dtype == Graph.FP16 and materialized.dtype != torch.float16: + materialized = materialized.to(torch.float16) + elif graph_dtype == Graph.INT8 and materialized.dtype != torch.int8: + materialized = materialized.to(torch.int8) + + tensor = g.input(shape=tuple(materialized.shape), dtype=graph_dtype) + g.set_input(tensor, materialized, dtype=graph_dtype) + materialized_constants = getattr(g, "_transpile_materialized_constants", None) + if isinstance(materialized_constants, list): + materialized_constants.append(tensor) + return tensor + + +def _materialize_constant_torch_dtype(dtype: Any) -> torch.dtype: + if dtype is None: + return torch.float32 + if isinstance(dtype, str): + if dtype.startswith("torch."): + return getattr(torch, dtype.split(".", 1)[1]) + if dtype in ("fp16", "bf16"): + return torch.float16 + if dtype in ("fp32", "fp64", "int16", "int32", "int64", "bool"): + return torch.float32 + if dtype == "int8": + return torch.int8 + if isinstance(dtype, torch.dtype): + return dtype + raise NotImplementedError(f"unsupported dtype for constant materialization: {dtype}") + + +def _torch_dtype_for_graph_dtype(dtype: int) -> torch.dtype: + if dtype == Graph.FP16: + return torch.float16 + if dtype == Graph.FP32: + return torch.float32 + if dtype == Graph.INT8: + return torch.int8 + raise NotImplementedError(f"unsupported graph dtype for constant materialization: {dtype}") diff --git a/python/src/transpile/model_adapters.py b/python/src/transpile/model_adapters.py new file mode 100644 index 000000000..c6aec13dd --- /dev/null +++ b/python/src/transpile/model_adapters.py @@ -0,0 +1,2477 @@ +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import inspect +import struct +import os +from pathlib import Path +from typing import Callable + +import numpy as np +import torch +import torch.nn.functional as F + +from src.transpile.component_pipeline import ComponentModuleSpec +from src.tensor_io import CACTUS_MAGIC +from src.tensor_io import align_offset + + +_GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE = 1.0 / 64.0 + + +@dataclass +class CanonicalizedModel: + module: torch.nn.Module + task: str + family: str + input_names: tuple[str, ...] = () + + +@dataclass(frozen=True) +class _Gemma4NativeMergeSegment: + kind: str + input_start: int + length: int + feature_start: int = 0 + + +@dataclass(frozen=True) +class _Gemma4NativeMergePlan: + segments: tuple[_Gemma4NativeMergeSegment, ...] + pli_token_ids: tuple[int, ...] + + +def _model_name_or_path(model: torch.nn.Module) -> str: + value = getattr(model, "name_or_path", None) + if isinstance(value, str) and value: + return value + config = getattr(model, "config", None) + value = getattr(config, "_name_or_path", None) + if isinstance(value, str) and value: + return value + return "" + + +def _transpile_graph_meta(model: torch.nn.Module, *, adapter_family: str, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: + return { + "adapter_family": adapter_family, + "adapter_type": adapter_type, + "model_name_or_path": _model_name_or_path(model), + "input_names": input_names, + } + + +def _extract_tensor_output(output: object, *, preferred_field: str | None = None) -> torch.Tensor: + if preferred_field is not None: + value = getattr(output, preferred_field, None) + if isinstance(value, torch.Tensor): + return value + + if isinstance(output, torch.Tensor): + return output + if isinstance(output, tuple) and output and isinstance(output[0], torch.Tensor): + return output[0] + + for field_name in ("last_hidden_state", "logits"): + value = getattr(output, field_name, None) + if isinstance(value, torch.Tensor): + return value + + raise TypeError(f"could not extract tensor output from {type(output).__name__}") + + +def _select_last_active_token(hidden_or_logits: torch.Tensor, attention_mask: torch.Tensor | None) -> torch.Tensor: + if attention_mask is None or hidden_or_logits.ndim < 3: + return hidden_or_logits[:, -1:, ...] + + if attention_mask.ndim != 2: + raise ValueError(f"expected 2D attention mask, got shape {tuple(attention_mask.shape)}") + if attention_mask.shape[1] != hidden_or_logits.shape[1]: + raise ValueError( + "attention mask / hidden sequence length mismatch: " + f"{tuple(attention_mask.shape)} vs {tuple(hidden_or_logits.shape)}" + ) + + trailing_zero = attention_mask[:, :1] - attention_mask[:, :1] + shifted_mask = torch.cat((attention_mask[:, 1:], trailing_zero), dim=1) + last_active_mask = torch.logical_and(attention_mask != 0, shifted_mask == 0) + expanded_mask = last_active_mask.to(dtype=hidden_or_logits.dtype) + for _ in range(hidden_or_logits.ndim - 2): + expanded_mask = expanded_mask.unsqueeze(-1) + return (hidden_or_logits * expanded_mask).sum(dim=1, keepdim=True) + + +def _module_floating_dtype(module: torch.nn.Module) -> torch.dtype | None: + for parameter in module.parameters(): + if parameter.is_floating_point(): + return parameter.dtype + for buffer in module.buffers(): + if buffer.is_floating_point(): + return buffer.dtype + return None + + +def _module_device(module: torch.nn.Module) -> torch.device | None: + for parameter in module.parameters(): + return parameter.device + for buffer in module.buffers(): + return buffer.device + return None + + +def _unique_modules(*candidates: object) -> tuple[torch.nn.Module, ...]: + modules: list[torch.nn.Module] = [] + seen: set[int] = set() + for candidate in candidates: + if not isinstance(candidate, torch.nn.Module): + continue + candidate_id = id(candidate) + if candidate_id in seen: + continue + seen.add(candidate_id) + modules.append(candidate) + return tuple(modules) + + +def _gemma4_vision_modules(multimodal_backbone: torch.nn.Module) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + getattr(multimodal_backbone, "vision_tower", None), + getattr(multimodal_backbone, "embed_vision", None), + ) + + +def _gemma4_audio_modules(multimodal_backbone: torch.nn.Module) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + getattr(multimodal_backbone, "audio_tower", None), + getattr(multimodal_backbone, "embed_audio", None), + ) + + +def _gemma4_text_config(multimodal_backbone: torch.nn.Module) -> object: + config = getattr(multimodal_backbone, "config", None) + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + return get_text_config() + except Exception: + pass + return config + + +def _gemma4_special_token_ids(multimodal_backbone: torch.nn.Module) -> tuple[int, int, int]: + config = getattr(multimodal_backbone, "config", None) + text_config = _gemma4_text_config(multimodal_backbone) + image_token_id = int(getattr(config, "image_token_id", 0) or 0) + audio_token_id = int(getattr(config, "audio_token_id", 0) or 0) + pad_token_id = getattr(text_config, "pad_token_id", None) + if pad_token_id is None: + pad_token_id = getattr(config, "pad_token_id", 0) + return image_token_id, audio_token_id, int(pad_token_id or 0) + + +@contextmanager +def _temporary_cpu_float32_modules(modules: tuple[torch.nn.Module, ...]): + promoted: list[tuple[torch.nn.Module, torch.dtype]] = [] + for module in modules: + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + promoted.append((module, dtype)) + module.to(dtype=torch.float32) + try: + yield + finally: + for module, dtype in reversed(promoted): + module.to(dtype=dtype) + + +def _torch_is_compiling() -> bool: + dynamo = getattr(torch, "_dynamo", None) + is_compiling = getattr(dynamo, "is_compiling", None) + if callable(is_compiling): + try: + return bool(is_compiling()) + except Exception: + return False + return False + + +def _gemma4_cpu_safe_text_mlp_enabled(layer: torch.nn.Module, hidden_states: torch.Tensor) -> bool: + if hidden_states.device.type != "cpu" or hidden_states.dtype != torch.float16: + return False + if bool(getattr(layer, "enable_moe_block", False)): + return False + mlp = getattr(layer, "mlp", None) + if mlp is None: + return False + for attr_name in ("gate_proj", "up_proj", "down_proj", "act_fn"): + if not hasattr(mlp, attr_name): + return False + return True + + +def _gemma4_cpu_safe_text_mlp_forward(mlp: torch.nn.Module, hidden_states: torch.Tensor) -> torch.Tensor: + gate = F.linear( + hidden_states, + mlp.gate_proj.weight, + mlp.gate_proj.bias, + ) + up = F.linear( + hidden_states, + mlp.up_proj.weight, + mlp.up_proj.bias, + ) + # Gemma4 text MLPs can overflow in FP16 at the gated product. + # Scaling one branch here keeps the product and down-projection finite. + # The following post-feedforward RMSNorm is scale-invariant, so this + # constant factor cancels back out in the normalized output. + activated = mlp.act_fn(gate) * _GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE + return F.linear( + activated * up, + mlp.down_proj.weight, + mlp.down_proj.bias, + ) + + +def _gemma4_text_attention_forward( + attn: torch.nn.Module, + hidden_states: torch.Tensor, + *, + position_embeddings: torch.Tensor | tuple[torch.Tensor, torch.Tensor] | None, + attention_mask: torch.Tensor | None, + position_ids: torch.LongTensor | None, + past_key_values: object | None, + use_cache: bool, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None, +) -> torch.Tensor: + if shared_kv_states is None: + attn_out, _ = attn( + hidden_states=hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + ) + return attn_out + + from transformers.models.gemma4.modeling_gemma4 import ALL_ATTENTION_FUNCTIONS # type: ignore + from transformers.models.gemma4.modeling_gemma4 import apply_rotary_pos_emb # type: ignore + from transformers.models.gemma4.modeling_gemma4 import eager_attention_forward # type: ignore + + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, attn.head_dim) + if position_embeddings is None: + raise ValueError("Gemma4 text attention requires position embeddings") + cos, sin = position_embeddings + + query_states = attn.q_proj(hidden_states).view(hidden_shape) + query_states = attn.q_norm(query_states) + query_states = apply_rotary_pos_emb(query_states, cos, sin, unsqueeze_dim=2) + query_states = query_states.transpose(1, 2) + + kv_shared_layer_index = getattr(attn, "kv_shared_layer_index", None) + if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_layer_index in shared_kv_states: + key_states, value_states = shared_kv_states[int(kv_shared_layer_index)] + key_states = key_states.to(query_states.device) + value_states = value_states.to(query_states.device) + else: + key_states = attn.k_proj(hidden_states).view(hidden_shape) + value_states = attn.v_proj(hidden_states).view(hidden_shape) if attn.v_proj is not None else key_states + + key_states = attn.k_norm(key_states) + key_states = apply_rotary_pos_emb(key_states, cos, sin, unsqueeze_dim=2) + key_states = key_states.transpose(1, 2) + + value_states = attn.v_norm(value_states) + value_states = value_states.transpose(1, 2) + + if bool(getattr(attn, "store_full_length_kv", False)): + shared_kv_states[int(attn.layer_idx)] = (key_states, value_states) + + attention_interface = eager_attention_forward + if getattr(attn.config, "_attn_implementation", "eager") != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[attn.config._attn_implementation] + + attn_output, _ = attention_interface( + attn, + query_states, + key_states, + value_states, + attention_mask, + dropout=attn.attention_dropout if attn.training else 0.0, + scaling=attn.scaling, + sliding_window=attn.sliding_window, + ) + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + return attn.o_proj(attn_output) + + +def _gemma4_text_decoder_layer_forward( + layer: torch.nn.Module, + hidden_states: torch.Tensor, + *, + position_embeddings: torch.Tensor | tuple[torch.Tensor, torch.Tensor] | None = None, + per_layer_input: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: object | None = None, + use_cache: bool = False, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, +) -> torch.Tensor: + if not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): + return layer( + hidden_states, + position_embeddings=position_embeddings, + per_layer_input=per_layer_input, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + ) + + residual = hidden_states + + hidden_states = layer.input_layernorm(hidden_states) + hidden_states = _gemma4_text_attention_forward( + layer.self_attn, + hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + shared_kv_states=shared_kv_states, + ) + hidden_states = layer.post_attention_layernorm(hidden_states) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = layer.pre_feedforward_layernorm(hidden_states) + mlp_out = _gemma4_cpu_safe_text_mlp_forward(layer.mlp, hidden_states) + hidden_states = layer.post_feedforward_layernorm(mlp_out) + hidden_states = residual.float() + hidden_states + hidden_states = hidden_states.to(dtype=residual.dtype) + + if getattr(layer, "hidden_size_per_layer_input", 0): + residual = hidden_states + hidden_states = layer.per_layer_input_gate(hidden_states) + hidden_states = layer.act_fn(hidden_states) + if per_layer_input is None: + raise ValueError("Gemma4 layer expected per_layer_input but none was provided") + hidden_states = torch.multiply(hidden_states, per_layer_input) + hidden_states = layer.per_layer_projection(hidden_states) + hidden_states = layer.post_per_layer_input_norm(hidden_states) + hidden_states = residual + hidden_states + + hidden_states = hidden_states * layer.layer_scalar + return hidden_states + + +def _gemma4_text_backbone_forward( + backbone: torch.nn.Module, + *, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor | None, + causal_mask_mapping: dict[str, torch.Tensor], + position_ids: torch.LongTensor, +) -> torch.Tensor: + hidden_states = inputs_embeds + layer_types = tuple(dict.fromkeys(getattr(backbone.config, "layer_types", ()))) + position_embeddings = { + layer_type: backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in layer_types + } + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + + for decoder_layer in backbone.layers[: backbone.config.num_hidden_layers]: + layer_per_input = None + if per_layer_inputs is not None: + layer_per_input = per_layer_inputs[:, :, decoder_layer.layer_idx, :] + hidden_states = _gemma4_text_decoder_layer_forward( + decoder_layer, + hidden_states, + per_layer_input=layer_per_input, + attention_mask=causal_mask_mapping[decoder_layer.attention_type], + position_embeddings=position_embeddings[decoder_layer.attention_type], + position_ids=position_ids, + past_key_values=None, + use_cache=False, + shared_kv_states=shared_kv_states, + ) + + return backbone.norm(hidden_states) + + +def _gemma4_strip_audio_padding(audio_output: object) -> torch.Tensor: + audio_features = getattr(audio_output, "pooler_output", None) + audio_mask_from_encoder = getattr(audio_output, "audio_mel_mask", None) + if not isinstance(audio_features, torch.Tensor): + raise TypeError("Gemma4 audio output did not expose tensor pooler_output") + if not isinstance(audio_mask_from_encoder, torch.Tensor): + return audio_features + all_real_tokens: list[torch.Tensor] = [] + for encodings, padding_mask in zip(audio_features, audio_mask_from_encoder, strict=True): + all_real_tokens.append(encodings[~padding_mask]) + return torch.cat(all_real_tokens, dim=0).unsqueeze(0) + + +def _gemma4_rms_norm_no_scale(hidden_states: torch.Tensor, *, eps: float) -> torch.Tensor: + hidden_states_f32 = hidden_states.float() + mean_squared = hidden_states_f32.pow(2).mean(-1, keepdim=True) + eps + return hidden_states_f32 * torch.pow(mean_squared, -0.5) + + +def _gemma4_rms_norm(hidden_states: torch.Tensor, weight: torch.Tensor, *, eps: float) -> torch.Tensor: + normed = _gemma4_rms_norm_no_scale(hidden_states, eps=eps) + return normed * weight.to(device=normed.device, dtype=normed.dtype) + + +def _gemma4_layer_norm_eps(multimodal_backbone: torch.nn.Module) -> float: + config = getattr(multimodal_backbone, "config", None) + text_config = None + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + text_config = get_text_config() + except Exception: + text_config = None + for candidate in (text_config, config): + value = getattr(candidate, "rms_norm_eps", None) + if value is not None: + return float(value) + value = getattr(candidate, "layer_norm_eps", None) + if value is not None: + return float(value) + return 1e-6 + + +def _gemma4_load_cactus_fp_tensor(path: Path) -> torch.Tensor: + with path.open("rb") as handle: + header = handle.read(84) + if len(header) < 84 or header[:4] != CACTUS_MAGIC: + raise RuntimeError(f"Gemma4 Cactus tensor is missing a valid header: {path}") + + alignment = max(1, int(struct.unpack_from(" 0) + precision = int(struct.unpack_from(" 0 else 0 + data_offset = ( + align_offset(scales_offset + scales_bytes, alignment) + if scales_bytes > 0 + else aligned_header + ) + data_count = byte_size // np.dtype(dtype).itemsize + array = np.memmap(path, mode="r", dtype=dtype, offset=data_offset, shape=(data_count,)) + tensor = torch.from_numpy(np.array(array, copy=True)) + if shape: + tensor = tensor.reshape(shape) + return tensor + + +def _gemma4_load_vision_post_proj_norm(weights_dir: str | Path | None) -> torch.Tensor | None: + if weights_dir is None: + return None + path = Path(weights_dir).expanduser() / "embed_vision_post_proj_norm.weights" + if not path.exists(): + return None + tensor = _gemma4_load_cactus_fp_tensor(path) + if tensor.ndim != 1: + raise RuntimeError(f"Gemma4 vision post-projection norm must be 1D, got shape={tuple(tensor.shape)}") + return tensor.float() + + +def _gemma4_can_use_native_like_vision_features(multimodal_backbone: torch.nn.Module) -> bool: + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + return ( + isinstance(vision_tower, torch.nn.Module) + and isinstance(embed_vision, torch.nn.Module) + and hasattr(vision_tower, "patch_embedder") + and hasattr(vision_tower, "encoder") + and hasattr(embed_vision, "embedding_projection") + ) + + +def _gemma4_can_use_native_like_audio_features(multimodal_backbone: torch.nn.Module) -> bool: + audio_tower = getattr(multimodal_backbone, "audio_tower", None) + embed_audio = getattr(multimodal_backbone, "embed_audio", None) + return ( + isinstance(audio_tower, torch.nn.Module) + and isinstance(embed_audio, torch.nn.Module) + and hasattr(audio_tower, "subsample_conv_projection") + and hasattr(audio_tower, "conformer") + and hasattr(embed_audio, "embedding_projection") + ) + + +def _gemma4_pool_vision_hidden_native_like( + vision_tower: torch.nn.Module, + hidden_states: torch.Tensor, + pixel_position_ids: torch.Tensor, +) -> torch.Tensor: + output_length = int(getattr(vision_tower, "default_output_length")) + pooling_kernel_size = int(getattr(vision_tower, "pooling_kernel_size")) + padding_positions = (pixel_position_ids == -1).all(dim=-1) + pooled_batches: list[torch.Tensor] = [] + for hidden_row, position_row, padding_row in zip(hidden_states, pixel_position_ids, padding_positions, strict=True): + valid_hidden = hidden_row[~padding_row].float() + valid_positions = position_row[~padding_row].clamp(min=0) + max_x = valid_positions[:, 0].max() + 1 + kernel_positions = torch.div(valid_positions, pooling_kernel_size, rounding_mode="floor") + kernel_indices = kernel_positions[:, 0] + torch.div( + max_x, + pooling_kernel_size, + rounding_mode="floor", + ) * kernel_positions[:, 1] + weights = F.one_hot(kernel_indices.long(), output_length).float() / float(pooling_kernel_size**2) + pooled_full = weights.transpose(0, 1) @ valid_hidden + valid_bins = torch.logical_not((weights == 0).all(dim=0)) + pooled_batches.append(pooled_full[valid_bins]) + return torch.cat(pooled_batches, dim=0) + + +def _gemma4_compute_native_like_image_features( + multimodal_backbone: torch.nn.Module, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + *, + post_proj_norm_weight: torch.Tensor | None = None, +) -> torch.Tensor: + if pixel_position_ids is None: + raise TypeError("Gemma4 native-like vision feature path requires pixel_position_ids") + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + if not isinstance(vision_tower, torch.nn.Module) or not isinstance(embed_vision, torch.nn.Module): + raise TypeError("Gemma4 multimodal backbone is missing native-like vision modules") + + padding_positions = (pixel_position_ids == -1).all(dim=-1) + vision_inputs = vision_tower.patch_embedder( + pixel_values, + pixel_position_ids, + padding_positions, + ) + vision_outputs = vision_tower.encoder( + inputs_embeds=vision_inputs, + attention_mask=~padding_positions, + pixel_position_ids=pixel_position_ids, + ) + vision_hidden = _extract_tensor_output(vision_outputs, preferred_field="last_hidden_state") + pooled_hidden = _gemma4_pool_vision_hidden_native_like( + vision_tower, + vision_hidden, + pixel_position_ids, + ) + projection = getattr(embed_vision, "embedding_projection", None) + if not isinstance(projection, torch.nn.Linear): + raise TypeError("Gemma4 vision embedder is missing embedding_projection") + projected = F.linear( + pooled_hidden, + projection.weight.float(), + None if projection.bias is None else projection.bias.float(), + ) + if post_proj_norm_weight is not None and post_proj_norm_weight.numel() > 0: + return _gemma4_rms_norm( + projected, + post_proj_norm_weight, + eps=_gemma4_layer_norm_eps(multimodal_backbone), + ) + return _gemma4_rms_norm_no_scale(projected, eps=float(getattr(embed_vision, "eps", 1e-6))) + + +def _gemma4_compute_native_like_audio_features( + multimodal_backbone: torch.nn.Module, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, +) -> torch.Tensor: + audio_tower = getattr(multimodal_backbone, "audio_tower", None) + embed_audio = getattr(multimodal_backbone, "embed_audio", None) + if not isinstance(audio_tower, torch.nn.Module) or not isinstance(embed_audio, torch.nn.Module): + raise TypeError("Gemma4 multimodal backbone is missing native-like audio modules") + + audio_output = audio_tower(input_features, ~input_features_mask) + if not isinstance(audio_output, tuple) or len(audio_output) != 2: + raise TypeError("Gemma4 audio tower did not return (audio_encodings, audio_mel_mask)") + audio_encodings, audio_mel_mask = audio_output + if not isinstance(audio_encodings, torch.Tensor): + raise TypeError("Gemma4 audio tower did not return tensor audio encodings") + if not isinstance(audio_mel_mask, torch.Tensor): + raise TypeError("Gemma4 audio tower did not return tensor audio mask") + + projection = getattr(embed_audio, "embedding_projection", None) + if not isinstance(projection, torch.nn.Linear): + raise TypeError("Gemma4 audio embedder is missing embedding_projection") + normed = _gemma4_rms_norm_no_scale(audio_encodings, eps=float(getattr(embed_audio, "eps", 1e-6))) + projected = F.linear( + normed, + projection.weight.float(), + None if projection.bias is None else projection.bias.float(), + ) + projected = projected * (1.0 / 16.0) + all_real_tokens: list[torch.Tensor] = [] + for encodings, padding_mask in zip(projected, audio_mel_mask, strict=True): + all_real_tokens.append(encodings[~padding_mask]) + return torch.cat(all_real_tokens, dim=0).unsqueeze(0) + + +def _gemma4_feature_token_count(features: torch.Tensor | None) -> int: + if features is None or features.numel() == 0: + return 0 + if features.ndim == 3: + return int(features.shape[1]) + if features.ndim == 2: + return int(features.shape[0]) + if features.ndim >= 1: + return int(features.reshape(-1, features.shape[-1]).shape[0]) + return 0 + + +def _gemma4_build_native_merge_plan( + multimodal_backbone: torch.nn.Module, + input_ids: torch.Tensor, + *, + image_feature_count: int, + audio_feature_count: int, +) -> _Gemma4NativeMergePlan: + if input_ids.ndim != 2 or int(input_ids.shape[0]) != 1: + raise ValueError( + "Gemma4 native multimodal merge currently expects a static batch-1 input_ids tensor, " + f"got shape {tuple(input_ids.shape)}" + ) + + image_token_id, audio_token_id, pad_token_id = _gemma4_special_token_ids(multimodal_backbone) + token_ids = [int(token) for token in input_ids.detach().cpu().reshape(-1).tolist()] + segments: list[_Gemma4NativeMergeSegment] = [] + pli_token_ids: list[int] = [] + image_offset = 0 + audio_offset = 0 + index = 0 + + while index < len(token_ids): + token_id = token_ids[index] + is_image = image_token_id != 0 and token_id == image_token_id + is_audio = audio_token_id != 0 and token_id == audio_token_id + + if is_image or is_audio: + region_start = index + while index < len(token_ids) and token_ids[index] == token_id: + index += 1 + placeholder_count = index - region_start + if is_image: + insert_count = min(placeholder_count, max(0, image_feature_count - image_offset)) + if insert_count > 0: + segments.append( + _Gemma4NativeMergeSegment( + kind="image", + input_start=region_start, + length=insert_count, + feature_start=image_offset, + ) + ) + pli_token_ids.extend([pad_token_id] * insert_count) + image_offset += insert_count + else: + insert_count = min(placeholder_count, max(0, audio_feature_count - audio_offset)) + if insert_count > 0: + segments.append( + _Gemma4NativeMergeSegment( + kind="audio", + input_start=region_start, + length=insert_count, + feature_start=audio_offset, + ) + ) + pli_token_ids.extend([pad_token_id] * insert_count) + audio_offset += insert_count + continue + + text_start = index + while index < len(token_ids): + next_token = token_ids[index] + if image_token_id != 0 and next_token == image_token_id: + break + if audio_token_id != 0 and next_token == audio_token_id: + break + index += 1 + text_tokens = token_ids[text_start:index] + if text_tokens: + segments.append( + _Gemma4NativeMergeSegment( + kind="text", + input_start=text_start, + length=len(text_tokens), + ) + ) + pli_token_ids.extend(text_tokens) + + if not segments: + raise RuntimeError("Gemma4 native multimodal merge built no input segments") + return _Gemma4NativeMergePlan( + segments=tuple(segments), + pli_token_ids=tuple(pli_token_ids), + ) + + +def _gemma4_feature_sequence( + features: torch.Tensor, + *, + batch_size: int, + feature_dim: int, +) -> torch.Tensor: + if features.ndim == 3: + if int(features.shape[0]) == batch_size: + return features + if int(features.shape[0]) == 1: + return features.expand(batch_size, -1, -1) + if features.ndim == 2: + if int(features.shape[-1]) != feature_dim: + raise ValueError(f"Gemma4 feature dim mismatch: expected {feature_dim}, got {tuple(features.shape)}") + return features.unsqueeze(0).expand(batch_size, -1, -1) + if features.ndim >= 1 and int(features.shape[-1]) == feature_dim: + flattened = features.reshape(-1, feature_dim) + return flattened.unsqueeze(0).expand(batch_size, -1, -1) + raise ValueError(f"unsupported Gemma4 feature tensor shape: {tuple(features.shape)}") + + +def _gemma4_apply_native_merge_plan( + model: torch.nn.Module, + *, + input_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + merge_plan: _Gemma4NativeMergePlan, + pli_token_ids: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + embedding = model.get_input_embeddings() + if not isinstance(embedding, torch.nn.Module): + raise TypeError("Gemma4 model is missing input embeddings") + batch_size = int(input_ids.shape[0]) + feature_dim = int(getattr(embedding, "embedding_dim", 0) or image_features.shape[-1] or audio_features.shape[-1]) + target_dtype = _module_floating_dtype(embedding) or image_features.dtype + image_sequence = _gemma4_feature_sequence( + image_features, + batch_size=batch_size, + feature_dim=feature_dim, + ).to(device=input_ids.device, dtype=target_dtype) + audio_sequence = _gemma4_feature_sequence( + audio_features, + batch_size=batch_size, + feature_dim=feature_dim, + ).to(device=input_ids.device, dtype=target_dtype) + + embedded_segments: list[torch.Tensor] = [] + model_config = getattr(model, "config", None) + text_config = getattr(model_config, "text_config", None) + hidden_scale = float(getattr(model_config, "hidden_size", 0) or 0) + if hidden_scale <= 0.0: + hidden_scale = float(getattr(text_config, "hidden_size", 0) or 0) + if hidden_scale <= 0.0: + hidden_scale = float(feature_dim) + hidden_scale = float(hidden_scale) ** 0.5 + for segment in merge_plan.segments: + if segment.kind == "text": + text_tokens = input_ids[:, segment.input_start : segment.input_start + segment.length] + embedded_segments.append(embedding(text_tokens) * hidden_scale) + elif segment.kind == "image": + embedded_segments.append( + image_sequence[:, segment.feature_start : segment.feature_start + segment.length, :] + ) + elif segment.kind == "audio": + embedded_segments.append( + audio_sequence[:, segment.feature_start : segment.feature_start + segment.length, :] + ) + else: + raise RuntimeError(f"unknown Gemma4 merge segment kind: {segment.kind!r}") + + inputs_embeds = torch.cat(embedded_segments, dim=1) + if pli_token_ids.numel() != inputs_embeds.shape[1]: + raise RuntimeError( + "Gemma4 native merge PLI token count mismatch: " + f"{int(pli_token_ids.numel())} vs {int(inputs_embeds.shape[1])}" + ) + pli_tokens = pli_token_ids.to(device=input_ids.device, dtype=input_ids.dtype).unsqueeze(0) + if batch_size != 1: + pli_tokens = pli_tokens.expand(batch_size, -1) + return inputs_embeds, pli_tokens + + +def _gemma4_remap_sequence_tensor( + tensor: torch.Tensor | None, + *, + merge_plan: _Gemma4NativeMergePlan, +) -> torch.Tensor | None: + if tensor is None: + return None + if tensor.ndim != 2: + raise ValueError(f"Gemma4 merge remap expects a rank-2 tensor, got shape {tuple(tensor.shape)}") + remapped_segments: list[torch.Tensor] = [] + for segment in merge_plan.segments: + remapped_segments.append( + tensor[:, segment.input_start : segment.input_start + segment.length] + ) + if not remapped_segments: + return tensor[:, :0] + return torch.cat(remapped_segments, dim=1) + + +def _gemma4_build_standard_causal_mask_mapping( + *, + create_causal_mask: Callable[..., torch.Tensor] | None, + create_sliding_window_causal_mask: Callable[..., torch.Tensor] | None, + config: object, + inputs_embeds: torch.Tensor, + attention_mask: torch.Tensor | None, + position_ids: torch.Tensor, +) -> dict[str, torch.Tensor]: + if not callable(create_causal_mask) or not callable(create_sliding_window_causal_mask): + raise RuntimeError("Gemma4 standard causal mask helpers are unavailable") + mask_kwargs = { + "config": config, + "inputs_embeds": inputs_embeds, + "attention_mask": attention_mask, + "past_key_values": None, + "position_ids": position_ids, + } + return { + "full_attention": create_causal_mask(**mask_kwargs), + "sliding_attention": create_sliding_window_causal_mask(**mask_kwargs), + } + + +def _infer_input_names(module: torch.nn.Module, *, preferred: tuple[str, ...]) -> tuple[str, ...]: + try: + signature = inspect.signature(module.forward) + except (TypeError, ValueError): + return preferred[:1] + + control_names = { + "self", + "return_dict", + "use_cache", + "past_key_values", + "cache_position", + "position_ids", + "labels", + "decoder_input_ids", + "decoder_attention_mask", + "output_attentions", + "output_hidden_states", + } + available = [ + name + for name, parameter in signature.parameters.items() + if name not in control_names + and parameter.kind in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + ] + matched = [name for name in preferred if name in available] + if matched: + return tuple(matched) + if available: + return tuple(available[: min(2, len(available))]) + return preferred[:1] + + +class BoundInputAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str, metadata_task: str): + super().__init__() + self.model = model + self.input_names = tuple(input_names) + self.family = family + self.metadata_task = metadata_task + + def _kwargs_from_bound_inputs(self, *bound_inputs: torch.Tensor | None) -> dict[str, torch.Tensor]: + provided = tuple(bound_inputs) + if len(self.input_names) > len(provided): + raise ValueError( + f"adapter expected at most {len(provided)} bound inputs, got {len(self.input_names)} names" + ) + kwargs: dict[str, torch.Tensor] = {} + for index, name in enumerate(self.input_names): + value = provided[index] + if value is None: + raise ValueError(f"missing required bound input {index} for {name}") + kwargs[name] = value + return kwargs + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family=self.family, + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + "task": self.metadata_task, + } + } + + +def _gemma4_import_hints(backbone: torch.nn.Module, *, module_path_suffix_prefix: str) -> list[dict[str, object]]: + sliding_window = getattr(backbone.config, "sliding_window", None) + layer_types = list(getattr(backbone.config, "layer_types", [])) + import_hints: list[dict[str, object]] = [] + for layer_index, layer_type in enumerate(layer_types): + attrs: dict[str, object] = {} + if layer_type == "sliding_attention" and sliding_window is not None: + attrs["window_size"] = int(sliding_window) + import_hints.append( + { + "module_path_suffix": f"{module_path_suffix_prefix}.layers.{layer_index}.self_attn", + "op": "scaled_dot_product_attention", + "attrs": attrs, + "meta": { + "attention_layer_type": layer_type, + "attention_layer_index": layer_index, + }, + } + ) + return import_hints + + +class CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + + def forward(self, input_ids: torch.Tensor): + return self.model( + input_ids=input_ids, + use_cache=False, + return_dict=False, + )[0] + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="generic", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + } + } + + +class GemmaCausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.gemma.modeling_gemma import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + causal_mask = self._create_causal_mask( + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=None, + past_key_values=None, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.backbone.rotary_emb(hidden_states, position_ids=position_ids) + checkpoints: list[torch.Tensor] = [] + + for decoder_layer in self.backbone.layers[: self.backbone.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=None, + use_cache=False, + position_embeddings=position_embeddings, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return _gemma4_apply_final_logit_softcapping(self.model, self.model.lm_head(hidden_states)), checkpoints + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + } + } + + +class Gemma3CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.gemma3.modeling_gemma3 import create_causal_mask # type: ignore + from transformers.models.gemma3.modeling_gemma3 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.config.layer_types + } + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], + position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], + position_ids=position_ids, + past_key_values=None, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma3", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + }, + "import_hints": _gemma4_import_hints(self.backbone, module_path_suffix_prefix="backbone"), + } + + +class Gemma4CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + model_backbone = model.model + self.backbone = getattr(model_backbone, "language_model", model_backbone) + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + per_layer_inputs = None + if self.backbone.hidden_size_per_layer_input: + per_layer_inputs = self.backbone.get_per_layer_inputs(input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.unique_layer_types + } + shared_kv_states: dict[str, torch.Tensor] = {} + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + per_layer_input = per_layer_inputs[:, :, i, :] if per_layer_inputs is not None else None + hidden_states = decoder_layer( + hidden_states, + per_layer_input, + shared_kv_states=shared_kv_states, + position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], + attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], + position_ids=position_ids, + past_key_values=None, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def debug_first_block(self, input_ids: torch.Tensor) -> dict[str, torch.Tensor]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + per_layer_inputs = None + if self.backbone.hidden_size_per_layer_input: + per_layer_inputs = self.backbone.get_per_layer_inputs(input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.unique_layer_types + } + shared_kv_states: dict[str, torch.Tensor] = {} + layer = self.backbone.layers[0] + layer_type = self.backbone.config.layer_types[0] + per_layer_input = per_layer_inputs[:, :, 0, :] if per_layer_inputs is not None else None + + checkpoints: dict[str, torch.Tensor] = {} + + residual = hidden_states + normed = layer.input_layernorm(hidden_states) + checkpoints["pre_attn_norm"] = normed + + attn_out = layer.self_attn( + normed, + position_embeddings=position_embeddings[layer_type], + attention_mask=causal_mask_mapping[layer_type], + position_ids=position_ids, + past_key_values=None, + shared_kv_states=shared_kv_states, + ) + if isinstance(attn_out, tuple): + attn_out = attn_out[0] + checkpoints["attn_o_proj"] = attn_out + + post_attn_norm = layer.post_attention_layernorm(attn_out) + checkpoints["post_attn_norm"] = post_attn_norm + + after_attention = residual + post_attn_norm + checkpoints["after_attention_residual"] = after_attention + + pre_ffn_norm = layer.pre_feedforward_layernorm(after_attention) + checkpoints["pre_ffn_norm"] = pre_ffn_norm + + mlp_out = layer.mlp(pre_ffn_norm) + checkpoints["mlp_down"] = mlp_out + + post_ffn_norm = layer.post_feedforward_layernorm(mlp_out) + checkpoints["post_ffn_norm"] = post_ffn_norm + + after_ffn = after_attention + post_ffn_norm + checkpoints["after_ffn_residual"] = after_ffn + + if per_layer_input is not None: + gated = layer.per_layer_input_gate(after_ffn) + gated = layer.act_fn(gated) + projected = gated * per_layer_input + per_layer_proj = layer.per_layer_projection(projected) + checkpoints["per_layer_input_proj"] = per_layer_proj + post_per_layer_input_norm = layer.post_per_layer_input_norm(per_layer_proj) + checkpoints["post_per_layer_input_norm"] = post_per_layer_input_norm + after_ffn = after_ffn + post_per_layer_input_norm + + layer_scalar = getattr(layer, "layer_scalar", None) + if layer_scalar is not None: + after_ffn = after_ffn * layer_scalar + checkpoints["layer_scalar_out"] = after_ffn + return checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + }, + "import_hints": _gemma4_import_hints(self.backbone, module_path_suffix_prefix="backbone"), + } + + +def _gemma4_apply_final_logit_softcapping(model: torch.nn.Module, logits: torch.Tensor) -> torch.Tensor: + config = getattr(model, "config", None) + text_config = None + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + text_config = get_text_config() + except Exception: + text_config = None + cap = getattr(text_config, "final_logit_softcapping", None) + if cap is None: + cap = getattr(config, "final_logit_softcapping", None) + if cap is None: + return logits + cap_value = float(cap) + if cap_value <= 0.0: + return logits + return torch.tanh(logits / cap_value) * cap_value + + +class Gemma4MultimodalCausalLMLogitsAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], weights_dir: str | None = None): + super().__init__( + model, + input_names=input_names, + family="gemma4", + metadata_task="multimodal_causal_lm_logits", + ) + model_backbone = model.model + self.multimodal_backbone = model_backbone + self.backbone = getattr(model_backbone, "language_model", model_backbone) + self.last_token_logits_only = False + self._use_cached_multimodal_features = False + self._native_merge_plan: _Gemma4NativeMergePlan | None = None + self.register_buffer("_cached_image_features", torch.empty(0), persistent=False) + self.register_buffer("_cached_audio_features", torch.empty(0), persistent=False) + self.register_buffer("_native_merge_pli_token_ids", torch.empty(0, dtype=torch.long), persistent=False) + vision_post_proj_norm = _gemma4_load_vision_post_proj_norm(weights_dir) + if vision_post_proj_norm is None: + vision_post_proj_norm = torch.empty(0) + self.register_buffer("_cactus_vision_post_proj_norm", vision_post_proj_norm, persistent=False) + self._capture_cpu_float32_text_modules: list[tuple[torch.nn.Module, torch.dtype]] = [] + self._create_causal_mask_mapping = None + self._create_masks_for_generate = None + self._create_causal_mask = None + self._create_sliding_window_causal_mask = None + try: + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_masks_for_generate # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_causal_mask_mapping = create_causal_mask_mapping + self._create_masks_for_generate = create_masks_for_generate + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + except Exception: + pass + + def _apply_final_logit_softcapping(self, logits: torch.Tensor) -> torch.Tensor: + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def _capture_text_modules(self) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + self.backbone, + getattr(self.model, "get_input_embeddings", lambda: None)(), + getattr(self.model, "lm_head", None), + ) + + def _compute_image_features( + self, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + get_image_features: Callable[..., object], + ) -> torch.Tensor: + multimodal_backbone = self.multimodal_backbone + if _gemma4_can_use_native_like_vision_features(multimodal_backbone): + return _gemma4_compute_native_like_image_features( + multimodal_backbone, + pixel_values, + pixel_position_ids, + post_proj_norm_weight=self._cactus_vision_post_proj_norm, + ) + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + vision_modules = _gemma4_vision_modules(multimodal_backbone) + if ( + pixel_values.device.type != "cpu" + or pixel_values.dtype != torch.float16 + or len(vision_modules) != 2 + or _torch_is_compiling() + ): + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + vision_dtype = _module_floating_dtype(vision_tower) + embed_dtype = _module_floating_dtype(embed_vision) + if vision_dtype != torch.float16 and embed_dtype != torch.float16: + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + # Gemma4's CPU float16 vision path can emit non-finite soft tokens; upcast only + # the static image feature extraction path and restore the original module dtypes. + with _temporary_cpu_float32_modules(vision_modules): + return get_image_features( + pixel_values.float(), + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + def _compute_audio_features( + self, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, + get_audio_features: Callable[..., object], + ) -> torch.Tensor: + multimodal_backbone = self.multimodal_backbone + if _gemma4_can_use_native_like_audio_features(multimodal_backbone): + return _gemma4_compute_native_like_audio_features( + multimodal_backbone, + input_features, + input_features_mask, + ) + audio_modules = _gemma4_audio_modules(multimodal_backbone) + if ( + input_features.device.type != "cpu" + or input_features.dtype != torch.float16 + or len(audio_modules) != 2 + or _torch_is_compiling() + ): + audio_output = get_audio_features(input_features, ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + if all(_module_floating_dtype(module) != torch.float16 for module in audio_modules): + audio_output = get_audio_features(input_features, ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + with _temporary_cpu_float32_modules(audio_modules): + audio_output = get_audio_features(input_features.float(), ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + def prepare_cpu_float32_capture(self) -> None: + self._capture_cpu_float32_text_modules.clear() + if os.environ.get("CACTUS_GEMMA4_CAPTURE_FP32") != "1": + return + for module in self._capture_text_modules(): + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + self._capture_cpu_float32_text_modules.append((module, dtype)) + module.to(dtype=torch.float32) + + def restore_cpu_float32_capture(self) -> None: + for module, dtype in reversed(self._capture_cpu_float32_text_modules): + module.to(dtype=dtype) + self._capture_cpu_float32_text_modules.clear() + + def _resolve_image_features( + self, + *, + inputs_embeds: torch.Tensor, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + get_image_features: Callable[..., object], + ) -> torch.Tensor | None: + if pixel_values is None: + return None + if self._use_cached_multimodal_features and self._cached_image_features.numel() > 0: + image_features = self._cached_image_features + else: + image_features = self._compute_image_features( + pixel_values, + pixel_position_ids, + get_image_features, + ) + return image_features.to(inputs_embeds.device, inputs_embeds.dtype) + + def _resolve_audio_features( + self, + *, + inputs_embeds: torch.Tensor, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_audio_features: Callable[..., object], + ) -> torch.Tensor | None: + if input_features is None or input_features_mask is None: + return None + if self._use_cached_multimodal_features and self._cached_audio_features.numel() > 0: + audio_features = self._cached_audio_features + else: + audio_features = self._compute_audio_features( + input_features, + input_features_mask, + get_audio_features, + ) + return audio_features.to(inputs_embeds.device, inputs_embeds.dtype) + + def _prepare_text_backbone_inputs( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor | None, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_placeholder_mask: Callable[..., object], + get_image_features: Callable[..., object], + get_audio_features: Callable[..., object], + ) -> tuple[torch.Tensor, torch.Tensor | None, dict[str, torch.Tensor], torch.LongTensor]: + inputs_embeds = self.model.get_input_embeddings()(input_ids) + + image_features = self._resolve_image_features( + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + get_image_features=get_image_features, + ) + audio_features = self._resolve_audio_features( + inputs_embeds=inputs_embeds, + input_features=input_features, + input_features_mask=input_features_mask, + get_audio_features=get_audio_features, + ) + + if image_features is not None and audio_features is not None and self._native_merge_plan is not None: + inputs_embeds, per_layer_inputs_tokens = _gemma4_apply_native_merge_plan( + self.model, + input_ids=input_ids, + image_features=image_features, + audio_features=audio_features, + merge_plan=self._native_merge_plan, + pli_token_ids=self._native_merge_pli_token_ids, + ) + attention_mask = _gemma4_remap_sequence_tensor( + attention_mask, + merge_plan=self._native_merge_plan, + ) + token_type_ids = _gemma4_remap_sequence_tensor( + token_type_ids, + merge_plan=self._native_merge_plan, + ) + if token_type_ids is None: + raise RuntimeError("Gemma4 native merge requires token_type_ids for multimodal attention masking") + else: + text_mask, image_mask, audio_mask = get_placeholder_mask( + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + if image_features is not None: + inputs_embeds = inputs_embeds.masked_scatter( + image_mask.unsqueeze(-1).expand_as(inputs_embeds), + image_features, + ) + if audio_features is not None: + inputs_embeds = inputs_embeds.masked_scatter( + audio_mask.unsqueeze(-1).expand_as(inputs_embeds), + audio_features, + ) + per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype) + + per_layer_inputs = None + text_config = _gemma4_text_config(self.multimodal_backbone) + if getattr(text_config, "hidden_size_per_layer_input", None): + per_layer_inputs = self.backbone.get_per_layer_inputs(per_layer_inputs_tokens) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + if self._native_merge_plan is not None: + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + elif getattr(text_config, "use_bidirectional_attention", None) == "vision": + causal_mask_mapping = self._create_causal_mask_mapping( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + token_type_ids, + pixel_values, + is_training=self.training, + ) + else: + causal_mask_mapping = self._create_masks_for_generate( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + ) + return inputs_embeds, per_layer_inputs, causal_mask_mapping, position_ids + + def _forward_hidden_states( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor | None, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_placeholder_mask: Callable[..., object], + get_image_features: Callable[..., object], + get_audio_features: Callable[..., object], + ) -> torch.Tensor: + inputs_embeds, per_layer_inputs, causal_mask_mapping, position_ids = self._prepare_text_backbone_inputs( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + input_features=input_features, + input_features_mask=input_features_mask, + get_placeholder_mask=get_placeholder_mask, + get_image_features=get_image_features, + get_audio_features=get_audio_features, + ) + return _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + + def prime_static_multimodal_features(self, *bound_inputs: torch.Tensor | None) -> None: + kwargs = self._kwargs_from_bound_inputs(*bound_inputs) + input_ids = kwargs["input_ids"] + token_type_ids = kwargs.get("token_type_ids") + pixel_values = kwargs.get("pixel_values") + pixel_position_ids = kwargs.get("pixel_position_ids") + input_features = kwargs.get("input_features") + input_features_mask = kwargs.get("input_features_mask") + + multimodal_backbone = self.multimodal_backbone + get_placeholder_mask = getattr(multimodal_backbone, "get_placeholder_mask", None) + get_image_features = getattr(multimodal_backbone, "get_image_features", None) + get_audio_features = getattr(multimodal_backbone, "get_audio_features", None) + if not callable(get_placeholder_mask): + return + + with torch.no_grad(): + inputs_embeds = self.model.get_input_embeddings()(input_ids) + _, image_mask, audio_mask = get_placeholder_mask( + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + + if pixel_values is not None and callable(get_image_features) and image_mask.any(): + image_features = self._compute_image_features( + pixel_values, + pixel_position_ids, + get_image_features, + ) + self._cached_image_features = image_features.detach() + else: + self._cached_image_features = self._cached_image_features.new_empty(0) + + if ( + input_features is not None + and input_features_mask is not None + and callable(get_audio_features) + and audio_mask.any() + ): + self._cached_audio_features = self._compute_audio_features( + input_features, + input_features_mask, + get_audio_features, + ).detach() + else: + self._cached_audio_features = self._cached_audio_features.new_empty(0) + + if self._cached_image_features.numel() > 0 and self._cached_audio_features.numel() > 0: + plan = _gemma4_build_native_merge_plan( + self.multimodal_backbone, + input_ids, + image_feature_count=_gemma4_feature_token_count(self._cached_image_features), + audio_feature_count=_gemma4_feature_token_count(self._cached_audio_features), + ) + self._native_merge_plan = plan + self._native_merge_pli_token_ids = torch.tensor( + plan.pli_token_ids, + dtype=torch.long, + ) + else: + self._native_merge_plan = None + self._native_merge_pli_token_ids = self._native_merge_pli_token_ids.new_empty(0) + + self._use_cached_multimodal_features = True + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + kwargs = self._kwargs_from_bound_inputs(*bound_inputs) + input_ids = kwargs["input_ids"] + attention_mask = kwargs.get("attention_mask") + token_type_ids = kwargs.get("token_type_ids") + pixel_values = kwargs.get("pixel_values") + pixel_position_ids = kwargs.get("pixel_position_ids") + input_features = kwargs.get("input_features") + input_features_mask = kwargs.get("input_features_mask") + + multimodal_backbone = self.multimodal_backbone + get_placeholder_mask = getattr(multimodal_backbone, "get_placeholder_mask", None) + get_image_features = getattr(multimodal_backbone, "get_image_features", None) + get_audio_features = getattr(multimodal_backbone, "get_audio_features", None) + lm_head = getattr(self.model, "lm_head", None) + if ( + not callable(get_placeholder_mask) + or not callable(get_image_features) + or not callable(get_audio_features) + or not callable(self._create_causal_mask_mapping) + or not callable(self._create_masks_for_generate) + or not isinstance(lm_head, torch.nn.Module) + ): + outputs = self.model( + return_dict=True, + use_cache=False, + **kwargs, + ) + logits = _extract_tensor_output(outputs, preferred_field="logits") + if self.last_token_logits_only and logits.ndim >= 3: + return _select_last_active_token(logits, attention_mask) + return logits + + hidden_states = self._forward_hidden_states( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + input_features=input_features, + input_features_mask=input_features_mask, + get_placeholder_mask=get_placeholder_mask, + get_image_features=get_image_features, + get_audio_features=get_audio_features, + ) + if self.last_token_logits_only: + hidden_states = _select_last_active_token(hidden_states, attention_mask) + return self._apply_final_logit_softcapping(lm_head(hidden_states)) + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + "task": self.metadata_task, + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + "last_token_logits_only": bool(self.last_token_logits_only), + }, + "import_hints": _gemma4_import_hints( + self.backbone, + module_path_suffix_prefix="backbone", + ), + } + + +_GEMMA4_DECODER_PIPELINE_IO_KEYS = ( + "inputs_embeds", + "per_layer_inputs", + "position_ids", +) + + +class _Gemma4MultimodalComponentBase(torch.nn.Module): + def __init__( + self, + model: torch.nn.Module, + *, + input_names: tuple[str, ...], + weights_dir: str | None = None, + native_merge_plan: _Gemma4NativeMergePlan | None = None, + ): + super().__init__() + self.model = model + self.input_names = input_names + self.multimodal_backbone = model.model + model_backbone = model.model + self.backbone = getattr(model_backbone, "language_model", model_backbone) + self._native_merge_plan = native_merge_plan + self.register_buffer( + "_native_merge_pli_token_ids", + torch.tensor(native_merge_plan.pli_token_ids, dtype=torch.long) + if native_merge_plan is not None + else torch.empty(0, dtype=torch.long), + persistent=False, + ) + vision_post_proj_norm = _gemma4_load_vision_post_proj_norm(weights_dir) + if vision_post_proj_norm is None: + vision_post_proj_norm = torch.empty(0) + self.register_buffer("_cactus_vision_post_proj_norm", vision_post_proj_norm, persistent=False) + self._create_causal_mask_mapping = None + self._create_masks_for_generate = None + self._create_causal_mask = None + self._create_sliding_window_causal_mask = None + try: + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_masks_for_generate # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_causal_mask_mapping = create_causal_mask_mapping + self._create_masks_for_generate = create_masks_for_generate + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + except Exception: + pass + self._capture_modules: list[tuple[torch.nn.Module, torch.dtype]] = [] + + def _base_graph_meta(self, *, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=adapter_type, + input_names=input_names, + ), + "task": "multimodal_causal_lm_logits", + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + } + + def _compute_image_features( + self, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + ) -> torch.Tensor: + if _gemma4_can_use_native_like_vision_features(self.multimodal_backbone): + return _gemma4_compute_native_like_image_features( + self.multimodal_backbone, + pixel_values, + pixel_position_ids, + post_proj_norm_weight=self._cactus_vision_post_proj_norm, + ) + get_image_features = getattr(self.multimodal_backbone, "get_image_features", None) + vision_tower = getattr(self.multimodal_backbone, "vision_tower", None) + embed_vision = getattr(self.multimodal_backbone, "embed_vision", None) + vision_modules = _gemma4_vision_modules(self.multimodal_backbone) + if not callable(get_image_features): + raise TypeError("Gemma4 multimodal backbone is missing get_image_features") + if ( + pixel_values.device.type != "cpu" + or pixel_values.dtype != torch.float16 + or len(vision_modules) != 2 + or _torch_is_compiling() + ): + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + vision_dtype = _module_floating_dtype(vision_tower) + embed_dtype = _module_floating_dtype(embed_vision) + if vision_dtype != torch.float16 and embed_dtype != torch.float16: + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + with _temporary_cpu_float32_modules(vision_modules): + return get_image_features( + pixel_values.float(), + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + def _compute_audio_features( + self, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, + ) -> torch.Tensor: + if _gemma4_can_use_native_like_audio_features(self.multimodal_backbone): + return _gemma4_compute_native_like_audio_features( + self.multimodal_backbone, + input_features, + input_features_mask, + ) + get_audio_features = getattr(self.multimodal_backbone, "get_audio_features", None) + if not callable(get_audio_features): + raise TypeError("Gemma4 multimodal backbone is missing get_audio_features") + audio_modules = _gemma4_audio_modules(self.multimodal_backbone) + if ( + input_features.device.type == "cpu" + and input_features.dtype == torch.float16 + and len(audio_modules) == 2 + and not _torch_is_compiling() + and any(_module_floating_dtype(module) == torch.float16 for module in audio_modules) + ): + with _temporary_cpu_float32_modules(audio_modules): + audio_output = get_audio_features(input_features.float(), ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + audio_output = get_audio_features(input_features, ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def prepare_for_capture(self, **_: object) -> None: + self.restore_after_capture() + modules = self._modules_to_prepare_for_capture() + if not modules: + return + promoted: list[tuple[torch.nn.Module, torch.dtype]] = [] + for module in modules: + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + promoted.append((module, dtype)) + module.to(dtype=torch.float32) + self._capture_modules = promoted + + def restore_after_capture(self, **_: object) -> None: + for module, dtype in reversed(self._capture_modules): + module.to(dtype=dtype) + self._capture_modules.clear() + + def _prepare_decoder_inputs( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.LongTensor]: + if self._native_merge_plan is not None: + inputs_embeds, per_layer_inputs_tokens = _gemma4_apply_native_merge_plan( + self.model, + input_ids=input_ids, + image_features=image_features, + audio_features=audio_features, + merge_plan=self._native_merge_plan, + pli_token_ids=self._native_merge_pli_token_ids, + ) + attention_mask = _gemma4_remap_sequence_tensor( + attention_mask, + merge_plan=self._native_merge_plan, + ) + token_type_ids = _gemma4_remap_sequence_tensor( + token_type_ids, + merge_plan=self._native_merge_plan, + ) + if token_type_ids is None: + raise RuntimeError("Gemma4 native merge requires token_type_ids for multimodal attention masking") + else: + get_placeholder_mask = getattr(self.multimodal_backbone, "get_placeholder_mask", None) + if not callable(get_placeholder_mask): + raise TypeError("Gemma4 multimodal backbone is missing get_placeholder_mask") + + inputs_embeds = self.model.get_input_embeddings()(input_ids) + text_mask, image_mask, audio_mask = get_placeholder_mask( + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + inputs_embeds = inputs_embeds.masked_scatter( + image_mask.unsqueeze(-1).expand_as(inputs_embeds), + image_features.to(inputs_embeds.device, inputs_embeds.dtype), + ) + inputs_embeds = inputs_embeds.masked_scatter( + audio_mask.unsqueeze(-1).expand_as(inputs_embeds), + audio_features.to(inputs_embeds.device, inputs_embeds.dtype), + ) + per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype) + + per_layer_inputs: torch.Tensor + text_config = _gemma4_text_config(self.multimodal_backbone) + if getattr(text_config, "hidden_size_per_layer_input", None): + per_layer_inputs = self.backbone.get_per_layer_inputs(per_layer_inputs_tokens) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + else: + per_layer_inputs = inputs_embeds.new_empty((inputs_embeds.shape[0], inputs_embeds.shape[1], 0, 0)) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + return ( + inputs_embeds, + per_layer_inputs, + position_ids, + ) + + +class Gemma4VisionEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=("pixel_values", "pixel_position_ids"), weights_dir=weights_dir) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def forward(self, pixel_values: torch.Tensor, pixel_position_ids: torch.Tensor | None) -> torch.Tensor: + return self._compute_image_features(pixel_values, pixel_position_ids) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("pixel_values", "pixel_position_ids"), + ), + } + + +class Gemma4AudioEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=("input_features", "input_features_mask"), weights_dir=weights_dir) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def forward(self, input_features: torch.Tensor, input_features_mask: torch.Tensor) -> torch.Tensor: + return self._compute_audio_features(input_features, input_features_mask) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("input_features", "input_features_mask"), + ), + } + + +class Gemma4LMEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__( + self, + model: torch.nn.Module, + *, + weights_dir: str | None = None, + native_merge_plan: _Gemma4NativeMergePlan | None = None, + ): + super().__init__( + model, + input_names=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + weights_dir=weights_dir, + native_merge_plan=native_merge_plan, + ) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.LongTensor]: + return self._prepare_decoder_inputs( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + image_features=image_features, + audio_features=audio_features, + ) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + ), + } + + +class Gemma4DecoderAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=_GEMMA4_DECODER_PIPELINE_IO_KEYS, weights_dir=weights_dir) + + def forward( + self, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor, + position_ids: torch.LongTensor, + ) -> torch.Tensor: + normalized_per_layer_inputs = per_layer_inputs + if normalized_per_layer_inputs.numel() == 0: + normalized_per_layer_inputs = None + attention_mask = torch.ones( + position_ids.shape, + dtype=torch.long, + device=position_ids.device, + ) + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + logits = self.model.lm_head(hidden_states[:, -1:, :]) + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + ), + "import_hints": _gemma4_import_hints( + self.backbone, + module_path_suffix_prefix="backbone", + ), + } + + +def _build_gemma4_multimodal_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None, + components: tuple[str, ...] | None = None, +) -> list[ComponentModuleSpec]: + pixel_values = named_tensors["pixel_values"] + pixel_position_ids = named_tensors.get("pixel_position_ids") + input_features = named_tensors["input_features"] + input_features_mask = named_tensors["input_features_mask"] + input_ids = named_tensors["input_ids"] + attention_mask = named_tensors.get("attention_mask") + token_type_ids = named_tensors["token_type_ids"] + + requested_components = tuple(components or ("vision_encoder", "audio_encoder", "lm_encoder", "decoder")) + requested_set = set(requested_components) + if not requested_set: + return [] + + expanded_components: list[str] = [] + + def _require(component: str) -> None: + if component not in expanded_components: + expanded_components.append(component) + + if "decoder" in requested_set: + _require("vision_encoder") + _require("audio_encoder") + _require("lm_encoder") + _require("decoder") + elif "lm_encoder" in requested_set: + _require("vision_encoder") + _require("audio_encoder") + _require("lm_encoder") + else: + if "vision_encoder" in requested_set: + _require("vision_encoder") + if "audio_encoder" in requested_set: + _require("audio_encoder") + + vision_encoder = Gemma4VisionEncoderAdapter(model, weights_dir=weights_dir).eval() + audio_encoder = Gemma4AudioEncoderAdapter(model, weights_dir=weights_dir).eval() + + image_features: torch.Tensor | None = None + audio_features: torch.Tensor | None = None + decoder_inputs: tuple[torch.Tensor, ...] | None = None + native_merge_plan: _Gemma4NativeMergePlan | None = None + + with torch.no_grad(): + if "vision_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): + image_features = vision_encoder(pixel_values, pixel_position_ids) + if "audio_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): + audio_features = audio_encoder(input_features, input_features_mask) + if "lm_encoder" in expanded_components or "decoder" in expanded_components: + if image_features is None: + image_features = vision_encoder(pixel_values, pixel_position_ids) + if audio_features is None: + audio_features = audio_encoder(input_features, input_features_mask) + native_merge_plan = _gemma4_build_native_merge_plan( + getattr(model, "model", model), + input_ids, + image_feature_count=_gemma4_feature_token_count(image_features), + audio_feature_count=_gemma4_feature_token_count(audio_features), + ) + lm_encoder = Gemma4LMEncoderAdapter( + model, + weights_dir=weights_dir, + native_merge_plan=native_merge_plan, + ).eval() + decoder = Gemma4DecoderAdapter(model, weights_dir=weights_dir).eval() + + with torch.no_grad(): + if "decoder" in expanded_components: + decoder_inputs = lm_encoder( + input_ids, + attention_mask, + token_type_ids, + image_features, + audio_features, + ) + + common_graph_meta = { + "weights_dir": weights_dir, + "task": "multimodal_causal_lm_logits", + "adapter_family": "gemma4", + } + specs: list[ComponentModuleSpec] = [] + if "vision_encoder" in expanded_components: + specs.append(ComponentModuleSpec( + component="vision_encoder", + module=vision_encoder, + example_inputs=(pixel_values, pixel_position_ids), + input_keys=("pixel_values", "pixel_position_ids"), + output_keys=("image_features",), + graph_meta={**common_graph_meta, "component": "vision_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "audio_encoder" in expanded_components: + specs.append(ComponentModuleSpec( + component="audio_encoder", + module=audio_encoder, + example_inputs=(input_features, input_features_mask), + input_keys=("input_features", "input_features_mask"), + output_keys=("audio_features",), + graph_meta={**common_graph_meta, "component": "audio_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "lm_encoder" in expanded_components: + if image_features is None or audio_features is None: + raise RuntimeError("Gemma4 lm_encoder spec requires precomputed image/audio features") + specs.append(ComponentModuleSpec( + component="lm_encoder", + module=lm_encoder, + example_inputs=(input_ids, attention_mask, token_type_ids, image_features, audio_features), + input_keys=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + graph_meta={**common_graph_meta, "component": "lm_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder" in expanded_components: + if decoder_inputs is None: + raise RuntimeError("Gemma4 decoder spec requires precomputed decoder inputs") + specs.append(ComponentModuleSpec( + component="decoder", + module=decoder, + example_inputs=tuple(decoder_inputs), + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + output_keys=("logits",), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + return specs + + +class Qwen35CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.qwen3_5.modeling_qwen3_5 import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1) + text_position_ids = position_ids[0] + multimodal_position_ids = position_ids[1:] + + causal_mask = self._create_causal_mask( + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=None, + past_key_values=None, + position_ids=text_position_ids, + ) + linear_attn_mask = self.backbone._update_linear_attn_mask(None, None) + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask + hidden_states = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + attention_mask=layer_mask, + position_ids=text_position_ids, + past_key_values=None, + use_cache=False, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + import_hints: list[dict[str, object]] = [] + for layer_index, layer_type in enumerate(layer_types): + import_hints.append( + { + "module_path_suffix": f"backbone.layers.{layer_index}.self_attn", + "meta": { + "attention_layer_type": layer_type, + "attention_layer_index": layer_index, + }, + } + ) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="qwen3_5", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": layer_types, + }, + "import_hints": import_hints, + } + + +class CTCLogitsAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): + super().__init__(model, input_names=input_names, family=family, metadata_task="ctc_logits") + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + outputs = self.model(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) + return _extract_tensor_output(outputs, preferred_field="logits") + + +class EncoderHiddenStatesAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): + encoder = None + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder = get_encoder() + if encoder is None: + encoder = getattr(model, "encoder", None) + if encoder is None: + model_attr = getattr(model, "model", None) + if model_attr is not None: + encoder = getattr(model_attr, "encoder", None) + if not isinstance(encoder, torch.nn.Module): + raise NotImplementedError(f"{type(model).__name__} does not expose an encoder module") + super().__init__(model, input_names=input_names, family=family, metadata_task="encoder_hidden_states") + self.encoder = encoder + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + outputs = self.encoder(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) + return _extract_tensor_output(outputs, preferred_field="last_hidden_state") + + +def _family_key(model: torch.nn.Module) -> str: + explicit_family = getattr(model, "family", None) + if isinstance(explicit_family, str) and explicit_family: + return explicit_family.lower() + module_name = type(model).__module__ + if module_name.startswith("transformers.models.whisper."): + return "whisper" + if module_name.startswith("transformers.models.gemma4."): + return "gemma4" + if module_name.startswith("transformers.models.gemma3."): + return "gemma3" + if module_name.startswith("transformers.models.gemma."): + return "gemma" + if module_name.startswith("transformers.models.qwen3_5."): + return "qwen3_5" + if module_name.startswith("transformers.models.lfm2_vl."): + return "lfm2_vl" + if module_name.startswith("transformers.models.lfm2_moe."): + return "lfm2_moe" + if module_name.startswith("transformers.models.lfm2."): + return "lfm2" + return "generic" + + +def build_component_module_specs( + model: torch.nn.Module, + *, + task: str, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, + components: tuple[str, ...] | None = None, +) -> list[ComponentModuleSpec] | None: + family = _family_key(model) + if family == "gemma4" and task == "multimodal_causal_lm_logits": + return _build_gemma4_multimodal_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + components=components, + ) + if family == "parakeet_tdt" and task == "tdt_transcription": + from src.transpile.parakeet_tdt_local import build_parakeet_tdt_component_specs + + return build_parakeet_tdt_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + ) + return None + + +def canonicalize_model_interface( + model: torch.nn.Module, + task: str = "causal_lm_logits", + *, + input_names: tuple[str, ...] | None = None, + weights_dir: str | None = None, +) -> CanonicalizedModel: + family = _family_key(model) + adapter_factory: Callable[[torch.nn.Module], torch.nn.Module] + resolved_input_names = tuple(input_names or ()) + + if task == "causal_lm_logits": + if family == "gemma": + adapter_factory = GemmaCausalLMLogitsAdapter + elif family == "gemma4": + adapter_factory = Gemma4CausalLMLogitsAdapter + elif family == "gemma3": + adapter_factory = Gemma3CausalLMLogitsAdapter + elif family == "qwen3_5": + adapter_factory = Qwen35CausalLMLogitsAdapter + else: + adapter_factory = CausalLMLogitsAdapter + resolved_input_names = ("input_ids",) + elif task == "multimodal_causal_lm_logits": + if family != "gemma4": + raise NotImplementedError(f"{type(model).__name__} does not support task={task}") + if not resolved_input_names: + resolved_input_names = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", + ) + adapter_factory = lambda inner_model: Gemma4MultimodalCausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + weights_dir=weights_dir, + ) + elif task == "ctc_logits": + if not resolved_input_names: + resolved_input_names = _infer_input_names( + model, + preferred=("input_values", "input_features", "attention_mask"), + ) + adapter_factory = lambda inner_model: CTCLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + elif task == "encoder_hidden_states": + encoder_module = None + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder_module = get_encoder() + if encoder_module is None: + encoder_module = getattr(model, "encoder", None) + if encoder_module is None and getattr(model, "model", None) is not None: + encoder_module = getattr(model.model, "encoder", None) + if not isinstance(encoder_module, torch.nn.Module): + raise NotImplementedError(f"{type(model).__name__} does not support task={task}") + if not resolved_input_names: + resolved_input_names = _infer_input_names( + encoder_module, + preferred=("input_features", "input_values", "attention_mask"), + ) + adapter_factory = lambda inner_model: EncoderHiddenStatesAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + else: + raise NotImplementedError(f"unsupported canonicalization task: {task}") + + return CanonicalizedModel( + module=adapter_factory(model).eval(), + task=task, + family=family, + input_names=resolved_input_names, + ) diff --git a/python/src/transpile/model_patterns.py b/python/src/transpile/model_patterns.py new file mode 100644 index 000000000..22be81bda --- /dev/null +++ b/python/src/transpile/model_patterns.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GoldPattern: + name: str + source_models: tuple[str, ...] + semantic_ops: tuple[str, ...] + description: str + + +GOLD_PATTERNS: tuple[GoldPattern, ...] = ( + GoldPattern( + name="decoder_attention_gqa", + source_models=("model_gemma.cpp", "model_qwen.cpp", "model_lfm2.cpp", "model_gemma4.cpp"), + semantic_ops=("rms_norm", "rope", "attention", "attention_int8_hybrid"), + description=( + "Q/K/V projections from the same normalized hidden state; Q and K optionally get per-head RMSNorm, " + "Q/K are rotary-embedded, then Cactus attention consumes native Q-head and KV-head counts." + ), + ), + GoldPattern( + name="gated_mlp_gelu", + source_models=("model_gemma.cpp", "model_gemma4.cpp"), + semantic_ops=("matmul", "gelu", "multiply"), + description="Two projection branches from the same normalized input: GELU(gate) * up, then a down projection.", + ), + GoldPattern( + name="gated_mlp_silu", + source_models=("model_qwen.cpp", "model_lfm2.cpp"), + semantic_ops=("matmul", "silu", "multiply"), + description="Two projection branches from the same normalized input: SiLU(gate) * up, then a down projection.", + ), + GoldPattern( + name="decoder_block_post_attn_norm", + source_models=("model_gemma.cpp", "model_gemma4.cpp", "model_gemma3n.cpp"), + semantic_ops=("rms_norm", "attention", "add_clipped"), + description=( + "Pre-norm attention block with post-attention RMSNorm before the residual add, followed by a second " + "pre-FFN RMSNorm and a post-FFN RMSNorm before the final residual add." + ), + ), + GoldPattern( + name="decoder_block_simple_residual", + source_models=("model_qwen.cpp", "model_lfm2.cpp"), + semantic_ops=("rms_norm", "attention", "add"), + description=( + "Pre-norm attention block with a direct residual add after attention, followed by a second RMSNorm and " + "a gated MLP branch with another direct residual add." + ), + ), + GoldPattern( + name="gemma4_partial_rope_attention", + source_models=("model_gemma4.cpp",), + semantic_ops=("rms_norm", "rope", "concat", "attention"), + description=( + "Gemma4 attention where only a prefix of the head dimension is rotary-embedded, optionally with shared " + "K/V heads and sliding-window attention." + ), + ), + GoldPattern( + name="sliding_window_attention_mask", + source_models=("model_gemma.cpp", "model_gemma4.cpp"), + semantic_ops=("attention",), + description=( + "Exported PyTorch may build a boolean mask with diff/cumsum/comparisons, but the handwritten Cactus " + "models encode the same behavior directly as an attention `window_size`." + ), + ), +) + + +def gold_patterns_by_name() -> dict[str, GoldPattern]: + return {pattern.name: pattern for pattern in GOLD_PATTERNS} diff --git a/python/src/transpile/capture/normalize.py b/python/src/transpile/normalize.py similarity index 75% rename from python/src/transpile/capture/normalize.py rename to python/src/transpile/normalize.py index f9d23b546..b2d3f2de3 100644 --- a/python/src/transpile/capture/normalize.py +++ b/python/src/transpile/normalize.py @@ -18,6 +18,10 @@ def normalize_target(target: str) -> str: return "identity" if target.startswith("aten.detach"): return "identity" + if target.startswith("aten.dropout"): + return "identity" + if target.startswith("aten.native_dropout"): + return "identity" if target.startswith("aten.arange.start"): return "arange" if target.startswith("aten.arange"): @@ -28,24 +32,46 @@ def normalize_target(target: str) -> str: return "subtract" if target.startswith("aten.mul"): return "multiply" + if target.startswith("aten.floor_divide"): + return "floor_divide" if target.startswith("aten.div"): return "divide" if target.startswith("aten.to.dtype"): return "precision_cast" + if target.startswith("aten.to.device"): + return "identity" if target.startswith("aten.type_as"): return "type_as" if target.startswith("aten.neg"): return "negate" if target.startswith("aten.ne.") or target == "aten.ne": return "not_equal" + if target.startswith("aten.eq.") or target == "aten.eq": + return "equal" + if target.startswith("aten.logical_or") or target.startswith("aten.__or__"): + return "logical_or" + if target.startswith("aten.logical_and") or target.startswith("aten.__and__"): + return "logical_and" + if target.startswith("aten.logical_not") or target.startswith("aten.bitwise_not"): + return "logical_not" if target.startswith("aten.abs"): return "abs" + if target.startswith("aten.clamp"): + return "clamp" + if target.startswith("aten.where"): + return "where" + if target.startswith("aten.masked_scatter"): + return "masked_scatter" + if target.startswith("aten.masked_fill"): + return "masked_fill" if target.startswith("aten.expand"): return "expand" if target.startswith("aten.exp"): return "scalar_exp" if target.startswith("aten.sqrt"): return "scalar_sqrt" + if target.startswith("aten.reciprocal"): + return "reciprocal" if target.startswith("aten.log"): return "scalar_log" if target.startswith("aten.rsqrt"): @@ -58,12 +84,18 @@ def normalize_target(target: str) -> str: return "reshape" if target == "aten.flatten.using_ints": return "flatten" + if target.startswith("aten.squeeze"): + return "squeeze" if target.startswith("aten.unsqueeze"): return "unsqueeze" if target == "aten.t.default" or target.startswith("aten.transpose"): return "transpose" if target.startswith("aten.permute"): return "permute" + if target.startswith("aten.numpy_T"): + return "numpy_T" + if target.startswith("aten.movedim"): + return "movedim" if target.startswith("aten.contiguous"): return "contiguous" if target.startswith("aten.mm") or target.startswith("aten.matmul") or target.startswith("aten.bmm"): @@ -82,6 +114,8 @@ def normalize_target(target: str) -> str: return "gelu" if target.startswith("aten.sigmoid"): return "sigmoid" + if target.startswith("aten.glu"): + return "glu" if target.startswith("aten.softplus"): return "softplus" if target.startswith("aten.tanh"): @@ -94,6 +128,8 @@ def normalize_target(target: str) -> str: return "sum" if target.startswith("aten.mean"): return "mean" + if target.startswith("aten.all"): + return "min" if target.startswith("aten.var"): return "variance" if target.startswith("aten.min") or target.startswith("aten.amin"): @@ -116,10 +152,18 @@ def normalize_target(target: str) -> str: return "gather" if target.startswith("aten.pad"): return "pad" + if target.startswith("aten.one_hot"): + return "one_hot" + if target.startswith("aten.tril"): + return "tril" + if target.startswith("aten.unfold"): + return "unfold" if target.startswith("aten.embedding"): return "embedding" if target.startswith("aten.conv1d"): return "conv1d" + if target.startswith("aten.conv2d"): + return "conv2d" if target.startswith("aten.pow"): return "pow" if target.startswith("aten.layer_norm"): diff --git a/python/src/transpile/capture/ops.py b/python/src/transpile/ops.py similarity index 69% rename from python/src/transpile/capture/ops.py rename to python/src/transpile/ops.py index b0340e4ed..924336fef 100644 --- a/python/src/transpile/capture/ops.py +++ b/python/src/transpile/ops.py @@ -24,7 +24,14 @@ class OpSchema: OpSchema("multiply", num_inputs=2, backend_op="multiply", is_fusible=True), OpSchema("divide", num_inputs=2, backend_op="divide", is_fusible=True), OpSchema("not_equal", num_inputs=2, backend_op="not_equal"), + OpSchema("equal", num_inputs=2), + OpSchema("greater", num_inputs=2), + OpSchema("less_equal", num_inputs=2), + OpSchema("logical_and", num_inputs=2), + OpSchema("logical_or", num_inputs=2), + OpSchema("logical_not", num_inputs=1), OpSchema("abs", num_inputs=1, backend_op="abs", is_fusible=True), + OpSchema("clamp", num_inputs=1, attrs=("min", "max")), OpSchema("pow", num_inputs=1, attrs=("exponent",), backend_op="pow"), OpSchema("negate", num_inputs=1, is_fusible=True), OpSchema("scalar_add", num_inputs=1, attrs=("value",), backend_op="scalar_add", is_fusible=True), @@ -32,8 +39,10 @@ class OpSchema: OpSchema("scalar_subtract_reverse", num_inputs=1, attrs=("value",)), OpSchema("scalar_multiply", num_inputs=1, attrs=("value",), backend_op="scalar_multiply", is_fusible=True), OpSchema("scalar_divide", num_inputs=1, attrs=("value",), backend_op="scalar_divide", is_fusible=True), + OpSchema("scalar_floor_divide", num_inputs=1, attrs=("value",), backend_op="scalar_floor_divide"), OpSchema("scalar_divide_reverse", num_inputs=1, attrs=("value",)), OpSchema("scalar_not_equal", num_inputs=1, attrs=("value",), backend_op="scalar_not_equal"), + OpSchema("scalar_equal", num_inputs=1, attrs=("value",)), OpSchema("scalar_exp", num_inputs=1, backend_op="scalar_exp", is_fusible=True), OpSchema("scalar_sqrt", num_inputs=1, backend_op="scalar_sqrt", is_fusible=True), OpSchema("scalar_log", num_inputs=1, backend_op="scalar_log", is_fusible=True), @@ -42,27 +51,43 @@ class OpSchema: OpSchema("precision_cast", num_inputs=1, attrs=("dtype",), backend_op="precision_cast"), OpSchema("view", num_inputs=1, attrs=("shape",), backend_op="reshape", is_layout=True), OpSchema("permute", num_inputs=1, attrs=("permutation",), backend_op="permute", is_layout=True), + OpSchema("squeeze", num_inputs=1, attrs=("dim",), is_layout=True), + OpSchema("movedim", num_inputs=1, attrs=("source", "destination"), is_layout=True), OpSchema("slice", num_inputs=1, attrs=("axis", "start", "end", "step"), backend_op="slice"), OpSchema("split_with_sizes", num_inputs=1, attrs=("sizes", "axis")), OpSchema("chunk", num_inputs=1, attrs=("chunks", "axis")), OpSchema("ones", num_inputs=0, attrs=("shape", "dtype")), + OpSchema("one_hot", num_inputs=1, attrs=("num_classes",)), + OpSchema("tril", num_inputs=1, attrs=("diagonal",)), + OpSchema("unfold", num_inputs=1, attrs=("dimension", "size", "step")), OpSchema("index", num_inputs=1, attrs=("axis", "index_value"), backend_op="index"), + OpSchema("advanced_index", num_inputs=None), + OpSchema( + "where", + num_inputs=None, + attrs=("true_is_scalar", "true_value", "false_is_scalar", "false_value"), + ), + OpSchema("masked_fill", num_inputs=2, attrs=("value",)), + OpSchema("masked_scatter", num_inputs=3), + OpSchema("getitem", num_inputs=1, attrs=("index",)), OpSchema("cat", num_inputs=None, attrs=("axis",), backend_op="cat"), OpSchema("matmul", num_inputs=2, backend_op="matmul"), OpSchema("linear", num_inputs=None, attrs=("has_bias",), backend_op="matmul"), OpSchema("addmm", num_inputs=3), - OpSchema("gather", num_inputs=2, backend_op="gather"), + OpSchema("gather", num_inputs=2, attrs=("axis",), backend_op="gather"), OpSchema("embedding", num_inputs=2, backend_op="embedding_from_tensor"), OpSchema("sum", num_inputs=1, attrs=("axis", "keepdim"), backend_op="sum", is_reduction=True), OpSchema("mean", num_inputs=1, attrs=("axis", "keepdim"), backend_op="mean", is_reduction=True), OpSchema("variance", num_inputs=1, attrs=("axis", "keepdim"), backend_op="variance", is_reduction=True), OpSchema("min", num_inputs=1, attrs=("axis", "keepdim"), backend_op="min", is_reduction=True), OpSchema("max", num_inputs=1, attrs=("axis", "keepdim"), backend_op="max", is_reduction=True), + OpSchema("cumsum", num_inputs=1, attrs=("axis",), backend_op="cumsum"), OpSchema("relu", num_inputs=1, backend_op="relu", is_fusible=True), OpSchema("silu", num_inputs=1, backend_op="silu", is_fusible=True), OpSchema("gelu", num_inputs=1, backend_op="gelu", is_fusible=True), OpSchema("gelu_erf", num_inputs=1, backend_op="gelu_erf", is_fusible=True), OpSchema("sigmoid", num_inputs=1, backend_op="sigmoid", is_fusible=True), + OpSchema("glu", num_inputs=1, attrs=("axis",), backend_op="glu"), OpSchema("tanh", num_inputs=1, backend_op="tanh", is_fusible=True), OpSchema("softplus", num_inputs=1), OpSchema("softmax", num_inputs=1, attrs=("axis",), backend_op="softmax"), @@ -70,11 +95,14 @@ class OpSchema: OpSchema("group_norm", num_inputs=3, attrs=("num_groups", "eps"), backend_op="group_norm"), OpSchema("batch_norm", num_inputs=5, attrs=("axis", "eps"), backend_op="batch_norm"), OpSchema("conv1d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv1d"), + OpSchema("conv2d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv2d"), OpSchema("pad", num_inputs=1, attrs=("pads", "value", "mode")), + OpSchema("lstm_cell", num_inputs=7, backend_op="lstm_cell"), # CUSTOM / FUSED OPS OpSchema("rms_norm", num_inputs=2, attrs=("eps",), backend_op="rms_norm"), OpSchema("rope", num_inputs=1, attrs=("theta", "position_offset"), backend_op="rope"), + OpSchema("rel_pos_bias", num_inputs=2, attrs=("scale",), backend_op="rel_pos_bias"), OpSchema( "attention", num_inputs=3, @@ -84,7 +112,56 @@ class OpSchema: OpSchema( "attention_block", num_inputs=None, - attrs=("scale", "is_causal", "window_size", "has_gate", "has_bias", "attention_output_shape"), + attrs=( + "scale", + "is_causal", + "window_size", + "has_mask", + "additive_mask", + "has_gate", + "has_bias", + "attention_output_shape", + ), + is_fusible=True, + ), + OpSchema( + "self_attention_block", + num_inputs=None, + attrs=( + "scale", + "is_causal", + "window_size", + "has_mask", + "additive_mask", + "has_gate", + "has_bias", + "has_query_projection_bias", + "has_query_add", + "has_rel_query_add", + "has_key_projection_bias", + "has_value_projection_bias", + "has_rel_pos_bias", + "has_relative_key_projection_bias", + "query_shape", + "key_shape", + "value_shape", + "relative_key_shape", + "rel_pos_scale", + "attention_output_shape", + ), + is_fusible=True, + ), + OpSchema( + "conv_module", + num_inputs=None, + attrs=( + "eps", + "has_pointwise1_bias", + "has_depthwise_bias", + "has_pointwise2_bias", + "depthwise_kernel_size", + "depthwise_padding", + ), is_fusible=True, ), OpSchema( diff --git a/python/src/transpile/optimize_graph.py b/python/src/transpile/optimize_graph.py new file mode 100644 index 000000000..c8822bdac --- /dev/null +++ b/python/src/transpile/optimize_graph.py @@ -0,0 +1,1280 @@ +from __future__ import annotations + +from dataclasses import dataclass +import re + +import torch + +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.canonicalize.utils import normalize_dtype_name +from src.transpile.canonicalize.utils import remove_node +from src.transpile.canonicalize.utils import rebuild_graph +from src.transpile.fusion import match_attention +from src.transpile.fusion import match_attention_block +from src.transpile.fusion import match_conv_module +from src.transpile.fusion import match_gated_deltanet +from src.transpile.fusion import match_gated_mlp +from src.transpile.fusion import match_lstm_cell +from src.transpile.fusion import match_rel_pos_bias +from src.transpile.fusion import match_rms_norm +from src.transpile.fusion import match_rope +from src.transpile.fusion import match_self_attention_block +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.model_patterns import GOLD_PATTERNS +from src.transpile.normalize import dtype_to_ir + + +@dataclass(frozen=True) +class DetectedPattern: + name: str + anchor_node_id: str + node_ids: tuple[str, ...] + value_ids: tuple[str, ...] + details: dict[str, object] + + +@dataclass(frozen=True) +class FusionConfig: + enable_gated_deltanet: bool = True + enable_lstm_cell: bool = True + enable_rms_norm: bool = True + enable_rope: bool = True + enable_rel_pos_bias: bool = True + enable_attention: bool = True + enable_self_attention_block: bool = True + enable_attention_block: bool = True + enable_conv_module: bool = True + enable_add_clipped: bool = True + + +_DECODER_SELF_ATTN_OUTPUT_WEIGHT_RE = re.compile(r"(?:^|.*\.)layers\.(\d+)\.self_attn\.(?:o_proj|out_proj)\.weight$") +_CAPTURED_LAYER_SLICE_WEIGHT_RE = re.compile( + r"(?:^|.*)backbone_layers_slice_none__\d+__none____modules__(\d+)___self_attn_(?:o_proj|out_proj)_weight$" +) + + +def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: + config = config or FusionConfig() + verify_ir(graph) + canonicalize_exported_graph(graph) + enable_gemma4_attention_block_fusions = not _is_gemma4_graph(graph) + + for _ in range(max_passes): + changed = False + if config.enable_gated_deltanet and fuse_gated_deltanet(graph): + changed = True + if config.enable_lstm_cell and fuse_lstm_cells(graph): + changed = True + if config.enable_rms_norm: + if fuse_rms_norm(graph): + changed = True + if fuse_rms_norm_scale_multiply(graph): + changed = True + if config.enable_rope and fuse_rope(graph): + changed = True + if config.enable_rel_pos_bias and fuse_rel_pos_bias(graph): + changed = True + if config.enable_attention and fuse_attention(graph): + changed = True + if ( + enable_gemma4_attention_block_fusions + and config.enable_self_attention_block + and fuse_self_attention_blocks(graph) + ): + changed = True + if ( + enable_gemma4_attention_block_fusions + and config.enable_attention_block + and fuse_attention_blocks(graph) + ): + changed = True + if normalize_gemma4_decoder_attention_semantics(graph): + changed = True + if config.enable_conv_module and fuse_conv_modules(graph): + changed = True + if config.enable_add_clipped and fuse_add_clipped(graph): + changed = True + if not changed: + break + canonicalize_exported_graph(graph) + + annotate_gold_patterns(graph) + _prune_unused_inputs(graph) + verify_ir(graph) + return graph + + +def fuse_gated_deltanet(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_gated_deltanet(graph, node) + if match is None: + continue + + inputs = [ + match.normalized_input_value_id, + match.qkv_weight_value_id, + match.a_weight_value_id, + match.b_weight_value_id, + match.norm_weight_value_id, + ] + if match.z_weight_value_id is not None: + inputs.append(match.z_weight_value_id) + if match.dt_bias_value_id is not None: + inputs.append(match.dt_bias_value_id) + if match.a_log_value_id is not None: + inputs.append(match.a_log_value_id) + if match.conv_weight_value_id is not None: + inputs.append(match.conv_weight_value_id) + + node.op = f"gated_deltanet_{match.mode}" + node.inputs = inputs + node.attrs = { + "num_k_heads": int(match.num_k_heads), + "num_v_heads": int(match.num_v_heads), + "key_dim": int(match.key_dim), + "value_dim": int(match.value_dim), + "eps": float(match.eps), + "chunk_size": int(match.chunk_size), + "has_z": bool(match.z_weight_value_id is not None), + "has_dt_bias": bool(match.dt_bias_value_id is not None), + "has_a_log": bool(match.a_log_value_id is not None), + "has_conv": bool(match.conv_weight_value_id is not None), + } + node.kind = "semantic" + node.meta["gated_deltanet_mode"] = match.mode + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_lstm_cells(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "multiply": + continue + + match = match_lstm_cell(graph, node) + if match is None: + continue + + bias_hh_value_id = match.bias_hh_value_id + if bias_hh_value_id is None: + bias_hh_value_id = _materialize_zero_like_constant( + graph, + match.bias_ih_value_id, + suffix="lstm_bias_hh_zero", + ) + + node.op = "lstm_cell" + node.inputs = [ + match.x_value_id, + match.h_prev_value_id, + match.c_prev_value_id, + match.weight_ih_value_id, + match.weight_hh_value_id, + match.bias_ih_value_id, + bias_hh_value_id, + ] + node.outputs = [match.h_output_value_id, match.c_output_value_id] + node.attrs = {} + node.kind = "semantic" + node.meta["lstm_cell_nodes"] = match.node_ids + node.meta["lstm_cell_bias_hh_zero"] = bool(match.bias_hh_value_id is None) + + for other_node_id in match.node_ids: + if other_node_id == node.id: + continue + graph.nodes.pop(other_node_id, None) + if other_node_id in graph.order: + graph.order.remove(other_node_id) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def annotate_gold_patterns(graph: IRGraph) -> list[DetectedPattern]: + _clear_gold_pattern_annotations(graph) + patterns = [ + *_detect_gated_mlps(graph), + *_detect_decoder_attentions(graph), + *_detect_transformer_blocks(graph), + ] + + for pattern in patterns: + for node_id in pattern.node_ids: + node = graph.nodes.get(node_id) + if node is None: + continue + node.meta.setdefault("gold_patterns", []) + node.meta["gold_patterns"].append(pattern.name) + + anchor = graph.nodes.get(pattern.anchor_node_id) + if anchor is not None: + anchor.meta["gold_pattern_anchor"] = True + anchor.meta["gold_pattern_details"] = pattern.details + + graph.meta["gold_patterns_catalog"] = tuple(pattern.name for pattern in GOLD_PATTERNS) + graph.meta["detected_gold_patterns"] = patterns + return patterns + + +def summarize_detected_gold_patterns(graph: IRGraph) -> dict[str, int]: + summary: dict[str, int] = {} + for pattern in annotate_gold_patterns(graph): + summary[pattern.name] = summary.get(pattern.name, 0) + 1 + return summary + + +def fuse_rms_norm(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"multiply", "type_as", "precision_cast"}: + continue + + match = match_rms_norm(graph, node) + if match is None: + continue + input_value = graph.values.get(match.input_value_id) + if input_value is not None and normalize_dtype_name(input_value.dtype) == "fp32": + # Keep FP32 normalization paths unfused so they can execute with FP32 + # reductions and elementwise math instead of the FP16-only RMS kernel. + continue + + weight_value_id = match.weight_value_id + if weight_value_id is None: + if input_value is None or input_value.shape is None or not input_value.shape: + continue + hidden_dim = input_value.shape[-1] + if not isinstance(hidden_dim, int) or hidden_dim <= 0: + continue + weight_value_id = _materialize_ones_constant( + graph, + hidden_dim, + dtype=input_value.dtype, + suffix="rms_norm_ones", + ) + if float(match.weight_offset) != 0.0: + weight_value_id = _materialize_shifted_constant( + graph, + weight_value_id, + float(match.weight_offset), + suffix="rms_norm_scale", + ) + + node.op = "rms_norm" + node.inputs = [match.input_value_id, weight_value_id] + node.attrs = {"eps": float(match.eps)} + node.kind = "semantic" + node.meta["rms_weight_offset"] = float(match.weight_offset) + node.meta["rms_input_value_id"] = match.input_value_id + node.meta["rms_weight_value_id"] = weight_value_id + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rms_norm_scale_multiply(graph: IRGraph) -> bool: + if not _is_gemma4_graph(graph): + return False + + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "multiply" or len(node.inputs) != 2 or len(node.outputs) != 1: + continue + + rms_node: IRNode | None = None + rms_output_id: str | None = None + scale_value_id: str | None = None + for input_id, other_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + candidate = producer(graph, input_id) + if candidate is None or candidate.op != "rms_norm" or len(candidate.inputs) != 2: + continue + if not _is_materialized_ones_constant(graph, candidate.inputs[1]): + continue + rms_node = candidate + rms_output_id = input_id + scale_value_id = other_id + break + + if rms_node is None or rms_output_id is None or scale_value_id is None: + continue + if rms_node.outputs != [rms_output_id]: + continue + if _has_other_users(graph, rms_output_id, excluding=node.id) or rms_output_id in graph.outputs: + continue + + node.op = "rms_norm" + node.inputs = [rms_node.inputs[0], scale_value_id] + node.attrs = dict(rms_node.attrs) + node.kind = "semantic" + node.meta["rms_folded_scale_multiply"] = True + node.meta["rms_input_value_id"] = rms_node.inputs[0] + node.meta["rms_weight_value_id"] = scale_value_id + remove_node(graph, rms_node.id) + graph.values.pop(rms_output_id, None) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rope(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or not node.outputs: + continue + + match = match_rope(graph, node.outputs[0]) + if match is None or match.partial: + continue + + node.op = "rope" + node.inputs = [match.input_value_id] + node.attrs = { + "theta": float(match.theta), + "position_offset": int(match.position_offset), + } + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_attention(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: + continue + + match = match_attention(graph, node) + if match is None: + continue + + window_size = int(node.attrs.get("window_size", 0)) + if len(node.inputs) > 3: + mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) + if mask_info is not None: + node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) + if window_size == 0: + window_size = int(mask_info["window_size"]) + node.meta["window_size_source"] = "mask_pattern" + + if window_size != 0 and "window_size_source" not in node.meta: + node.meta["window_size_source"] = "import_attr" + + semantic_attrs = { + key: value + for key, value in node.attrs.items() + if key not in {"mask", "dropout_p", "scale", "is_causal"} + } + semantic_attrs.update( + { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", True)), + "window_size": window_size, + } + ) + + if ( + node.op == "attention" + and node.inputs[:3] == [match.query_value_id, match.key_value_id, match.value_value_id] + and node.attrs == semantic_attrs + ): + continue + + node.op = "attention" + node.inputs = [match.query_value_id, match.key_value_id, match.value_value_id] + node.attrs = semantic_attrs + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rel_pos_bias(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_rel_pos_bias(graph, node) + if match is None: + continue + + node.op = "rel_pos_bias" + node.inputs = [match.query_value_id, match.relative_key_value_id] + node.attrs = {"scale": float(match.scale)} + node.kind = "semantic" + node.meta["rel_pos_bias_nodes"] = match.node_ids + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_attention_blocks(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_attention_block(graph, node) + if match is None: + continue + + inputs = [match.query_value_id, match.key_value_id, match.value_value_id] + if match.mask_value_id is not None: + inputs.append(match.mask_value_id) + if match.gate_value_id is not None: + inputs.append(match.gate_value_id) + inputs.append(match.output_projection_weight_value_id) + if match.output_projection_bias_value_id is not None: + inputs.append(match.output_projection_bias_value_id) + + node.op = "attention_block" + node.inputs = inputs + node.attrs = { + "scale": float(match.scale), + "is_causal": bool(match.is_causal), + "window_size": int(match.window_size), + "has_mask": bool(match.mask_value_id is not None), + "additive_mask": bool(match.additive_mask), + "has_gate": bool(match.gate_value_id is not None), + "has_bias": bool(match.output_projection_bias_value_id is not None), + "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), + } + node.kind = "semantic" + node.meta["attention_block_source"] = match.attention_node_id + _apply_decoder_attention_layer_hints( + graph, + node, + output_projection_weight_value_id=match.output_projection_weight_value_id, + ) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: + if not _is_gemma4_graph(graph): + return False + + changed = False + if _assign_gemma4_decoder_attention_hints_from_graph_meta(graph): + changed = True + sliding_window = _graph_sliding_window(graph) + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"attention", "scaled_dot_product_attention", "attention_block"}: + continue + layer_type = str(node.meta.get("attention_layer_type") or "").strip().lower() + if layer_type == "full_attention": + mask_input_index: int | None = None + if node.op == "attention_block": + if bool(node.attrs.get("has_mask", False)) and len(node.inputs) > 3: + mask_input_index = 3 + elif len(node.inputs) > 3: + mask_input_index = 3 + + if mask_input_index is not None: + mask_value_id = node.inputs[mask_input_index] + if _gemma4_can_elide_attention_mask(graph, mask_value_id): + del node.inputs[mask_input_index] + node.meta["gemma4_full_mask_elided"] = True + changed = True + + if node.op == "attention_block" and bool(node.attrs.get("has_mask", False)): + node.attrs["has_mask"] = False + changed = True + if not bool(node.attrs.get("is_causal", False)): + node.attrs["is_causal"] = True + changed = True + if "additive_mask" in node.attrs and not bool(node.attrs.get("additive_mask", False)): + node.attrs.pop("additive_mask", None) + changed = True + if int(node.attrs.get("window_size", 0)) == 0: + seq_len = 0 + attention_output_shape = node.attrs.get("attention_output_shape") + if isinstance(attention_output_shape, (list, tuple)) and len(attention_output_shape) >= 2: + try: + seq_len = int(attention_output_shape[1]) + except (TypeError, ValueError): + seq_len = 0 + if seq_len <= 0 and node.inputs: + query_value = graph.values.get(node.inputs[0]) + if query_value is not None and query_value.shape is not None and len(query_value.shape) >= 3: + seq_len = int(query_value.shape[2]) + if seq_len > 0: + # Gemma4 full-attention layers are semantically unwindowed, but using a + # window_size >= seq_len avoids the Apple Accelerate full-window fast path + # that has been unstable for transpiled grouped-query Gemma4 graphs. + node.attrs["window_size"] = seq_len + node.meta["gemma4_full_attention_window_compat"] = True + changed = True + continue + if layer_type != "sliding_attention": + continue + if bool(node.attrs.get("additive_mask", False)): + continue + + mask_input_index: int | None = None + if node.op == "attention_block": + if bool(node.attrs.get("has_mask", False)) and len(node.inputs) > 3: + mask_input_index = 3 + elif len(node.inputs) > 3: + mask_input_index = 3 + + if mask_input_index is not None: + mask_value_id = node.inputs[mask_input_index] + if _gemma4_can_elide_attention_mask(graph, mask_value_id): + mask_info = _extract_sliding_window_mask(graph, mask_value_id) + mask_input_name = _graph_input_name(graph, mask_value_id) + del node.inputs[mask_input_index] + node.meta["gemma4_sliding_mask_elided"] = True + if mask_info is not None: + node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) + if int(node.attrs.get("window_size", 0)) == 0 and int(mask_info["window_size"]) > 0: + node.attrs["window_size"] = int(mask_info["window_size"]) + node.meta.setdefault("window_size_source", "mask_pattern") + elif mask_input_name is not None: + node.meta["gemma4_elided_mask_input_name"] = mask_input_name + changed = True + + if node.op == "attention_block" and bool(node.attrs.get("has_mask", False)): + node.attrs["has_mask"] = False + changed = True + if not bool(node.attrs.get("is_causal", False)): + node.attrs["is_causal"] = True + changed = True + if ( + int(node.attrs.get("window_size", 0)) == 0 + and sliding_window is not None + and sliding_window > 0 + ): + node.attrs["window_size"] = int(sliding_window) + node.meta.setdefault("window_size_source", "layer_type_config") + changed = True + if "additive_mask" in node.attrs and not bool(node.attrs.get("additive_mask", False)): + node.attrs.pop("additive_mask", None) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bool: + component = str(graph.meta.get("component", "") or "").strip().lower() + if component != "decoder": + return False + + layer_types = _graph_layer_types(graph) + if not layer_types: + return False + + attention_nodes: list[IRNode] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"attention", "scaled_dot_product_attention", "attention_block"}: + continue + attention_nodes.append(node) + + if len(attention_nodes) != len(layer_types): + return False + + changed = False + for layer_index, (node, layer_type) in enumerate(zip(attention_nodes, layer_types, strict=True)): + if "attention_layer_type" not in node.meta: + node.meta["attention_layer_type"] = str(layer_type) + changed = True + if "attention_layer_index" not in node.meta: + node.meta["attention_layer_index"] = int(layer_index) + changed = True + return changed + + +def _gemma4_can_elide_attention_mask( + graph: IRGraph, + value_id: str, + *, + _visited: set[str] | None = None, +) -> bool: + mask_value = graph.values.get(value_id) + if mask_value is None: + return False + if mask_value.dtype in (None, "bool"): + return True + + input_name = _graph_input_name(graph, value_id) + if isinstance(input_name, str) and "attention_mask" in input_name: + return True + + if _visited is None: + _visited = set() + if value_id in _visited: + return False + _visited.add(value_id) + + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + return False + if node.op in { + "logical_and", + "logical_or", + "scalar_equal", + "scalar_not_equal", + "greater", + "greater_equal", + "less", + "less_equal", + "aten.__and__.Tensor", + }: + return True + if node.op in {"precision_cast", "view", "expand", "permute", "slice", "index", "where"}: + return any(_gemma4_can_elide_attention_mask(graph, input_id, _visited=_visited) for input_id in node.inputs) + return False + + +def fuse_self_attention_blocks(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_self_attention_block(graph, node) + if match is None: + continue + + inputs = [match.hidden_value_id, match.query_weight_value_id] + if match.query_projection_bias_value_id is not None: + inputs.append(match.query_projection_bias_value_id) + if match.query_add_value_id is not None: + inputs.append(match.query_add_value_id) + if match.rel_query_add_value_id is not None: + inputs.append(match.rel_query_add_value_id) + inputs.append(match.key_weight_value_id) + if match.key_projection_bias_value_id is not None: + inputs.append(match.key_projection_bias_value_id) + inputs.append(match.value_weight_value_id) + if match.value_projection_bias_value_id is not None: + inputs.append(match.value_projection_bias_value_id) + if match.mask_value_id is not None: + inputs.append(match.mask_value_id) + if match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None: + inputs.append(match.relative_key_input_value_id) + inputs.append(match.relative_key_weight_value_id) + if match.relative_key_projection_bias_value_id is not None: + inputs.append(match.relative_key_projection_bias_value_id) + if match.gate_value_id is not None: + inputs.append(match.gate_value_id) + inputs.append(match.output_projection_weight_value_id) + if match.output_projection_bias_value_id is not None: + inputs.append(match.output_projection_bias_value_id) + + node.op = "self_attention_block" + node.inputs = inputs + node.attrs = { + "scale": float(match.scale), + "is_causal": bool(match.is_causal), + "window_size": int(match.window_size), + "has_mask": bool(match.mask_value_id is not None), + "additive_mask": bool(match.additive_mask), + "has_gate": bool(match.gate_value_id is not None), + "has_bias": bool(match.output_projection_bias_value_id is not None), + "has_query_projection_bias": bool(match.query_projection_bias_value_id is not None), + "has_query_add": bool(match.query_add_value_id is not None), + "has_rel_query_add": bool(match.rel_query_add_value_id is not None), + "has_key_projection_bias": bool(match.key_projection_bias_value_id is not None), + "has_value_projection_bias": bool(match.value_projection_bias_value_id is not None), + "has_rel_pos_bias": bool(match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None), + "has_relative_key_projection_bias": bool(match.relative_key_projection_bias_value_id is not None), + "query_shape": tuple(int(dim) for dim in match.query_shape), + "key_shape": tuple(int(dim) for dim in match.key_shape), + "value_shape": tuple(int(dim) for dim in match.value_shape), + "relative_key_shape": tuple(int(dim) for dim in match.relative_key_shape) if match.relative_key_shape is not None else (), + "rel_pos_scale": float(match.rel_pos_scale) if match.rel_pos_scale is not None else 1.0, + "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), + } + node.kind = "semantic" + node.meta["self_attention_block_source"] = match.attention_node_id + _apply_decoder_attention_layer_hints( + graph, + node, + output_projection_weight_value_id=match.output_projection_weight_value_id, + ) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_add_clipped(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "add" or len(node.inputs) != 2: + continue + + lhs = strip_passthrough(graph, node.inputs[0]) + rhs = strip_passthrough(graph, node.inputs[1]) + if not (_looks_like_gemma_residual_add(graph, lhs, rhs) or _looks_like_gemma_residual_add(graph, rhs, lhs)): + continue + + node.op = "add_clipped" + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_conv_modules(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_conv_module(graph, node) + if match is None: + continue + + inputs = [ + match.input_value_id, + match.pointwise1_weight_value_id, + ] + if match.pointwise1_bias_value_id is not None: + inputs.append(match.pointwise1_bias_value_id) + inputs.append(match.depthwise_weight_value_id) + if match.depthwise_bias_value_id is not None: + inputs.append(match.depthwise_bias_value_id) + inputs.extend( + [ + match.batch_norm_weight_value_id, + match.batch_norm_bias_value_id, + match.batch_norm_running_mean_value_id, + match.batch_norm_running_var_value_id, + match.pointwise2_weight_value_id, + ] + ) + if match.pointwise2_bias_value_id is not None: + inputs.append(match.pointwise2_bias_value_id) + + node.op = "conv_module" + node.inputs = inputs + node.attrs = { + "eps": float(match.eps), + "has_pointwise1_bias": bool(match.pointwise1_bias_value_id is not None), + "has_depthwise_bias": bool(match.depthwise_bias_value_id is not None), + "has_pointwise2_bias": bool(match.pointwise2_bias_value_id is not None), + "depthwise_kernel_size": int(match.depthwise_kernel_size), + "depthwise_padding": int(match.depthwise_padding), + } + node.kind = "semantic" + node.meta["conv_module_nodes"] = match.node_ids + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _materialize_shifted_constant(graph: IRGraph, value_id: str, offset: float, *, suffix: str) -> str: + base = graph.constants[value_id] + if not isinstance(base, torch.Tensor): + raise NotImplementedError(f"expected tensor constant for {value_id}, got {type(base).__name__}") + + new_value_id = f"{value_id}_{suffix}" + if new_value_id in graph.constants: + return new_value_id + + shifted = base.detach().cpu() + offset + graph.constants[new_value_id] = shifted + meta: dict[str, object] = {} + source_value = graph.values.get(value_id) + if _is_gemma4_graph(graph) and float(offset) == 1.0 and source_value is not None: + path = source_value.meta.get("path") + kind = source_value.meta.get("kind") + source_name = source_value.meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + meta = dict(source_value.meta) + meta["materialized_from_value_id"] = value_id + meta["materialized_by_op"] = "scalar_add" + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=tuple(shifted.shape), + dtype=dtype_to_ir(shifted.dtype), + producer=None, + users=[], + meta=meta, + ) + return new_value_id + + +def _materialize_zero_like_constant(graph: IRGraph, value_id: str, *, suffix: str) -> str: + base = graph.constants[value_id] + new_value_id = f"{value_id}_{suffix}" + if new_value_id in graph.constants: + return new_value_id + + if isinstance(base, torch.Tensor): + zero = torch.zeros_like(base.detach().cpu()) + else: + value = graph.values.get(value_id) + if value is None or value.shape is None or value.dtype is None: + raise NotImplementedError( + f"expected tensor constant metadata for {value_id}, got {type(base).__name__}" + ) + dtype_map = { + "bool": torch.bool, + "u8": torch.uint8, + "i8": torch.int8, + "i16": torch.int16, + "i32": torch.int32, + "i64": torch.int64, + "fp16": torch.float16, + "bf16": torch.bfloat16, + "fp32": torch.float32, + "fp64": torch.float64, + } + torch_dtype = dtype_map.get(str(value.dtype)) + if torch_dtype is None: + raise NotImplementedError(f"unsupported IR dtype for zero-like constant: {value.dtype}") + zero = torch.zeros(tuple(int(dim) for dim in value.shape), dtype=torch_dtype) + + graph.constants[new_value_id] = zero + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=tuple(zero.shape), + dtype=dtype_to_ir(zero.dtype), + producer=None, + users=[], + ) + return new_value_id + + +def _materialize_ones_constant(graph: IRGraph, size: int, *, dtype: str | None, suffix: str) -> str: + dtype_map = { + "fp16": torch.float16, + "bf16": torch.bfloat16, + "fp32": torch.float32, + "fp64": torch.float64, + } + torch_dtype = dtype_map.get(dtype, torch.float32) + new_value_id = f"c_{suffix}_{size}_{dtype or 'fp32'}" + if new_value_id in graph.constants: + return new_value_id + + value = torch.ones((size,), dtype=torch_dtype) + graph.constants[new_value_id] = value + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=(size,), + dtype=dtype_to_ir(value.dtype), + producer=None, + users=[], + ) + return new_value_id + + +def _is_gemma4_graph(graph: IRGraph) -> bool: + return str(graph.meta.get("adapter_family") or graph.meta.get("family") or "").lower() == "gemma4" + + +def _prune_unused_inputs(graph: IRGraph) -> bool: + kept_inputs: list[str] = [] + changed = False + for value_id in graph.inputs: + value = graph.values.get(value_id) + if value_id in graph.outputs: + kept_inputs.append(value_id) + continue + if value is not None and value.users: + kept_inputs.append(value_id) + continue + changed = True + + if not changed: + return False + + graph.inputs = kept_inputs + rebuild_graph(graph) + return True + + +def _graph_input_name(graph: IRGraph, value_id: str) -> str | None: + input_names = _coerce_string_tuple(graph.meta.get("input_names")) + match = re.match(r"^v_args_(\d+)$", value_id) + if match is not None: + input_index = int(match.group(1)) + if input_index < len(input_names): + return input_names[input_index] + try: + input_index = list(graph.inputs).index(value_id) + except ValueError: + return None + if input_index < len(input_names): + return input_names[input_index] + return None + + +def _is_materialized_ones_constant(graph: IRGraph, value_id: str) -> bool: + value = graph.constants.get(value_id) + if not isinstance(value, torch.Tensor): + return False + if value.numel() == 0: + return False + return bool(torch.all(value.detach().cpu() == 1).item()) + + +def _has_other_users(graph: IRGraph, value_id: str, *, excluding: str) -> bool: + for other_id, other in graph.nodes.items(): + if other_id == excluding: + continue + if value_id in other.inputs: + return True + return False + + +def _clear_gold_pattern_annotations(graph: IRGraph) -> None: + for node in graph.nodes.values(): + node.meta.pop("gold_patterns", None) + node.meta.pop("gold_pattern_anchor", None) + node.meta.pop("gold_pattern_details", None) + + +def _detect_gated_mlps(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_gated_mlp(graph, node) + if match is None: + continue + + pattern_name = "gated_mlp_gelu" if match.activation == "gelu" else "gated_mlp_silu" + patterns.append( + DetectedPattern( + name=pattern_name, + anchor_node_id=node.id, + node_ids=match.node_ids, + value_ids=(match.input_value_id,), + details={ + "activation": match.activation, + "input_value_id": match.input_value_id, + "gate_weight_value_id": match.gate_weight_value_id, + "up_weight_value_id": match.up_weight_value_id, + "down_weight_value_id": match.down_weight_value_id, + }, + ) + ) + return patterns + + +def _detect_decoder_attentions(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: + continue + + match = match_attention(graph, node) + if match is None: + continue + + q_input, k_input, v_input = match.source_input_value_ids + pattern_name = "decoder_attention_gqa" if match.has_gqa_repeat else "decoder_attention" + + node_ids = set(match.node_ids) + details = { + "input_value_ids": (q_input, k_input, v_input), + "q_linear_weight_value_id": match.weight_value_ids[0], + "k_linear_weight_value_id": match.weight_value_ids[1], + "v_linear_weight_value_id": match.weight_value_ids[2], + "has_rope": bool(match.has_rope), + "has_qk_rms_norm": bool(match.has_qk_norm), + "has_gqa_repeat": bool(match.has_gqa_repeat), + "is_causal": bool(match.is_causal), + "scale": float(match.scale), + "window_size_hint": int(match.window_size), + } + + if len(node.inputs) > 3: + details["mask_value_id"] = node.inputs[3] + mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) + if mask_info is not None: + details["mask_pattern"] = "sliding_window_attention_mask" + details["window_size_hint"] = int(mask_info["window_size"]) + node_ids.update(mask_info["node_ids"]) + + patterns.append( + DetectedPattern( + name=pattern_name, + anchor_node_id=node.id, + node_ids=tuple(sorted(node_ids)), + value_ids=(q_input, k_input, v_input), + details=details, + ) + ) + return patterns + + +def _detect_transformer_blocks(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"add", "add_clipped"} or len(node.inputs) != 2: + continue + + rhs_pattern = _find_anchor_pattern(graph, node.inputs[1], {"gated_mlp_gelu", "gated_mlp_silu"}) + lhs_pattern = _find_anchor_pattern(graph, node.inputs[0], {"gated_mlp_gelu", "gated_mlp_silu"}) + mlp_pattern = rhs_pattern or lhs_pattern + if mlp_pattern is None: + continue + + residual_value_id = node.inputs[0] if rhs_pattern is not None else node.inputs[1] + attn_pattern = _find_anchor_pattern( + graph, + residual_value_id, + {"decoder_attention_gqa", "decoder_attention", "gemma4_partial_rope_attention"}, + ) + if attn_pattern is None: + continue + + node_ids = {node.id} + for candidate in graph.nodes.values(): + gold_patterns = candidate.meta.get("gold_patterns", []) + if attn_pattern in gold_patterns or mlp_pattern in gold_patterns: + node_ids.add(candidate.id) + + patterns.append( + DetectedPattern( + name="decoder_block_post_attn_norm" if node.op == "add_clipped" else "decoder_block_simple_residual", + anchor_node_id=node.id, + node_ids=tuple(sorted(node_ids)), + value_ids=(residual_value_id,), + details={ + "residual_value_id": residual_value_id, + "attention_pattern": attn_pattern, + "mlp_pattern": mlp_pattern, + "residual_op": node.op, + }, + ) + ) + return patterns + + +def _looks_like_gemma_residual_add(graph: IRGraph, residual_value_id: str, branch_value_id: str) -> bool: + branch_node = producer(graph, strip_passthrough(graph, branch_value_id)) + if branch_node is None or branch_node.op != "rms_norm": + return False + return strip_passthrough(graph, residual_value_id) != strip_passthrough(graph, branch_node.inputs[0]) + + +def _find_anchor_pattern(graph: IRGraph, value_id: str, names: set[str]) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + + for name in node.meta.get("gold_patterns", []): + if name in names and node.meta.get("gold_pattern_anchor"): + return name + + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _extract_sliding_window_mask(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node_ids: set[str] = set() + top_and = producer(graph, strip_passthrough(graph, value_id)) + if top_and is None or top_and.op not in {"aten.__and__.Tensor", "logical_and"}: + return None + node_ids.add(top_and.id) + + stack = list(top_and.inputs) + saw_diff = False + saw_cumsum = False + window_candidates: list[int] = [] + + while stack: + current = strip_passthrough(graph, stack.pop()) + node = producer(graph, current) + if node is None or node.id in node_ids: + continue + node_ids.add(node.id) + + if node.op in {"aten.diff.default", "diff"}: + saw_diff = True + elif node.op in {"aten.cumsum.default", "cumsum"}: + saw_cumsum = True + elif node.op == "scalar_subtract": + maybe_window = int(node.attrs.get("value", 0)) + if maybe_window > 0: + window_candidates.append(maybe_window) + + stack.extend(node.inputs) + + if not saw_diff or not saw_cumsum: + return None + + return { + "window_size": max((candidate for candidate in window_candidates if candidate > 1), default=0), + "node_ids": tuple(sorted(node_ids)), + } + + +def _apply_decoder_attention_layer_hints( + graph: IRGraph, + node: IRNode, + *, + output_projection_weight_value_id: str, +) -> None: + layer_config = _decoder_attention_layer_config( + graph, + output_projection_weight_value_id=output_projection_weight_value_id, + ) + if layer_config is None: + return + + layer_index, layer_type, sliding_window = layer_config + node.meta.setdefault("attention_layer_index", layer_index) + node.meta.setdefault("attention_layer_type", layer_type) + node.meta.setdefault("attention_layer_source", "weight_source_name") + + if ( + layer_type == "sliding_attention" + and int(node.attrs.get("window_size", 0)) == 0 + and sliding_window is not None + and sliding_window > 0 + ): + node.attrs["window_size"] = int(sliding_window) + node.meta.setdefault("window_size_source", "layer_type_config") + + +def _decoder_attention_layer_config( + graph: IRGraph, + *, + output_projection_weight_value_id: str, +) -> tuple[int, str, int | None] | None: + layer_types = _graph_layer_types(graph) + if not layer_types: + return None + + layer_index = _decoder_attention_layer_index(graph, output_projection_weight_value_id) + if layer_index is None or layer_index < 0 or layer_index >= len(layer_types): + return None + + sliding_window = _graph_sliding_window(graph) + return (layer_index, layer_types[layer_index], sliding_window) + + +def _decoder_attention_layer_index(graph: IRGraph, output_projection_weight_value_id: str) -> int | None: + weight_value = graph.values.get(output_projection_weight_value_id) + candidates: list[str] = [output_projection_weight_value_id] + if weight_value is not None and isinstance(weight_value.meta, dict): + source_name = weight_value.meta.get("source_name") + if isinstance(source_name, str) and source_name: + candidates.insert(0, source_name) + + for candidate in candidates: + match = _DECODER_SELF_ATTN_OUTPUT_WEIGHT_RE.match(candidate) + if match is not None: + return int(match.group(1)) + match = _CAPTURED_LAYER_SLICE_WEIGHT_RE.match(candidate) + if match is not None: + return int(match.group(1)) + return None + + +def _graph_layer_types(graph: IRGraph) -> tuple[str, ...]: + layer_types = _coerce_string_tuple(graph.meta.get("layer_types")) + if layer_types: + return layer_types + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + layer_types = _coerce_string_tuple(provider_meta.get("layer_types")) + if layer_types: + return layer_types + return () + + +def _graph_sliding_window(graph: IRGraph) -> int | None: + sliding_window = _coerce_optional_int(graph.meta.get("sliding_window")) + if sliding_window is not None: + return sliding_window + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + sliding_window = _coerce_optional_int(provider_meta.get("sliding_window")) + if sliding_window is not None: + return sliding_window + return None + + +def _coerce_string_tuple(value: object) -> tuple[str, ...]: + if not isinstance(value, (tuple, list)): + return () + result: list[str] = [] + for item in value: + if not isinstance(item, str): + return () + result.append(item) + return tuple(result) + + +def _coerce_optional_int(value: object) -> int | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return int(value) + return None diff --git a/python/src/transpile/parakeet_tdt_local.py b/python/src/transpile/parakeet_tdt_local.py new file mode 100644 index 000000000..ed2e0a275 --- /dev/null +++ b/python/src/transpile/parakeet_tdt_local.py @@ -0,0 +1,756 @@ +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import replace +import json +from pathlib import Path +import re +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from src.transpile.component_pipeline import ComponentModuleSpec +from src.transpile.audio_preprocess import prepare_native_parakeet_audio_features + + +def _cfg_get(config: dict[str, Any], key: str, default: Any = None) -> Any: + value = config.get(key, default) + return default if value is None else value + + +def _load_tensor_state_dict(model_source: str) -> dict[str, torch.Tensor]: + root = Path(model_source) + safetensors_path = root / "model.safetensors" + if safetensors_path.exists(): + from safetensors.torch import load_file + + return dict(load_file(str(safetensors_path))) + + bin_path = root / "pytorch_model.bin" + if bin_path.exists(): + loaded = torch.load(bin_path, map_location="cpu") + if isinstance(loaded, dict): + return { + str(key): value + for key, value in loaded.items() + if isinstance(value, torch.Tensor) + } + raise RuntimeError(f"unsupported Parakeet TDT checkpoint format in {model_source}") + + +@dataclass +class ParakeetTDTConfig: + model_source: str + sample_rate: int + num_mel_bins: int + hidden_dim: int + num_layers: int + attention_heads: int + attention_head_dim: int + attention_scale: float + ff_intermediate_dim: int + conv_kernel_size: int + subsampling_factor: int + subsampling_conv_channels: int + predictor_hidden_dim: int + predictor_num_layers: int + joint_dim: int + num_tdt_durations: int + tdt_durations: tuple[int, ...] + blank_id: int + vocabulary: tuple[str, ...] + encoder_hidden_act: str + + +def prepare_parakeet_tdt_audio_features( + audio_file: str | Path, + *, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, +) -> tuple[torch.Tensor, int]: + return prepare_native_parakeet_audio_features( + audio_file, + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + + +def greedy_decode_parakeet_tdt_token_ids( + *, + config: ParakeetTDTConfig, + encoder_hidden_states: np.ndarray, + initial_states: tuple[np.ndarray, ...], + step, +) -> list[int]: + hidden = np.ascontiguousarray(np.asarray(encoder_hidden_states)) + if hidden.ndim != 3 or hidden.shape[0] != 1: + raise ValueError( + f"expected encoder_hidden_states with shape [1, T, D], got {tuple(hidden.shape)}" + ) + predictor_layers = int(config.predictor_num_layers) + if len(initial_states) != predictor_layers * 2: + raise ValueError( + f"expected {predictor_layers * 2} initial state tensors, got {len(initial_states)}" + ) + + states = tuple(np.ascontiguousarray(np.asarray(state)) for state in initial_states) + durations = [int(value) for value in config.tdt_durations] + if not durations and int(config.num_tdt_durations) > 0: + durations = list(range(int(config.num_tdt_durations))) + duration_class_count = max(len(durations), int(config.num_tdt_durations)) + last_token = _initial_parakeet_tdt_blank_id(config) + emitted: list[int] = [] + time_index = 0 + + while time_index < int(hidden.shape[1]): + frame = np.ascontiguousarray(hidden[:, time_index, :]) + advanced = False + symbols_added = 0 + + while symbols_added < 10: + logits, next_states = step(frame, last_token, states) + logits_array = np.asarray(logits, dtype=np.float32) + total_classes = int(logits_array.shape[-1]) + active_duration_count = duration_class_count + if active_duration_count <= 0 or active_duration_count >= total_classes: + active_duration_count = max(1, min(total_classes - 1, int(config.num_tdt_durations) or len(durations) or 1)) + if len(durations) != active_duration_count: + durations = list(range(active_duration_count)) + token_class_count = total_classes - active_duration_count + blank_id = _effective_parakeet_tdt_blank_id( + config, + token_class_count=token_class_count, + ) + if last_token < 0 or last_token >= token_class_count: + last_token = blank_id + token_scores = logits_array[:, :token_class_count] + duration_scores = logits_array[:, token_class_count:] + next_token = int(np.argmax(token_scores[0])) + duration_index = int(np.argmax(duration_scores[0])) + skip = int(durations[min(duration_index, len(durations) - 1)]) if durations else 1 + + if next_token != blank_id: + emitted.append(next_token) + last_token = next_token + states = tuple(np.ascontiguousarray(np.asarray(state).copy()) for state in next_states) + + symbols_added += 1 + + if skip > 0: + time_index += skip + advanced = True + break + if next_token == blank_id: + time_index += 1 + advanced = True + break + + if not advanced: + time_index += 1 + + return emitted + + +def _initial_parakeet_tdt_blank_id(config: ParakeetTDTConfig) -> int: + blank_id = int(config.blank_id) + vocab_size = len(config.vocabulary) + if vocab_size > 0 and blank_id == vocab_size - 1: + return vocab_size + return blank_id + + +def _effective_parakeet_tdt_blank_id( + config: ParakeetTDTConfig, + *, + token_class_count: int, +) -> int: + blank_id = int(config.blank_id) + vocab_size = len(config.vocabulary) + if vocab_size > 0 and blank_id == vocab_size - 1 and token_class_count == vocab_size + 1: + return token_class_count - 1 + if 0 <= blank_id < token_class_count: + return blank_id + if vocab_size > 0 and token_class_count == vocab_size + 1: + return token_class_count - 1 + return max(0, token_class_count - 1) + + +def load_parakeet_tdt_config(model_source: str) -> ParakeetTDTConfig: + config_path = Path(model_source) / "config.json" + if not config_path.exists(): + raise FileNotFoundError(f"missing config.json for Parakeet TDT: {config_path}") + root = json.loads(config_path.read_text()) + encoder = root.get("encoder") or {} + decoder = root.get("decoder") or {} + prediction = decoder.get("prediction") or decoder.get("prednet") or {} + joint = root.get("joint") or {} + jointnet = joint.get("jointnet") or {} + model_defaults = root.get("model_defaults") or {} + preprocessor = root.get("preprocessor") or {} + + hidden_dim = int(_cfg_get(encoder, "d_model", _cfg_get(encoder, "hidden_size", 0))) + attention_heads = int(_cfg_get(encoder, "n_heads", _cfg_get(encoder, "num_attention_heads", 0))) + attention_head_dim = hidden_dim // max(attention_heads, 1) + tdt_durations = tuple(int(value) for value in _cfg_get(model_defaults, "tdt_durations", (0, 1, 2, 3, 4))) + vocabulary = tuple(str(value) for value in _cfg_get(joint, "vocabulary", ())) + decoder_vocab_size = int(_cfg_get(decoder, "vocab_size", len(vocabulary))) + decoding = root.get("decoding") or {} + blank_id_raw = _cfg_get(decoding, "blank_id", _cfg_get(decoder, "blank_id", None)) + if blank_id_raw is None: + blank_id = decoder_vocab_size + else: + try: + blank_id = int(blank_id_raw) + except (TypeError, ValueError): + blank_id = decoder_vocab_size + if blank_id < 0: + blank_id = decoder_vocab_size + + return ParakeetTDTConfig( + model_source=model_source, + sample_rate=int(_cfg_get(preprocessor, "sample_rate", 16000)), + num_mel_bins=int(_cfg_get(preprocessor, "features", _cfg_get(encoder, "feat_in", 128))), + hidden_dim=hidden_dim, + num_layers=int(_cfg_get(encoder, "n_layers", _cfg_get(encoder, "num_hidden_layers", 0))), + attention_heads=attention_heads, + attention_head_dim=attention_head_dim, + attention_scale=float(attention_head_dim ** -0.5 if attention_head_dim > 0 else 1.0), + ff_intermediate_dim=int( + _cfg_get( + encoder, + "ffn_hidden_size", + round(hidden_dim * float(_cfg_get(encoder, "ff_expansion_factor", 4.0))), + ) + ), + conv_kernel_size=int(_cfg_get(encoder, "conv_kernel_size", 9)), + subsampling_factor=int(_cfg_get(encoder, "subsampling_factor", 8)), + subsampling_conv_channels=int(_cfg_get(encoder, "subsampling_conv_channels", 256)), + predictor_hidden_dim=int(_cfg_get(prediction, "pred_hidden", _cfg_get(model_defaults, "pred_hidden", 640))), + predictor_num_layers=int(_cfg_get(prediction, "pred_rnn_layers", 1)), + joint_dim=int(_cfg_get(jointnet, "joint_hidden", _cfg_get(model_defaults, "joint_hidden", 640))), + num_tdt_durations=int(_cfg_get(model_defaults, "num_tdt_durations", len(tdt_durations))), + tdt_durations=tdt_durations, + blank_id=blank_id, + vocabulary=vocabulary, + encoder_hidden_act=str(_cfg_get(encoder, "activation", "silu")).lower(), + ) + + +def _copy_linear_weight(linear: nn.Linear, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + linear.weight.data.copy_(weight.to(dtype=linear.weight.dtype, device=linear.weight.device)) + if linear.bias is not None: + if bias is None: + linear.bias.data.zero_() + else: + linear.bias.data.copy_(bias.to(dtype=linear.bias.dtype, device=linear.bias.device)) + + +def _copy_conv2d_weight(conv: nn.Conv2d, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + conv.weight.data.copy_(weight.permute(0, 3, 1, 2).contiguous().to(dtype=conv.weight.dtype, device=conv.weight.device)) + if conv.bias is not None: + if bias is None: + conv.bias.data.zero_() + else: + conv.bias.data.copy_(bias.to(dtype=conv.bias.dtype, device=conv.bias.device)) + + +def _copy_conv1d_weight(conv: nn.Conv1d, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + tensor = weight + if tensor.ndim == 3 and tensor.shape[1] != conv.in_channels // conv.groups: + tensor = tensor.permute(0, 2, 1).contiguous() + conv.weight.data.copy_(tensor.to(dtype=conv.weight.dtype, device=conv.weight.device)) + if conv.bias is not None: + if bias is None: + conv.bias.data.zero_() + else: + conv.bias.data.copy_(bias.to(dtype=conv.bias.dtype, device=conv.bias.device)) + + +def _apply_activation(x: torch.Tensor, activation: str) -> torch.Tensor: + if "gelu" in activation: + return F.gelu(x) + if activation == "relu": + return F.relu(x) + return F.silu(x) + + +def _relative_position_embeddings(*, seq_len: int, hidden_dim: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + rel_positions = torch.arange(seq_len - 1, -seq_len, -1, device=device, dtype=torch.float32) + half_dim = hidden_dim // 2 + inv_freq = 1.0 / ( + 10000.0 + ** (torch.arange(0, half_dim, device=device, dtype=torch.float32) * 2.0 / float(max(hidden_dim, 1))) + ) + angles = rel_positions.unsqueeze(1) * inv_freq.unsqueeze(0) + sin = torch.sin(angles) + cos = torch.cos(angles) + embeddings = torch.stack((sin, cos), dim=-1).reshape(2 * seq_len - 1, half_dim * 2) + if embeddings.shape[1] < hidden_dim: + pad = torch.zeros( + (embeddings.shape[0], hidden_dim - embeddings.shape[1]), + device=device, + dtype=embeddings.dtype, + ) + embeddings = torch.cat((embeddings, pad), dim=1) + elif embeddings.shape[1] > hidden_dim: + embeddings = embeddings[:, :hidden_dim] + return embeddings.to(dtype=dtype) + + +def _relative_position_bias(query: torch.Tensor, relative_key: torch.Tensor, *, scale: float) -> torch.Tensor: + batch, heads, seq_len, _ = query.shape + scores = torch.matmul(query, relative_key.transpose(-1, -2)) + rel_index = ( + torch.arange(seq_len, device=query.device).view(seq_len, 1) + - torch.arange(seq_len, device=query.device).view(1, seq_len) + + (seq_len - 1) + ) + rel_index = rel_index.view(1, 1, seq_len, seq_len).expand(batch, heads, seq_len, seq_len) + gathered = scores.gather(-1, rel_index) + return gathered * float(scale) + + +class ParakeetTDTFeedForward(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.linear1 = nn.Linear(config.hidden_dim, config.ff_intermediate_dim, bias=False) + self.linear2 = nn.Linear(config.ff_intermediate_dim, config.hidden_dim, bias=False) + _copy_linear_weight(self.linear1, state_dict[f"{prefix}.linear1.weight"]) + _copy_linear_weight(self.linear2, state_dict[f"{prefix}.linear2.weight"]) + + def forward(self, x: torch.Tensor, *, activation: str) -> torch.Tensor: + return self.linear2(_apply_activation(self.linear1(x), activation)) + + +class ParakeetTDTSelfAttention(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.linear_q = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_k = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_v = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_out = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_pos = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + _copy_linear_weight(self.linear_q, state_dict[f"{prefix}.linear_q.weight"]) + _copy_linear_weight(self.linear_k, state_dict[f"{prefix}.linear_k.weight"]) + _copy_linear_weight(self.linear_v, state_dict[f"{prefix}.linear_v.weight"]) + _copy_linear_weight(self.linear_out, state_dict[f"{prefix}.linear_out.weight"]) + _copy_linear_weight(self.linear_pos, state_dict[f"{prefix}.linear_pos.weight"]) + self.pos_bias_u = nn.Parameter(state_dict[f"{prefix}.pos_bias_u"].clone()) + self.pos_bias_v = nn.Parameter(state_dict[f"{prefix}.pos_bias_v"].clone()) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + batch, seq_len, _ = x.shape + num_heads = self.config.attention_heads + head_dim = self.config.attention_head_dim + + q = self.linear_q(x).view(batch, seq_len, num_heads, head_dim) + k = self.linear_k(x).view(batch, seq_len, num_heads, head_dim) + v = self.linear_v(x).view(batch, seq_len, num_heads, head_dim) + + rel_pos = _relative_position_embeddings( + seq_len=seq_len, + hidden_dim=self.config.hidden_dim, + device=x.device, + dtype=x.dtype, + ) + rel_k = self.linear_pos(rel_pos).view(1, 2 * seq_len - 1, num_heads, head_dim) + + q_u = q + self.pos_bias_u.view(1, 1, num_heads, head_dim).to(dtype=x.dtype, device=x.device) + q_v = q + self.pos_bias_v.view(1, 1, num_heads, head_dim).to(dtype=x.dtype, device=x.device) + + q_u_heads = q_u.permute(0, 2, 1, 3) + q_v_heads = q_v.permute(0, 2, 1, 3) + k_heads = k.permute(0, 2, 1, 3) + v_heads = v.permute(0, 2, 1, 3) + rel_k_heads = rel_k.permute(0, 2, 1, 3) + + rel_bias = _relative_position_bias( + q_v_heads, + rel_k_heads, + scale=self.config.attention_scale, + ) + attn = F.scaled_dot_product_attention( + q_u_heads, + k_heads, + v_heads, + attn_mask=rel_bias, + dropout_p=0.0, + is_causal=False, + ) + attn = attn.permute(0, 2, 1, 3).reshape(batch, seq_len, self.config.hidden_dim) + return self.linear_out(attn) + + +class ParakeetTDTConformerConv(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + hidden_dim = config.hidden_dim + kernel = config.conv_kernel_size + self.pointwise_conv1 = nn.Conv1d(hidden_dim, hidden_dim * 2, kernel_size=1, bias=False) + self.depthwise_conv = nn.Conv1d( + hidden_dim, + hidden_dim, + kernel_size=kernel, + padding=kernel // 2, + groups=hidden_dim, + bias=False, + ) + self.batch_norm = nn.BatchNorm1d(hidden_dim, eps=1e-5, affine=True, track_running_stats=True) + self.pointwise_conv2 = nn.Conv1d(hidden_dim, hidden_dim, kernel_size=1, bias=False) + _copy_conv1d_weight(self.pointwise_conv1, state_dict[f"{prefix}.pointwise_conv1.weight"]) + _copy_conv1d_weight(self.depthwise_conv, state_dict[f"{prefix}.depthwise_conv.weight"]) + _copy_conv1d_weight(self.pointwise_conv2, state_dict[f"{prefix}.pointwise_conv2.weight"]) + self.batch_norm.weight.data.copy_(state_dict[f"{prefix}.batch_norm.weight"].to(dtype=self.batch_norm.weight.dtype)) + self.batch_norm.bias.data.copy_(state_dict[f"{prefix}.batch_norm.bias"].to(dtype=self.batch_norm.bias.dtype)) + self.batch_norm.running_mean.data.copy_( + state_dict[f"{prefix}.batch_norm.running_mean"].to(dtype=self.batch_norm.running_mean.dtype) + ) + self.batch_norm.running_var.data.copy_( + state_dict[f"{prefix}.batch_norm.running_var"].to(dtype=self.batch_norm.running_var.dtype) + ) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.transpose(1, 2) + x = self.pointwise_conv1(x) + x = F.glu(x, dim=1) + x = self.depthwise_conv(x) + x = self.batch_norm(x) + x = _apply_activation(x, self.config.encoder_hidden_act) + x = self.pointwise_conv2(x) + return x.transpose(1, 2) + + +class ParakeetTDTEncoderLayer(nn.Module): + def __init__(self, config: ParakeetTDTConfig, layer_index: int, state_dict: dict[str, torch.Tensor]): + super().__init__() + prefix = f"encoder.layers.{layer_index}" + self.feed_forward1 = ParakeetTDTFeedForward(config, f"{prefix}.feed_forward1", state_dict) + self.feed_forward2 = ParakeetTDTFeedForward(config, f"{prefix}.feed_forward2", state_dict) + self.self_attn = ParakeetTDTSelfAttention(config, f"{prefix}.self_attn", state_dict) + self.conv = ParakeetTDTConformerConv(config, f"{prefix}.conv", state_dict) + self.norm_feed_forward1 = nn.LayerNorm(config.hidden_dim) + self.norm_self_att = nn.LayerNorm(config.hidden_dim) + self.norm_conv = nn.LayerNorm(config.hidden_dim) + self.norm_feed_forward2 = nn.LayerNorm(config.hidden_dim) + self.norm_out = nn.LayerNorm(config.hidden_dim) + for name in ("norm_feed_forward1", "norm_self_att", "norm_conv", "norm_feed_forward2", "norm_out"): + module = getattr(self, name) + module.weight.data.copy_(state_dict[f"{prefix}.{name}.weight"].to(dtype=module.weight.dtype)) + module.bias.data.copy_(state_dict[f"{prefix}.{name}.bias"].to(dtype=module.bias.dtype)) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + 0.5 * self.feed_forward1(self.norm_feed_forward1(x), activation=self.config.encoder_hidden_act) + x = x + self.self_attn(self.norm_self_att(x)) + x = x + self.conv(self.norm_conv(x)) + x = x + 0.5 * self.feed_forward2(self.norm_feed_forward2(x), activation=self.config.encoder_hidden_act) + return self.norm_out(x) + + +class ParakeetTDTPreEncode(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + channels = config.subsampling_conv_channels + self.conv = nn.ModuleList( + [ + nn.Conv2d(1, channels, kernel_size=3, stride=2, padding=1), + nn.Identity(), + nn.Conv2d(channels, channels, kernel_size=3, stride=2, padding=1, groups=channels), + nn.Conv2d(channels, channels, kernel_size=1, stride=1, padding=0), + nn.Identity(), + nn.Conv2d(channels, channels, kernel_size=3, stride=2, padding=1, groups=channels), + nn.Conv2d(channels, channels, kernel_size=1, stride=1, padding=0), + ] + ) + _copy_conv2d_weight(self.conv[0], state_dict["encoder.pre_encode.conv.0.weight"], bias=state_dict["encoder.pre_encode.conv.0.bias"]) + _copy_conv2d_weight(self.conv[2], state_dict["encoder.pre_encode.conv.2.weight"], bias=state_dict["encoder.pre_encode.conv.2.bias"]) + _copy_conv2d_weight(self.conv[3], state_dict["encoder.pre_encode.conv.3.weight"], bias=state_dict["encoder.pre_encode.conv.3.bias"]) + _copy_conv2d_weight(self.conv[5], state_dict["encoder.pre_encode.conv.5.weight"], bias=state_dict["encoder.pre_encode.conv.5.bias"]) + _copy_conv2d_weight(self.conv[6], state_dict["encoder.pre_encode.conv.6.weight"], bias=state_dict["encoder.pre_encode.conv.6.bias"]) + projected_width = config.num_mel_bins + for _ in range(3): + projected_width = (projected_width + 2 - 3) // 2 + 1 + self.out = nn.Linear(channels * projected_width, config.hidden_dim) + _copy_linear_weight( + self.out, + state_dict["encoder.pre_encode.out.weight"], + bias=state_dict["encoder.pre_encode.out.bias"], + ) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + if input_features.ndim != 3: + raise ValueError(f"expected input_features [batch, frames, mels], got {tuple(input_features.shape)}") + x = input_features.unsqueeze(1) + x = F.relu(self.conv[0](x)) + x = F.relu(self.conv[3](self.conv[2](x))) + x = F.relu(self.conv[6](self.conv[5](x))) + x = x.permute(0, 2, 1, 3).reshape(x.shape[0], x.shape[2], -1) + return self.out(x) + + +class ParakeetTDTEncoder(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.pre_encode = ParakeetTDTPreEncode(config, state_dict) + self.layers = nn.ModuleList( + [ParakeetTDTEncoderLayer(config, index, state_dict) for index in range(config.num_layers)] + ) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + x = self.pre_encode(input_features) + for layer in self.layers: + x = layer(x) + return x + + +class ParakeetTDTDecoderCell(nn.Module): + def __init__(self, hidden_dim: int, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.Wx = nn.Parameter(state_dict[f"{prefix}.Wx"].clone()) + self.Wh = nn.Parameter(state_dict[f"{prefix}.Wh"].clone()) + self.bias = nn.Parameter(state_dict[f"{prefix}.bias"].clone()) + self.hidden_dim = hidden_dim + + def forward(self, x: torch.Tensor, h: torch.Tensor, c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + gates = F.linear(x, self.Wx) + F.linear(h, self.Wh) + self.bias + i, f, g, o = torch.chunk(gates, 4, dim=-1) + i = torch.sigmoid(i) + f = torch.sigmoid(f) + g = torch.tanh(g) + o = torch.sigmoid(o) + c_next = f * c + i * g + h_next = o * torch.tanh(c_next) + return h_next, c_next + + +class ParakeetTDTDecoderPrediction(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + vocab_size = max(config.blank_id + 1, state_dict["decoder.prediction.embed.weight"].shape[0]) + self.embed = nn.Embedding(vocab_size, config.predictor_hidden_dim) + self.embed.weight.data.copy_(state_dict["decoder.prediction.embed.weight"].to(dtype=self.embed.weight.dtype)) + self.dec_rnn = nn.Module() + self.dec_rnn.lstm = nn.ModuleList( + [ + ParakeetTDTDecoderCell( + config.predictor_hidden_dim, + f"decoder.prediction.dec_rnn.lstm.{index}", + state_dict, + ) + for index in range(config.predictor_num_layers) + ] + ) + + def forward( + self, + token_ids: torch.Tensor, + state_h: tuple[torch.Tensor, ...], + state_c: tuple[torch.Tensor, ...], + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...], tuple[torch.Tensor, ...]]: + x = self.embed(token_ids) + if x.ndim == 3 and x.shape[1] == 1: + x = x[:, 0, :] + next_h: list[torch.Tensor] = [] + next_c: list[torch.Tensor] = [] + for index, cell in enumerate(self.dec_rnn.lstm): + h_new, c_new = cell(x, state_h[index], state_c[index]) + next_h.append(h_new) + next_c.append(c_new) + x = h_new + return x, tuple(next_h), tuple(next_c) + + +class ParakeetTDTJoint(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.enc = nn.Linear(config.hidden_dim, config.joint_dim) + self.pred = nn.Linear(config.predictor_hidden_dim, config.joint_dim) + output_classes = int(state_dict["joint.joint_net.2.weight"].shape[0]) + self.joint_net = nn.Sequential( + nn.Identity(), + nn.ReLU(), + nn.Linear( + config.joint_dim, + output_classes, + ), + ) + _copy_linear_weight(self.enc, state_dict["joint.enc.weight"], bias=state_dict["joint.enc.bias"]) + _copy_linear_weight(self.pred, state_dict["joint.pred.weight"], bias=state_dict["joint.pred.bias"]) + _copy_linear_weight( + self.joint_net[2], + state_dict["joint.joint_net.2.weight"], + bias=state_dict["joint.joint_net.2.bias"], + ) + + def forward(self, encoder_frame: torch.Tensor, predictor_hidden: torch.Tensor) -> torch.Tensor: + return self.joint_net(F.relu(self.enc(encoder_frame) + self.pred(predictor_hidden))) + + +class ParakeetTDTDecoderStep(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.prediction = ParakeetTDTDecoderPrediction(config, state_dict) + self.joint = ParakeetTDTJoint(config, state_dict) + self.config = config + + def forward(self, encoder_frame: torch.Tensor, token_ids: torch.Tensor, *state_tensors: torch.Tensor) -> tuple[torch.Tensor, ...]: + if len(state_tensors) != self.config.predictor_num_layers * 2: + raise ValueError( + f"expected {self.config.predictor_num_layers * 2} predictor state tensors, got {len(state_tensors)}" + ) + state_h = tuple(state_tensors[0::2]) + state_c = tuple(state_tensors[1::2]) + predictor_hidden, next_h, next_c = self.prediction(token_ids, state_h, state_c) + logits = self.joint(encoder_frame, predictor_hidden) + outputs: list[torch.Tensor] = [logits] + for h, c in zip(next_h, next_c, strict=True): + outputs.append(h) + outputs.append(c) + return tuple(outputs) + + +class ParakeetTDTLocalModel(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.name_or_path = config.model_source + self.family = "parakeet_tdt" + self.config = config + self.encoder = ParakeetTDTEncoder(config, state_dict) + self.decoder = nn.Module() + self.decoder.prediction = ParakeetTDTDecoderPrediction(config, state_dict) + self.joint = ParakeetTDTJoint(config, state_dict) + self.decoder_step = ParakeetTDTDecoderStep(config, state_dict) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + return self.encoder(input_features) + + def initial_decoder_state(self, *, batch_size: int, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, ...]: + state: list[torch.Tensor] = [] + for _ in range(self.config.predictor_num_layers): + state.append(torch.zeros((batch_size, self.config.predictor_hidden_dim), device=device, dtype=dtype)) + state.append(torch.zeros((batch_size, self.config.predictor_hidden_dim), device=device, dtype=dtype)) + return tuple(state) + + def greedy_decode_token_ids(self, input_features: torch.Tensor) -> list[int]: + with torch.no_grad(): + encoder_hidden = self.encoder(input_features) + batch = int(encoder_hidden.shape[0]) + if batch != 1: + raise ValueError("Parakeet TDT local greedy decode currently expects batch size 1") + states = self.initial_decoder_state( + batch_size=batch, + device=encoder_hidden.device, + dtype=encoder_hidden.dtype, + ) + state_arrays = tuple(state.detach().cpu().numpy() for state in states) + encoder_hidden_np = encoder_hidden.detach().cpu().numpy() + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + frame_tensor = torch.from_numpy(frame).to(device=encoder_hidden.device, dtype=encoder_hidden.dtype) + token_tensor = torch.tensor([[token_id]], device=encoder_hidden.device, dtype=torch.long) + state_tensors = tuple( + torch.from_numpy(value).to(device=encoder_hidden.device, dtype=encoder_hidden.dtype) + for value in state_values + ) + outputs = self.decoder_step(frame_tensor, token_tensor, *state_tensors) + logits = outputs[0].detach().cpu().numpy() + next_states = tuple(output.detach().cpu().numpy() for output in outputs[1:]) + return logits, next_states + + return greedy_decode_parakeet_tdt_token_ids( + config=self.config, + encoder_hidden_states=encoder_hidden_np, + initial_states=state_arrays, + step=_step, + ) + + def decode_token_ids(self, token_ids: list[int]) -> str: + pieces: list[str] = [] + for token_id in token_ids: + if token_id < 0 or token_id >= len(self.config.vocabulary): + continue + piece = self.config.vocabulary[token_id] + if piece.startswith("<|") and piece.endswith("|>"): + continue + pieces.append(piece) + text = "".join(pieces).replace("▁", " ") + return re.sub(r"\s+", " ", text).strip() + + +def load_parakeet_tdt_local_model(model_source: str, *, torch_dtype: torch.dtype) -> ParakeetTDTLocalModel: + config = load_parakeet_tdt_config(model_source) + state_dict = _load_tensor_state_dict(model_source) + predictor_vocab_size = int(state_dict["decoder.prediction.embed.weight"].shape[0]) + if config.blank_id < 0 or config.blank_id >= predictor_vocab_size: + config = replace(config, blank_id=predictor_vocab_size - 1) + elif config.blank_id == len(config.vocabulary) - 1 and predictor_vocab_size == len(config.vocabulary) + 1: + config = replace(config, blank_id=predictor_vocab_size - 1) + model = ParakeetTDTLocalModel(config, state_dict).eval() + model.to(dtype=torch_dtype) + return model + + +def build_parakeet_tdt_component_specs( + model: ParakeetTDTLocalModel, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, +) -> list[ComponentModuleSpec]: + input_features = named_tensors["input_features"] + example_hidden = model.encoder(input_features) + batch_size = int(example_hidden.shape[0]) + initial_states = model.initial_decoder_state( + batch_size=batch_size, + device=example_hidden.device, + dtype=example_hidden.dtype, + ) + example_frame = example_hidden[:, :1, :].reshape(batch_size, example_hidden.shape[-1]) + example_token_id = torch.full( + (batch_size,), + int(model.config.blank_id), + device=example_hidden.device, + dtype=torch.long, + ) + + state_input_keys: list[str] = [] + state_output_keys: list[str] = [] + for index in range(model.config.predictor_num_layers): + state_input_keys.extend((f"state_h_{index}", f"state_c_{index}")) + state_output_keys.extend((f"state_h_{index}", f"state_c_{index}")) + + common_graph_meta = { + "weights_dir": weights_dir, + "task": "tdt_transcription", + "adapter_family": "parakeet_tdt", + } + return [ + ComponentModuleSpec( + component="audio_encoder", + module=model.encoder, + example_inputs=(input_features,), + input_keys=("input_features",), + output_keys=("encoder_hidden_states",), + graph_meta={**common_graph_meta, "component": "audio_encoder"}, + metadata={"family": "parakeet_tdt", "task": "tdt_transcription"}, + ), + ComponentModuleSpec( + component="decoder", + module=model.decoder_step, + example_inputs=(example_frame, example_token_id, *initial_states), + input_keys=("encoder_frame", "token_ids", *state_input_keys), + output_keys=("step_logits", *state_output_keys), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "parakeet_tdt", "task": "tdt_transcription"}, + ), + ] diff --git a/python/src/transpile/runtime_compat.py b/python/src/transpile/runtime_compat.py new file mode 100644 index 000000000..f0baa4b19 --- /dev/null +++ b/python/src/transpile/runtime_compat.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +import ctypes +import importlib +import sys +from typing import Any + + +class _MissingFFIFunction: + _cactus_missing_symbol = True + + def __init__(self, name: str): + self.__name__ = name + self.argtypes = None + self.restype = ctypes.c_int + + def __call__(self, *args: Any, **kwargs: Any) -> int: + raise RuntimeError(f"Cactus runtime is missing required symbol: {self.__name__}") + + +_ORIG_CDLL_GETATTR = ctypes.CDLL.__getattr__ + + +def _patched_cdll_getattr(self: ctypes.CDLL, name: str): + try: + return _ORIG_CDLL_GETATTR(self, name) + except AttributeError: + if not name.startswith("cactus_"): + raise + missing = _MissingFFIFunction(name) + setattr(self, name, missing) + return missing + + +def _load_runtime_modules(): + if "src.cactus" in sys.modules and "src.graph" in sys.modules: + return sys.modules["src.cactus"], sys.modules["src.graph"] + + ctypes.CDLL.__getattr__ = _patched_cdll_getattr + try: + cactus_module = importlib.import_module("src.cactus") + graph_module = importlib.import_module("src.graph") + finally: + ctypes.CDLL.__getattr__ = _ORIG_CDLL_GETATTR + return cactus_module, graph_module + + +def _patch_graph_runtime(graph_module, cactus_module) -> None: + Graph = graph_module.Graph + if getattr(Graph, "_transpile_runtime_compat_patched", False): + return + + _lib = cactus_module._lib + _err = cactus_module._err + cactus_node_t = cactus_module.cactus_node_t + + def _has_symbol(name: str) -> bool: + symbol = getattr(_lib, name, None) + return symbol is not None and not getattr(symbol, "_cactus_missing_symbol", False) + + def _ensure_compare_tensor(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP16): + return tensor + return self.precision_cast(tensor, Graph.FP16) + + def _ensure_scalar_tensor(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP16): + return tensor + return self.precision_cast(tensor, Graph.FP16) + + def _ensure_fp16_activation(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP32): + return self.precision_cast(tensor, Graph.FP16) + return tensor + + def _approx_nonzero_mask(self, tensor): + tensor = _ensure_scalar_tensor(self, tensor) + # Gemma4 compare paths in v2 only need stable 0/1-style masks for + # discrete values such as token-type ids and prebuilt boolean masks. + magnitude = self.abs(tensor) + shifted = self.scalar_add(magnitude, -0.5) + sharpened = self.scalar_multiply(shifted, 16.0) + return self.sigmoid(sharpened) + + orig_conv2d = Graph.conv2d + orig_not_equal = Graph.not_equal + orig_scalar_not_equal = Graph.scalar_not_equal + orig_transpose = Graph.transpose + orig_permute = Graph.permute + orig_scalar_add = Graph.scalar_add + orig_scalar_subtract = Graph.scalar_subtract + orig_scalar_multiply = Graph.scalar_multiply + orig_scalar_divide = Graph.scalar_divide + orig_scalar_exp = Graph.scalar_exp + orig_scalar_sqrt = Graph.scalar_sqrt + orig_scalar_cos = Graph.scalar_cos + orig_scalar_sin = Graph.scalar_sin + orig_scalar_log = Graph.scalar_log + orig_add = Graph.add + orig_subtract = Graph.subtract + orig_multiply = Graph.multiply + orig_divide = Graph.divide + orig_concat = Graph.concat + orig_cat = Graph.cat + orig_abs = Graph.abs + orig_pow = Graph.pow + + def matmul(self, a, b, pretransposed_rhs=False, backend=Graph.CPU, output_dtype=None): + a = _ensure_fp16_activation(self, a) + b = _ensure_fp16_activation(self, b) + out = cactus_node_t() + rc = _lib.cactus_graph_matmul( + self.h, + cactus_node_t(a.id), + cactus_node_t(b.id), + ctypes.c_bool(bool(pretransposed_rhs)), + ctypes.c_int32(int(backend)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_matmul failed")) + result = self._tensor_from_node(out.value) + if output_dtype is not None and int(output_dtype) != int(result.dtype): + result = self.precision_cast(result, int(output_dtype)) + return result + + def gather(self, tensor, indices, axis=0): + tensor = self._ensure_tensor(tensor) + indices = self._ensure_tensor(indices) + if int(axis) != 0: + raise NotImplementedError( + f"transpiler runtime compatibility only supports gather(axis=0), got axis={axis}" + ) + out = cactus_node_t() + rc = _lib.cactus_graph_gather( + self.h, + cactus_node_t(tensor.id), + cactus_node_t(indices.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_gather failed") + return self._tensor_from_node(out.value) + + def conv2d(self, x, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + def _pair(value: Any) -> tuple[int, int]: + if isinstance(value, (tuple, list)): + if len(value) != 2: + raise ValueError(f"expected pair for conv2d parameter, got {value!r}") + return int(value[0]), int(value[1]) + return int(value), int(value) + + stride_hw = _pair(stride) + padding_hw = _pair(padding) + dilation_hw = _pair(dilation) + groups_int = int(groups) + kernel_hw = tuple(int(dim) for dim in self._ensure_tensor(weight).shape[-2:]) + + if _has_symbol("cactus_graph_conv2d"): + return orig_conv2d( + self, + x, + weight, + bias=bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + ) + + if ( + kernel_hw == (3, 3) + and stride_hw == (2, 2) + and padding_hw == (1, 1) + and dilation_hw == (1, 1) + ): + if groups_int == 1: + return self.conv2d_k3s2p1(x, weight, bias=bias) + if len(weight.shape) >= 1 and groups_int == int(weight.shape[0]): + return self.conv2d_depthwise_k3s2p1(x, weight, bias=bias) + + if ( + kernel_hw == (3, 3) + and stride_hw == (1, 1) + and padding_hw == (1, 1) + and dilation_hw == (1, 1) + and groups_int == 1 + and _has_symbol("cactus_graph_conv2d_k3s1p1") + ): + return self.conv2d_k3s1p1(x, weight, bias=bias) + + if ( + kernel_hw == (1, 1) + and stride_hw == (1, 1) + and padding_hw == (0, 0) + and dilation_hw == (1, 1) + and groups_int == 1 + ): + return self.conv2d_pointwise_1x1(x, weight, bias=bias) + + raise NotImplementedError( + "v2 runtime does not expose generic conv2d for this configuration: " + f"kernel={kernel_hw} stride={stride_hw} padding={padding_hw} " + f"dilation={dilation_hw} groups={groups_int}" + ) + + def scalar_not_equal(self, x, value): + if _has_symbol("cactus_graph_scalar_not_equal"): + return orig_scalar_not_equal(self, x, value) + x = _ensure_compare_tensor(self, x) + delta = self.scalar_add(x, -float(value)) + return _approx_nonzero_mask(self, delta) + + def not_equal(self, a, b): + if _has_symbol("cactus_graph_not_equal"): + return orig_not_equal(self, a, b) + a = _ensure_compare_tensor(self, a) + b = _ensure_compare_tensor(self, b) + delta = self.subtract(a, b) + return _approx_nonzero_mask(self, delta) + + def transpose(self, x, backend=Graph.CPU): + return orig_transpose(self, _ensure_scalar_tensor(self, x), backend=backend) + + def permute(self, x, permutation, backend=Graph.CPU): + return orig_permute(self, _ensure_scalar_tensor(self, x), permutation, backend=backend) + + def scalar_add(self, x, value): + return orig_scalar_add(self, _ensure_scalar_tensor(self, x), value) + + def scalar_subtract(self, x, value): + return orig_scalar_subtract(self, _ensure_scalar_tensor(self, x), value) + + def scalar_multiply(self, x, value): + return orig_scalar_multiply(self, _ensure_scalar_tensor(self, x), value) + + def scalar_divide(self, x, value): + return orig_scalar_divide(self, _ensure_scalar_tensor(self, x), value) + + def scalar_exp(self, x): + return orig_scalar_exp(self, _ensure_scalar_tensor(self, x)) + + def scalar_sqrt(self, x): + return orig_scalar_sqrt(self, _ensure_scalar_tensor(self, x)) + + def scalar_cos(self, x): + return orig_scalar_cos(self, _ensure_scalar_tensor(self, x)) + + def scalar_sin(self, x): + return orig_scalar_sin(self, _ensure_scalar_tensor(self, x)) + + def scalar_log(self, x): + return orig_scalar_log(self, _ensure_scalar_tensor(self, x)) + + def add(self, a, b): + return orig_add(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def subtract(self, a, b): + return orig_subtract(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def multiply(self, a, b): + return orig_multiply(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def divide(self, a, b): + return orig_divide(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def concat(self, a, b, axis=0): + return orig_concat(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b), axis=axis) + + def cat(self, tensors, axis=0): + legalized = [_ensure_scalar_tensor(self, tensor) for tensor in tensors] + return orig_cat(self, legalized, axis=axis) + + def abs(self, x): + return orig_abs(self, _ensure_scalar_tensor(self, x)) + + def pow(self, x, exponent): + return orig_pow(self, _ensure_scalar_tensor(self, x), exponent) + + Graph.matmul = matmul + Graph.gather = gather + Graph.conv2d = conv2d + Graph.scalar_not_equal = scalar_not_equal + Graph.not_equal = not_equal + Graph.transpose = transpose + Graph.permute = permute + Graph.scalar_add = scalar_add + Graph.scalar_subtract = scalar_subtract + Graph.scalar_multiply = scalar_multiply + Graph.scalar_divide = scalar_divide + Graph.scalar_exp = scalar_exp + Graph.scalar_sqrt = scalar_sqrt + Graph.scalar_cos = scalar_cos + Graph.scalar_sin = scalar_sin + Graph.scalar_log = scalar_log + Graph.add = add + Graph.subtract = subtract + Graph.multiply = multiply + Graph.divide = divide + Graph.concat = concat + Graph.cat = cat + Graph.abs = abs + Graph.pow = pow + Graph._transpile_runtime_compat_patched = True + + +_cactus_module, _graph_module = _load_runtime_modules() + +if hasattr(_cactus_module, "_lib"): + _lib_obj = _cactus_module._lib + if hasattr(_lib_obj, "cactus_graph_matmul"): + _lib_obj.cactus_graph_matmul.argtypes = [ + ctypes.c_void_p, + _cactus_module.cactus_node_t, + _cactus_module.cactus_node_t, + ctypes.c_bool, + ctypes.c_int32, + ctypes.POINTER(_cactus_module.cactus_node_t), + ] + _lib_obj.cactus_graph_matmul.restype = ctypes.c_int + if hasattr(_lib_obj, "cactus_graph_gather"): + _lib_obj.cactus_graph_gather.argtypes = [ + ctypes.c_void_p, + _cactus_module.cactus_node_t, + _cactus_module.cactus_node_t, + ctypes.POINTER(_cactus_module.cactus_node_t), + ] + _lib_obj.cactus_graph_gather.restype = ctypes.c_int + +_patch_graph_runtime(_graph_module, _cactus_module) + +_lib = _cactus_module._lib +_err = _cactus_module._err +cactus_node_t = _cactus_module.cactus_node_t +cactus_tensor_info_t = _cactus_module.cactus_tensor_info_t +Graph = _graph_module.Graph +Tensor = _graph_module.Tensor diff --git a/python/src/transpile/test_capture.py b/python/src/transpile/test_capture.py deleted file mode 100644 index 539d6f437..000000000 --- a/python/src/transpile/test_capture.py +++ /dev/null @@ -1,187 +0,0 @@ -from __future__ import annotations - -import json -import re -from dataclasses import asdict -from pathlib import Path -from typing import Any - -import torch -from transformers import AutoModelForCausalLM -from transformers import AutoTokenizer - -from src.transpile.capture.capture_pytorch import capture_model_with_fallback -from src.transpile.capture.capture_pytorch import dump_graph -from src.transpile.capture.graph_ir import verify_ir - - -MODELS = ( - "google/gemma-4-E2B", - "google/gemma-3-270m-it", - "google/gemma-2b-it", - "google/gemma-2-2b", - "Qwen/Qwen3.5-2B", - "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "HuggingFaceTB/SmolLM2-360M-Instruct", -) - -PROMPT = "The capital of France is" -ARTIFACTS_DIR = Path(__file__).resolve().parents[3] / "artifacts" / "python_capture_tests" - - -def slugify(value: str) -> str: - return re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("._-") or "model" - - -def resolve_local_snapshot(model_id: str) -> str | None: - explicit = Path(model_id) - if explicit.exists(): - return str(explicit) - - snapshots_dir = ( - Path.home() - / ".cache" - / "huggingface" - / "hub" - / ("models--" + model_id.replace("/", "--")) - / "snapshots" - ) - if not snapshots_dir.exists(): - return None - - snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) - if not snapshots: - return None - return str(snapshots[-1]) - - -def jsonify(value: Any) -> Any: - if isinstance(value, dict): - return {str(key): jsonify(inner) for key, inner in value.items()} - if isinstance(value, (list, tuple)): - return [jsonify(inner) for inner in value] - if isinstance(value, Path): - return str(value) - if isinstance(value, torch.dtype): - return str(value) - if isinstance(value, torch.Tensor): - return { - "type": "torch.Tensor", - "dtype": str(value.dtype), - "shape": list(value.shape), - } - if hasattr(value, "__dataclass_fields__"): - return jsonify(asdict(value)) - if isinstance(value, (str, int, float, bool)) or value is None: - return value - return repr(value) - - -def graph_to_dict(graph) -> dict[str, Any]: - return { - "meta": jsonify(graph.meta), - "inputs": list(graph.inputs), - "outputs": list(graph.outputs), - "constants": {value_id: jsonify(constant) for value_id, constant in graph.constants.items()}, - "values": {value_id: jsonify(asdict(value)) for value_id, value in graph.values.items()}, - "nodes": [jsonify(asdict(graph.nodes[node_id])) for node_id in graph.order], - } - - -def write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - - -class CausalLMCaptureWrapper(torch.nn.Module): - def __init__(self, model: torch.nn.Module): - super().__init__() - self.model = model.eval() - - def forward(self, input_ids: torch.Tensor) -> torch.Tensor: - outputs = self.model( - input_ids=input_ids, - use_cache=False, - return_dict=False, - ) - if isinstance(outputs, torch.Tensor): - return outputs - if isinstance(outputs, (tuple, list)) and outputs and isinstance(outputs[0], torch.Tensor): - return outputs[0] - logits = getattr(outputs, "logits", None) - if isinstance(logits, torch.Tensor): - return logits - raise TypeError(f"unsupported model output type: {type(outputs).__name__}") - - -def capture_one_model(model_id: str) -> dict[str, Any]: - snapshot = resolve_local_snapshot(model_id) - if snapshot is None: - raise FileNotFoundError(f"no local snapshot found for {model_id}") - - artifact_dir = ARTIFACTS_DIR / slugify(model_id) - tokenizer = AutoTokenizer.from_pretrained(snapshot, local_files_only=True) - encoded = tokenizer(PROMPT, return_tensors="pt") - input_ids = encoded["input_ids"] - - model = AutoModelForCausalLM.from_pretrained( - snapshot, - torch_dtype="auto", - device_map=None, - low_cpu_mem_usage=True, - local_files_only=True, - ).eval() - - captured = capture_model_with_fallback( - CausalLMCaptureWrapper(model), - (input_ids,), - ) - verify_ir(captured.ir_graph) - - write_json( - artifact_dir / "graph_ir.json", - { - "model_id": model_id, - "snapshot": snapshot, - "prompt": PROMPT, - "strict": captured.strict, - "graph": graph_to_dict(captured.ir_graph), - }, - ) - (artifact_dir / "fx_graph.txt").write_text(dump_graph(captured) + "\n") - - return { - "model_id": model_id, - "snapshot": snapshot, - "artifact_dir": str(artifact_dir), - "strict": captured.strict, - "ir_nodes": len(captured.ir_graph.order), - "ir_values": len(captured.ir_graph.values), - } - - -def main() -> None: - results: list[dict[str, Any]] = [] - for model_id in MODELS: - print(f"capturing {model_id}") - try: - result = capture_one_model(model_id) - except Exception as exc: - result = { - "model_id": model_id, - "error": str(exc), - } - results.append(result) - print(json.dumps(result, indent=2)) - - write_json( - ARTIFACTS_DIR / "capture_manifest.json", - { - "prompt": PROMPT, - "results": results, - }, - ) - - -if __name__ == "__main__": - main() diff --git a/python/src/transpile/weight_binding.py b/python/src/transpile/weight_binding.py new file mode 100644 index 000000000..a04a0978d --- /dev/null +++ b/python/src/transpile/weight_binding.py @@ -0,0 +1,561 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re + + +@dataclass(frozen=True) +class WeightBinding: + path: str + kind: str # "weight" | "embedding" + source_name: str + + +_PROJECT_ROOT = Path(__file__).resolve().parents[3] + +_PARAKEET_GLOBAL_FILENAMES: dict[str, str] = { + "ctc_head.weight": "ctc_head_weight.weights", + "ctc_head.bias": "ctc_head_bias.bias", + "encoder.subsampling.layers.0.weight": "subsampling_conv0_weight.weights", + "encoder.subsampling.layers.0.bias": "subsampling_conv0_bias.bias", + "encoder.subsampling.layers.2.weight": "subsampling_depthwise1_weight.weights", + "encoder.subsampling.layers.2.bias": "subsampling_depthwise1_bias.bias", + "encoder.subsampling.layers.3.weight": "subsampling_pointwise1_weight.weights", + "encoder.subsampling.layers.3.bias": "subsampling_pointwise1_bias.bias", + "encoder.subsampling.layers.5.weight": "subsampling_depthwise2_weight.weights", + "encoder.subsampling.layers.5.bias": "subsampling_depthwise2_bias.bias", + "encoder.subsampling.layers.6.weight": "subsampling_pointwise2_weight.weights", + "encoder.subsampling.layers.6.bias": "subsampling_pointwise2_bias.bias", + "encoder.subsampling.linear.weight": "subsampling_linear_weight.weights", + "encoder.subsampling.linear.bias": "subsampling_linear_bias.bias", + "encoder.pre_encode.conv.0.weight": "subsampling_conv0_weight.weights", + "encoder.pre_encode.conv.0.bias": "subsampling_conv0_bias.bias", + "encoder.pre_encode.conv.2.weight": "subsampling_depthwise1_weight.weights", + "encoder.pre_encode.conv.2.bias": "subsampling_depthwise1_bias.bias", + "encoder.pre_encode.conv.3.weight": "subsampling_pointwise1_weight.weights", + "encoder.pre_encode.conv.3.bias": "subsampling_pointwise1_bias.bias", + "encoder.pre_encode.conv.5.weight": "subsampling_depthwise2_weight.weights", + "encoder.pre_encode.conv.5.bias": "subsampling_depthwise2_bias.bias", + "encoder.pre_encode.conv.6.weight": "subsampling_pointwise2_weight.weights", + "encoder.pre_encode.conv.6.bias": "subsampling_pointwise2_bias.bias", + "encoder.pre_encode.out.weight": "subsampling_linear_weight.weights", + "encoder.pre_encode.out.bias": "subsampling_linear_bias.bias", + "decoder.prediction.embed.weight": "tdt_predictor_embed.weights", + "joint.enc.weight": "tdt_joint_enc.weights", + "joint.enc.bias": "tdt_joint_enc.bias", + "joint.pred.weight": "tdt_joint_pred.weights", + "joint.pred.bias": "tdt_joint_pred.bias", + "joint.joint_net.2.weight": "tdt_joint_out.weights", + "joint.joint_net.2.bias": "tdt_joint_out.bias", +} + +_PARAKEET_LAYER_FILENAMES: dict[str, str] = { + "feed_forward1.linear1.weight": "ff1_linear1.weights", + "feed_forward1.linear1.bias": "ff1_linear1.bias", + "feed_forward1.linear2.weight": "ff1_linear2.weights", + "feed_forward1.linear2.bias": "ff1_linear2.bias", + "feed_forward2.linear1.weight": "ff2_linear1.weights", + "feed_forward2.linear1.bias": "ff2_linear1.bias", + "feed_forward2.linear2.weight": "ff2_linear2.weights", + "feed_forward2.linear2.bias": "ff2_linear2.bias", + "self_attn.q_proj.weight": "self_attn_q.weights", + "self_attn.q_proj.bias": "self_attn_q.bias", + "self_attn.k_proj.weight": "self_attn_k.weights", + "self_attn.k_proj.bias": "self_attn_k.bias", + "self_attn.v_proj.weight": "self_attn_v.weights", + "self_attn.v_proj.bias": "self_attn_v.bias", + "self_attn.o_proj.weight": "self_attn_output.weights", + "self_attn.o_proj.bias": "self_attn_output.bias", + "self_attn.relative_k_proj.weight": "self_attn_relative_k.weights", + "self_attn.linear_q.weight": "self_attn_q.weights", + "self_attn.linear_q.bias": "self_attn_q.bias", + "self_attn.linear_k.weight": "self_attn_k.weights", + "self_attn.linear_k.bias": "self_attn_k.bias", + "self_attn.linear_v.weight": "self_attn_v.weights", + "self_attn.linear_v.bias": "self_attn_v.bias", + "self_attn.linear_out.weight": "self_attn_output.weights", + "self_attn.linear_out.bias": "self_attn_output.bias", + "self_attn.linear_pos.weight": "self_attn_relative_k.weights", + "self_attn.bias_u": "self_attn_bias_u.weights", + "self_attn.bias_v": "self_attn_bias_v.weights", + "self_attn.pos_bias_u": "self_attn_bias_u.weights", + "self_attn.pos_bias_v": "self_attn_bias_v.weights", + "conv.pointwise_conv1.weight": "conv_pointwise1.weights", + "conv.pointwise_conv1.bias": "conv_pointwise1.bias", + "conv.depthwise_conv.weight": "conv_depthwise.weights", + "conv.depthwise_conv.bias": "conv_depthwise.bias", + "conv.pointwise_conv2.weight": "conv_pointwise2.weights", + "conv.pointwise_conv2.bias": "conv_pointwise2.bias", + "conv.norm.weight": "conv_batchnorm_weight.weights", + "conv.norm.bias": "conv_batchnorm_bias.bias", + "conv.norm.running_mean": "conv_batchnorm_running_mean.weights", + "conv.norm.running_var": "conv_batchnorm_running_var.weights", + "conv.batch_norm.weight": "conv_batchnorm_weight.weights", + "conv.batch_norm.bias": "conv_batchnorm_bias.bias", + "conv.batch_norm.running_mean": "conv_batchnorm_running_mean.weights", + "conv.batch_norm.running_var": "conv_batchnorm_running_var.weights", + "norm_feed_forward1.weight": "norm_ff1.weights", + "norm_feed_forward1.bias": "norm_ff1.bias", + "norm_self_att.weight": "norm_self_attn.weights", + "norm_self_att.bias": "norm_self_attn.bias", + "norm_conv.weight": "norm_conv.weights", + "norm_conv.bias": "norm_conv.bias", + "norm_feed_forward2.weight": "norm_ff2.weights", + "norm_feed_forward2.bias": "norm_ff2.bias", + "norm_out.weight": "norm_out.weights", + "norm_out.bias": "norm_out.bias", +} + +_WHISPER_GLOBAL_FILENAMES: dict[str, tuple[str, str]] = { + "decoder.embed_tokens.weight": ("decoder_token_embeddings.weights", "embedding"), + "decoder.embed_positions.weight": ("decoder_position_embeddings.weights", "embedding"), + "decoder.layer_norm.weight": ("decoder_norm.weights", "weight"), + "decoder.layer_norm.bias": ("decoder_norm.bias", "weight"), + "proj_out.weight": ("output_weight.weights", "weight"), + "encoder.embed_positions.weight": ("encoder_position_embeddings.weights", "embedding"), + "encoder.conv1.bias": ("encoder_conv1_bias.bias", "weight"), + "encoder.conv1.weight": ("encoder_conv1_weight.weights", "weight"), + "encoder.conv2.bias": ("encoder_conv2_bias.bias", "weight"), + "encoder.conv2.weight": ("encoder_conv2_weight.weights", "weight"), + "encoder.layer_norm.bias": ("encoder_norm_bias.bias", "weight"), + "encoder.layer_norm.weight": ("encoder_norm_weight.weights", "weight"), +} + +_WHISPER_LAYER_FILENAMES: dict[str, str] = { + "self_attn.q_proj.weight": "self_attn_q.weights", + "self_attn.k_proj.weight": "self_attn_k.weights", + "self_attn.v_proj.weight": "self_attn_v.weights", + "self_attn.q_proj.bias": "self_attn_q.bias", + "self_attn.v_proj.bias": "self_attn_v.bias", + "self_attn.out_proj.weight": "self_attn_output.weights", + "self_attn.out_proj.bias": "self_attn_output.bias", + "self_attn_layer_norm.weight": "self_attn_norm.weights", + "self_attn_layer_norm.bias": "self_attn_norm.bias", + "encoder_attn.q_proj.weight": "encoder_attn_q.weights", + "encoder_attn.k_proj.weight": "encoder_attn_k.weights", + "encoder_attn.v_proj.weight": "encoder_attn_v.weights", + "encoder_attn.out_proj.weight": "encoder_attn_output.weights", + "encoder_attn.o_proj.weight": "encoder_attn_output.weights", + "encoder_attn.q_proj.bias": "encoder_attn_q.bias", + "encoder_attn.v_proj.bias": "encoder_attn_v.bias", + "encoder_attn.out_proj.bias": "encoder_attn_output.bias", + "encoder_attn.o_proj.bias": "encoder_attn_output.bias", + "encoder_attn_layer_norm.weight": "encoder_attn_norm.weights", + "encoder_attn_layer_norm.bias": "encoder_attn_norm.bias", + "fc1.weight": "mlp_fc1.weights", + "fc1.bias": "mlp_fc1.bias", + "fc2.weight": "mlp_fc2.weights", + "fc2.bias": "mlp_fc2.bias", + "final_layer_norm.weight": "final_norm.weights", + "final_layernorm.weight": "final_norm.weights", + "final_layer_norm.bias": "final_norm.bias", + "final_layernorm.bias": "final_norm.bias", +} + +_GEMMA3N_VISION_TOWER_PREFIX = "model.vision_tower.timm_model." +_GEMMA3N_AUDIO_TOWER_PREFIX = "model.audio_tower." +_GEMMA4_VISION_TOWER_PREFIX = "model.vision_tower." +_GEMMA4_AUDIO_TOWER_PREFIX = "model.audio_tower." + +_GEMMA_GLOBAL_FILENAMES: dict[str, tuple[str, str]] = { + "model.embed_tokens.weight": ("token_embeddings.weights", "embedding"), + "model.language_model.embed_tokens.weight": ("token_embeddings.weights", "embedding"), + "model.language_model.embed_tokens_per_layer.weight": ("embed_tokens_per_layer.weights", "embedding"), + "model.language_model.per_layer_model_projection.weight": ("per_layer_model_proj.weights", "weight"), + "model.language_model.per_layer_projection_norm.weight": ("per_layer_proj_norm.weights", "weight"), + "model.embed_vision.embedding.weight": ("embed_vision_embedding.weights", "embedding"), + "model.embed_vision.embedding_projection.weight": ("embed_vision_proj.weights", "weight"), + "model.embed_vision.soft_embedding_norm.weight": ("embed_vision_soft_norm.weights", "weight"), + "model.embed_vision.hard_embedding_norm.weight": ("embed_vision_hard_norm.weights", "weight"), + "model.embed_audio.embedding.weight": ("embed_audio_embedding.weights", "embedding"), + "model.embed_audio.embedding_projection.weight": ("embed_audio_proj.weights", "weight"), + "model.embed_audio.soft_embedding_norm.weight": ("embed_audio_soft_norm.weights", "weight"), + "model.embed_audio.hard_embedding_norm.weight": ("embed_audio_hard_norm.weights", "weight"), +} + +_GEMMA_DECODER_LAYER_FILENAMES: dict[str, tuple[str, str]] = { + "layer_scalar": ("layer_scalar.weights", "weight"), + "self_attn.q_proj.weight": ("attn_q.weights", "weight"), + "self_attn.k_proj.weight": ("attn_k.weights", "weight"), + "self_attn.v_proj.weight": ("attn_v.weights", "weight"), + "self_attn.o_proj.weight": ("attn_output.weights", "weight"), + "self_attn.q_norm.weight": ("attn_q_norm.weights", "weight"), + "self_attn.k_norm.weight": ("attn_k_norm.weights", "weight"), + "mlp.gate_proj.weight": ("ffn_gate.weights", "weight"), + "mlp.up_proj.weight": ("ffn_up.weights", "weight"), + "mlp.down_proj.weight": ("ffn_down.weights", "weight"), + "input_layernorm.weight": ("input_norm.weights", "weight"), + "post_attention_layernorm.weight": ("post_attn_norm.weights", "weight"), + "pre_feedforward_layernorm.weight": ("pre_ffn_norm.weights", "weight"), + "post_feedforward_layernorm.weight": ("post_ffn_norm.weights", "weight"), + "post_per_layer_projection_layernorm.weight": ("post_per_layer_norm.weights", "weight"), + "per_layer_input_gate.weight": ("per_layer_gate.weights", "weight"), + "per_layer_projection.weight": ("per_layer_proj.weights", "weight"), +} + + + +def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: + candidates: list[str] = [] + raw = model_name_or_path.strip() + if not raw: + return candidates + + def _add(name: str) -> None: + name = name.strip().lower() + if name and name not in candidates: + candidates.append(name) + + _add(raw.split("/")[-1]) + + path = Path(raw) + parts = path.parts + for part in parts: + if part.startswith("models--"): + _add(part[len("models--"):].split("--")[-1]) + break + + return candidates + + +def _default_weights_dir_for_model_name(model_name_or_path: str) -> str | None: + if not model_name_or_path: + return None + for model_dir_name in _candidate_model_dir_names(model_name_or_path): + candidate = _PROJECT_ROOT / "weights" / model_dir_name + if candidate.exists(): + return str(candidate) + return None + + +def _normalized_source_candidates(source_name: str) -> list[str]: + candidates: list[str] = [] + + def _add(name: str) -> None: + if name and name not in candidates: + candidates.append(name) + + def _add_backbone_aliases(name: str) -> None: + if name.startswith("backbone."): + tail = name[len("backbone.") :] + _add(f"model.{tail}") + _add(f"model.language_model.{tail}") + if name.startswith("language_model."): + tail = name[len("language_model.") :] + _add(f"model.language_model.{tail}") + if name.startswith("multimodal_backbone."): + tail = name[len("multimodal_backbone.") :] + _add(tail) + _add(f"model.{tail}") + _add(f"model.multimodal_backbone.{tail}") + if name.startswith("model.encoder."): + _add(name[len("model.") :]) + if name.startswith("model.decoder."): + _add(name[len("model.") :]) + if name.startswith("layers."): + _add(f"encoder.{name}") + if name.startswith("pre_encode."): + _add(f"encoder.{name}") + if name.startswith("prediction."): + _add(f"decoder.{name}") + + raw = source_name.strip() + _add(raw) + _add_backbone_aliases(raw) + + for prefix in ("p_", "b_", "c_"): + if raw.startswith(prefix): + stripped = raw[len(prefix):] + _add(stripped) + _add_backbone_aliases(stripped) + + stripped = raw + while True: + for prefix in ("adapter.model.", "adapter.module.", "adapter.encoder.", "adapter.decoder.", "module."): + if stripped.startswith(prefix): + tail = stripped[len(prefix) :] + if prefix == "adapter.encoder.": + stripped = f"encoder.{tail}" + elif prefix == "adapter.decoder.": + stripped = f"decoder.{tail}" + else: + stripped = tail + _add(stripped) + _add_backbone_aliases(stripped) + break + else: + break + + for candidate in tuple(candidates): + for repeated_prefix, collapsed_prefix in ( + ("model.model.", "model."), + ("language_model.language_model.", "language_model."), + ("backbone.backbone.", "backbone."), + ): + if candidate.startswith(repeated_prefix): + tail = candidate[len(repeated_prefix) :] + _add(f"{collapsed_prefix}{tail}") + + for prefix in ( + "module.backbone.", + "adapter.backbone.", + "adapter.model.model.", + "adapter.model.language_model.", + ): + if not candidate.startswith(prefix): + continue + tail = candidate[len(prefix):] + _add(tail) + _add_backbone_aliases(tail) + _add(f"model.{tail}") + _add(f"model.language_model.{tail}") + if tail.startswith("layers."): + _add(f"model.{tail}") + _add(f"model.language_model.{tail}") + break + + if stripped == "adapter.model.lm_head.weight": + _add("lm_head.weight") + _add("model.embed_tokens.weight") + _add("model.language_model.embed_tokens.weight") + if stripped == "lm_head.weight": + _add("model.embed_tokens.weight") + _add("model.language_model.embed_tokens.weight") + + layer_match = re.match( + r"^(?:(?:module|adapter)_)?backbone_layers_slice_none__\d+__none____modules__(\d+)___(.+)$", + stripped, + ) + if layer_match: + layer_index = int(layer_match.group(1)) + tail = layer_match.group(2) + tail_map = { + "self_attn_q_proj_weight": "self_attn.q_proj.weight", + "self_attn_k_proj_weight": "self_attn.k_proj.weight", + "self_attn_v_proj_weight": "self_attn.v_proj.weight", + "self_attn_o_proj_weight": "self_attn.o_proj.weight", + "self_attn_q_norm_weight": "self_attn.q_norm.weight", + "self_attn_k_norm_weight": "self_attn.k_norm.weight", + "mlp_gate_proj_weight": "mlp.gate_proj.weight", + "mlp_up_proj_weight": "mlp.up_proj.weight", + "mlp_down_proj_weight": "mlp.down_proj.weight", + "input_layernorm_weight": "input_layernorm.weight", + "post_attention_layernorm_weight": "post_attention_layernorm.weight", + "linear_attn_in_proj_qkv_weight": "linear_attn.in_proj_qkv.weight", + "linear_attn_conv1d_weight": "linear_attn.conv1d.weight", + "linear_attn_norm_weight": "linear_attn.norm.weight", + "linear_attn_dt_bias": "linear_attn.dt_bias", + "linear_attn_A_log": "linear_attn.A_log", + } + dotted_tail = tail_map.get(tail) + if dotted_tail is not None: + _add(f"model.layers.{layer_index}.{dotted_tail}") + _add(f"model.language_model.layers.{layer_index}.{dotted_tail}") + + embed_map = { + "module_backbone_embed_tokens_weight": [ + "model.embed_tokens.weight", + "model.language_model.embed_tokens.weight", + ], + "adapter_backbone_embed_tokens_weight": [ + "model.embed_tokens.weight", + "model.language_model.embed_tokens.weight", + ], + "module_backbone_embed_tokens_per_layer_weight": [ + "model.embed_tokens_per_layer.weight", + "model.language_model.embed_tokens_per_layer.weight", + ], + "adapter_backbone_embed_tokens_per_layer_weight": [ + "model.embed_tokens_per_layer.weight", + "model.language_model.embed_tokens_per_layer.weight", + ], + "module_backbone_per_layer_model_projection_weight": [ + "model.per_layer_model_projection.weight", + "model.language_model.per_layer_model_projection.weight", + ], + "adapter_backbone_per_layer_model_projection_weight": [ + "model.per_layer_model_projection.weight", + "model.language_model.per_layer_model_projection.weight", + ], + "module_backbone_norm_weight": [ + "model.norm.weight", + "model.language_model.norm.weight", + ], + "adapter_backbone_norm_weight": [ + "model.norm.weight", + "model.language_model.norm.weight", + ], + "module_model_lm_head_weight": [ + "lm_head.weight", + ], + "adapter_model_lm_head_weight": [ + "lm_head.weight", + ], + } + for key, mapped in embed_map.items(): + if stripped == key: + for item in mapped: + _add(item) + + return candidates + + +def _gemma_tower_output_name(hf_key: str, strip_prefix: str, add_prefix: str) -> str: + name = hf_key[len(strip_prefix) :] + if name.endswith(".weight"): + name = name[: -len(".weight")] + ext = ".weights" + elif name.endswith(".bias"): + name = name[: -len(".bias")] + ext = ".bias" + else: + ext = ".weights" + if name.endswith(".linear"): + name = name[: -len(".linear")] + elif name.endswith("_linear"): + name = name[: -len("_linear")] + name = name.replace(".", "_") + return add_prefix + name + ext + + +def _fallback_filename_candidates(source_name: str) -> list[tuple[str, str]]: + filenames: list[tuple[str, str]] = [] + + def _add(filename: str | None, *, kind: str = "weight") -> None: + if not filename: + return + candidate = (filename, kind) + if candidate not in filenames: + filenames.append(candidate) + + for candidate in _normalized_source_candidates(source_name): + _add(_PARAKEET_GLOBAL_FILENAMES.get(candidate)) + whisper_global = _WHISPER_GLOBAL_FILENAMES.get(candidate) + if whisper_global is not None: + _add(whisper_global[0], kind=whisper_global[1]) + gemma_global = _GEMMA_GLOBAL_FILENAMES.get(candidate) + if gemma_global is not None: + _add(gemma_global[0], kind=gemma_global[1]) + if candidate in {"model.embed_tokens_per_layer.weight", "model.language_model.embed_tokens_per_layer.weight"}: + _add("embed_tokens_per_layer.weights", kind="embedding") + elif candidate in {"model.per_layer_model_projection.weight", "model.language_model.per_layer_model_projection.weight"}: + _add("per_layer_model_proj.weights") + elif candidate in {"model.per_layer_projection_norm.weight", "model.language_model.per_layer_projection_norm.weight"}: + _add("per_layer_proj_norm.weights") + elif candidate in {"lm_head.weight", "model.lm_head.weight"}: + _add("token_embeddings.weights", kind="embedding") + elif candidate in { + "_cactus_vision_post_proj_norm", + "model._cactus_vision_post_proj_norm", + }: + _add("embed_vision_post_proj_norm.weights") + + if candidate.startswith(_GEMMA3N_VISION_TOWER_PREFIX): + _add(_gemma_tower_output_name(candidate, _GEMMA3N_VISION_TOWER_PREFIX, "vision_")) + elif candidate.startswith(_GEMMA3N_AUDIO_TOWER_PREFIX): + _add(_gemma_tower_output_name(candidate, _GEMMA3N_AUDIO_TOWER_PREFIX, "audio_")) + elif candidate.startswith(_GEMMA4_VISION_TOWER_PREFIX): + _add(_gemma_tower_output_name(candidate, _GEMMA4_VISION_TOWER_PREFIX, "vision_")) + elif candidate.startswith(_GEMMA4_AUDIO_TOWER_PREFIX): + _add(_gemma_tower_output_name(candidate, _GEMMA4_AUDIO_TOWER_PREFIX, "audio_")) + + predictor_match = re.match(r"^decoder\.prediction\.dec_rnn\.lstm\.(\d+)\.(Wx|Wh|bias)$", candidate) + if predictor_match: + layer_index = int(predictor_match.group(1)) + suffix = predictor_match.group(2) + if suffix == "Wx": + _add(f"tdt_predictor_lstm_{layer_index}_weight_ih.weights") + elif suffix == "Wh": + _add(f"tdt_predictor_lstm_{layer_index}_weight_hh.weights") + elif suffix == "bias": + _add(f"tdt_predictor_lstm_{layer_index}_bias.weights") + + layer_match = re.match(r"^encoder\.layers\.(\d+)\.(.+)$", candidate) + if layer_match: + layer_index = int(layer_match.group(1)) + suffix = layer_match.group(2) + mapped = _PARAKEET_LAYER_FILENAMES.get(suffix) + if mapped is not None: + _add(f"layer_{layer_index}_{mapped}") + + whisper_layer_match = re.match(r"^(encoder|decoder)\.layers\.(\d+)\.(.+)$", candidate) + if whisper_layer_match: + block = whisper_layer_match.group(1) + layer_index = int(whisper_layer_match.group(2)) + suffix = whisper_layer_match.group(3) + mapped = _WHISPER_LAYER_FILENAMES.get(suffix) + if mapped is not None: + _add(f"{block}.layer_{layer_index}_{mapped}") + + gemma_layer_match = re.match( + r"^(?:model(?:\.language_model)?\.)?layers\.(\d+)\.(.+)$", + candidate, + ) + if gemma_layer_match: + layer_index = int(gemma_layer_match.group(1)) + suffix = gemma_layer_match.group(2) + mapped = _GEMMA_DECODER_LAYER_FILENAMES.get(suffix) + if mapped is not None: + _add(f"layer_{layer_index}_{mapped[0]}", kind=mapped[1]) + + return filenames + + +def resolve_transpile_weights_dir(graph_meta: dict[str, object]) -> str | None: + explicit = graph_meta.get("weights_dir") + if isinstance(explicit, str) and explicit: + return explicit + + family = str(graph_meta.get("adapter_family", "")).upper() + family_env = f"CACTUS_TRANSPILER_WEIGHTS_DIR_{family}" + if family and family_env in os.environ and os.environ[family_env]: + return os.environ[family_env] + + generic = os.environ.get("CACTUS_TRANSPILER_WEIGHTS_DIR") + if generic: + return generic + + model_name_or_path = graph_meta.get("model_name_or_path") + if isinstance(model_name_or_path, str) and model_name_or_path: + default_dir = _default_weights_dir_for_model_name(model_name_or_path) + if default_dir: + return default_dir + return None + + +def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> WeightBinding | None: + if not weights_dir: + return None + root = Path(weights_dir) + if not root.exists(): + return None + manifest: dict[str, object] = {} + manifest_path = root / "weights_manifest.json" + if manifest_path.exists(): + try: + loaded_manifest = json.loads(manifest_path.read_text()) + except Exception: + loaded_manifest = None + if isinstance(loaded_manifest, dict): + manifest = loaded_manifest + for candidate_name in _normalized_source_candidates(source_name): + entry = manifest.get(candidate_name) + if not isinstance(entry, dict): + continue + filename = entry.get("filename") + kind = entry.get("kind", "weight") + if not isinstance(filename, str) or not isinstance(kind, str): + continue + candidate = root / filename + if not candidate.exists(): + continue + return WeightBinding(path=str(candidate), kind=kind, source_name=candidate_name) + + for filename, kind in _fallback_filename_candidates(source_name): + candidate = root / filename + if candidate.exists(): + return WeightBinding(path=str(candidate), kind=kind, source_name=source_name) + return None diff --git a/python/src/transpile/weight_compat.py b/python/src/transpile/weight_compat.py new file mode 100644 index 000000000..47e61ed7d --- /dev/null +++ b/python/src/transpile/weight_compat.py @@ -0,0 +1,522 @@ +from __future__ import annotations + +from dataclasses import replace +import shutil +import struct +from pathlib import Path + +import numpy as np + +from src.tensor_io import CACTUS_ALIGNMENT +from src.tensor_io import CACTUS_MAGIC +from src.tensor_io import FLAG_INTERLEAVED +from src.tensor_io import align_offset +from src.cq_convert.model_adapters.naming import gemma4_scale_factor +from src.transpile.runtime_compat import Graph +from src.transpile.weight_binding import WeightBinding +from src.cq_convert.quantization.cq import FLAG_ORTHOGONAL_ROTATION +from src.cq_convert.quantization.cq import GROUP_SIZE as CQ_GROUP_SIZE +from src.cq_convert.quantization.cq import PRECISION_CQ +from src.cq_convert.quantization.cq import make_codebook +from src.cq_convert.quantization.cq import make_hadamard_components +from src.cq_convert.quantization.cq import make_hadamard_matrix +from src.cq_convert.quantization.cq import make_orthogonal_rotation +from src.cq_convert.quantization.cq import pack_indices_lsb +from src.cq_convert.quantization.cq import quantize_hadamard +from src.cq_convert.quantization.cq import quantize_orthogonal +from src.cq_convert.quantization.cq import write_cq_tensor + + +_HEADER_SIZE = 84 +_INT8 = int(Graph.INT8) +_INT4 = int(Graph.INT4) + +_TOKEN_EMBEDDING_FILENAMES = { + "token_embeddings.weights", + "decoder_token_embeddings.weights", +} +_PER_LAYER_EMBEDDING_FILENAMES = { + "embed_tokens_per_layer.weights", +} + + +class _OpenedTensor: + def __init__( + self, + *, + path: Path, + precision: int, + shape: tuple[int, ...], + data: np.memmap, + scales: np.memmap | None, + group_size: int, + num_groups: int, + is_interleaved: bool, + original_n: int, + alignment: int, + ) -> None: + self.path = path + self.precision = precision + self.shape = shape + self.data = data + self.scales = scales + self.group_size = group_size + self.num_groups = num_groups + self.is_interleaved = is_interleaved + self.original_n = original_n + self.alignment = alignment + + +def ensure_binding_compatible(binding: WeightBinding, source_tensor: object | None = None) -> WeightBinding: + binding = ensure_embedding_binding_compatible(binding) + return _ensure_legacy_int4_weight_binding_compatible(binding, source_tensor=source_tensor) + + +def ensure_embedding_binding_compatible(binding: WeightBinding) -> WeightBinding: + if binding.kind != "embedding": + return binding + + source_path = Path(binding.path).expanduser().resolve() + opened = _open_cactus_tensor_file(source_path) + if opened.precision != _INT8: + return binding + if len(opened.shape) != 2 or opened.scales is None or opened.group_size <= 0: + return binding + + config = _embedding_cache_config(source_path.name, source_name=binding.source_name) + if config is None: + return binding + + compat_path = source_path.with_name(source_path.stem + f".cq{config['bits']}.weights") + _materialize_cq_embedding_cache( + opened, + compat_path, + bits=int(config["bits"]), + rotation=str(config["rotation"]), + ) + _cleanup_legacy_fp16_cache(source_path) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + +def _embedding_cache_config(filename: str, *, source_name: str) -> dict[str, object] | None: + normalized_source = str(source_name or "") + if filename in _PER_LAYER_EMBEDDING_FILENAMES or normalized_source.endswith("embed_tokens_per_layer.weight"): + return {"bits": 2, "rotation": "hadamard"} + if filename in _TOKEN_EMBEDDING_FILENAMES: + return {"bits": 4, "rotation": "orthogonal"} + return {"bits": 4, "rotation": "orthogonal"} + + +def _ensure_legacy_int4_weight_binding_compatible( + binding: WeightBinding, + *, + source_tensor: object | None, +) -> WeightBinding: + source_path = Path(binding.path).expanduser().resolve() + opened = _open_cactus_tensor_file(source_path) + if not _is_legacy_packed_int4_tensor(opened): + return binding + + compat_path = source_path.with_name(source_path.stem + ".cq4.weights") + if compat_path.exists(): + if _can_materialize_compat_weight(source_tensor): + _materialize_source_cq_weight( + source_tensor, + compat_path, + bits=4, + rotation="hadamard", + scale_factor=_compat_weight_scale_factor(source_path), + source_mtime_ns=source_path.stat().st_mtime_ns, + ) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + if not _can_materialize_compat_weight(source_tensor): + raise RuntimeError( + "legacy packed INT4 weight is not directly executable in the v2 runtime and is " + f"missing a CQ companion file: {source_path.name}. " + "Rerun `cactus transpile ...` so the transpiler can materialize the CQ weights, " + "or generate them through `cq_convert` first." + ) + + _materialize_source_cq_weight( + source_tensor, + compat_path, + bits=4, + rotation="hadamard", + scale_factor=_compat_weight_scale_factor(source_path), + source_mtime_ns=source_path.stat().st_mtime_ns, + ) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + +def _is_legacy_packed_int4_tensor(opened: _OpenedTensor) -> bool: + if opened.precision != _INT4: + return False + if len(opened.shape) != 2 or opened.scales is None or opened.group_size <= 0: + return False + if not opened.is_interleaved: + return False + return True + + +def _can_materialize_compat_weight(source_tensor: object | None) -> bool: + if source_tensor is None: + return False + if isinstance(source_tensor, (str, bytes, bytearray, Path)): + return False + if np.isscalar(source_tensor): + return False + return hasattr(source_tensor, "shape") + + +def _compat_weight_scale_factor(source_path: Path) -> float: + parent_name = source_path.parent.name.lower() + if "gemma-4" in parent_name or "gemma4" in parent_name: + return gemma4_scale_factor(source_path.name) + return 1.0 + +def _cleanup_legacy_fp16_cache(source_path: Path) -> None: + legacy = source_path.with_name(source_path.stem + ".fp16.weights") + if not legacy.exists(): + return + try: + legacy.unlink() + except OSError: + pass + + +def _materialize_source_cq_weight( + tensor: object, + out_path: Path, + *, + bits: int, + rotation: str, + scale_factor: float = 1.0, + source_mtime_ns: int | None = None, +) -> None: + if source_mtime_ns is not None and out_path.exists() and out_path.stat().st_mtime_ns >= source_mtime_ns: + return + + if rotation == "orthogonal": + cq = quantize_orthogonal(tensor, bits=bits) + else: + cq = quantize_hadamard(tensor, bits=bits, use_gptq=False) + if float(scale_factor) != 1.0: + cq = replace(cq, norms=(cq.norms.astype(np.float32) * float(scale_factor)).astype(np.float16)) + write_cq_tensor(out_path, cq) + + +def _open_cactus_tensor_file(path: str | Path) -> _OpenedTensor: + tensor_path = Path(path).expanduser().resolve() + with tensor_path.open("rb") as handle: + header = handle.read(_HEADER_SIZE) + if len(header) < _HEADER_SIZE: + raise RuntimeError(f"tensor file is too small for a Cactus header: {tensor_path}") + if header[:4] != CACTUS_MAGIC: + raise RuntimeError(f"tensor file is missing the CACT header: {tensor_path}") + + flags = struct.unpack_from(" 0) + precision = int(struct.unpack_from(" 0 else 0 + data_offset = align_offset(scales_offset + scales_bytes, alignment) if scales_bytes > 0 else aligned_header + + data_element_count = byte_size // np.dtype(dtype).itemsize + data = np.memmap(tensor_path, mode="r", dtype=dtype, offset=data_offset, shape=(data_element_count,)) + scales = None + if scales_bytes > 0: + scales = np.memmap( + tensor_path, + mode="r", + dtype=np.float16, + offset=scales_offset, + shape=(scales_bytes // np.dtype(np.float16).itemsize,), + ) + return _OpenedTensor( + path=tensor_path, + precision=precision, + shape=shape, + data=data, + scales=scales, + group_size=group_size, + num_groups=num_groups, + is_interleaved=bool(flags & FLAG_INTERLEAVED), + original_n=original_n, + alignment=alignment, + ) + + +def _materialize_cq_embedding_cache( + opened: _OpenedTensor, + out_path: Path, + *, + bits: int, + rotation: str, +) -> None: + src_mtime_ns = opened.path.stat().st_mtime_ns + if out_path.exists() and out_path.stat().st_mtime_ns >= src_mtime_ns: + return + + if len(opened.shape) != 2: + raise RuntimeError(f"expected rank-2 embedding tensor, got shape={opened.shape}") + + rows = int(opened.original_n or opened.shape[0]) + cols = int(opened.shape[1]) + if rotation == "hadamard" and cols % CQ_GROUP_SIZE != 0: + raise RuntimeError( + f"Hadamard CQ embedding conversion requires hidden dim divisible by {CQ_GROUP_SIZE}, " + f"got {cols} for {opened.path.name}" + ) + + input_scale = _estimate_embedding_input_scale(opened, rows=rows, cols=cols) + precision = int(PRECISION_CQ[int(bits)]) + group_size = cols if rotation == "orthogonal" else CQ_GROUP_SIZE + num_groups = cols // group_size + codebook = make_codebook(group_size, bits).astype(np.float16) + input_scale = np.asarray(input_scale, dtype=np.float16).reshape(cols) + recip = np.minimum(1.0 / np.maximum(input_scale.astype(np.float32), 1e-8), 65504.0).astype(np.float16) + + chunk_rows = _recommended_chunk_rows(cols=cols, rotation=rotation) + norms_tmp = out_path.with_suffix(out_path.suffix + ".norms.tmp") + data_tmp = out_path.with_suffix(out_path.suffix + ".data.tmp") + final_tmp = out_path.with_suffix(out_path.suffix + ".tmp") + + flags = 0 + trailer_parts: list[bytes] = [] + if rotation == "orthogonal": + flags |= FLAG_ORTHOGONAL_ROTATION + trailer_parts.append(make_orthogonal_rotation(group_size, seed=1234).astype(np.float16).tobytes()) + else: + left, right, perm = make_hadamard_components(group_size, seed=1234) + trailer_parts.extend([ + left.tobytes(), + right.tobytes(), + perm.astype(" current: + handle.write(b"\x00" * (data_offset - current)) + with data_tmp.open("rb") as packed_handle: + shutil.copyfileobj(packed_handle, handle, length=8 * 1024 * 1024) + final_tmp.replace(out_path) + finally: + for temp_path in (norms_tmp, data_tmp): + try: + temp_path.unlink() + except FileNotFoundError: + pass + + +def _recommended_chunk_rows(*, cols: int, rotation: str) -> int: + if rotation == "orthogonal": + if cols >= 8192: + return 8 + if cols >= 4096: + return 16 + return 64 + if cols >= 8192: + return 32 + if cols >= 4096: + return 48 + return 96 + + +def _estimate_embedding_input_scale( + opened: _OpenedTensor, + *, + rows: int, + cols: int, + max_sample_rows: int = 2048, + sample_chunk_rows: int = 64, +) -> np.ndarray: + if rows <= 0 or cols <= 0: + return np.ones((cols,), dtype=np.float16) + + stride = max(1, rows // max(1, max_sample_rows)) + sampled = 0 + accum = np.zeros((cols,), dtype=np.float64) + for start in range(0, rows, stride * sample_chunk_rows): + stop = min(start + sample_chunk_rows, rows) + chunk = _dequantize_int8_rows(opened, start, stop).astype(np.float32, copy=False) + accum += np.abs(chunk).sum(axis=0, dtype=np.float64) + sampled += int(stop - start) + if sampled >= max_sample_rows: + break + + if sampled <= 0: + return np.ones((cols,), dtype=np.float16) + + mean_abs = np.maximum(accum / float(sampled), 1e-6) + raw = np.power(mean_abs, -0.5, dtype=np.float64) + raw /= np.exp(np.mean(np.log(np.clip(raw, 1e-6, None)))) + return np.clip(raw, 1.0 / 8.0, 8.0).astype(np.float16) + + +def _quantize_hadamard_chunk( + chunk: np.ndarray, + *, + bits: int, + input_scale: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + work = chunk.astype(np.float32, copy=False) * input_scale.astype(np.float32, copy=False)[None, :] + rows, cols = work.shape + group_size = CQ_GROUP_SIZE + groups = cols // group_size + rotation = make_hadamard_matrix(group_size, seed=1234).astype(np.float32) + codebook = make_codebook(group_size, bits).astype(np.float32) + + indices = np.empty((rows, cols), dtype=np.uint8) + norms = np.empty((rows, groups), dtype=np.float16) + for group_index in range(groups): + lo = group_index * group_size + hi = lo + group_size + group = work[:, lo:hi] + row_norms = np.linalg.norm(group, axis=1).clip(min=1e-8).astype(np.float32, copy=False) + rotated = (group / row_norms[:, None]) @ rotation + idx = np.abs(rotated[..., None] - codebook[None, None, :]).argmin(axis=-1).astype(np.uint8) + indices[:, lo:hi] = idx + norms[:, group_index] = row_norms.astype(np.float16) + return pack_indices_lsb(indices, group_size, bits), norms + + +def _quantize_orthogonal_chunk( + chunk: np.ndarray, + *, + bits: int, + input_scale: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + work = chunk.astype(np.float32, copy=False) * input_scale.astype(np.float32, copy=False)[None, :] + rows, cols = work.shape + rotation = make_orthogonal_rotation(cols, seed=1234).astype(np.float32) + codebook = make_codebook(cols, bits).astype(np.float32) + norms = np.linalg.norm(work, axis=1).clip(min=1e-8).astype(np.float32, copy=False) + rotated = (work / norms[:, None]) @ rotation + indices = np.abs(rotated[..., None] - codebook[None, None, :]).argmin(axis=-1).astype(np.uint8) + return pack_indices_lsb(indices, cols, bits), norms.astype(np.float16).reshape(rows, 1) + + +def _write_cq_header( + handle, + *, + rows: int, + cols: int, + precision: int, + packed_bytes: int, + scales_bytes: int, + group_size: int, + num_groups: int, + flags: int, +) -> None: + handle.write(CACTUS_MAGIC) + handle.write(struct.pack(" np.ndarray: + if opened.scales is None or opened.group_size <= 0: + raise RuntimeError(f"INT8 embedding file is missing grouped scales: {opened.path}") + + rows = stop - start + cols = int(opened.shape[1]) + num_groups = int(opened.num_groups) + row_group = int(opened.group_size) + padded_rows = int(opened.shape[0]) + + if opened.is_interleaved: + block = 4 + padded_cols = ((cols + block - 1) // block) * block + q_blocks = opened.data.reshape(padded_rows // block, padded_cols // block, block, block) + s_blocks = opened.scales.reshape(padded_rows // block, num_groups, block) + block_start = start // block + block_end = (stop + block - 1) // block + q = np.asarray(q_blocks[block_start:block_end], dtype=np.int8).transpose(0, 2, 1, 3).reshape(-1, padded_cols) + s = np.asarray(s_blocks[block_start:block_end], dtype=np.float16).astype(np.float32).transpose(0, 2, 1).reshape(-1, num_groups) + row_offset = start - block_start * block + q = q[row_offset : row_offset + rows, :cols] + s = s[row_offset : row_offset + rows] + else: + q = np.asarray(opened.data, dtype=np.int8).reshape(padded_rows, cols)[start:stop] + s = np.asarray(opened.scales, dtype=np.float16).astype(np.float32).reshape(padded_rows, num_groups)[start:stop] + + out = np.empty((rows, cols), dtype=np.float16) + for group_index in range(num_groups): + lo = group_index * row_group + hi = min(lo + row_group, cols) + if lo >= hi: + break + scales = s[:, group_index : group_index + 1] + out[:, lo:hi] = (q[:, lo:hi].astype(np.float32) * scales).astype(np.float16) + return out diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py new file mode 100644 index 000000000..4e0f3b3ce --- /dev/null +++ b/python/tests/test_cli_transpile_defaults.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from pathlib import Path + +from src import cli + + +def _fake_completed_process(returncode: int = 0): + class _Result: + def __init__(self, code: int): + self.returncode = code + + return _Result(returncode) + + +def test_cmd_transpile_defaults_to_skip_execute(monkeypatch) -> None: + parser = cli.create_parser() + args = cli.preprocess_eval_args(parser, ["transpile", "gemma4"]) + + captured: list[tuple[list[str], Path, dict[str, str]]] = [] + + monkeypatch.setattr(cli, "_ensure_python_runtime_library", lambda: Path("/tmp/libcactus.dylib")) + monkeypatch.setattr(cli, "get_weights_dir", lambda model_id: Path("/tmp/missing-weights")) + + def _fake_run(command, cwd=None, env=None): + captured.append((list(command), Path(cwd), dict(env or {}))) + return _fake_completed_process(0) + + monkeypatch.setattr(cli.subprocess, "run", _fake_run) + + rc = cli.cmd_transpile(args) + + assert rc == 0 + assert captured + command, cwd, env = captured[0] + assert command[:4] == [ + cli.sys.executable, + str(cli.PROJECT_ROOT / "python" / "examples" / "transpile_hf_model.py"), + "--model-id", + cli.DEFAULT_MODEL_ID, + ] + assert "--skip-execute" in command + assert cwd == cli.PROJECT_ROOT + assert env["CACTUS_LIB_PATH"] == "/tmp/libcactus.dylib" + + +def test_cmd_transpile_can_execute_immediately_when_requested(monkeypatch) -> None: + parser = cli.create_parser() + args = cli.preprocess_eval_args( + parser, + ["transpile", "gemma4", "--execute-after-transpile", "--artifact-dir", "/tmp/gemma4-bundle"], + ) + + captured: list[list[str]] = [] + + monkeypatch.setattr(cli, "_ensure_python_runtime_library", lambda: Path("/tmp/libcactus.dylib")) + monkeypatch.setattr(cli, "get_weights_dir", lambda model_id: Path("/tmp/missing-weights")) + + def _fake_run(command, cwd=None, env=None): + captured.append(list(command)) + return _fake_completed_process(0) + + monkeypatch.setattr(cli.subprocess, "run", _fake_run) + + rc = cli.cmd_transpile(args) + + assert rc == 0 + assert captured + command = captured[0] + assert "--skip-execute" not in command + assert "--artifact-dir" in command + assert "/tmp/gemma4-bundle" in command diff --git a/python/tests/test_graph.py b/python/tests/test_graph.py index e83b9e90d..e1f0c8615 100644 --- a/python/tests/test_graph.py +++ b/python/tests/test_graph.py @@ -453,6 +453,46 @@ def test_softmax_2d(self): np.testing.assert_allclose(out, expected, atol=5e-2) +class TestGraphCumsum(unittest.TestCase): + + def test_cumsum_last_axis(self): + g = Graph() + a = g.input((2, 4)) + y = a.cumsum(axis=-1) + + data = np.array([[1, 2, 3, 4], [0.5, 1.5, 2.5, 3.5]], dtype=np.float16) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = np.cumsum(data.astype(np.float32), axis=-1).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=5e-2) + + def test_save_load_roundtrip_cumsum(self): + g = Graph() + a = g.input((3, 2)) + y = a.cumsum(axis=0) + + data = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float16) + g.set_input(a, data) + g.execute() + expected = y.numpy() + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "cumsum_graph.cg" + g.save(path) + + loaded = Graph.load(path) + loaded_a = Tensor(loaded, a.id, a.shape, a.dtype) + loaded_y = Tensor(loaded, y.id, y.shape, y.dtype) + + loaded.set_input(loaded_a, data) + loaded.execute() + + out = loaded_y.numpy() + np.testing.assert_allclose(out, expected, atol=1e-2) + + class TestGraphSaveLoad(unittest.TestCase): def _rebind_tensor(self, graph, tensor): diff --git a/python/tests/test_optimize_gemma4_attention.py b/python/tests/test_optimize_gemma4_attention.py new file mode 100644 index 000000000..65a7e745b --- /dev/null +++ b/python/tests/test_optimize_gemma4_attention.py @@ -0,0 +1,224 @@ +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.optimize_graph import normalize_gemma4_decoder_attention_semantics + + +def test_normalize_gemma4_full_attention_uses_sequence_window_compat() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 512), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 1, 800, 512), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 1, 800, 512), dtype="fp16"), + "full_attention_mask": IRValue( + id="full_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "proj": IRValue(id="proj", shape=(1536, 4096), dtype="fp16"), + "out": IRValue(id="out", shape=(1, 800, 1536), dtype="fp16", producer="attn"), + }, + nodes={ + "attn": IRNode( + id="attn", + op="attention_block", + inputs=["query", "key", "value", "full_attention_mask", "proj"], + outputs=["out"], + attrs={ + "has_mask": True, + "is_causal": False, + "window_size": 0, + "attention_output_shape": (1, 800, 4096), + }, + meta={"attention_layer_type": "full_attention"}, + ) + }, + order=["attn"], + inputs=["query", "key", "value", "full_attention_mask"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "input_names": ("query", "key", "value", "full_attention_mask"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value", "proj"] + assert graph.nodes["attn"].attrs["has_mask"] is False + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 800 + assert graph.nodes["attn"].meta["gemma4_full_attention_window_compat"] is True + + +def test_normalize_gemma4_sliding_attention_elides_runtime_mask() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 256), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 8, 800, 256), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 8, 800, 256), dtype="fp16"), + "sliding_attention_mask": IRValue( + id="sliding_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "proj": IRValue(id="proj", shape=(1536, 2048), dtype="fp16"), + "out": IRValue(id="out", shape=(1, 800, 1536), dtype="fp16", producer="attn"), + }, + nodes={ + "attn": IRNode( + id="attn", + op="attention_block", + inputs=["query", "key", "value", "sliding_attention_mask", "proj"], + outputs=["out"], + attrs={ + "has_mask": True, + "is_causal": False, + "window_size": 0, + "attention_output_shape": (1, 800, 2048), + }, + meta={"attention_layer_type": "sliding_attention"}, + ) + }, + order=["attn"], + inputs=["query", "key", "value", "sliding_attention_mask"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + "input_names": ("query", "key", "value", "sliding_attention_mask"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value", "proj"] + assert graph.nodes["attn"].attrs["has_mask"] is False + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 512 + + +def test_normalize_gemma4_attention_recovers_layer_hints_from_graph_meta() -> None: + graph = IRGraph( + values={ + "query_0": IRValue(id="query_0", shape=(1, 8, 800, 256), dtype="fp16"), + "key_0": IRValue(id="key_0", shape=(1, 8, 800, 256), dtype="fp16"), + "value_0": IRValue(id="value_0", shape=(1, 8, 800, 256), dtype="fp16"), + "sliding_attention_mask": IRValue( + id="sliding_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "out_0": IRValue(id="out_0", shape=(1, 8, 800, 256), dtype="fp16", producer="attn_0"), + "query_1": IRValue(id="query_1", shape=(1, 8, 800, 256), dtype="fp16"), + "key_1": IRValue(id="key_1", shape=(1, 8, 800, 256), dtype="fp16"), + "value_1": IRValue(id="value_1", shape=(1, 8, 800, 256), dtype="fp16"), + "full_attention_mask": IRValue( + id="full_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "out_1": IRValue(id="out_1", shape=(1, 8, 800, 256), dtype="fp16", producer="attn_1"), + }, + nodes={ + "attn_0": IRNode( + id="attn_0", + op="attention", + inputs=["query_0", "key_0", "value_0", "sliding_attention_mask"], + outputs=["out_0"], + attrs={"is_causal": False, "window_size": 0}, + ), + "attn_1": IRNode( + id="attn_1", + op="attention", + inputs=["query_1", "key_1", "value_1", "full_attention_mask"], + outputs=["out_1"], + attrs={"is_causal": False, "window_size": 0}, + ), + }, + order=["attn_0", "attn_1"], + inputs=[ + "query_0", + "key_0", + "value_0", + "sliding_attention_mask", + "query_1", + "key_1", + "value_1", + "full_attention_mask", + ], + outputs=["out_1"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + "layer_types": ("sliding_attention", "full_attention"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn_0"].meta["attention_layer_type"] == "sliding_attention" + assert graph.nodes["attn_0"].meta["attention_layer_index"] == 0 + assert graph.nodes["attn_0"].inputs == ["query_0", "key_0", "value_0"] + assert graph.nodes["attn_0"].attrs["is_causal"] is True + assert graph.nodes["attn_0"].attrs["window_size"] == 512 + assert graph.nodes["attn_1"].meta["attention_layer_type"] == "full_attention" + assert graph.nodes["attn_1"].meta["attention_layer_index"] == 1 + assert graph.nodes["attn_1"].inputs == ["query_1", "key_1", "value_1"] + assert graph.nodes["attn_1"].attrs["is_causal"] is True + assert graph.nodes["attn_1"].attrs["window_size"] == 800 + + +def test_normalize_gemma4_attention_elides_logical_fp16_mask() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 256), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 8, 800, 256), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 8, 800, 256), dtype="fp16"), + "mask_a": IRValue(id="mask_a", shape=(1, 1, 800, 800), dtype="bool"), + "mask_b": IRValue(id="mask_b", shape=(1, 1, 800, 800), dtype="bool"), + "mask_fp16": IRValue(id="mask_fp16", shape=(1, 1, 800, 800), dtype="fp16", producer="mask_and"), + "out": IRValue(id="out", shape=(1, 8, 800, 256), dtype="fp16", producer="attn"), + }, + nodes={ + "mask_and": IRNode( + id="mask_and", + op="logical_and", + inputs=["mask_a", "mask_b"], + outputs=["mask_fp16"], + ), + "attn": IRNode( + id="attn", + op="attention", + inputs=["query", "key", "value", "mask_fp16"], + outputs=["out"], + attrs={"is_causal": False, "window_size": 0}, + meta={"attention_layer_type": "sliding_attention"}, + ), + }, + order=["mask_and", "attn"], + inputs=["query", "key", "value", "mask_a", "mask_b"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value"] + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 512 diff --git a/python/tests/test_transpile_qwen35_layer_diff.py b/python/tests/test_transpile_qwen35_layer_diff.py new file mode 100644 index 000000000..b5dc0f817 --- /dev/null +++ b/python/tests/test_transpile_qwen35_layer_diff.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import os +import sys +import unittest +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +try: + import torch +except Exception: # pragma: no cover + torch = None # type: ignore[assignment] + +try: + from transformers import AutoModelForCausalLM +except Exception: # pragma: no cover + AutoModelForCausalLM = None # type: ignore[assignment] + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +if torch is not None and AutoModelForCausalLM is not None: + from src.transpile.capture_pytorch import capture_model + from src.transpile.canonicalize.cleanup import canonicalize_exported_graph + from src.transpile.model_adapters import canonicalize_model_interface + from src.transpile.optimize_graph import optimize_graph + from src.transpile.lower import transpile_preoptimized_ir + + +_RUN_QWEN35_LAYER_DIFF = os.environ.get("CACTUS_RUN_LOCAL_QWEN35_LAYER_DIFF_TEST") == "1" +_DEFAULT_MODEL_ID = os.environ.get("CACTUS_TEST_QWEN35_MODEL_ID", "Qwen/Qwen3.5-2B") +_DEFAULT_INPUT_IDS = os.environ.get("CACTUS_TEST_QWEN35_INPUT_IDS", "151644,8948,198,9707") +_DEFAULT_WEIGHTS_DIR = os.environ.get( + "CACTUS_TEST_QWEN35_WEIGHTS_DIR", + str((Path(__file__).resolve().parents[2] / "weights" / "qwen3.5-2b").resolve()), +) + + +def _resolve_local_model_path(model_id: str) -> str: + candidate = Path(model_id).expanduser() + if candidate.exists(): + return str(candidate.resolve()) + + cache_root = ( + Path.home() + / ".cache" + / "huggingface" + / "hub" + / ("models--" + model_id.replace("/", "--")) + ) + snapshots_dir = cache_root / "snapshots" + if not snapshots_dir.exists(): + raise FileNotFoundError(f"could not find local snapshot for {model_id!r} under {snapshots_dir}") + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + raise FileNotFoundError(f"no snapshots found for {model_id!r} under {snapshots_dir}") + return str(snapshots[-1]) + + +def _parse_input_ids(raw: str) -> list[int]: + token_ids = [int(part.strip()) for part in raw.split(",") if part.strip()] + if not token_ids: + raise ValueError("CACTUS_TEST_QWEN35_INPUT_IDS produced no token ids") + return token_ids + + +def _summarize_ops(ir) -> str: + counts: dict[str, int] = {} + for node_id in ir.order: + op_name = str(ir.nodes[node_id].op) + counts[op_name] = counts.get(op_name, 0) + 1 + items = sorted(counts.items(), key=lambda item: (-item[1], item[0])) + preview = items[:8] + return ",".join(f"{name}:{count}" for name, count in preview) + + +@dataclass +class DiffStats: + label: str + shape: tuple[int, ...] + max_abs_diff: float + mean_abs_diff: float + argmax_ref: int | None + argmax_transpiled: int | None + raw_ir_nodes: int + optimized_ir_nodes: int + top_ops: str + + +if torch is not None and AutoModelForCausalLM is not None: + class _WeightsBoundTranspileWrapper(torch.nn.Module): + def __init__(self, module: torch.nn.Module, *, weights_dir: str | None): + super().__init__() + self.module = module + self.weights_dir = weights_dir + + def forward(self, *args: torch.Tensor) -> torch.Tensor: + return self.module(*args) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.module, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta: dict[str, object] = {} + base_graph = metadata.get("graph", {}) + if isinstance(base_graph, dict): + graph_meta.update(base_graph) + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir + metadata["graph"] = graph_meta + return metadata + + + class Qwen35CheckpointWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module, checkpoint_index: int, *, weights_dir: str | None = None): + super().__init__() + adapter = canonicalize_model_interface(model, task="causal_lm_logits", weights_dir=weights_dir) + if adapter.family != "qwen3_5": + raise ValueError(f"expected qwen3_5 adapter, got family={adapter.family}") + self.adapter = adapter.module + self.checkpoint_index = int(checkpoint_index) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + _, checkpoints = self.adapter.debug_forward(input_ids) + return checkpoints[self.checkpoint_index] + + + class Qwen35LogitsWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__() + adapter = canonicalize_model_interface(model, task="causal_lm_logits", weights_dir=weights_dir) + if adapter.family != "qwen3_5": + raise ValueError(f"expected qwen3_5 adapter, got family={adapter.family}") + self.adapter = adapter.module + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + logits, _ = self.adapter.debug_forward(input_ids) + return logits + + +class TestTranspileQwen35LayerDiff(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + if not _RUN_QWEN35_LAYER_DIFF: + raise unittest.SkipTest("set CACTUS_RUN_LOCAL_QWEN35_LAYER_DIFF_TEST=1 to run") + if torch is None: + raise unittest.SkipTest("torch is not available") + if AutoModelForCausalLM is None: + raise unittest.SkipTest("transformers is not available") + + local_model_path = _resolve_local_model_path(_DEFAULT_MODEL_ID) + cls.model_id = _DEFAULT_MODEL_ID + cls.model_path = local_model_path + cls.input_ids_list = _parse_input_ids(_DEFAULT_INPUT_IDS) + cls.input_ids = torch.tensor([cls.input_ids_list], dtype=torch.long) + cls.weights_dir = None + candidate_weights_dir = Path(_DEFAULT_WEIGHTS_DIR).expanduser().resolve() + if candidate_weights_dir.exists(): + cls.weights_dir = str(candidate_weights_dir) + + common_kwargs: dict[str, object] = { + "local_files_only": True, + "torch_dtype": torch.float16, + "device_map": None, + "low_cpu_mem_usage": True, + } + token = os.environ.get("HF_TOKEN") + if token: + common_kwargs["token"] = token + + cls.model = AutoModelForCausalLM.from_pretrained(local_model_path, **common_kwargs).eval() + + adapter = canonicalize_model_interface(cls.model, task="causal_lm_logits", weights_dir=cls.weights_dir) + if adapter.family != "qwen3_5": + raise unittest.SkipTest(f"expected qwen3_5 adapter, got {adapter.family}") + cls.num_layers = int(adapter.module.backbone.config.num_hidden_layers) + + print("") + print(f"[qwen35-layer-diff] model_id={cls.model_id}") + print(f"[qwen35-layer-diff] model_path={cls.model_path}") + print(f"[qwen35-layer-diff] input_ids={cls.input_ids_list}") + print(f"[qwen35-layer-diff] num_layers={cls.num_layers}") + print(f"[qwen35-layer-diff] weights_dir={cls.weights_dir}") + + @classmethod + def tearDownClass(cls) -> None: + model = getattr(cls, "model", None) + if model is not None: + del cls.model + + def _run_wrapper_diff(self, label: str, wrapper: torch.nn.Module) -> DiffStats: + wrapper = wrapper.eval() + with torch.no_grad(): + reference = wrapper(self.input_ids).detach().float().cpu().numpy() + + transpile_wrapper = _WeightsBoundTranspileWrapper(wrapper, weights_dir=self.weights_dir).eval() + captured = capture_model(transpile_wrapper, (self.input_ids,)) + raw_ir_nodes = len(captured.ir_graph.order) + canonicalize_exported_graph(captured.ir_graph) + optimize_graph(captured.ir_graph) + optimized_ir_nodes = len(captured.ir_graph.order) + top_ops = _summarize_ops(captured.ir_graph) + transpiled = transpile_preoptimized_ir(captured.ir_graph) + transpiled.set_inputs([self.input_ids.cpu().numpy()]) + actual = transpiled.execute()[0].numpy().astype(np.float32) + + diff = np.abs(reference.astype(np.float32) - actual) + max_abs_diff = float(np.max(diff)) + mean_abs_diff = float(np.mean(diff)) + + argmax_ref: int | None = None + argmax_transpiled: int | None = None + if reference.ndim == 3: + argmax_ref = int(np.argmax(reference[0, -1])) + argmax_transpiled = int(np.argmax(actual[0, -1])) + + print( + "[qwen35-layer-diff] " + f"{label} " + f"shape={tuple(reference.shape)} " + f"raw_ir_nodes={raw_ir_nodes} " + f"optimized_ir_nodes={optimized_ir_nodes} " + f"max_abs_diff={max_abs_diff:.6f} " + f"mean_abs_diff={mean_abs_diff:.6f} " + f"top_ops={top_ops}" + ) + if argmax_ref is not None and argmax_transpiled is not None: + print( + "[qwen35-layer-diff] " + f"{label} " + f"argmax_ref={argmax_ref} " + f"argmax_transpiled={argmax_transpiled}" + ) + + return DiffStats( + label=label, + shape=tuple(reference.shape), + max_abs_diff=max_abs_diff, + mean_abs_diff=mean_abs_diff, + argmax_ref=argmax_ref, + argmax_transpiled=argmax_transpiled, + raw_ir_nodes=raw_ir_nodes, + optimized_ir_nodes=optimized_ir_nodes, + top_ops=top_ops, + ) + + def test_qwen35_layer_by_layer_debug_diff(self) -> None: + layer_stats: list[DiffStats] = [] + for layer_index in range(self.num_layers): + wrapper = Qwen35CheckpointWrapper(self.model, checkpoint_index=layer_index, weights_dir=self.weights_dir) + layer_stats.append(self._run_wrapper_diff(f"layer_{layer_index}", wrapper)) + + norm_wrapper = Qwen35CheckpointWrapper(self.model, checkpoint_index=self.num_layers, weights_dir=self.weights_dir) + norm_stats = self._run_wrapper_diff("final_norm", norm_wrapper) + logits_stats = self._run_wrapper_diff("logits", Qwen35LogitsWrapper(self.model, weights_dir=self.weights_dir)) + + worst_layer = max(layer_stats, key=lambda item: item.max_abs_diff) + print( + "[qwen35-layer-diff] " + f"worst_layer={worst_layer.label} " + f"worst_layer_max_abs_diff={worst_layer.max_abs_diff:.6f} " + f"worst_layer_mean_abs_diff={worst_layer.mean_abs_diff:.6f}" + ) + print( + "[qwen35-layer-diff] " + f"final_norm_max_abs_diff={norm_stats.max_abs_diff:.6f} " + f"logits_max_abs_diff={logits_stats.max_abs_diff:.6f}" + ) + + self.assertEqual(len(layer_stats), self.num_layers) + self.assertTrue(np.isfinite(logits_stats.max_abs_diff)) + self.assertTrue(np.isfinite(logits_stats.mean_abs_diff)) diff --git a/python/tests/test_transpile_weight_compat.py b/python/tests/test_transpile_weight_compat.py new file mode 100644 index 000000000..d5c09a970 --- /dev/null +++ b/python/tests/test_transpile_weight_compat.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +from src.tensor_io import save_tensor_with_header +from src.transpile.weight_binding import WeightBinding +from src.transpile.weight_compat import _open_cactus_tensor_file +from src.transpile.weight_compat import ensure_binding_compatible +from src.transpile.weight_compat import ensure_embedding_binding_compatible + + +def _write_grouped_int8_embedding(path: Path, rows: int, cols: int) -> None: + rng = np.random.default_rng(1234) + tensor = rng.standard_normal((rows, cols), dtype=np.float32) + save_tensor_with_header(tensor, path, precision="INT8", model_type="generic") + + +def test_token_embedding_binding_upgrades_to_cq4(tmp_path: Path) -> None: + source = tmp_path / "token_embeddings.weights" + _write_grouped_int8_embedding(source, rows=8, cols=128) + + legacy_fp16 = tmp_path / "token_embeddings.fp16.weights" + legacy_fp16.write_bytes(b"legacy") + + binding = WeightBinding( + path=str(source), + kind="embedding", + source_name="model.embed_tokens.weight", + ) + compat = ensure_embedding_binding_compatible(binding) + + assert compat.path.endswith(".cq4.weights") + assert compat.kind == "embedding" + assert not legacy_fp16.exists() + + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 6 + assert opened.shape == (8, 128) + assert opened.scales is not None + + +def test_per_layer_embedding_binding_upgrades_to_cq2(tmp_path: Path) -> None: + source = tmp_path / "embed_tokens_per_layer.weights" + _write_grouped_int8_embedding(source, rows=8, cols=256) + + binding = WeightBinding( + path=str(source), + kind="embedding", + source_name="model.language_model.embed_tokens_per_layer.weight", + ) + compat = ensure_embedding_binding_compatible(binding) + + assert compat.path.endswith(".cq2.weights") + + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 4 + assert opened.shape == (8, 256) + assert opened.scales is not None + + +def test_gemma4_per_layer_projection_binding_upgrades_legacy_int4(tmp_path: Path) -> None: + rng = np.random.default_rng(1234) + source = tmp_path / "per_layer_model_proj.weights" + tensor = rng.standard_normal((8, 128), dtype=np.float32) + save_tensor_with_header(tensor, source, precision="INT4", model_type="generic") + + binding = WeightBinding( + path=str(source), + kind="weight", + source_name="module.model.model.language_model.per_layer_model_projection.weight", + ) + compat = ensure_binding_compatible(binding, source_tensor=tensor) + + assert compat.path.endswith(".cq4.weights") + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 6 + assert opened.shape == (8, 128) + assert opened.scales is not None + + +def test_decoder_binding_prefers_existing_cq4_companion(tmp_path: Path) -> None: + rng = np.random.default_rng(4321) + source = tmp_path / "layer_0_attn_q.weights" + tensor = rng.standard_normal((8, 128), dtype=np.float32) + save_tensor_with_header(tensor, source, precision="INT4", model_type="generic") + + companion = tmp_path / "layer_0_attn_q.cq4.weights" + save_tensor_with_header(tensor, companion, precision="FP16", model_type="generic") + + binding = WeightBinding( + path=str(source), + kind="weight", + source_name="module.backbone.layers.0.self_attn.q_proj.weight", + ) + compat = ensure_binding_compatible(binding, source_tensor=0) + + assert compat.path == str(companion) diff --git a/tests/ios/CactusTest/CactusTest.xcodeproj/project.pbxproj b/tests/ios/CactusTest/CactusTest.xcodeproj/project.pbxproj new file mode 100644 index 000000000..1db38c954 --- /dev/null +++ b/tests/ios/CactusTest/CactusTest.xcodeproj/project.pbxproj @@ -0,0 +1,488 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + 0CE3609610F41EE57C3EF450 /* test_rag.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F2321D2FF06C53F142C9E4A0 /* test_rag.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_rag_main"; }; }; + 1297BDC23284CC2F68DC3F0D /* test_telemetry.cpp in Sources */ = {isa = PBXBuildFile; fileRef = A43ACF25D2BEECF06382F746 /* test_telemetry.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_telemetry_main"; }; }; + 12CC126B7285C7E7B260F354 /* test_curl.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F993F80BF90226984FE5FA46 /* test_curl.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_curl_main"; }; }; + 159396ABC5D7C56EBD60787C /* test_kv_cache.cpp in Sources */ = {isa = PBXBuildFile; fileRef = EEE3AAC3B98C5A4D7934390B /* test_kv_cache.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_kv_cache_main"; }; }; + 1AA51A651CB30DFBB7A91B32 /* test_audio_cosine.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1F972A3BCEA50EA0944B93F1 /* test_audio_cosine.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_audio_cosine_main"; }; }; + 1B84962019170487AAE57354 /* test_embed.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6792059E996574F9932267B6 /* test_embed.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_embed_main"; }; }; + 406CB7FEF241AED3E35F3E5F /* test_gemma4_thinking.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 36DEA043863AF7F99EE0DBD6 /* test_gemma4_thinking.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_thinking_main"; }; }; + 44DADDEB1F5068DF48AF51A9 /* test_gemma4_suite.cpp in Sources */ = {isa = PBXBuildFile; fileRef = A24768DA8A7AA0FB613B0E47 /* test_gemma4_suite.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_suite_main"; }; }; + 5CCAE950625131F3B367E23D /* test_graph.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2D0B741B8364E6DEB7880ECC /* test_graph.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_graph_main"; }; }; + 64A62BB21BE64C469F1A1910 /* test_audio_chat_pcm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 756477DDFCDD7FDAFE86BBB8 /* test_audio_chat_pcm.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_audio_chat_pcm_main"; }; }; + 703EA361EBF4536C94115D6D /* test_kernel.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 7F29F0F824D84E357FD6780B /* test_kernel.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_kernel_main"; }; }; + 70E04EF839DE4444310D4651 /* test_stt.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2AD904FE8AF16AC361E3FB86 /* test_stt.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_stt_main"; }; }; + 72965BE8BCFDA801E76EEB31 /* test_llm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 23BA649FAF464F37FCCC6B94 /* test_llm.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_llm_main"; }; }; + B2B5CECE2E2477A23192ACCF /* test_vlm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 934A65226FA379ED9D8E8706 /* test_vlm.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_vlm_main"; }; }; + B74B4DBFE7341E92AC8F01E9 /* test_utils.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 344D0D09F78DD280BE1E102E /* test_utils.cpp */; }; + BABC16B78AE1B36F177463BD /* test_index.cpp in Sources */ = {isa = PBXBuildFile; fileRef = BAF5F34E04D4261AAECEA8A1 /* test_index.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_index_main"; }; }; + CD0618CF2D40DEF51EFE9570 /* test_performance.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 68746E7164704D873563D27B /* test_performance.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_performance_main"; }; }; + D8DB20D73F9153B5AFD710AD /* test_gemma4_audio.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 35E791CDD8BA48ED65F41D72 /* test_gemma4_audio.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_audio_main"; }; }; + DFEA8971CFB31E6A890B956F /* test_model_loading.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 257E9B342ABCA116053E18C2 /* test_model_loading.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_model_loading_main"; }; }; + E0F525364DF8BB570ABCD421 /* test_gemma4_vision.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 68B50E60C97A4675E06FA03D /* test_gemma4_vision.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_vision_main"; }; }; + EE0A6D1035305A79B11CA900 /* test_gemma4_audio_layers.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 841D05B43AF13CCB9FC6156A /* test_gemma4_audio_layers.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_audio_layers_main"; }; }; + FD46DB3D01A60AF2DED26E5A /* test_exhaustive.cpp in Sources */ = {isa = PBXBuildFile; fileRef = CC335EA1CA171C8BBE20E50F /* test_exhaustive.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_exhaustive_main"; }; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 1695BEEDC52BAA73FD8D27F1 /* test_utils.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; name = test_utils.h; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_utils.h; sourceTree = ""; }; + 1F972A3BCEA50EA0944B93F1 /* test_audio_cosine.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_audio_cosine.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_audio_cosine.cpp; sourceTree = ""; }; + 23BA649FAF464F37FCCC6B94 /* test_llm.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_llm.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_llm.cpp; sourceTree = ""; }; + 257E9B342ABCA116053E18C2 /* test_model_loading.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_model_loading.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_model_loading.cpp; sourceTree = ""; }; + 2AA889CA2EE510D50037A29D /* CactusTest.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CactusTest.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 2AD904FE8AF16AC361E3FB86 /* test_stt.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_stt.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_stt.cpp; sourceTree = ""; }; + 2D0B741B8364E6DEB7880ECC /* test_graph.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_graph.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_graph.cpp; sourceTree = ""; }; + 344D0D09F78DD280BE1E102E /* test_utils.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_utils.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_utils.cpp; sourceTree = ""; }; + 35E791CDD8BA48ED65F41D72 /* test_gemma4_audio.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_audio.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_audio.cpp; sourceTree = ""; }; + 36DEA043863AF7F99EE0DBD6 /* test_gemma4_thinking.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_thinking.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_thinking.cpp; sourceTree = ""; }; + 6792059E996574F9932267B6 /* test_embed.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_embed.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_embed.cpp; sourceTree = ""; }; + 68746E7164704D873563D27B /* test_performance.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_performance.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_performance.cpp; sourceTree = ""; }; + 68B50E60C97A4675E06FA03D /* test_gemma4_vision.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_vision.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_vision.cpp; sourceTree = ""; }; + 756477DDFCDD7FDAFE86BBB8 /* test_audio_chat_pcm.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_audio_chat_pcm.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_audio_chat_pcm.cpp; sourceTree = ""; }; + 7F29F0F824D84E357FD6780B /* test_kernel.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_kernel.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_kernel.cpp; sourceTree = ""; }; + 841D05B43AF13CCB9FC6156A /* test_gemma4_audio_layers.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_audio_layers.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_audio_layers.cpp; sourceTree = ""; }; + 934A65226FA379ED9D8E8706 /* test_vlm.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_vlm.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_vlm.cpp; sourceTree = ""; }; + A24768DA8A7AA0FB613B0E47 /* test_gemma4_suite.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_suite.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_suite.cpp; sourceTree = ""; }; + A43ACF25D2BEECF06382F746 /* test_telemetry.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_telemetry.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_telemetry.cpp; sourceTree = ""; }; + BAF5F34E04D4261AAECEA8A1 /* test_index.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_index.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_index.cpp; sourceTree = ""; }; + CC335EA1CA171C8BBE20E50F /* test_exhaustive.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_exhaustive.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_exhaustive.cpp; sourceTree = ""; }; + EEE3AAC3B98C5A4D7934390B /* test_kv_cache.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_kv_cache.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_kv_cache.cpp; sourceTree = ""; }; + F2321D2FF06C53F142C9E4A0 /* test_rag.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_rag.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_rag.cpp; sourceTree = ""; }; + F993F80BF90226984FE5FA46 /* test_curl.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_curl.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_curl.cpp; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ + 2AA889E12EE510D60037A29D /* Exceptions for "CactusTest" folder in "CactusTest" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + Info.plist, + ); + target = 2AA889C92EE510D50037A29D /* CactusTest */; + }; +/* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ + +/* Begin PBXFileSystemSynchronizedRootGroup section */ + 2AA889CC2EE510D50037A29D /* CactusTest */ = { + isa = PBXFileSystemSynchronizedRootGroup; + exceptions = ( + 2AA889E12EE510D60037A29D /* Exceptions for "CactusTest" folder in "CactusTest" target */, + ); + path = CactusTest; + sourceTree = ""; + }; +/* End PBXFileSystemSynchronizedRootGroup section */ + +/* Begin PBXFrameworksBuildPhase section */ + 2AA889C72EE510D50037A29D /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 2AA889C12EE510D50037A29D = { + isa = PBXGroup; + children = ( + 2AA889CC2EE510D50037A29D /* CactusTest */, + 2AA889CB2EE510D50037A29D /* Products */, + 5E90E676965D10AF854DF5E1 /* Tests */, + ); + sourceTree = ""; + }; + 2AA889CB2EE510D50037A29D /* Products */ = { + isa = PBXGroup; + children = ( + 2AA889CA2EE510D50037A29D /* CactusTest.app */, + ); + name = Products; + sourceTree = ""; + }; + 5E90E676965D10AF854DF5E1 /* Tests */ = { + isa = PBXGroup; + children = ( + 756477DDFCDD7FDAFE86BBB8 /* test_audio_chat_pcm.cpp */, + 1F972A3BCEA50EA0944B93F1 /* test_audio_cosine.cpp */, + F993F80BF90226984FE5FA46 /* test_curl.cpp */, + 6792059E996574F9932267B6 /* test_embed.cpp */, + CC335EA1CA171C8BBE20E50F /* test_exhaustive.cpp */, + 35E791CDD8BA48ED65F41D72 /* test_gemma4_audio.cpp */, + 841D05B43AF13CCB9FC6156A /* test_gemma4_audio_layers.cpp */, + A24768DA8A7AA0FB613B0E47 /* test_gemma4_suite.cpp */, + 36DEA043863AF7F99EE0DBD6 /* test_gemma4_thinking.cpp */, + 68B50E60C97A4675E06FA03D /* test_gemma4_vision.cpp */, + 2D0B741B8364E6DEB7880ECC /* test_graph.cpp */, + BAF5F34E04D4261AAECEA8A1 /* test_index.cpp */, + 7F29F0F824D84E357FD6780B /* test_kernel.cpp */, + EEE3AAC3B98C5A4D7934390B /* test_kv_cache.cpp */, + 23BA649FAF464F37FCCC6B94 /* test_llm.cpp */, + 257E9B342ABCA116053E18C2 /* test_model_loading.cpp */, + 68746E7164704D873563D27B /* test_performance.cpp */, + F2321D2FF06C53F142C9E4A0 /* test_rag.cpp */, + 2AD904FE8AF16AC361E3FB86 /* test_stt.cpp */, + A43ACF25D2BEECF06382F746 /* test_telemetry.cpp */, + 934A65226FA379ED9D8E8706 /* test_vlm.cpp */, + 344D0D09F78DD280BE1E102E /* test_utils.cpp */, + 1695BEEDC52BAA73FD8D27F1 /* test_utils.h */, + ); + name = Tests; + path = ../..; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 2AA889C92EE510D50037A29D /* CactusTest */ = { + isa = PBXNativeTarget; + buildConfigurationList = 2AA889E22EE510D60037A29D /* Build configuration list for PBXNativeTarget "CactusTest" */; + buildPhases = ( + 2AA889C62EE510D50037A29D /* Sources */, + 2AA889C72EE510D50037A29D /* Frameworks */, + 2AA889C82EE510D50037A29D /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + 2AA889CC2EE510D50037A29D /* CactusTest */, + ); + name = CactusTest; + productName = CactusTest; + productReference = 2AA889CA2EE510D50037A29D /* CactusTest.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 2AA889C22EE510D50037A29D /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastUpgradeCheck = 2600; + TargetAttributes = { + 2AA889C92EE510D50037A29D = { + CreatedOnToolsVersion = 26.0.1; + }; + }; + }; + buildConfigurationList = 2AA889C52EE510D50037A29D /* Build configuration list for PBXProject "CactusTest" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 2AA889C12EE510D50037A29D; + minimizedProjectReferenceProxies = 1; + preferredProjectObjectVersion = 77; + productRefGroup = 2AA889CB2EE510D50037A29D /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 2AA889C92EE510D50037A29D /* CactusTest */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 2AA889C82EE510D50037A29D /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 2AA889C62EE510D50037A29D /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 64A62BB21BE64C469F1A1910 /* test_audio_chat_pcm.cpp in Sources */, + 1AA51A651CB30DFBB7A91B32 /* test_audio_cosine.cpp in Sources */, + 12CC126B7285C7E7B260F354 /* test_curl.cpp in Sources */, + 1B84962019170487AAE57354 /* test_embed.cpp in Sources */, + FD46DB3D01A60AF2DED26E5A /* test_exhaustive.cpp in Sources */, + D8DB20D73F9153B5AFD710AD /* test_gemma4_audio.cpp in Sources */, + EE0A6D1035305A79B11CA900 /* test_gemma4_audio_layers.cpp in Sources */, + 44DADDEB1F5068DF48AF51A9 /* test_gemma4_suite.cpp in Sources */, + 406CB7FEF241AED3E35F3E5F /* test_gemma4_thinking.cpp in Sources */, + E0F525364DF8BB570ABCD421 /* test_gemma4_vision.cpp in Sources */, + 5CCAE950625131F3B367E23D /* test_graph.cpp in Sources */, + BABC16B78AE1B36F177463BD /* test_index.cpp in Sources */, + 703EA361EBF4536C94115D6D /* test_kernel.cpp in Sources */, + 159396ABC5D7C56EBD60787C /* test_kv_cache.cpp in Sources */, + 72965BE8BCFDA801E76EEB31 /* test_llm.cpp in Sources */, + DFEA8971CFB31E6A890B956F /* test_model_loading.cpp in Sources */, + CD0618CF2D40DEF51EFE9570 /* test_performance.cpp in Sources */, + 0CE3609610F41EE57C3EF450 /* test_rag.cpp in Sources */, + 70E04EF839DE4444310D4651 /* test_stt.cpp in Sources */, + 1297BDC23284CC2F68DC3F0D /* test_telemetry.cpp in Sources */, + B2B5CECE2E2477A23192ACCF /* test_vlm.cpp in Sources */, + B74B4DBFE7341E92AC8F01E9 /* test_utils.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + 2AA889E32EE510D60037A29D /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_CXX_LANGUAGE_STANDARD = "c++20"; + CLANG_CXX_LIBRARY = "libc++"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = K3DTVW668Q; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests, + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/cactus, + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/libs/curl/include, + ); + INFOPLIST_FILE = CactusTest/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + OTHER_CPLUSPLUSFLAGS = ( + "$(inherited)", + "-pthread", + "-Wall", + "-Wextra", + "-pedantic", + "-O3", + ); + OTHER_LDFLAGS = ( + "$(inherited)", + "/Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/apple/libcactus-device.a", + "-framework CoreML", + "-framework Foundation", + "-framework Accelerate", + "-framework Security", + "-framework SystemConfiguration", + "-framework CFNetwork", + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/libs/curl/ios/device/libcurl.a, + ); + PRODUCT_BUNDLE_IDENTIFIER = com.cactus.test.parkiratsandhu; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 2AA889E42EE510D60037A29D /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_CXX_LANGUAGE_STANDARD = "c++20"; + CLANG_CXX_LIBRARY = "libc++"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = K3DTVW668Q; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests, + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/cactus, + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/libs/curl/include, + ); + INFOPLIST_FILE = CactusTest/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + OTHER_CPLUSPLUSFLAGS = ( + "$(inherited)", + "-pthread", + "-Wall", + "-Wextra", + "-pedantic", + "-O3", + ); + OTHER_LDFLAGS = ( + "$(inherited)", + "/Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/apple/libcactus-device.a", + "-framework CoreML", + "-framework Foundation", + "-framework Accelerate", + "-framework Security", + "-framework SystemConfiguration", + "-framework CFNetwork", + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/libs/curl/ios/device/libcurl.a, + ); + PRODUCT_BUNDLE_IDENTIFIER = com.cactus.test.parkiratsandhu; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + 2AA889E52EE510D60037A29D /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + }; + name = Debug; + }; + 2AA889E62EE510D60037A29D /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 2AA889C52EE510D50037A29D /* Build configuration list for PBXProject "CactusTest" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 2AA889E52EE510D60037A29D /* Debug */, + 2AA889E62EE510D60037A29D /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 2AA889E22EE510D60037A29D /* Build configuration list for PBXNativeTarget "CactusTest" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 2AA889E32EE510D60037A29D /* Debug */, + 2AA889E42EE510D60037A29D /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 2AA889C22EE510D50037A29D /* Project object */; +}