From 86fe8d5703127e1e018a9c4d8fd8762d2b96b83b Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Tue, 5 May 2026 09:14:29 -0700 Subject: [PATCH 01/11] Adding transpiler base code --- python/src/transpile/canonicalize/cleanup.py | 629 +++++ python/src/transpile/canonicalize/utils.py | 228 ++ python/src/transpile/capture_pytorch.py | 302 +++ python/src/transpile/fusion/__init__.py | 43 + python/src/transpile/fusion/attention.py | 550 +++++ python/src/transpile/fusion/common.py | 73 + python/src/transpile/fusion/conv.py | 123 + python/src/transpile/fusion/deltanet.py | 357 +++ python/src/transpile/fusion/linear.py | 58 + python/src/transpile/fusion/mlp.py | 53 + python/src/transpile/fusion/rel_pos_bias.py | 104 + python/src/transpile/fusion/rms_norm.py | 112 + python/src/transpile/fusion/rope.py | 252 ++ python/src/transpile/graph_ir.py | 201 ++ python/src/transpile/import_ir.py | 266 ++ python/src/transpile/import_semantics.py | 337 +++ python/src/transpile/importers.py | 1754 ++++++++++++++ python/src/transpile/lower.py | 2142 +++++++++++++++++ python/src/transpile/model_adapters.py | 803 ++++++ python/src/transpile/model_patterns.py | 76 + python/src/transpile/normalize.py | 199 ++ python/src/transpile/ops.py | 234 ++ python/src/transpile/optimize_graph.py | 737 ++++++ python/src/transpile/weight_binding.py | 330 +++ .../CactusTest.xcodeproj/project.pbxproj | 488 ++++ 25 files changed, 10451 insertions(+) create mode 100644 python/src/transpile/canonicalize/cleanup.py create mode 100644 python/src/transpile/canonicalize/utils.py create mode 100644 python/src/transpile/capture_pytorch.py create mode 100644 python/src/transpile/fusion/__init__.py create mode 100644 python/src/transpile/fusion/attention.py create mode 100644 python/src/transpile/fusion/common.py create mode 100644 python/src/transpile/fusion/conv.py create mode 100644 python/src/transpile/fusion/deltanet.py create mode 100644 python/src/transpile/fusion/linear.py create mode 100644 python/src/transpile/fusion/mlp.py create mode 100644 python/src/transpile/fusion/rel_pos_bias.py create mode 100644 python/src/transpile/fusion/rms_norm.py create mode 100644 python/src/transpile/fusion/rope.py create mode 100644 python/src/transpile/graph_ir.py create mode 100644 python/src/transpile/import_ir.py create mode 100644 python/src/transpile/import_semantics.py create mode 100644 python/src/transpile/importers.py create mode 100644 python/src/transpile/lower.py create mode 100644 python/src/transpile/model_adapters.py create mode 100644 python/src/transpile/model_patterns.py create mode 100644 python/src/transpile/normalize.py create mode 100644 python/src/transpile/ops.py create mode 100644 python/src/transpile/optimize_graph.py create mode 100644 python/src/transpile/weight_binding.py create mode 100644 tests/ios/CactusTest/CactusTest.xcodeproj/project.pbxproj diff --git a/python/src/transpile/canonicalize/cleanup.py b/python/src/transpile/canonicalize/cleanup.py new file mode 100644 index 000000000..d72da0b3c --- /dev/null +++ b/python/src/transpile/canonicalize/cleanup.py @@ -0,0 +1,629 @@ +from __future__ import annotations + +from collections import Counter + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.ops import canonicalize_op +from src.transpile.ops import get_op +from src.transpile.ops import has_op +from src.transpile.canonicalize.utils import bypass_unary_node +from src.transpile.canonicalize.utils import dce +from src.transpile.canonicalize.utils import ir_dtype_to_torch +from src.transpile.canonicalize.utils import materialize_constant_output +from src.transpile.canonicalize.utils import normalize_dim +from src.transpile.canonicalize.utils import normalize_dtype_name +from src.transpile.canonicalize.utils import numel +from src.transpile.canonicalize.utils import rebuild_graph + + +COMPILER_SUPPORTED_OPS = { + "arange", + "ones", +} + +CANONICAL_UNSUPPORTED_RENAMES = { + "aten.__and__.Tensor": "logical_and", + "aten.__or__.Tensor": "logical_or", + "aten.cumsum.default": "cumsum", + "aten.eq.Scalar": "equal", + "aten.eq.Tensor": "equal", + "aten.gt.Tensor": "greater", + "aten.index.Tensor": "advanced_index", + "aten.le.Tensor": "less_equal", +} + +FP32_SUPPORTED_ALL_INPUT_OPS = { + "precision_cast", + "view", + "slice", + "index", + "gather", + "layer_norm", + "batch_norm", + "glu", +} + +FP32_SUPPORTED_INPUT_INDICES = { + "embedding": {1}, +} + +FP16_ONLY_OUTPUT_OPS = { + "add", + "subtract", + "multiply", + "divide", + "scalar_add", + "scalar_subtract", + "scalar_subtract_reverse", + "scalar_multiply", + "scalar_divide", + "scalar_divide_reverse", + "not_equal", + "equal", + "greater", + "less_equal", + "logical_and", + "logical_or", + "logical_not", + "one_hot", + "scalar_not_equal", + "scalar_equal", + "abs", + "negate", + "pow", + "scalar_exp", + "scalar_sqrt", + "scalar_log", + "scalar_cos", + "scalar_sin", + "permute", + "matmul", + "cat", + "softmax", + "sum", + "mean", + "variance", + "min", + "max", + "rms_norm", + "embedding", + "group_norm", + "relu", + "silu", + "gelu", + "gelu_erf", + "sigmoid", + "tanh", + "linear", + "addmm", + "attention", + "attention_block", + "conv_module", + "rope", +} + + +def canonicalize_exported_graph(graph: IRGraph, *, max_passes: int = 8) -> IRGraph: + verify_ir(graph) + + for _ in range(max_passes): + changed = False + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if _canonicalize_node(graph, node): + changed = True + + rebuild_graph(graph) + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if _simplify_node(graph, node): + changed = True + + rebuild_graph(graph) + + if _legalize_precisions(graph): + changed = True + rebuild_graph(graph) + + dce(graph) + + if not changed: + break + + unsupported_counts = summarize_unsupported_ops(graph) + graph.meta = dict(graph.meta) + graph.meta["canonical_unsupported_ops"] = tuple(sorted(unsupported_counts)) + graph.meta["canonical_unsupported_op_counts"] = unsupported_counts + verify_ir(graph) + return graph + + +def summarize_unsupported_ops(graph: IRGraph) -> dict[str, int]: + counts: Counter[str] = Counter() + for node_id in graph.order: + node = graph.nodes[node_id] + if not _is_supported_without_backend_kernel(node.op): + counts[node.op] += 1 + return dict(sorted(counts.items())) + + +def _canonicalize_node(graph: IRGraph, node: IRNode) -> bool: + original_op = node.op + + if original_op == "aten.new_ones.default": + return _materialize_new_ones_constant(graph, node) + + if original_op in {"aten.new_zeros.default", "new_zeros"}: + return _materialize_new_zeros_constant(graph, node) + + if original_op == "arange": + return _materialize_arange_constant(graph, node) + + if original_op in {"identity", "contiguous"}: + return bypass_unary_node(graph, node) + + if original_op in CANONICAL_UNSUPPORTED_RENAMES: + node.op = CANONICAL_UNSUPPORTED_RENAMES[original_op] + return True + + if original_op == "multiply_inplace": + node.op = "multiply" + return True + + if original_op == "view" and "shape" not in node.attrs: + return _rewrite_view_from_output_shape(graph, node) + + if original_op in {"reshape", "flatten", "unsqueeze", "squeeze"}: + return _rewrite_view_from_output_shape(graph, node) + + if original_op == "expand": + return _rewrite_expand_if_view_equivalent(graph, node) + + if original_op == "transpose": + return _rewrite_transpose_as_permute(graph, node) + + if original_op == "movedim": + return _rewrite_movedim_as_permute(graph, node) + + if original_op == "type_as": + return _rewrite_type_as(graph, node) + + if original_op == "concat": + node.op = "cat" + if "dim" in node.attrs and "axis" not in node.attrs: + node.attrs = {**node.attrs, "axis": int(node.attrs["dim"])} + return True + + if original_op == "scaled_dot_product_attention": + node.op = "attention" + node.attrs = { + "scale": float(node.attrs.get("scale", 1.0)), + "is_causal": bool(node.attrs.get("is_causal", False)), + "window_size": int(node.attrs.get("window_size", 0)), + **({"mask": node.attrs["mask"]} if "mask" in node.attrs else {}), + **({"additive_mask": bool(node.attrs.get("additive_mask", False))} if "additive_mask" in node.attrs else {}), + } + return True + + canonical_op = canonicalize_op(original_op) + if canonical_op != original_op: + node.op = canonical_op + _normalize_attr_spellings(node) + if has_op(node.op): + _filter_attrs_to_schema(node) + return True + + if has_op(node.op): + _normalize_attr_spellings(node) + _filter_attrs_to_schema(node) + return True + + return False + + +def _simplify_node(graph: IRGraph, node: IRNode) -> bool: + if _fold_constant_node(graph, node): + return True + + if node.op == "precision_cast": + return _remove_noop_precision_cast(graph, node) + + if node.op == "view": + if _remove_noop_view(graph, node): + return True + return _collapse_view_chain(graph, node) + + if node.op == "slice": + return _remove_noop_slice(graph, node) + + if node.op == "scalar_add": + return _remove_noop_scalar(graph, node, neutral=0.0) + + if node.op == "scalar_subtract": + return _remove_noop_scalar(graph, node, neutral=0.0) + + if node.op == "scalar_multiply": + return _remove_noop_scalar(graph, node, neutral=1.0) + + if node.op == "scalar_divide": + return _remove_noop_scalar(graph, node, neutral=1.0) + + return False + + +def _fold_constant_node(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + + output_value = graph.values.get(node.outputs[0]) + if output_value is None: + return False + + if node.op in FP16_ONLY_OUTPUT_OPS: + output_value.dtype = "fp16" + node.meta["dtype"] = "fp16" + + output_torch_dtype = ir_dtype_to_torch(output_value.dtype) if output_value.dtype is not None else None + + constant_inputs = [graph.constants.get(value_id) for value_id in node.inputs] + if node.op == "view" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + if output_value.shape is None: + return False + return materialize_constant_output( + graph, + node, + constant_inputs[0].reshape(tuple(int(dim) for dim in output_value.shape)).to(dtype=output_torch_dtype), + ) + + if node.op == "expand" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + if output_value.shape is None: + return False + return materialize_constant_output( + graph, + node, + constant_inputs[0].expand(tuple(int(dim) for dim in output_value.shape)).clone().to(dtype=output_torch_dtype), + ) + + if node.op == "precision_cast" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + dtype = normalize_dtype_name(node.attrs.get("dtype")) + if dtype is None: + return False + return materialize_constant_output(graph, node, constant_inputs[0].to(dtype=ir_dtype_to_torch(dtype))) + + if node.op in {"cos", "scalar_cos"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + return materialize_constant_output(graph, node, torch.cos(constant_inputs[0])) + + if node.op in {"sin", "scalar_sin"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + return materialize_constant_output(graph, node, torch.sin(constant_inputs[0])) + + if node.op in {"scalar_add", "scalar_subtract", "scalar_multiply", "scalar_divide"}: + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + scalar = float(node.attrs["value"]) + if node.op == "scalar_add": + result = constant_inputs[0] + scalar + elif node.op == "scalar_subtract": + result = constant_inputs[0] - scalar + elif node.op == "scalar_multiply": + result = constant_inputs[0] * scalar + else: + result = constant_inputs[0] / scalar + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op in {"scalar_not_equal", "scalar_equal"}: + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + scalar = float(node.attrs["value"]) + if node.op == "scalar_not_equal": + result = constant_inputs[0] != scalar + else: + result = constant_inputs[0] == scalar + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op in {"not_equal", "equal", "greater", "less_equal", "logical_and", "logical_or"}: + if len(constant_inputs) != 2 or not all(isinstance(value, torch.Tensor) for value in constant_inputs): + return False + lhs, rhs = constant_inputs + if node.op == "not_equal": + result = lhs != rhs + elif node.op == "equal": + result = lhs == rhs + elif node.op == "greater": + result = lhs > rhs + elif node.op == "less_equal": + result = lhs <= rhs + elif node.op == "logical_and": + result = torch.logical_and(lhs != 0, rhs != 0) + else: + result = torch.logical_or(lhs != 0, rhs != 0) + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op == "logical_not": + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + result = torch.logical_not(constant_inputs[0] != 0) + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + return False + + +def _legalize_precisions(graph: IRGraph) -> bool: + changed = False + counter = 0 + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + if node.op in FP32_SUPPORTED_ALL_INPUT_OPS: + continue + + inserted: list[IRNode] = [] + allowed_indices = FP32_SUPPORTED_INPUT_INDICES.get(node.op, set()) + for input_index, value_id in enumerate(list(node.inputs)): + if input_index in allowed_indices: + continue + value = graph.values.get(value_id) + if value is None or normalize_dtype_name(value.dtype) != "fp32": + continue + + counter += 1 + cast_node_id = f"{node.id}_legalize_fp16_{counter}" + cast_value_id = f"{value_id}_legalize_fp16_{counter}" + cast_node = IRNode( + id=cast_node_id, + op="precision_cast", + inputs=[value_id], + outputs=[cast_value_id], + attrs={"dtype": "fp16"}, + meta={"inserted_by": "canonicalize_exported_graph"}, + ) + graph.values[cast_value_id] = IRValue( + id=cast_value_id, + shape=value.shape, + dtype="fp16", + producer=cast_node_id, + users=[], + ) + node.inputs[input_index] = cast_value_id + inserted.append(cast_node) + changed = True + + if inserted: + insert_at = graph.order.index(node_id) + for offset, cast_node in enumerate(inserted): + graph.nodes[cast_node.id] = cast_node + graph.order.insert(insert_at + offset, cast_node.id) + + if node.op in FP16_ONLY_OUTPUT_OPS: + for output_id in node.outputs: + output_value = graph.values.get(output_id) + if output_value is not None: + output_value.dtype = "fp16" + node.meta["dtype"] = "fp16" + + return changed + + +def _rewrite_view_from_output_shape(graph: IRGraph, node: IRNode) -> bool: + if not node.outputs: + return False + output_value = graph.values.get(node.outputs[0]) + if output_value is None or output_value.shape is None: + return False + node.op = "view" + node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} + return True + + +def _rewrite_expand_if_view_equivalent(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs or not node.outputs: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if numel(input_value.shape) != numel(output_value.shape): + node.meta["canonicalization_blocked"] = "expand_requires_broadcast" + return False + node.op = "view" + node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} + return True + + +def _rewrite_transpose_as_permute(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.shape is None: + return False + rank = len(input_value.shape) + dim0 = normalize_dim(int(node.attrs["dim0"]), rank) + dim1 = normalize_dim(int(node.attrs["dim1"]), rank) + permutation = list(range(rank)) + permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] + node.op = "permute" + node.attrs = {"permutation": tuple(permutation)} + return True + + +def _rewrite_movedim_as_permute(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.shape is None: + return False + rank = len(input_value.shape) + source = [normalize_dim(int(dim), rank) for dim in node.attrs["source"]] + destination = [normalize_dim(int(dim), rank) for dim in node.attrs["destination"]] + if len(source) != len(destination): + return False + + remaining = [dim for dim in range(rank) if dim not in source] + permutation = list(remaining) + for dst, src in sorted(zip(destination, source)): + permutation.insert(dst, src) + + node.op = "permute" + node.attrs = {"permutation": tuple(permutation)} + return True + + +def _rewrite_type_as(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + target_dtype = None + if node.outputs: + output_value = graph.values.get(node.outputs[0]) + if output_value is not None: + target_dtype = output_value.dtype + if target_dtype is None and len(node.inputs) > 1: + target_value = graph.values.get(node.inputs[1]) + if target_value is not None: + target_dtype = target_value.dtype + if target_dtype is None: + return False + node.op = "precision_cast" + node.inputs = [node.inputs[0]] + node.attrs = {"dtype": normalize_dtype_name(target_dtype)} + return True + + +def _normalize_attr_spellings(node: IRNode) -> None: + attrs = dict(node.attrs) + if "dim" in attrs and "axis" not in attrs and node.op in {"softmax", "sum", "mean", "variance", "min", "max", "cat", "slice", "index"}: + attrs["axis"] = attrs.pop("dim") + if node.op == "permute" and "dims" in attrs and "permutation" not in attrs: + attrs["permutation"] = tuple(int(dim) for dim in attrs.pop("dims")) + if node.op == "slice": + attrs.setdefault("step", 1) + if node.op == "precision_cast" and "dtype" in attrs: + attrs["dtype"] = normalize_dtype_name(attrs["dtype"]) + node.attrs = attrs + + +def _filter_attrs_to_schema(node: IRNode) -> None: + schema = get_op(node.op) + if not schema.attrs: + node.attrs = {} + return + node.attrs = {name: node.attrs[name] for name in schema.attrs if name in node.attrs} + + +def _remove_noop_precision_cast(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + target_dtype = normalize_dtype_name(node.attrs.get("dtype")) + input_dtype = None if input_value is None else normalize_dtype_name(input_value.dtype) + output_dtype = None if output_value is None else normalize_dtype_name(output_value.dtype) + if target_dtype is None or target_dtype == input_dtype or (input_dtype is not None and output_dtype == input_dtype): + return bypass_unary_node(graph, node) + return False + + +def _remove_noop_view(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if tuple(input_value.shape) == tuple(output_value.shape): + return bypass_unary_node(graph, node) + return False + + +def _collapse_view_chain(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.producer is None: + return False + producer = graph.nodes.get(input_value.producer) + if producer is None or producer.op != "view" or len(producer.inputs) != 1: + return False + node.inputs = [producer.inputs[0]] + return True + + +def _remove_noop_slice(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if tuple(input_value.shape) == tuple(output_value.shape): + return bypass_unary_node(graph, node) + return False + + +def _remove_noop_scalar(graph: IRGraph, node: IRNode, *, neutral: float) -> bool: + value = node.attrs.get("value") + if value is None: + return False + if float(value) != float(neutral): + return False + return bypass_unary_node(graph, node) + + +def _materialize_new_ones_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None or output_value.dtype is None: + return False + torch_dtype = ir_dtype_to_torch(output_value.dtype) + const = torch.ones(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _materialize_new_zeros_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None or output_value.dtype is None: + return False + torch_dtype = ir_dtype_to_torch(output_value.dtype) + const = torch.zeros(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _materialize_arange_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + start = int(node.attrs.get("start", 0)) + end = node.attrs.get("end") + if end is None: + return False + step = int(node.attrs.get("step", 1)) + dtype = normalize_dtype_name(node.attrs.get("dtype", "fp32")) + torch_dtype = ir_dtype_to_torch(dtype) + const = torch.arange(start, int(end), step=step, dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _is_supported_without_backend_kernel(op_name: str) -> bool: + return has_op(op_name) or op_name in COMPILER_SUPPORTED_OPS diff --git a/python/src/transpile/canonicalize/utils.py b/python/src/transpile/canonicalize/utils.py new file mode 100644 index 000000000..30c4da76f --- /dev/null +++ b/python/src/transpile/canonicalize/utils.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +from typing import Any + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir + + +def normalize_dtype_name(dtype: Any) -> str | None: + if dtype is None: + return None + if not isinstance(dtype, str): + return str(dtype) + if dtype.startswith("torch."): + torch_name = dtype.split(".", 1)[1] + mapping = { + "bool": "bool", + "int8": "int8", + "int16": "int16", + "int32": "int32", + "int64": "int64", + "float16": "fp16", + "bfloat16": "bf16", + "float32": "fp32", + "float64": "fp64", + } + return mapping.get(torch_name, dtype) + return dtype + + +def ir_dtype_to_torch(dtype: str) -> torch.dtype: + normalized = normalize_dtype_name(dtype) + if normalized == "bool": + return torch.bool + if normalized == "int8": + return torch.int8 + if normalized == "int16": + return torch.int16 + if normalized == "int32": + return torch.int32 + if normalized == "int64": + return torch.int64 + if normalized == "fp16": + return torch.float16 + if normalized == "bf16": + return torch.bfloat16 + if normalized == "fp32": + return torch.float32 + if normalized == "fp64": + return torch.float64 + raise NotImplementedError(f"unsupported IR dtype for constant materialization: {dtype}") + + +def normalize_dim(dim: int, rank: int) -> int: + if dim < 0: + dim += rank + return dim + + +def numel(shape: tuple[int, ...]) -> int: + total = 1 + for dim in shape: + total *= int(dim) + return total + + +def refresh_value_users(graph: IRGraph) -> None: + for value in graph.values.values(): + value.users = [] + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + value = graph.values.get(input_id) + if value is not None: + value.users.append(node_id) + + +def rebuild_graph(graph: IRGraph) -> None: + new_values: dict[str, IRValue] = {} + + for value_id in graph.inputs: + old = graph.values.get(value_id, IRValue(id=value_id)) + new_values[value_id] = IRValue( + id=value_id, + shape=old.shape, + dtype=old.dtype, + producer=None, + users=[], + meta=dict(old.meta), + ) + + for value_id in graph.constants.keys(): + old = graph.values.get(value_id, IRValue(id=value_id)) + new_values[value_id] = IRValue( + id=value_id, + shape=old.shape, + dtype=old.dtype, + producer=None, + users=[], + meta=dict(old.meta), + ) + + for node_id in graph.order: + node = graph.nodes[node_id] + for output_id in node.outputs: + old = graph.values.get(output_id, IRValue(id=output_id)) + new_values[output_id] = IRValue( + id=output_id, + shape=old.shape, + dtype=old.dtype, + producer=node_id, + users=[], + meta=dict(old.meta), + ) + + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + if input_id not in new_values: + old = graph.values.get(input_id, IRValue(id=input_id)) + new_values[input_id] = IRValue( + id=input_id, + shape=old.shape, + dtype=old.dtype, + producer=old.producer, + users=[], + meta=dict(old.meta), + ) + new_values[input_id].users.append(node_id) + + for output_id in graph.outputs: + if output_id not in new_values: + old = graph.values.get(output_id, IRValue(id=output_id)) + new_values[output_id] = IRValue( + id=output_id, + shape=old.shape, + dtype=old.dtype, + producer=old.producer, + users=[], + meta=dict(old.meta), + ) + + graph.values = new_values + verify_ir(graph) + + +def dce(graph: IRGraph) -> None: + live_values = set(graph.outputs) + live_nodes: set[str] = set() + + changed = True + while changed: + changed = False + for node_id in reversed(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if not any(output in live_values for output in node.outputs): + continue + if node_id not in live_nodes: + live_nodes.add(node_id) + changed = True + for input_id in node.inputs: + if input_id not in live_values: + live_values.add(input_id) + changed = True + + graph.order = [node_id for node_id in graph.order if node_id in live_nodes] + graph.nodes = {node_id: node for node_id, node in graph.nodes.items() if node_id in live_nodes} + + live_value_ids = set(graph.inputs) | set(graph.outputs) + for node in graph.nodes.values(): + live_value_ids.update(node.inputs) + live_value_ids.update(node.outputs) + + graph.values = {value_id: value for value_id, value in graph.values.items() if value_id in live_value_ids} + graph.constants = {value_id: value for value_id, value in graph.constants.items() if value_id in live_value_ids} + rebuild_graph(graph) + + +def remove_node(graph: IRGraph, node_id: str) -> None: + if node_id in graph.nodes: + del graph.nodes[node_id] + graph.order = [existing_id for existing_id in graph.order if existing_id != node_id] + + +def drop_value_if_unused(graph: IRGraph, value_id: str) -> None: + if value_id in graph.inputs or value_id in graph.outputs or value_id in graph.constants: + return + value = graph.values.get(value_id) + if value is None: + return + if value.producer is None and not value.users: + del graph.values[value_id] + + +def bypass_unary_node(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_id = node.inputs[0] + output_id = node.outputs[0] + + for other in graph.nodes.values(): + other.inputs = [input_id if value_id == output_id else value_id for value_id in other.inputs] + graph.outputs = [input_id if value_id == output_id else value_id for value_id in graph.outputs] + + remove_node(graph, node.id) + if output_id in graph.values: + del graph.values[output_id] + drop_value_if_unused(graph, input_id) + return True + + +def materialize_constant_output(graph: IRGraph, node: IRNode, value: Any) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None: + return False + graph.constants[output_id] = value + output_value.producer = None + remove_node(graph, node.id) + return True diff --git a/python/src/transpile/capture_pytorch.py b/python/src/transpile/capture_pytorch.py new file mode 100644 index 000000000..7a9bb3cd8 --- /dev/null +++ b/python/src/transpile/capture_pytorch.py @@ -0,0 +1,302 @@ +from dataclasses import dataclass +import inspect +from typing import Any +from typing import Iterable + +import torch +from torch.export import export + +from src.transpile.graph_ir import IRGraph + + +class CapturePhaseError(RuntimeError): + def __init__(self, phase: str, message: str, *, cause: Exception | None = None): + super().__init__(f"[capture:{phase}] {message}") + self.phase = phase + self.cause = cause + + +@dataclass +class TensorMetadata: + shape: tuple[Any, ...] + dtype: Any + stride: tuple[Any, ...] | None = None + device: Any | None = None + + +@dataclass +class CapturedModel: + exported_program: Any + graph_module: Any + graph: Any + state_dict: dict[str, Any] + ir_graph: IRGraph + source_module: Any + example_args: tuple[Any, ...] + example_kwargs: dict[str, Any] + strict: bool + transpile_metadata: dict[str, Any] + + def named_parameters(self): + return self.exported_program.named_parameters() + + def named_buffers(self): + return self.exported_program.named_buffers() + + def parameters_dict(self) -> dict[str, Any]: + return dict(self.named_parameters()) + + def buffers_dict(self) -> dict[str, Any]: + return dict(self.named_buffers()) + + def placeholders(self) -> list[Any]: + return [node for node in self.graph.nodes if node.op == "placeholder"] + + def outputs(self) -> list[Any]: + return [node for node in self.graph.nodes if node.op == "output"] + + +def _normalize_args(args: Any) -> tuple[Any, ...]: + if args is None: + return () + if isinstance(args, tuple): + return args + if isinstance(args, list): + return tuple(args) + return (args,) + + +def _normalize_kwargs(kwargs: dict[str, Any] | None) -> dict[str, Any]: + return {} if kwargs is None else dict(kwargs) + + +def _clone_example_value(value: Any) -> Any: + if isinstance(value, torch.Tensor): + return value.detach().clone() + if isinstance(value, tuple): + return tuple(_clone_example_value(v) for v in value) + if isinstance(value, list): + return [_clone_example_value(v) for v in value] + if isinstance(value, dict): + return {k: _clone_example_value(v) for k, v in value.items()} + return value + + +def _clone_examples(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[tuple[Any, ...], dict[str, Any]]: + return ( + tuple(_clone_example_value(arg) for arg in args), + {k: _clone_example_value(v) for k, v in kwargs.items()}, + ) + + +def _inject_export_safe_kwargs(model: torch.nn.Module, kwargs: dict[str, Any]) -> dict[str, Any]: + try: + signature = inspect.signature(model.forward) + except (TypeError, ValueError): + return kwargs + + updated = dict(kwargs) + parameters = signature.parameters + + # HF causal LM forwards often default to returning cache objects and ModelOutput + # containers, which torch.export cannot serialize cleanly. Keep the capture + # boundary tensor-only unless the caller explicitly asked otherwise. + if "use_cache" in parameters and "use_cache" not in updated: + updated["use_cache"] = False + if "return_dict" in parameters and "return_dict" not in updated: + updated["return_dict"] = False + return updated + + +def _call_transpile_metadata_provider(module: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: + for attr_name in ("get_transpile_metadata", "transpile_metadata"): + provider = getattr(module, attr_name, None) + if not callable(provider): + continue + try: + metadata = provider(args=args, kwargs=kwargs) + except TypeError: + metadata = provider() + if metadata is None: + return {} + if not isinstance(metadata, dict): + raise TypeError(f"{type(module).__name__}.{attr_name} must return a dict or None") + return metadata + return {} + + +def _collect_transpile_metadata(model: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: + graph_meta: dict[str, Any] = {} + provider_graph_meta: dict[str, dict[str, Any]] = {} + import_hints: list[dict[str, Any]] = [] + seen_modules: set[int] = set() + + for module_path, module in model.named_modules(): + module_id = id(module) + if module_id in seen_modules: + continue + seen_modules.add(module_id) + + metadata = _call_transpile_metadata_provider(module, args, kwargs) + if not metadata: + continue + + provider_key = module_path or type(module).__name__ + provider_meta = metadata.get("graph", {}) + if isinstance(provider_meta, dict) and provider_meta: + provider_graph_meta[provider_key] = dict(provider_meta) + if not module_path: + graph_meta.update(provider_meta) + + raw_hints = metadata.get("import_hints", ()) + for raw_hint in raw_hints: + if not isinstance(raw_hint, dict): + raise TypeError(f"{type(module).__name__} import hints must be dictionaries") + hint = dict(raw_hint) + hint.setdefault("provider", provider_key) + import_hints.append(hint) + + if provider_graph_meta: + graph_meta.setdefault("transpile_metadata_providers", provider_graph_meta) + if import_hints: + graph_meta["import_hint_count"] = len(import_hints) + + if not graph_meta and not import_hints: + return {} + return { + "graph": graph_meta, + "import_hints": import_hints, + } + +def capture_model(model, args, kwargs=None, *, strict=True) -> CapturedModel: + if not isinstance(model, torch.nn.Module): + raise TypeError("model must be a torch.nn.Module") + + normalized_args = _normalize_args(args) + normalized_kwargs = _inject_export_safe_kwargs(model, _normalize_kwargs(kwargs)) + + if not normalized_args and not normalized_kwargs: + raise ValueError("capture_model requires example args or kwargs") + + model = model.eval() + example_args, example_kwargs = _clone_examples(normalized_args, normalized_kwargs) + transpile_metadata = _collect_transpile_metadata(model, example_args, example_kwargs) + + try: + ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) + except Exception as exc: + raise CapturePhaseError( + "export", + f"torch.export failed for model={type(model).__name__} strict={strict}: {exc}", + cause=exc, + ) from exc + from src.transpile.import_ir import import_captured_to_ir + + raw_captured = CapturedModel( + exported_program=ep, + graph_module=ep.graph_module, + graph=ep.graph, + state_dict=dict(ep.state_dict), + ir_graph=IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[]), + source_module=model, + example_args=example_args, + example_kwargs=example_kwargs, + strict=strict, + transpile_metadata=transpile_metadata, + ) + try: + ir_graph = import_captured_to_ir(raw_captured, strict=strict) + except Exception as exc: + raise CapturePhaseError( + "import", + f"failed to import exported graph to IR for model={type(model).__name__} strict={strict}: {exc}", + cause=exc, + ) from exc + + return CapturedModel( + exported_program=ep, + graph_module=ep.graph_module, + graph=ep.graph, + state_dict=dict(ep.state_dict), + ir_graph=ir_graph, + source_module=model, + example_args=example_args, + example_kwargs=example_kwargs, + strict=strict, + transpile_metadata=transpile_metadata, + ) + + +def capture_model_with_fallback(model, args, kwargs=None) -> CapturedModel: + try: + return capture_model(model, args, kwargs=kwargs, strict=True) + except Exception: + return capture_model(model, args, kwargs=kwargs, strict=False) + + +def resolve_attr(root: Any, target: str) -> Any: + obj = root + for atom in target.split("."): + obj = getattr(obj, atom) + return obj + + +def format_target(node: Any) -> str: + return str(node.target) + +def get_tensor_metadata(node: Any) -> TensorMetadata | None: + meta = getattr(node, "meta", {}) or {} + value = meta.get("val") + if value is None: + return None + + if isinstance(value, torch.Tensor): + stride = tuple(int(s) for s in value.stride()) if value.layout == torch.strided else None + return TensorMetadata( + shape=tuple(value.shape), + dtype=value.dtype, + stride=stride, + device=value.device, + ) + + shape = tuple(getattr(value, "shape", ())) + dtype = getattr(value, "dtype", None) + stride_value = getattr(value, "stride", None) + stride = tuple(stride_value) if isinstance(stride_value, Iterable) else None + device = getattr(value, "device", None) + if not shape and dtype is None and stride is None and device is None: + return None + return TensorMetadata(shape=shape, dtype=dtype, stride=stride, device=device) + + +def get_shape(node: Any) -> tuple[Any, ...] | None: + metadata = get_tensor_metadata(node) + return None if metadata is None else metadata.shape + + +def get_dtype(node: Any) -> Any | None: + metadata = get_tensor_metadata(node) + return None if metadata is None else metadata.dtype + + +def dump_graph(captured: CapturedModel, *, include_meta: bool = True) -> str: + lines: list[str] = [] + for index, node in enumerate(captured.graph.nodes): + lines.append(f"[{index}] {node.op} {node.name} target={node.target}") + lines.append(f" args={node.args}") + lines.append(f" kwargs={node.kwargs}") + if include_meta: + tensor_meta = get_tensor_metadata(node) + if tensor_meta is not None: + lines.append( + " tensor_meta=" + f"shape={tensor_meta.shape} dtype={tensor_meta.dtype} " + f"stride={tensor_meta.stride} device={tensor_meta.device}" + ) + else: + lines.append(f" meta={getattr(node, 'meta', {})}") + return "\n".join(lines) + + +def print_graph(captured: CapturedModel, *, include_meta: bool = True) -> None: + print(dump_graph(captured, include_meta=include_meta)) diff --git a/python/src/transpile/fusion/__init__.py b/python/src/transpile/fusion/__init__.py new file mode 100644 index 000000000..9a8dc325a --- /dev/null +++ b/python/src/transpile/fusion/__init__.py @@ -0,0 +1,43 @@ +from src.transpile.fusion.conv import ConvModuleMatch +from src.transpile.fusion.conv import match_conv_module +from src.transpile.fusion.attention import AttentionBlockMatch +from src.transpile.fusion.attention import AttentionMatch +from src.transpile.fusion.attention import SelfAttentionBlockMatch +from src.transpile.fusion.attention import match_attention +from src.transpile.fusion.attention import match_attention_block +from src.transpile.fusion.attention import match_self_attention_block +from src.transpile.fusion.deltanet import GatedDeltaNetMatch +from src.transpile.fusion.deltanet import match_gated_deltanet +from src.transpile.fusion.linear import LinearMatch +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.mlp import GatedMLPMatch +from src.transpile.fusion.mlp import match_gated_mlp +from src.transpile.fusion.rel_pos_bias import RelPosBiasMatch +from src.transpile.fusion.rel_pos_bias import match_rel_pos_bias +from src.transpile.fusion.rms_norm import RMSNormMatch +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.fusion.rope import RoPEMatch +from src.transpile.fusion.rope import match_rope + +__all__ = [ + "ConvModuleMatch", + "AttentionBlockMatch", + "AttentionMatch", + "SelfAttentionBlockMatch", + "GatedDeltaNetMatch", + "GatedMLPMatch", + "LinearMatch", + "RelPosBiasMatch", + "RMSNormMatch", + "RoPEMatch", + "match_attention", + "match_attention_block", + "match_self_attention_block", + "match_conv_module", + "match_gated_deltanet", + "match_gated_mlp", + "match_linear", + "match_rel_pos_bias", + "match_rms_norm", + "match_rope", +] diff --git a/python/src/transpile/fusion/attention.py b/python/src/transpile/fusion/attention.py new file mode 100644 index 000000000..eac7ee9d3 --- /dev/null +++ b/python/src/transpile/fusion/attention.py @@ -0,0 +1,550 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import collect_node_ids +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.fusion.rope import match_rope + + +@dataclass(frozen=True) +class AttentionMatch: + query_value_id: str + key_value_id: str + value_value_id: str + source_input_value_ids: tuple[str, str, str] + weight_value_ids: tuple[str | None, str | None, str | None] + has_rope: bool + has_qk_norm: bool + has_gqa_repeat: bool + is_causal: bool + scale: float + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class AttentionBlockMatch: + attention_node_id: str + query_value_id: str + key_value_id: str + value_value_id: str + mask_value_id: str | None + gate_value_id: str | None + output_projection_weight_value_id: str + output_projection_bias_value_id: str | None + attention_output_shape: tuple[int, ...] + is_causal: bool + additive_mask: bool + scale: float + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class SelfAttentionBlockMatch: + attention_node_id: str + hidden_value_id: str + query_weight_value_id: str + query_projection_bias_value_id: str | None + query_add_value_id: str | None + rel_query_add_value_id: str | None + key_weight_value_id: str + key_projection_bias_value_id: str | None + value_weight_value_id: str + value_projection_bias_value_id: str | None + mask_value_id: str | None + relative_key_input_value_id: str | None + relative_key_weight_value_id: str | None + relative_key_projection_bias_value_id: str | None + gate_value_id: str | None + output_projection_weight_value_id: str + output_projection_bias_value_id: str | None + query_shape: tuple[int, ...] + key_shape: tuple[int, ...] + value_shape: tuple[int, ...] + relative_key_shape: tuple[int, ...] | None + attention_output_shape: tuple[int, ...] + is_causal: bool + additive_mask: bool + scale: float + rel_pos_scale: float | None + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _ProjectedAttentionInputMatch: + hidden_value_id: str + weight_value_id: str + projection_bias_value_id: str | None + projected_shape: tuple[int, ...] + add_value_id: str | None + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _RelativeKeyProjectionMatch: + input_value_id: str + weight_value_id: str + projection_bias_value_id: str | None + projected_shape: tuple[int, ...] + node_ids: tuple[str, ...] + + +def match_attention(graph: IRGraph, node: IRNode) -> AttentionMatch | None: + if node.op not in {"attention", "scaled_dot_product_attention"} or len(node.inputs) < 3: + return None + + q_info = _extract_attention_input(graph, node.inputs[0], role="q") + k_info = _extract_attention_input(graph, node.inputs[1], role="k") + v_info = _extract_attention_input(graph, node.inputs[2], role="v") + if q_info is None or k_info is None or v_info is None: + return None + + has_rope = bool(q_info["has_rope"] and k_info["has_rope"]) + has_qk_norm = bool(q_info["has_rms_norm"] and k_info["has_rms_norm"]) + has_gqa = bool(k_info["has_gqa_repeat"] or v_info["has_gqa_repeat"] or node.attrs.get("enable_gqa", False)) + node_ids = {node.id, *q_info["node_ids"], *k_info["node_ids"], *v_info["node_ids"]} + + return AttentionMatch( + query_value_id=node.inputs[0], + key_value_id=node.inputs[1], + value_value_id=node.inputs[2], + source_input_value_ids=(q_info["source_input"], k_info["source_input"], v_info["source_input"]), + weight_value_ids=(q_info.get("weight_value_id"), k_info.get("weight_value_id"), v_info.get("weight_value_id")), + has_rope=has_rope, + has_qk_norm=has_qk_norm, + has_gqa_repeat=has_gqa, + is_causal=bool(node.attrs.get("is_causal", False)), + scale=float(node.attrs.get("scale", 1.0)), + window_size=int(node.attrs.get("window_size", 0)), + node_ids=tuple(sorted(node_ids)), + ) + + +def match_self_attention_block(graph: IRGraph, node: IRNode) -> SelfAttentionBlockMatch | None: + projection = match_linear(graph, node) + if projection is None: + return None + + output_path = _extract_attention_output_path(graph, projection.input_value_id) + if output_path is None: + return None + attention_node = graph.nodes.get(output_path["attention_node_id"]) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: + return None + + query_match = _extract_projected_attention_input(graph, attention_node.inputs[0], role="q") + key_match = _extract_projected_attention_input(graph, attention_node.inputs[1], role="k") + value_match = _extract_projected_attention_input(graph, attention_node.inputs[2], role="v") + if query_match is None or key_match is None or value_match is None: + return None + + if not ( + query_match.hidden_value_id == key_match.hidden_value_id == value_match.hidden_value_id + ): + return None + + mask_value_id = attention_node.inputs[3] if len(attention_node.inputs) > 3 else None + rel_query_add_value_id: str | None = None + relative_key_input_value_id: str | None = None + relative_key_weight_value_id: str | None = None + relative_key_projection_bias_value_id: str | None = None + relative_key_shape: tuple[int, ...] | None = None + rel_pos_scale: float | None = None + rel_pos_node_ids: tuple[str, ...] = () + + if mask_value_id is not None: + rel_pos_node = producer(graph, strip_passthrough(graph, mask_value_id)) + if rel_pos_node is not None and rel_pos_node.op == "rel_pos_bias" and len(rel_pos_node.inputs) == 2: + rel_query_match = _extract_projected_attention_input(graph, rel_pos_node.inputs[0], role="q") + relative_key_match = _extract_relative_key_projection(graph, rel_pos_node.inputs[1]) + if rel_query_match is None or relative_key_match is None: + return None + if not ( + rel_query_match.hidden_value_id == query_match.hidden_value_id + and rel_query_match.weight_value_id == query_match.weight_value_id + and rel_query_match.projection_bias_value_id == query_match.projection_bias_value_id + and rel_query_match.projected_shape == query_match.projected_shape + ): + return None + rel_query_add_value_id = rel_query_match.add_value_id + relative_key_input_value_id = relative_key_match.input_value_id + relative_key_weight_value_id = relative_key_match.weight_value_id + relative_key_projection_bias_value_id = relative_key_match.projection_bias_value_id + relative_key_shape = relative_key_match.projected_shape + rel_pos_scale = float(rel_pos_node.attrs.get("scale", 1.0)) + rel_pos_node_ids = collect_node_ids( + rel_pos_node.id, + rel_pos_node.meta.get("rel_pos_bias_nodes", ()), + rel_query_match.node_ids, + relative_key_match.node_ids, + ) + mask_value_id = None + + return SelfAttentionBlockMatch( + attention_node_id=attention_node.id, + hidden_value_id=query_match.hidden_value_id, + query_weight_value_id=query_match.weight_value_id, + query_projection_bias_value_id=query_match.projection_bias_value_id, + query_add_value_id=query_match.add_value_id, + rel_query_add_value_id=rel_query_add_value_id, + key_weight_value_id=key_match.weight_value_id, + key_projection_bias_value_id=key_match.projection_bias_value_id, + value_weight_value_id=value_match.weight_value_id, + value_projection_bias_value_id=value_match.projection_bias_value_id, + mask_value_id=mask_value_id, + relative_key_input_value_id=relative_key_input_value_id, + relative_key_weight_value_id=relative_key_weight_value_id, + relative_key_projection_bias_value_id=relative_key_projection_bias_value_id, + gate_value_id=output_path["gate_value_id"], + output_projection_weight_value_id=projection.weight_value_id, + output_projection_bias_value_id=projection.bias_value_id, + query_shape=query_match.projected_shape, + key_shape=key_match.projected_shape, + value_shape=value_match.projected_shape, + relative_key_shape=relative_key_shape, + attention_output_shape=output_path["attention_output_shape"], + is_causal=bool(attention_node.attrs.get("is_causal", True)), + additive_mask=bool(attention_node.attrs.get("additive_mask", False)), + scale=float(attention_node.attrs.get("scale", 1.0)), + rel_pos_scale=rel_pos_scale, + window_size=int(attention_node.attrs.get("window_size", 0)), + node_ids=collect_node_ids( + attention_node.id, + projection.node_ids, + output_path["node_ids"], + query_match.node_ids, + key_match.node_ids, + value_match.node_ids, + rel_pos_node_ids, + ), + ) + + +def match_attention_block(graph: IRGraph, node: IRNode) -> AttentionBlockMatch | None: + projection = match_linear(graph, node) + if projection is None: + return None + + output_path = _extract_attention_output_path(graph, projection.input_value_id) + if output_path is None: + return None + attention_node = graph.nodes.get(output_path["attention_node_id"]) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: + return None + + return AttentionBlockMatch( + attention_node_id=attention_node.id, + query_value_id=attention_node.inputs[0], + key_value_id=attention_node.inputs[1], + value_value_id=attention_node.inputs[2], + mask_value_id=attention_node.inputs[3] if len(attention_node.inputs) > 3 else None, + gate_value_id=output_path["gate_value_id"], + output_projection_weight_value_id=projection.weight_value_id, + output_projection_bias_value_id=projection.bias_value_id, + attention_output_shape=output_path["attention_output_shape"], + is_causal=bool(attention_node.attrs.get("is_causal", True)), + additive_mask=bool(attention_node.attrs.get("additive_mask", False)), + scale=float(attention_node.attrs.get("scale", 1.0)), + window_size=int(attention_node.attrs.get("window_size", 0)), + node_ids=tuple(sorted({attention_node.id, *projection.node_ids, *output_path["node_ids"]})), + ) + + +def _extract_attention_output_path(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + gate_value_id: str | None = None + + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + break + + node = producer(graph, current) + if node is not None and node.op == "multiply" and len(node.inputs) == 2: + for attn_candidate, gate_candidate in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + attn_match = _extract_attention_output_path(graph, attn_candidate) + if attn_match is None: + continue + attn_match["gate_value_id"] = gate_candidate + attn_match["node_ids"] = tuple(sorted({node.id, *attn_match["node_ids"]})) + return attn_match + + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + attention_output_shape = tuple(int(v) for v in reshape_node.attrs.get("shape", ())) + current = reshape_node.inputs[0] + transpose_node = producer(graph, current) + if transpose_node is None: + return None + + if transpose_node.op == "transpose": + if int(transpose_node.attrs.get("dim0", -1)) != 1 or int(transpose_node.attrs.get("dim1", -1)) != 2: + return None + current = transpose_node.inputs[0] + elif transpose_node.op == "permute": + permutation = tuple(int(v) for v in transpose_node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + current = transpose_node.inputs[0] + else: + return None + + attention_node = producer(graph, current) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"}: + return None + + return { + "attention_node_id": attention_node.id, + "gate_value_id": gate_value_id, + "attention_output_shape": attention_output_shape, + "node_ids": tuple(sorted({reshape_node.id, transpose_node.id})), + } + + +def _extract_projected_attention_input(graph: IRGraph, value_id: str, *, role: str) -> _ProjectedAttentionInputMatch | None: + current = strip_passthrough(graph, value_id) + add_node = producer(graph, current) + if add_node is not None and add_node.op == "add" and len(add_node.inputs) == 2: + for projected_candidate, add_candidate in ((add_node.inputs[0], add_node.inputs[1]), (add_node.inputs[1], add_node.inputs[0])): + projected = _extract_transposed_projection(graph, projected_candidate) + if projected is None: + continue + if not _looks_like_attention_addend(graph, add_candidate, projected.projected_shape): + continue + return _ProjectedAttentionInputMatch( + hidden_value_id=projected.hidden_value_id, + weight_value_id=projected.weight_value_id, + projection_bias_value_id=projected.projection_bias_value_id, + projected_shape=projected.projected_shape, + add_value_id=strip_passthrough(graph, add_candidate), + node_ids=collect_node_ids(add_node.id, projected.node_ids), + ) + + if role == "q": + return None + + return _extract_transposed_projection(graph, current) + + +def _extract_transposed_projection(graph: IRGraph, value_id: str) -> _ProjectedAttentionInputMatch | None: + current = strip_passthrough(graph, value_id) + layout_node = producer(graph, current) + if layout_node is None or len(layout_node.inputs) != 1: + return None + + if layout_node.op == "permute": + permutation = tuple(int(v) for v in layout_node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + elif layout_node.op == "transpose": + if int(layout_node.attrs.get("dim0", -1)) != 1 or int(layout_node.attrs.get("dim1", -1)) != 2: + return None + else: + return None + + projected_value_id = strip_passthrough(graph, layout_node.inputs[0]) + projected_value = graph.values.get(projected_value_id) + if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: + return None + + reshape_node = producer(graph, projected_value_id) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + linear_node = producer(graph, reshape_node.inputs[0]) + if linear_node is None: + return None + linear = match_linear(graph, linear_node) + if linear is None: + return None + + return _ProjectedAttentionInputMatch( + hidden_value_id=linear.input_value_id, + weight_value_id=linear.weight_value_id, + projection_bias_value_id=linear.bias_value_id, + projected_shape=tuple(int(v) for v in projected_value.shape), + add_value_id=None, + node_ids=collect_node_ids(layout_node.id, reshape_node.id, linear.node_ids), + ) + + +def _extract_relative_key_projection(graph: IRGraph, value_id: str) -> _RelativeKeyProjectionMatch | None: + current = strip_passthrough(graph, value_id) + projected_value = graph.values.get(current) + if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: + return None + + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + linear_node = producer(graph, reshape_node.inputs[0]) + if linear_node is None: + return None + linear = match_linear(graph, linear_node) + if linear is None: + return None + + return _RelativeKeyProjectionMatch( + input_value_id=linear.input_value_id, + weight_value_id=linear.weight_value_id, + projection_bias_value_id=linear.bias_value_id, + projected_shape=tuple(int(v) for v in projected_value.shape), + node_ids=collect_node_ids(reshape_node.id, linear.node_ids), + ) + + +def _looks_like_attention_addend(graph: IRGraph, value_id: str, target_shape: tuple[int, ...]) -> bool: + value = graph.values.get(strip_passthrough(graph, value_id)) + if value is None or value.shape is None: + return False + + shape = tuple(int(v) for v in value.shape) + if len(shape) != 4 or len(target_shape) != 4: + return False + + batch, seq_len, heads, head_dim = target_shape + if shape[0] not in (1, batch) or shape[3] != head_dim: + return False + + # Allow either BHSD-style tensors like [B, H, 1, D] or BSHD-style tensors like [B, 1, H, D]. + if shape[1] == heads and shape[2] in (1, seq_len): + return True + if shape[1] in (1, seq_len) and shape[2] == heads: + return True + return False + + +def _extract_attention_input(graph: IRGraph, value_id: str, *, role: str) -> dict[str, object] | None: + node_ids: set[str] = set() + current = strip_passthrough(graph, value_id) + has_gqa_repeat = False + + while True: + node = producer(graph, current) + if node is None: + return { + "source_input": current, + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + node_ids.add(node.id) + + if node.op in {"reshape", "view", "transpose", "permute"}: + current = node.inputs[0] + continue + + rope = match_rope(graph, current) + if rope is not None: + rope_info = _extract_attention_input(graph, rope.input_value_id, role=role) + if rope_info is None: + return None + rope_info["has_rope"] = True + rope_info["node_ids"] = tuple(sorted(set(rope_info["node_ids"]) | node_ids | set(rope.node_ids))) + rope_info["has_gqa_repeat"] = bool(rope_info["has_gqa_repeat"] or has_gqa_repeat) + return rope_info + + rms = match_rms_norm(graph, node) + if rms is not None: + if role == "v": + return None + rms_info = _extract_attention_input(graph, rms.input_value_id, role=role) + if rms_info is None: + return None + rms_info["has_rms_norm"] = True + rms_info["node_ids"] = tuple(sorted(set(rms_info["node_ids"]) | node_ids | set(rms.node_ids))) + rms_info["has_gqa_repeat"] = bool(rms_info["has_gqa_repeat"] or has_gqa_repeat) + return rms_info + + if _looks_like_gqa_repeat(graph, current): + has_gqa_repeat = True + current = _unwrap_gqa_repeat(graph, current) + if current is None: + return None + continue + + linear_node = producer(graph, current) + if linear_node is None: + return { + "source_input": strip_passthrough(graph, current), + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + linear = match_linear(graph, linear_node) + if linear is None: + return { + "source_input": strip_passthrough(graph, current), + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + return { + "source_input": strip_passthrough(graph, linear.input_value_id), + "weight_value_id": linear.weight_value_id, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(set(node_ids) | set(linear.node_ids))), + } + + +def _looks_like_gqa_repeat(graph: IRGraph, value_id: str) -> bool: + return _unwrap_gqa_repeat(graph, value_id) is not None + + +def _unwrap_gqa_repeat(graph: IRGraph, value_id: str) -> str | None: + current = strip_passthrough(graph, value_id) + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"}: + return None + current = reshape_node.inputs[0] + expand_node = producer(graph, current) + if expand_node is None or expand_node.op != "expand": + return None + current = expand_node.inputs[0] + while True: + node = producer(graph, current) + if node is None: + return None + if node.op == "slice": + current = node.inputs[0] + continue + if node.op != "unsqueeze": + return None + if int(node.attrs.get("dim", -999)) != 2: + return None + break + base_value_id = strip_passthrough(graph, node.inputs[0]) + base_shape = graph.values.get(base_value_id).shape if graph.values.get(base_value_id) is not None else None + target_shape = tuple(int(v) for v in expand_node.attrs.get("shape", ())) + if base_shape is None or len(base_shape) != 4 or len(target_shape) != 5: + return None + expected = (int(base_shape[0]), int(base_shape[1]), int(target_shape[2]), int(base_shape[2]), int(base_shape[3])) + if expected != target_shape: + return None + return base_value_id diff --git a/python/src/transpile/fusion/common.py b/python/src/transpile/fusion/common.py new file mode 100644 index 000000000..828783b7e --- /dev/null +++ b/python/src/transpile/fusion/common.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class FusionMatch: + kind: str + anchor_node_id: str + node_ids: tuple[str, ...] + value_ids: tuple[str, ...] + attrs: dict[str, object] + + +def producer(graph: IRGraph, value_id: str) -> IRNode | None: + value = graph.values.get(value_id) + if value is None or value.producer is None: + return None + return graph.nodes.get(value.producer) + + +def strip_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def collect_node_ids(*parts: Any) -> tuple[str, ...]: + node_ids: set[str] = set() + for part in parts: + if part is None: + continue + if isinstance(part, str): + node_ids.add(part) + continue + if isinstance(part, IRNode): + node_ids.add(part.id) + continue + if isinstance(part, (list, tuple, set)): + for item in part: + if isinstance(item, str): + node_ids.add(item) + elif isinstance(item, IRNode): + node_ids.add(item.id) + return tuple(sorted(node_ids)) diff --git a/python/src/transpile/fusion/conv.py b/python/src/transpile/fusion/conv.py new file mode 100644 index 000000000..1f75d6e79 --- /dev/null +++ b/python/src/transpile/fusion/conv.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import collect_node_ids +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class ConvModuleMatch: + input_value_id: str + pointwise1_weight_value_id: str + pointwise1_bias_value_id: str | None + depthwise_weight_value_id: str + depthwise_bias_value_id: str | None + batch_norm_weight_value_id: str + batch_norm_bias_value_id: str + batch_norm_running_mean_value_id: str + batch_norm_running_var_value_id: str + pointwise2_weight_value_id: str + pointwise2_bias_value_id: str | None + eps: float + depthwise_kernel_size: int + depthwise_padding: int + node_ids: tuple[str, ...] + + +def match_conv_module(graph: IRGraph, node: IRNode) -> ConvModuleMatch | None: + if node.op != "permute" or len(node.inputs) != 1 or len(node.outputs) != 1: + return None + if tuple(node.attrs.get("permutation", ())) != (0, 2, 1): + return None + + pointwise2 = producer(graph, strip_passthrough(graph, node.inputs[0])) + if not _is_pointwise_conv1d(pointwise2): + return None + + silu = producer(graph, strip_passthrough(graph, pointwise2.inputs[0])) + if silu is None or silu.op != "silu" or len(silu.inputs) != 1: + return None + + batch_norm = producer(graph, strip_passthrough(graph, silu.inputs[0])) + if batch_norm is None or batch_norm.op != "batch_norm" or len(batch_norm.inputs) != 5: + return None + + depthwise = producer(graph, strip_passthrough(graph, batch_norm.inputs[0])) + if not _is_same_depthwise_conv1d(depthwise): + return None + + glu = producer(graph, strip_passthrough(graph, depthwise.inputs[0])) + if glu is None or glu.op != "glu" or len(glu.inputs) != 1: + return None + if int(glu.attrs.get("axis", -1)) not in {1, -2}: + return None + + pointwise1 = producer(graph, strip_passthrough(graph, glu.inputs[0])) + if not _is_pointwise_conv1d(pointwise1): + return None + + input_permute = producer(graph, strip_passthrough(graph, pointwise1.inputs[0])) + if input_permute is None or input_permute.op != "permute" or len(input_permute.inputs) != 1: + return None + if tuple(input_permute.attrs.get("permutation", ())) != (0, 2, 1): + return None + + depthwise_weight_shape = graph.values.get(depthwise.inputs[1]).shape if depthwise.inputs[1] in graph.values else None + if depthwise_weight_shape is None or len(depthwise_weight_shape) != 3: + return None + depthwise_kernel_size = int(depthwise_weight_shape[2]) + + return ConvModuleMatch( + input_value_id=strip_passthrough(graph, input_permute.inputs[0]), + pointwise1_weight_value_id=pointwise1.inputs[1], + pointwise1_bias_value_id=pointwise1.inputs[2] if len(pointwise1.inputs) > 2 else None, + depthwise_weight_value_id=depthwise.inputs[1], + depthwise_bias_value_id=depthwise.inputs[2] if len(depthwise.inputs) > 2 else None, + batch_norm_weight_value_id=batch_norm.inputs[1], + batch_norm_bias_value_id=batch_norm.inputs[2], + batch_norm_running_mean_value_id=batch_norm.inputs[3], + batch_norm_running_var_value_id=batch_norm.inputs[4], + pointwise2_weight_value_id=pointwise2.inputs[1], + pointwise2_bias_value_id=pointwise2.inputs[2] if len(pointwise2.inputs) > 2 else None, + eps=float(batch_norm.attrs.get("eps", 1e-5)), + depthwise_kernel_size=depthwise_kernel_size, + depthwise_padding=int(depthwise.attrs.get("padding", 0)), + node_ids=collect_node_ids( + node, + pointwise2, + silu, + batch_norm, + depthwise, + glu, + pointwise1, + input_permute, + ), + ) + + +def _is_pointwise_conv1d(node: IRNode | None) -> bool: + return ( + node is not None + and node.op == "conv1d" + and len(node.inputs) >= 2 + and int(node.attrs.get("stride", 1)) == 1 + and int(node.attrs.get("padding", 0)) == 0 + and int(node.attrs.get("dilation", 1)) == 1 + and int(node.attrs.get("groups", 1)) == 1 + ) + + +def _is_same_depthwise_conv1d(node: IRNode | None) -> bool: + if node is None or node.op != "conv1d" or len(node.inputs) < 2: + return False + groups = int(node.attrs.get("groups", 1)) + padding = int(node.attrs.get("padding", 0)) + stride = int(node.attrs.get("stride", 1)) + dilation = int(node.attrs.get("dilation", 1)) + if stride != 1 or dilation != 1: + return False + return groups > 1 and padding >= 0 diff --git a/python/src/transpile/fusion/deltanet.py b/python/src/transpile/fusion/deltanet.py new file mode 100644 index 000000000..cef656a5d --- /dev/null +++ b/python/src/transpile/fusion/deltanet.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class GatedDeltaNetMatch: + output_value_id: str + normalized_input_value_id: str + qkv_weight_value_id: str + a_weight_value_id: str + b_weight_value_id: str + norm_weight_value_id: str + z_weight_value_id: str | None + dt_bias_value_id: str | None + a_log_value_id: str | None + conv_weight_value_id: str | None + num_k_heads: int + num_v_heads: int + key_dim: int + value_dim: int + eps: float + chunk_size: int + mode: str + node_ids: tuple[str, ...] + + +def match_gated_deltanet(graph: IRGraph, node: IRNode) -> GatedDeltaNetMatch | None: + if node.op not in {"view", "reshape"} or len(node.outputs) != 1: + return None + output_value = graph.values.get(node.outputs[0]) + if output_value is None or output_value.shape is None or len(output_value.shape) != 3: + return None + + body_value_id = strip_passthrough(graph, node.inputs[0]) + body_node = producer(graph, body_value_id) + if body_node is None: + return None + + z_weight_value_id: str | None = None + rms_match = None + normalized_input_value_id: str | None = None + node_ids: set[str] = {node.id} + + if body_node.op == "multiply" and len(body_node.inputs) == 2: + for rms_value_id, z_value_id in ((body_node.inputs[0], body_node.inputs[1]), (body_node.inputs[1], body_node.inputs[0])): + rms_match = _match_rms_from_value(graph, rms_value_id) + if rms_match is None: + continue + z_weight_value_id, normalized_input_value_id, z_node_ids = _extract_z_branch(graph, z_value_id) + if normalized_input_value_id is None: + continue + node_ids.update(z_node_ids) + node_ids.add(body_node.id) + break + if rms_match is None or normalized_input_value_id is None: + return None + else: + rms_match = _match_rms_from_value(graph, body_value_id) + if rms_match is None: + return None + + y2d_value_id = rms_match.input_value_id + if rms_match.weight_value_id is None: + return None + norm_weight_value_id = rms_match.weight_value_id + eps = float(rms_match.eps) + node_ids.update(rms_match.node_ids) + + if normalized_input_value_id is None: + normalized_input_value_id = _infer_normalized_input_from_y_branch(graph, y2d_value_id) + if normalized_input_value_id is None: + return None + + qkv_info = _find_qkv_linear(graph, normalized_input_value_id) + if qkv_info is None: + return None + qkv_weight_value_id, conv_weight_value_id, num_k_heads, key_dim, num_v_heads, value_dim, qkv_node_ids = qkv_info + node_ids.update(qkv_node_ids) + + a_info = _find_a_linear(graph, normalized_input_value_id) + b_info = _find_b_linear(graph, normalized_input_value_id) + if a_info is None or b_info is None: + return None + a_weight_value_id, dt_bias_value_id, a_log_value_id, a_node_ids = a_info + b_weight_value_id, b_node_ids = b_info + node_ids.update(a_node_ids) + node_ids.update(b_node_ids) + + mode, chunk_size = _classify_mode(graph, y2d_value_id, match_node_ids=node_ids) + + return GatedDeltaNetMatch( + output_value_id=node.outputs[0], + normalized_input_value_id=normalized_input_value_id, + qkv_weight_value_id=qkv_weight_value_id, + a_weight_value_id=a_weight_value_id, + b_weight_value_id=b_weight_value_id, + norm_weight_value_id=norm_weight_value_id, + z_weight_value_id=z_weight_value_id, + dt_bias_value_id=dt_bias_value_id, + a_log_value_id=a_log_value_id, + conv_weight_value_id=conv_weight_value_id, + num_k_heads=num_k_heads, + num_v_heads=num_v_heads, + key_dim=key_dim, + value_dim=value_dim, + eps=eps, + chunk_size=chunk_size, + mode=mode, + node_ids=tuple(sorted(node_ids)), + ) + + +def _match_rms_from_value(graph: IRGraph, value_id: str): + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + return None + return match_rms_norm(graph, node) + + +def _extract_z_branch(graph: IRGraph, value_id: str) -> tuple[str | None, str | None, tuple[str, ...]]: + current = strip_passthrough(graph, value_id) + silu_node = producer(graph, current) + if silu_node is None or silu_node.op != "silu" or len(silu_node.inputs) != 1: + return None, None, () + linear_value_id = strip_passthrough(graph, silu_node.inputs[0]) + linear_node = producer(graph, linear_value_id) + while linear_node is not None and linear_node.op in {"view", "reshape"} and len(linear_node.inputs) == 1: + linear_value_id = strip_passthrough(graph, linear_node.inputs[0]) + linear_node = producer(graph, linear_value_id) + if linear_node is None: + return None, None, () + linear_match = match_linear(graph, linear_node) + if linear_match is None: + return None, None, () + return ( + linear_match.weight_value_id, + linear_match.input_value_id, + tuple(sorted({silu_node.id, *linear_match.node_ids})), + ) + + +def _infer_normalized_input_from_y_branch(graph: IRGraph, value_id: str) -> str | None: + current = strip_passthrough(graph, value_id) + node = producer(graph, current) + visited: set[str] = set() + while node is not None and node.id not in visited: + visited.add(node.id) + for input_value_id in node.inputs: + candidate = producer(graph, strip_passthrough(graph, input_value_id)) + if candidate is None: + continue + linear_match = match_linear(graph, candidate) + if linear_match is not None: + return linear_match.input_value_id + if len(node.inputs) != 1: + return None + node = producer(graph, strip_passthrough(graph, node.inputs[0])) + return None + + +def _find_qkv_linear( + graph: IRGraph, + normalized_input_value_id: str, +) -> tuple[str, str | None, int, int, int, int, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + split_node = _find_descendant_op(graph, linear_match.output_value_id, {"split_with_sizes"}, max_depth=8) + if split_node is None: + continue + split_sizes = tuple(int(v) for v in split_node.attrs.get("sizes", ())) + if len(split_sizes) != 3: + continue + + conv_node = _find_descendant_op(graph, linear_match.output_value_id, {"conv1d"}, max_depth=6) + conv_weight_value_id = conv_node.inputs[1] if conv_node is not None and len(conv_node.inputs) >= 2 else None + + q_shape = None + v_shape = None + node_ids = {node.id, split_node.id} + if conv_node is not None: + node_ids.add(conv_node.id) + + split_value = graph.values.get(split_node.outputs[0]) + if split_value is None: + continue + for getitem_node_id in split_value.users: + getitem_node = graph.nodes.get(getitem_node_id) + if getitem_node is None or getitem_node.op != "getitem" or not getitem_node.outputs: + continue + shape = _find_rank4_shape(graph, getitem_node.outputs[0], max_depth=5) + if shape is None: + continue + branch_has_sum = _find_descendant_op(graph, getitem_node.outputs[0], {"sum"}, max_depth=8) is not None + node_ids.add(getitem_node.id) + if branch_has_sum and q_shape is None: + q_shape = shape + elif not branch_has_sum and v_shape is None: + v_shape = shape + + if q_shape is None or v_shape is None: + continue + + num_k_heads = int(q_shape[2]) + key_dim = int(q_shape[3]) + num_v_heads = int(v_shape[2]) + value_dim = int(v_shape[3]) + if num_k_heads <= 0 or key_dim <= 0 or num_v_heads <= 0 or value_dim <= 0: + continue + + return ( + linear_match.weight_value_id, + conv_weight_value_id, + num_k_heads, + key_dim, + num_v_heads, + value_dim, + tuple(sorted(node_ids | set(linear_match.node_ids))), + ) + return None + + +def _find_a_linear( + graph: IRGraph, + normalized_input_value_id: str, +) -> tuple[str, str | None, str | None, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + softplus_node = _find_descendant_op(graph, linear_match.output_value_id, {"softplus"}, max_depth=6) + if softplus_node is None: + continue + dt_bias_value_id = None + softplus_input_node = producer(graph, strip_passthrough(graph, softplus_node.inputs[0])) + if softplus_input_node is not None and softplus_input_node.op == "add": + for input_value_id in softplus_input_node.inputs: + value = graph.values.get(input_value_id) + if value is not None and value.producer is None and input_value_id in graph.constants: + dt_bias_value_id = input_value_id + break + a_log_value_id = None + multiply_node = _find_descendant_op(graph, softplus_node.outputs[0], {"multiply"}, max_depth=6) + if multiply_node is not None: + for input_value_id in multiply_node.inputs: + a_log_value_id = _extract_exp_constant_input(graph, input_value_id) + if a_log_value_id is not None: + break + node_ids = {softplus_node.id, *linear_match.node_ids} + if softplus_input_node is not None: + node_ids.add(softplus_input_node.id) + if multiply_node is not None: + node_ids.add(multiply_node.id) + return linear_match.weight_value_id, dt_bias_value_id, a_log_value_id, tuple(sorted(node_ids)) + return None + + +def _find_b_linear(graph: IRGraph, normalized_input_value_id: str) -> tuple[str, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + sigmoid_node = _find_descendant_op(graph, linear_match.output_value_id, {"sigmoid"}, max_depth=4) + if sigmoid_node is None: + continue + return linear_match.weight_value_id, tuple(sorted({sigmoid_node.id, *linear_match.node_ids})) + return None + + +def _extract_exp_constant_input(graph: IRGraph, value_id: str) -> str | None: + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + value = graph.values.get(value_id) + if value is not None and value.producer is None and value_id in graph.constants: + return value_id + return None + if node.op == "scalar_multiply": + return _extract_exp_constant_input(graph, node.inputs[0]) + if node.op == "scalar_exp": + source = strip_passthrough(graph, node.inputs[0]) + value = graph.values.get(source) + if value is not None and value.producer is None and source in graph.constants: + return source + return None + + +def _find_rank4_shape(graph: IRGraph, value_id: str, *, max_depth: int) -> tuple[int, int, int, int] | None: + frontier = [(value_id, 0)] + seen: set[str] = set() + while frontier: + current, depth = frontier.pop(0) + if current in seen or depth > max_depth: + continue + seen.add(current) + value = graph.values.get(current) + if value is not None and value.shape is not None and len(value.shape) == 4: + shape = tuple(int(dim) for dim in value.shape) + return shape # type: ignore[return-value] + for user_id in graph.values.get(current, None).users if graph.values.get(current, None) is not None else (): + user = graph.nodes.get(user_id) + if user is None or not user.outputs: + continue + frontier.extend((output_id, depth + 1) for output_id in user.outputs) + return None + + +def _find_descendant_op(graph: IRGraph, value_id: str, targets: set[str], *, max_depth: int) -> IRNode | None: + frontier = [(value_id, 0)] + seen: set[str] = set() + while frontier: + current, depth = frontier.pop(0) + if current in seen or depth > max_depth: + continue + seen.add(current) + value = graph.values.get(current) + if value is None: + continue + for user_id in value.users: + user = graph.nodes.get(user_id) + if user is None or not user.outputs: + continue + if user.op in targets: + return user + frontier.extend((output_id, depth + 1) for output_id in user.outputs) + return None + + +def _classify_mode(graph: IRGraph, y2d_value_id: str, *, match_node_ids: set[str]) -> tuple[str, int]: + del y2d_value_id + chunk_size = _infer_prefill_chunk_size(graph, match_node_ids) + # Current exported graphs are uncached/full-sequence captures, so fuse the + # linear-attention block to the prefill kernel by default. Decode should be + # emitted only once the capture path models cache/state explicitly. + return "prefill", chunk_size + + +def _infer_prefill_chunk_size(graph: IRGraph, match_node_ids: set[str]) -> int: + for node_id in match_node_ids: + node = graph.nodes.get(node_id) + if node is None or node.op != "aten.triu.default" or not node.inputs: + continue + ones_value = graph.values.get(node.inputs[0]) + if ones_value is not None and ones_value.shape and len(ones_value.shape) >= 1: + return int(ones_value.shape[-1]) + return 64 diff --git a/python/src/transpile/fusion/linear.py b/python/src/transpile/fusion/linear.py new file mode 100644 index 000000000..df5346730 --- /dev/null +++ b/python/src/transpile/fusion/linear.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class LinearMatch: + input_value_id: str + weight_value_id: str + bias_value_id: str | None + output_value_id: str + node_ids: tuple[str, ...] + kind: str + + +def match_linear(graph: IRGraph, node: IRNode) -> LinearMatch | None: + if node.op == "linear" and len(node.inputs) >= 2 and len(node.outputs) == 1: + return LinearMatch( + input_value_id=strip_passthrough(graph, node.inputs[0]), + weight_value_id=node.inputs[1], + bias_value_id=node.inputs[2] if bool(node.attrs.get("has_bias", False)) and len(node.inputs) > 2 else None, + output_value_id=node.outputs[0], + node_ids=(node.id,), + kind="linear", + ) + + if node.op == "addmm" and len(node.inputs) == 3 and len(node.outputs) == 1: + return LinearMatch( + input_value_id=strip_passthrough(graph, node.inputs[1]), + weight_value_id=node.inputs[2], + bias_value_id=node.inputs[0], + output_value_id=node.outputs[0], + node_ids=(node.id,), + kind="addmm", + ) + + if node.op != "add" or len(node.inputs) != 2 or len(node.outputs) != 1: + return None + + for matmul_value_id, bias_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + matmul_node = producer(graph, strip_passthrough(graph, matmul_value_id)) + if matmul_node is None or matmul_node.op != "matmul" or len(matmul_node.inputs) != 2: + continue + return LinearMatch( + input_value_id=strip_passthrough(graph, matmul_node.inputs[0]), + weight_value_id=matmul_node.inputs[1], + bias_value_id=bias_value_id, + output_value_id=node.outputs[0], + node_ids=tuple(sorted({node.id, matmul_node.id})), + kind="matmul_add", + ) + + return None diff --git a/python/src/transpile/fusion/mlp.py b/python/src/transpile/fusion/mlp.py new file mode 100644 index 000000000..1c3e0ab34 --- /dev/null +++ b/python/src/transpile/fusion/mlp.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear + + +@dataclass(frozen=True) +class GatedMLPMatch: + input_value_id: str + gate_weight_value_id: str + up_weight_value_id: str + down_weight_value_id: str + activation: str + bias_value_ids: tuple[str | None, str | None, str | None] + node_ids: tuple[str, ...] + + +def match_gated_mlp(graph: IRGraph, node: IRNode) -> GatedMLPMatch | None: + down = match_linear(graph, node) + if down is None: + return None + + mul_node = graph.nodes.get(graph.values[down.input_value_id].producer) if graph.values.get(down.input_value_id) is not None else None + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for activated_value_id, up_value_id in ((mul_node.inputs[0], mul_node.inputs[1]), (mul_node.inputs[1], mul_node.inputs[0])): + activated_node = graph.nodes.get(graph.values[strip_passthrough(graph, activated_value_id)].producer) if graph.values.get(strip_passthrough(graph, activated_value_id)) is not None else None + if activated_node is None or activated_node.op not in {"gelu", "silu"}: + continue + + gate = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer]) if graph.values.get(strip_passthrough(graph, activated_node.inputs[0])) is not None and graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer in graph.nodes else None + up = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, up_value_id)].producer]) if graph.values.get(strip_passthrough(graph, up_value_id)) is not None and graph.values[strip_passthrough(graph, up_value_id)].producer in graph.nodes else None + if gate is None or up is None: + continue + if strip_passthrough(graph, gate.input_value_id) != strip_passthrough(graph, up.input_value_id): + continue + + return GatedMLPMatch( + input_value_id=strip_passthrough(graph, gate.input_value_id), + gate_weight_value_id=gate.weight_value_id, + up_weight_value_id=up.weight_value_id, + down_weight_value_id=down.weight_value_id, + activation=activated_node.op, + bias_value_ids=(gate.bias_value_id, up.bias_value_id, down.bias_value_id), + node_ids=tuple(sorted({*gate.node_ids, activated_node.id, *up.node_ids, mul_node.id, *down.node_ids})), + ) + + return None diff --git a/python/src/transpile/fusion/rel_pos_bias.py b/python/src/transpile/fusion/rel_pos_bias.py new file mode 100644 index 000000000..d9b5a3def --- /dev/null +++ b/python/src/transpile/fusion/rel_pos_bias.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class RelPosBiasMatch: + query_value_id: str + relative_key_value_id: str + scale: float + node_ids: tuple[str, ...] + + +def match_rel_pos_bias(graph: IRGraph, node: IRNode) -> RelPosBiasMatch | None: + if node.op != "scalar_multiply" or len(node.inputs) != 1: + return None + + final_slice = producer(graph, strip_passthrough(graph, node.inputs[0])) + if final_slice is None or final_slice.op != "slice": + return None + if int(final_slice.attrs.get("axis", -1)) != 3 or int(final_slice.attrs.get("start", -1)) != 0: + return None + + reshape_after_shift = producer(graph, strip_passthrough(graph, final_slice.inputs[0])) + if reshape_after_shift is None or reshape_after_shift.op not in {"reshape", "view"} or len(reshape_after_shift.inputs) != 1: + return None + + shift_slice = producer(graph, strip_passthrough(graph, reshape_after_shift.inputs[0])) + if shift_slice is None or shift_slice.op != "slice": + return None + if int(shift_slice.attrs.get("axis", -1)) != 2 or int(shift_slice.attrs.get("start", -1)) != 1: + return None + + reshape_for_shift = producer(graph, strip_passthrough(graph, shift_slice.inputs[0])) + if reshape_for_shift is None or reshape_for_shift.op not in {"reshape", "view"} or len(reshape_for_shift.inputs) != 1: + return None + + pad = producer(graph, strip_passthrough(graph, reshape_for_shift.inputs[0])) + if pad is None or pad.op != "pad" or len(pad.inputs) != 1: + return None + if tuple(int(v) for v in pad.attrs.get("pads", ())) != (1, 0): + return None + if str(pad.attrs.get("mode", "constant")) != "constant": + return None + if float(pad.attrs.get("value", 0.0)) != 0.0: + return None + + matmul = producer(graph, strip_passthrough(graph, pad.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + + rhs_permute = producer(graph, strip_passthrough(graph, matmul.inputs[1])) + if rhs_permute is None or rhs_permute.op != "permute" or len(rhs_permute.inputs) != 1: + return None + if tuple(int(v) for v in rhs_permute.attrs.get("permutation", ())) != (0, 2, 3, 1): + return None + + query_value_id = strip_passthrough(graph, matmul.inputs[0]) + relative_key_value_id = strip_passthrough(graph, rhs_permute.inputs[0]) + + query_shape = graph.values.get(query_value_id).shape if query_value_id in graph.values else None + relative_key_shape = graph.values.get(relative_key_value_id).shape if relative_key_value_id in graph.values else None + output_shape = graph.values.get(node.outputs[0]).shape if node.outputs and node.outputs[0] in graph.values else None + if query_shape is None or relative_key_shape is None or output_shape is None: + return None + if len(query_shape) != 4 or len(relative_key_shape) != 4 or len(output_shape) != 4: + return None + + batch, heads, seq_len, head_dim = (int(v) for v in query_shape) + rel_batch, rel_len, rel_heads, rel_dim = (int(v) for v in relative_key_shape) + out_batch, out_heads, out_t, out_s = (int(v) for v in output_shape) + if rel_batch not in {1, batch}: + return None + if rel_heads != heads or rel_dim != head_dim: + return None + if rel_len < 2 * seq_len - 1: + return None + if (out_batch, out_heads, out_t, out_s) != (batch, heads, seq_len, seq_len): + return None + + return RelPosBiasMatch( + query_value_id=query_value_id, + relative_key_value_id=relative_key_value_id, + scale=float(node.attrs.get("value", 1.0)), + node_ids=tuple( + sorted( + { + node.id, + final_slice.id, + reshape_after_shift.id, + shift_slice.id, + reshape_for_shift.id, + pad.id, + matmul.id, + rhs_permute.id, + } + ) + ), + ) diff --git a/python/src/transpile/fusion/rms_norm.py b/python/src/transpile/fusion/rms_norm.py new file mode 100644 index 000000000..793f1aff2 --- /dev/null +++ b/python/src/transpile/fusion/rms_norm.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class RMSNormMatch: + input_value_id: str + weight_value_id: str | None + weight_offset: float + eps: float + node_ids: tuple[str, ...] + + +def match_rms_norm(graph: IRGraph, node: IRNode) -> RMSNormMatch | None: + anchor_value_id = node.inputs[0] if node.op == "type_as" and node.inputs else node.outputs[0] + anchor_value_id = strip_passthrough(graph, anchor_value_id) + anchor_node = producer(graph, anchor_value_id) + if anchor_node is None or anchor_node.op != "multiply" or len(anchor_node.inputs) != 2: + return None + + for normed_value_id, scale_value_id in ( + (anchor_node.inputs[0], anchor_node.inputs[1]), + (anchor_node.inputs[1], anchor_node.inputs[0]), + ): + scale = _extract_scale(graph, scale_value_id) + norm = _extract_normed_value(graph, normed_value_id) + if scale is None or norm is None: + continue + if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): + continue + return RMSNormMatch( + input_value_id=norm["input_value_id"], + weight_value_id=scale["weight_value_id"], + weight_offset=float(scale["offset"]), + eps=float(norm["eps"]), + node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"], *scale["node_ids"]})), + ) + + norm = _extract_normed_value(graph, anchor_node.outputs[0]) + if norm is None: + return None + if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): + return None + return RMSNormMatch( + input_value_id=norm["input_value_id"], + weight_value_id=None, + weight_offset=0.0, + eps=float(norm["eps"]), + node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"]})), + ) + + +def _extract_scale(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + offset = 0.0 + node_ids: set[str] = set() + producer_node = producer(graph, current) + if producer_node is not None and producer_node.op == "scalar_add" and float(producer_node.attrs.get("value", 0.0)) == 1.0: + offset = 1.0 + node_ids.add(producer_node.id) + current = producer_node.inputs[0] + current = strip_passthrough(graph, current) + if current not in graph.constants or not isinstance(graph.constants[current], torch.Tensor): + return None + return {"weight_value_id": current, "offset": offset, "node_ids": tuple(sorted(node_ids))} + + +def _extract_normed_value(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul = producer(graph, strip_passthrough(graph, value_id)) + if mul is None or mul.op != "multiply" or len(mul.inputs) != 2: + return None + for x_candidate, rsqrt_candidate in ((mul.inputs[0], mul.inputs[1]), (mul.inputs[1], mul.inputs[0])): + rsqrt = _extract_rsqrt_chain(graph, rsqrt_candidate) + if rsqrt is None: + continue + if strip_passthrough(graph, x_candidate) != strip_passthrough(graph, rsqrt["pow_input_value_id"]): + continue + return { + "input_value_id": strip_passthrough(graph, x_candidate), + "pow_input_value_id": rsqrt["pow_input_value_id"], + "eps": rsqrt["eps"], + "node_ids": tuple(sorted({mul.id, *rsqrt["node_ids"]})), + } + return None + + +def _extract_rsqrt_chain(graph: IRGraph, value_id: str) -> dict[str, object] | None: + pow_node = producer(graph, strip_passthrough(graph, value_id)) + if pow_node is None or pow_node.op != "pow" or float(pow_node.attrs.get("exponent", 0.0)) != -0.5: + return None + add_node = producer(graph, pow_node.inputs[0]) + if add_node is None or add_node.op != "scalar_add": + return None + mean_node = producer(graph, add_node.inputs[0]) + if mean_node is None or mean_node.op != "mean" or not bool(mean_node.attrs.get("keepdim", False)): + return None + pow2_node = producer(graph, mean_node.inputs[0]) + if pow2_node is None or pow2_node.op != "pow" or float(pow2_node.attrs.get("exponent", 0.0)) != 2.0: + return None + return { + "pow_input_value_id": pow2_node.inputs[0], + "eps": float(add_node.attrs.get("value", 0.0)), + "node_ids": (pow_node.id, add_node.id, mean_node.id, pow2_node.id), + } diff --git a/python/src/transpile/fusion/rope.py b/python/src/transpile/fusion/rope.py new file mode 100644 index 000000000..592ee2933 --- /dev/null +++ b/python/src/transpile/fusion/rope.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_layout_passthrough +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class RoPEMatch: + input_value_id: str + theta: float + position_offset: int + partial: bool + node_ids: tuple[str, ...] + + +def match_rope(graph: IRGraph, value_id: str) -> RoPEMatch | None: + desc = _extract_rope_descriptor(graph, value_id) + if desc is None: + return None + return RoPEMatch( + input_value_id=strip_passthrough(graph, desc["input_value_id"]), + theta=float(desc["theta"]), + position_offset=int(desc["position_offset"]), + partial=bool(desc.get("partial_rope", False)), + node_ids=tuple(sorted(set(desc.get("node_ids", ())))), + ) + + +def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = strip_layout_passthrough(graph, value_id) + node = producer(graph, current) + if node is None: + return None + if node.op == "rope": + return { + "input_value_id": node.inputs[0], + "theta": float(node.attrs.get("theta", 0.0)), + "position_offset": int(node.attrs.get("position_offset", 0)), + "partial_rope": False, + "node_ids": (node.id,), + } + classic = _extract_classic_rope_descriptor(graph, current) + if classic is not None: + classic["partial_rope"] = False + return classic + if node.op == "cat" and len(node.inputs) == 2: + for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) + if rope_branch is None: + continue + passthrough_node = producer(graph, strip_layout_passthrough(graph, passthrough_value_id)) + if passthrough_node is None or passthrough_node.op != "slice": + continue + if strip_passthrough(graph, passthrough_node.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): + continue + rope_branch["partial_rope"] = True + rope_branch["node_ids"] = tuple(sorted({node.id, passthrough_node.id, *rope_branch["node_ids"]})) + return rope_branch + return None + + +def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node = producer(graph, strip_layout_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return None + for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + direct_mult = producer(graph, direct_mult_id) + rotated_mult = producer(graph, rotated_mult_id) + if direct_mult is None or direct_mult.op != "multiply" or rotated_mult is None or rotated_mult.op != "multiply": + continue + direct_match = _extract_direct_rope_branch(graph, direct_mult) + rotated_match = _extract_rotated_rope_branch(graph, rotated_mult) + if direct_match is None or rotated_match is None: + continue + input_value_id = strip_passthrough(graph, direct_match["input_value_id"]) + if input_value_id != strip_passthrough(graph, rotated_match["input_value_id"]): + continue + cos_info = _extract_rope_trig(graph, direct_match["trig_value_id"], expected="scalar_cos") + sin_info = _extract_rope_trig(graph, rotated_match["trig_value_id"], expected="scalar_sin") + if cos_info is None or sin_info is None: + continue + if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: + continue + if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): + continue + return { + "input_value_id": input_value_id, + "theta": cos_info["theta"], + "position_offset": cos_info["position_offset"], + "node_ids": tuple(sorted({node.id, *direct_match["node_ids"], *rotated_match["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), + } + return None + + +def _extract_direct_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + if _unwrap_trig_value(graph, trig_value_id, expected="scalar_cos") is not None: + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotated_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + input_value_id = _extract_rotate_half_source(graph, rotated_value_id) + if input_value_id is None: + continue + if _unwrap_trig_value(graph, trig_value_id, expected="scalar_sin") is None: + continue + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + neg_node = producer(graph, cat_node.inputs[0]) + right_slice = producer(graph, cat_node.inputs[1]) + if neg_node is None or neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: + return None + left_slice = producer(graph, neg_node.inputs[0]) + if left_slice is None or right_slice is None or left_slice.op != "slice" or right_slice.op != "slice": + return None + if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): + return None + return left_slice.inputs[0] + + +def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: + trig_info = _unwrap_trig_value(graph, value_id, expected=expected) + if trig_info is None: + return None + trig_node = trig_info["trig_node"] + cat_node = producer(graph, trig_node.inputs[0]) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: + return None + angle_info = _extract_rope_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + matmul_node = angle_info["matmul_node"] + inv_freq_const_id = None + arange_node = None + for input_id in matmul_node.inputs: + if inv_freq_const_id is None: + inv_freq_const_id = _find_constant_ancestor(graph, input_id) + if arange_node is None: + arange_node = _find_arange_ancestor(graph, input_id) + if inv_freq_const_id is None or arange_node is None: + return None + theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) + if theta is None: + return None + return { + "theta": theta, + "position_offset": int(arange_node.attrs.get("start", 0)), + "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), + } + + +def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"unsqueeze", "precision_cast", "reshape", "view", "expand"}: + current = node.inputs[0] + continue + if node.op == "type_as": + current = node.inputs[0] + continue + if node.op == "scalar_multiply": + current = node.inputs[0] + continue + if node.op != expected: + return None + return {"trig_node": node} + + +def _extract_rope_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + while True: + current = strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "permute": + matmul_node = producer(graph, node.inputs[0]) + if matmul_node is None or matmul_node.op != "matmul": + return None + return {"matmul_node": matmul_node} + if node.op == "matmul": + return {"matmul_node": node} + if node.op in {"index", "slice"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + return None + + +def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_layout_passthrough(graph, current) + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return current + node = producer(graph, current) + if node is None or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "arange": + return node + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return None + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _infer_rope_theta(value: Any) -> float | None: + if not isinstance(value, torch.Tensor): + return None + flat = value.detach().cpu().float().reshape(-1) + if flat.numel() < 2: + return None + second = float(flat[1].item()) + if second <= 0.0: + return None + inv_count = flat.numel() + return float((1.0 / second) ** inv_count) diff --git a/python/src/transpile/graph_ir.py b/python/src/transpile/graph_ir.py new file mode 100644 index 000000000..632585c57 --- /dev/null +++ b/python/src/transpile/graph_ir.py @@ -0,0 +1,201 @@ +import copy +from dataclasses import dataclass +from dataclasses import field + +import torch + + +def _clone_ir_payload(value, memo): + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.Tensor): + # IR constants may be real tensors or export-time tensor wrappers. + # They are treated as immutable graph payloads, so preserve them by + # reference instead of triggering Tensor.__deepcopy__. + return value + if isinstance(value, tuple): + return tuple(_clone_ir_payload(item, memo) for item in value) + if isinstance(value, list): + return [_clone_ir_payload(item, memo) for item in value] + if isinstance(value, dict): + return { + copy.deepcopy(key, memo): _clone_ir_payload(inner, memo) + for key, inner in value.items() + } + try: + return copy.deepcopy(value, memo) + except Exception: + return value + + +@dataclass +class IRValue: + id: str + shape: tuple[int, ...] | None = None + dtype: str | None = None + producer: str | None = None + users: list[str] = field(default_factory=list) + meta: dict[str, object] = field(default_factory=dict) + + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRValue( + id=self.id, + shape=_clone_ir_payload(self.shape, memo), + dtype=self.dtype, + producer=self.producer, + users=list(self.users), + meta=_clone_ir_payload(self.meta, memo), + ) + memo[id(self)] = cloned + return cloned + +@dataclass +class IRNode: + id: str + op: str + inputs: list[str] + outputs: list[str] + attrs: dict[str, object] = field(default_factory=dict) + meta: dict[str, object] = field(default_factory=dict) + kind: str = "generic" + + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRNode( + id=self.id, + op=self.op, + inputs=list(self.inputs), + outputs=list(self.outputs), + attrs=_clone_ir_payload(self.attrs, memo), + meta=_clone_ir_payload(self.meta, memo), + kind=self.kind, + ) + memo[id(self)] = cloned + return cloned + +@dataclass +class IRGraph: + values: dict[str, IRValue] + nodes: dict[str, IRNode] + order: list[str] + inputs: list[str] + outputs: list[str] + constants: dict[str, object] = field(default_factory=dict) + meta: dict[str, object] = field(default_factory=dict) + + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRGraph( + values={value_id: copy.deepcopy(value, memo) for value_id, value in self.values.items()}, + nodes={node_id: copy.deepcopy(node, memo) for node_id, node in self.nodes.items()}, + order=list(self.order), + inputs=list(self.inputs), + outputs=list(self.outputs), + constants={ + value_id: _clone_ir_payload(constant, memo) + for value_id, constant in self.constants.items() + }, + meta=_clone_ir_payload(self.meta, memo), + ) + memo[id(self)] = cloned + return cloned + + def add_node(self, node: IRNode) -> None: + if node.id in self.nodes: + raise ValueError(f"duplicate IR node id: {node.id}") + self.nodes[node.id] = node + for output in node.outputs: + if output in self.values: + raise ValueError(f"duplicate IR value id: {output}") + self.values[output] = IRValue(id=output, producer=node.id) + + def add_value(self, value: IRValue) -> None: + if value.id in self.values: + raise ValueError(f"duplicate IR value id: {value.id}") + self.values[value.id] = value + + +def verify_ir(graph: IRGraph) -> None: + order_set = set(graph.order) + node_keys = set(graph.nodes.keys()) + if len(order_set) != len(graph.order): + raise ValueError("IR order contains duplicate node ids") + if order_set != node_keys: + missing_in_order = node_keys - order_set + missing_in_nodes = order_set - node_keys + raise ValueError( + "IR order/nodes mismatch: " + f"missing_in_order={sorted(missing_in_order)} " + f"missing_in_nodes={sorted(missing_in_nodes)}" + ) + + seen_nodes: set[str] = set() + seen_outputs: set[str] = set() + expected_users: dict[str, list[str]] = {} + + for value_id in graph.inputs: + if value_id not in graph.values: + raise ValueError(f"IR input missing value entry: {value_id}") + expected_users.setdefault(value_id, []) + + for value_id in graph.constants: + if value_id not in graph.values: + raise ValueError(f"IR constant missing value entry: {value_id}") + expected_users.setdefault(value_id, []) + + for node_id in graph.order: + if node_id in seen_nodes: + raise ValueError(f"duplicate node in IR order: {node_id}") + seen_nodes.add(node_id) + + node = graph.nodes[node_id] + if not node.outputs: + raise ValueError(f"IR node has no outputs: {node_id}") + + for input_id in node.inputs: + if input_id not in graph.values: + raise ValueError(f"IR node {node_id} references missing input value {input_id}") + expected_users.setdefault(input_id, []).append(node_id) + producer = graph.values[input_id].producer + if producer is not None and producer not in graph.nodes: + raise ValueError( + f"IR value {input_id} for node {node_id} has missing producer {producer}" + ) + + for output_id in node.outputs: + if output_id in seen_outputs: + raise ValueError(f"IR value produced more than once: {output_id}") + seen_outputs.add(output_id) + if output_id not in graph.values: + raise ValueError(f"IR node {node_id} missing output value entry {output_id}") + value = graph.values[output_id] + if value.producer != node_id: + raise ValueError( + f"IR value {output_id} producer mismatch: expected {node_id}, got {value.producer}" + ) + expected_users.setdefault(output_id, []) + + for output_id in graph.outputs: + if output_id not in graph.values: + raise ValueError(f"IR graph output missing value entry: {output_id}") + + for value_id, value in graph.values.items(): + expected = expected_users.get(value_id, []) + if value.users != expected: + raise ValueError( + f"IR value {value_id} user list mismatch: expected {expected}, got {value.users}" + ) + if value.producer is None and value_id not in graph.inputs and value_id not in graph.constants: + if value_id in seen_outputs: + raise ValueError(f"IR value {value_id} is produced but has no producer") + if value.producer is not None and value.producer not in graph.nodes: + raise ValueError( + f"IR value {value_id} references unknown producer {value.producer}" + ) diff --git a/python/src/transpile/import_ir.py b/python/src/transpile/import_ir.py new file mode 100644 index 000000000..1052b464e --- /dev/null +++ b/python/src/transpile/import_ir.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import torch +from torch.export.graph_signature import ConstantArgument +from torch.export.graph_signature import InputKind + +from src.transpile.capture_pytorch import format_target +from src.transpile.capture_pytorch import get_dtype +from src.transpile.capture_pytorch import get_shape +from src.transpile.capture_pytorch import resolve_attr +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import verify_ir +from src.transpile.importers import extract_literals +from src.transpile.importers import ImportContext +from src.transpile.importers import import_call_function +from src.transpile.importers import import_get_attr +from src.transpile.importers import import_output +from src.transpile.importers import import_placeholder +from src.transpile.importers import value_id +from src.transpile.normalize import dtype_to_ir +from src.transpile.import_semantics import apply_import_semantics +from src.transpile.weight_binding import resolve_transpile_weights_dir +from src.transpile.weight_binding import resolve_weight_binding + + +def import_captured_to_ir(captured: Any, *, strict: bool = True) -> IRGraph: + transpile_metadata = getattr(captured, "transpile_metadata", {}) or {} + graph_meta = transpile_metadata.get("graph", {}) + ir = IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[], constants={}, meta=dict(graph_meta)) + ctx = ImportContext(strict=strict, transpile_metadata=transpile_metadata) + weights_dir = resolve_transpile_weights_dir(ir.meta) + if weights_dir: + ir.meta["weights_dir"] = weights_dir + placeholder_specs: dict[str, Any] = {} + inline_counter = 0 + + def _try_register_weight_binding(value_id_str: str, target: str, value: Any) -> None: + if not isinstance(value, torch.Tensor): + return + binding = resolve_weight_binding( + weights_dir=weights_dir, + source_name=target, + ) + if binding is None: + return + bindings = ir.meta.setdefault("weight_bindings", {}) + if isinstance(bindings, dict): + bindings[value_id_str] = { + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + if value_id_str in ir.values: + ir.values[value_id_str].meta.update( + { + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + ) + + def _make_inlined_node(node: Any, *, inline_index: int) -> Any: + return SimpleNamespace( + name=f"inl_{inline_index}_{node.name}", + op=node.op, + target=node.target, + args=node.args, + kwargs=getattr(node, "kwargs", {}), + meta=getattr(node, "meta", {}), + ) + + def _resolve_target(target: str) -> Any: + candidates: list[str] = [] + parts = target.split(".") + for i in range(len(parts)): + candidate = ".".join(parts[i:]) + if candidate and candidate not in candidates: + candidates.append(candidate) + + exported_program = getattr(captured, "exported_program", None) + if exported_program is not None: + tensor_constants = getattr(exported_program, "tensor_constants", None) + if tensor_constants is not None: + for candidate in candidates: + if candidate in tensor_constants: + return tensor_constants[candidate] + constants = getattr(exported_program, "constants", None) + if constants is not None: + for candidate in candidates: + if candidate in constants: + return constants[candidate] + + for candidate in candidates: + try: + return resolve_attr(captured.graph_module, candidate) + except AttributeError: + pass + + for candidate in candidates: + if candidate in captured.state_dict: + return captured.state_dict[candidate] + + source_module = getattr(captured, "source_module", None) + if source_module is not None: + for candidate in candidates: + try: + return resolve_attr(source_module, candidate) + except AttributeError: + pass + + raise NotImplementedError(f"could not resolve get_attr target: {target}") from None + + def _collect_output_ids(arg: Any, local_alias: dict[str, str]) -> list[str]: + if hasattr(arg, "name") and hasattr(arg, "op"): + name = arg.name + if name in local_alias: + return [local_alias[name]] + return [ctx.resolve_value_id(arg)] + if isinstance(arg, (tuple, list)): + out: list[str] = [] + for item in arg: + out.extend(_collect_output_ids(item, local_alias)) + return out + return [] + + def _inline_wrap_with_set_grad_enabled(node: Any, shape: Any, dtype: Any) -> None: + nonlocal inline_counter + subgraph_ref = node.args[1] + if not hasattr(subgraph_ref, "op") or subgraph_ref.op != "get_attr": + raise NotImplementedError("wrap_with_set_grad_enabled without get_attr submodule is not supported") + + subgraph_module = _resolve_target(subgraph_ref.target) + if not hasattr(subgraph_module, "graph"): + raise NotImplementedError("wrap_with_set_grad_enabled target is not a GraphModule") + + inline_counter += 1 + inline_index = inline_counter + local_alias: dict[str, str] = {} + subgraph_inputs = list(node.args[2:]) + input_index = 0 + + ctx.push_local_aliases(local_alias) + try: + for sub_node in subgraph_module.graph.nodes: + sub_shape = get_shape(sub_node) + sub_dtype = dtype_to_ir(get_dtype(sub_node)) + sub_desc = ( + f"inlined:{node.name}:{sub_node.op}:{getattr(sub_node, 'name', '')}:" + f"{format_target(sub_node) if sub_node.op == 'call_function' else getattr(sub_node, 'target', '')}" + ) + ctx.push_node(sub_desc) + try: + if sub_node.op == "placeholder": + if input_index >= len(subgraph_inputs): + raise NotImplementedError("subgraph placeholder/input arity mismatch during inlining") + local_alias[sub_node.name] = ctx.resolve_value_id(subgraph_inputs[input_index]) + input_index += 1 + continue + + if sub_node.op == "get_attr": + prefixed = _make_inlined_node(sub_node, inline_index=inline_index) + value = _resolve_target(sub_node.target) + import_get_attr(ir, prefixed, ctx, value, shape=sub_shape, dtype=sub_dtype, source_name=str(sub_node.target)) + _try_register_weight_binding(f"v_{prefixed.name}", str(sub_node.target), value) + local_alias[sub_node.name] = f"v_{prefixed.name}" + continue + + if sub_node.op == "call_function": + prefixed = _make_inlined_node(sub_node, inline_index=inline_index) + import_call_function( + ir, + prefixed, + ctx, + shape=sub_shape, + dtype=sub_dtype, + torch_op=format_target(sub_node), + ) + local_alias[sub_node.name] = f"v_{prefixed.name}" + continue + + if sub_node.op == "output": + ctx.tuple_aliases[node.name] = _collect_output_ids(sub_node.args[0], local_alias) + return + + raise NotImplementedError(f"unsupported inlined subgraph node op: {sub_node.op}") + finally: + ctx.pop_node() + finally: + ctx.pop_local_aliases() + + raise NotImplementedError("inlined subgraph had no output node") + + for spec in captured.exported_program.graph_signature.input_specs: + arg = getattr(spec, "arg", None) + name = getattr(arg, "name", None) + if name is not None: + placeholder_specs[name] = spec + + for node in captured.graph.nodes: + shape = get_shape(node) + dtype = dtype_to_ir(get_dtype(node)) + node_desc = f"top:{node.op}:{getattr(node, 'name', '')}:{getattr(node, 'target', '')}" + ctx.push_node(node_desc) + try: + if node.op == "placeholder": + spec = placeholder_specs.get(node.name) + if spec is not None and isinstance(getattr(spec, "arg", None), ConstantArgument): + import_get_attr( + ir, + node, + ctx, + spec.arg.value, + shape=shape, + dtype=dtype, + source_name=node.name, + ) + continue + if spec is not None and spec.kind in {InputKind.PARAMETER, InputKind.BUFFER, InputKind.CONSTANT_TENSOR}: + target = getattr(spec, "target", None) + if target is None: + raise NotImplementedError(f"placeholder {node.name} has no target for lifted constant input") + value = _resolve_target(target) + import_get_attr(ir, node, ctx, value, shape=shape, dtype=dtype, source_name=str(target)) + _try_register_weight_binding(value_id(node, ctx), str(target), value) + continue + import_placeholder(ir, node, ctx, shape=shape, dtype=dtype) + continue + + if node.op == "get_attr": + if str(node.target).startswith("submod_"): + continue + value = _resolve_target(node.target) + import_get_attr(ir, node, ctx, value, shape=shape, dtype=dtype, source_name=str(node.target)) + _try_register_weight_binding(value_id(node, ctx), str(node.target), value) + continue + + if node.op == "call_function": + torch_op = format_target(node) + if "wrap_with_set_grad_enabled" in torch_op: + _inline_wrap_with_set_grad_enabled(node, shape, dtype) + continue + if torch_op == "": + source = node.args[0] + if hasattr(source, "name") and source.name in ctx.tuple_aliases: + index_value = extract_literals(node.args[1]) + if not isinstance(index_value, int): + raise NotImplementedError(f"unsupported tuple getitem index: {node.args!r}") + ctx.alias_values[node.name] = ctx.tuple_aliases[source.name][index_value] + continue + import_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) + continue + + if node.op == "output": + import_output(ir, node, ctx) + continue + + raise NotImplementedError(f"unsupported FX node op: {node.op}") + finally: + ctx.pop_node() + + apply_import_semantics(ir) + verify_ir(ir) + return ir diff --git a/python/src/transpile/import_semantics.py b/python/src/transpile/import_semantics.py new file mode 100644 index 000000000..7ef37ed1f --- /dev/null +++ b/python/src/transpile/import_semantics.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.canonicalize.utils import rebuild_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import verify_ir + + +@dataclass(frozen=True) +class EarlyRoPEMatch: + anchor_node_id: str + input_value_id: str + theta: float + position_offset: int + partial: bool + node_ids: tuple[str, ...] + + +def apply_import_semantics(graph: IRGraph) -> IRGraph: + _rewrite_explicit_attention(graph) + _tag_explicit_linear(graph) + _rewrite_early_rope(graph) + rebuild_graph(graph) + verify_ir(graph) + return graph + + +def _rewrite_explicit_attention(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "scaled_dot_product_attention": + continue + additive_mask = bool(node.attrs.get("additive_mask", False)) + if not additive_mask and len(node.inputs) > 3: + mask_value = graph.values.get(node.inputs[3]) + if mask_value is not None and mask_value.dtype not in (None, "bool"): + additive_mask = True + node.op = "attention" + node.kind = "semantic" + node.meta["semantic_source"] = "import" + node.attrs = { + "scale": float(node.attrs.get("scale", 1.0)), + "is_causal": bool(node.attrs.get("is_causal", False)), + "window_size": int(node.attrs.get("window_size", 0)), + **({"enable_gqa": bool(node.attrs.get("enable_gqa", False))} if "enable_gqa" in node.attrs else {}), + **({"additive_mask": True} if additive_mask else {}), + } + + +def _tag_explicit_linear(graph: IRGraph) -> None: + for node in graph.nodes.values(): + if node.op not in {"linear", "addmm"}: + continue + node.meta.setdefault("semantic_source", "import") + + +def _rewrite_early_rope(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or not node.outputs: + continue + match = match_early_rope(graph, node.outputs[0]) + if match is None or match.anchor_node_id != node.id: + continue + if match.partial: + node.meta["semantic_hint"] = "partial_rope" + node.meta["rope_theta_hint"] = float(match.theta) + node.meta["rope_position_offset_hint"] = int(match.position_offset) + continue + node.op = "rope" + node.inputs = [match.input_value_id] + node.attrs = { + "theta": float(match.theta), + "position_offset": int(match.position_offset), + } + node.kind = "semantic" + node.meta["semantic_source"] = "import" + + +def match_early_rope(graph: IRGraph, value_id: str) -> EarlyRoPEMatch | None: + desc = _extract_rope_descriptor(graph, value_id) + if desc is None: + return None + return EarlyRoPEMatch( + anchor_node_id=str(desc["anchor_node_id"]), + input_value_id=str(desc["input_value_id"]), + theta=float(desc["theta"]), + position_offset=int(desc["position_offset"]), + partial=bool(desc.get("partial_rope", False)), + node_ids=tuple(sorted(set(desc.get("node_ids", ())))), + ) + + +def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = _strip_layout_passthrough(graph, value_id) + node = producer(graph, current) + if node is None: + return None + if node.op == "rope": + return { + "anchor_node_id": node.id, + "input_value_id": node.inputs[0], + "theta": float(node.attrs.get("theta", 0.0)), + "position_offset": int(node.attrs.get("position_offset", 0)), + "partial_rope": False, + "node_ids": (node.id,), + } + classic = _extract_classic_rope_descriptor(graph, current) + if classic is not None: + classic["partial_rope"] = False + return classic + if node.op == "cat" and len(node.inputs) == 2: + for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) + if rope_branch is None: + continue + passthrough = producer(graph, _strip_layout_passthrough(graph, passthrough_value_id)) + if passthrough is None or passthrough.op != "slice": + continue + if strip_passthrough(graph, passthrough.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): + continue + rope_branch["anchor_node_id"] = node.id + rope_branch["partial_rope"] = True + rope_branch["node_ids"] = tuple(sorted({node.id, passthrough.id, *rope_branch["node_ids"]})) + return rope_branch + return None + + +def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node = producer(graph, _strip_layout_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return None + for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + direct_mult = producer(graph, direct_mult_id) + rotated_mult = producer(graph, rotated_mult_id) + if direct_mult is None or rotated_mult is None: + continue + if direct_mult.op != "multiply" or rotated_mult.op != "multiply": + continue + direct = _extract_direct_branch(graph, direct_mult) + rotated = _extract_rotated_branch(graph, rotated_mult) + if direct is None or rotated is None: + continue + input_value_id = strip_passthrough(graph, direct["input_value_id"]) + if input_value_id != strip_passthrough(graph, rotated["input_value_id"]): + continue + cos_info = _extract_rope_trig(graph, direct["trig_value_id"], expected=("cos", "scalar_cos")) + sin_info = _extract_rope_trig(graph, rotated["trig_value_id"], expected=("sin", "scalar_sin")) + if cos_info is None or sin_info is None: + continue + if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: + continue + if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): + continue + return { + "anchor_node_id": node.id, + "input_value_id": input_value_id, + "theta": float(cos_info["theta"]), + "position_offset": int(cos_info["position_offset"]), + "node_ids": tuple(sorted({node.id, *direct["node_ids"], *rotated["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), + } + return None + + +def _extract_direct_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + if _unwrap_trig_value(graph, trig_value_id, expected=("cos", "scalar_cos")) is not None: + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotated_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + input_value_id = _extract_rotate_half_source(graph, rotated_value_id) + if input_value_id is None: + continue + if _unwrap_trig_value(graph, trig_value_id, expected=("sin", "scalar_sin")) is None: + continue + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + neg_node = producer(graph, cat_node.inputs[0]) + right_slice = producer(graph, cat_node.inputs[1]) + if neg_node is None or right_slice is None: + return None + if neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: + return None + left_slice = producer(graph, neg_node.inputs[0]) + if left_slice is None or left_slice.op != "slice" or right_slice.op != "slice": + return None + if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): + return None + return left_slice.inputs[0] + + +def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: + trig = _unwrap_trig_value(graph, value_id, expected=expected) + if trig is None: + return None + trig_node = trig["trig_node"] + cat_node = producer(graph, trig_node.inputs[0]) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: + return None + angle_info = _extract_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + matmul_node = angle_info["matmul_node"] + inv_freq_const_id = None + arange_node = None + for input_id in matmul_node.inputs: + if inv_freq_const_id is None: + inv_freq_const_id = _find_constant_ancestor(graph, input_id) + if arange_node is None: + arange_node = _find_arange_ancestor(graph, input_id) + if inv_freq_const_id is None or arange_node is None: + return None + theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) + if theta is None: + return None + return { + "theta": theta, + "position_offset": int(arange_node.attrs.get("start", 0)), + "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), + } + + +def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"unsqueeze", "reshape", "view", "expand", "precision_cast"}: + current = node.inputs[0] + continue + if node.op == "type_as": + current = node.inputs[0] + continue + if node.op == "scalar_multiply": + current = node.inputs[0] + continue + if node.op not in expected: + return None + return {"trig_node": node} + + +def _extract_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + while True: + current = _strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op in {"transpose", "permute"}: + matmul_node = producer(graph, node.inputs[0]) + if matmul_node is None or matmul_node.op != "matmul": + return None + return {"matmul_node": matmul_node} + if node.op == "matmul": + return {"matmul_node": node} + if node.op in {"index", "slice"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + return None + + +def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = _strip_layout_passthrough(graph, current) + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return current + node = producer(graph, current) + if node is None or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = _strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "arange": + return node + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return None + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def _infer_rope_theta(value: Any) -> float | None: + if not isinstance(value, torch.Tensor): + return None + flat = value.detach().cpu().float().reshape(-1) + if flat.numel() < 2: + return None + second = float(flat[1].item()) + if second <= 0.0: + return None + return float((1.0 / second) ** flat.numel()) diff --git a/python/src/transpile/importers.py b/python/src/transpile/importers.py new file mode 100644 index 000000000..f63abb433 --- /dev/null +++ b/python/src/transpile/importers.py @@ -0,0 +1,1754 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from dataclasses import field +from types import SimpleNamespace +from typing import Any + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.normalize import dtype_to_ir +from src.transpile.normalize import normalize_target + + +class UnsupportedImportError(NotImplementedError): + pass + + +@dataclass +class ImportContext: + strict: bool = True + alias_values: dict[str, str] = field(default_factory=dict) + tuple_aliases: dict[str, list[str]] = field(default_factory=dict) + local_alias_stack: list[dict[str, str]] = field(default_factory=list) + node_stack: list[str] = field(default_factory=list) + transpile_metadata: dict[str, object] = field(default_factory=dict) + + def push_local_aliases(self, aliases: dict[str, str]) -> None: + self.local_alias_stack.append(aliases) + + def pop_local_aliases(self) -> None: + if not self.local_alias_stack: + raise RuntimeError("local alias stack underflow") + self.local_alias_stack.pop() + + def resolve_value_id(self, node: Any) -> str: + if is_fx_node(node): + name = getattr(node, "name", "") + for aliases in reversed(self.local_alias_stack): + if name in aliases: + return aliases[name] + resolved = self.alias_values.get(name) + if resolved is not None: + return resolved + return f"v_{name}" + raise TypeError(f"cannot resolve value id for non-FX node: {type(node).__name__}") + + def fail(self, message: str) -> None: + if self.node_stack: + message = f"{message} [context: {' > '.join(self.node_stack)}]" + if self.strict: + raise UnsupportedImportError(message) + raise UnsupportedImportError(f"{message} [non-strict import has no generic fallback]") + + def push_node(self, description: str) -> None: + self.node_stack.append(description) + + def pop_node(self) -> None: + if self.node_stack: + self.node_stack.pop() + + def _node_module_paths(self, node: Any) -> tuple[str, ...]: + meta = getattr(node, "meta", {}) or {} + raw_stack = meta.get("nn_module_stack") + if not isinstance(raw_stack, dict): + return () + paths: list[str] = [] + for entry in raw_stack.values(): + if not isinstance(entry, (tuple, list)) or not entry: + continue + module_path = entry[0] + if isinstance(module_path, str) and module_path: + paths.append(module_path) + # Prefer the deepest module path first. + paths.reverse() + deduped: list[str] = [] + for path in paths: + if path not in deduped: + deduped.append(path) + return tuple(deduped) + + def lookup_import_hint(self, node: Any, *, torch_op: str, op_name: str) -> dict[str, object]: + hints = self.transpile_metadata.get("import_hints", ()) + if not isinstance(hints, list): + return {} + + module_paths = self._node_module_paths(node) + matched_attrs: dict[str, object] = {} + matched_meta: dict[str, object] = {} + matched_provider: list[str] = [] + + for hint in hints: + if not isinstance(hint, dict): + continue + hint_op = hint.get("op") + if hint_op is not None and hint_op != op_name: + continue + hint_torch_op = hint.get("torch_op") + if hint_torch_op is not None and hint_torch_op != torch_op: + continue + + module_path = hint.get("module_path") + if module_path is not None and module_path not in module_paths: + continue + module_path_suffix = hint.get("module_path_suffix") + if module_path_suffix is not None and not any( + path == module_path_suffix or path.endswith(f".{module_path_suffix}") for path in module_paths + ): + continue + + attrs = hint.get("attrs", {}) + if isinstance(attrs, dict): + matched_attrs.update(attrs) + meta = hint.get("meta", {}) + if isinstance(meta, dict): + matched_meta.update(meta) + provider = hint.get("provider") + if isinstance(provider, str) and provider not in matched_provider: + matched_provider.append(provider) + + if not matched_attrs and not matched_meta and not matched_provider: + return {} + + if matched_provider: + matched_meta.setdefault("import_hint_providers", tuple(matched_provider)) + matched_meta.setdefault("import_hint_source", "capture_metadata") + return {"attrs": matched_attrs, "meta": matched_meta} + + +def value_id(node: Any, ctx: ImportContext) -> str: + return ctx.resolve_value_id(node) + + +def node_id(node: Any) -> str: + return f"n_{node.name}" + + +def is_fx_node(value: Any) -> bool: + return hasattr(value, "name") and hasattr(value, "op") + + +def extract_input_ids(arg: Any, ctx: ImportContext) -> list[str]: + if is_fx_node(arg): + return [value_id(arg, ctx)] + if isinstance(arg, (tuple, list)): + ids: list[str] = [] + for item in arg: + ids.extend(extract_input_ids(item, ctx)) + return ids + if isinstance(arg, dict): + ids: list[str] = [] + for item in arg.values(): + ids.extend(extract_input_ids(item, ctx)) + return ids + return [] + + +def extract_literals(value: Any) -> Any: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return extract_literals(meta_value) + return None + if isinstance(value, torch.Size): + return tuple(extract_literals(v) for v in value) + if isinstance(value, (tuple, list)): + return type(value)(extract_literals(v) for v in value) + if isinstance(value, dict): + return {k: extract_literals(v) for k, v in value.items()} + if isinstance(value, torch.dtype): + return str(value) + if isinstance(value, (int, float, bool, str)) or value is None: + return value + return None + + +def make_prefixed_node(node: Any, prefix: str) -> Any: + return SimpleNamespace( + name=f"{prefix}{node.name}", + op=node.op, + target=node.target, + args=node.args, + kwargs=getattr(node, "kwargs", {}), + meta=getattr(node, "meta", {}), + ) + + +def add_value_if_missing(ir: IRGraph, value: IRValue) -> None: + if value.id not in ir.values: + ir.add_value(value) + + +def register_node( + ir: IRGraph, + ir_node: IRNode, + *, + shape: tuple[int, ...] | None, + dtype: str | None, +) -> None: + ir.add_node(ir_node) + ir.order.append(ir_node.id) + for output_id in ir_node.outputs: + ir.values[output_id].shape = shape + ir.values[output_id].dtype = dtype + for input_id in ir_node.inputs: + if input_id in ir.values: + ir.values[input_id].users.append(ir_node.id) + + +def _extract_numeric_literal(value: Any) -> float | int | None: + literal = extract_literals(value) + if isinstance(literal, bool): + return int(literal) + if isinstance(literal, (int, float)): + return literal + return None + + +def _extract_static_int(value: Any) -> int | None: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return _extract_static_int(meta_value) + return None + + literal = extract_literals(value) + if isinstance(literal, bool): + return int(literal) + if isinstance(literal, int): + return int(literal) + if isinstance(literal, float) and float(literal).is_integer(): + return int(literal) + + try: + return int(value) + except Exception: + return None + + +def _extract_int_sequence_literal(value: Any) -> tuple[int, ...] | None: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return _extract_int_sequence_literal(meta_value) + return None + + if isinstance(value, torch.Size): + value = tuple(value) + + if isinstance(value, (tuple, list)): + result: list[int] = [] + for item in value: + item_value = _extract_static_int(item) + if item_value is None: + return None + result.append(item_value) + return tuple(result) + + scalar_value = _extract_static_int(value) + if scalar_value is not None: + return (scalar_value,) + return None + + +def _extract_fx_tensor_shape(value: Any) -> tuple[Any, ...] | None: + if not is_fx_node(value): + return None + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if isinstance(meta_value, torch.Tensor): + return tuple(meta_value.shape) + tensor_meta = meta.get("tensor_meta") + tensor_shape = getattr(tensor_meta, "shape", None) + if tensor_shape is not None: + return tuple(tensor_shape) + return None + + +def _base_meta(shape: tuple[int, ...] | None, dtype: str | None, torch_op: str, node: Any) -> dict[str, object]: + return { + "shape": shape, + "dtype": dtype, + "torch_op": torch_op, + "torch_name": node.name, + } + + +def _extract_fx_torch_dtype(node: Any) -> torch.dtype | None: + if not is_fx_node(node): + return None + meta = getattr(node, "meta", {}) or {} + value = meta.get("val") + dtype = getattr(value, "dtype", None) + if isinstance(dtype, torch.dtype): + return dtype + tensor_meta = meta.get("tensor_meta") + dtype = getattr(tensor_meta, "dtype", None) + if isinstance(dtype, torch.dtype): + return dtype + return None + + +def _import_scalar_binary( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, + op_name: str, +) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if op_name == "add": + if lhs_is_node and rhs_literal is not None: + scalar_op = "scalar_add" + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + scalar_op = "scalar_add" + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + scalar_op = None + elif op_name == "multiply": + if lhs_is_node and rhs_literal is not None: + scalar_op = "scalar_multiply" + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + scalar_op = "scalar_multiply" + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + scalar_op = None + elif op_name == "subtract": + if lhs_is_node and rhs_literal is not None: + scalar_op = "scalar_subtract" + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + scalar_op = "scalar_subtract_reverse" + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + scalar_op = None + elif op_name == "divide": + if lhs_is_node and rhs_literal is not None: + scalar_op = "scalar_divide" + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + scalar_op = "scalar_divide_reverse" + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + scalar_op = None + else: + scalar_op = None + + if scalar_op is None: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op=scalar_op, + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_placeholder( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, +) -> None: + add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) + ir.inputs.append(value_id(node, ctx)) + + +def import_get_attr( + ir: IRGraph, + node: Any, + ctx: ImportContext, + value: Any, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + source_name: str | None = None, +) -> None: + add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) + ir.constants[value_id(node, ctx)] = value.detach().cpu() if isinstance(value, torch.Tensor) else value + ir.values[value_id(node, ctx)].meta["source_name"] = source_name + if source_name is not None: + bindings = ir.meta.setdefault("weight_bindings", {}) + if isinstance(bindings, dict): + bindings.setdefault(value_id(node, ctx), {})["source_name"] = source_name + + +def import_output(ir: IRGraph, node: Any, ctx: ImportContext) -> None: + ir.outputs.extend(extract_input_ids(node.args[0], ctx)) + + +def import_call_function( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + op = normalize_target(torch_op) + import_hint = ctx.lookup_import_hint(node, torch_op=torch_op, op_name=op) + importer = OP_IMPORTERS.get(op) + if importer is None: + import_opaque_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name=op) + else: + importer(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) + _apply_import_hint(ir, node, import_hint) + + +def _apply_import_hint(ir: IRGraph, node: Any, hint: dict[str, object]) -> None: + if not hint: + return + ir_node = ir.nodes.get(node_id(node)) + if ir_node is None: + return + attrs = hint.get("attrs", {}) + if isinstance(attrs, dict): + ir_node.attrs.update(attrs) + meta = hint.get("meta", {}) + if isinstance(meta, dict): + ir_node.meta.update(meta) + + +def import_opaque_call_function( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, + op_name: str, +) -> None: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=extract_input_ids(getattr(node, "args", ()), ctx) + extract_input_ids(getattr(node, "kwargs", {}), ctx), + outputs=[value_id(node, ctx)], + attrs={ + "opaque": True, + "torch_op": torch_op, + "args": extract_literals(getattr(node, "args", ())), + "kwargs": extract_literals(getattr(node, "kwargs", {})), + }, + meta=_base_meta(shape, dtype, torch_op, node), + kind="opaque", + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_add(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="add") + + +def import_subtract(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="subtract") + + +def import_multiply(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") + + +def import_multiply_inplace(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") + + +def import_divide(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="divide") + + +def import_not_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op="not_equal", + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if lhs_is_node and rhs_literal is not None: + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op="scalar_not_equal", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op="equal", + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if lhs_is_node and rhs_literal is not None: + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op="scalar_equal", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_binary(op_name: str): + def _importer( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, + ) -> None: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + return _importer + + +def import_where( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + condition = node.args[0] + true_value = node.args[1] + false_value = node.args[2] + + inputs = [value_id(condition, ctx)] + attrs: dict[str, object] = {} + + if is_fx_node(true_value): + inputs.append(value_id(true_value, ctx)) + attrs["true_is_scalar"] = False + else: + literal = _extract_numeric_literal(true_value) + if literal is None: + ctx.fail(f"unsupported true branch for {torch_op}: {node.args!r}") + attrs["true_is_scalar"] = True + attrs["true_value"] = float(literal) + + if is_fx_node(false_value): + inputs.append(value_id(false_value, ctx)) + attrs["false_is_scalar"] = False + else: + literal = _extract_numeric_literal(false_value) + if literal is None: + ctx.fail(f"unsupported false branch for {torch_op}: {node.args!r}") + attrs["false_is_scalar"] = True + attrs["false_value"] = float(literal) + + ir_node = IRNode( + id=node_id(node), + op="where", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_negate(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="scalar_multiply", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": -1.0}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_precision_cast(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + target_dtype = extract_literals(node.args[1]) if len(node.args) > 1 else None + ir_node = IRNode( + id=node_id(node), + op="precision_cast", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"dtype": target_dtype}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_arange(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + attrs: dict[str, object] = {} + if len(node.args) == 1: + attrs["start"] = 0 + attrs["end"] = int(node.args[0]) + elif len(node.args) >= 2: + attrs["start"] = int(node.args[0]) + attrs["end"] = int(node.args[1]) + else: + ctx.fail(f"unsupported arange signature for {torch_op}: {node.args!r}") + + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + attrs["step"] = int(node.args[2]) + + target_dtype = None + if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None: + target_dtype = extract_literals(node.kwargs["dtype"]) + elif dtype is not None: + target_dtype = dtype + if target_dtype is not None: + attrs["dtype"] = target_dtype + + ir_node = IRNode( + id=node_id(node), + op="arange", + inputs=[], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_type_as(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="type_as", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_identity(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="identity", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_reshape(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + target_shape = _extract_int_sequence_literal(node.args[1]) + if target_shape is None: + ctx.fail(f"unsupported reshape shape for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="reshape", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"shape": target_shape}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_flatten(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="flatten", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"start_dim": int(node.args[1]), "end_dim": int(node.args[2])}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_unsqueeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + dim = int(node.args[1]) + attrs: dict[str, object] = {"dim": dim} + if shape is not None: + attrs["shape"] = tuple(int(v) for v in shape) + ir_node = IRNode( + id=node_id(node), + op="unsqueeze", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_squeeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + dim = int(node.args[1]) if len(node.args) > 1 and node.args[1] is not None else 0 + attrs: dict[str, object] = {"dim": dim} + if shape is not None: + attrs["shape"] = tuple(int(v) for v in shape) + ir_node = IRNode( + id=node_id(node), + op="squeeze", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_masked_scatter( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + ir_node = IRNode( + id=node_id(node), + op="masked_scatter", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_masked_fill( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + value = _extract_numeric_literal(node.args[2]) + if value is None: + ctx.fail(f"unsupported masked_fill value for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="masked_fill", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": float(value)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_expand(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + target_shape = _extract_int_sequence_literal(node.args[1]) + if target_shape is None and shape is not None: + target_shape = _extract_int_sequence_literal(shape) + if target_shape is None: + ctx.fail(f"unsupported expand shape for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="expand", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"shape": target_shape}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_one_hot(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + num_classes = _extract_static_int(node.args[1]) if len(node.args) > 1 else None + if (num_classes is None or num_classes <= 0) and shape is not None and shape: + num_classes = int(shape[-1]) + if num_classes is None or num_classes <= 0: + ctx.fail(f"unsupported one_hot num_classes for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="one_hot", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"num_classes": int(num_classes)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_tril(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + diagonal = int(extract_literals(node.args[1])) if len(node.args) > 1 and extract_literals(node.args[1]) is not None else 0 + ir_node = IRNode( + id=node_id(node), + op="tril", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"diagonal": diagonal}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_unfold(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 4: + ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") + dimension = extract_literals(node.args[1]) + size = extract_literals(node.args[2]) + step = extract_literals(node.args[3]) + if not isinstance(dimension, int) or not isinstance(size, int) or not isinstance(step, int): + ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="unfold", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"dimension": dimension, "size": size, "step": step}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_transpose(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + attrs = {"dim0": 0, "dim1": 1} if torch_op == "aten.t.default" else {"dim0": int(node.args[1]), "dim1": int(node.args[2])} + ir_node = IRNode( + id=node_id(node), + op="transpose", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_permute(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + permutation = _extract_int_sequence_literal(node.args[1]) + if permutation is None: + ctx.fail(f"unsupported permute dims for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="permute", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"permutation": permutation}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_numpy_t(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + input_shape = _extract_fx_tensor_shape(node.args[0]) + rank_source = input_shape if input_shape is not None else shape + if rank_source is None: + ctx.fail(f"unsupported numpy_T rank inference for {torch_op}: {node.args!r}") + permutation = tuple(range(len(rank_source) - 1, -1, -1)) + ir_node = IRNode( + id=node_id(node), + op="permute", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"permutation": permutation}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_movedim(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + source = _extract_int_sequence_literal(node.args[1]) + destination = _extract_int_sequence_literal(node.args[2]) + if source is None or destination is None: + ctx.fail(f"unsupported movedim dims for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="movedim", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"source": source, "destination": destination}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_matmul(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="matmul", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_linear(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + attrs = {"has_bias": False} + if len(node.args) > 2 and node.args[2] is not None: + inputs.append(value_id(node.args[2], ctx)) + attrs["has_bias"] = True + ir_node = IRNode( + id=node_id(node), + op="linear", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_addmm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="addmm", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_unary(op_name: str): + def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return _importer + + +def import_clamp(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + min_value = extract_literals(node.args[1]) if len(node.args) > 1 else None + max_value = extract_literals(node.args[2]) if len(node.args) > 2 else None + attrs: dict[str, object] = {} + if isinstance(min_value, (int, float)): + attrs["min"] = float(min_value) + if isinstance(max_value, (int, float)): + attrs["max"] = float(max_value) + ir_node = IRNode( + id=node_id(node), + op="clamp", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_pow_rsqrt(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="pow", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"exponent": -0.5}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_reciprocal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="pow", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"exponent": -1.0}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_floor_divide( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported floor_divide signature for {torch_op}: {node.args!r}") + divisor = _extract_numeric_literal(node.args[1]) + if divisor is None: + ctx.fail(f"unsupported floor_divide divisor for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="scalar_floor_divide", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": float(divisor)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_softmax(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="softmax", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"axis": int(node.args[1])}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_glu(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + axis = -1 + if len(node.args) > 1 and extract_literals(node.args[1]) is not None: + axis = int(node.args[1]) + ir_node = IRNode( + id=node_id(node), + op="glu", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_scaled_dot_product_attention( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + attrs: dict[str, object] = {} + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)] + if len(node.args) > 3: + mask_literal = extract_literals(node.args[3]) + if mask_literal is not None: + attrs["mask"] = mask_literal + elif is_fx_node(node.args[3]): + inputs.append(value_id(node.args[3], ctx)) + mask_dtype = _extract_fx_torch_dtype(node.args[3]) + if mask_dtype is not None and mask_dtype != torch.bool: + attrs["additive_mask"] = True + if len(node.args) > 4 and extract_literals(node.args[4]) is not None: + attrs["dropout_p"] = float(node.args[4]) + if len(node.args) > 5 and extract_literals(node.args[5]) is not None: + attrs["is_causal"] = bool(node.args[5]) + if "scale" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["scale"]) is not None: + attrs["scale"] = float(node.kwargs["scale"]) + if "enable_gqa" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["enable_gqa"]) is not None: + attrs["enable_gqa"] = bool(node.kwargs["enable_gqa"]) + ir_node = IRNode( + id=node_id(node), + op="scaled_dot_product_attention", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_reduce(op_name: str): + def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + attrs: dict[str, object] = {} + if len(node.args) > 1 and extract_literals(node.args[1]) is not None: + attrs["axis"] = extract_literals(node.args[1]) + keepdim = None + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + keepdim = bool(extract_literals(node.args[2])) + elif "keepdim" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["keepdim"]) is not None: + keepdim = bool(extract_literals(node.kwargs["keepdim"])) + if keepdim is not None: + attrs["keepdim"] = keepdim + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return _importer + + +def import_cat(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + tensors_arg = node.args[0] + axis = int(node.args[1]) if len(node.args) > 1 else 0 + ir_node = IRNode( + id=node_id(node), + op="cat", + inputs=extract_input_ids(tensors_arg, ctx), + outputs=[value_id(node, ctx)], + attrs={"axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_split_with_sizes(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported split_with_sizes signature for {torch_op}: {node.args!r}") + sizes = _extract_int_sequence_literal(node.args[1]) + if sizes is None: + ctx.fail(f"unsupported split sizes for {torch_op}: {node.args!r}") + axis = -1 + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + axis = int(extract_literals(node.args[2])) + ir_node = IRNode( + id=node_id(node), + op="split_with_sizes", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"sizes": sizes, "axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_chunk(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported chunk signature for {torch_op}: {node.args!r}") + chunks = extract_literals(node.args[1]) + if not isinstance(chunks, int): + ctx.fail(f"unsupported chunk count for {torch_op}: {node.args!r}") + axis = 0 + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + axis = int(extract_literals(node.args[2])) + ir_node = IRNode( + id=node_id(node), + op="chunk", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"chunks": int(chunks), "axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_pow(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + base_literal = _extract_numeric_literal(node.args[0]) if len(node.args) > 0 else None + attrs: dict[str, object] = {} + exponent_literal = extract_literals(node.args[1]) if len(node.args) > 1 else None + + if base_literal is not None and isinstance(exponent_literal, (int, float)): + add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) + ir.constants[value_id(node, ctx)] = base_literal**exponent_literal + return + + if base_literal is not None: + if not (len(node.args) > 1 and is_fx_node(node.args[1])): + ctx.fail(f"unsupported pow signature for {torch_op}: scalar base with non-literal exponent {node.args!r}") + if float(base_literal) <= 0.0: + ctx.fail(f"unsupported pow signature for {torch_op}: non-positive scalar base {base_literal!r}") + + exponent_value_id = value_id(node.args[1], ctx) + mul_output_id = f"{value_id(node, ctx)}__pow_scalar_base_log_mul" + mul_node = IRNode( + id=f"{node_id(node)}__pow_scalar_base_log_mul", + op="scalar_multiply", + inputs=[exponent_value_id], + outputs=[mul_output_id], + attrs={"value": float(math.log(float(base_literal)))}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, mul_node, shape=shape, dtype=dtype) + + exp_node = IRNode( + id=node_id(node), + op="scalar_exp", + inputs=[mul_output_id], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, exp_node, shape=shape, dtype=dtype) + return + + inputs = [value_id(node.args[0], ctx)] + if exponent_literal is None and len(node.args) > 1 and is_fx_node(node.args[1]): + inputs.append(value_id(node.args[1], ctx)) + else: + attrs["exponent"] = exponent_literal + ir_node = IRNode( + id=node_id(node), + op="pow", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_slice(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + attrs = {"axis": int(node.args[1]), "start": int(node.args[2]), "end": int(node.args[3])} + if len(node.args) > 4 and extract_literals(node.args[4]) is not None: + attrs["step"] = int(node.args[4]) + ir_node = IRNode( + id=node_id(node), + op="slice", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = list(getattr(node, "args", ())) + if not inputs: + ctx.fail(f"unsupported diff signature for {torch_op}: missing input tensor") + + source = inputs[0] + if not is_fx_node(source): + ctx.fail(f"unsupported diff signature for {torch_op}: source is not FX node") + + n_value = 1 + if len(inputs) > 1 and extract_literals(inputs[1]) is not None: + n_value = int(extract_literals(inputs[1])) + elif "n" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["n"]) is not None: + n_value = int(extract_literals(node.kwargs["n"])) + if n_value != 1: + ctx.fail(f"unsupported diff order for {torch_op}: n={n_value}") + + dim_value = None + if len(inputs) > 2 and extract_literals(inputs[2]) is not None: + dim_value = int(extract_literals(inputs[2])) + elif "dim" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dim"]) is not None: + dim_value = int(extract_literals(node.kwargs["dim"])) + if dim_value is None: + ctx.fail(f"unsupported diff signature for {torch_op}: missing integer dim") + + prepend_value = None + append_value = None + if len(inputs) > 3: + prepend_value = extract_literals(inputs[3]) + if len(inputs) > 4: + append_value = extract_literals(inputs[4]) + if "prepend" in getattr(node, "kwargs", {}): + prepend_value = extract_literals(node.kwargs["prepend"]) + if "append" in getattr(node, "kwargs", {}): + append_value = extract_literals(node.kwargs["append"]) + if prepend_value is not None or append_value is not None: + ctx.fail(f"unsupported diff signature for {torch_op}: prepend/append are not supported") + + source_value_id = value_id(source, ctx) + source_shape = None + if source_value_id in ir.values: + source_shape = ir.values[source_value_id].shape + if source_shape is None: + source_shape = shape + if source_shape is None: + ctx.fail(f"unsupported diff import for {torch_op}: missing source shape") + + rank = len(source_shape) + normalized_dim = dim_value if dim_value >= 0 else dim_value + rank + if normalized_dim < 0 or normalized_dim >= rank: + ctx.fail(f"unsupported diff dimension for {torch_op}: dim={dim_value} rank={rank}") + + dim_extent = int(source_shape[normalized_dim]) + if dim_extent < 1: + ctx.fail(f"unsupported diff input extent for {torch_op}: dim size {dim_extent}") + + left_shape = list(source_shape) + left_shape[normalized_dim] = max(0, dim_extent - 1) + left_shape_tuple = tuple(int(v) for v in left_shape) + + left_node = IRNode( + id=f"{node_id(node)}__diff_left", + op="slice", + inputs=[source_value_id], + outputs=[f"{value_id(node, ctx)}__diff_left"], + attrs={"axis": normalized_dim, "start": 0, "end": dim_extent - 1, "step": 1}, + meta=_base_meta(left_shape_tuple, dtype, torch_op, node), + ) + register_node(ir, left_node, shape=left_shape_tuple, dtype=dtype) + + right_node = IRNode( + id=f"{node_id(node)}__diff_right", + op="slice", + inputs=[source_value_id], + outputs=[f"{value_id(node, ctx)}__diff_right"], + attrs={"axis": normalized_dim, "start": 1, "end": dim_extent, "step": 1}, + meta=_base_meta(left_shape_tuple, dtype, torch_op, node), + ) + register_node(ir, right_node, shape=left_shape_tuple, dtype=dtype) + + diff_node = IRNode( + id=node_id(node), + op="subtract", + inputs=[right_node.outputs[0], left_node.outputs[0]], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, diff_node, shape=shape, dtype=dtype) + + +def import_index(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if torch_op.startswith("aten.select"): + axis = int(node.args[1]) + index_value = int(node.args[2]) + else: + index_value = int(node.args[1]) + axis = int(node.args[2]) if len(node.args) > 2 else 0 + ir_node = IRNode( + id=node_id(node), + op="index", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"index_value": index_value, "axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_gather(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="gather", + inputs=[value_id(node.args[0], ctx), value_id(node.args[2], ctx)], + outputs=[value_id(node, ctx)], + attrs={"axis": int(node.args[1])}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_embedding(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + attrs: dict[str, object] = {} + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + attrs["padding_idx"] = int(node.args[2]) + if len(node.args) > 3 and extract_literals(node.args[3]) is not None: + attrs["scale_grad_by_freq"] = bool(node.args[3]) + if len(node.args) > 4 and extract_literals(node.args[4]) is not None: + attrs["sparse"] = bool(node.args[4]) + ir_node = IRNode( + id=node_id(node), + op="embedding", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def _extract_int_list_or_scalar(value: Any, *, default: int) -> int: + literal = extract_literals(value) + if literal is None: + return default + if isinstance(literal, int): + return int(literal) + if isinstance(literal, (tuple, list)) and literal: + first = literal[0] + if isinstance(first, int): + return int(first) + raise UnsupportedImportError(f"unsupported integer/list literal: {value!r}") + + +def import_conv1d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported conv1d signature for {torch_op}: {node.args!r}") + + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + if len(node.args) > 2 and is_fx_node(node.args[2]): + inputs.append(value_id(node.args[2], ctx)) + + stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 + padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 + dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 + groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 + + ir_node = IRNode( + id=node_id(node), + op="conv1d", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_conv2d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported conv2d signature for {torch_op}: {node.args!r}") + + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + if len(node.args) > 2 and is_fx_node(node.args[2]): + inputs.append(value_id(node.args[2], ctx)) + + stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 + padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 + dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 + groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 + + ir_node = IRNode( + id=node_id(node), + op="conv2d", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_pad(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported pad signature for {torch_op}: {node.args!r}") + + pads = extract_literals(node.args[1]) + if not isinstance(pads, (tuple, list)) or not all(isinstance(v, int) for v in pads): + ctx.fail(f"unsupported pads for {torch_op}: {node.args!r}") + + mode = "constant" + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + mode = str(extract_literals(node.args[2])) + if "mode" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["mode"]) is not None: + mode = str(extract_literals(node.kwargs["mode"])) + + value = 0.0 + if len(node.args) > 3 and extract_literals(node.args[3]) is not None: + value = float(extract_literals(node.args[3])) + if "value" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["value"]) is not None: + value = float(extract_literals(node.kwargs["value"])) + + ir_node = IRNode( + id=node_id(node), + op="pad", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"pads": tuple(int(v) for v in pads), "mode": mode, "value": value}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if not node.args: + ctx.fail(f"unsupported ones signature for {torch_op}: {node.args!r}") + + output_shape = _extract_int_sequence_literal(node.args[0]) + if output_shape is None: + ctx.fail(f"unsupported ones shape for {torch_op}: {node.args!r}") + + node_dtype = dtype + if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None: + node_dtype = dtype_to_ir(extract_literals(node.kwargs["dtype"])) + elif len(node.args) > 1 and extract_literals(node.args[1]) is not None: + node_dtype = dtype_to_ir(extract_literals(node.args[1])) + if node_dtype is None: + node_dtype = "fp32" + + ir_node = IRNode( + id=node_id(node), + op="ones", + inputs=[], + outputs=[value_id(node, ctx)], + attrs={"shape": output_shape, "dtype": node_dtype}, + meta=_base_meta(shape or output_shape, node_dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape or output_shape, dtype=node_dtype) + + +def import_layer_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if not node.args: + ctx.fail(f"unsupported layer_norm signature for {torch_op}: {node.args!r}") + + normalized_shape_value = None + if len(node.args) > 1: + normalized_shape_value = _extract_int_sequence_literal(node.args[1]) + if normalized_shape_value is None: + normalized_shape_value = _extract_int_sequence_literal(getattr(node, "kwargs", {}).get("normalized_shape")) + if normalized_shape_value is None: + ctx.fail(f"unsupported normalized_shape for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + + eps_value = None + if len(node.args) > 4: + eps_value = _extract_numeric_literal(node.args[4]) + if eps_value is None: + eps_value = _extract_numeric_literal(getattr(node, "kwargs", {}).get("eps")) + if eps_value is None: + eps_value = 1e-5 + + inputs = [value_id(node.args[0], ctx)] + weight = node.args[2] if len(node.args) > 2 else getattr(node, "kwargs", {}).get("weight") + bias = node.args[3] if len(node.args) > 3 else getattr(node, "kwargs", {}).get("bias") + if not is_fx_node(weight): + ctx.fail(f"unsupported layer_norm weight form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + inputs.append(value_id(weight, ctx)) + if bias is None: + pass + elif is_fx_node(bias): + inputs.append(value_id(bias, ctx)) + else: + ctx.fail(f"unsupported layer_norm bias form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + + attrs = {"normalized_shape": tuple(int(v) for v in normalized_shape_value), "eps": float(eps_value)} + ir_node = IRNode( + id=node_id(node), + op="layer_norm", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_rms_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx)] + attrs = {"normalized_shape": tuple(int(v) for v in node.args[1]), "eps": float(node.args[3])} + ir_node = IRNode( + id=node_id(node), + op="rms_norm", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_group_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx), value_id(node.args[3], ctx)] + attrs = {"num_groups": int(node.args[1]), "eps": float(node.args[4])} + ir_node = IRNode( + id=node_id(node), + op="group_norm", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_batch_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [ + value_id(node.args[0], ctx), + value_id(node.args[1], ctx), + value_id(node.args[2], ctx), + value_id(node.args[3], ctx), + value_id(node.args[4], ctx), + ] + attrs = {"training": bool(node.args[5]), "momentum": float(node.args[6]), "eps": float(node.args[7])} + ir_node = IRNode( + id=node_id(node), + op="batch_norm", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_contiguous(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="contiguous", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + index_value = extract_literals(node.args[1]) + if not isinstance(index_value, int): + ctx.fail(f"unsupported getitem index for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="getitem", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"index": index_value}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +OP_IMPORTERS = { + "arange": import_arange, + "identity": import_identity, + "add": import_add, + "subtract": import_subtract, + "multiply": import_multiply, + "multiply_inplace": import_multiply_inplace, + "divide": import_divide, + "precision_cast": import_precision_cast, + "type_as": import_type_as, + "negate": import_negate, + "abs": import_unary("abs"), + "clamp": import_clamp, + "not_equal": import_not_equal, + "equal": import_equal, + "logical_and": import_binary("logical_and"), + "logical_or": import_binary("logical_or"), + "logical_not": import_unary("logical_not"), + "where": import_where, + "masked_scatter": import_masked_scatter, + "masked_fill": import_masked_fill, + "cos": import_unary("cos"), + "sin": import_unary("sin"), + "floor_divide": import_floor_divide, + "scalar_exp": import_unary("scalar_exp"), + "scalar_sqrt": import_unary("scalar_sqrt"), + "reciprocal": import_reciprocal, + "scalar_log": import_unary("scalar_log"), + "rsqrt": import_pow_rsqrt, + "reshape": import_reshape, + "flatten": import_flatten, + "unsqueeze": import_unsqueeze, + "squeeze": import_squeeze, + "expand": import_expand, + "one_hot": import_one_hot, + "tril": import_tril, + "unfold": import_unfold, + "numpy_T": import_numpy_t, + "transpose": import_transpose, + "permute": import_permute, + "movedim": import_movedim, + "matmul": import_matmul, + "linear": import_linear, + "addmm": import_addmm, + "relu": import_unary("relu"), + "silu": import_unary("silu"), + "gelu": import_unary("gelu"), + "gelu_erf": import_unary("gelu_erf"), + "sigmoid": import_unary("sigmoid"), + "glu": import_glu, + "softplus": import_unary("softplus"), + "tanh": import_unary("tanh"), + "softmax": import_softmax, + "scaled_dot_product_attention": import_scaled_dot_product_attention, + "sum": import_reduce("sum"), + "mean": import_reduce("mean"), + "variance": import_reduce("variance"), + "min": import_reduce("min"), + "max": import_reduce("max"), + "cat": import_cat, + "split_with_sizes": import_split_with_sizes, + "chunk": import_chunk, + "ones": import_ones, + "pad": import_pad, + "pow": import_pow, + "aten.diff.default": import_diff, + "slice": import_slice, + "index": import_index, + "gather": import_gather, + "embedding": import_embedding, + "conv1d": import_conv1d, + "conv2d": import_conv2d, + "layer_norm": import_layer_norm, + "rms_norm": import_rms_norm, + "group_norm": import_group_norm, + "batch_norm": import_batch_norm, + "contiguous": import_contiguous, + "getitem": import_getitem, +} diff --git a/python/src/transpile/lower.py b/python/src/transpile/lower.py new file mode 100644 index 000000000..5784e4cac --- /dev/null +++ b/python/src/transpile/lower.py @@ -0,0 +1,2142 @@ +from __future__ import annotations + +from dataclasses import dataclass +import os +from typing import Any + +import torch + +from src.graph import Graph +from src.graph import Tensor +from src.transpile.capture_pytorch import CapturedModel +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.optimize_graph import optimize_graph +from src.transpile.weight_binding import WeightBinding + + +@dataclass +class TranspiledGraph: + graph: Graph + runtime_inputs: list[Tensor] + bound_constants: list[Tensor] + outputs: list[Tensor] + + def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: + if index < 0 or index >= len(self.runtime_inputs): + raise IndexError( + f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" + ) + self.graph.set_input(self.runtime_inputs[index], data, dtype=dtype) + + def set_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> None: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" + ) + for index, value in enumerate(inputs): + self.set_input(index, value) + + def execute(self) -> list[Tensor]: + self.graph.execute() + return self.outputs + + +@dataclass +class BroadcastAlias: + tensor: Tensor + logical_shape: tuple[int, ...] + kind: str + + +def transpile_captured(captured: CapturedModel) -> TranspiledGraph: + return transpile_ir(captured.ir_graph) + + +def transpile_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + canonicalize_exported_graph(ir) + optimize_graph(ir) + g = Graph() + env: dict[str, Any] = {} + runtime_inputs: list[Tensor] = [] + bound_constants: list[Tensor] = [] + + for value_id in ir.inputs: + value = ir.values[value_id] + tensor = _lower_input_value(g, value) + env[value_id] = tensor + runtime_inputs.append(tensor) + + for value_id, const in ir.constants.items(): + value = ir.values[value_id] + lowered_const = _lower_constant_value(g, value, const) + env[value_id] = lowered_const + if isinstance(lowered_const, Tensor): + bound_constants.append(lowered_const) + + for node_id in ir.order: + node = ir.nodes[node_id] + outputs = _lower_ir_node(g, node, env, ir) + if len(outputs) != len(node.outputs): + raise ValueError( + f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" + ) + for output_id, tensor in zip(node.outputs, outputs): + env[output_id] = tensor + + outputs = [env[value_id] for value_id in ir.outputs] + return TranspiledGraph( + graph=g, + runtime_inputs=runtime_inputs, + bound_constants=bound_constants, + outputs=outputs, + ) + + +def _lower_input_value(g: Graph, value: IRValue) -> Tensor: + if value.shape is None or value.dtype is None: + raise ValueError(f"IR input missing shape or dtype: {value.id}") + return g.input(shape=value.shape, dtype=_map_ir_dtype(value.dtype)) + +#Need to work on this for mmmaps and constants that can be bound as tensors vs materialized as literals -- Fuck this shit I fucking hate my life +def _lower_constant_value(g: Graph, value: IRValue, const: Any) -> Any: + binding = _lookup_weight_binding(value) + if binding is not None: + _debug_mmap_binding(value.id, binding) + if binding.kind == "embedding": + print("generating embedding tensor") # Observe which shit actually has embeddings generated + return g.mmap_embeddings(binding.path) + return g.mmap_weights(binding.path) + + _debug_constant_fallback(value, const) + + if isinstance(const, torch.nn.Parameter): + const = const.detach() + if isinstance(const, torch.Tensor): + tensor_value = const.detach().cpu() + else: + raise NotImplementedError( + f"unsupported IR constant type for {value.id}: {type(const).__name__}" + ) + + if tensor_value.numel() == 1: + return tensor_value.item() + + return _materialize_constant_tensor(g, tensor_value) + + +def _lookup_weight_binding(value: IRValue) -> WeightBinding | None: + meta = getattr(value, "meta", None) + if isinstance(meta, dict): + path = meta.get("path") + kind = meta.get("kind") + source_name = meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + return WeightBinding(path=path, kind=kind, source_name=source_name) + return None + + +def _debug_mmap_binding(value_id: str, binding: WeightBinding) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + print( + "[transpile:mmap] " + f"value={value_id} " + f"kind={binding.kind} " + f"source={binding.source_name} " + f"path={binding.path}" + ) + + +def _debug_constant_fallback(value: IRValue, const: Any) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + const_type = type(const).__name__ + shape = None + dtype = None + if isinstance(const, torch.Tensor): + shape = tuple(const.shape) + dtype = str(const.dtype) + source_name = None + if isinstance(value.meta, dict): + source_name = value.meta.get("source_name") + print( + "[transpile:fallback] " + f"value={value.id} " + f"source={source_name} " + f"const_type={const_type} " + f"shape={shape} " + f"dtype={dtype}" + ) + + +def _debug_embedding_lowering(node: IRNode, embedding_tensor: Tensor, indices_tensor: Tensor) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + print( + "[transpile:embedding] " + f"node={node.id} " + f"embedding_id={embedding_tensor.id} " + f"embedding_shape={tuple(embedding_tensor.shape)} " + f"embedding_dtype={embedding_tensor.dtype} " + f"indices_id={indices_tensor.id} " + f"indices_shape={tuple(indices_tensor.shape)} " + f"indices_dtype={indices_tensor.dtype}" + ) + + +def _matmul_with_quantized_rhs_legalization( + g: Graph, + lhs: Tensor, + rhs: Tensor, + *, + pretransposed_rhs: bool = False, +) -> Tensor: + if rhs.dtype in (Graph.INT8, Graph.INT4) and lhs.dtype == Graph.FP32: + lhs = g.precision_cast(lhs, Graph.FP16) + return g.matmul(lhs, rhs, pretransposed_rhs=pretransposed_rhs) + + +def _legalize_for_transpose(g: Graph, x: Tensor) -> Tensor: + if x.dtype == Graph.FP32: + return g.precision_cast(x, Graph.FP16) + return x + + +def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> list[Any]: + op = node.op + + if op == "arange": + start = int(node.attrs.get("start", 0)) + end = int(node.attrs["end"]) + step = int(node.attrs.get("step", 1)) + dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) + tensor_value = torch.arange(start, end, step=step, dtype=dtype) + return [_materialize_constant_tensor(g, tensor_value)] + + if op == "add": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "add")] + + if op == "add_clipped": + lhs, rhs = _legalize_elementwise_binary_inputs(g, _tensor(env, node.inputs[0]), _tensor(env, node.inputs[1])) + return [g.add_clipped(lhs, rhs)] + + if op == "subtract": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "subtract")] + + if op == "multiply": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] + + if op == "multiply_inplace": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] + + if op == "divide": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "divide")] + + if op == "not_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "not_equal")] + + if op == "equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "equal")] + + if op == "greater": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "greater")] + + if op == "less_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "less_equal")] + + if op == "logical_and": + lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + return [_lower_binary_op(g, lhs, rhs, "multiply")] + + if op == "logical_or": + lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + return [_lower_compare_op(g, _lower_binary_op(g, lhs, rhs, "add"), 0.0, "not_equal")] + + if op == "logical_not": + return [_lower_compare_op(g, env[node.inputs[0]], 0.0, "equal")] + + if op == "where": + return [_lower_where_op(g, node, env)] + + if op == "scalar_add": + return [g.scalar_add(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] + + if op == "scalar_subtract": + return [g.scalar_subtract(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] + + if op == "scalar_subtract_reverse": + x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_add(g.scalar_multiply(x, -1.0), float(node.attrs["value"]))] + + if op == "scalar_multiply": + return [g.scalar_multiply(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] + + if op == "scalar_divide": + return [g.scalar_divide(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] + + if op == "scalar_floor_divide": + source = _tensor(env, node.inputs[0]) + x = _ensure_fp16_tensor(g, source) + y = g.scalar_floor_divide(x, float(node.attrs["value"])) + output_dtype = ir.values[node.outputs[0]].dtype + if output_dtype is None: + return [y] + target_dtype = _map_ir_dtype(output_dtype) + if y.dtype == target_dtype: + return [y] + return [g.precision_cast(y, target_dtype)] + + if op == "scalar_not_equal": + return [g.scalar_not_equal(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] + + if op == "scalar_equal": + not_equal = g.scalar_not_equal(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"])) + return [g.scalar_not_equal(not_equal, 1.0)] + + if op == "scalar_divide_reverse": + raise NotImplementedError("scalar_divide_reverse is not directly supported by Cactus graph ops") + + if op == "precision_cast": + target_dtype = node.attrs.get("dtype") + if target_dtype is None: + return [_tensor(env, node.inputs[0])] + return [g.precision_cast(_tensor(env, node.inputs[0]), _map_ir_or_torch_dtype(target_dtype))] + + if op == "type_as": + source = _tensor(env, node.inputs[0]) + target = _tensor(env, node.inputs[1]) + return [g.precision_cast(source, target.dtype)] + + if op == "abs": + return [g.abs(_tensor(env, node.inputs[0]))] + + if op == "clamp": + x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) + min_value = node.attrs.get("min") + max_value = node.attrs.get("max") + if min_value is not None: + x = g.scalar_add(g.relu(g.scalar_subtract(x, float(min_value))), float(min_value)) + if max_value is not None: + upper_delta = g.relu(g.scalar_add(g.scalar_multiply(x, -1.0), float(max_value))) + x = g.scalar_add(g.scalar_multiply(upper_delta, -1.0), float(max_value)) + return [x] + + if op == "negate": + return [g.scalar_multiply(_tensor(env, node.inputs[0]), -1.0)] + + if op == "pow": + exponent = node.attrs.get("exponent") + if exponent is None: + raise NotImplementedError("tensor-tensor pow is not supported by Cactus graph ops") + return [g.pow(_tensor(env, node.inputs[0]), float(exponent))] + + if op == "scalar_exp": + return [g.scalar_exp(_tensor(env, node.inputs[0]))] + + if op == "scalar_sqrt": + return [g.scalar_sqrt(_tensor(env, node.inputs[0]))] + + if op == "scalar_log": + return [g.scalar_log(_tensor(env, node.inputs[0]))] + + if op == "scalar_cos": + return [g.scalar_cos(_tensor(env, node.inputs[0]))] + + if op == "scalar_sin": + return [g.scalar_sin(_tensor(env, node.inputs[0]))] + + if op in {"reshape", "view"}: + source = env[node.inputs[0]] + if isinstance(source, BroadcastAlias): + input_shape = tuple(source.logical_shape) + else: + input_shape = tuple(_tensor(env, node.inputs[0]).shape) + target_shape = _resolve_reshape_shape(input_shape, tuple(node.attrs["shape"])) + if isinstance(source, BroadcastAlias): + if source.kind != "gqa_repeat_kv": + raise NotImplementedError(f"unsupported broadcast alias kind in reshape: {source.kind}") + base_shape = tuple(source.tensor.shape) + if len(base_shape) != 4: + raise NotImplementedError(f"gqa_repeat_kv alias requires 4D base tensor, got {base_shape}") + if len(target_shape) != 4: + raise NotImplementedError(f"gqa_repeat_kv reshape requires 4D target shape, got {target_shape}") + batch, kv_heads, seq_len, head_dim = base_shape + if target_shape[0] != batch or target_shape[2] != seq_len or target_shape[3] != head_dim: + raise NotImplementedError( + f"gqa_repeat_kv reshape mismatch: base {base_shape}, target {target_shape}" + ) + if target_shape[1] % max(kv_heads, 1) != 0: + raise NotImplementedError( + f"gqa_repeat_kv target head count must be a multiple of kv heads: {base_shape} -> {target_shape}" + ) + return [BroadcastAlias(source.tensor, target_shape, source.kind)] + return [g.reshape(_tensor(env, node.inputs[0]), target_shape)] + + if op == "flatten": + return [ + g.flatten( + _tensor(env, node.inputs[0]), + start_dim=int(node.attrs["start_dim"]), + end_dim=int(node.attrs["end_dim"]), + ) + ] + + if op == "unsqueeze": + x = _tensor(env, node.inputs[0]) + target_shape = node.attrs.get("shape") + if target_shape is None: + dim = _normalize_dim(int(node.attrs["dim"]), len(x.shape) + 1) + shape_list = list(x.shape) + shape_list.insert(dim, 1) + target_shape = tuple(shape_list) + return [g.reshape(x, tuple(target_shape))] + + if op == "expand": + matched_base = _match_gqa_expand_alias(ir, node) + if matched_base is not None: + return [BroadcastAlias(_tensor(env, matched_base), tuple(node.attrs["shape"]), "gqa_repeat_kv")] + x = _tensor(env, node.inputs[0]) + target_shape = _resolve_expand_shape(tuple(x.shape), tuple(node.attrs["shape"])) + if target_shape == tuple(x.shape): + return [x] + return [g.expand(x, target_shape)] + + if op == "one_hot": + indices = _tensor(env, node.inputs[0]) + num_classes = int(node.attrs["num_classes"]) + eye = torch.eye(num_classes, dtype=torch.float16) + embedding = _materialize_constant_tensor(g, eye) + return [g.embedding_from_tensor(embedding, indices)] + + if op == "transpose": + x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + dim0 = _normalize_dim(int(node.attrs["dim0"]), len(x.shape)) + dim1 = _normalize_dim(int(node.attrs["dim1"]), len(x.shape)) + rank = len(x.shape) + permutation = list(range(rank)) + permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] + if rank == 2 and permutation == [1, 0]: + return [g.transpose(x)] + return [g.permute(x, permutation)] + + if op == "permute": + x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + permutation = tuple(_normalize_dim(int(dim), len(x.shape)) for dim in node.attrs["permutation"]) + if len(permutation) == 2 and permutation == (1, 0): + return [g.transpose(x)] + return [g.permute(x, permutation)] + + if op == "matmul": + lhs = _tensor(env, node.inputs[0]) + rhs = _tensor(env, node.inputs[1]) + if len(lhs.shape) == 2 and len(rhs.shape) == 2: + return [_matmul_with_quantized_rhs_legalization(g, lhs, rhs)] + legalized = _legalize_matmul_inputs(g, lhs, rhs, node) + if legalized is None: + static_batched = _lower_static_batched_matmul(g, lhs, rhs) + if static_batched is not None: + return [static_batched] + return [_matmul_with_quantized_rhs_legalization(g, lhs, rhs)] + lhs_2d, rhs_2d, output_shape = legalized + out = _matmul_with_quantized_rhs_legalization(g, lhs_2d, rhs_2d) + return [g.reshape(out, output_shape)] + + if op == "linear": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + reshape_back: tuple[int, ...] | None = None + if len(x.shape) > 2: + x = _flatten_to_2d_for_linear(g, x) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"linear missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, x, weight, pretransposed_rhs=True) + if node.attrs.get("has_bias"): + out = g.add(out, _tensor(env, node.inputs[2])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "addmm": + bias = _tensor(env, node.inputs[0]) + lhs = _tensor(env, node.inputs[1]) + rhs = _tensor(env, node.inputs[2]) + return [g.add(bias, _matmul_with_quantized_rhs_legalization(g, lhs, rhs))] + + if op == "relu": + return [g.relu(_tensor(env, node.inputs[0]))] + + if op == "silu": + return [g.silu(_tensor(env, node.inputs[0]))] + + if op == "gelu": + return [g.gelu(_tensor(env, node.inputs[0]))] + + if op == "gelu_erf": + return [g.gelu_erf(_tensor(env, node.inputs[0]))] + + if op == "sigmoid": + return [g.sigmoid(_tensor(env, node.inputs[0]))] + + if op == "glu": + return [g.glu(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] + + if op == "softplus": + x = _tensor(env, node.inputs[0]) + return [_lower_softplus(g, x)] + + if op == "tanh": + return [g.tanh(_tensor(env, node.inputs[0]))] + + if op == "softmax": + return [g.softmax(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] + + if op in {"scaled_dot_product_attention", "attention"}: + mask = node.attrs.get("mask") + mask_tensor: Tensor | None = None + additive_mask = bool(node.attrs.get("additive_mask", False)) + if len(node.inputs) > 3: + mask_tensor = _tensor(env, node.inputs[3]) + elif mask is not None: + raise NotImplementedError(f"{op} with literal mask is not supported yet") + if op == "scaled_dot_product_attention": + dropout_p = float(node.attrs.get("dropout_p", 0.0)) + if dropout_p != 0.0: + raise NotImplementedError("scaled_dot_product_attention with dropout is not supported yet") + + # PyTorch SDPA exports tensors as [batch, heads, seq, dim], while Cactus attention + # expects [batch, seq, heads, dim]. Convert into Cactus layout, call attention, then + # convert back so downstream exported ops still see PyTorch layout. + query = g.permute(_attention_tensor(env, node.inputs[0]), (0, 2, 1, 3)) + key = g.permute(_attention_tensor(env, node.inputs[1]), (0, 2, 1, 3)) + value = g.permute(_attention_tensor(env, node.inputs[2]), (0, 2, 1, 3)) + out = g.attention( + query, + key, + value, + scale=float(node.attrs.get("scale", 1.0)), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=additive_mask, + ) + return [g.permute(out, (0, 2, 1, 3))] + + if op == "attention_block": + has_mask = bool(node.attrs.get("has_mask", False)) + has_gate = bool(node.attrs.get("has_gate", False)) + has_bias = bool(node.attrs.get("has_bias", False)) + input_index = 0 + query = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + key = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + value = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + mask_tensor: Tensor | None = None + if has_mask: + mask_tensor = _tensor(env, node.inputs[input_index]) + input_index += 1 + + attn_out = g.attention( + query, + key, + value, + scale=float(node.attrs.get("scale", 1.0)), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=bool(node.attrs.get("additive_mask", False)), + ) + flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) + flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () + if flat_shape: + attn_out = g.reshape(attn_out, flat_shape) + + if has_gate: + gate = _tensor(env, node.inputs[input_index]) + input_index += 1 + attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) + attn_out = g.multiply(attn_out, gate) + + weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + reshape_back: tuple[int, ...] | None = None + linear_input = attn_out + if len(linear_input.shape) > 2: + linear_input = _flatten_to_2d_for_linear(g, linear_input) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"attention_block missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) + if has_bias: + out = g.add(out, _tensor(env, node.inputs[input_index])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "self_attention_block": + has_mask = bool(node.attrs.get("has_mask", False)) + has_gate = bool(node.attrs.get("has_gate", False)) + has_bias = bool(node.attrs.get("has_bias", False)) + has_query_projection_bias = bool(node.attrs.get("has_query_projection_bias", False)) + has_query_add = bool(node.attrs.get("has_query_add", False)) + has_rel_query_add = bool(node.attrs.get("has_rel_query_add", False)) + has_key_projection_bias = bool(node.attrs.get("has_key_projection_bias", False)) + has_value_projection_bias = bool(node.attrs.get("has_value_projection_bias", False)) + has_rel_pos_bias = bool(node.attrs.get("has_rel_pos_bias", False)) + has_relative_key_projection_bias = bool(node.attrs.get("has_relative_key_projection_bias", False)) + + input_index = 0 + hidden = _tensor(env, node.inputs[input_index]) + input_index += 1 + + query_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + query_projection_bias = None + if has_query_projection_bias: + query_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + query_add = None + if has_query_add: + query_add = _tensor(env, node.inputs[input_index]) + input_index += 1 + rel_query_add = None + if has_rel_query_add: + rel_query_add = _tensor(env, node.inputs[input_index]) + input_index += 1 + + key_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + key_projection_bias = None + if has_key_projection_bias: + key_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + value_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + value_projection_bias = None + if has_value_projection_bias: + value_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + mask_tensor: Tensor | None = None + if has_mask: + mask_tensor = _tensor(env, node.inputs[input_index]) + input_index += 1 + + relative_key_input = None + relative_key_weight = None + relative_key_projection_bias = None + if has_rel_pos_bias: + relative_key_input = _tensor(env, node.inputs[input_index]) + input_index += 1 + relative_key_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + if has_relative_key_projection_bias: + relative_key_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + gate = None + if has_gate: + gate = _tensor(env, node.inputs[input_index]) + input_index += 1 + + output_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + output_bias = None + if has_bias: + output_bias = _tensor(env, node.inputs[input_index]) + + query_shape = tuple(int(v) for v in node.attrs.get("query_shape", ())) + key_shape = tuple(int(v) for v in node.attrs.get("key_shape", ())) + value_shape = tuple(int(v) for v in node.attrs.get("value_shape", ())) + relative_key_shape = tuple(int(v) for v in node.attrs.get("relative_key_shape", ())) + + query_base = _lower_projected_attention_tensor(g, hidden, query_weight, query_projection_bias, query_shape) + query = query_base + if query_add is not None: + query_add = _normalize_attention_add_tensor(g, query_add, query_shape) + query, query_add = _legalize_elementwise_binary_inputs(g, query, query_add) + query = g.add(query, query_add) + + key = _lower_projected_attention_tensor(g, hidden, key_weight, key_projection_bias, key_shape) + value = _lower_projected_attention_tensor(g, hidden, value_weight, value_projection_bias, value_shape) + + if has_rel_pos_bias: + if relative_key_input is None or relative_key_weight is None or not relative_key_shape: + raise NotImplementedError(f"self_attention_block missing relative position bias inputs for node {node.id}") + rel_query = query_base + if rel_query_add is not None: + rel_query_add = _normalize_attention_add_tensor(g, rel_query_add, query_shape) + rel_query, rel_query_add = _legalize_elementwise_binary_inputs(g, rel_query, rel_query_add) + rel_query = g.add(rel_query, rel_query_add) + relative_key = _lower_projected_attention_tensor( + g, + relative_key_input, + relative_key_weight, + relative_key_projection_bias, + relative_key_shape, + ) + rel_mask = g.rel_pos_bias(rel_query, relative_key, float(node.attrs.get("rel_pos_scale", 1.0))) + if mask_tensor is None: + mask_tensor = rel_mask + else: + mask_tensor, rel_mask = _legalize_elementwise_binary_inputs(g, mask_tensor, rel_mask) + mask_tensor = g.add(mask_tensor, rel_mask) + + attn_out = g.attention( + query, + key, + value, + scale=float(node.attrs.get("scale", 1.0)), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=bool(node.attrs.get("additive_mask", False)), + ) + flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) + flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () + if flat_shape: + attn_out = g.reshape(attn_out, flat_shape) + + if gate is not None: + attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) + attn_out = g.multiply(attn_out, gate) + + linear_input = attn_out + reshape_back: tuple[int, ...] | None = None + if len(linear_input.shape) > 2: + linear_input = _flatten_to_2d_for_linear(g, linear_input) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"self_attention_block missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, linear_input, output_weight, pretransposed_rhs=True) + if output_bias is not None: + out = g.add(out, output_bias) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op in ("sum", "mean", "variance", "min", "max"): + axis = node.attrs.get("axis") + if isinstance(axis, (list, tuple)) and len(axis) == 1 and isinstance(axis[0], int): + axis = int(axis[0]) + if not isinstance(axis, int): + raise NotImplementedError(f"{op} currently requires a single integer axis") + fn = getattr(g, op) + x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) + out = fn(x, axis) + if bool(node.attrs.get("keepdim", False)): + reduced_axis = _normalize_dim(axis, len(x.shape)) + new_shape = list(x.shape) + new_shape[reduced_axis] = 1 + out = g.reshape(out, tuple(new_shape)) + return [out] + + if op == "cat": + tensors = [_tensor(env, value_id) for value_id in node.inputs] + output_dtype: int | None = None + if node.outputs: + output_value = ir.values.get(node.outputs[0]) + if output_value is not None and output_value.dtype is not None: + output_dtype = _map_ir_or_torch_dtype(output_value.dtype) + return [_cat_with_legalized_dtype(g, tensors, axis=int(node.attrs.get("axis", 0)), dtype=output_dtype)] + + if op in {"stack", "aten.stack.default"}: + tensors = [_tensor(env, value_id) for value_id in node.inputs] + if not tensors: + raise NotImplementedError("stack requires at least one input tensor") + + raw_axis = node.attrs.get("axis") + if not isinstance(raw_axis, int): + args = node.attrs.get("args", ()) + if isinstance(args, (list, tuple)) and len(args) > 1 and isinstance(args[1], int): + raw_axis = int(args[1]) + else: + raw_axis = 0 + + base_shape = tuple(int(dim) for dim in tensors[0].shape) + axis = _normalize_dim(int(raw_axis), len(base_shape) + 1) + reshaped: list[Tensor] = [] + for tensor in tensors: + if tuple(int(dim) for dim in tensor.shape) != base_shape: + raise NotImplementedError( + f"stack requires equal input shapes, got {[tuple(int(dim) for dim in t.shape) for t in tensors]}" + ) + expanded_shape = list(base_shape) + expanded_shape.insert(axis, 1) + reshaped.append(g.reshape(tensor, tuple(expanded_shape))) + return [g.cat(reshaped, axis=axis)] + + if op == "slice": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs["axis"]), len(x.shape)) + start = int(node.attrs["start"]) + end = int(node.attrs["end"]) + step = int(node.attrs.get("step", 1)) + dim_size = x.shape[axis] + if step == 0: + raise NotImplementedError("slice with step == 0 is invalid") + if step != 1: + indices = list(range(*slice(start, end, step).indices(dim_size))) + if not indices: + return [g.slice(x, axis=axis, start=0, length=0)] + expanded_shape = list(int(dim) for dim in x.shape) + expanded_shape[axis] = 1 + pieces: list[Tensor] = [] + for idx in indices: + piece = g.index(x, index_value=idx, axis=axis) + piece = g.reshape(piece, tuple(expanded_shape)) + pieces.append(piece) + if len(pieces) == 1: + return [pieces[0]] + return [g.cat(pieces, axis=axis)] + start = _normalize_index(start, dim_size) + end = _normalize_slice_end(end, dim_size) + length = max(0, end - start) + return [g.slice(x, axis=axis, start=start, length=length)] + + if op == "split_with_sizes": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", -1)), len(x.shape)) + sizes = tuple(int(v) for v in node.attrs["sizes"]) + start = 0 + outputs: list[Tensor] = [] + for size in sizes: + outputs.append(g.slice(x, axis=axis, start=start, length=int(size))) + start += int(size) + return [outputs] + + if op == "chunk": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) + chunks = int(node.attrs["chunks"]) + if chunks <= 0: + raise NotImplementedError(f"chunk requires a positive chunk count, got {chunks}") + dim_size = int(x.shape[axis]) + if dim_size <= 0: + return [[g.slice(x, axis=axis, start=0, length=0)]] + chunk_size = (dim_size + chunks - 1) // chunks + outputs: list[Tensor] = [] + start = 0 + while start < dim_size: + length = min(chunk_size, dim_size - start) + outputs.append(g.slice(x, axis=axis, start=start, length=length)) + start += length + return [outputs] + + if op == "ones": + shape = tuple(int(v) for v in node.attrs.get("shape", ())) + if not shape: + raise NotImplementedError(f"ones requires a static shape, got {shape}") + torch_dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) + return [_materialize_constant_tensor(g, torch.ones(shape, dtype=torch_dtype))] + + if op == "tril": + x = _tensor(env, node.inputs[0]) + shape = tuple(int(dim) for dim in x.shape) + if len(shape) < 2: + raise NotImplementedError(f"tril expects rank >= 2, got shape {shape}") + diagonal = int(node.attrs.get("diagonal", 0)) + mask_dtype = _torch_dtype_for_graph_dtype(x.dtype) + mask = torch.ones(shape, dtype=mask_dtype).tril(diagonal=diagonal) + mask_tensor = _materialize_constant_tensor(g, mask) + return [_lower_binary_op(g, x, mask_tensor, "multiply")] + + if op == "unfold": + x = _tensor(env, node.inputs[0]) + rank = len(x.shape) + axis = _normalize_dim(int(node.attrs["dimension"]), rank) + size = int(node.attrs["size"]) + step = int(node.attrs["step"]) + if size <= 0: + raise NotImplementedError(f"unfold requires a positive size, got {size}") + if step <= 0: + raise NotImplementedError(f"unfold requires a positive step, got {step}") + dim_size = int(x.shape[axis]) + if size > dim_size: + raise NotImplementedError( + f"unfold size {size} exceeds dimension size {dim_size} for axis {axis}" + ) + + window_starts = list(range(0, dim_size - size + 1, step)) + if not window_starts: + raise NotImplementedError( + f"unfold produced no windows for axis={axis}, size={size}, step={step}, dim_size={dim_size}" + ) + + base_shape = [int(dim) for dim in x.shape] + trailing_shape = [base_shape[idx] for idx in range(rank) if idx != axis] + [size] + expanded_shape = tuple(trailing_shape[:axis] + [1] + trailing_shape[axis:]) + permutation = tuple(idx for idx in range(rank) if idx != axis) + (axis,) + + windows: list[Tensor] = [] + for start in window_starts: + piece = g.slice(x, axis=axis, start=start, length=size) + if axis != rank - 1: + piece = g.permute(piece, permutation=permutation) + piece = g.reshape(piece, expanded_shape) + windows.append(piece) + if len(windows) == 1: + return [windows[0]] + return [g.cat(windows, axis=axis)] + + if op == "pad": + x = _tensor(env, node.inputs[0]) + mode = str(node.attrs.get("mode", "constant")) + if mode != "constant": + raise NotImplementedError(f"pad mode is unsupported: {mode}") + pads = tuple(int(v) for v in node.attrs.get("pads", ())) + if len(pads) % 2 != 0: + raise NotImplementedError(f"pad expects an even-length pads tuple, got {pads}") + value = float(node.attrs.get("value", 0.0)) + + current = x + current_shape = list(int(dim) for dim in x.shape) + pad_dims = len(pads) // 2 + if pad_dims > len(current_shape): + raise NotImplementedError( + f"pad rank mismatch: pads={pads} for input shape {tuple(current_shape)}" + ) + + for pad_index in range(pad_dims): + before = pads[2 * pad_index] + after = pads[2 * pad_index + 1] + axis = len(current_shape) - 1 - pad_index + if before < 0 or after < 0: + raise NotImplementedError(f"negative pad is unsupported: {pads}") + pieces = [] + torch_dtype = _torch_dtype_for_graph_dtype(current.dtype) + if before > 0: + left_shape = list(current_shape) + left_shape[axis] = before + left = _materialize_constant_tensor( + g, + torch.full(tuple(left_shape), value, dtype=torch_dtype), + ) + pieces.append(left) + pieces.append(current) + if after > 0: + right_shape = list(current_shape) + right_shape[axis] = after + right = _materialize_constant_tensor( + g, + torch.full(tuple(right_shape), value, dtype=torch_dtype), + ) + pieces.append(right) + if len(pieces) > 1: + current = g.cat(pieces, axis=axis) + current_shape[axis] += before + after + return [current] + + if op == "index": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) + index_value = _normalize_index(int(node.attrs["index_value"]), x.shape[axis]) + return [g.index(x, index_value=index_value, axis=axis)] + + if op == "gather": + return [g.gather(_tensor(env, node.inputs[0]), _tensor(env, node.inputs[1]))] + + if op == "embedding": + embedding_tensor = _tensor(env, node.inputs[0]) + indices_tensor = _tensor(env, node.inputs[1]) + _debug_embedding_lowering(node, embedding_tensor, indices_tensor) + return [g.embedding_from_tensor(embedding_tensor, indices_tensor)] + + if op == "conv_module": + input_index = 0 + x_nlc = _tensor(env, node.inputs[input_index]) + input_index += 1 + + pointwise1_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + pointwise1_bias = None + if bool(node.attrs.get("has_pointwise1_bias", False)): + pointwise1_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + depthwise_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + depthwise_bias = None + if bool(node.attrs.get("has_depthwise_bias", False)): + depthwise_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + batch_norm_weight = _tensor(env, node.inputs[input_index]) + batch_norm_bias = _tensor(env, node.inputs[input_index + 1]) + batch_norm_running_mean = _tensor(env, node.inputs[input_index + 2]) + batch_norm_running_var = _tensor(env, node.inputs[input_index + 3]) + input_index += 4 + + pointwise2_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + pointwise2_bias = None + if bool(node.attrs.get("has_pointwise2_bias", False)): + pointwise2_bias = _tensor(env, node.inputs[input_index]) + + if len(x_nlc.shape) != 3: + raise NotImplementedError(f"conv_module expects rank-3 NLC input, got {x_nlc.shape}") + kernel_size = int(node.attrs.get("depthwise_kernel_size", 0)) + padding = int(node.attrs.get("depthwise_padding", 0)) + if len(depthwise_weight.shape) != 3 or kernel_size != 9 or padding != 4: + raise NotImplementedError( + f"conv_module currently supports same depthwise kernel-9 lowering, got " + f"kernel_size={kernel_size} padding={padding} weight_shape={depthwise_weight.shape}" + ) + + current = g.conv1d_pointwise(x_nlc, pointwise1_weight, bias=pointwise1_bias) + current = g.glu(current, axis=-1) + current = g.conv1d_same_depthwise_k9(current, depthwise_weight, bias=depthwise_bias) + current = g.batch_norm( + current, + batch_norm_weight, + batch_norm_bias, + batch_norm_running_mean, + batch_norm_running_var, + axis=len(current.shape) - 1, + eps=float(node.attrs.get("eps", 1e-5)), + ) + current = g.silu(current) + current = g.conv1d_pointwise(current, pointwise2_weight, bias=pointwise2_bias) + return [current] + + if op == "conv1d": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + stride = int(node.attrs.get("stride", 1)) + padding = int(node.attrs.get("padding", 0)) + dilation = int(node.attrs.get("dilation", 1)) + groups = int(node.attrs.get("groups", 1)) + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and weight.shape[2] == 1 + and stride == 1 + and padding == 0 + and dilation == 1 + and groups == 1 + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_pointwise(x_nlc, weight, bias=bias) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and weight.shape[2] == 9 + and stride == 1 + and padding == 4 + and dilation == 1 + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_same_depthwise_k9(x_nlc, weight, bias=bias) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and stride == 1 + and padding == dilation * max(weight.shape[2] - 1, 0) + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_causal(x_nlc, weight, kernel_size=weight.shape[2], dilation=dilation) + if bias is not None: + bias_reshaped = g.reshape(bias, (1, 1, int(weight.shape[0]))) + out_nlc, bias_reshaped = _legalize_elementwise_binary_inputs(g, out_nlc, bias_reshaped) + out_nlc = g.add(out_nlc, bias_reshaped) + return [g.permute(out_nlc, (0, 2, 1))] + + if dilation != 1: + raise NotImplementedError(f"conv1d with dilation != 1 is unsupported by generic lowering: {dilation}") + if padding != 0: + if len(x.shape) != 3: + raise NotImplementedError(f"generic conv1d padding expects rank-3 input, got {x.shape}") + batch_size, channels, _ = (int(dim) for dim in x.shape) + pad_dtype = _torch_dtype_for_graph_dtype(x.dtype) + pieces: list[Tensor] = [] + if padding > 0: + left = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, padding), dtype=pad_dtype), + ) + pieces.append(left) + pieces.append(x) + if padding > 0: + right = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, padding), dtype=pad_dtype), + ) + pieces.append(right) + x = g.cat(pieces, axis=2) + if groups != 1: + if len(x.shape) != 3 or len(weight.shape) != 3: + raise NotImplementedError( + f"grouped conv1d lowering expects rank-3 input and weight, got {x.shape} and {weight.shape}" + ) + input_channels = int(x.shape[1]) + if input_channels % groups != 0: + raise NotImplementedError( + f"grouped conv1d requires input channels divisible by groups, got C_in={input_channels}, groups={groups}" + ) + output_channels = int(weight.shape[0]) + if output_channels % groups != 0: + raise NotImplementedError( + f"grouped conv1d requires output channels divisible by groups, got C_out={output_channels}, groups={groups}" + ) + input_channels_per_group = input_channels // groups + output_channels_per_group = output_channels // groups + if int(weight.shape[1]) != input_channels_per_group: + raise NotImplementedError( + "grouped conv1d weight has incompatible per-group input channels: " + f"weight C_in/group={int(weight.shape[1])}, expected {input_channels_per_group}" + ) + outputs: list[Tensor] = [] + for group_index in range(groups): + x_group = g.slice( + x, + axis=1, + start=group_index * input_channels_per_group, + length=input_channels_per_group, + ) + weight_group = g.slice( + weight, + axis=0, + start=group_index * output_channels_per_group, + length=output_channels_per_group, + ) + bias_group = None + if bias is not None: + bias_group = g.slice( + bias, + axis=0, + start=group_index * output_channels_per_group, + length=output_channels_per_group, + ) + outputs.append(g.conv1d(x_group, weight_group, bias=bias_group, stride=stride)) + if len(outputs) == 1: + return [outputs[0]] + return [g.cat(outputs, axis=1)] + return [g.conv1d(x, weight, bias=bias, stride=stride)] + + if op == "conv2d": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + stride = int(node.attrs.get("stride", 1)) + padding = int(node.attrs.get("padding", 0)) + dilation = int(node.attrs.get("dilation", 1)) + groups = int(node.attrs.get("groups", 1)) + + if len(x.shape) != 4 or len(weight.shape) != 4: + raise NotImplementedError(f"conv2d lowering expects rank-4 tensors, got {x.shape} and {weight.shape}") + if dilation != 1: + raise NotImplementedError(f"conv2d with dilation != 1 is unsupported: {dilation}") + + if stride == 2 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): + return [g.conv2d_k3s2p1(x, weight, bias=bias)] + + if ( + stride == 2 + and padding == 1 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and weight.shape[2:] == (3, 3) + ): + return [g.conv2d_depthwise_k3s2p1(x, weight, bias=bias)] + + if ( + stride == 1 + and padding == 0 + and groups == 1 + and weight.shape[2:] == (1, 1) + ): + return [g.conv2d_pointwise_1x1(x, weight, bias=bias)] + + if stride == 1 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): + return [g.conv2d_k3s1p1(x, weight, bias=bias)] + + return [g.conv2d(x, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups)] + + if op == "layer_norm": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + return [g.layer_norm(x, weight, bias=bias, eps=float(node.attrs["eps"]))] + + if op == "rms_norm": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + reshape_back: tuple[int, ...] | None = None + if len(x.shape) > 2: + reshape_back = tuple(int(dim) for dim in x.shape) + x = _flatten_to_2d_for_linear(g, x) + out = g.rms_norm(x, weight, eps=float(node.attrs["eps"])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "rope": + rope_input = _tensor(env, node.inputs[0]) + if _rope_input_is_bhsd(ir, node.inputs[0]): + rope_input = g.permute(rope_input, (0, 2, 1, 3)) + rope_out = g.rope( + rope_input, + float(node.attrs["theta"]), + position_offset=int(node.attrs.get("position_offset", 0)), + ) + return [g.permute(rope_out, (0, 2, 1, 3))] + return [ + g.rope( + rope_input, + float(node.attrs["theta"]), + position_offset=int(node.attrs.get("position_offset", 0)), + ) + ] + + if op == "rel_pos_bias": + query = _attention_tensor(env, node.inputs[0]) + if len(query.shape) != 4: + raise NotImplementedError(f"rel_pos_bias expects rank-4 query input, got {query.shape}") + query = g.permute(query, (0, 2, 1, 3)) + relative_key = _tensor(env, node.inputs[1]) + return [g.rel_pos_bias(query, relative_key, float(node.attrs.get("scale", 1.0)))] + + if op in {"gated_deltanet_prefill", "gated_deltanet_decode"}: + x = _tensor(env, node.inputs[0]) + qkv_weight = _tensor(env, node.inputs[1]) + a_weight = _tensor(env, node.inputs[2]) + b_weight = _tensor(env, node.inputs[3]) + norm_weight = _tensor(env, node.inputs[4]) + + input_index = 5 + z_weight = None + if bool(node.attrs.get("has_z", False)): + z_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + dt_bias = None + if bool(node.attrs.get("has_dt_bias", False)): + dt_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + a_log = None + if bool(node.attrs.get("has_a_log", False)): + a_log = _tensor(env, node.inputs[input_index]) + input_index += 1 + conv_weight = None + if bool(node.attrs.get("has_conv", False)): + conv_weight = _tensor(env, node.inputs[input_index]) + + if len(x.shape) != 3: + raise NotImplementedError(f"{op} currently expects rank-3 normalized input, got {x.shape}") + + batch_size, seq_len, hidden_dim = (int(dim) for dim in x.shape) + if batch_size != 1: + raise NotImplementedError(f"{op} currently supports batch size 1, got {x.shape}") + + num_k_heads = int(node.attrs["num_k_heads"]) + num_v_heads = int(node.attrs["num_v_heads"]) + key_dim = int(node.attrs["key_dim"]) + value_dim = int(node.attrs["value_dim"]) + eps = float(node.attrs.get("eps", 1e-6)) + chunk_size = int(node.attrs.get("chunk_size", 64)) + + mixed_qkv = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + qkv_weight, + pretransposed_rhs=True, + ) + mixed_qkv_dim = int(qkv_weight.shape[0]) + mixed_qkv = g.reshape(mixed_qkv, (batch_size, seq_len, mixed_qkv_dim)) + if conv_weight is not None: + kernel_size = int(conv_weight.shape[2]) + mixed_qkv = g.conv1d_causal(mixed_qkv, conv_weight, kernel_size=kernel_size, dilation=1) + mixed_qkv = g.silu(mixed_qkv) + + q_proj_dim = num_k_heads * key_dim + v_proj_dim = num_v_heads * value_dim + k_proj_dim = num_k_heads * key_dim + q_proj = g.slice(mixed_qkv, axis=2, start=0, length=q_proj_dim) + k_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim, length=k_proj_dim) + v_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim + k_proj_dim, length=v_proj_dim) + + q_4d = g.reshape(q_proj, (batch_size, seq_len, num_k_heads, key_dim)) + k_4d = g.reshape(k_proj, (batch_size, seq_len, num_k_heads, key_dim)) + v_4d = g.reshape(v_proj, (batch_size, seq_len, num_v_heads, value_dim)) + + q_norm = g.sum(g.multiply(q_4d, q_4d), axis=3) + q_norm = g.scalar_sqrt(g.scalar_add(q_norm, eps)) + q_norm = g.reshape(q_norm, (batch_size, seq_len, num_k_heads, 1)) + q_4d = g.divide(q_4d, q_norm) + + k_norm = g.sum(g.multiply(k_4d, k_4d), axis=3) + k_norm = g.scalar_sqrt(g.scalar_add(k_norm, eps)) + k_norm = g.reshape(k_norm, (batch_size, seq_len, num_k_heads, 1)) + k_4d = g.divide(k_4d, k_norm) + + a_logits = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + a_weight, + pretransposed_rhs=True, + ) + a_logits = g.reshape(a_logits, (batch_size, seq_len, int(a_weight.shape[0]))) + b_logits = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + b_weight, + pretransposed_rhs=True, + ) + b_logits = g.reshape(b_logits, (batch_size, seq_len, int(b_weight.shape[0]))) + + if dt_bias is not None: + dt_bias_2d = g.reshape(dt_bias, (1, int(dt_bias.shape[0]))) + a_logits, dt_bias_2d = _legalize_elementwise_binary_inputs(g, a_logits, dt_bias_2d) + a_logits = g.add(a_logits, dt_bias_2d) + a_softplus = _lower_softplus(g, a_logits) + + if a_log is not None: + a_log_2d = g.reshape(a_log, (1, int(a_log.shape[0]))) + neg_exp_a = g.scalar_multiply(g.scalar_exp(a_log_2d), -1.0) + neg_exp_a, a_softplus = _legalize_elementwise_binary_inputs(g, neg_exp_a, a_softplus) + gate_log = g.multiply(neg_exp_a, a_softplus) + else: + gate_log = g.scalar_multiply(a_softplus, -1.0) + beta = g.sigmoid(b_logits) + + initial_state = _materialize_constant_tensor( + g, + torch.zeros((batch_size, key_dim, num_v_heads, value_dim), dtype=torch.float16), + ) + + if op == "gated_deltanet_decode": + deltanet_out = g.gated_deltanet_decode(q_4d, k_4d, v_4d, gate_log, beta, initial_state, 0.0) + else: + deltanet_out = g.gated_deltanet_prefill(q_4d, k_4d, v_4d, gate_log, beta, initial_state, chunk_size, 0.0) + + y_4d = g.slice(deltanet_out, axis=1, start=0, length=seq_len) + y_2d = g.reshape(y_4d, (seq_len * num_v_heads, value_dim)) + + if z_weight is not None: + z_proj = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + z_weight, + pretransposed_rhs=True, + ) + z_proj = g.reshape(z_proj, (seq_len * num_v_heads, value_dim)) + y_2d = g.multiply(g.rms_norm(y_2d, norm_weight, eps=eps), g.silu(z_proj)) + + return [g.reshape(y_2d, (batch_size, seq_len, num_v_heads * value_dim))] + + if op == "group_norm": + return [ + g.group_norm( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + _tensor(env, node.inputs[2]), + num_groups=int(node.attrs["num_groups"]), + eps=float(node.attrs["eps"]), + ) + ] + + if op == "batch_norm": + axis = int(node.attrs.get("axis", 1)) + return [ + g.batch_norm( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + _tensor(env, node.inputs[2]), + _tensor(env, node.inputs[3]), + _tensor(env, node.inputs[4]), + axis=axis, + eps=float(node.attrs["eps"]), + ) + ] + + if op == "identity": + return [env[node.inputs[0]]] + + if op == "contiguous": + return [_tensor(env, node.inputs[0])] + + if op == "advanced_index": + lowered = _try_lower_advanced_index(g, node, env, ir) + if lowered is not None: + return [lowered] + raise NotImplementedError(f"unsupported advanced_index pattern for node {node.id}") + + if op in {"masked_scatter", "aten.masked_scatter.default"}: + base = _tensor(env, node.inputs[0]) + mask = _tensor(env, node.inputs[1]) + source = _tensor(env, node.inputs[2]) + if source.dtype != base.dtype: + source = _ensure_tensor_dtype(g, source, base.dtype) + return [g.masked_scatter(base, mask, source)] + + if op == "masked_fill": + base = _tensor(env, node.inputs[0]) + mask = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + inverse_mask = g.scalar_add(g.scalar_multiply(mask, -1.0), 1.0) + fill_term = g.scalar_multiply(_ensure_tensor_dtype(g, mask, base.dtype), float(node.attrs["value"])) + base_term = _lower_binary_op(g, base, _ensure_tensor_dtype(g, inverse_mask, base.dtype), "multiply") + return [_lower_binary_op(g, fill_term, base_term, "add")] + + if op == "getitem": + source = env[node.inputs[0]] + index = int(node.attrs["index"]) + if isinstance(source, (tuple, list)): + return [source[index]] + if isinstance(source, Tensor): + normalized_index = _normalize_index(index, int(source.shape[0])) + return [g.index(source, normalized_index, axis=0)] + raise NotImplementedError(f"getitem source is not tuple/list for node {node.id}") + + raise NotImplementedError(f"unsupported IR op in lowering: {op}") + + +def _tensor(env: dict[str, Any], value_id: str) -> Tensor: + try: + value = env[value_id] + except KeyError as exc: + raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc + if not isinstance(value, Tensor): + raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") + return value + + +def _attention_tensor(env: dict[str, Any], value_id: str) -> Tensor: + try: + value = env[value_id] + except KeyError as exc: + raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc + if isinstance(value, BroadcastAlias): + if value.kind != "gqa_repeat_kv": + raise TypeError(f"unsupported broadcast alias for attention input {value_id}: {value.kind}") + return value.tensor + if not isinstance(value, Tensor): + raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") + return value + + +def _lower_projected_attention_tensor( + g: Graph, + hidden: Tensor, + weight: Tensor, + bias: Tensor | None, + target_shape: tuple[int, ...], +) -> Tensor: + if len(target_shape) != 4: + raise NotImplementedError(f"projected attention tensor expects rank-4 target shape, got {target_shape}") + + linear_input = _flatten_to_2d_for_linear(g, hidden) + out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) + if bias is not None: + out = g.add(out, bias) + return g.reshape(out, target_shape) + + +def _normalize_attention_add_tensor(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: + if tuple(tensor.shape) == target_shape: + return tensor + + if len(tensor.shape) != 4 or len(target_shape) != 4: + raise NotImplementedError( + f"unsupported attention add tensor shape {tuple(tensor.shape)} for target {target_shape}" + ) + + batch, seq_len, heads, head_dim = target_shape + shape = tuple(int(v) for v in tensor.shape) + + if shape[0] in (1, batch) and shape[1] == heads and shape[2] in (1, seq_len) and shape[3] == head_dim: + return g.permute(tensor, (0, 2, 1, 3)) + if shape[0] in (1, batch) and shape[1] in (1, seq_len) and shape[2] == heads and shape[3] == head_dim: + return tensor + + raise NotImplementedError( + f"unsupported attention add tensor shape {shape} for target {target_shape}" + ) + + +def _rope_input_is_bhsd(ir: IRGraph, value_id: str) -> bool: + value = ir.values.get(value_id) + if value is None or value.producer is None: + return False + node = ir.nodes.get(value.producer) + if node is None: + return False + if node.op == "permute": + permutation = tuple(int(dim) for dim in node.attrs.get("permutation", ())) + return permutation == (0, 2, 1, 3) + if node.op == "transpose": + return int(node.attrs.get("dim0", -1)) == 1 and int(node.attrs.get("dim1", -1)) == 2 + return False + + +def _lower_softplus(g: Graph, x: Tensor) -> Tensor: + # Stable fp16 softplus: relu(x) + log(1 + exp(-abs(x))). + abs_x = g.abs(x) + neg_abs_x = g.scalar_multiply(abs_x, -1.0) + exp_term = g.scalar_exp(neg_abs_x) + log_term = g.scalar_log(g.scalar_add(exp_term, 1.0)) + return g.add(g.relu(x), log_term) + + +def _normalize_dim(dim: int, rank: int) -> int: + if dim < 0: + dim += rank + return dim + + +def _normalize_index(index: int, dim_size: int) -> int: + if index < 0: + index += dim_size + return index + + +def _static_shape(shape: Any) -> tuple[int, ...] | None: + if not isinstance(shape, tuple): + return None + dims: list[int] = [] + for dim in shape: + try: + dims.append(int(dim)) + except Exception: + return None + return tuple(dims) + + +def _constant_tensor_from_ir(ir: IRGraph, value_id: str) -> torch.Tensor | None: + const = ir.constants.get(value_id) + if isinstance(const, torch.nn.Parameter): + const = const.detach() + if isinstance(const, torch.Tensor): + return const.detach().cpu() + return None + + +def _try_lower_identity_broadcast_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + if len(node.inputs) != 3: + return None + + source = _tensor(env, node.inputs[0]) + source_shape = tuple(int(dim) for dim in source.shape) + if len(source_shape) != 2 or int(source_shape[0]) != 1: + return None + + output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + if output_shape is None: + return None + + batch_index = _constant_tensor_from_ir(ir, node.inputs[1]) + token_index = _constant_tensor_from_ir(ir, node.inputs[2]) + if batch_index is None or token_index is None: + return None + + try: + batch_index = torch.broadcast_to(batch_index.to(torch.int64), output_shape) + token_index = torch.broadcast_to(token_index.to(torch.int64), output_shape) + except RuntimeError: + return None + + if torch.count_nonzero(batch_index).item() != 0: + return None + + flat_index = token_index.reshape(-1) + expected = torch.arange(int(source_shape[1]), dtype=torch.int64) + if flat_index.numel() != expected.numel() or not torch.equal(flat_index, expected): + return None + + return g.reshape(source, output_shape) + + +def _try_lower_prefix_mask_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], +) -> Tensor | None: + if len(node.inputs) != 2: + return None + + source = _tensor(env, node.inputs[0]) + mask = _tensor(env, node.inputs[1]) + source_shape = tuple(int(dim) for dim in source.shape) + mask_shape = tuple(int(dim) for dim in mask.shape) + if not mask_shape or len(mask_shape) > len(source_shape): + return None + if source_shape[: len(mask_shape)] != mask_shape: + return None + return g.masked_select_prefix(source, mask) + + +def _try_lower_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + lowered = _try_lower_identity_broadcast_advanced_index(g, node, env, ir) + if lowered is not None: + return lowered + return _try_lower_prefix_mask_advanced_index(g, node, env) + + +def _normalize_slice_end(end: int, dim_size: int) -> int: + if end < 0: + end += dim_size + return max(0, min(end, dim_size)) + + +def _legalize_elementwise_binary_inputs(g: Graph, lhs: Tensor, rhs: Tensor) -> tuple[Tensor, Tensor]: + if lhs.shape == rhs.shape: + return lhs, rhs + + lhs_rank = len(lhs.shape) + rhs_rank = len(rhs.shape) + + if lhs_rank > rhs_rank: + rhs = _reshape_for_trailing_broadcast(g, rhs, lhs.shape) + elif rhs_rank > lhs_rank: + lhs = _reshape_for_trailing_broadcast(g, lhs, rhs.shape) + + if lhs.shape == rhs.shape: + return lhs, rhs + + return lhs, rhs + + +def _is_scalar_like(value: Any) -> bool: + return isinstance(value, (int, float, bool)) + + +def _lower_binary_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: + if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): + lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) + target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 + lhs = _ensure_tensor_dtype(g, lhs, target_dtype) + rhs = _ensure_tensor_dtype(g, rhs, target_dtype) + if op == "add": + return g.add(lhs, rhs) + if op == "subtract": + return g.subtract(lhs, rhs) + if op == "multiply": + return g.multiply(lhs, rhs) + if op == "divide": + return g.divide(lhs, rhs) + raise NotImplementedError(f"unsupported binary op: {op}") + + if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): + lhs_value = _ensure_tensor_dtype( + g, + lhs_value, + Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + scalar = float(rhs_value) + if op == "add": + return g.scalar_add(lhs_value, scalar) + if op == "subtract": + return g.scalar_subtract(lhs_value, scalar) + if op == "multiply": + return g.scalar_multiply(lhs_value, scalar) + if op == "divide": + return g.scalar_divide(lhs_value, scalar) + raise NotImplementedError(f"unsupported binary op: {op}") + + if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): + rhs_value = _ensure_tensor_dtype( + g, + rhs_value, + Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + scalar = float(lhs_value) + if op == "add": + return g.scalar_add(rhs_value, scalar) + if op == "subtract": + return g.scalar_add(g.scalar_multiply(rhs_value, -1.0), scalar) + if op == "multiply": + return g.scalar_multiply(rhs_value, scalar) + if op == "divide": + raise NotImplementedError("scalar/tensor divide is not directly supported by Cactus graph ops") + raise NotImplementedError(f"unsupported binary op: {op}") + + raise TypeError( + f"unsupported lowered operand types for {op}: " + f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" + ) + + +def _lower_compare_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: + if op == "equal": + not_equal = _lower_compare_op(g, lhs_value, rhs_value, "not_equal") + return g.scalar_not_equal(not_equal, 1.0) + + if op == "greater": + delta = _lower_binary_op(g, lhs_value, rhs_value, "subtract") + delta = _ensure_fp16_tensor(g, delta) + return g.scalar_not_equal(g.relu(delta), 0.0) + + if op == "less_equal": + greater = _lower_compare_op(g, lhs_value, rhs_value, "greater") + return g.scalar_not_equal(greater, 1.0) + + if op != "not_equal": + raise NotImplementedError(f"unsupported compare op: {op}") + + if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): + lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) + target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 + lhs = _ensure_tensor_dtype(g, lhs, target_dtype) + rhs = _ensure_tensor_dtype(g, rhs, target_dtype) + return g.not_equal(lhs, rhs) + + if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): + lhs_value = _ensure_tensor_dtype( + g, + lhs_value, + Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + return g.scalar_not_equal(lhs_value, float(rhs_value)) + + if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): + rhs_value = _ensure_tensor_dtype( + g, + rhs_value, + Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + return g.scalar_not_equal(rhs_value, float(lhs_value)) + + raise TypeError( + f"unsupported lowered operand types for {op}: " + f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" + ) + + +def _reshape_for_trailing_broadcast(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: + tensor_shape = tuple(tensor.shape) + target_rank = len(target_shape) + tensor_rank = len(tensor_shape) + + if tensor_rank > target_rank: + return tensor + + padded_shape = (1,) * (target_rank - tensor_rank) + tensor_shape + + # Only legalize cases that are valid trailing broadcasts, e.g.: + # (H,) -> (1, H), (1, 1, H), etc. + for src_dim, tgt_dim in zip(padded_shape, target_shape): + if src_dim != 1 and src_dim != tgt_dim: + return tensor + + if padded_shape == tensor_shape: + return tensor + + return g.reshape(tensor, padded_shape) + + +def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_tensor_dtype(g: Graph, tensor: Tensor, dtype: int) -> Tensor: + if tensor.dtype == dtype: + return tensor + return g.precision_cast(tensor, dtype) + + +def _cat_with_legalized_dtype(g: Graph, tensors: list[Tensor], *, axis: int, dtype: int | None) -> Tensor: + if not tensors: + raise NotImplementedError("cat requires at least one tensor") + if dtype is None: + dtypes = {tensor.dtype for tensor in tensors} + dtype = tensors[0].dtype if len(dtypes) == 1 else Graph.FP16 + legalized = [_ensure_tensor_dtype(g, tensor, dtype) for tensor in tensors] + return g.cat(legalized, axis=axis) + + +def _lower_where_branch_term(g: Graph, branch_value: Any, mask: Tensor, *, dtype: int | None) -> Tensor: + if dtype is not None: + mask = _ensure_tensor_dtype(g, mask, dtype) + if isinstance(branch_value, Tensor): + branch_mask = mask if dtype is None else _ensure_tensor_dtype(g, mask, branch_value.dtype) + return _lower_binary_op(g, branch_value, branch_mask, "multiply") + return g.scalar_multiply(mask, float(branch_value)) + + +def _lower_where_op(g: Graph, node: IRNode, env: dict[str, Any]) -> Tensor: + condition = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + false_mask = g.scalar_add(g.scalar_multiply(condition, -1.0), 1.0) + + input_index = 1 + if bool(node.attrs.get("true_is_scalar", False)): + true_value: Any = float(node.attrs["true_value"]) + else: + true_value = env[node.inputs[input_index]] + input_index += 1 + + if bool(node.attrs.get("false_is_scalar", False)): + false_value: Any = float(node.attrs["false_value"]) + else: + false_value = env[node.inputs[input_index]] + + result_dtype: int | None = None + if isinstance(true_value, Tensor): + result_dtype = true_value.dtype + elif isinstance(false_value, Tensor): + result_dtype = false_value.dtype + + true_term = _lower_where_branch_term(g, true_value, condition, dtype=result_dtype) + false_term = _lower_where_branch_term(g, false_value, false_mask, dtype=result_dtype) + return _lower_binary_op(g, true_term, false_term, "add") + + +def _flatten_to_2d_for_linear(g: Graph, tensor: Tensor) -> Tensor: + shape = tuple(tensor.shape) + if len(shape) <= 2: + return tensor + leading = 1 + for dim in shape[:-1]: + leading *= int(dim) + return g.reshape(tensor, (leading, int(shape[-1]))) + + +def _broadcast_batch_shape(lhs_batch: tuple[int, ...], rhs_batch: tuple[int, ...]) -> tuple[int, ...] | None: + rank = max(len(lhs_batch), len(rhs_batch)) + lhs_padded = (1,) * (rank - len(lhs_batch)) + lhs_batch + rhs_padded = (1,) * (rank - len(rhs_batch)) + rhs_batch + result: list[int] = [] + for lhs_dim, rhs_dim in zip(lhs_padded, rhs_padded): + if lhs_dim == rhs_dim: + result.append(int(lhs_dim)) + elif lhs_dim == 1: + result.append(int(rhs_dim)) + elif rhs_dim == 1: + result.append(int(lhs_dim)) + else: + return None + return tuple(result) + + +def _project_broadcast_coord(coord: tuple[int, ...], operand_batch: tuple[int, ...]) -> tuple[int, ...]: + if not operand_batch: + return () + offset = len(coord) - len(operand_batch) + projected: list[int] = [] + for axis, dim in enumerate(operand_batch): + value = coord[offset + axis] + projected.append(0 if int(dim) == 1 else int(value)) + return tuple(projected) + + +def _index_batch_prefix(g: Graph, tensor: Tensor, coord: tuple[int, ...]) -> Tensor: + for index_value in coord: + tensor = g.index(tensor, index_value, axis=0) + return tensor + + +def _lower_static_batched_matmul(g: Graph, lhs: Tensor, rhs: Tensor) -> Tensor | None: + lhs_shape = tuple(int(dim) for dim in lhs.shape) + rhs_shape = tuple(int(dim) for dim in rhs.shape) + if len(lhs_shape) < 3 or len(rhs_shape) < 3: + return None + if lhs_shape[-1] != rhs_shape[-2]: + return None + + lhs_batch = lhs_shape[:-2] + rhs_batch = rhs_shape[:-2] + batch_shape = _broadcast_batch_shape(lhs_batch, rhs_batch) + if batch_shape is None: + return None + + output_matrix_shape = (int(lhs_shape[-2]), int(rhs_shape[-1])) + rank = len(batch_shape) + + def _build(dim: int, coord_prefix: tuple[int, ...]) -> Tensor: + if dim == rank: + lhs_coord = _project_broadcast_coord(coord_prefix, lhs_batch) + rhs_coord = _project_broadcast_coord(coord_prefix, rhs_batch) + lhs_slice = _index_batch_prefix(g, lhs, lhs_coord) + rhs_slice = _index_batch_prefix(g, rhs, rhs_coord) + out = _matmul_with_quantized_rhs_legalization(g, lhs_slice, rhs_slice) + return g.reshape(out, (1,) * rank + output_matrix_shape) + + pieces = [_build(dim + 1, coord_prefix + (index_value,)) for index_value in range(int(batch_shape[dim]))] + if len(pieces) == 1: + return pieces[0] + return g.cat(pieces, axis=dim) + + return _build(0, ()) + + +def _legalize_matmul_inputs( + g: Graph, + lhs: Tensor, + rhs: Tensor, + node: IRNode, +) -> tuple[Tensor, Tensor, tuple[int, ...]] | None: + lhs_shape = tuple(lhs.shape) + rhs_shape = tuple(rhs.shape) + if len(lhs_shape) <= 2 and len(rhs_shape) <= 2: + return None + + if len(lhs_shape) > 2 and len(rhs_shape) == 2 and lhs_shape[-1] == rhs_shape[0]: + leading = 1 + for dim in lhs_shape[:-1]: + leading *= int(dim) + lhs_2d = g.reshape(lhs, (leading, int(lhs_shape[-1]))) + output_shape = lhs_shape[:-1] + (int(rhs_shape[1]),) + return lhs_2d, rhs, output_shape + + # Cactus matmul is 2D-only. Legalize the narrow case where both operands have + # only singleton leading dims, e.g. rotary helper matmuls: + # (1, M, K) @ (1, K, N) -> reshape to (M, K) @ (K, N) -> reshape back. + if any(dim != 1 for dim in lhs_shape[:-2]): + return None + if any(dim != 1 for dim in rhs_shape[:-2]): + return None + + lhs_2d_shape = lhs_shape[-2:] + rhs_2d_shape = rhs_shape[-2:] + if lhs_2d_shape[-1] != rhs_2d_shape[0]: + return None + + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + output_shape = lhs_shape[:-2] + (lhs_2d_shape[0], rhs_2d_shape[1]) + output_shape = tuple(int(v) for v in output_shape) + + lhs_2d = g.reshape(lhs, lhs_2d_shape) + rhs_2d = g.reshape(rhs, rhs_2d_shape) + return lhs_2d, rhs_2d, output_shape + + +def _resolve_expand_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: + if len(requested_shape) < len(input_shape): + raise NotImplementedError(f"expand cannot reduce rank: {input_shape} -> {requested_shape}") + + padded_input = (1,) * (len(requested_shape) - len(input_shape)) + input_shape + resolved: list[int] = [] + for in_dim, req_dim in zip(padded_input, requested_shape): + if req_dim == -1: + resolved.append(in_dim) + continue + if req_dim < -1: + raise NotImplementedError(f"invalid expand dimension: {req_dim}") + resolved.append(int(req_dim)) + return tuple(resolved) + + +def _resolve_reshape_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: + resolved = [int(v) for v in requested_shape] + unknown_indices = [idx for idx, dim in enumerate(resolved) if dim == -1] + if len(unknown_indices) > 1: + raise NotImplementedError(f"reshape with multiple inferred dimensions is unsupported: {requested_shape}") + if not unknown_indices: + return tuple(resolved) + + input_elements = 1 + for dim in input_shape: + input_elements *= int(dim) + + known_elements = 1 + for dim in resolved: + if dim != -1: + known_elements *= int(dim) + + if known_elements == 0 or input_elements % known_elements != 0: + raise NotImplementedError(f"cannot infer reshape target {requested_shape} from input shape {input_shape}") + + resolved[unknown_indices[0]] = input_elements // known_elements + return tuple(resolved) + + +def _match_gqa_expand_alias(ir: IRGraph, expand_node: IRNode) -> str | None: + target_shape = tuple(int(v) for v in expand_node.attrs["shape"]) + if len(target_shape) != 5: + return None + + current_value_id = expand_node.inputs[0] + base_value_id: str | None = None + + while True: + producer_id = ir.values[current_value_id].producer + if producer_id is None: + return None + producer = ir.nodes[producer_id] + if producer.op == "slice": + axis = int(producer.attrs.get("axis", -1)) + input_shape = ir.values[producer.inputs[0]].shape + if input_shape is None: + return None + normalized_axis = _normalize_dim(axis, len(input_shape)) + start = int(producer.attrs.get("start", 0)) + end = int(producer.attrs.get("end", input_shape[normalized_axis])) + if start != 0 or end < input_shape[normalized_axis]: + return None + current_value_id = producer.inputs[0] + continue + if producer.op == "unsqueeze": + base_value_id = producer.inputs[0] + base_shape = ir.values[base_value_id].shape + if base_shape is None: + return None + unsqueezed_dim = _normalize_dim(int(producer.attrs.get("dim", 0)), len(base_shape) + 1) + if unsqueezed_dim != 2: + return None + break + if producer.op in {"reshape", "view"}: + base_value_id = producer.inputs[0] + base_shape = ir.values[base_value_id].shape + view_shape = ir.values[current_value_id].shape + if base_shape is None or view_shape is None: + return None + if len(base_shape) != 4 or len(view_shape) != 5: + return None + if tuple(int(v) for v in view_shape) != ( + int(base_shape[0]), + int(base_shape[1]), + 1, + int(base_shape[2]), + int(base_shape[3]), + ): + return None + break + return None + + if base_value_id is None: + return None + base_shape = ir.values[base_value_id].shape + if base_shape is None or len(base_shape) != 4: + return None + + expected_target = ( + int(base_shape[0]), + int(base_shape[1]), + int(target_shape[2]), + int(base_shape[2]), + int(base_shape[3]), + ) + if target_shape != expected_target: + return None + if int(target_shape[2]) <= 1: + return None + + return base_value_id + + +def _map_ir_dtype(dtype: str) -> int: + if dtype == "bf16": + return Graph.FP32 + if dtype == "fp16": + return Graph.FP16 + if dtype in ("fp32", "fp64"): + return Graph.FP32 + if dtype == "int8": + return Graph.INT8 + if dtype in ("int32", "int64", "bool"): + return Graph.FP32 + + raise NotImplementedError(f"unsupported IR dtype: {dtype}") + + +def _map_ir_or_torch_dtype(dtype: Any) -> int: + if dtype is None: + raise NotImplementedError("missing dtype for precision_cast") + if isinstance(dtype, str): + if dtype.startswith("torch."): + return _map_torch_dtype(getattr(torch, dtype.split(".", 1)[1])) + return _map_ir_dtype(dtype) + return _map_torch_dtype(dtype) + + +def _map_torch_dtype(dtype: Any) -> int: + if dtype == torch.bfloat16: + return Graph.FP32 + if dtype == torch.float16: + return Graph.FP16 + if dtype == torch.float32: + return Graph.FP32 + if dtype == torch.float64: + return Graph.FP32 + if dtype == torch.int8: + return Graph.INT8 + if dtype in (torch.int16, torch.int32, torch.int64, torch.bool): + return Graph.FP32 + raise NotImplementedError(f"unsupported torch dtype: {dtype}") + + +def _materialize_constant_tensor(g: Graph, tensor_value: torch.Tensor) -> Tensor: + graph_dtype = _map_torch_dtype(tensor_value.dtype) + materialized = tensor_value.detach().cpu() + if graph_dtype == Graph.FP32 and materialized.dtype not in (torch.float32, torch.float64): + materialized = materialized.to(torch.float32) + elif graph_dtype == Graph.FP16 and materialized.dtype != torch.float16: + materialized = materialized.to(torch.float16) + elif graph_dtype == Graph.INT8 and materialized.dtype != torch.int8: + materialized = materialized.to(torch.int8) + + tensor = g.input(shape=tuple(materialized.shape), dtype=graph_dtype) + g.set_input(tensor, materialized, dtype=graph_dtype) + return tensor + + +def _materialize_constant_torch_dtype(dtype: Any) -> torch.dtype: + if dtype is None: + return torch.float32 + if isinstance(dtype, str): + if dtype.startswith("torch."): + return getattr(torch, dtype.split(".", 1)[1]) + if dtype in ("fp16", "bf16"): + return torch.float16 + if dtype in ("fp32", "fp64", "int16", "int32", "int64", "bool"): + return torch.float32 + if dtype == "int8": + return torch.int8 + if isinstance(dtype, torch.dtype): + return dtype + raise NotImplementedError(f"unsupported dtype for constant materialization: {dtype}") + + +def _torch_dtype_for_graph_dtype(dtype: int) -> torch.dtype: + if dtype == Graph.FP16: + return torch.float16 + if dtype == Graph.FP32: + return torch.float32 + if dtype == Graph.INT8: + return torch.int8 + raise NotImplementedError(f"unsupported graph dtype for constant materialization: {dtype}") diff --git a/python/src/transpile/model_adapters.py b/python/src/transpile/model_adapters.py new file mode 100644 index 000000000..46f215895 --- /dev/null +++ b/python/src/transpile/model_adapters.py @@ -0,0 +1,803 @@ +from __future__ import annotations + +from dataclasses import dataclass +import inspect +from typing import Callable + +import torch + + +@dataclass +class CanonicalizedModel: + module: torch.nn.Module + task: str + family: str + input_names: tuple[str, ...] = () + + +def _model_name_or_path(model: torch.nn.Module) -> str: + value = getattr(model, "name_or_path", None) + if isinstance(value, str) and value: + return value + config = getattr(model, "config", None) + value = getattr(config, "_name_or_path", None) + if isinstance(value, str) and value: + return value + return "" + + +def _transpile_graph_meta(model: torch.nn.Module, *, adapter_family: str, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: + return { + "adapter_family": adapter_family, + "adapter_type": adapter_type, + "model_name_or_path": _model_name_or_path(model), + "input_names": input_names, + } + + +def _extract_tensor_output(output: object, *, preferred_field: str | None = None) -> torch.Tensor: + if preferred_field is not None: + value = getattr(output, preferred_field, None) + if isinstance(value, torch.Tensor): + return value + + if isinstance(output, torch.Tensor): + return output + if isinstance(output, tuple) and output and isinstance(output[0], torch.Tensor): + return output[0] + + for field_name in ("last_hidden_state", "logits"): + value = getattr(output, field_name, None) + if isinstance(value, torch.Tensor): + return value + + raise TypeError(f"could not extract tensor output from {type(output).__name__}") + + +def _infer_input_names(module: torch.nn.Module, *, preferred: tuple[str, ...]) -> tuple[str, ...]: + try: + signature = inspect.signature(module.forward) + except (TypeError, ValueError): + return preferred[:1] + + control_names = { + "self", + "return_dict", + "use_cache", + "past_key_values", + "cache_position", + "position_ids", + "labels", + "decoder_input_ids", + "decoder_attention_mask", + "output_attentions", + "output_hidden_states", + } + available = [ + name + for name, parameter in signature.parameters.items() + if name not in control_names + and parameter.kind in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + ] + matched = [name for name in preferred if name in available] + if matched: + return tuple(matched) + if available: + return tuple(available[: min(2, len(available))]) + return preferred[:1] + + +class BoundInputAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str, metadata_task: str): + super().__init__() + self.model = model + self.input_names = tuple(input_names) + self.family = family + self.metadata_task = metadata_task + + def _kwargs_from_bound_inputs(self, *bound_inputs: torch.Tensor | None) -> dict[str, torch.Tensor]: + provided = tuple(bound_inputs) + if len(self.input_names) > len(provided): + raise ValueError( + f"adapter expected at most {len(provided)} bound inputs, got {len(self.input_names)} names" + ) + kwargs: dict[str, torch.Tensor] = {} + for index, name in enumerate(self.input_names): + value = provided[index] + if value is None: + raise ValueError(f"missing required bound input {index} for {name}") + kwargs[name] = value + return kwargs + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family=self.family, + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + "task": self.metadata_task, + } + } + + +def _gemma4_import_hints(backbone: torch.nn.Module, *, module_path_suffix_prefix: str) -> list[dict[str, object]]: + sliding_window = getattr(backbone.config, "sliding_window", None) + layer_types = list(getattr(backbone.config, "layer_types", [])) + import_hints: list[dict[str, object]] = [] + for layer_index, layer_type in enumerate(layer_types): + attrs: dict[str, object] = {} + if layer_type == "sliding_attention" and sliding_window is not None: + attrs["window_size"] = int(sliding_window) + import_hints.append( + { + "module_path_suffix": f"{module_path_suffix_prefix}.layers.{layer_index}.self_attn", + "op": "scaled_dot_product_attention", + "attrs": attrs, + "meta": { + "attention_layer_type": layer_type, + "attention_layer_index": layer_index, + }, + } + ) + return import_hints + + +class CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + + def forward(self, input_ids: torch.Tensor): + return self.model( + input_ids=input_ids, + use_cache=False, + return_dict=False, + )[0] + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="generic", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + } + } + + +class GemmaCausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.gemma.modeling_gemma import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + causal_mask = self._create_causal_mask( + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=None, + past_key_values=None, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.backbone.rotary_emb(hidden_states, position_ids=position_ids) + checkpoints: list[torch.Tensor] = [] + + for decoder_layer in self.backbone.layers[: self.backbone.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=None, + use_cache=False, + position_embeddings=position_embeddings, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + } + } + + +class Gemma3CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.gemma3.modeling_gemma3 import create_causal_mask # type: ignore + from transformers.models.gemma3.modeling_gemma3 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.config.layer_types + } + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], + position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], + position_ids=position_ids, + past_key_values=None, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma3", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + }, + "import_hints": _gemma4_import_hints(self.backbone, module_path_suffix_prefix="backbone"), + } + + +class Gemma4CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + model_backbone = model.model + self.backbone = getattr(model_backbone, "language_model", model_backbone) + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + per_layer_inputs = None + if self.backbone.hidden_size_per_layer_input: + per_layer_inputs = self.backbone.get_per_layer_inputs(input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.unique_layer_types + } + shared_kv_states: dict[str, torch.Tensor] = {} + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + per_layer_input = per_layer_inputs[:, :, i, :] if per_layer_inputs is not None else None + hidden_states = decoder_layer( + hidden_states, + per_layer_input, + shared_kv_states=shared_kv_states, + position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], + attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], + position_ids=position_ids, + past_key_values=None, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def debug_first_block(self, input_ids: torch.Tensor) -> dict[str, torch.Tensor]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + per_layer_inputs = None + if self.backbone.hidden_size_per_layer_input: + per_layer_inputs = self.backbone.get_per_layer_inputs(input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.unique_layer_types + } + shared_kv_states: dict[str, torch.Tensor] = {} + layer = self.backbone.layers[0] + layer_type = self.backbone.config.layer_types[0] + per_layer_input = per_layer_inputs[:, :, 0, :] if per_layer_inputs is not None else None + + checkpoints: dict[str, torch.Tensor] = {} + + residual = hidden_states + normed = layer.input_layernorm(hidden_states) + checkpoints["pre_attn_norm"] = normed + + attn_out = layer.self_attn( + normed, + position_embeddings=position_embeddings[layer_type], + attention_mask=causal_mask_mapping[layer_type], + position_ids=position_ids, + past_key_values=None, + shared_kv_states=shared_kv_states, + ) + if isinstance(attn_out, tuple): + attn_out = attn_out[0] + checkpoints["attn_o_proj"] = attn_out + + post_attn_norm = layer.post_attention_layernorm(attn_out) + checkpoints["post_attn_norm"] = post_attn_norm + + after_attention = residual + post_attn_norm + checkpoints["after_attention_residual"] = after_attention + + pre_ffn_norm = layer.pre_feedforward_layernorm(after_attention) + checkpoints["pre_ffn_norm"] = pre_ffn_norm + + mlp_out = layer.mlp(pre_ffn_norm) + checkpoints["mlp_down"] = mlp_out + + post_ffn_norm = layer.post_feedforward_layernorm(mlp_out) + checkpoints["post_ffn_norm"] = post_ffn_norm + + after_ffn = after_attention + post_ffn_norm + checkpoints["after_ffn_residual"] = after_ffn + + if per_layer_input is not None: + gated = layer.per_layer_input_gate(after_ffn) + gated = layer.act_fn(gated) + projected = gated * per_layer_input + per_layer_proj = layer.per_layer_projection(projected) + checkpoints["per_layer_input_proj"] = per_layer_proj + post_per_layer_input_norm = layer.post_per_layer_input_norm(per_layer_proj) + checkpoints["post_per_layer_input_norm"] = post_per_layer_input_norm + after_ffn = after_ffn + post_per_layer_input_norm + + layer_scalar = getattr(layer, "layer_scalar", None) + if layer_scalar is not None: + after_ffn = after_ffn * layer_scalar + checkpoints["layer_scalar_out"] = after_ffn + return checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + }, + "import_hints": _gemma4_import_hints(self.backbone, module_path_suffix_prefix="backbone"), + } + + +class Gemma4MultimodalCausalLMLogitsAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...]): + super().__init__( + model, + input_names=input_names, + family="gemma4", + metadata_task="multimodal_causal_lm_logits", + ) + model_backbone = model.model + self.multimodal_backbone = model_backbone + self.backbone = getattr(model_backbone, "language_model", model_backbone) + self._create_causal_mask_mapping = None + self._create_masks_for_generate = None + try: + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_masks_for_generate # type: ignore + + self._create_causal_mask_mapping = create_causal_mask_mapping + self._create_masks_for_generate = create_masks_for_generate + except Exception: + pass + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + kwargs = self._kwargs_from_bound_inputs(*bound_inputs) + input_ids = kwargs["input_ids"] + attention_mask = kwargs.get("attention_mask") + token_type_ids = kwargs.get("token_type_ids") + pixel_values = kwargs.get("pixel_values") + pixel_position_ids = kwargs.get("pixel_position_ids") + input_features = kwargs.get("input_features") + input_features_mask = kwargs.get("input_features_mask") + + multimodal_backbone = self.multimodal_backbone + get_placeholder_mask = getattr(multimodal_backbone, "get_placeholder_mask", None) + get_image_features = getattr(multimodal_backbone, "get_image_features", None) + get_audio_features = getattr(multimodal_backbone, "get_audio_features", None) + lm_head = getattr(self.model, "lm_head", None) + if ( + not callable(get_placeholder_mask) + or not callable(get_image_features) + or not callable(get_audio_features) + or not callable(self._create_causal_mask_mapping) + or not callable(self._create_masks_for_generate) + or not isinstance(lm_head, torch.nn.Module) + ): + outputs = self.model( + return_dict=True, + use_cache=False, + **kwargs, + ) + return _extract_tensor_output(outputs, preferred_field="logits") + + inputs_embeds = self.model.get_input_embeddings()(input_ids) + text_mask, image_mask, audio_mask = get_placeholder_mask( + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + + per_layer_inputs = None + text_config = self.multimodal_backbone.config.get_text_config() + if getattr(text_config, "hidden_size_per_layer_input", None): + per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype) + per_layer_inputs = self.backbone.get_per_layer_inputs(per_layer_inputs_tokens) + + if pixel_values is not None: + image_features = get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) + image_mask_expanded = image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) + inputs_embeds = inputs_embeds.masked_scatter(image_mask_expanded, image_features) + + if input_features is not None and input_features_mask is not None: + audio_output = get_audio_features(input_features, ~input_features_mask, return_dict=True) + audio_features = audio_output.pooler_output.to(inputs_embeds.device, inputs_embeds.dtype) + audio_mask_expanded = audio_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) + inputs_embeds = inputs_embeds.masked_scatter(audio_mask_expanded, audio_features) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + if getattr(text_config, "use_bidirectional_attention", None) == "vision": + causal_mask_mapping = self._create_causal_mask_mapping( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + token_type_ids, + pixel_values, + is_training=self.training, + ) + else: + causal_mask_mapping = self._create_masks_for_generate( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + ) + + hidden_states = self.backbone( + input_ids=None, + per_layer_inputs=per_layer_inputs, + attention_mask=causal_mask_mapping, + position_ids=position_ids, + past_key_values=None, + inputs_embeds=inputs_embeds, + use_cache=False, + return_dict=True, + ).last_hidden_state + return lm_head(hidden_states) + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + "task": self.metadata_task, + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + }, + "import_hints": _gemma4_import_hints( + self.backbone, + module_path_suffix_prefix="model.model.language_model", + ), + } + + +class Qwen35CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.qwen3_5.modeling_qwen3_5 import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1) + text_position_ids = position_ids[0] + multimodal_position_ids = position_ids[1:] + + causal_mask = self._create_causal_mask( + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=None, + past_key_values=None, + position_ids=text_position_ids, + ) + linear_attn_mask = self.backbone._update_linear_attn_mask(None, None) + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask + hidden_states = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + attention_mask=layer_mask, + position_ids=text_position_ids, + past_key_values=None, + use_cache=False, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + import_hints: list[dict[str, object]] = [] + for layer_index, layer_type in enumerate(layer_types): + import_hints.append( + { + "module_path_suffix": f"backbone.layers.{layer_index}.self_attn", + "meta": { + "attention_layer_type": layer_type, + "attention_layer_index": layer_index, + }, + } + ) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="qwen3_5", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": layer_types, + }, + "import_hints": import_hints, + } + + +class CTCLogitsAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): + super().__init__(model, input_names=input_names, family=family, metadata_task="ctc_logits") + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + outputs = self.model(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) + return _extract_tensor_output(outputs, preferred_field="logits") + + +class EncoderHiddenStatesAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): + encoder = None + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder = get_encoder() + if encoder is None: + encoder = getattr(model, "encoder", None) + if encoder is None: + model_attr = getattr(model, "model", None) + if model_attr is not None: + encoder = getattr(model_attr, "encoder", None) + if not isinstance(encoder, torch.nn.Module): + raise NotImplementedError(f"{type(model).__name__} does not expose an encoder module") + super().__init__(model, input_names=input_names, family=family, metadata_task="encoder_hidden_states") + self.encoder = encoder + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + outputs = self.encoder(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) + return _extract_tensor_output(outputs, preferred_field="last_hidden_state") + + +def _family_key(model: torch.nn.Module) -> str: + module_name = type(model).__module__ + if module_name.startswith("transformers.models.whisper."): + return "whisper" + if module_name.startswith("transformers.models.gemma4."): + return "gemma4" + if module_name.startswith("transformers.models.gemma3."): + return "gemma3" + if module_name.startswith("transformers.models.gemma."): + return "gemma" + if module_name.startswith("transformers.models.qwen3_5."): + return "qwen3_5" + return "generic" + + +def canonicalize_model_interface( + model: torch.nn.Module, + task: str = "causal_lm_logits", + *, + input_names: tuple[str, ...] | None = None, +) -> CanonicalizedModel: + family = _family_key(model) + adapter_factory: Callable[[torch.nn.Module], torch.nn.Module] + resolved_input_names = tuple(input_names or ()) + + if task == "causal_lm_logits": + if family == "gemma": + adapter_factory = GemmaCausalLMLogitsAdapter + elif family == "gemma4": + adapter_factory = Gemma4CausalLMLogitsAdapter + elif family == "gemma3": + adapter_factory = Gemma3CausalLMLogitsAdapter + elif family == "qwen3_5": + adapter_factory = Qwen35CausalLMLogitsAdapter + else: + adapter_factory = CausalLMLogitsAdapter + resolved_input_names = ("input_ids",) + elif task == "multimodal_causal_lm_logits": + if family != "gemma4": + raise NotImplementedError(f"{type(model).__name__} does not support task={task}") + if not resolved_input_names: + resolved_input_names = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", + ) + adapter_factory = lambda inner_model: Gemma4MultimodalCausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + ) + elif task == "ctc_logits": + if not resolved_input_names: + resolved_input_names = _infer_input_names( + model, + preferred=("input_values", "input_features", "attention_mask"), + ) + adapter_factory = lambda inner_model: CTCLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + elif task == "encoder_hidden_states": + encoder_module = None + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder_module = get_encoder() + if encoder_module is None: + encoder_module = getattr(model, "encoder", None) + if encoder_module is None and getattr(model, "model", None) is not None: + encoder_module = getattr(model.model, "encoder", None) + if not isinstance(encoder_module, torch.nn.Module): + raise NotImplementedError(f"{type(model).__name__} does not support task={task}") + if not resolved_input_names: + resolved_input_names = _infer_input_names( + encoder_module, + preferred=("input_features", "input_values", "attention_mask"), + ) + adapter_factory = lambda inner_model: EncoderHiddenStatesAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + else: + raise NotImplementedError(f"unsupported canonicalization task: {task}") + + return CanonicalizedModel( + module=adapter_factory(model).eval(), + task=task, + family=family, + input_names=resolved_input_names, + ) diff --git a/python/src/transpile/model_patterns.py b/python/src/transpile/model_patterns.py new file mode 100644 index 000000000..22be81bda --- /dev/null +++ b/python/src/transpile/model_patterns.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GoldPattern: + name: str + source_models: tuple[str, ...] + semantic_ops: tuple[str, ...] + description: str + + +GOLD_PATTERNS: tuple[GoldPattern, ...] = ( + GoldPattern( + name="decoder_attention_gqa", + source_models=("model_gemma.cpp", "model_qwen.cpp", "model_lfm2.cpp", "model_gemma4.cpp"), + semantic_ops=("rms_norm", "rope", "attention", "attention_int8_hybrid"), + description=( + "Q/K/V projections from the same normalized hidden state; Q and K optionally get per-head RMSNorm, " + "Q/K are rotary-embedded, then Cactus attention consumes native Q-head and KV-head counts." + ), + ), + GoldPattern( + name="gated_mlp_gelu", + source_models=("model_gemma.cpp", "model_gemma4.cpp"), + semantic_ops=("matmul", "gelu", "multiply"), + description="Two projection branches from the same normalized input: GELU(gate) * up, then a down projection.", + ), + GoldPattern( + name="gated_mlp_silu", + source_models=("model_qwen.cpp", "model_lfm2.cpp"), + semantic_ops=("matmul", "silu", "multiply"), + description="Two projection branches from the same normalized input: SiLU(gate) * up, then a down projection.", + ), + GoldPattern( + name="decoder_block_post_attn_norm", + source_models=("model_gemma.cpp", "model_gemma4.cpp", "model_gemma3n.cpp"), + semantic_ops=("rms_norm", "attention", "add_clipped"), + description=( + "Pre-norm attention block with post-attention RMSNorm before the residual add, followed by a second " + "pre-FFN RMSNorm and a post-FFN RMSNorm before the final residual add." + ), + ), + GoldPattern( + name="decoder_block_simple_residual", + source_models=("model_qwen.cpp", "model_lfm2.cpp"), + semantic_ops=("rms_norm", "attention", "add"), + description=( + "Pre-norm attention block with a direct residual add after attention, followed by a second RMSNorm and " + "a gated MLP branch with another direct residual add." + ), + ), + GoldPattern( + name="gemma4_partial_rope_attention", + source_models=("model_gemma4.cpp",), + semantic_ops=("rms_norm", "rope", "concat", "attention"), + description=( + "Gemma4 attention where only a prefix of the head dimension is rotary-embedded, optionally with shared " + "K/V heads and sliding-window attention." + ), + ), + GoldPattern( + name="sliding_window_attention_mask", + source_models=("model_gemma.cpp", "model_gemma4.cpp"), + semantic_ops=("attention",), + description=( + "Exported PyTorch may build a boolean mask with diff/cumsum/comparisons, but the handwritten Cactus " + "models encode the same behavior directly as an attention `window_size`." + ), + ), +) + + +def gold_patterns_by_name() -> dict[str, GoldPattern]: + return {pattern.name: pattern for pattern in GOLD_PATTERNS} diff --git a/python/src/transpile/normalize.py b/python/src/transpile/normalize.py new file mode 100644 index 000000000..b2d3f2de3 --- /dev/null +++ b/python/src/transpile/normalize.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from typing import Any + +import torch + + +def normalize_target(target: str) -> str: + if target.startswith("aten.mul_"): + return "multiply_inplace" + if target.startswith("aten._assert_tensor_metadata"): + return "identity" + if target.startswith("aten.lift_fresh_copy"): + return "identity" + if target.startswith("aten.clone"): + return "identity" + if target.startswith("aten.alias"): + return "identity" + if target.startswith("aten.detach"): + return "identity" + if target.startswith("aten.dropout"): + return "identity" + if target.startswith("aten.native_dropout"): + return "identity" + if target.startswith("aten.arange.start"): + return "arange" + if target.startswith("aten.arange"): + return "arange" + if target.startswith("aten.add"): + return "add" + if target.startswith("aten.sub"): + return "subtract" + if target.startswith("aten.mul"): + return "multiply" + if target.startswith("aten.floor_divide"): + return "floor_divide" + if target.startswith("aten.div"): + return "divide" + if target.startswith("aten.to.dtype"): + return "precision_cast" + if target.startswith("aten.to.device"): + return "identity" + if target.startswith("aten.type_as"): + return "type_as" + if target.startswith("aten.neg"): + return "negate" + if target.startswith("aten.ne.") or target == "aten.ne": + return "not_equal" + if target.startswith("aten.eq.") or target == "aten.eq": + return "equal" + if target.startswith("aten.logical_or") or target.startswith("aten.__or__"): + return "logical_or" + if target.startswith("aten.logical_and") or target.startswith("aten.__and__"): + return "logical_and" + if target.startswith("aten.logical_not") or target.startswith("aten.bitwise_not"): + return "logical_not" + if target.startswith("aten.abs"): + return "abs" + if target.startswith("aten.clamp"): + return "clamp" + if target.startswith("aten.where"): + return "where" + if target.startswith("aten.masked_scatter"): + return "masked_scatter" + if target.startswith("aten.masked_fill"): + return "masked_fill" + if target.startswith("aten.expand"): + return "expand" + if target.startswith("aten.exp"): + return "scalar_exp" + if target.startswith("aten.sqrt"): + return "scalar_sqrt" + if target.startswith("aten.reciprocal"): + return "reciprocal" + if target.startswith("aten.log"): + return "scalar_log" + if target.startswith("aten.rsqrt"): + return "rsqrt" + if target.startswith("aten.cos"): + return "cos" + if target.startswith("aten.sin"): + return "sin" + if target in ("aten.view.default", "aten.reshape.default", "aten._unsafe_view.default"): + return "reshape" + if target == "aten.flatten.using_ints": + return "flatten" + if target.startswith("aten.squeeze"): + return "squeeze" + if target.startswith("aten.unsqueeze"): + return "unsqueeze" + if target == "aten.t.default" or target.startswith("aten.transpose"): + return "transpose" + if target.startswith("aten.permute"): + return "permute" + if target.startswith("aten.numpy_T"): + return "numpy_T" + if target.startswith("aten.movedim"): + return "movedim" + if target.startswith("aten.contiguous"): + return "contiguous" + if target.startswith("aten.mm") or target.startswith("aten.matmul") or target.startswith("aten.bmm"): + return "matmul" + if target.startswith("aten.linear"): + return "linear" + if target.startswith("aten.addmm"): + return "addmm" + if target.startswith("aten.relu"): + return "relu" + if target.startswith("aten.silu"): + return "silu" + if target == "aten.gelu.erf": + return "gelu_erf" + if target.startswith("aten.gelu"): + return "gelu" + if target.startswith("aten.sigmoid"): + return "sigmoid" + if target.startswith("aten.glu"): + return "glu" + if target.startswith("aten.softplus"): + return "softplus" + if target.startswith("aten.tanh"): + return "tanh" + if target.startswith("aten.softmax"): + return "softmax" + if target.startswith("aten.scaled_dot_product_attention"): + return "scaled_dot_product_attention" + if target.startswith("aten.sum"): + return "sum" + if target.startswith("aten.mean"): + return "mean" + if target.startswith("aten.all"): + return "min" + if target.startswith("aten.var"): + return "variance" + if target.startswith("aten.min") or target.startswith("aten.amin"): + return "min" + if target.startswith("aten.max") or target.startswith("aten.amax"): + return "max" + if target.startswith("aten.cat"): + return "cat" + if target.startswith("aten.split_with_sizes"): + return "split_with_sizes" + if target.startswith("aten.chunk"): + return "chunk" + if target.startswith("aten.ones"): + return "ones" + if target.startswith("aten.slice"): + return "slice" + if target.startswith("aten.select"): + return "index" + if target.startswith("aten.gather"): + return "gather" + if target.startswith("aten.pad"): + return "pad" + if target.startswith("aten.one_hot"): + return "one_hot" + if target.startswith("aten.tril"): + return "tril" + if target.startswith("aten.unfold"): + return "unfold" + if target.startswith("aten.embedding"): + return "embedding" + if target.startswith("aten.conv1d"): + return "conv1d" + if target.startswith("aten.conv2d"): + return "conv2d" + if target.startswith("aten.pow"): + return "pow" + if target.startswith("aten.layer_norm"): + return "layer_norm" + if target.startswith("aten.native_layer_norm"): + return "layer_norm" + if target.startswith("aten.group_norm"): + return "group_norm" + if target.startswith("aten.batch_norm") or target.startswith("aten._native_batch_norm_legit_no_training"): + return "batch_norm" + if target.startswith("aten.rms_norm"): + return "rms_norm" + if target == "": + return "getitem" + return target + + +def dtype_to_ir(dtype: Any | None) -> str | None: + if dtype is None: + return None + + dtype_map = { + torch.float16: "fp16", + torch.float32: "fp32", + torch.float64: "fp64", + torch.bfloat16: "bf16", + torch.int8: "int8", + torch.int16: "int16", + torch.int32: "int32", + torch.int64: "int64", + torch.bool: "bool", + } + return dtype_map.get(dtype, str(dtype)) diff --git a/python/src/transpile/ops.py b/python/src/transpile/ops.py new file mode 100644 index 000000000..e4441acb1 --- /dev/null +++ b/python/src/transpile/ops.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class OpSchema: + name: str + num_inputs: int | None = None + attrs: tuple[str, ...] = () + backend_op: str | None = None + is_layout: bool = False + is_reduction: bool = False + is_fusible: bool = False + + +# Small canonical op registry for the transpiler. +# This is metadata for canonical IR ops, not another IR layer. + +OPS: tuple[OpSchema, ...] = ( + OpSchema("add", num_inputs=2, backend_op="add", is_fusible=True), + OpSchema("add_clipped", num_inputs=2, backend_op="add_clipped", is_fusible=True), + OpSchema("subtract", num_inputs=2, backend_op="subtract", is_fusible=True), + OpSchema("multiply", num_inputs=2, backend_op="multiply", is_fusible=True), + OpSchema("divide", num_inputs=2, backend_op="divide", is_fusible=True), + OpSchema("not_equal", num_inputs=2, backend_op="not_equal"), + OpSchema("equal", num_inputs=2), + OpSchema("greater", num_inputs=2), + OpSchema("less_equal", num_inputs=2), + OpSchema("logical_and", num_inputs=2), + OpSchema("logical_or", num_inputs=2), + OpSchema("logical_not", num_inputs=1), + OpSchema("abs", num_inputs=1, backend_op="abs", is_fusible=True), + OpSchema("clamp", num_inputs=1, attrs=("min", "max")), + OpSchema("pow", num_inputs=1, attrs=("exponent",), backend_op="pow"), + OpSchema("negate", num_inputs=1, is_fusible=True), + OpSchema("scalar_add", num_inputs=1, attrs=("value",), backend_op="scalar_add", is_fusible=True), + OpSchema("scalar_subtract", num_inputs=1, attrs=("value",), backend_op="scalar_subtract", is_fusible=True), + OpSchema("scalar_subtract_reverse", num_inputs=1, attrs=("value",)), + OpSchema("scalar_multiply", num_inputs=1, attrs=("value",), backend_op="scalar_multiply", is_fusible=True), + OpSchema("scalar_divide", num_inputs=1, attrs=("value",), backend_op="scalar_divide", is_fusible=True), + OpSchema("scalar_floor_divide", num_inputs=1, attrs=("value",), backend_op="scalar_floor_divide"), + OpSchema("scalar_divide_reverse", num_inputs=1, attrs=("value",)), + OpSchema("scalar_not_equal", num_inputs=1, attrs=("value",), backend_op="scalar_not_equal"), + OpSchema("scalar_equal", num_inputs=1, attrs=("value",)), + OpSchema("scalar_exp", num_inputs=1, backend_op="scalar_exp", is_fusible=True), + OpSchema("scalar_sqrt", num_inputs=1, backend_op="scalar_sqrt", is_fusible=True), + OpSchema("scalar_log", num_inputs=1, backend_op="scalar_log", is_fusible=True), + OpSchema("scalar_cos", num_inputs=1, backend_op="scalar_cos"), + OpSchema("scalar_sin", num_inputs=1, backend_op="scalar_sin"), + OpSchema("precision_cast", num_inputs=1, attrs=("dtype",), backend_op="precision_cast"), + OpSchema("view", num_inputs=1, attrs=("shape",), backend_op="reshape", is_layout=True), + OpSchema("permute", num_inputs=1, attrs=("permutation",), backend_op="permute", is_layout=True), + OpSchema("squeeze", num_inputs=1, attrs=("dim",), is_layout=True), + OpSchema("movedim", num_inputs=1, attrs=("source", "destination"), is_layout=True), + OpSchema("slice", num_inputs=1, attrs=("axis", "start", "end", "step"), backend_op="slice"), + OpSchema("split_with_sizes", num_inputs=1, attrs=("sizes", "axis")), + OpSchema("chunk", num_inputs=1, attrs=("chunks", "axis")), + OpSchema("ones", num_inputs=0, attrs=("shape", "dtype")), + OpSchema("one_hot", num_inputs=1, attrs=("num_classes",)), + OpSchema("tril", num_inputs=1, attrs=("diagonal",)), + OpSchema("unfold", num_inputs=1, attrs=("dimension", "size", "step")), + OpSchema("index", num_inputs=1, attrs=("axis", "index_value"), backend_op="index"), + OpSchema("advanced_index", num_inputs=None), + OpSchema( + "where", + num_inputs=None, + attrs=("true_is_scalar", "true_value", "false_is_scalar", "false_value"), + ), + OpSchema("masked_fill", num_inputs=2, attrs=("value",)), + OpSchema("masked_scatter", num_inputs=3), + OpSchema("getitem", num_inputs=1, attrs=("index",)), + OpSchema("cat", num_inputs=None, attrs=("axis",), backend_op="cat"), + OpSchema("matmul", num_inputs=2, backend_op="matmul"), + OpSchema("linear", num_inputs=None, attrs=("has_bias",), backend_op="matmul"), + OpSchema("addmm", num_inputs=3), + OpSchema("gather", num_inputs=2, backend_op="gather"), + OpSchema("embedding", num_inputs=2, backend_op="embedding_from_tensor"), + OpSchema("sum", num_inputs=1, attrs=("axis", "keepdim"), backend_op="sum", is_reduction=True), + OpSchema("mean", num_inputs=1, attrs=("axis", "keepdim"), backend_op="mean", is_reduction=True), + OpSchema("variance", num_inputs=1, attrs=("axis", "keepdim"), backend_op="variance", is_reduction=True), + OpSchema("min", num_inputs=1, attrs=("axis", "keepdim"), backend_op="min", is_reduction=True), + OpSchema("max", num_inputs=1, attrs=("axis", "keepdim"), backend_op="max", is_reduction=True), + OpSchema("relu", num_inputs=1, backend_op="relu", is_fusible=True), + OpSchema("silu", num_inputs=1, backend_op="silu", is_fusible=True), + OpSchema("gelu", num_inputs=1, backend_op="gelu", is_fusible=True), + OpSchema("gelu_erf", num_inputs=1, backend_op="gelu_erf", is_fusible=True), + OpSchema("sigmoid", num_inputs=1, backend_op="sigmoid", is_fusible=True), + OpSchema("glu", num_inputs=1, attrs=("axis",), backend_op="glu"), + OpSchema("tanh", num_inputs=1, backend_op="tanh", is_fusible=True), + OpSchema("softplus", num_inputs=1), + OpSchema("softmax", num_inputs=1, attrs=("axis",), backend_op="softmax"), + OpSchema("layer_norm", num_inputs=None, attrs=("eps",), backend_op="layer_norm"), + OpSchema("group_norm", num_inputs=3, attrs=("num_groups", "eps"), backend_op="group_norm"), + OpSchema("batch_norm", num_inputs=5, attrs=("axis", "eps"), backend_op="batch_norm"), + OpSchema("conv1d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv1d"), + OpSchema("conv2d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv2d"), + OpSchema("pad", num_inputs=1, attrs=("pads", "value", "mode")), + + # CUSTOM / FUSED OPS + OpSchema("rms_norm", num_inputs=2, attrs=("eps",), backend_op="rms_norm"), + OpSchema("rope", num_inputs=1, attrs=("theta", "position_offset"), backend_op="rope"), + OpSchema("rel_pos_bias", num_inputs=2, attrs=("scale",), backend_op="rel_pos_bias"), + OpSchema( + "attention", + num_inputs=3, + attrs=("scale", "is_causal", "window_size", "mask", "additive_mask"), + backend_op="attention", + ), + OpSchema( + "attention_block", + num_inputs=None, + attrs=( + "scale", + "is_causal", + "window_size", + "has_mask", + "additive_mask", + "has_gate", + "has_bias", + "attention_output_shape", + ), + is_fusible=True, + ), + OpSchema( + "self_attention_block", + num_inputs=None, + attrs=( + "scale", + "is_causal", + "window_size", + "has_mask", + "additive_mask", + "has_gate", + "has_bias", + "has_query_projection_bias", + "has_query_add", + "has_rel_query_add", + "has_key_projection_bias", + "has_value_projection_bias", + "has_rel_pos_bias", + "has_relative_key_projection_bias", + "query_shape", + "key_shape", + "value_shape", + "relative_key_shape", + "rel_pos_scale", + "attention_output_shape", + ), + is_fusible=True, + ), + OpSchema( + "conv_module", + num_inputs=None, + attrs=( + "eps", + "has_pointwise1_bias", + "has_depthwise_bias", + "has_pointwise2_bias", + "depthwise_kernel_size", + "depthwise_padding", + ), + is_fusible=True, + ), + OpSchema( + "gated_deltanet_prefill", + num_inputs=None, + attrs=( + "num_k_heads", + "num_v_heads", + "key_dim", + "value_dim", + "eps", + "chunk_size", + "has_z", + "has_dt_bias", + "has_a_log", + "has_conv", + ), + is_fusible=True, + ), + OpSchema( + "gated_deltanet_decode", + num_inputs=None, + attrs=( + "num_k_heads", + "num_v_heads", + "key_dim", + "value_dim", + "eps", + "has_z", + "has_dt_bias", + "has_a_log", + "has_conv", + ), + is_fusible=True, + ), +) + + +ALL_OPS: dict[str, OpSchema] = {op.name: op for op in OPS} + + +# Import/capture spellings reduced into the canonical registry above. +ALIASES: dict[str, str] = { + "reshape": "view", + "flatten": "view", + "unsqueeze": "view", + "expand": "view", + "contiguous": "view", + "identity": "view", + "transpose": "permute", + "concat": "cat", + "type_as": "precision_cast", + "scaled_dot_product_attention": "attention", + "cos": "scalar_cos", + "sin": "scalar_sin", +} + + +PRECISION_OP = "precision_cast" +SUPPORTED_PRECISIONS: tuple[str, ...] = ("int4", "int8", "fp16", "fp32") + + +def canonicalize_op(name: str) -> str: + return ALIASES.get(name, name) + + +def get_op(name: str) -> OpSchema: + return ALL_OPS[canonicalize_op(name)] + + +def has_op(name: str) -> bool: + return canonicalize_op(name) in ALL_OPS diff --git a/python/src/transpile/optimize_graph.py b/python/src/transpile/optimize_graph.py new file mode 100644 index 000000000..474c1439d --- /dev/null +++ b/python/src/transpile/optimize_graph.py @@ -0,0 +1,737 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.canonicalize.utils import rebuild_graph +from src.transpile.fusion import match_attention +from src.transpile.fusion import match_attention_block +from src.transpile.fusion import match_conv_module +from src.transpile.fusion import match_gated_deltanet +from src.transpile.fusion import match_gated_mlp +from src.transpile.fusion import match_rel_pos_bias +from src.transpile.fusion import match_rms_norm +from src.transpile.fusion import match_rope +from src.transpile.fusion import match_self_attention_block +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.model_patterns import GOLD_PATTERNS +from src.transpile.normalize import dtype_to_ir + + +@dataclass(frozen=True) +class DetectedPattern: + name: str + anchor_node_id: str + node_ids: tuple[str, ...] + value_ids: tuple[str, ...] + details: dict[str, object] + + +@dataclass(frozen=True) +class FusionConfig: + enable_gated_deltanet: bool = True + enable_rms_norm: bool = True + enable_rope: bool = True + enable_rel_pos_bias: bool = True + enable_attention: bool = True + enable_self_attention_block: bool = True + enable_attention_block: bool = True + enable_conv_module: bool = True + enable_add_clipped: bool = True + + +def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: + config = config or FusionConfig() + verify_ir(graph) + canonicalize_exported_graph(graph) + + for _ in range(max_passes): + changed = False + if config.enable_gated_deltanet and fuse_gated_deltanet(graph): + changed = True + if config.enable_rms_norm and fuse_rms_norm(graph): + changed = True + if config.enable_rope and fuse_rope(graph): + changed = True + if config.enable_rel_pos_bias and fuse_rel_pos_bias(graph): + changed = True + if config.enable_attention and fuse_attention(graph): + changed = True + if config.enable_self_attention_block and fuse_self_attention_blocks(graph): + changed = True + if config.enable_attention_block and fuse_attention_blocks(graph): + changed = True + if config.enable_conv_module and fuse_conv_modules(graph): + changed = True + if config.enable_add_clipped and fuse_add_clipped(graph): + changed = True + if not changed: + break + canonicalize_exported_graph(graph) + + annotate_gold_patterns(graph) + verify_ir(graph) + return graph + + +def fuse_gated_deltanet(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_gated_deltanet(graph, node) + if match is None: + continue + + inputs = [ + match.normalized_input_value_id, + match.qkv_weight_value_id, + match.a_weight_value_id, + match.b_weight_value_id, + match.norm_weight_value_id, + ] + if match.z_weight_value_id is not None: + inputs.append(match.z_weight_value_id) + if match.dt_bias_value_id is not None: + inputs.append(match.dt_bias_value_id) + if match.a_log_value_id is not None: + inputs.append(match.a_log_value_id) + if match.conv_weight_value_id is not None: + inputs.append(match.conv_weight_value_id) + + node.op = f"gated_deltanet_{match.mode}" + node.inputs = inputs + node.attrs = { + "num_k_heads": int(match.num_k_heads), + "num_v_heads": int(match.num_v_heads), + "key_dim": int(match.key_dim), + "value_dim": int(match.value_dim), + "eps": float(match.eps), + "chunk_size": int(match.chunk_size), + "has_z": bool(match.z_weight_value_id is not None), + "has_dt_bias": bool(match.dt_bias_value_id is not None), + "has_a_log": bool(match.a_log_value_id is not None), + "has_conv": bool(match.conv_weight_value_id is not None), + } + node.kind = "semantic" + node.meta["gated_deltanet_mode"] = match.mode + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def annotate_gold_patterns(graph: IRGraph) -> list[DetectedPattern]: + _clear_gold_pattern_annotations(graph) + patterns = [ + *_detect_gated_mlps(graph), + *_detect_decoder_attentions(graph), + *_detect_transformer_blocks(graph), + ] + + for pattern in patterns: + for node_id in pattern.node_ids: + node = graph.nodes.get(node_id) + if node is None: + continue + node.meta.setdefault("gold_patterns", []) + node.meta["gold_patterns"].append(pattern.name) + + anchor = graph.nodes.get(pattern.anchor_node_id) + if anchor is not None: + anchor.meta["gold_pattern_anchor"] = True + anchor.meta["gold_pattern_details"] = pattern.details + + graph.meta["gold_patterns_catalog"] = tuple(pattern.name for pattern in GOLD_PATTERNS) + graph.meta["detected_gold_patterns"] = patterns + return patterns + + +def summarize_detected_gold_patterns(graph: IRGraph) -> dict[str, int]: + summary: dict[str, int] = {} + for pattern in annotate_gold_patterns(graph): + summary[pattern.name] = summary.get(pattern.name, 0) + 1 + return summary + + +def fuse_rms_norm(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"multiply", "type_as", "precision_cast"}: + continue + + match = match_rms_norm(graph, node) + if match is None: + continue + + weight_value_id = match.weight_value_id + if weight_value_id is None: + input_value = graph.values.get(match.input_value_id) + if input_value is None or input_value.shape is None or not input_value.shape: + continue + hidden_dim = input_value.shape[-1] + if not isinstance(hidden_dim, int) or hidden_dim <= 0: + continue + weight_value_id = _materialize_ones_constant( + graph, + hidden_dim, + dtype=input_value.dtype, + suffix="rms_norm_ones", + ) + if float(match.weight_offset) != 0.0: + weight_value_id = _materialize_shifted_constant( + graph, + weight_value_id, + float(match.weight_offset), + suffix="rms_norm_scale", + ) + + node.op = "rms_norm" + node.inputs = [match.input_value_id, weight_value_id] + node.attrs = {"eps": float(match.eps)} + node.kind = "semantic" + node.meta["rms_weight_offset"] = float(match.weight_offset) + node.meta["rms_input_value_id"] = match.input_value_id + node.meta["rms_weight_value_id"] = weight_value_id + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rope(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or not node.outputs: + continue + + match = match_rope(graph, node.outputs[0]) + if match is None or match.partial: + continue + + node.op = "rope" + node.inputs = [match.input_value_id] + node.attrs = { + "theta": float(match.theta), + "position_offset": int(match.position_offset), + } + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_attention(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: + continue + + match = match_attention(graph, node) + if match is None: + continue + + window_size = int(node.attrs.get("window_size", 0)) + if len(node.inputs) > 3: + mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) + if mask_info is not None: + node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) + if window_size == 0: + window_size = int(mask_info["window_size"]) + node.meta["window_size_source"] = "mask_pattern" + + if window_size != 0 and "window_size_source" not in node.meta: + node.meta["window_size_source"] = "import_attr" + + semantic_attrs = { + key: value + for key, value in node.attrs.items() + if key not in {"mask", "dropout_p", "scale", "is_causal"} + } + semantic_attrs.update( + { + "scale": float(node.attrs.get("scale", 1.0)), + "is_causal": bool(node.attrs.get("is_causal", True)), + "window_size": window_size, + } + ) + + if ( + node.op == "attention" + and node.inputs[:3] == [match.query_value_id, match.key_value_id, match.value_value_id] + and node.attrs == semantic_attrs + ): + continue + + node.op = "attention" + node.inputs = [match.query_value_id, match.key_value_id, match.value_value_id] + node.attrs = semantic_attrs + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rel_pos_bias(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_rel_pos_bias(graph, node) + if match is None: + continue + + node.op = "rel_pos_bias" + node.inputs = [match.query_value_id, match.relative_key_value_id] + node.attrs = {"scale": float(match.scale)} + node.kind = "semantic" + node.meta["rel_pos_bias_nodes"] = match.node_ids + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_attention_blocks(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_attention_block(graph, node) + if match is None: + continue + + inputs = [match.query_value_id, match.key_value_id, match.value_value_id] + if match.mask_value_id is not None: + inputs.append(match.mask_value_id) + if match.gate_value_id is not None: + inputs.append(match.gate_value_id) + inputs.append(match.output_projection_weight_value_id) + if match.output_projection_bias_value_id is not None: + inputs.append(match.output_projection_bias_value_id) + + node.op = "attention_block" + node.inputs = inputs + node.attrs = { + "scale": float(match.scale), + "is_causal": bool(match.is_causal), + "window_size": int(match.window_size), + "has_mask": bool(match.mask_value_id is not None), + "additive_mask": bool(match.additive_mask), + "has_gate": bool(match.gate_value_id is not None), + "has_bias": bool(match.output_projection_bias_value_id is not None), + "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), + } + node.kind = "semantic" + node.meta["attention_block_source"] = match.attention_node_id + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_self_attention_blocks(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_self_attention_block(graph, node) + if match is None: + continue + + inputs = [match.hidden_value_id, match.query_weight_value_id] + if match.query_projection_bias_value_id is not None: + inputs.append(match.query_projection_bias_value_id) + if match.query_add_value_id is not None: + inputs.append(match.query_add_value_id) + if match.rel_query_add_value_id is not None: + inputs.append(match.rel_query_add_value_id) + inputs.append(match.key_weight_value_id) + if match.key_projection_bias_value_id is not None: + inputs.append(match.key_projection_bias_value_id) + inputs.append(match.value_weight_value_id) + if match.value_projection_bias_value_id is not None: + inputs.append(match.value_projection_bias_value_id) + if match.mask_value_id is not None: + inputs.append(match.mask_value_id) + if match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None: + inputs.append(match.relative_key_input_value_id) + inputs.append(match.relative_key_weight_value_id) + if match.relative_key_projection_bias_value_id is not None: + inputs.append(match.relative_key_projection_bias_value_id) + if match.gate_value_id is not None: + inputs.append(match.gate_value_id) + inputs.append(match.output_projection_weight_value_id) + if match.output_projection_bias_value_id is not None: + inputs.append(match.output_projection_bias_value_id) + + node.op = "self_attention_block" + node.inputs = inputs + node.attrs = { + "scale": float(match.scale), + "is_causal": bool(match.is_causal), + "window_size": int(match.window_size), + "has_mask": bool(match.mask_value_id is not None), + "additive_mask": bool(match.additive_mask), + "has_gate": bool(match.gate_value_id is not None), + "has_bias": bool(match.output_projection_bias_value_id is not None), + "has_query_projection_bias": bool(match.query_projection_bias_value_id is not None), + "has_query_add": bool(match.query_add_value_id is not None), + "has_rel_query_add": bool(match.rel_query_add_value_id is not None), + "has_key_projection_bias": bool(match.key_projection_bias_value_id is not None), + "has_value_projection_bias": bool(match.value_projection_bias_value_id is not None), + "has_rel_pos_bias": bool(match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None), + "has_relative_key_projection_bias": bool(match.relative_key_projection_bias_value_id is not None), + "query_shape": tuple(int(dim) for dim in match.query_shape), + "key_shape": tuple(int(dim) for dim in match.key_shape), + "value_shape": tuple(int(dim) for dim in match.value_shape), + "relative_key_shape": tuple(int(dim) for dim in match.relative_key_shape) if match.relative_key_shape is not None else (), + "rel_pos_scale": float(match.rel_pos_scale) if match.rel_pos_scale is not None else 1.0, + "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), + } + node.kind = "semantic" + node.meta["self_attention_block_source"] = match.attention_node_id + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_add_clipped(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "add" or len(node.inputs) != 2: + continue + + lhs = strip_passthrough(graph, node.inputs[0]) + rhs = strip_passthrough(graph, node.inputs[1]) + if not (_looks_like_gemma_residual_add(graph, lhs, rhs) or _looks_like_gemma_residual_add(graph, rhs, lhs)): + continue + + node.op = "add_clipped" + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_conv_modules(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_conv_module(graph, node) + if match is None: + continue + + inputs = [ + match.input_value_id, + match.pointwise1_weight_value_id, + ] + if match.pointwise1_bias_value_id is not None: + inputs.append(match.pointwise1_bias_value_id) + inputs.append(match.depthwise_weight_value_id) + if match.depthwise_bias_value_id is not None: + inputs.append(match.depthwise_bias_value_id) + inputs.extend( + [ + match.batch_norm_weight_value_id, + match.batch_norm_bias_value_id, + match.batch_norm_running_mean_value_id, + match.batch_norm_running_var_value_id, + match.pointwise2_weight_value_id, + ] + ) + if match.pointwise2_bias_value_id is not None: + inputs.append(match.pointwise2_bias_value_id) + + node.op = "conv_module" + node.inputs = inputs + node.attrs = { + "eps": float(match.eps), + "has_pointwise1_bias": bool(match.pointwise1_bias_value_id is not None), + "has_depthwise_bias": bool(match.depthwise_bias_value_id is not None), + "has_pointwise2_bias": bool(match.pointwise2_bias_value_id is not None), + "depthwise_kernel_size": int(match.depthwise_kernel_size), + "depthwise_padding": int(match.depthwise_padding), + } + node.kind = "semantic" + node.meta["conv_module_nodes"] = match.node_ids + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _materialize_shifted_constant(graph: IRGraph, value_id: str, offset: float, *, suffix: str) -> str: + base = graph.constants[value_id] + if not isinstance(base, torch.Tensor): + raise NotImplementedError(f"expected tensor constant for {value_id}, got {type(base).__name__}") + + new_value_id = f"{value_id}_{suffix}" + if new_value_id in graph.constants: + return new_value_id + + shifted = base.detach().cpu() + offset + graph.constants[new_value_id] = shifted + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=tuple(shifted.shape), + dtype=dtype_to_ir(shifted.dtype), + producer=None, + users=[], + ) + return new_value_id + + +def _materialize_ones_constant(graph: IRGraph, size: int, *, dtype: str | None, suffix: str) -> str: + dtype_map = { + "fp16": torch.float16, + "bf16": torch.bfloat16, + "fp32": torch.float32, + "fp64": torch.float64, + } + torch_dtype = dtype_map.get(dtype, torch.float32) + new_value_id = f"c_{suffix}_{size}_{dtype or 'fp32'}" + if new_value_id in graph.constants: + return new_value_id + + value = torch.ones((size,), dtype=torch_dtype) + graph.constants[new_value_id] = value + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=(size,), + dtype=dtype_to_ir(value.dtype), + producer=None, + users=[], + ) + return new_value_id + + +def _clear_gold_pattern_annotations(graph: IRGraph) -> None: + for node in graph.nodes.values(): + node.meta.pop("gold_patterns", None) + node.meta.pop("gold_pattern_anchor", None) + node.meta.pop("gold_pattern_details", None) + + +def _detect_gated_mlps(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_gated_mlp(graph, node) + if match is None: + continue + + pattern_name = "gated_mlp_gelu" if match.activation == "gelu" else "gated_mlp_silu" + patterns.append( + DetectedPattern( + name=pattern_name, + anchor_node_id=node.id, + node_ids=match.node_ids, + value_ids=(match.input_value_id,), + details={ + "activation": match.activation, + "input_value_id": match.input_value_id, + "gate_weight_value_id": match.gate_weight_value_id, + "up_weight_value_id": match.up_weight_value_id, + "down_weight_value_id": match.down_weight_value_id, + }, + ) + ) + return patterns + + +def _detect_decoder_attentions(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: + continue + + match = match_attention(graph, node) + if match is None: + continue + + q_input, k_input, v_input = match.source_input_value_ids + pattern_name = "decoder_attention_gqa" if match.has_gqa_repeat else "decoder_attention" + + node_ids = set(match.node_ids) + details = { + "input_value_ids": (q_input, k_input, v_input), + "q_linear_weight_value_id": match.weight_value_ids[0], + "k_linear_weight_value_id": match.weight_value_ids[1], + "v_linear_weight_value_id": match.weight_value_ids[2], + "has_rope": bool(match.has_rope), + "has_qk_rms_norm": bool(match.has_qk_norm), + "has_gqa_repeat": bool(match.has_gqa_repeat), + "is_causal": bool(match.is_causal), + "scale": float(match.scale), + "window_size_hint": int(match.window_size), + } + + if len(node.inputs) > 3: + details["mask_value_id"] = node.inputs[3] + mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) + if mask_info is not None: + details["mask_pattern"] = "sliding_window_attention_mask" + details["window_size_hint"] = int(mask_info["window_size"]) + node_ids.update(mask_info["node_ids"]) + + patterns.append( + DetectedPattern( + name=pattern_name, + anchor_node_id=node.id, + node_ids=tuple(sorted(node_ids)), + value_ids=(q_input, k_input, v_input), + details=details, + ) + ) + return patterns + + +def _detect_transformer_blocks(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"add", "add_clipped"} or len(node.inputs) != 2: + continue + + rhs_pattern = _find_anchor_pattern(graph, node.inputs[1], {"gated_mlp_gelu", "gated_mlp_silu"}) + lhs_pattern = _find_anchor_pattern(graph, node.inputs[0], {"gated_mlp_gelu", "gated_mlp_silu"}) + mlp_pattern = rhs_pattern or lhs_pattern + if mlp_pattern is None: + continue + + residual_value_id = node.inputs[0] if rhs_pattern is not None else node.inputs[1] + attn_pattern = _find_anchor_pattern( + graph, + residual_value_id, + {"decoder_attention_gqa", "decoder_attention", "gemma4_partial_rope_attention"}, + ) + if attn_pattern is None: + continue + + node_ids = {node.id} + for candidate in graph.nodes.values(): + gold_patterns = candidate.meta.get("gold_patterns", []) + if attn_pattern in gold_patterns or mlp_pattern in gold_patterns: + node_ids.add(candidate.id) + + patterns.append( + DetectedPattern( + name="decoder_block_post_attn_norm" if node.op == "add_clipped" else "decoder_block_simple_residual", + anchor_node_id=node.id, + node_ids=tuple(sorted(node_ids)), + value_ids=(residual_value_id,), + details={ + "residual_value_id": residual_value_id, + "attention_pattern": attn_pattern, + "mlp_pattern": mlp_pattern, + "residual_op": node.op, + }, + ) + ) + return patterns + + +def _looks_like_gemma_residual_add(graph: IRGraph, residual_value_id: str, branch_value_id: str) -> bool: + branch_node = producer(graph, strip_passthrough(graph, branch_value_id)) + if branch_node is None or branch_node.op != "rms_norm": + return False + return strip_passthrough(graph, residual_value_id) != strip_passthrough(graph, branch_node.inputs[0]) + + +def _find_anchor_pattern(graph: IRGraph, value_id: str, names: set[str]) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + + for name in node.meta.get("gold_patterns", []): + if name in names and node.meta.get("gold_pattern_anchor"): + return name + + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _extract_sliding_window_mask(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node_ids: set[str] = set() + top_and = producer(graph, strip_passthrough(graph, value_id)) + if top_and is None or top_and.op not in {"aten.__and__.Tensor", "logical_and"}: + return None + node_ids.add(top_and.id) + + stack = list(top_and.inputs) + saw_diff = False + saw_cumsum = False + window_candidates: list[int] = [] + + while stack: + current = strip_passthrough(graph, stack.pop()) + node = producer(graph, current) + if node is None or node.id in node_ids: + continue + node_ids.add(node.id) + + if node.op in {"aten.diff.default", "diff"}: + saw_diff = True + elif node.op in {"aten.cumsum.default", "cumsum"}: + saw_cumsum = True + elif node.op == "scalar_subtract": + maybe_window = int(node.attrs.get("value", 0)) + if maybe_window > 0: + window_candidates.append(maybe_window) + + stack.extend(node.inputs) + + if not saw_diff or not saw_cumsum: + return None + + return { + "window_size": max((candidate for candidate in window_candidates if candidate > 1), default=0), + "node_ids": tuple(sorted(node_ids)), + } diff --git a/python/src/transpile/weight_binding.py b/python/src/transpile/weight_binding.py new file mode 100644 index 000000000..f0a8221c9 --- /dev/null +++ b/python/src/transpile/weight_binding.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re + + +@dataclass(frozen=True) +class WeightBinding: + path: str + kind: str # "weight" | "embedding" + source_name: str + + +_PROJECT_ROOT = Path(__file__).resolve().parents[3] + +_PARAKEET_GLOBAL_FILENAMES: dict[str, str] = { + "ctc_head.weight": "ctc_head_weight.weights", + "ctc_head.bias": "ctc_head_bias.bias", + "encoder.subsampling.layers.0.weight": "subsampling_conv0_weight.weights", + "encoder.subsampling.layers.0.bias": "subsampling_conv0_bias.bias", + "encoder.subsampling.layers.2.weight": "subsampling_depthwise1_weight.weights", + "encoder.subsampling.layers.2.bias": "subsampling_depthwise1_bias.bias", + "encoder.subsampling.layers.3.weight": "subsampling_pointwise1_weight.weights", + "encoder.subsampling.layers.3.bias": "subsampling_pointwise1_bias.bias", + "encoder.subsampling.layers.5.weight": "subsampling_depthwise2_weight.weights", + "encoder.subsampling.layers.5.bias": "subsampling_depthwise2_bias.bias", + "encoder.subsampling.layers.6.weight": "subsampling_pointwise2_weight.weights", + "encoder.subsampling.layers.6.bias": "subsampling_pointwise2_bias.bias", + "encoder.subsampling.linear.weight": "subsampling_linear_weight.weights", + "encoder.subsampling.linear.bias": "subsampling_linear_bias.bias", + "encoder.pre_encode.conv.0.weight": "subsampling_conv0_weight.weights", + "encoder.pre_encode.conv.0.bias": "subsampling_conv0_bias.bias", + "encoder.pre_encode.conv.2.weight": "subsampling_depthwise1_weight.weights", + "encoder.pre_encode.conv.2.bias": "subsampling_depthwise1_bias.bias", + "encoder.pre_encode.conv.3.weight": "subsampling_pointwise1_weight.weights", + "encoder.pre_encode.conv.3.bias": "subsampling_pointwise1_bias.bias", + "encoder.pre_encode.conv.5.weight": "subsampling_depthwise2_weight.weights", + "encoder.pre_encode.conv.5.bias": "subsampling_depthwise2_bias.bias", + "encoder.pre_encode.conv.6.weight": "subsampling_pointwise2_weight.weights", + "encoder.pre_encode.conv.6.bias": "subsampling_pointwise2_bias.bias", + "encoder.pre_encode.out.weight": "subsampling_linear_weight.weights", + "encoder.pre_encode.out.bias": "subsampling_linear_bias.bias", +} + +_PARAKEET_LAYER_FILENAMES: dict[str, str] = { + "feed_forward1.linear1.weight": "ff1_linear1.weights", + "feed_forward1.linear1.bias": "ff1_linear1.bias", + "feed_forward1.linear2.weight": "ff1_linear2.weights", + "feed_forward1.linear2.bias": "ff1_linear2.bias", + "feed_forward2.linear1.weight": "ff2_linear1.weights", + "feed_forward2.linear1.bias": "ff2_linear1.bias", + "feed_forward2.linear2.weight": "ff2_linear2.weights", + "feed_forward2.linear2.bias": "ff2_linear2.bias", + "self_attn.q_proj.weight": "self_attn_q.weights", + "self_attn.q_proj.bias": "self_attn_q.bias", + "self_attn.k_proj.weight": "self_attn_k.weights", + "self_attn.k_proj.bias": "self_attn_k.bias", + "self_attn.v_proj.weight": "self_attn_v.weights", + "self_attn.v_proj.bias": "self_attn_v.bias", + "self_attn.o_proj.weight": "self_attn_output.weights", + "self_attn.o_proj.bias": "self_attn_output.bias", + "self_attn.relative_k_proj.weight": "self_attn_relative_k.weights", + "self_attn.linear_q.weight": "self_attn_q.weights", + "self_attn.linear_q.bias": "self_attn_q.bias", + "self_attn.linear_k.weight": "self_attn_k.weights", + "self_attn.linear_k.bias": "self_attn_k.bias", + "self_attn.linear_v.weight": "self_attn_v.weights", + "self_attn.linear_v.bias": "self_attn_v.bias", + "self_attn.linear_out.weight": "self_attn_output.weights", + "self_attn.linear_out.bias": "self_attn_output.bias", + "self_attn.linear_pos.weight": "self_attn_relative_k.weights", + "self_attn.bias_u": "self_attn_bias_u.weights", + "self_attn.bias_v": "self_attn_bias_v.weights", + "self_attn.pos_bias_u": "self_attn_bias_u.weights", + "self_attn.pos_bias_v": "self_attn_bias_v.weights", + "conv.pointwise_conv1.weight": "conv_pointwise1.weights", + "conv.pointwise_conv1.bias": "conv_pointwise1.bias", + "conv.depthwise_conv.weight": "conv_depthwise.weights", + "conv.depthwise_conv.bias": "conv_depthwise.bias", + "conv.pointwise_conv2.weight": "conv_pointwise2.weights", + "conv.pointwise_conv2.bias": "conv_pointwise2.bias", + "conv.norm.weight": "conv_batchnorm_weight.weights", + "conv.norm.bias": "conv_batchnorm_bias.bias", + "conv.norm.running_mean": "conv_batchnorm_running_mean.weights", + "conv.norm.running_var": "conv_batchnorm_running_var.weights", + "conv.batch_norm.weight": "conv_batchnorm_weight.weights", + "conv.batch_norm.bias": "conv_batchnorm_bias.bias", + "conv.batch_norm.running_mean": "conv_batchnorm_running_mean.weights", + "conv.batch_norm.running_var": "conv_batchnorm_running_var.weights", + "norm_feed_forward1.weight": "norm_ff1.weights", + "norm_feed_forward1.bias": "norm_ff1.bias", + "norm_self_att.weight": "norm_self_attn.weights", + "norm_self_att.bias": "norm_self_attn.bias", + "norm_conv.weight": "norm_conv.weights", + "norm_conv.bias": "norm_conv.bias", + "norm_feed_forward2.weight": "norm_ff2.weights", + "norm_feed_forward2.bias": "norm_ff2.bias", + "norm_out.weight": "norm_out.weights", + "norm_out.bias": "norm_out.bias", +} + + +def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: + candidates: list[str] = [] + raw = model_name_or_path.strip() + if not raw: + return candidates + + def _add(name: str) -> None: + name = name.strip().lower() + if name and name not in candidates: + candidates.append(name) + + _add(raw.split("/")[-1]) + + path = Path(raw) + parts = path.parts + for part in parts: + if part.startswith("models--"): + _add(part[len("models--"):].split("--")[-1]) + break + + return candidates + + +def _default_weights_dir_for_model_name(model_name_or_path: str) -> str | None: + if not model_name_or_path: + return None + for model_dir_name in _candidate_model_dir_names(model_name_or_path): + candidate = _PROJECT_ROOT / "weights" / model_dir_name + if candidate.exists(): + return str(candidate) + return None + + +def _normalized_source_candidates(source_name: str) -> list[str]: + candidates: list[str] = [] + + def _add(name: str) -> None: + if name and name not in candidates: + candidates.append(name) + + raw = source_name.strip() + _add(raw) + + for prefix in ("p_", "b_", "c_"): + if raw.startswith(prefix): + _add(raw[len(prefix):]) + + for prefix in ("adapter.model.", "adapter.module.", "module."): + if raw.startswith(prefix): + _add(raw[len(prefix):]) + + stripped = candidates[-1] + + for prefix in ( + "module.backbone.", + "adapter.backbone.", + "adapter.model.model.", + "adapter.model.language_model.", + ): + if stripped.startswith(prefix): + tail = stripped[len(prefix):] + _add(f"model.{tail}") + _add(f"model.language_model.{tail}") + if tail.startswith("layers."): + _add(f"model.{tail}") + _add(f"model.language_model.{tail}") + break + + if stripped == "adapter.model.lm_head.weight": + _add("lm_head.weight") + + layer_match = re.match( + r"^(?:(?:module|adapter)_)?backbone_layers_slice_none__\d+__none____modules__(\d+)___(.+)$", + stripped, + ) + if layer_match: + layer_index = int(layer_match.group(1)) + tail = layer_match.group(2) + tail_map = { + "self_attn_q_proj_weight": "self_attn.q_proj.weight", + "self_attn_k_proj_weight": "self_attn.k_proj.weight", + "self_attn_v_proj_weight": "self_attn.v_proj.weight", + "self_attn_o_proj_weight": "self_attn.o_proj.weight", + "self_attn_q_norm_weight": "self_attn.q_norm.weight", + "self_attn_k_norm_weight": "self_attn.k_norm.weight", + "mlp_gate_proj_weight": "mlp.gate_proj.weight", + "mlp_up_proj_weight": "mlp.up_proj.weight", + "mlp_down_proj_weight": "mlp.down_proj.weight", + "input_layernorm_weight": "input_layernorm.weight", + "post_attention_layernorm_weight": "post_attention_layernorm.weight", + "linear_attn_in_proj_qkv_weight": "linear_attn.in_proj_qkv.weight", + "linear_attn_conv1d_weight": "linear_attn.conv1d.weight", + "linear_attn_norm_weight": "linear_attn.norm.weight", + "linear_attn_dt_bias": "linear_attn.dt_bias", + "linear_attn_A_log": "linear_attn.A_log", + } + dotted_tail = tail_map.get(tail) + if dotted_tail is not None: + _add(f"model.layers.{layer_index}.{dotted_tail}") + _add(f"model.language_model.layers.{layer_index}.{dotted_tail}") + + embed_map = { + "module_backbone_embed_tokens_weight": [ + "model.embed_tokens.weight", + "model.language_model.embed_tokens.weight", + ], + "adapter_backbone_embed_tokens_weight": [ + "model.embed_tokens.weight", + "model.language_model.embed_tokens.weight", + ], + "module_backbone_embed_tokens_per_layer_weight": [ + "model.embed_tokens_per_layer.weight", + "model.language_model.embed_tokens_per_layer.weight", + ], + "adapter_backbone_embed_tokens_per_layer_weight": [ + "model.embed_tokens_per_layer.weight", + "model.language_model.embed_tokens_per_layer.weight", + ], + "module_backbone_per_layer_model_projection_weight": [ + "model.per_layer_model_projection.weight", + "model.language_model.per_layer_model_projection.weight", + ], + "adapter_backbone_per_layer_model_projection_weight": [ + "model.per_layer_model_projection.weight", + "model.language_model.per_layer_model_projection.weight", + ], + "module_backbone_norm_weight": [ + "model.norm.weight", + "model.language_model.norm.weight", + ], + "adapter_backbone_norm_weight": [ + "model.norm.weight", + "model.language_model.norm.weight", + ], + "module_model_lm_head_weight": [ + "lm_head.weight", + ], + "adapter_model_lm_head_weight": [ + "lm_head.weight", + ], + } + for key, mapped in embed_map.items(): + if stripped == key: + for item in mapped: + _add(item) + + return candidates + + +def _fallback_filename_candidates(source_name: str) -> list[str]: + filenames: list[str] = [] + + def _add(filename: str | None) -> None: + if filename and filename not in filenames: + filenames.append(filename) + + for candidate in _normalized_source_candidates(source_name): + _add(_PARAKEET_GLOBAL_FILENAMES.get(candidate)) + + layer_match = re.match(r"^encoder\.layers\.(\d+)\.(.+)$", candidate) + if layer_match: + layer_index = int(layer_match.group(1)) + suffix = layer_match.group(2) + mapped = _PARAKEET_LAYER_FILENAMES.get(suffix) + if mapped is not None: + _add(f"layer_{layer_index}_{mapped}") + + return filenames + + +def resolve_transpile_weights_dir(graph_meta: dict[str, object]) -> str | None: + explicit = graph_meta.get("weights_dir") + if isinstance(explicit, str) and explicit: + return explicit + + family = str(graph_meta.get("adapter_family", "")).upper() + family_env = f"CACTUS_TRANSPILER_WEIGHTS_DIR_{family}" + if family and family_env in os.environ and os.environ[family_env]: + return os.environ[family_env] + + generic = os.environ.get("CACTUS_TRANSPILER_WEIGHTS_DIR") + if generic: + return generic + + model_name_or_path = graph_meta.get("model_name_or_path") + if isinstance(model_name_or_path, str) and model_name_or_path: + default_dir = _default_weights_dir_for_model_name(model_name_or_path) + if default_dir: + return default_dir + return None + + +def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> WeightBinding | None: + if not weights_dir: + return None + root = Path(weights_dir) + if not root.exists(): + return None + manifest_path = root / "weights_manifest.json" + if not manifest_path.exists(): + return None + try: + manifest = json.loads(manifest_path.read_text()) + except Exception: + return None + if not isinstance(manifest, dict): + return None + for candidate_name in _normalized_source_candidates(source_name): + entry = manifest.get(candidate_name) + if not isinstance(entry, dict): + continue + filename = entry.get("filename") + kind = entry.get("kind", "weight") + if not isinstance(filename, str) or not isinstance(kind, str): + continue + candidate = root / filename + if not candidate.exists(): + continue + return WeightBinding(path=str(candidate), kind=kind, source_name=candidate_name) + + for filename in _fallback_filename_candidates(source_name): + candidate = root / filename + if candidate.exists(): + return WeightBinding(path=str(candidate), kind="weight", source_name=source_name) + return None diff --git a/tests/ios/CactusTest/CactusTest.xcodeproj/project.pbxproj b/tests/ios/CactusTest/CactusTest.xcodeproj/project.pbxproj new file mode 100644 index 000000000..1db38c954 --- /dev/null +++ b/tests/ios/CactusTest/CactusTest.xcodeproj/project.pbxproj @@ -0,0 +1,488 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + 0CE3609610F41EE57C3EF450 /* test_rag.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F2321D2FF06C53F142C9E4A0 /* test_rag.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_rag_main"; }; }; + 1297BDC23284CC2F68DC3F0D /* test_telemetry.cpp in Sources */ = {isa = PBXBuildFile; fileRef = A43ACF25D2BEECF06382F746 /* test_telemetry.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_telemetry_main"; }; }; + 12CC126B7285C7E7B260F354 /* test_curl.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F993F80BF90226984FE5FA46 /* test_curl.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_curl_main"; }; }; + 159396ABC5D7C56EBD60787C /* test_kv_cache.cpp in Sources */ = {isa = PBXBuildFile; fileRef = EEE3AAC3B98C5A4D7934390B /* test_kv_cache.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_kv_cache_main"; }; }; + 1AA51A651CB30DFBB7A91B32 /* test_audio_cosine.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1F972A3BCEA50EA0944B93F1 /* test_audio_cosine.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_audio_cosine_main"; }; }; + 1B84962019170487AAE57354 /* test_embed.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6792059E996574F9932267B6 /* test_embed.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_embed_main"; }; }; + 406CB7FEF241AED3E35F3E5F /* test_gemma4_thinking.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 36DEA043863AF7F99EE0DBD6 /* test_gemma4_thinking.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_thinking_main"; }; }; + 44DADDEB1F5068DF48AF51A9 /* test_gemma4_suite.cpp in Sources */ = {isa = PBXBuildFile; fileRef = A24768DA8A7AA0FB613B0E47 /* test_gemma4_suite.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_suite_main"; }; }; + 5CCAE950625131F3B367E23D /* test_graph.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2D0B741B8364E6DEB7880ECC /* test_graph.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_graph_main"; }; }; + 64A62BB21BE64C469F1A1910 /* test_audio_chat_pcm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 756477DDFCDD7FDAFE86BBB8 /* test_audio_chat_pcm.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_audio_chat_pcm_main"; }; }; + 703EA361EBF4536C94115D6D /* test_kernel.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 7F29F0F824D84E357FD6780B /* test_kernel.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_kernel_main"; }; }; + 70E04EF839DE4444310D4651 /* test_stt.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2AD904FE8AF16AC361E3FB86 /* test_stt.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_stt_main"; }; }; + 72965BE8BCFDA801E76EEB31 /* test_llm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 23BA649FAF464F37FCCC6B94 /* test_llm.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_llm_main"; }; }; + B2B5CECE2E2477A23192ACCF /* test_vlm.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 934A65226FA379ED9D8E8706 /* test_vlm.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_vlm_main"; }; }; + B74B4DBFE7341E92AC8F01E9 /* test_utils.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 344D0D09F78DD280BE1E102E /* test_utils.cpp */; }; + BABC16B78AE1B36F177463BD /* test_index.cpp in Sources */ = {isa = PBXBuildFile; fileRef = BAF5F34E04D4261AAECEA8A1 /* test_index.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_index_main"; }; }; + CD0618CF2D40DEF51EFE9570 /* test_performance.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 68746E7164704D873563D27B /* test_performance.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_performance_main"; }; }; + D8DB20D73F9153B5AFD710AD /* test_gemma4_audio.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 35E791CDD8BA48ED65F41D72 /* test_gemma4_audio.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_audio_main"; }; }; + DFEA8971CFB31E6A890B956F /* test_model_loading.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 257E9B342ABCA116053E18C2 /* test_model_loading.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_model_loading_main"; }; }; + E0F525364DF8BB570ABCD421 /* test_gemma4_vision.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 68B50E60C97A4675E06FA03D /* test_gemma4_vision.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_vision_main"; }; }; + EE0A6D1035305A79B11CA900 /* test_gemma4_audio_layers.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 841D05B43AF13CCB9FC6156A /* test_gemma4_audio_layers.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_gemma4_audio_layers_main"; }; }; + FD46DB3D01A60AF2DED26E5A /* test_exhaustive.cpp in Sources */ = {isa = PBXBuildFile; fileRef = CC335EA1CA171C8BBE20E50F /* test_exhaustive.cpp */; settings = {COMPILER_FLAGS = "-Dmain=test_exhaustive_main"; }; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 1695BEEDC52BAA73FD8D27F1 /* test_utils.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; name = test_utils.h; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_utils.h; sourceTree = ""; }; + 1F972A3BCEA50EA0944B93F1 /* test_audio_cosine.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_audio_cosine.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_audio_cosine.cpp; sourceTree = ""; }; + 23BA649FAF464F37FCCC6B94 /* test_llm.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_llm.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_llm.cpp; sourceTree = ""; }; + 257E9B342ABCA116053E18C2 /* test_model_loading.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_model_loading.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_model_loading.cpp; sourceTree = ""; }; + 2AA889CA2EE510D50037A29D /* CactusTest.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CactusTest.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 2AD904FE8AF16AC361E3FB86 /* test_stt.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_stt.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_stt.cpp; sourceTree = ""; }; + 2D0B741B8364E6DEB7880ECC /* test_graph.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_graph.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_graph.cpp; sourceTree = ""; }; + 344D0D09F78DD280BE1E102E /* test_utils.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_utils.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_utils.cpp; sourceTree = ""; }; + 35E791CDD8BA48ED65F41D72 /* test_gemma4_audio.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_audio.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_audio.cpp; sourceTree = ""; }; + 36DEA043863AF7F99EE0DBD6 /* test_gemma4_thinking.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_thinking.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_thinking.cpp; sourceTree = ""; }; + 6792059E996574F9932267B6 /* test_embed.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_embed.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_embed.cpp; sourceTree = ""; }; + 68746E7164704D873563D27B /* test_performance.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_performance.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_performance.cpp; sourceTree = ""; }; + 68B50E60C97A4675E06FA03D /* test_gemma4_vision.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_vision.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_vision.cpp; sourceTree = ""; }; + 756477DDFCDD7FDAFE86BBB8 /* test_audio_chat_pcm.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_audio_chat_pcm.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_audio_chat_pcm.cpp; sourceTree = ""; }; + 7F29F0F824D84E357FD6780B /* test_kernel.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_kernel.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_kernel.cpp; sourceTree = ""; }; + 841D05B43AF13CCB9FC6156A /* test_gemma4_audio_layers.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_audio_layers.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_audio_layers.cpp; sourceTree = ""; }; + 934A65226FA379ED9D8E8706 /* test_vlm.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_vlm.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_vlm.cpp; sourceTree = ""; }; + A24768DA8A7AA0FB613B0E47 /* test_gemma4_suite.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_gemma4_suite.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_gemma4_suite.cpp; sourceTree = ""; }; + A43ACF25D2BEECF06382F746 /* test_telemetry.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_telemetry.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_telemetry.cpp; sourceTree = ""; }; + BAF5F34E04D4261AAECEA8A1 /* test_index.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_index.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_index.cpp; sourceTree = ""; }; + CC335EA1CA171C8BBE20E50F /* test_exhaustive.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_exhaustive.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_exhaustive.cpp; sourceTree = ""; }; + EEE3AAC3B98C5A4D7934390B /* test_kv_cache.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_kv_cache.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_kv_cache.cpp; sourceTree = ""; }; + F2321D2FF06C53F142C9E4A0 /* test_rag.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_rag.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_rag.cpp; sourceTree = ""; }; + F993F80BF90226984FE5FA46 /* test_curl.cpp */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; name = test_curl.cpp; path = /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests/test_curl.cpp; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ + 2AA889E12EE510D60037A29D /* Exceptions for "CactusTest" folder in "CactusTest" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + Info.plist, + ); + target = 2AA889C92EE510D50037A29D /* CactusTest */; + }; +/* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ + +/* Begin PBXFileSystemSynchronizedRootGroup section */ + 2AA889CC2EE510D50037A29D /* CactusTest */ = { + isa = PBXFileSystemSynchronizedRootGroup; + exceptions = ( + 2AA889E12EE510D60037A29D /* Exceptions for "CactusTest" folder in "CactusTest" target */, + ); + path = CactusTest; + sourceTree = ""; + }; +/* End PBXFileSystemSynchronizedRootGroup section */ + +/* Begin PBXFrameworksBuildPhase section */ + 2AA889C72EE510D50037A29D /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 2AA889C12EE510D50037A29D = { + isa = PBXGroup; + children = ( + 2AA889CC2EE510D50037A29D /* CactusTest */, + 2AA889CB2EE510D50037A29D /* Products */, + 5E90E676965D10AF854DF5E1 /* Tests */, + ); + sourceTree = ""; + }; + 2AA889CB2EE510D50037A29D /* Products */ = { + isa = PBXGroup; + children = ( + 2AA889CA2EE510D50037A29D /* CactusTest.app */, + ); + name = Products; + sourceTree = ""; + }; + 5E90E676965D10AF854DF5E1 /* Tests */ = { + isa = PBXGroup; + children = ( + 756477DDFCDD7FDAFE86BBB8 /* test_audio_chat_pcm.cpp */, + 1F972A3BCEA50EA0944B93F1 /* test_audio_cosine.cpp */, + F993F80BF90226984FE5FA46 /* test_curl.cpp */, + 6792059E996574F9932267B6 /* test_embed.cpp */, + CC335EA1CA171C8BBE20E50F /* test_exhaustive.cpp */, + 35E791CDD8BA48ED65F41D72 /* test_gemma4_audio.cpp */, + 841D05B43AF13CCB9FC6156A /* test_gemma4_audio_layers.cpp */, + A24768DA8A7AA0FB613B0E47 /* test_gemma4_suite.cpp */, + 36DEA043863AF7F99EE0DBD6 /* test_gemma4_thinking.cpp */, + 68B50E60C97A4675E06FA03D /* test_gemma4_vision.cpp */, + 2D0B741B8364E6DEB7880ECC /* test_graph.cpp */, + BAF5F34E04D4261AAECEA8A1 /* test_index.cpp */, + 7F29F0F824D84E357FD6780B /* test_kernel.cpp */, + EEE3AAC3B98C5A4D7934390B /* test_kv_cache.cpp */, + 23BA649FAF464F37FCCC6B94 /* test_llm.cpp */, + 257E9B342ABCA116053E18C2 /* test_model_loading.cpp */, + 68746E7164704D873563D27B /* test_performance.cpp */, + F2321D2FF06C53F142C9E4A0 /* test_rag.cpp */, + 2AD904FE8AF16AC361E3FB86 /* test_stt.cpp */, + A43ACF25D2BEECF06382F746 /* test_telemetry.cpp */, + 934A65226FA379ED9D8E8706 /* test_vlm.cpp */, + 344D0D09F78DD280BE1E102E /* test_utils.cpp */, + 1695BEEDC52BAA73FD8D27F1 /* test_utils.h */, + ); + name = Tests; + path = ../..; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 2AA889C92EE510D50037A29D /* CactusTest */ = { + isa = PBXNativeTarget; + buildConfigurationList = 2AA889E22EE510D60037A29D /* Build configuration list for PBXNativeTarget "CactusTest" */; + buildPhases = ( + 2AA889C62EE510D50037A29D /* Sources */, + 2AA889C72EE510D50037A29D /* Frameworks */, + 2AA889C82EE510D50037A29D /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + 2AA889CC2EE510D50037A29D /* CactusTest */, + ); + name = CactusTest; + productName = CactusTest; + productReference = 2AA889CA2EE510D50037A29D /* CactusTest.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 2AA889C22EE510D50037A29D /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastUpgradeCheck = 2600; + TargetAttributes = { + 2AA889C92EE510D50037A29D = { + CreatedOnToolsVersion = 26.0.1; + }; + }; + }; + buildConfigurationList = 2AA889C52EE510D50037A29D /* Build configuration list for PBXProject "CactusTest" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 2AA889C12EE510D50037A29D; + minimizedProjectReferenceProxies = 1; + preferredProjectObjectVersion = 77; + productRefGroup = 2AA889CB2EE510D50037A29D /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 2AA889C92EE510D50037A29D /* CactusTest */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 2AA889C82EE510D50037A29D /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 2AA889C62EE510D50037A29D /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 64A62BB21BE64C469F1A1910 /* test_audio_chat_pcm.cpp in Sources */, + 1AA51A651CB30DFBB7A91B32 /* test_audio_cosine.cpp in Sources */, + 12CC126B7285C7E7B260F354 /* test_curl.cpp in Sources */, + 1B84962019170487AAE57354 /* test_embed.cpp in Sources */, + FD46DB3D01A60AF2DED26E5A /* test_exhaustive.cpp in Sources */, + D8DB20D73F9153B5AFD710AD /* test_gemma4_audio.cpp in Sources */, + EE0A6D1035305A79B11CA900 /* test_gemma4_audio_layers.cpp in Sources */, + 44DADDEB1F5068DF48AF51A9 /* test_gemma4_suite.cpp in Sources */, + 406CB7FEF241AED3E35F3E5F /* test_gemma4_thinking.cpp in Sources */, + E0F525364DF8BB570ABCD421 /* test_gemma4_vision.cpp in Sources */, + 5CCAE950625131F3B367E23D /* test_graph.cpp in Sources */, + BABC16B78AE1B36F177463BD /* test_index.cpp in Sources */, + 703EA361EBF4536C94115D6D /* test_kernel.cpp in Sources */, + 159396ABC5D7C56EBD60787C /* test_kv_cache.cpp in Sources */, + 72965BE8BCFDA801E76EEB31 /* test_llm.cpp in Sources */, + DFEA8971CFB31E6A890B956F /* test_model_loading.cpp in Sources */, + CD0618CF2D40DEF51EFE9570 /* test_performance.cpp in Sources */, + 0CE3609610F41EE57C3EF450 /* test_rag.cpp in Sources */, + 70E04EF839DE4444310D4651 /* test_stt.cpp in Sources */, + 1297BDC23284CC2F68DC3F0D /* test_telemetry.cpp in Sources */, + B2B5CECE2E2477A23192ACCF /* test_vlm.cpp in Sources */, + B74B4DBFE7341E92AC8F01E9 /* test_utils.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + 2AA889E32EE510D60037A29D /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_CXX_LANGUAGE_STANDARD = "c++20"; + CLANG_CXX_LIBRARY = "libc++"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = K3DTVW668Q; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests, + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/cactus, + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/libs/curl/include, + ); + INFOPLIST_FILE = CactusTest/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + OTHER_CPLUSPLUSFLAGS = ( + "$(inherited)", + "-pthread", + "-Wall", + "-Wextra", + "-pedantic", + "-O3", + ); + OTHER_LDFLAGS = ( + "$(inherited)", + "/Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/apple/libcactus-device.a", + "-framework CoreML", + "-framework Foundation", + "-framework Accelerate", + "-framework Security", + "-framework SystemConfiguration", + "-framework CFNetwork", + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/libs/curl/ios/device/libcurl.a, + ); + PRODUCT_BUNDLE_IDENTIFIER = com.cactus.test.parkiratsandhu; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 2AA889E42EE510D60037A29D /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_CXX_LANGUAGE_STANDARD = "c++20"; + CLANG_CXX_LIBRARY = "libc++"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = K3DTVW668Q; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/tests, + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/cactus, + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/libs/curl/include, + ); + INFOPLIST_FILE = CactusTest/Info.plist; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + OTHER_CPLUSPLUSFLAGS = ( + "$(inherited)", + "-pthread", + "-Wall", + "-Wextra", + "-pedantic", + "-O3", + ); + OTHER_LDFLAGS = ( + "$(inherited)", + "/Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/apple/libcactus-device.a", + "-framework CoreML", + "-framework Foundation", + "-framework Accelerate", + "-framework Security", + "-framework SystemConfiguration", + "-framework CFNetwork", + /Users/parkiratsandhu/Documents/programming_projects/cactus/cactus/libs/curl/ios/device/libcurl.a, + ); + PRODUCT_BUNDLE_IDENTIFIER = com.cactus.test.parkiratsandhu; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + 2AA889E52EE510D60037A29D /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + }; + name = Debug; + }; + 2AA889E62EE510D60037A29D /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 2AA889C52EE510D50037A29D /* Build configuration list for PBXProject "CactusTest" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 2AA889E52EE510D60037A29D /* Debug */, + 2AA889E62EE510D60037A29D /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 2AA889E22EE510D60037A29D /* Build configuration list for PBXNativeTarget "CactusTest" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 2AA889E32EE510D60037A29D /* Debug */, + 2AA889E42EE510D60037A29D /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 2AA889C22EE510D50037A29D /* Project object */; +} From dae4d8836e951ebd8d7fb0c4b76aef4393ac1db3 Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Mon, 11 May 2026 21:08:44 -0700 Subject: [PATCH 02/11] Examples --- python/examples/compare_gemma4_multimodal.py | 779 ++++++++ .../examples/compare_parakeet_performance.py | 793 ++++++++ python/examples/inspect_gemma4_cleanup.py | 241 +++ python/examples/inspect_transpile_cleanup.py | 138 ++ python/examples/test_torch_capture.py | 70 + python/examples/transpile_hf_causal_lm.py | 17 + python/examples/transpile_hf_model.py | 1734 +++++++++++++++++ 7 files changed, 3772 insertions(+) create mode 100644 python/examples/compare_gemma4_multimodal.py create mode 100644 python/examples/compare_parakeet_performance.py create mode 100644 python/examples/inspect_gemma4_cleanup.py create mode 100644 python/examples/inspect_transpile_cleanup.py create mode 100644 python/examples/test_torch_capture.py create mode 100644 python/examples/transpile_hf_causal_lm.py create mode 100644 python/examples/transpile_hf_model.py diff --git a/python/examples/compare_gemma4_multimodal.py b/python/examples/compare_gemma4_multimodal.py new file mode 100644 index 000000000..0970f2496 --- /dev/null +++ b/python/examples/compare_gemma4_multimodal.py @@ -0,0 +1,779 @@ +from __future__ import annotations + +import argparse +import copy +import gc +import json +import os +import sys +import time +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +EXAMPLES_DIR = Path(__file__).resolve().parent +PYTHON_ROOT = EXAMPLES_DIR.parent +PROJECT_ROOT = PYTHON_ROOT.parent + +sys.path.insert(0, str(PYTHON_ROOT)) +sys.path.insert(0, str(EXAMPLES_DIR)) + +from src.cactus import cactus_complete +from src.cactus import cactus_destroy +from src.cactus import cactus_init +from src.cactus import cactus_reset +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph +from transpile_hf_model import TranspileWrapper +from transpile_hf_model import _graph_to_dict +from transpile_hf_model import _load_transformers_bundle +from transpile_hf_model import _lower_preoptimized_ir +from transpile_hf_model import _parse_dtype +from transpile_hf_model import _prepare_gemma4_multimodal_inputs +from transpile_hf_model import _serialize_json_compatible +from transpile_hf_model import _validate_weights_dir +from transpile_hf_model import _write_json + + +_DEFAULT_STOP_SEQUENCES = ("", "", "", "<|im_end|>") + + +@dataclass +class NumericSummary: + mean: float + min: float + max: float + median: float + + +@dataclass +class HandwrittenRun: + wall_ms: float + total_time_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + total_tokens: int + response: str + response_json: dict[str, Any] + + +@dataclass +class TranspiledRun: + preprocess_ms: float + execute_ms: float + total_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + total_tokens: int + response: str + stop_reason: str + generated_token_ids: list[int] + input_shapes: dict[str, list[int]] + + +def _numeric_summary(values: list[float]) -> NumericSummary: + arr = np.asarray(values, dtype=np.float64) + return NumericSummary( + mean=float(np.mean(arr)), + min=float(np.min(arr)), + max=float(np.max(arr)), + median=float(np.median(arr)), + ) + + +def _roundtrip_jsonable(value: Any) -> Any: + return json.loads(json.dumps(_serialize_json_compatible(value))) + + +def _resolve_default_weights_dir(model_id: str) -> Path | None: + env_path = os.environ.get("CACTUS_TEST_GEMMA4_MODEL") + if env_path: + candidate = Path(env_path).resolve() + if (candidate / "config.txt").exists(): + return candidate + + candidates = ( + PROJECT_ROOT / "weights" / "gemma4_int4", + PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower(), + PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower().replace("_", "-"), + ) + for candidate in candidates: + if (candidate / "config.txt").exists(): + return candidate + return None + + +def _require_processor_tokenizer(processor_or_tokenizer: object) -> object: + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + if not hasattr(tokenizer_like, "decode") or not hasattr(tokenizer_like, "encode"): + raise RuntimeError( + "Gemma4 compare script requires a tokenizer-like object with encode/decode support." + ) + return tokenizer_like + + +def _build_messages_payload( + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + system_prompt: str = "", +) -> list[dict[str, object]]: + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system: + messages.append({"role": "system", "content": normalized_system}) + + user_message: dict[str, object] = { + "role": "user", + "content": prompt, + } + if image_files: + user_message["images"] = [str(Path(path).resolve()) for path in image_files] + if audio_file: + user_message["audio"] = [str(Path(audio_file).resolve())] + messages.append(user_message) + return messages + + +def _encode_stop_sequences(tokenizer: object, stop_sequences: tuple[str, ...]) -> list[list[int]]: + encoded: list[list[int]] = [] + encode = getattr(tokenizer, "encode", None) + if not callable(encode): + return encoded + for stop_sequence in stop_sequences: + try: + token_ids = list(encode(stop_sequence, add_special_tokens=False)) + except TypeError: + token_ids = list(encode(stop_sequence)) + if token_ids: + encoded.append([int(token_id) for token_id in token_ids]) + return encoded + + +def _has_suffix(token_ids: list[int], suffix: list[int]) -> bool: + if not suffix or len(token_ids) < len(suffix): + return False + return token_ids[-len(suffix) :] == suffix + + +def _trim_stop_suffix(token_ids: list[int], stop_sequences: list[list[int]]) -> bool: + for stop_sequence in stop_sequences: + if _has_suffix(token_ids, stop_sequence): + del token_ids[-len(stop_sequence) :] + return True + return False + + +def _decode_generated_text(tokenizer: object, token_ids: list[int], *, skip_special_tokens: bool) -> str: + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + raise RuntimeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") + try: + return str(decode(token_ids, skip_special_tokens=skip_special_tokens)) + except TypeError: + return str(decode(token_ids)) + + +def _summarize_handwritten_runs( + runs: list[HandwrittenRun], + *, + weights_dir: Path, +) -> dict[str, Any]: + if not runs: + raise ValueError("expected at least one handwritten run") + return { + "weights_dir": str(weights_dir), + "latest_response": runs[-1].response, + "total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), + "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), + "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], + } + + +def _summarize_transpiled_runs( + runs: list[TranspiledRun], + *, + compile_time_ms: float, + model_source: str, + weight_bindings: int, + prepared_input_shapes: dict[str, list[int]], +) -> dict[str, Any]: + if not runs: + raise ValueError("expected at least one transpiled run") + return { + "compile_time_ms": compile_time_ms, + "model_source": model_source, + "weight_bindings": weight_bindings, + "prepared_input_shapes": prepared_input_shapes, + "latest_response": runs[-1].response, + "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), + "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), + "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), + "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], + } + + +def _prepare_repeat_inputs( + processor: object, + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str, + enable_thinking_if_supported: bool, +) -> tuple[object, Any]: + start = time.perf_counter() + prepared = _prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + use_gemma4_chat_template=True, + ) + preprocess_ms = (time.perf_counter() - start) * 1000.0 + return prepared, preprocess_ms + + +def _run_handwritten_once( + *, + model_handle: Any, + messages_json: str, + options_json: str, +) -> HandwrittenRun: + cactus_reset(model_handle) + streamed_token_ids: list[int] = [] + + def _collect(_text: str, token_id: int) -> None: + streamed_token_ids.append(int(token_id)) + + start = time.perf_counter() + response_json_text = cactus_complete( + model_handle, + messages_json, + options_json, + "[]", + _collect, + pcm_data=None, + ) + wall_ms = (time.perf_counter() - start) * 1000.0 + response_payload = json.loads(response_json_text) + return HandwrittenRun( + wall_ms=wall_ms, + total_time_ms=float(response_payload.get("total_time_ms", wall_ms)), + time_to_first_token_ms=float(response_payload.get("time_to_first_token_ms", 0.0)), + prefill_tps=float(response_payload.get("prefill_tps", 0.0)), + decode_tps=float(response_payload.get("decode_tps", 0.0)), + decode_tokens=len(streamed_token_ids), + total_tokens=int(response_payload.get("total_tokens", len(streamed_token_ids))), + response=str(response_payload.get("response", "")), + response_json=response_payload, + ) + + +def _run_transpiled_generation_once( + *, + tg, + processor: object, + tokenizer: object, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str, + enable_thinking_if_supported: bool, + max_new_tokens: int, + stop_sequences: tuple[str, ...], +) -> TranspiledRun: + prepared, preprocess_ms = _prepare_repeat_inputs( + processor, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + inputs_by_name = { + name: tensor.detach().cpu().numpy() + for name, tensor in zip(prepared.names, prepared.tensors) + } + if "input_ids" not in inputs_by_name: + raise RuntimeError("transpiled Gemma4 generation requires input_ids") + + input_ids = np.asarray(inputs_by_name["input_ids"], dtype=np.int64) + prompt_tokens = int(input_ids.shape[1]) + target_tokens = prompt_tokens + max_new_tokens + + pad_token_id = getattr(tokenizer, "pad_token_id", None) + if pad_token_id is None: + pad_token_id = 0 + eos_token_id = getattr(tokenizer, "eos_token_id", None) + + padded_inputs: dict[str, np.ndarray] = {} + for name, value in inputs_by_name.items(): + if name in {"input_ids", "attention_mask", "token_type_ids"}: + padded_shape = list(value.shape) + padded_shape[1] = target_tokens + if name == "input_ids": + padded_value = np.full(padded_shape, int(pad_token_id), dtype=value.dtype) + else: + padded_value = np.zeros(padded_shape, dtype=value.dtype) + padded_value[:, : value.shape[1]] = value + padded_inputs[name] = padded_value + else: + padded_inputs[name] = value + + if "attention_mask" not in padded_inputs: + padded_inputs["attention_mask"] = np.zeros((1, target_tokens), dtype=np.int64) + padded_inputs["attention_mask"][:, :prompt_tokens] = 1 + if "token_type_ids" not in padded_inputs: + padded_inputs["token_type_ids"] = np.zeros((1, target_tokens), dtype=np.int64) + + tg.graph.hard_reset() + generated_ids: list[int] = [] + encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) + current_length = prompt_tokens + first_token_ms = 0.0 + execute_start = time.perf_counter() + stop_reason = "max_new_tokens" + + for step_index in range(max_new_tokens): + runtime_inputs = [padded_inputs[name] for name in prepared.names] + tg.set_inputs(runtime_inputs) + logits = tg.execute()[0].numpy().astype(np.float32) + next_token_id = int(np.argmax(logits[0, current_length - 1])) + generated_ids.append(next_token_id) + + if step_index == 0: + first_token_ms = preprocess_ms + (time.perf_counter() - execute_start) * 1000.0 + + if eos_token_id is not None and next_token_id == int(eos_token_id): + stop_reason = "eos_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + + if current_length >= target_tokens: + stop_reason = "context_limit" + break + + padded_inputs["input_ids"][0, current_length] = next_token_id + padded_inputs["attention_mask"][0, current_length] = 1 + padded_inputs["token_type_ids"][0, current_length] = 0 + current_length += 1 + + execute_ms = (time.perf_counter() - execute_start) * 1000.0 + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not response: + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() + decode_time_ms = max(0.0, execute_ms + preprocess_ms - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + prefill_tps = (prompt_tokens * 1000.0) / first_token_ms if first_token_ms > 0.0 else 0.0 + + return TranspiledRun( + preprocess_ms=preprocess_ms, + execute_ms=execute_ms, + total_ms=preprocess_ms + execute_ms, + time_to_first_token_ms=first_token_ms, + prefill_tps=prefill_tps, + decode_tps=decode_tps, + decode_tokens=len(generated_ids), + total_tokens=prompt_tokens + len(generated_ids), + response=response, + stop_reason=stop_reason, + generated_token_ids=generated_ids, + input_shapes={ + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + ) + + +def _print_summary(summary: dict[str, Any]) -> None: + print(f"model_id={summary['model_id']}") + print(f"prompt={summary['prompt']!r}") + if summary.get("image_files"): + print(f"image_files={summary['image_files']}") + if summary.get("audio_file"): + print(f"audio_file={summary['audio_file']}") + + handwritten = summary.get("handwritten") + if handwritten is not None: + hw_total = handwritten["total_time_ms"]["mean"] + hw_ttft = handwritten["time_to_first_token_ms"]["mean"] + hw_tokens = handwritten["decode_tokens"]["mean"] + hw_tps = hw_tokens * 1000.0 / max(hw_total, 1e-9) + print("handwritten:") + print(f" total_ms mean={hw_total:.3f}") + print(f" time_to_first_token_ms mean={hw_ttft:.3f}") + print(f" decode_tokens mean={hw_tokens:.3f}") + print(f" output_tokens_per_sec={hw_tps:.3f}") + print(f" latest_response={handwritten['latest_response']!r}") + + transpiled = summary.get("transpiled") + if transpiled is not None: + tr_total = transpiled["total_ms"]["mean"] + tr_ttft = transpiled["time_to_first_token_ms"]["mean"] + tr_tokens = transpiled["decode_tokens"]["mean"] + tr_tps = tr_tokens * 1000.0 / max(tr_total, 1e-9) + print("transpiled:") + print(f" compile_time_ms={transpiled['compile_time_ms']:.3f}") + print(f" total_ms mean={tr_total:.3f}") + print(f" time_to_first_token_ms mean={tr_ttft:.3f}") + print(f" decode_tokens mean={tr_tokens:.3f}") + print(f" output_tokens_per_sec={tr_tps:.3f}") + print(f" weight_bindings={transpiled['weight_bindings']}") + print(f" latest_response={transpiled['latest_response']!r}") + + comparison = summary.get("comparison") + if comparison is not None: + print("comparison:") + print( + " end_to_end_speedup_vs_handwritten=" + f"{comparison['end_to_end_speedup_vs_handwritten']:.6f}" + ) + print(f" responses_match={comparison['responses_match']}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Compare handwritten Cactus Gemma4 multimodal generation against the " + "transpiled Gemma4 multimodal graph on the same prompt, image, and audio." + ) + ) + parser.add_argument("--model-id", default="google/gemma-4-E2B-it") + parser.add_argument("--prompt", required=True) + parser.add_argument("--system-prompt", default="") + parser.add_argument("--image-file", action="append", default=[]) + parser.add_argument("--audio-file", default="") + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus weights directory. Required for handwritten comparison and reused for transpiled mmap bindings.", + ) + parser.add_argument("--artifact-dir", default="", help="Optional directory for summary and transpile artifacts.") + parser.add_argument("--graph-filename", default="graph.cactus") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=1) + parser.add_argument("--max-new-tokens", type=int, default=96) + parser.add_argument("--torch-dtype", default="float16") + parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--skip-handwritten", action="store_true") + parser.add_argument("--skip-transpiled", action="store_true") + parser.add_argument("--enable-thinking", action="store_true") + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--top-p", type=float, default=1.0) + parser.add_argument("--top-k", type=int, default=1) + parser.add_argument("--stop-sequence", action="append", default=[]) + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + args = parser.parse_args() + + if args.skip_handwritten and args.skip_transpiled: + raise RuntimeError("cannot skip both handwritten and transpiled runs") + + image_files = tuple(str(Path(path).resolve()) for path in args.image_file if str(path).strip()) + audio_file = str(Path(args.audio_file).resolve()) if args.audio_file.strip() else None + if audio_file and not Path(audio_file).exists(): + raise FileNotFoundError(f"audio file does not exist: {audio_file}") + for image_file in image_files: + if not Path(image_file).exists(): + raise FileNotFoundError(f"image file does not exist: {image_file}") + + torch_dtype = _parse_dtype(args.torch_dtype) + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + + weights_dir_path = None + if args.weights_dir.strip(): + weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) + else: + weights_dir_path = _resolve_default_weights_dir(args.model_id) + + if not args.skip_handwritten and weights_dir_path is None: + raise RuntimeError( + "handwritten Gemma4 comparison requires a converted weights directory.\n" + "\n" + "Create one with something like:\n" + f" cactus convert {args.model_id} /path/to/gemma4_weights\n" + "\n" + "Then rerun with --weights-dir /path/to/gemma4_weights, or rerun with --skip-handwritten." + ) + + stop_sequences = tuple(args.stop_sequence) if args.stop_sequence else _DEFAULT_STOP_SEQUENCES + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + summary: dict[str, Any] = { + "model_id": args.model_id, + "prompt": args.prompt, + "system_prompt": args.system_prompt, + "image_files": list(image_files), + "audio_file": audio_file, + "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, + "warmup": args.warmup, + "repeats": args.repeats, + "max_new_tokens": args.max_new_tokens, + "torch_dtype": args.torch_dtype, + "trust_remote_code": bool(args.trust_remote_code), + "local_files_only": bool(args.local_files_only), + "enable_thinking": bool(args.enable_thinking), + "temperature": float(args.temperature), + "top_p": float(args.top_p), + "top_k": int(args.top_k), + "stop_sequences": list(stop_sequences), + } + + messages_payload = _build_messages_payload( + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + system_prompt=args.system_prompt, + ) + messages_json = json.dumps(messages_payload) + options_json = json.dumps( + { + "temperature": float(args.temperature), + "top_p": float(args.top_p), + "top_k": int(args.top_k), + "max_tokens": int(args.max_new_tokens), + "stop_sequences": list(stop_sequences), + "enable_thinking_if_supported": bool(args.enable_thinking), + "auto_handoff": False, + "telemetry_enabled": False, + } + ) + + handwritten_runs: list[HandwrittenRun] = [] + if not args.skip_handwritten and weights_dir_path is not None: + model_handle = cactus_init(str(weights_dir_path), None, False) + try: + for _ in range(args.warmup): + _run_handwritten_once( + model_handle=model_handle, + messages_json=messages_json, + options_json=options_json, + ) + for _ in range(args.repeats): + handwritten_runs.append( + _run_handwritten_once( + model_handle=model_handle, + messages_json=messages_json, + options_json=options_json, + ) + ) + finally: + cactus_destroy(model_handle) + summary["handwritten"] = _summarize_handwritten_runs( + handwritten_runs, + weights_dir=weights_dir_path, + ) + + transpiled_runs: list[TranspiledRun] = [] + if not args.skip_transpiled: + model_source, processor_or_tokenizer, model, _ = _load_transformers_bundle( + model_id=args.model_id, + task="multimodal_causal_lm_logits", + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + processor = processor_or_tokenizer + tokenizer = _require_processor_tokenizer(processor_or_tokenizer) + initial_prepared, _ = _prepare_repeat_inputs( + processor, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + ) + canonical = canonicalize_model_interface( + model, + task="multimodal_causal_lm_logits", + input_names=initial_prepared.names, + ) + wrapper = TranspileWrapper( + canonical.module, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + ).eval() + + compile_start = time.perf_counter() + captured = capture_model(wrapper, initial_prepared.tensors) + raw_ir_graph = copy.deepcopy(captured.ir_graph) + canonicalize_exported_graph(captured.ir_graph) + optimize_graph(captured.ir_graph, config=fusion_config) + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + tg = _lower_preoptimized_ir(captured.ir_graph) + compile_time_ms = (time.perf_counter() - compile_start) * 1000.0 + + op_counts: dict[str, int] = {} + for node_id in optimized_ir_graph.order: + op = optimized_ir_graph.nodes[node_id].op + op_counts[op] = op_counts.get(op, 0) + 1 + weight_bindings = sum( + 1 + for value in optimized_ir_graph.values.values() + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) + ) + + summary["transpile_compile"] = { + "model_source": model_source, + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": weight_bindings, + "compile_time_ms": compile_time_ms, + "ops": op_counts, + } + + if weights_dir_path is not None and weight_bindings == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir_path}\n" + "\n" + "The converted weights folder exists, but none of the captured constants matched weights_manifest.json.\n" + "Re-convert Gemma4 with the current converter before benchmarking." + ) + + if artifact_dir is not None: + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": "multimodal_causal_lm_logits", + "inputs": _serialize_json_compatible(initial_prepared.metadata), + "graph": _graph_to_dict(raw_ir_graph), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": "multimodal_causal_lm_logits", + "inputs": _serialize_json_compatible(initial_prepared.metadata), + "graph": _graph_to_dict(optimized_ir_graph), + }, + ) + graph_path = artifact_dir / args.graph_filename + tg.graph.save(graph_path) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + print(f"saved_graph={graph_path}") + + del model + del wrapper + del canonical + del captured + gc.collect() + + for _ in range(args.warmup): + _run_transpiled_generation_once( + tg=tg, + processor=processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + stop_sequences=stop_sequences, + ) + for _ in range(args.repeats): + transpiled_runs.append( + _run_transpiled_generation_once( + tg=tg, + processor=processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + stop_sequences=stop_sequences, + ) + ) + + summary["transpiled"] = _summarize_transpiled_runs( + transpiled_runs, + compile_time_ms=compile_time_ms, + model_source=model_source, + weight_bindings=weight_bindings, + prepared_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + ) + + if handwritten_runs and transpiled_runs: + handwritten_latest = handwritten_runs[-1].response.strip() + transpiled_latest = transpiled_runs[-1].response.strip() + summary["comparison"] = { + "end_to_end_speedup_vs_handwritten": ( + summary["handwritten"]["total_time_ms"]["mean"] + / max(summary["transpiled"]["total_ms"]["mean"], 1e-9) + ), + "responses_match": handwritten_latest == transpiled_latest, + "handwritten_latest_response": handwritten_latest, + "transpiled_latest_response": transpiled_latest, + } + + if artifact_dir is not None: + summary_path = artifact_dir / "summary.json" + summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") + print(f"saved_summary={summary_path}") + + _print_summary(summary) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/compare_parakeet_performance.py b/python/examples/compare_parakeet_performance.py new file mode 100644 index 000000000..621f1e1f5 --- /dev/null +++ b/python/examples/compare_parakeet_performance.py @@ -0,0 +1,793 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from collections import defaultdict +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from scipy.io import wavfile + +EXAMPLES_DIR = Path(__file__).resolve().parent +PYTHON_ROOT = EXAMPLES_DIR.parent +PROJECT_ROOT = PYTHON_ROOT.parent + +sys.path.insert(0, str(PYTHON_ROOT)) +sys.path.insert(0, str(EXAMPLES_DIR)) + +from src.cactus import cactus_destroy +from src.cactus import cactus_init +from src.cactus import cactus_transcribe +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph +from transpile_hf_model import TranspileWrapper +from transpile_hf_model import _ctc_greedy_decode_token_ids +from transpile_hf_model import _decode_token_ids +from transpile_hf_model import _infer_task_from_config +from transpile_hf_model import _load_optional_json +from transpile_hf_model import _load_optional_tokenizer +from transpile_hf_model import _load_transformers_bundle +from transpile_hf_model import _lower_preoptimized_ir +from transpile_hf_model import _parse_dtype +from transpile_hf_model import _prepare_audio_inputs +from transpile_hf_model import _validate_weights_dir + + +@dataclass +class NumericSummary: + mean: float + min: float + max: float + median: float + + +@dataclass +class ProfileEntry: + op: str + time_ms: float + shape: str + raw_line: str + + +@dataclass +class ProfileSection: + entries: list[ProfileEntry] + total_ms: float + + +@dataclass +class HandwrittenRun: + wall_ms: float + total_time_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + prefill_tokens: int + total_tokens: int + confidence: float + transcript: str + response_json: dict[str, Any] + + +@dataclass +class TranspiledRun: + preprocess_ms: float + execute_ms: float + decode_ms: float + total_ms: float + graph_only_ms: float + output_frames: int + output_vocab: int + decode_tokens: int + transcript: str | None + input_shapes: dict[str, list[int]] + + +def _numeric_summary(values: list[float]) -> NumericSummary: + arr = np.asarray(values, dtype=np.float64) + return NumericSummary( + mean=float(np.mean(arr)), + min=float(np.min(arr)), + max=float(np.max(arr)), + median=float(np.median(arr)), + ) + + +def _resolve_default_weights_dir(model_id: str) -> Path | None: + candidate = PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower() + if (candidate / "config.txt").exists(): + return candidate + return None + + +def _audio_duration_seconds(audio_file: str) -> float: + sample_rate, samples = wavfile.read(audio_file) + if sample_rate <= 0: + raise ValueError(f"invalid sample rate in {audio_file}: {sample_rate}") + return float(len(samples)) / float(sample_rate) + + +def _profile_env(profile_file: Path | None): + class _ProfileContext: + def __enter__(self_inner): + self_inner.old_profile = os.environ.get("CACTUS_PROFILE") + self_inner.old_profile_file = os.environ.get("CACTUS_PROFILE_FILE") + if profile_file is None: + os.environ.pop("CACTUS_PROFILE", None) + os.environ.pop("CACTUS_PROFILE_FILE", None) + return self_inner + profile_file.parent.mkdir(parents=True, exist_ok=True) + if profile_file.exists(): + profile_file.unlink() + os.environ.pop("CACTUS_PROFILE", None) + os.environ["CACTUS_PROFILE_FILE"] = str(profile_file) + return self_inner + + def __exit__(self_inner, exc_type, exc, tb): + if self_inner.old_profile is None: + os.environ.pop("CACTUS_PROFILE", None) + else: + os.environ["CACTUS_PROFILE"] = self_inner.old_profile + + if self_inner.old_profile_file is None: + os.environ.pop("CACTUS_PROFILE_FILE", None) + else: + os.environ["CACTUS_PROFILE_FILE"] = self_inner.old_profile_file + return False + + return _ProfileContext() + + +def _parse_profile_file(path: Path) -> list[ProfileSection]: + if not path.exists(): + return [] + + lines = path.read_text().splitlines() + sections: list[ProfileSection] = [] + idx = 0 + while idx < len(lines): + if lines[idx].strip() != "=== Graph Execution Profile ===": + idx += 1 + continue + + entries: list[ProfileEntry] = [] + total_ms = 0.0 + idx += 1 + while idx < len(lines): + line = lines[idx].rstrip() + stripped = line.strip() + if stripped.startswith("Total execution time:"): + try: + total_ms = float(stripped.split(":", 1)[1].strip().split()[0]) + except Exception: + total_ms = 0.0 + elif stripped == "================================": + break + elif ( + stripped + and not stripped.startswith("Operation") + and not set(stripped).issubset({"-"}) + and not stripped.startswith("Total execution time:") + ): + parts = stripped.split() + if len(parts) >= 3: + op = parts[0] + try: + time_ms = float(parts[1]) + except ValueError: + time_ms = 0.0 + shape = parts[2] if parts[2].startswith("[") else "" + entries.append(ProfileEntry(op=op, time_ms=time_ms, shape=shape, raw_line=stripped)) + idx += 1 + + sections.append(ProfileSection(entries=entries, total_ms=total_ms)) + idx += 1 + + return sections + + +def _summarize_profile_sections(sections: list[ProfileSection]) -> dict[str, Any]: + op_total_ms: dict[str, float] = defaultdict(float) + op_count: dict[str, int] = defaultdict(int) + all_entries: list[ProfileEntry] = [] + total_ms_values: list[float] = [] + + for section in sections: + total_ms_values.append(section.total_ms) + all_entries.extend(section.entries) + for entry in section.entries: + op_total_ms[entry.op] += entry.time_ms + op_count[entry.op] += 1 + + by_op = [] + for op, total_ms in sorted(op_total_ms.items(), key=lambda item: item[1], reverse=True): + count = op_count[op] + by_op.append( + { + "op": op, + "count": count, + "total_ms": round(total_ms, 6), + "mean_ms": round(total_ms / max(count, 1), 6), + } + ) + + slowest_nodes = [ + { + "op": entry.op, + "time_ms": round(entry.time_ms, 6), + "shape": entry.shape, + "raw_line": entry.raw_line, + } + for entry in sorted(all_entries, key=lambda item: item.time_ms, reverse=True)[:50] + ] + + return { + "runs_profiled": len(sections), + "graph_total_ms": asdict(_numeric_summary(total_ms_values)) if total_ms_values else None, + "entries_per_run": [len(section.entries) for section in sections], + "by_op": by_op, + "slowest_nodes": slowest_nodes, + } + + +def _run_handwritten_once( + *, + model_handle: Any, + audio_file: str, +) -> HandwrittenRun: + options_json = json.dumps( + { + "use_vad": False, + "max_tokens": 4096, + } + ) + start = time.perf_counter() + response = cactus_transcribe( + model_handle, + audio_file, + "", + options_json, + None, + None, + ) + end = time.perf_counter() + + payload = json.loads(response) + return HandwrittenRun( + wall_ms=(end - start) * 1000.0, + total_time_ms=float(payload.get("total_time_ms", 0.0)), + time_to_first_token_ms=float(payload.get("time_to_first_token_ms", 0.0)), + prefill_tps=float(payload.get("prefill_tps", 0.0)), + decode_tps=float(payload.get("decode_tps", 0.0)), + decode_tokens=int(payload.get("decode_tokens", 0)), + prefill_tokens=int(payload.get("prefill_tokens", 0)), + total_tokens=int(payload.get("total_tokens", 0)), + confidence=float(payload.get("confidence", 0.0)), + transcript=str(payload.get("response", "")), + response_json=payload, + ) + + +def _run_transpiled_once( + *, + tg: Any, + processor: object | None, + input_names: tuple[str, ...], + model_config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + torch_dtype: torch.dtype, + audio_file: str, + tokenizer: object | None, + blank_token_id: int | None, +) -> TranspiledRun: + start_pre = time.perf_counter() + prepared = _prepare_audio_inputs( + processor, + input_names=input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task="ctc_logits", + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + end_pre = time.perf_counter() + + input_arrays = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] + + start_exec = time.perf_counter() + tg.set_inputs(input_arrays) + outputs = tg.execute() + end_exec = time.perf_counter() + + start_decode = time.perf_counter() + logits = outputs[0].numpy().astype(np.float32) + token_ids = _ctc_greedy_decode_token_ids(logits, blank_token_id=blank_token_id) + transcript = _decode_token_ids(tokenizer, token_ids) if tokenizer is not None else None + end_decode = time.perf_counter() + + return TranspiledRun( + preprocess_ms=(end_pre - start_pre) * 1000.0, + execute_ms=(end_exec - start_exec) * 1000.0, + decode_ms=(end_decode - start_decode) * 1000.0, + total_ms=(end_decode - start_pre) * 1000.0, + graph_only_ms=(end_exec - start_exec) * 1000.0, + output_frames=int(logits.shape[1]), + output_vocab=int(logits.shape[2]), + decode_tokens=len(token_ids), + transcript=transcript, + input_shapes={ + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + ) + + +def _run_transpiled_graph_only_once( + *, + tg: Any, + cached_inputs: list[np.ndarray], +) -> float: + start = time.perf_counter() + tg.set_inputs(cached_inputs) + tg.execute() + end = time.perf_counter() + return (end - start) * 1000.0 + + +def _warmup(fn, count: int) -> None: + for _ in range(max(count, 0)): + fn() + + +def _roundtrip_jsonable(obj: Any) -> Any: + if isinstance(obj, dict): + return {str(key): _roundtrip_jsonable(value) for key, value in obj.items()} + if isinstance(obj, list): + return [_roundtrip_jsonable(value) for value in obj] + if isinstance(obj, tuple): + return [_roundtrip_jsonable(value) for value in obj] + if isinstance(obj, Path): + return str(obj) + if isinstance(obj, NumericSummary): + return asdict(obj) + if hasattr(obj, "__dataclass_fields__"): + return asdict(obj) + return obj + + +def _build_transpiled_graph( + *, + model_id: str, + audio_file: str, + weights_dir: str | None, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, + fusion_config: FusionConfig, +): + task = _infer_task_from_config(model_id) + if task != "ctc_logits": + raise RuntimeError(f"expected a CTC model, got task={task}") + + model_source, processor, model, model_config = _load_transformers_bundle( + model_id=model_id, + task=task, + torch_dtype=torch_dtype, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) + + preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") + if not preprocessor_config: + preprocessor_config = _load_optional_json(model_id, "preprocessor_config.json") + + tokenizer = _load_optional_tokenizer( + model_id=model_id, + model_source=model_source, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) + + canonical = canonicalize_model_interface(model, task=task) + prepared = _prepare_audio_inputs( + processor, + input_names=canonical.input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface(model, task=task, input_names=prepared.names) + + wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + + capture_started = time.perf_counter() + captured = capture_model(wrapper, prepared.tensors) + canonicalize_exported_graph(captured.ir_graph) + optimize_graph(captured.ir_graph, config=fusion_config) + tg = _lower_preoptimized_ir(captured.ir_graph) + capture_finished = time.perf_counter() + + cached_inputs = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] + blank_token_id = getattr(getattr(model, "config", None), "pad_token_id", None) + + return { + "tg": tg, + "model": model, + "processor": processor, + "model_config": model_config, + "preprocessor_config": preprocessor_config, + "tokenizer": tokenizer, + "blank_token_id": int(blank_token_id) if blank_token_id is not None else None, + "compile_time_ms": (capture_finished - capture_started) * 1000.0, + "cached_inputs": cached_inputs, + "input_names": tuple(prepared.names), + "prepared_input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + "model_source": model_source, + "weight_bindings": sum( + 1 + for value in captured.ir_graph.values.values() + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) + ), + } + + +def _benchmark_handwritten( + *, + weights_dir: Path, + audio_file: str, + warmup: int, + repeats: int, + profile_repeats: int, + artifact_dir: Path | None, +) -> dict[str, Any]: + model_handle = cactus_init(str(weights_dir), None, False) + try: + _warmup(lambda: _run_handwritten_once(model_handle=model_handle, audio_file=audio_file), warmup) + + runs: list[HandwrittenRun] = [] + for _ in range(max(repeats, 1)): + runs.append(_run_handwritten_once(model_handle=model_handle, audio_file=audio_file)) + + profile_path = ( + artifact_dir / "handwritten_profile.txt" + if artifact_dir is not None and profile_repeats > 0 + else None + ) + with _profile_env(profile_path): + for _ in range(max(profile_repeats, 0)): + _run_handwritten_once(model_handle=model_handle, audio_file=audio_file) + + profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] + profile_summary = _summarize_profile_sections(profile_sections) + + return { + "weights_dir": str(weights_dir), + "runs": [_roundtrip_jsonable(run) for run in runs], + "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), + "internal_total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "confidence": asdict(_numeric_summary([run.confidence for run in runs])), + "latest_transcript": runs[-1].transcript if runs else "", + "profile_file": str(profile_path) if profile_path is not None else None, + "profile_summary": profile_summary, + } + finally: + cactus_destroy(model_handle) + + +def _benchmark_transpiled( + *, + model_id: str, + audio_file: str, + weights_dir: str | None, + warmup: int, + repeats: int, + profile_repeats: int, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, + artifact_dir: Path | None, + fusion_config: FusionConfig, +) -> dict[str, Any]: + built = _build_transpiled_graph( + model_id=model_id, + audio_file=audio_file, + weights_dir=weights_dir, + torch_dtype=torch_dtype, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + fusion_config=fusion_config, + ) + + tg = built["tg"] + model = built["model"] + processor = built["processor"] + input_names = built["input_names"] + model_config = built["model_config"] + preprocessor_config = built["preprocessor_config"] + tokenizer = built["tokenizer"] + blank_token_id = built["blank_token_id"] + cached_inputs = built["cached_inputs"] + + _warmup( + lambda: _run_transpiled_once( + tg=tg, + processor=processor, + input_names=input_names, + model_config=model_config, + preprocessor_config=preprocessor_config, + model=model, + torch_dtype=torch_dtype, + audio_file=audio_file, + tokenizer=tokenizer, + blank_token_id=blank_token_id, + ), + warmup, + ) + _warmup(lambda: _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs), warmup) + + runs: list[TranspiledRun] = [] + graph_only_ms: list[float] = [] + for _ in range(max(repeats, 1)): + runs.append( + _run_transpiled_once( + tg=tg, + processor=processor, + input_names=input_names, + model_config=model_config, + preprocessor_config=preprocessor_config, + model=model, + torch_dtype=torch_dtype, + audio_file=audio_file, + tokenizer=tokenizer, + blank_token_id=blank_token_id, + ) + ) + graph_only_ms.append(_run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs)) + + profile_path = ( + artifact_dir / "transpiled_profile.txt" + if artifact_dir is not None and profile_repeats > 0 + else None + ) + with _profile_env(profile_path): + for _ in range(max(profile_repeats, 0)): + _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs) + + profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] + profile_summary = _summarize_profile_sections(profile_sections) + + return { + "model_source": built["model_source"], + "compile_time_ms": built["compile_time_ms"], + "weight_bindings": built["weight_bindings"], + "prepared_input_shapes": built["prepared_input_shapes"], + "runs": [_roundtrip_jsonable(run) for run in runs], + "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), + "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), + "decode_ms": asdict(_numeric_summary([run.decode_ms for run in runs])), + "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), + "graph_only_ms": asdict(_numeric_summary(graph_only_ms)), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "output_frames": asdict(_numeric_summary([float(run.output_frames) for run in runs])), + "latest_transcript": runs[-1].transcript if runs else "", + "profile_file": str(profile_path) if profile_path is not None else None, + "profile_summary": profile_summary, + } + + +def _print_summary( + *, + audio_duration_sec: float, + handwritten: dict[str, Any] | None, + transpiled: dict[str, Any] | None, +) -> None: + print() + print(f"audio_duration_sec={audio_duration_sec:.3f}") + + if handwritten is not None: + hw_total = handwritten["internal_total_time_ms"]["mean"] + hw_graph = None + profile_total = handwritten.get("profile_summary", {}).get("graph_total_ms") + if isinstance(profile_total, dict): + hw_graph = profile_total.get("mean") + hw_tok = handwritten["decode_tokens"]["mean"] + hw_rtf = hw_total / max(audio_duration_sec * 1000.0, 1e-9) + hw_xrt = (audio_duration_sec * 1000.0) / max(hw_total, 1e-9) + hw_tok_s = hw_tok * 1000.0 / max(hw_total, 1e-9) + print("handwritten:") + print(f" total_ms mean={hw_total:.3f}") + print(f" graph_profile_ms mean={hw_graph:.3f}" if hw_graph is not None else " graph_profile_ms unavailable") + print(f" decode_tokens mean={hw_tok:.3f}") + print(f" output_tokens_per_sec={hw_tok_s:.3f}") + print(f" real_time_factor={hw_rtf:.3f}") + print(f" audio_x_realtime={hw_xrt:.3f}") + + if transpiled is not None: + tr_total = transpiled["total_ms"]["mean"] + tr_graph = transpiled["graph_only_ms"]["mean"] + tr_tok = transpiled["decode_tokens"]["mean"] + tr_rtf = tr_total / max(audio_duration_sec * 1000.0, 1e-9) + tr_xrt = (audio_duration_sec * 1000.0) / max(tr_total, 1e-9) + tr_tok_s = tr_tok * 1000.0 / max(tr_total, 1e-9) + tr_graph_tok_s = tr_tok * 1000.0 / max(tr_graph, 1e-9) + print("transpiled:") + print(f" total_ms mean={tr_total:.3f}") + print(f" graph_only_ms mean={tr_graph:.3f}") + print(f" decode_tokens mean={tr_tok:.3f}") + print(f" output_tokens_per_sec={tr_tok_s:.3f}") + print(f" graph_only_tokens_per_sec={tr_graph_tok_s:.3f}") + print(f" real_time_factor={tr_rtf:.3f}") + print(f" audio_x_realtime={tr_xrt:.3f}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Compare handwritten Cactus Parakeet performance against the transpiled " + "Parakeet graph on the same WAV input, including per-op graph profiles." + ) + ) + parser.add_argument("--model-id", default="nvidia/parakeet-ctc-1.1b") + parser.add_argument("--audio-file", required=True) + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus weights directory. Reused for handwritten runtime and transpiled mmap bindings.", + ) + parser.add_argument("--artifact-dir", default="", help="Optional directory for JSON summaries and raw profiles.") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--profile-repeats", type=int, default=1) + parser.add_argument("--torch-dtype", default="float16") + parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--skip-handwritten", action="store_true") + parser.add_argument("--skip-transpiled", action="store_true") + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + args = parser.parse_args() + + audio_file = str(Path(args.audio_file).resolve()) + if not Path(audio_file).exists(): + raise FileNotFoundError(f"audio file does not exist: {audio_file}") + + torch_dtype = _parse_dtype(args.torch_dtype) + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + + weights_dir_path = None + if args.weights_dir.strip(): + weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) + else: + weights_dir_path = _resolve_default_weights_dir(args.model_id) + + if not args.skip_handwritten and weights_dir_path is None: + raise RuntimeError( + "handwritten comparison requires a converted weights directory.\n" + "\n" + f"Create one with:\n" + f" cactus convert {args.model_id} {PROJECT_ROOT / 'weights' / args.model_id.split('/')[-1].lower()}\n" + "\n" + "Or rerun with --skip-handwritten." + ) + + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + summary: dict[str, Any] = { + "model_id": args.model_id, + "audio_file": audio_file, + "audio_duration_sec": _audio_duration_seconds(audio_file), + "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, + "warmup": args.warmup, + "repeats": args.repeats, + "profile_repeats": args.profile_repeats, + "torch_dtype": args.torch_dtype, + "trust_remote_code": bool(args.trust_remote_code), + "local_files_only": bool(args.local_files_only), + } + + handwritten_summary = None + if not args.skip_handwritten and weights_dir_path is not None: + handwritten_summary = _benchmark_handwritten( + weights_dir=weights_dir_path, + audio_file=audio_file, + warmup=args.warmup, + repeats=args.repeats, + profile_repeats=args.profile_repeats, + artifact_dir=artifact_dir, + ) + summary["handwritten"] = handwritten_summary + + transpiled_summary = None + if not args.skip_transpiled: + transpiled_summary = _benchmark_transpiled( + model_id=args.model_id, + audio_file=audio_file, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + warmup=args.warmup, + repeats=args.repeats, + profile_repeats=args.profile_repeats, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + artifact_dir=artifact_dir, + fusion_config=fusion_config, + ) + summary["transpiled"] = transpiled_summary + + if handwritten_summary is not None and transpiled_summary is not None: + hw_total = handwritten_summary["internal_total_time_ms"]["mean"] + tr_total = transpiled_summary["total_ms"]["mean"] + hw_graph = handwritten_summary.get("profile_summary", {}).get("graph_total_ms") + tr_graph = transpiled_summary["graph_only_ms"]["mean"] + summary["comparison"] = { + "end_to_end_speedup_vs_handwritten": hw_total / max(tr_total, 1e-9), + "graph_only_speedup_vs_handwritten": ( + float(hw_graph["mean"]) / max(tr_graph, 1e-9) + if isinstance(hw_graph, dict) and "mean" in hw_graph + else None + ), + } + + if artifact_dir is not None: + summary_path = artifact_dir / "summary.json" + summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") + print(f"saved_summary={summary_path}") + if handwritten_summary is not None and handwritten_summary.get("profile_file"): + print(f"saved_handwritten_profile={handwritten_summary['profile_file']}") + if transpiled_summary is not None and transpiled_summary.get("profile_file"): + print(f"saved_transpiled_profile={transpiled_summary['profile_file']}") + + _print_summary( + audio_duration_sec=float(summary["audio_duration_sec"]), + handwritten=handwritten_summary, + transpiled=transpiled_summary, + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/inspect_gemma4_cleanup.py b/python/examples/inspect_gemma4_cleanup.py new file mode 100644 index 000000000..ec9e78d05 --- /dev/null +++ b/python/examples/inspect_gemma4_cleanup.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +import torch +from transformers import AutoModelForCausalLM + +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.model_adapters import canonicalize_model_interface + + +class Gemma4FullModelWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = canonicalize_model_interface(model, task="causal_lm_logits").module + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model(input_ids) + + +class Gemma4FirstBlockCheckpointWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module, checkpoint_name: str): + super().__init__() + adapter = canonicalize_model_interface(model, task="causal_lm_logits") + if adapter.family != "gemma4": + raise ValueError(f"expected gemma4 adapter, got family={adapter.family}") + if not hasattr(adapter.module, "debug_first_block"): + raise ValueError("Gemma4 adapter does not expose debug_first_block()") + self.adapter = adapter.module + self.checkpoint_name = checkpoint_name + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + checkpoints = self.adapter.debug_first_block(input_ids) + if self.checkpoint_name not in checkpoints: + available = ", ".join(sorted(checkpoints.keys())) + raise KeyError(f"unknown checkpoint {self.checkpoint_name!r}; available: {available}") + return checkpoints[self.checkpoint_name] + + +def format_ir_graph(graph: IRGraph, *, max_nodes: int | None = None) -> str: + lines: list[str] = [] + lines.append("IR Summary") + lines.append(f" inputs={graph.inputs}") + lines.append(f" outputs={graph.outputs}") + lines.append(f" nodes={len(graph.order)}") + if graph.meta: + lines.append(f" meta={graph.meta}") + lines.append("") + lines.append("IR Nodes") + + node_ids = graph.order if max_nodes is None else graph.order[:max_nodes] + for index, node_id in enumerate(node_ids): + node = graph.nodes[node_id] + lines.append(f"[{index}] {node.id} op={node.op}") + lines.append(f" inputs={node.inputs}") + lines.append(f" outputs={node.outputs}") + if node.attrs: + lines.append(f" attrs={node.attrs}") + if node.kind != "generic": + lines.append(f" kind={node.kind}") + if node.meta: + lines.append(f" meta={node.meta}") + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + lines.append( + f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" + ) + + if max_nodes is not None and len(graph.order) > max_nodes: + lines.append(f"... ({len(graph.order) - max_nodes} more nodes omitted)") + + if graph.constants: + lines.append("") + lines.append("IR Constants") + constant_ids = sorted(graph.constants.keys()) + preview_ids = constant_ids if max_nodes is None else constant_ids[: max(8, min(len(constant_ids), max_nodes))] + for value_id in preview_ids: + value = graph.constants[value_id] + value_meta = graph.values.get(value_id) + shape = None if value_meta is None else value_meta.shape + dtype = None if value_meta is None else value_meta.dtype + lines.append(f"- {value_id}: type={type(value).__name__} shape={shape} dtype={dtype}") + if len(preview_ids) < len(constant_ids): + lines.append(f"... ({len(constant_ids) - len(preview_ids)} more constants omitted)") + + return "\n".join(lines) + + +def format_unsupported_ops(graph: IRGraph) -> str: + counts = graph.meta.get("canonical_unsupported_op_counts", {}) + if not isinstance(counts, dict): + counts = {} + + examples: dict[str, list[str]] = {} + for node_id in graph.order: + node = graph.nodes[node_id] + print(f"node {node_id} op={node.op}") + if node.op not in counts: + continue + bucket = examples.setdefault(node.op, []) + if len(bucket) < 5: + bucket.append(node.id) + + total = sum(int(count) for count in counts.values()) + lines = ["Unsupported Ops", f" unique={len(counts)}", f" total_nodes={total}"] + if not counts: + lines.append(" none") + return "\n".join(lines) + + for op_name, count in counts.items(): + sample_nodes = ", ".join(examples.get(op_name, ())) + lines.append(f"- {op_name}: {count}") + if sample_nodes: + lines.append(f" examples={sample_nodes}") + return "\n".join(lines) + + +def resolve_local_model_path(model_id: str) -> str: + model_path = Path(model_id) + if model_path.exists(): + return str(model_path) + + cache_root = Path.home() / ".cache" / "huggingface" / "hub" / ("models--" + model_id.replace("/", "--")) + snapshots_dir = cache_root / "snapshots" + if not snapshots_dir.exists(): + raise FileNotFoundError( + f"could not find local snapshot for {model_id!r} under {snapshots_dir}" + ) + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + raise FileNotFoundError(f"no snapshots found for {model_id!r} under {snapshots_dir}") + return str(snapshots[-1]) + + +def load_model(model_id: str) -> torch.nn.Module: + token = os.environ.get("HF_TOKEN") + common_kwargs: dict[str, object] = { + "local_files_only": True, + } + if token: + common_kwargs["token"] = token + + local_model_path = resolve_local_model_path(model_id) + model = AutoModelForCausalLM.from_pretrained( + local_model_path, + torch_dtype=torch.float16, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model + + +def build_module( + model: torch.nn.Module, + *, + mode: str, + checkpoint: str, +) -> torch.nn.Module: + if mode == "full": + return Gemma4FullModelWrapper(model).eval() + if mode == "first_block": + return Gemma4FirstBlockCheckpointWrapper(model, checkpoint).eval() + raise ValueError(f"unknown mode: {mode}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture Gemma4, run canonicalize/cleanup.py, and report unsupported ops.", + ) + parser.add_argument( + "--model-id", + default=os.environ.get("CACTUS_GEMMA_HF_MODEL_ID", "google/gemma-4-E2B"), + help="Hugging Face model id. Uses local cache only.", + ) + parser.add_argument( + "--mode", + choices=("first_block", "full"), + default="first_block", + help="Capture a first-block checkpoint or the full Gemma4 adapter graph.", + ) + parser.add_argument( + "--checkpoint", + default="after_ffn_residual", + help="First-block checkpoint name when --mode=first_block.", + ) + parser.add_argument( + "--input-ids", + default="2,818,5279,529,7001,563", + help="Comma-separated integer token ids to capture.", + ) + parser.add_argument( + "--max-nodes", + type=int, + default=120, + help="Maximum IR nodes to print when --print-ir is enabled.", + ) + parser.add_argument( + "--print-ir", + action="store_true", + help="Also print the IR before and after canonical cleanup.", + ) + args = parser.parse_args() + + model = load_model(args.model_id) + module = build_module(model, mode=args.mode, checkpoint=args.checkpoint) + token_ids = [int(token.strip()) for token in args.input_ids.split(",") if token.strip()] + input_ids = torch.tensor([token_ids], dtype=torch.long) + + print(f"Model: {args.model_id}") + print(f"Mode: {args.mode}") + print(f"Checkpoint: {args.checkpoint if args.mode == 'first_block' else '(full model)'}") + print(f"Input IDs shape: {tuple(input_ids.shape)}") + print("") + + captured = capture_model(module, (input_ids,)) + + if args.print_ir: + print("=== IR Before Canonical Cleanup ===") + print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) + print("") + + canonicalize_exported_graph(captured.ir_graph) + + print("=== Unsupported Ops After Canonical Cleanup ===") + print(format_unsupported_ops(captured.ir_graph)) + + if args.print_ir: + print("") + print("=== IR After Canonical Cleanup ===") + print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) + + +if __name__ == "__main__": + main() diff --git a/python/examples/inspect_transpile_cleanup.py b/python/examples/inspect_transpile_cleanup.py new file mode 100644 index 000000000..366765f62 --- /dev/null +++ b/python/examples/inspect_transpile_cleanup.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import argparse + +import torch +import torch.nn as nn + +from src.transpile.capture_pytorch import capture_model +from src.transpile.capture_pytorch import dump_graph +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph + + +class Toy(nn.Module): + def forward(self, x, y): + z = x + y + z = z * 0.5 + z = torch.nn.functional.gelu(z) + z = torch.softmax(z, dim=-1) + return z + + +class AttentionBlockToy(nn.Module): + def __init__(self, hidden_size: int = 8): + super().__init__() + self.out_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=torch.float16) + + def forward(self, q, k, v, gate): + attn = torch.nn.functional.scaled_dot_product_attention(q, k, v, is_causal=True, scale=1.0) + attn = attn.transpose(1, 2).reshape(q.shape[0], q.shape[2], -1) + attn = attn * torch.sigmoid(gate) + attn = attn.to(self.out_proj.weight.dtype) + return self.out_proj(attn) + + +def build_case(name: str) -> tuple[nn.Module, tuple[torch.Tensor, ...]]: + if name == "toy": + model = Toy().eval() + args = ( + torch.randn(2, 4, dtype=torch.float16), + torch.randn(2, 4, dtype=torch.float16), + ) + return model, args + + if name == "attention_block": + model = AttentionBlockToy().eval() + args = ( + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 3, 8, dtype=torch.float16), + ) + return model, args + + raise ValueError(f"unknown case: {name}") + + +def format_ir_graph(graph: IRGraph) -> str: + lines: list[str] = [] + lines.append("IR Summary") + lines.append(f" inputs={graph.inputs}") + lines.append(f" outputs={graph.outputs}") + lines.append(f" nodes={len(graph.order)}") + if graph.meta: + lines.append(f" meta={graph.meta}") + lines.append("") + lines.append("IR Nodes") + + for index, node_id in enumerate(graph.order): + node = graph.nodes[node_id] + lines.append(f"[{index}] {node.id} op={node.op}") + lines.append(f" inputs={node.inputs}") + lines.append(f" outputs={node.outputs}") + if node.attrs: + lines.append(f" attrs={node.attrs}") + if node.kind != "generic": + lines.append(f" kind={node.kind}") + if node.meta: + lines.append(f" meta={node.meta}") + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + lines.append( + f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" + ) + + if graph.constants: + lines.append("") + lines.append("IR Constants") + for value_id, value in graph.constants.items(): + value_meta = graph.values.get(value_id) + shape = None if value_meta is None else value_meta.shape + dtype = None if value_meta is None else value_meta.dtype + typename = type(value).__name__ + lines.append(f"- {value_id}: type={typename} shape={shape} dtype={dtype}") + + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture a small PyTorch model, dump the exported graph, and print IR before/after cleanup.", + ) + parser.add_argument( + "--case", + choices=("toy", "attention_block"), + default="toy", + help="Which built-in model to inspect.", + ) + parser.add_argument( + "--no-meta", + action="store_true", + help="Hide exported graph metadata to keep the dump shorter.", + ) + args = parser.parse_args() + + model, example_args = build_case(args.case) + captured = capture_model(model, example_args) + + print(f"Case: {args.case}") + print(f"Model: {type(model).__name__}") + print("") + print("=== Exported Graph ===") + print(dump_graph(captured, include_meta=not args.no_meta)) + print("") + print("=== IR Before Cleanup ===") + print(format_ir_graph(captured.ir_graph)) + print("") + + canonicalize_exported_graph(captured.ir_graph) + + print("=== IR After Cleanup ===") + print(format_ir_graph(captured.ir_graph)) + + +if __name__ == "__main__": + main() diff --git a/python/examples/test_torch_capture.py b/python/examples/test_torch_capture.py new file mode 100644 index 000000000..c2ff748b4 --- /dev/null +++ b/python/examples/test_torch_capture.py @@ -0,0 +1,70 @@ +import torch +import torch.nn as nn + +from src.transpile.capture_pytorch import ( + capture_model_with_fallback, + dump_graph, + get_dtype, + get_shape, +) +from src.transpile.lower import transpile_captured + + +class TinyBlock(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(8, 16) + self.fc2 = nn.Linear(16, 4) + + def forward(self, x): + h = self.fc1(x) + h = torch.relu(h) + y = self.fc2(h) + return y + + +def main(): + model = TinyBlock().eval() + x = torch.randn(2, 8) + + captured = capture_model_with_fallback(model, args=(x,)) + + print("strict:", captured.strict) + print("graph_module type:", type(captured.graph_module).__name__) + print("state_dict keys:", list(captured.state_dict.keys())) + print() + + print("=== FX Graph ===") + print(captured.graph) + print() + + print("=== Node Walk ===") + for i, node in enumerate(captured.graph.nodes): + print(f"[{i}] op={node.op} name={node.name} target={node.target}") + print(f" args={node.args}") + print(f" kwargs={node.kwargs}") + print(f" shape={get_shape(node)} dtype={get_dtype(node)}") + print() + + print("=== Full Dump ===") + print(dump_graph(captured)) + print() + + print("=== Lower To Cactus ===") + try: + transpiled = transpile_captured(captured) + print("lowering succeeded") + print("lowered graph type:", type(transpiled.graph).__name__) + print("runtime inputs:", transpiled.runtime_inputs) + print("bound constants:", transpiled.bound_constants) + print("outputs:", transpiled.outputs) + print() + print("binding runtime input 0 from example tensor") + transpiled.set_input(0, x) + except NotImplementedError as exc: + print("lowering stopped on unimplemented case:") + print(f" {exc}") + + +if __name__ == "__main__": + main() diff --git a/python/examples/transpile_hf_causal_lm.py b/python/examples/transpile_hf_causal_lm.py new file mode 100644 index 000000000..05e70b8c9 --- /dev/null +++ b/python/examples/transpile_hf_causal_lm.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import runpy +import sys +from pathlib import Path + + +def main() -> int: + if "--task" not in sys.argv[1:]: + sys.argv[1:1] = ["--task", "causal_lm_logits"] + script_path = Path(__file__).with_name("transpile_hf_model.py") + runpy.run_path(str(script_path), run_name="__main__") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/transpile_hf_model.py b/python/examples/transpile_hf_model.py new file mode 100644 index 000000000..8a86c7d7d --- /dev/null +++ b/python/examples/transpile_hf_model.py @@ -0,0 +1,1734 @@ +from __future__ import annotations + +import argparse +import builtins +import copy +import gc +import importlib.util +import json +import os +import re +import sys +from collections import Counter +from dataclasses import dataclass +from dataclasses import fields +from functools import lru_cache +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from scipy.io import wavfile +from scipy.signal import resample_poly + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.graph import Graph +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import verify_ir +from src.transpile.lower import TranspiledGraph +from src.transpile.lower import _lower_constant_value +from src.transpile.lower import _lower_input_value +from src.transpile.lower import _lower_ir_node +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph + + +@dataclass +class PreparedInputs: + names: tuple[str, ...] + tensors: tuple[torch.Tensor, ...] + metadata: dict[str, object] + + +def _transformers_supports_model_module(module_name: str) -> bool: + try: + return importlib.util.find_spec(module_name) is not None + except Exception: + return False + + +def _candidate_external_site_packages() -> list[Path]: + candidates: list[Path] = [] + major_minor = f"python{sys.version_info.major}.{sys.version_info.minor}" + pyenv_versions = Path.home() / ".pyenv" / "versions" + if pyenv_versions.exists(): + for version_dir in sorted(pyenv_versions.iterdir(), reverse=True): + site_packages = version_dir / "lib" / major_minor / "site-packages" + if site_packages.exists(): + candidates.append(site_packages) + return candidates + + +def _ensure_transformers_supports_model_type(model_type: str) -> str | None: + normalized = str(model_type or "").strip().lower() + target_module = { + "gemma4": "transformers.models.gemma4.modeling_gemma4", + }.get(normalized) + if not target_module: + return None + if _transformers_supports_model_module(target_module): + return None + + for site_packages in _candidate_external_site_packages(): + candidate = site_packages / Path(*target_module.split(".")) + if not candidate.with_suffix(".py").exists() and not candidate.is_dir(): + continue + sys.path.insert(0, str(site_packages)) + for module_name in list(sys.modules): + root_name = module_name.split(".", 1)[0] + if root_name in {"transformers", "huggingface_hub", "tokenizers"}: + del sys.modules[module_name] + if _transformers_supports_model_module(target_module): + return str(site_packages) + try: + sys.path.remove(str(site_packages)) + except ValueError: + pass + return None + + +def _resolve_local_snapshot(model_id_or_path: str) -> str | None: + explicit = Path(model_id_or_path) + if explicit.exists(): + return str(explicit) + + snapshots_dir = ( + Path.home() + / ".cache" + / "huggingface" + / "hub" + / ("models--" + model_id_or_path.replace("/", "--")) + / "snapshots" + ) + if not snapshots_dir.exists(): + return None + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + return None + return str(snapshots[-1]) + + +def _snapshot_has_model_weights(path: str | Path) -> bool: + root = Path(path) + if not root.exists() or not root.is_dir(): + return False + candidates = ( + "model.safetensors", + "model.safetensors.index.json", + "pytorch_model.bin", + "pytorch_model.bin.index.json", + ) + return any((root / name).exists() for name in candidates) + + +def _validate_weights_dir(weights_dir: str | None, *, model_id: str) -> Path | None: + if not weights_dir: + return None + + root = Path(weights_dir).resolve() + if not root.exists(): + raise RuntimeError( + f"weights_dir does not exist: {root}\n" + "\n" + f"Create the folder first with:\n" + f" cactus convert {model_id} {root}\n" + ) + + manifest_path = root / "weights_manifest.json" + if not manifest_path.exists(): + raise RuntimeError( + f"weights_dir is missing weights_manifest.json: {manifest_path}\n" + "\n" + f"Re-convert with the current converter:\n" + f" cactus convert {model_id} {root}\n" + ) + + return root + + +def _serialize_json_compatible(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _serialize_json_compatible(inner) for key, inner in value.items()} + if isinstance(value, (list, tuple)): + return [_serialize_json_compatible(inner) for inner in value] + if isinstance(value, Path): + return str(value) + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.dtype): + return str(value) + if isinstance(value, torch.Tensor): + return { + "type": "torch.Tensor", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if isinstance(value, np.ndarray): + return { + "type": "numpy.ndarray", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if hasattr(value, "__dataclass_fields__"): + return { + field.name: _serialize_json_compatible(getattr(value, field.name)) + for field in fields(value) + } + try: + return repr(value) + except Exception: + return f"<{type(value).__module__}.{type(value).__name__}>" + + +def _graph_to_dict(graph) -> dict[str, object]: + return { + "meta": _serialize_json_compatible(graph.meta), + "inputs": list(graph.inputs), + "outputs": list(graph.outputs), + "constants": { + value_id: _serialize_json_compatible(constant) + for value_id, constant in graph.constants.items() + }, + "values": { + value_id: _serialize_json_compatible(value) + for value_id, value in graph.values.items() + }, + "nodes": [ + _serialize_json_compatible(graph.nodes[node_id]) + for node_id in graph.order + ], + } + + +def _write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def _parse_dtype(name: str) -> torch.dtype: + normalized = name.strip().lower() + mapping = { + "float16": torch.float16, + "fp16": torch.float16, + "float32": torch.float32, + "fp32": torch.float32, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + } + if normalized not in mapping: + raise ValueError(f"unsupported torch dtype: {name}") + return mapping[normalized] + + +class TranspileWrapper(torch.nn.Module): + def __init__(self, adapter_module: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__() + self.adapter = adapter_module + self.weights_dir = weights_dir + + def forward(self, *bound_inputs: torch.Tensor) -> torch.Tensor: + return self.adapter(*bound_inputs) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.adapter, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta: dict[str, object] = {} + base_graph = metadata.get("graph", {}) + if isinstance(base_graph, dict): + graph_meta.update(base_graph) + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir + metadata["graph"] = graph_meta + return metadata + + +def _patch_transformers_torchvision_probe() -> str | None: + has_torchvision = importlib.util.find_spec("torchvision") is not None + has_lzma = importlib.util.find_spec("_lzma") is not None + + if not has_torchvision or has_lzma: + return None + + class _InterpolationModeStub: + NEAREST = "nearest" + BILINEAR = "bilinear" + BICUBIC = "bicubic" + LANCZOS = "lanczos" + + class _TorchvisionFunctionalStub: + InterpolationMode = _InterpolationModeStub + + def __getattr__(self, name: str): + raise RuntimeError( + "torchvision functionality is unavailable because this Python build is missing _lzma; " + f"attempted to access torchvision.transforms.functional.{name}" + ) + + # Some Transformers multimodal image/video modules reference `F.InterpolationMode` + # or `tvF.InterpolationMode` in annotations even when torchvision probing is disabled. + # Installing a tiny builtins-level stub lets those modules import so the PIL fallback + # classes can be selected instead. + builtins.F = _TorchvisionFunctionalStub() + builtins.tvF = builtins.F + + import transformers.utils as tf_utils # type: ignore + import transformers.utils.import_utils as tf_import_utils # type: ignore + + @lru_cache + def _disabled() -> bool: + return False + + tf_import_utils.is_torchvision_available = _disabled + tf_import_utils.is_torchvision_v2_available = _disabled + tf_utils.is_torchvision_available = _disabled + tf_utils.is_torchvision_v2_available = _disabled + return "disabled torchvision import checks because this Python build is missing _lzma" + + +def _patch_torch_flex_attention_compat() -> str | None: + try: + import torch.nn.attention.flex_attention as flex_attention # type: ignore + except Exception: + return None + + if hasattr(flex_attention, "AuxRequest"): + return None + + class _AuxRequest: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + flex_attention.AuxRequest = _AuxRequest # type: ignore[attr-defined] + return "installed torch flex_attention AuxRequest compatibility stub" + + +def _load_local_torch_state_dict(model_source: str) -> dict[str, torch.Tensor] | None: + root = Path(model_source) + if not root.exists() or not root.is_dir(): + return None + + safetensors_path = root / "model.safetensors" + if safetensors_path.exists(): + try: + from safetensors.torch import load_file # type: ignore + + return load_file(str(safetensors_path)) + except Exception: + return None + + pytorch_path = root / "pytorch_model.bin" + if pytorch_path.exists(): + try: + loaded = torch.load(str(pytorch_path), map_location="cpu", weights_only=True) + except TypeError: + loaded = torch.load(str(pytorch_path), map_location="cpu") + return loaded if isinstance(loaded, dict) else None + + return None + + +def _remap_gemma4_checkpoint_state_dict(state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + remapped: dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + if "audio_tower" in new_key: + new_key = re.sub(r"subsample_conv_projection\.layer(\d+)\.", r"subsample_conv_projection.conv_\1.", new_key) + new_key = new_key.replace("audio_tower.layers.", "audio_tower.conformer.") + new_key = new_key.replace(".feed_forward1.", ".ffw_layer_start.") + new_key = new_key.replace(".feed_forward2.", ".ffw_layer_end.") + new_key = re.sub(r"\.self_attn\.(q_proj|k_proj|v_proj)\.", r".attention.attn.\1.", new_key) + new_key = new_key.replace(".self_attn.per_dim_scale", ".attention.attn.per_dim_scale") + new_key = new_key.replace(".self_attn.relative_k_proj.", ".attention.attn.relative_position_embedding.pos_proj.") + new_key = new_key.replace(".self_attn.post.", ".attention.post.") + new_key = new_key.replace(".norm_pre_attn.", ".attention.pre_attn_norm.") + new_key = new_key.replace(".norm_post_attn.", ".attention.post_norm.") + new_key = new_key.replace(".norm_out.", ".norm.") + new_key = new_key.replace(".linear.weight", ".weight") + remapped[new_key] = value + if new_key.endswith(".attention.attn.per_dim_scale"): + remapped[new_key.replace(".per_dim_scale", ".per_dim_key_scale")] = value + if "lm_head.weight" not in remapped: + tied_embedding = remapped.get("model.language_model.embed_tokens.weight") + if tied_embedding is not None: + remapped["lm_head.weight"] = tied_embedding + return remapped + + +def _repair_gemma4_checkpoint_weights(model: torch.nn.Module, model_source: str) -> dict[str, object]: + raw_state_dict = _load_local_torch_state_dict(model_source) + if raw_state_dict is None: + return {"applied": False, "reason": "no local checkpoint state_dict"} + + remapped_state_dict = _remap_gemma4_checkpoint_state_dict(raw_state_dict) + load_result = model.load_state_dict(remapped_state_dict, strict=False) + return { + "applied": True, + "missing_keys": list(load_result.missing_keys), + "unexpected_keys": list(load_result.unexpected_keys), + } + + +def _load_config_json(model_id_or_path: str) -> dict[str, object]: + local_snapshot = _resolve_local_snapshot(model_id_or_path) + config_source = Path(local_snapshot) / "config.json" if local_snapshot else None + if config_source is not None and config_source.exists(): + return json.loads(config_source.read_text()) + explicit = Path(model_id_or_path) / "config.json" + if explicit.exists(): + return json.loads(explicit.read_text()) + return {} + + +def _load_optional_json(model_id_or_path: str, filename: str) -> dict[str, object]: + local_snapshot = _resolve_local_snapshot(model_id_or_path) + candidate = Path(local_snapshot) / filename if local_snapshot else None + if candidate is not None and candidate.exists(): + return json.loads(candidate.read_text()) + explicit = Path(model_id_or_path) / filename + if explicit.exists(): + return json.loads(explicit.read_text()) + return {} + + +def _load_gemma4_tokenizer_fallback(source_candidates: list[str]) -> object | None: + try: + from transformers import PreTrainedTokenizerFast # type: ignore + except Exception: + return None + + for source in source_candidates: + root = Path(source) + if not root.exists() or not root.is_dir(): + continue + tokenizer_json = root / "tokenizer.json" + tokenizer_config_path = root / "tokenizer_config.json" + if not tokenizer_json.exists() or not tokenizer_config_path.exists(): + continue + + tokenizer_config = json.loads(tokenizer_config_path.read_text()) + tokenizer = PreTrainedTokenizerFast( + tokenizer_file=str(tokenizer_json), + bos_token=tokenizer_config.get("bos_token"), + eos_token=tokenizer_config.get("eos_token"), + unk_token=tokenizer_config.get("unk_token"), + pad_token=tokenizer_config.get("pad_token"), + mask_token=tokenizer_config.get("mask_token"), + padding_side=str(tokenizer_config.get("padding_side", "right")), + additional_special_tokens=list(tokenizer_config.get("extra_special_tokens", []) or []), + ) + model_max_length = tokenizer_config.get("model_max_length") + if isinstance(model_max_length, int): + tokenizer.model_max_length = model_max_length + + chat_template_path = root / "chat_template.jinja" + if chat_template_path.exists(): + tokenizer.chat_template = chat_template_path.read_text() + + for token_attr in ( + "image_token", + "audio_token", + "boi_token", + "eoi_token", + "boa_token", + "eoa_token", + ): + token_value = tokenizer_config.get(token_attr) + if isinstance(token_value, str): + setattr(tokenizer, token_attr, token_value) + setattr(tokenizer, f"{token_attr}_id", tokenizer.convert_tokens_to_ids(token_value)) + return tokenizer + + return None + + +def _load_gemma4_processor_fallback( + *, + source_candidates: list[str], + common_kwargs: dict[str, object], +) -> object | None: + try: + from transformers.models.gemma4.feature_extraction_gemma4 import Gemma4AudioFeatureExtractor # type: ignore + from transformers.models.gemma4.image_processing_gemma4 import Gemma4ImageProcessor # type: ignore + from transformers.models.gemma4.processing_gemma4 import Gemma4Processor # type: ignore + except Exception: + return None + + tokenizer = None + processor_config: dict[str, object] = {} + for source in source_candidates: + try: + try: + from transformers import AutoTokenizer # type: ignore + + tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) + except Exception: + tokenizer = _load_gemma4_tokenizer_fallback([source]) + processor_config = _load_optional_json(source, "processor_config.json") + if tokenizer is not None and processor_config: + break + except Exception: + continue + if tokenizer is None or not processor_config: + return None + + feature_config = dict(processor_config.get("feature_extractor", {}) or {}) + image_config = dict(processor_config.get("image_processor", {}) or {}) + feature_config.pop("feature_extractor_type", None) + image_config.pop("image_processor_type", None) + + feature_extractor = Gemma4AudioFeatureExtractor(**feature_config) + image_processor = Gemma4ImageProcessor(**image_config) + + processor_kwargs = {} + for key in ("image_seq_length", "audio_seq_length", "audio_ms_per_token"): + if key in processor_config: + processor_kwargs[key] = processor_config[key] + + return Gemma4Processor( + feature_extractor=feature_extractor, + image_processor=image_processor, + tokenizer=tokenizer, + **processor_kwargs, + ) + + +def _infer_task_from_config(model_id_or_path: str) -> str: + config = _load_config_json(model_id_or_path) + architectures = [str(value) for value in config.get("architectures", []) if isinstance(value, str)] + model_type = str(config.get("model_type", "") or "").lower() + + if any("CausalLM" in value for value in architectures): + return "causal_lm_logits" + if any("CTC" in value for value in architectures): + return "ctc_logits" + if model_type == "whisper": + return "encoder_hidden_states" + if any("ConditionalGeneration" in value and "Whisper" in value for value in architectures): + return "encoder_hidden_states" + + lowered_id = model_id_or_path.lower() + if "whisper" in lowered_id: + return "encoder_hidden_states" + if "ctc" in lowered_id: + return "ctc_logits" + if any(token in lowered_id for token in ("qwen", "gemma", "llama", "mistral")): + return "causal_lm_logits" + + raise RuntimeError( + f"Could not infer transpile task for {model_id_or_path}.\n" + "\n" + "Pass one explicitly with --task, for example:\n" + " --task causal_lm_logits\n" + " --task ctc_logits\n" + " --task encoder_hidden_states\n" + ) + + +def _normalize_audio_samples(samples: np.ndarray) -> np.ndarray: + if samples.dtype == np.uint8: + return ((samples.astype(np.float32) - 128.0) / 128.0).astype(np.float32) + if np.issubdtype(samples.dtype, np.integer): + info = np.iinfo(samples.dtype) + scale = float(max(abs(info.min), info.max)) + if scale <= 0: + scale = 1.0 + return (samples.astype(np.float32) / scale).astype(np.float32) + return samples.astype(np.float32) + + +def _hz_to_mel(hz: np.ndarray) -> np.ndarray: + return 2595.0 * np.log10(1.0 + hz / 700.0) + + +def _mel_to_hz(mel: np.ndarray) -> np.ndarray: + return 700.0 * (np.power(10.0, mel / 2595.0) - 1.0) + + +@lru_cache(maxsize=16) +def _mel_filter_bank(num_mels: int, sample_rate: int, n_fft: int) -> np.ndarray: + try: + from transformers.audio_utils import mel_filter_bank as transformers_mel_filter_bank # type: ignore + + return transformers_mel_filter_bank( + num_frequency_bins=n_fft // 2 + 1, + num_mel_filters=num_mels, + min_frequency=0.0, + max_frequency=sample_rate / 2.0, + sampling_rate=sample_rate, + norm="slaney", + mel_scale="slaney", + ).astype(np.float32) + except Exception: + pass + + num_freq_bins = n_fft // 2 + 1 + mel_points = np.linspace( + _hz_to_mel(np.array([0.0], dtype=np.float32))[0], + _hz_to_mel(np.array([sample_rate / 2.0], dtype=np.float32))[0], + num_mels + 2, + ) + hz_points = _mel_to_hz(mel_points) + bins = np.floor((n_fft + 1) * hz_points / sample_rate).astype(np.int64) + + filters = np.zeros((num_mels, num_freq_bins), dtype=np.float32) + for idx in range(1, num_mels + 1): + left = max(0, min(num_freq_bins - 1, int(bins[idx - 1]))) + center = max(left + 1, min(num_freq_bins - 1, int(bins[idx]))) + right = max(center + 1, min(num_freq_bins, int(bins[idx + 1]))) + for freq in range(left, center): + filters[idx - 1, freq] = float(freq - left) / float(max(center - left, 1)) + for freq in range(center, right): + filters[idx - 1, freq] = float(right - freq) / float(max(right - center, 1)) + return filters + + +def _load_audio_waveform(audio_file: str | Path, *, target_sample_rate: int) -> np.ndarray: + sample_rate, waveform = wavfile.read(str(audio_file)) + if waveform.ndim > 1: + waveform = waveform.mean(axis=1) + waveform = _normalize_audio_samples(np.asarray(waveform)) + + if int(sample_rate) != int(target_sample_rate): + gcd = int(np.gcd(int(sample_rate), int(target_sample_rate))) + waveform = resample_poly( + waveform, + int(target_sample_rate) // gcd, + int(sample_rate) // gcd, + ).astype(np.float32) + + return waveform.astype(np.float32) + + +def _generic_log_mel_features( + waveform: np.ndarray, + *, + sample_rate: int, + num_mels: int, + n_fft: int, + hop_length: int, + frame_length: int, + preemphasis: float | None = None, +) -> tuple[np.ndarray, int]: + waveform_tensor = torch.from_numpy(waveform).to(torch.float32).unsqueeze(0) + if preemphasis is not None and abs(preemphasis) > 0.0: + waveform_tensor = torch.cat( + [ + waveform_tensor[:, :1], + waveform_tensor[:, 1:] - float(preemphasis) * waveform_tensor[:, :-1], + ], + dim=1, + ) + + window = torch.hann_window(frame_length, periodic=False, dtype=torch.float32) + stft = torch.stft( + waveform_tensor, + n_fft=n_fft, + hop_length=hop_length, + win_length=frame_length, + window=window, + return_complex=True, + pad_mode="constant", + ) + magnitudes = torch.view_as_real(stft) + magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1)) + power = magnitudes.pow(2) + + mel_filters = torch.from_numpy(_mel_filter_bank(num_mels, sample_rate, n_fft)).to(torch.float32) + mel_spec = power.transpose(1, 2) @ mel_filters + mel_spec = torch.log(mel_spec + np.float32(2**-24)) + + feature_lengths = max(1, int(waveform.shape[0] // hop_length)) + attention_mask = ( + torch.arange(mel_spec.shape[1], dtype=torch.long, device=mel_spec.device).unsqueeze(0) < feature_lengths + ) + mask = attention_mask.unsqueeze(-1).to(torch.float32) + masked = mel_spec * mask + mean = masked.sum(dim=1) / float(feature_lengths) + centered = masked - mean.unsqueeze(1) + denom = float(max(1, feature_lengths - 1)) + variance = (centered.pow(2) * mask).sum(dim=1) / denom + mel_spec = (mel_spec - mean.unsqueeze(1)) / (torch.sqrt(variance).unsqueeze(1) + np.float32(1e-5)) + mel_spec = mel_spec * mask + return mel_spec[0].cpu().numpy().astype(np.float32), feature_lengths + + +def _resolve_audio_sample_rate(processor: object) -> int: + for attr_name in ("feature_extractor", "tokenizer"): + child = getattr(processor, attr_name, None) + sample_rate = getattr(child, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + sample_rate = getattr(processor, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + return 16000 + + +def _infer_fallback_audio_input_names(config: dict[str, object], task: str) -> tuple[str, ...]: + model_type = str(config.get("model_type", "") or "").lower() + if model_type == "whisper": + return ("input_features",) + if task == "encoder_hidden_states": + return ("input_features",) + audio_cfg = config.get("audio_config") + if isinstance(audio_cfg, dict) and any(key in audio_cfg for key in ("features", "input_feat_size", "num_mel_bins")): + return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) + encoder_cfg = config.get("encoder") + if isinstance(encoder_cfg, dict) and any(key in encoder_cfg for key in ("feat_in", "num_mel_bins")): + return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) + return ("input_values", "attention_mask") if task == "ctc_logits" else ("input_values",) + + +def _resolve_encoder_module(model: torch.nn.Module) -> torch.nn.Module | None: + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder = get_encoder() + if isinstance(encoder, torch.nn.Module): + return encoder + encoder = getattr(model, "encoder", None) + if isinstance(encoder, torch.nn.Module): + return encoder + model_attr = getattr(model, "model", None) + if model_attr is not None: + encoder = getattr(model_attr, "encoder", None) + if isinstance(encoder, torch.nn.Module): + return encoder + return None + + +def _infer_expected_input_feature_frames(model: torch.nn.Module) -> int | None: + config = getattr(model, "config", None) + max_source_positions = getattr(config, "max_source_positions", None) + if not isinstance(max_source_positions, int) or max_source_positions <= 0: + return None + + encoder = _resolve_encoder_module(model) + if encoder is None: + return None + + stride_product = 1 + found_conv = False + for child in encoder.children(): + if isinstance(child, torch.nn.Conv1d): + stride = child.stride[0] if isinstance(child.stride, tuple) else child.stride + stride_product *= int(stride) + found_conv = True + + if not found_conv: + return None + return int(max_source_positions) * stride_product + + +def _prepare_fallback_audio_inputs( + *, + input_names: tuple[str, ...], + config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + task: str, + audio_file: str, + torch_dtype: torch.dtype, +) -> PreparedInputs: + if not input_names: + input_names = _infer_fallback_audio_input_names(config, task) + sample_rate = int(preprocessor_config.get("sampling_rate", config.get("sampling_rate", 16000)) or 16000) + num_mels = int(preprocessor_config.get("feature_size", config.get("num_mel_bins", config.get("feature_size", 80))) or 80) + encoder_cfg = config.get("encoder_config") + if isinstance(encoder_cfg, dict): + num_mels = int(encoder_cfg.get("num_mel_bins", encoder_cfg.get("feat_in", num_mels)) or num_mels) + hop_length = int(preprocessor_config.get("hop_length", config.get("hop_length", 160)) or 160) + n_fft = int(preprocessor_config.get("n_fft", config.get("n_fft", 400)) or 400) + frame_length = int(preprocessor_config.get("win_length", preprocessor_config.get("frame_length", config.get("frame_length", n_fft))) or n_fft) + preemphasis = preprocessor_config.get("preemphasis") + if preemphasis is not None: + preemphasis = float(preemphasis) + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + + tensors: list[torch.Tensor] = [] + if input_names and input_names[0] == "input_features": + features, feature_length = _generic_log_mel_features( + waveform, + sample_rate=sample_rate, + num_mels=num_mels, + n_fft=n_fft, + hop_length=hop_length, + frame_length=frame_length, + preemphasis=preemphasis, + ) + expected_frames = _infer_expected_input_feature_frames(model) + attention_mask: np.ndarray | None = None + target_frames = feature_length + if isinstance(expected_frames, int) and expected_frames > 0: + target_frames = expected_frames + + active_frames = min(feature_length, target_frames) + features = features[:active_frames, :] + if target_frames > active_frames: + pad_width = target_frames - active_frames + features = np.pad(features, ((0, pad_width), (0, 0)), mode="constant") + attention_mask = np.zeros((target_frames,), dtype=np.bool_) + attention_mask[:active_frames] = True + + tensors.append(torch.from_numpy(features).unsqueeze(0).to(dtype=torch_dtype)) + if attention_mask is not None and len(input_names) > 1 and input_names[1] == "attention_mask": + tensors.append(torch.from_numpy(attention_mask).unsqueeze(0)) + else: + input_values = torch.from_numpy(waveform).unsqueeze(0).to(dtype=torch_dtype) + tensors.append(input_values) + if len(input_names) > 1 and input_names[1] == "attention_mask": + tensors.append(torch.ones_like(input_values, dtype=torch.float32)) + + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + + +def _prepare_audio_inputs( + processor: object | None, + *, + input_names: tuple[str, ...], + config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + task: str, + audio_file: str, + torch_dtype: torch.dtype, +) -> PreparedInputs: + if processor is None: + return _prepare_fallback_audio_inputs( + input_names=input_names, + config=config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + + sample_rate = _resolve_audio_sample_rate(processor) + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + batch = processor( + waveform, + sampling_rate=sample_rate, + return_tensors="pt", + ) + + preferred_keys = tuple(input_names) + tuple( + key for key in ("input_features", "input_values", "attention_mask") if key not in input_names + ) + tensor_keys = [key for key, value in batch.items() if isinstance(value, torch.Tensor)] + ordered_keys = [key for key in preferred_keys if key in tensor_keys] + ordered_keys.extend(key for key in tensor_keys if key not in ordered_keys) + if not ordered_keys: + raise RuntimeError(f"processor did not return tensor inputs for audio file: {audio_file}") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": False, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + }, + ) + + +def _prepare_text_inputs(tokenizer: object, *, prompt: str, input_ids_text: str | None) -> PreparedInputs: + if input_ids_text: + token_ids = [int(part.strip()) for part in input_ids_text.split(",") if part.strip()] + if not token_ids: + raise ValueError("--input-ids was provided but no ids were parsed") + input_ids = torch.tensor([token_ids], dtype=torch.long) + else: + input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"] + return PreparedInputs( + names=("input_ids",), + tensors=(input_ids,), + metadata={ + "prompt": prompt, + "input_ids": input_ids.tolist(), + }, + ) + + +_GEMMA4_MULTIMODAL_INPUT_ORDER = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", +) + + +def _normalize_multimodal_prompt( + prompt: str, + *, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, +) -> str: + normalized = prompt.strip() + prefixes: list[str] = [] + + if image_token and num_images > 0: + image_count = normalized.count(image_token) + if image_count < num_images: + prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) + if audio_token and num_audio_segments > 0: + audio_count = normalized.count(audio_token) + if audio_count < num_audio_segments: + prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) + + if prefixes: + prefix = "\n".join(part for part in prefixes if part) + if normalized: + return f"{prefix}\n{normalized}" + return prefix + return normalized + + +def _build_gemma4_chat_prompt( + *, + prompt: str, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> str: + result = "" + normalized_system = system_prompt.strip() + if enable_thinking_if_supported or normalized_system: + result += "<|turn>system\n" + if enable_thinking_if_supported: + result += "<|think|>" + result += normalized_system + result += "\n" + + result += "<|turn>user\n" + if image_token and num_images > 0: + for _ in range(num_images): + result += f"\n\n{image_token}\n\n" + result += prompt + if audio_token and num_audio_segments > 0: + result += "".join(audio_token for _ in range(num_audio_segments)) + result += "\n" + result += "<|turn>model\n" + return result + + +def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: + if not image_files: + return [] + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for --image-file: {exc}") from exc + + images: list[object] = [] + for image_file in image_files: + path = Path(image_file).resolve() + if not path.exists(): + raise RuntimeError(f"image_file does not exist: {path}") + with Image.open(path) as image: + images.append(image.convert("RGB").copy()) + return images + + +def _prepare_gemma4_multimodal_inputs( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, + use_gemma4_chat_template: bool = False, +) -> PreparedInputs: + if processor is None: + raise RuntimeError("multimodal Gemma4 transpile requires an AutoProcessor with image and audio support") + + images = _load_image_inputs(image_files) + audio_waveforms: list[np.ndarray] = [] + sample_rate: int | None = None + if audio_file: + sample_rate = _resolve_audio_sample_rate(processor) + audio_waveforms.append(_load_audio_waveform(audio_file, target_sample_rate=sample_rate)) + + image_token = getattr(processor, "image_token", None) + audio_token = getattr(processor, "audio_token", None) + processor_prompt = _normalize_multimodal_prompt( + prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + ) + normalized_prompt = processor_prompt + if use_gemma4_chat_template: + normalized_prompt = prompt.strip() + processor_prompt = _build_gemma4_chat_prompt( + prompt=normalized_prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + + batch = processor( + text=processor_prompt, + images=images or None, + audio=audio_waveforms or None, + return_tensors="pt", + ) + + ordered_keys = [ + key + for key in _GEMMA4_MULTIMODAL_INPUT_ORDER + if isinstance(batch.get(key), torch.Tensor) + ] + if not ordered_keys: + raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + metadata: dict[str, object] = { + "prompt": normalized_prompt, + "processor_prompt": processor_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + } + if audio_file: + metadata["audio_file"] = str(Path(audio_file).resolve()) + if sample_rate is not None: + metadata["sample_rate"] = sample_rate + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata=metadata, + ) + + +def _load_model_source(model_id: str, *, local_files_only: bool) -> str: + local_snapshot = _resolve_local_snapshot(model_id) + if local_snapshot and _snapshot_has_model_weights(local_snapshot): + return local_snapshot + if local_snapshot and local_files_only: + raise RuntimeError( + f"Found local snapshot for {model_id}, but it is incomplete and has no model weights:\n" + f" {local_snapshot}\n" + "\n" + "Re-run without --local-files-only to let transformers download the missing weights." + ) + return model_id + + +def _load_transformers_bundle( + *, + model_id: str, + task: str, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, +): + config = _load_config_json(model_id) + config_model_type = str(config.get("model_type", "") or "").lower() + external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) + patch_note = _patch_transformers_torchvision_probe() + if patch_note: + print(f"note={patch_note}") + flex_patch_note = _patch_torch_flex_attention_compat() + if flex_patch_note: + print(f"note={flex_patch_note}") + if external_transformers_site_packages: + print(f"note=using external transformers install for {config_model_type}: {external_transformers_site_packages}") + + try: + from transformers import AutoFeatureExtractor # type: ignore + from transformers import AutoModel # type: ignore + from transformers import AutoModelForCTC # type: ignore + from transformers import AutoModelForCausalLM # type: ignore + from transformers import AutoModelForSeq2SeqLM # type: ignore + from transformers import AutoModelForSpeechSeq2Seq # type: ignore + from transformers import AutoProcessor # type: ignore + from transformers import AutoTokenizer # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"transformers is not available: {exc}") from exc + + model_source = _load_model_source(model_id, local_files_only=local_files_only) + source_candidates = [] + for candidate in (model_source, model_id): + if candidate not in source_candidates: + source_candidates.append(candidate) + common_kwargs: dict[str, object] = { + "trust_remote_code": trust_remote_code, + "local_files_only": local_files_only, + } + if token: + common_kwargs["token"] = token + + if task == "causal_lm_logits": + tokenizer = None + tokenizer_errors: list[str] = [] + for source in source_candidates: + try: + tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + tokenizer_errors.append(f"{source}: {exc}") + if tokenizer is None: + raise RuntimeError( + f"Could not load tokenizer for {model_id}:\n" + + "\n".join(tokenizer_errors) + ) + model = AutoModelForCausalLM.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model_source, tokenizer, model, config + if task == "multimodal_causal_lm_logits": + processor = None + processor_errors: list[str] = [] + for source in source_candidates: + try: + processor = AutoProcessor.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + processor_errors.append(f"{source}: {exc}") + if processor is None and config_model_type == "gemma4": + processor = _load_gemma4_processor_fallback( + source_candidates=source_candidates, + common_kwargs=common_kwargs, + ) + if processor is not None: + print("note=using manual gemma4 processor fallback") + if processor is None: + processor_config_hint = "" + if config_model_type == "gemma4": + processor_config_hint = ( + "\n" + "Gemma4 multimodal transpile needs a processor bundle, not just tokenizer/model weights.\n" + "Your local snapshot may be missing files such as `processor_config.json` or modality-specific\n" + "preprocessor configs. Use an official Gemma4 snapshot that includes the processor, or let\n" + "transformers download one by re-running without `--local-files-only`.\n" + ) + raise RuntimeError( + f"Could not load processor for {model_id}:\n" + + "\n".join(processor_errors) + + processor_config_hint + ) + + model = AutoModelForCausalLM.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + if config_model_type == "gemma4": + repair_result = _repair_gemma4_checkpoint_weights(model, model_source) + if repair_result.get("applied"): + missing = repair_result.get("missing_keys", []) + unexpected = repair_result.get("unexpected_keys", []) + print( + "note=applied gemma4 checkpoint key remap" + f" missing_after={len(missing)} unexpected_after={len(unexpected)}" + ) + return model_source, processor, model, config + + processor = None + processor_errors: list[str] = [] + missing_optional_audio_dep: str | None = None + for source in source_candidates: + for loader in (AutoProcessor, AutoFeatureExtractor): + try: + processor = loader.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + processor_errors.append(f"{loader.__name__}@{source}: {exc}") + if isinstance(exc, ImportError) and "requires the librosa library" in str(exc): + missing_optional_audio_dep = "librosa" + break + if processor is not None: + break + if missing_optional_audio_dep is not None: + break + if processor is None: + if missing_optional_audio_dep == "librosa": + print("note=falling back to built-in audio preprocessing because the HF feature extractor requires librosa") + else: + print("note=falling back to built-in audio preprocessing because no HF processor/feature extractor was available") + + if task == "ctc_logits": + model_loaders = (AutoModelForCTC, AutoModel) + elif task == "encoder_hidden_states": + model_loaders = (AutoModelForSpeechSeq2Seq, AutoModelForSeq2SeqLM, AutoModel) + else: + raise NotImplementedError(f"unsupported generic HF task: {task}") + + load_errors: list[str] = [] + for loader in model_loaders: + try: + model = loader.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model_source, processor, model, config + except Exception as exc: + load_errors.append(f"{loader.__name__}: {exc}") + + raise RuntimeError( + f"Could not load model for task={task} from {model_source}.\n" + "\n".join(load_errors) + ) + + +def _load_optional_tokenizer( + *, + model_id: str, + model_source: str, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, +): + try: + from transformers import AutoTokenizer # type: ignore + except Exception: + return None + + source_candidates = [] + for candidate in (model_source, model_id): + if candidate not in source_candidates: + source_candidates.append(candidate) + + common_kwargs: dict[str, object] = { + "trust_remote_code": trust_remote_code, + "local_files_only": local_files_only, + } + if token: + common_kwargs["token"] = token + + for source in source_candidates: + try: + return AutoTokenizer.from_pretrained(source, **common_kwargs) + except Exception: + continue + return None + + +def _ctc_greedy_decode_token_ids(logits: np.ndarray, *, blank_token_id: int | None) -> list[int]: + if logits.ndim != 3 or logits.shape[0] < 1: + raise ValueError(f"expected CTC logits with shape [batch, time, vocab], got {list(logits.shape)}") + + raw_ids = np.argmax(logits[0], axis=-1).tolist() + collapsed: list[int] = [] + previous: int | None = None + for token_id in raw_ids: + if token_id != previous: + collapsed.append(int(token_id)) + previous = int(token_id) + + if blank_token_id is None: + return collapsed + return [token_id for token_id in collapsed if int(token_id) != int(blank_token_id)] + + +def _decode_token_ids(tokenizer: object, token_ids: list[int]) -> str: + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + raise TypeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") + try: + return str(decode(token_ids, skip_special_tokens=True)) + except TypeError: + return str(decode(token_ids)) + + +def _count_weight_bindings(ir_graph: IRGraph) -> int: + count = 0 + for value in ir_graph.values.values(): + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str): + count += 1 + return count + + +def _lower_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + graph = Graph() + env: dict[str, Any] = {} + runtime_inputs = [] + bound_constants = [] + + for value_id in ir.inputs: + value = ir.values[value_id] + tensor = _lower_input_value(graph, value) + env[value_id] = tensor + runtime_inputs.append(tensor) + + for value_id, const in ir.constants.items(): + value = ir.values[value_id] + lowered_const = _lower_constant_value(graph, value, const) + env[value_id] = lowered_const + if hasattr(lowered_const, "g") and hasattr(lowered_const, "id"): + bound_constants.append(lowered_const) + + for node_id in ir.order: + node = ir.nodes[node_id] + outputs = _lower_ir_node(graph, node, env, ir) + if len(outputs) != len(node.outputs): + raise ValueError( + f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" + ) + for output_id, tensor in zip(node.outputs, outputs): + env[output_id] = tensor + + outputs = [env[value_id] for value_id in ir.outputs] + return TranspiledGraph( + graph=graph, + runtime_inputs=runtime_inputs, + bound_constants=bound_constants, + outputs=outputs, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Load a Hugging Face model, canonicalize it into a generic transpile task, " + "capture it with the Cactus transpiler, lower it to a Cactus Graph, and " + "optionally save artifacts or run the lowered graph." + ) + ) + parser.add_argument( + "--model-id", + required=True, + help="Hugging Face model id or local snapshot path.", + ) + parser.add_argument( + "--task", + default="auto", + choices=("auto", "causal_lm_logits", "multimodal_causal_lm_logits", "ctc_logits", "encoder_hidden_states"), + help="Transpile task. Use auto to infer from config/model id.", + ) + parser.add_argument( + "--prompt", + default="The capital of France is", + help="Prompt used for causal_lm_logits or multimodal_causal_lm_logits when --input-ids is not set.", + ) + parser.add_argument( + "--input-ids", + default="", + help="Optional comma-separated token ids for causal_lm_logits.", + ) + parser.add_argument( + "--audio-file", + default="", + help="Path to a WAV file for audio or multimodal tasks.", + ) + parser.add_argument( + "--image-file", + action="append", + default=[], + help="Path to an image file for multimodal tasks. Repeat to pass multiple images.", + ) + parser.add_argument( + "--torch-dtype", + default="float16", + help="Torch dtype for model loading: float16, float32, or bfloat16.", + ) + parser.add_argument( + "--token", + default=os.environ.get("HF_TOKEN"), + help="Optional Hugging Face token. Defaults to HF_TOKEN.", + ) + parser.add_argument( + "--trust-remote-code", + action="store_true", + help="Pass trust_remote_code=True to transformers loaders.", + ) + parser.add_argument( + "--local-files-only", + action="store_true", + help="Require the model/processor to already exist locally.", + ) + parser.add_argument( + "--weights-dir", + default="", + help="Optional converted Cactus weights directory for mmap weight binding.", + ) + parser.add_argument( + "--artifact-dir", + default="", + help="Optional directory where raw_ir.json, optimized_ir.json, graph.cactus, and result.json are saved.", + ) + parser.add_argument( + "--graph-filename", + default="graph.cactus", + help="Filename to use for Graph.save() inside --artifact-dir.", + ) + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + parser.add_argument( + "--skip-execute", + action="store_true", + help="Stop after lowering instead of running the transpiled graph.", + ) + parser.add_argument( + "--skip-reference-compare", + action="store_true", + help="Run the transpiled graph but skip the follow-up PyTorch reference pass.", + ) + args = parser.parse_args() + + image_files = tuple(str(path) for path in args.image_file if str(path).strip()) + if args.task == "auto": + inferred_task = _infer_task_from_config(args.model_id) + config_for_auto = _load_config_json(args.model_id) + model_type_for_auto = str(config_for_auto.get("model_type", "") or "").lower() + has_multimodal_config = ( + model_type_for_auto == "gemma4" + and ( + isinstance(config_for_auto.get("vision_config"), dict) + or isinstance(config_for_auto.get("audio_config"), dict) + ) + ) + if image_files or (args.audio_file and has_multimodal_config and inferred_task == "causal_lm_logits"): + task = "multimodal_causal_lm_logits" + else: + task = inferred_task + else: + task = args.task + torch_dtype = _parse_dtype(args.torch_dtype) + validated_weights_dir = _validate_weights_dir(args.weights_dir.strip() or None, model_id=args.model_id) + weights_dir = str(validated_weights_dir) if validated_weights_dir is not None else None + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + + model_source, processor_or_tokenizer, model, model_config = _load_transformers_bundle( + model_id=args.model_id, + task=task, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") + if not preprocessor_config: + preprocessor_config = _load_optional_json(args.model_id, "preprocessor_config.json") + auxiliary_tokenizer = None + if task == "ctc_logits": + auxiliary_tokenizer = _load_optional_tokenizer( + model_id=args.model_id, + model_source=model_source, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + + if task == "causal_lm_logits": + prepared = _prepare_text_inputs( + processor_or_tokenizer, + prompt=args.prompt, + input_ids_text=args.input_ids.strip() or None, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + ) + elif task == "multimodal_causal_lm_logits": + prepared = _prepare_gemma4_multimodal_inputs( + processor_or_tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=args.audio_file.strip() or None, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + ) + else: + if not args.audio_file: + raise RuntimeError(f"--audio-file is required for task={task}") + canonical = canonicalize_model_interface(model, task=task) + prepared = _prepare_audio_inputs( + processor_or_tokenizer, + input_names=canonical.input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=args.audio_file, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + ) + wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + + print(f"model_id={args.model_id}") + print(f"model_source={model_source}") + print(f"task={task}") + print(f"adapter_family={canonical.family}") + print(f"adapter_module={type(canonical.module).__name__}") + print(f"input_names={','.join(prepared.names)}") + for name, tensor in zip(prepared.names, prepared.tensors): + print(f"input_{name}_shape={list(tensor.shape)}") + if weights_dir: + print(f"weights_dir={weights_dir}") + + captured = capture_model(wrapper, prepared.tensors) + raw_ir_graph = copy.deepcopy(captured.ir_graph) + + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + canonicalize_exported_graph(captured.ir_graph) + optimize_graph(captured.ir_graph, config=fusion_config) + tg = _lower_preoptimized_ir(captured.ir_graph) + + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + binding_count = _count_weight_bindings(optimized_ir_graph) + op_counts = Counter(optimized_ir_graph.nodes[node_id].op for node_id in optimized_ir_graph.order) + + print(f"raw_ir_nodes={len(raw_ir_graph.order)}") + print(f"optimized_ir_nodes={len(optimized_ir_graph.order)}") + print(f"weight_bindings={binding_count}") + print( + "ops=" + f"attention:{op_counts.get('attention', 0)} " + f"conv1d:{op_counts.get('conv1d', 0)} " + f"conv2d:{op_counts.get('conv2d', 0)} " + f"batch_norm:{op_counts.get('batch_norm', 0)} " + f"layer_norm:{op_counts.get('layer_norm', 0)} " + f"rms_norm:{op_counts.get('rms_norm', 0)} " + f"rope:{op_counts.get('rope', 0)} " + f"linear:{op_counts.get('linear', 0)}" + ) + + if weights_dir and binding_count == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir}\n" + "\n" + "The weights folder exists, but none of the captured constants matched entries in weights_manifest.json.\n" + "\n" + f"Recommended fix:\n" + f" cactus convert {args.model_id} {weights_dir}\n" + ) + + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(raw_ir_graph), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(optimized_ir_graph), + }, + ) + graph_path = artifact_dir / args.graph_filename + tg.graph.save(graph_path) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + print(f"saved_graph={graph_path}") + + if args.skip_execute: + return 0 + + if args.skip_reference_compare: + del model + del wrapper + gc.collect() + + tg.set_inputs([tensor.cpu().numpy() for tensor in prepared.tensors]) + print("execute_begin=true") + transpiled_output = tg.execute()[0].numpy().astype(np.float32) + print("execute_done=true") + + if args.skip_reference_compare: + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(transpiled_output.shape), + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": binding_count, + "reference_compare_skipped": True, + } + print(f"output_shape={list(transpiled_output.shape)}") + if task in {"causal_lm_logits", "multimodal_causal_lm_logits"}: + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"transpiled_next_token_id={transpiled_next}") + result_payload["transpiled_next_token_id"] = transpiled_next + if hasattr(tokenizer_like, "decode"): + transpiled_token = tokenizer_like.decode([transpiled_next]) + print(f"transpiled_next_token={transpiled_token!r}") + result_payload["transpiled_next_token"] = transpiled_token + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + print("reference_begin=true") + with torch.no_grad(): + reference_output = wrapper(*prepared.tensors).detach().float().cpu().numpy() + print("reference_done=true") + + max_abs_diff = float(np.max(np.abs(reference_output - transpiled_output))) + mean_abs_diff = float(np.mean(np.abs(reference_output - transpiled_output))) + result_payload: dict[str, object] = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(reference_output.shape), + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": binding_count, + "max_abs_diff": max_abs_diff, + "mean_abs_diff": mean_abs_diff, + } + + if task == "causal_lm_logits": + tokenizer = processor_or_tokenizer + hf_next = int(np.argmax(reference_output[0, -1])) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"hf_next_token_id={hf_next}") + print(f"transpiled_next_token_id={transpiled_next}") + print(f"logits_max_abs_diff={max_abs_diff:.6f}") + print(f"logits_mean_abs_diff={mean_abs_diff:.6f}") + print(f"hf_next_token={tokenizer.decode([hf_next])!r}") + print(f"transpiled_next_token={tokenizer.decode([transpiled_next])!r}") + result_payload.update( + { + "hf_next_token_id": hf_next, + "transpiled_next_token_id": transpiled_next, + "hf_next_token": tokenizer.decode([hf_next]), + "transpiled_next_token": tokenizer.decode([transpiled_next]), + } + ) + elif task == "ctc_logits": + blank_token_id = getattr(model.config, "pad_token_id", None) + if blank_token_id is None and auxiliary_tokenizer is not None: + blank_token_id = getattr(auxiliary_tokenizer, "pad_token_id", None) + print(f"output_shape={list(reference_output.shape)}") + print(f"output_max_abs_diff={max_abs_diff:.6f}") + print(f"output_mean_abs_diff={mean_abs_diff:.6f}") + if auxiliary_tokenizer is not None: + hf_token_ids = _ctc_greedy_decode_token_ids(reference_output, blank_token_id=blank_token_id) + transpiled_token_ids = _ctc_greedy_decode_token_ids(transpiled_output, blank_token_id=blank_token_id) + hf_transcript = _decode_token_ids(auxiliary_tokenizer, hf_token_ids) + transpiled_transcript = _decode_token_ids(auxiliary_tokenizer, transpiled_token_ids) + print(f"hf_transcript={hf_transcript!r}") + print(f"transpiled_transcript={transpiled_transcript!r}") + result_payload.update( + { + "blank_token_id": None if blank_token_id is None else int(blank_token_id), + "hf_transcript_token_ids": hf_token_ids, + "transpiled_transcript_token_ids": transpiled_token_ids, + "hf_transcript": hf_transcript, + "transpiled_transcript": transpiled_transcript, + } + ) + else: + print(f"output_shape={list(reference_output.shape)}") + print(f"output_max_abs_diff={max_abs_diff:.6f}") + print(f"output_mean_abs_diff={mean_abs_diff:.6f}") + + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 858cf80b4d9f637faa371aecb2e802c6fa3d4b18 Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Mon, 11 May 2026 21:10:22 -0700 Subject: [PATCH 03/11] Removed old transpiler logic --- python/examples/compare_gemma4_multimodal.py | 779 ------ .../examples/compare_parakeet_performance.py | 793 ------ python/examples/inspect_gemma4_cleanup.py | 241 -- python/examples/inspect_transpile_cleanup.py | 138 -- python/examples/test_torch_capture.py | 70 - python/examples/transpile_hf_causal_lm.py | 17 - python/examples/transpile_hf_model.py | 1734 ------------- python/src/transpile/canonicalize/cleanup.py | 629 ----- python/src/transpile/canonicalize/utils.py | 228 -- .../src/transpile/capture/capture_pytorch.py | 302 --- python/src/transpile/capture/graph_ir.py | 125 - python/src/transpile/capture/import_ir.py | 266 -- .../src/transpile/capture/import_semantics.py | 13 - python/src/transpile/capture/importers.py | 1236 ---------- python/src/transpile/capture/normalize.py | 155 -- python/src/transpile/capture/ops.py | 159 -- .../src/transpile/capture/weight_binding.py | 185 -- python/src/transpile/capture_pytorch.py | 302 --- python/src/transpile/fusion/__init__.py | 43 - python/src/transpile/fusion/attention.py | 550 ----- python/src/transpile/fusion/common.py | 73 - python/src/transpile/fusion/conv.py | 123 - python/src/transpile/fusion/deltanet.py | 357 --- python/src/transpile/fusion/linear.py | 58 - python/src/transpile/fusion/mlp.py | 53 - python/src/transpile/fusion/rel_pos_bias.py | 104 - python/src/transpile/fusion/rms_norm.py | 112 - python/src/transpile/fusion/rope.py | 252 -- python/src/transpile/graph_ir.py | 201 -- python/src/transpile/import_ir.py | 266 -- python/src/transpile/import_semantics.py | 337 --- python/src/transpile/importers.py | 1754 -------------- python/src/transpile/lower.py | 2142 ----------------- python/src/transpile/model_adapters.py | 803 ------ python/src/transpile/model_patterns.py | 76 - python/src/transpile/normalize.py | 199 -- python/src/transpile/ops.py | 234 -- python/src/transpile/optimize_graph.py | 737 ------ python/src/transpile/test_capture.py | 187 -- python/src/transpile/weight_binding.py | 330 --- 40 files changed, 16363 deletions(-) delete mode 100644 python/examples/compare_gemma4_multimodal.py delete mode 100644 python/examples/compare_parakeet_performance.py delete mode 100644 python/examples/inspect_gemma4_cleanup.py delete mode 100644 python/examples/inspect_transpile_cleanup.py delete mode 100644 python/examples/test_torch_capture.py delete mode 100644 python/examples/transpile_hf_causal_lm.py delete mode 100644 python/examples/transpile_hf_model.py delete mode 100644 python/src/transpile/canonicalize/cleanup.py delete mode 100644 python/src/transpile/canonicalize/utils.py delete mode 100644 python/src/transpile/capture/capture_pytorch.py delete mode 100644 python/src/transpile/capture/graph_ir.py delete mode 100644 python/src/transpile/capture/import_ir.py delete mode 100644 python/src/transpile/capture/import_semantics.py delete mode 100644 python/src/transpile/capture/importers.py delete mode 100644 python/src/transpile/capture/normalize.py delete mode 100644 python/src/transpile/capture/ops.py delete mode 100644 python/src/transpile/capture/weight_binding.py delete mode 100644 python/src/transpile/capture_pytorch.py delete mode 100644 python/src/transpile/fusion/__init__.py delete mode 100644 python/src/transpile/fusion/attention.py delete mode 100644 python/src/transpile/fusion/common.py delete mode 100644 python/src/transpile/fusion/conv.py delete mode 100644 python/src/transpile/fusion/deltanet.py delete mode 100644 python/src/transpile/fusion/linear.py delete mode 100644 python/src/transpile/fusion/mlp.py delete mode 100644 python/src/transpile/fusion/rel_pos_bias.py delete mode 100644 python/src/transpile/fusion/rms_norm.py delete mode 100644 python/src/transpile/fusion/rope.py delete mode 100644 python/src/transpile/graph_ir.py delete mode 100644 python/src/transpile/import_ir.py delete mode 100644 python/src/transpile/import_semantics.py delete mode 100644 python/src/transpile/importers.py delete mode 100644 python/src/transpile/lower.py delete mode 100644 python/src/transpile/model_adapters.py delete mode 100644 python/src/transpile/model_patterns.py delete mode 100644 python/src/transpile/normalize.py delete mode 100644 python/src/transpile/ops.py delete mode 100644 python/src/transpile/optimize_graph.py delete mode 100644 python/src/transpile/test_capture.py delete mode 100644 python/src/transpile/weight_binding.py diff --git a/python/examples/compare_gemma4_multimodal.py b/python/examples/compare_gemma4_multimodal.py deleted file mode 100644 index 0970f2496..000000000 --- a/python/examples/compare_gemma4_multimodal.py +++ /dev/null @@ -1,779 +0,0 @@ -from __future__ import annotations - -import argparse -import copy -import gc -import json -import os -import sys -import time -from dataclasses import asdict -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import numpy as np -import torch - -EXAMPLES_DIR = Path(__file__).resolve().parent -PYTHON_ROOT = EXAMPLES_DIR.parent -PROJECT_ROOT = PYTHON_ROOT.parent - -sys.path.insert(0, str(PYTHON_ROOT)) -sys.path.insert(0, str(EXAMPLES_DIR)) - -from src.cactus import cactus_complete -from src.cactus import cactus_destroy -from src.cactus import cactus_init -from src.cactus import cactus_reset -from src.transpile.capture_pytorch import capture_model -from src.transpile.canonicalize.cleanup import canonicalize_exported_graph -from src.transpile.model_adapters import canonicalize_model_interface -from src.transpile.optimize_graph import FusionConfig -from src.transpile.optimize_graph import optimize_graph -from transpile_hf_model import TranspileWrapper -from transpile_hf_model import _graph_to_dict -from transpile_hf_model import _load_transformers_bundle -from transpile_hf_model import _lower_preoptimized_ir -from transpile_hf_model import _parse_dtype -from transpile_hf_model import _prepare_gemma4_multimodal_inputs -from transpile_hf_model import _serialize_json_compatible -from transpile_hf_model import _validate_weights_dir -from transpile_hf_model import _write_json - - -_DEFAULT_STOP_SEQUENCES = ("", "", "", "<|im_end|>") - - -@dataclass -class NumericSummary: - mean: float - min: float - max: float - median: float - - -@dataclass -class HandwrittenRun: - wall_ms: float - total_time_ms: float - time_to_first_token_ms: float - prefill_tps: float - decode_tps: float - decode_tokens: int - total_tokens: int - response: str - response_json: dict[str, Any] - - -@dataclass -class TranspiledRun: - preprocess_ms: float - execute_ms: float - total_ms: float - time_to_first_token_ms: float - prefill_tps: float - decode_tps: float - decode_tokens: int - total_tokens: int - response: str - stop_reason: str - generated_token_ids: list[int] - input_shapes: dict[str, list[int]] - - -def _numeric_summary(values: list[float]) -> NumericSummary: - arr = np.asarray(values, dtype=np.float64) - return NumericSummary( - mean=float(np.mean(arr)), - min=float(np.min(arr)), - max=float(np.max(arr)), - median=float(np.median(arr)), - ) - - -def _roundtrip_jsonable(value: Any) -> Any: - return json.loads(json.dumps(_serialize_json_compatible(value))) - - -def _resolve_default_weights_dir(model_id: str) -> Path | None: - env_path = os.environ.get("CACTUS_TEST_GEMMA4_MODEL") - if env_path: - candidate = Path(env_path).resolve() - if (candidate / "config.txt").exists(): - return candidate - - candidates = ( - PROJECT_ROOT / "weights" / "gemma4_int4", - PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower(), - PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower().replace("_", "-"), - ) - for candidate in candidates: - if (candidate / "config.txt").exists(): - return candidate - return None - - -def _require_processor_tokenizer(processor_or_tokenizer: object) -> object: - tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) - if not hasattr(tokenizer_like, "decode") or not hasattr(tokenizer_like, "encode"): - raise RuntimeError( - "Gemma4 compare script requires a tokenizer-like object with encode/decode support." - ) - return tokenizer_like - - -def _build_messages_payload( - *, - prompt: str, - image_files: tuple[str, ...], - audio_file: str | None, - system_prompt: str = "", -) -> list[dict[str, object]]: - messages: list[dict[str, object]] = [] - normalized_system = system_prompt.strip() - if normalized_system: - messages.append({"role": "system", "content": normalized_system}) - - user_message: dict[str, object] = { - "role": "user", - "content": prompt, - } - if image_files: - user_message["images"] = [str(Path(path).resolve()) for path in image_files] - if audio_file: - user_message["audio"] = [str(Path(audio_file).resolve())] - messages.append(user_message) - return messages - - -def _encode_stop_sequences(tokenizer: object, stop_sequences: tuple[str, ...]) -> list[list[int]]: - encoded: list[list[int]] = [] - encode = getattr(tokenizer, "encode", None) - if not callable(encode): - return encoded - for stop_sequence in stop_sequences: - try: - token_ids = list(encode(stop_sequence, add_special_tokens=False)) - except TypeError: - token_ids = list(encode(stop_sequence)) - if token_ids: - encoded.append([int(token_id) for token_id in token_ids]) - return encoded - - -def _has_suffix(token_ids: list[int], suffix: list[int]) -> bool: - if not suffix or len(token_ids) < len(suffix): - return False - return token_ids[-len(suffix) :] == suffix - - -def _trim_stop_suffix(token_ids: list[int], stop_sequences: list[list[int]]) -> bool: - for stop_sequence in stop_sequences: - if _has_suffix(token_ids, stop_sequence): - del token_ids[-len(stop_sequence) :] - return True - return False - - -def _decode_generated_text(tokenizer: object, token_ids: list[int], *, skip_special_tokens: bool) -> str: - decode = getattr(tokenizer, "decode", None) - if not callable(decode): - raise RuntimeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") - try: - return str(decode(token_ids, skip_special_tokens=skip_special_tokens)) - except TypeError: - return str(decode(token_ids)) - - -def _summarize_handwritten_runs( - runs: list[HandwrittenRun], - *, - weights_dir: Path, -) -> dict[str, Any]: - if not runs: - raise ValueError("expected at least one handwritten run") - return { - "weights_dir": str(weights_dir), - "latest_response": runs[-1].response, - "total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), - "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), - "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), - "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), - "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), - "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), - "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), - "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], - } - - -def _summarize_transpiled_runs( - runs: list[TranspiledRun], - *, - compile_time_ms: float, - model_source: str, - weight_bindings: int, - prepared_input_shapes: dict[str, list[int]], -) -> dict[str, Any]: - if not runs: - raise ValueError("expected at least one transpiled run") - return { - "compile_time_ms": compile_time_ms, - "model_source": model_source, - "weight_bindings": weight_bindings, - "prepared_input_shapes": prepared_input_shapes, - "latest_response": runs[-1].response, - "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), - "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), - "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), - "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), - "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), - "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), - "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), - "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), - "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], - } - - -def _prepare_repeat_inputs( - processor: object, - *, - prompt: str, - image_files: tuple[str, ...], - audio_file: str | None, - torch_dtype: torch.dtype, - system_prompt: str, - enable_thinking_if_supported: bool, -) -> tuple[object, Any]: - start = time.perf_counter() - prepared = _prepare_gemma4_multimodal_inputs( - processor, - prompt=prompt, - image_files=image_files, - audio_file=audio_file, - torch_dtype=torch_dtype, - system_prompt=system_prompt, - enable_thinking_if_supported=enable_thinking_if_supported, - use_gemma4_chat_template=True, - ) - preprocess_ms = (time.perf_counter() - start) * 1000.0 - return prepared, preprocess_ms - - -def _run_handwritten_once( - *, - model_handle: Any, - messages_json: str, - options_json: str, -) -> HandwrittenRun: - cactus_reset(model_handle) - streamed_token_ids: list[int] = [] - - def _collect(_text: str, token_id: int) -> None: - streamed_token_ids.append(int(token_id)) - - start = time.perf_counter() - response_json_text = cactus_complete( - model_handle, - messages_json, - options_json, - "[]", - _collect, - pcm_data=None, - ) - wall_ms = (time.perf_counter() - start) * 1000.0 - response_payload = json.loads(response_json_text) - return HandwrittenRun( - wall_ms=wall_ms, - total_time_ms=float(response_payload.get("total_time_ms", wall_ms)), - time_to_first_token_ms=float(response_payload.get("time_to_first_token_ms", 0.0)), - prefill_tps=float(response_payload.get("prefill_tps", 0.0)), - decode_tps=float(response_payload.get("decode_tps", 0.0)), - decode_tokens=len(streamed_token_ids), - total_tokens=int(response_payload.get("total_tokens", len(streamed_token_ids))), - response=str(response_payload.get("response", "")), - response_json=response_payload, - ) - - -def _run_transpiled_generation_once( - *, - tg, - processor: object, - tokenizer: object, - prompt: str, - image_files: tuple[str, ...], - audio_file: str | None, - torch_dtype: torch.dtype, - system_prompt: str, - enable_thinking_if_supported: bool, - max_new_tokens: int, - stop_sequences: tuple[str, ...], -) -> TranspiledRun: - prepared, preprocess_ms = _prepare_repeat_inputs( - processor, - prompt=prompt, - image_files=image_files, - audio_file=audio_file, - torch_dtype=torch_dtype, - system_prompt=system_prompt, - enable_thinking_if_supported=enable_thinking_if_supported, - ) - inputs_by_name = { - name: tensor.detach().cpu().numpy() - for name, tensor in zip(prepared.names, prepared.tensors) - } - if "input_ids" not in inputs_by_name: - raise RuntimeError("transpiled Gemma4 generation requires input_ids") - - input_ids = np.asarray(inputs_by_name["input_ids"], dtype=np.int64) - prompt_tokens = int(input_ids.shape[1]) - target_tokens = prompt_tokens + max_new_tokens - - pad_token_id = getattr(tokenizer, "pad_token_id", None) - if pad_token_id is None: - pad_token_id = 0 - eos_token_id = getattr(tokenizer, "eos_token_id", None) - - padded_inputs: dict[str, np.ndarray] = {} - for name, value in inputs_by_name.items(): - if name in {"input_ids", "attention_mask", "token_type_ids"}: - padded_shape = list(value.shape) - padded_shape[1] = target_tokens - if name == "input_ids": - padded_value = np.full(padded_shape, int(pad_token_id), dtype=value.dtype) - else: - padded_value = np.zeros(padded_shape, dtype=value.dtype) - padded_value[:, : value.shape[1]] = value - padded_inputs[name] = padded_value - else: - padded_inputs[name] = value - - if "attention_mask" not in padded_inputs: - padded_inputs["attention_mask"] = np.zeros((1, target_tokens), dtype=np.int64) - padded_inputs["attention_mask"][:, :prompt_tokens] = 1 - if "token_type_ids" not in padded_inputs: - padded_inputs["token_type_ids"] = np.zeros((1, target_tokens), dtype=np.int64) - - tg.graph.hard_reset() - generated_ids: list[int] = [] - encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) - current_length = prompt_tokens - first_token_ms = 0.0 - execute_start = time.perf_counter() - stop_reason = "max_new_tokens" - - for step_index in range(max_new_tokens): - runtime_inputs = [padded_inputs[name] for name in prepared.names] - tg.set_inputs(runtime_inputs) - logits = tg.execute()[0].numpy().astype(np.float32) - next_token_id = int(np.argmax(logits[0, current_length - 1])) - generated_ids.append(next_token_id) - - if step_index == 0: - first_token_ms = preprocess_ms + (time.perf_counter() - execute_start) * 1000.0 - - if eos_token_id is not None and next_token_id == int(eos_token_id): - stop_reason = "eos_token" - break - if _trim_stop_suffix(generated_ids, encoded_stop_sequences): - stop_reason = "stop_sequence" - break - - if current_length >= target_tokens: - stop_reason = "context_limit" - break - - padded_inputs["input_ids"][0, current_length] = next_token_id - padded_inputs["attention_mask"][0, current_length] = 1 - padded_inputs["token_type_ids"][0, current_length] = 0 - current_length += 1 - - execute_ms = (time.perf_counter() - execute_start) * 1000.0 - response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() - if not response: - response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() - decode_time_ms = max(0.0, execute_ms + preprocess_ms - first_token_ms) - decode_tps = ( - ((len(generated_ids) - 1) * 1000.0) / decode_time_ms - if len(generated_ids) > 1 and decode_time_ms > 0.0 - else 0.0 - ) - prefill_tps = (prompt_tokens * 1000.0) / first_token_ms if first_token_ms > 0.0 else 0.0 - - return TranspiledRun( - preprocess_ms=preprocess_ms, - execute_ms=execute_ms, - total_ms=preprocess_ms + execute_ms, - time_to_first_token_ms=first_token_ms, - prefill_tps=prefill_tps, - decode_tps=decode_tps, - decode_tokens=len(generated_ids), - total_tokens=prompt_tokens + len(generated_ids), - response=response, - stop_reason=stop_reason, - generated_token_ids=generated_ids, - input_shapes={ - name: list(tensor.shape) - for name, tensor in zip(prepared.names, prepared.tensors) - }, - ) - - -def _print_summary(summary: dict[str, Any]) -> None: - print(f"model_id={summary['model_id']}") - print(f"prompt={summary['prompt']!r}") - if summary.get("image_files"): - print(f"image_files={summary['image_files']}") - if summary.get("audio_file"): - print(f"audio_file={summary['audio_file']}") - - handwritten = summary.get("handwritten") - if handwritten is not None: - hw_total = handwritten["total_time_ms"]["mean"] - hw_ttft = handwritten["time_to_first_token_ms"]["mean"] - hw_tokens = handwritten["decode_tokens"]["mean"] - hw_tps = hw_tokens * 1000.0 / max(hw_total, 1e-9) - print("handwritten:") - print(f" total_ms mean={hw_total:.3f}") - print(f" time_to_first_token_ms mean={hw_ttft:.3f}") - print(f" decode_tokens mean={hw_tokens:.3f}") - print(f" output_tokens_per_sec={hw_tps:.3f}") - print(f" latest_response={handwritten['latest_response']!r}") - - transpiled = summary.get("transpiled") - if transpiled is not None: - tr_total = transpiled["total_ms"]["mean"] - tr_ttft = transpiled["time_to_first_token_ms"]["mean"] - tr_tokens = transpiled["decode_tokens"]["mean"] - tr_tps = tr_tokens * 1000.0 / max(tr_total, 1e-9) - print("transpiled:") - print(f" compile_time_ms={transpiled['compile_time_ms']:.3f}") - print(f" total_ms mean={tr_total:.3f}") - print(f" time_to_first_token_ms mean={tr_ttft:.3f}") - print(f" decode_tokens mean={tr_tokens:.3f}") - print(f" output_tokens_per_sec={tr_tps:.3f}") - print(f" weight_bindings={transpiled['weight_bindings']}") - print(f" latest_response={transpiled['latest_response']!r}") - - comparison = summary.get("comparison") - if comparison is not None: - print("comparison:") - print( - " end_to_end_speedup_vs_handwritten=" - f"{comparison['end_to_end_speedup_vs_handwritten']:.6f}" - ) - print(f" responses_match={comparison['responses_match']}") - - -def main() -> int: - parser = argparse.ArgumentParser( - description=( - "Compare handwritten Cactus Gemma4 multimodal generation against the " - "transpiled Gemma4 multimodal graph on the same prompt, image, and audio." - ) - ) - parser.add_argument("--model-id", default="google/gemma-4-E2B-it") - parser.add_argument("--prompt", required=True) - parser.add_argument("--system-prompt", default="") - parser.add_argument("--image-file", action="append", default=[]) - parser.add_argument("--audio-file", default="") - parser.add_argument( - "--weights-dir", - default="", - help="Converted Cactus weights directory. Required for handwritten comparison and reused for transpiled mmap bindings.", - ) - parser.add_argument("--artifact-dir", default="", help="Optional directory for summary and transpile artifacts.") - parser.add_argument("--graph-filename", default="graph.cactus") - parser.add_argument("--warmup", type=int, default=1) - parser.add_argument("--repeats", type=int, default=1) - parser.add_argument("--max-new-tokens", type=int, default=96) - parser.add_argument("--torch-dtype", default="float16") - parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) - parser.add_argument("--trust-remote-code", action="store_true") - parser.add_argument("--local-files-only", action="store_true") - parser.add_argument("--skip-handwritten", action="store_true") - parser.add_argument("--skip-transpiled", action="store_true") - parser.add_argument("--enable-thinking", action="store_true") - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--top-p", type=float, default=1.0) - parser.add_argument("--top-k", type=int, default=1) - parser.add_argument("--stop-sequence", action="append", default=[]) - parser.add_argument("--no-fuse-gated-deltanet", action="store_true") - parser.add_argument("--no-fuse-rms-norm", action="store_true") - parser.add_argument("--no-fuse-rope", action="store_true") - parser.add_argument("--no-fuse-attention", action="store_true") - parser.add_argument("--no-fuse-attention-block", action="store_true") - parser.add_argument("--no-fuse-add-clipped", action="store_true") - args = parser.parse_args() - - if args.skip_handwritten and args.skip_transpiled: - raise RuntimeError("cannot skip both handwritten and transpiled runs") - - image_files = tuple(str(Path(path).resolve()) for path in args.image_file if str(path).strip()) - audio_file = str(Path(args.audio_file).resolve()) if args.audio_file.strip() else None - if audio_file and not Path(audio_file).exists(): - raise FileNotFoundError(f"audio file does not exist: {audio_file}") - for image_file in image_files: - if not Path(image_file).exists(): - raise FileNotFoundError(f"image file does not exist: {image_file}") - - torch_dtype = _parse_dtype(args.torch_dtype) - artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None - if artifact_dir is not None: - artifact_dir.mkdir(parents=True, exist_ok=True) - - weights_dir_path = None - if args.weights_dir.strip(): - weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) - else: - weights_dir_path = _resolve_default_weights_dir(args.model_id) - - if not args.skip_handwritten and weights_dir_path is None: - raise RuntimeError( - "handwritten Gemma4 comparison requires a converted weights directory.\n" - "\n" - "Create one with something like:\n" - f" cactus convert {args.model_id} /path/to/gemma4_weights\n" - "\n" - "Then rerun with --weights-dir /path/to/gemma4_weights, or rerun with --skip-handwritten." - ) - - stop_sequences = tuple(args.stop_sequence) if args.stop_sequence else _DEFAULT_STOP_SEQUENCES - fusion_config = FusionConfig( - enable_gated_deltanet=not args.no_fuse_gated_deltanet, - enable_rms_norm=not args.no_fuse_rms_norm, - enable_rope=not args.no_fuse_rope, - enable_attention=not args.no_fuse_attention, - enable_attention_block=not args.no_fuse_attention_block, - enable_add_clipped=not args.no_fuse_add_clipped, - ) - - summary: dict[str, Any] = { - "model_id": args.model_id, - "prompt": args.prompt, - "system_prompt": args.system_prompt, - "image_files": list(image_files), - "audio_file": audio_file, - "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, - "warmup": args.warmup, - "repeats": args.repeats, - "max_new_tokens": args.max_new_tokens, - "torch_dtype": args.torch_dtype, - "trust_remote_code": bool(args.trust_remote_code), - "local_files_only": bool(args.local_files_only), - "enable_thinking": bool(args.enable_thinking), - "temperature": float(args.temperature), - "top_p": float(args.top_p), - "top_k": int(args.top_k), - "stop_sequences": list(stop_sequences), - } - - messages_payload = _build_messages_payload( - prompt=args.prompt, - image_files=image_files, - audio_file=audio_file, - system_prompt=args.system_prompt, - ) - messages_json = json.dumps(messages_payload) - options_json = json.dumps( - { - "temperature": float(args.temperature), - "top_p": float(args.top_p), - "top_k": int(args.top_k), - "max_tokens": int(args.max_new_tokens), - "stop_sequences": list(stop_sequences), - "enable_thinking_if_supported": bool(args.enable_thinking), - "auto_handoff": False, - "telemetry_enabled": False, - } - ) - - handwritten_runs: list[HandwrittenRun] = [] - if not args.skip_handwritten and weights_dir_path is not None: - model_handle = cactus_init(str(weights_dir_path), None, False) - try: - for _ in range(args.warmup): - _run_handwritten_once( - model_handle=model_handle, - messages_json=messages_json, - options_json=options_json, - ) - for _ in range(args.repeats): - handwritten_runs.append( - _run_handwritten_once( - model_handle=model_handle, - messages_json=messages_json, - options_json=options_json, - ) - ) - finally: - cactus_destroy(model_handle) - summary["handwritten"] = _summarize_handwritten_runs( - handwritten_runs, - weights_dir=weights_dir_path, - ) - - transpiled_runs: list[TranspiledRun] = [] - if not args.skip_transpiled: - model_source, processor_or_tokenizer, model, _ = _load_transformers_bundle( - model_id=args.model_id, - task="multimodal_causal_lm_logits", - torch_dtype=torch_dtype, - token=args.token, - trust_remote_code=args.trust_remote_code, - local_files_only=args.local_files_only, - ) - processor = processor_or_tokenizer - tokenizer = _require_processor_tokenizer(processor_or_tokenizer) - initial_prepared, _ = _prepare_repeat_inputs( - processor, - prompt=args.prompt, - image_files=image_files, - audio_file=audio_file, - torch_dtype=torch_dtype, - system_prompt=args.system_prompt, - enable_thinking_if_supported=args.enable_thinking, - ) - canonical = canonicalize_model_interface( - model, - task="multimodal_causal_lm_logits", - input_names=initial_prepared.names, - ) - wrapper = TranspileWrapper( - canonical.module, - weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, - ).eval() - - compile_start = time.perf_counter() - captured = capture_model(wrapper, initial_prepared.tensors) - raw_ir_graph = copy.deepcopy(captured.ir_graph) - canonicalize_exported_graph(captured.ir_graph) - optimize_graph(captured.ir_graph, config=fusion_config) - optimized_ir_graph = copy.deepcopy(captured.ir_graph) - tg = _lower_preoptimized_ir(captured.ir_graph) - compile_time_ms = (time.perf_counter() - compile_start) * 1000.0 - - op_counts: dict[str, int] = {} - for node_id in optimized_ir_graph.order: - op = optimized_ir_graph.nodes[node_id].op - op_counts[op] = op_counts.get(op, 0) + 1 - weight_bindings = sum( - 1 - for value in optimized_ir_graph.values.values() - if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) - ) - - summary["transpile_compile"] = { - "model_source": model_source, - "raw_ir_nodes": len(raw_ir_graph.order), - "optimized_ir_nodes": len(optimized_ir_graph.order), - "weight_bindings": weight_bindings, - "compile_time_ms": compile_time_ms, - "ops": op_counts, - } - - if weights_dir_path is not None and weight_bindings == 0: - raise RuntimeError( - f"No weight bindings were resolved from {weights_dir_path}\n" - "\n" - "The converted weights folder exists, but none of the captured constants matched weights_manifest.json.\n" - "Re-convert Gemma4 with the current converter before benchmarking." - ) - - if artifact_dir is not None: - _write_json( - artifact_dir / "raw_ir.json", - { - "model_id": args.model_id, - "model_source": model_source, - "task": "multimodal_causal_lm_logits", - "inputs": _serialize_json_compatible(initial_prepared.metadata), - "graph": _graph_to_dict(raw_ir_graph), - }, - ) - _write_json( - artifact_dir / "optimized_ir.json", - { - "model_id": args.model_id, - "model_source": model_source, - "task": "multimodal_causal_lm_logits", - "inputs": _serialize_json_compatible(initial_prepared.metadata), - "graph": _graph_to_dict(optimized_ir_graph), - }, - ) - graph_path = artifact_dir / args.graph_filename - tg.graph.save(graph_path) - print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") - print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") - print(f"saved_graph={graph_path}") - - del model - del wrapper - del canonical - del captured - gc.collect() - - for _ in range(args.warmup): - _run_transpiled_generation_once( - tg=tg, - processor=processor, - tokenizer=tokenizer, - prompt=args.prompt, - image_files=image_files, - audio_file=audio_file, - torch_dtype=torch_dtype, - system_prompt=args.system_prompt, - enable_thinking_if_supported=args.enable_thinking, - max_new_tokens=args.max_new_tokens, - stop_sequences=stop_sequences, - ) - for _ in range(args.repeats): - transpiled_runs.append( - _run_transpiled_generation_once( - tg=tg, - processor=processor, - tokenizer=tokenizer, - prompt=args.prompt, - image_files=image_files, - audio_file=audio_file, - torch_dtype=torch_dtype, - system_prompt=args.system_prompt, - enable_thinking_if_supported=args.enable_thinking, - max_new_tokens=args.max_new_tokens, - stop_sequences=stop_sequences, - ) - ) - - summary["transpiled"] = _summarize_transpiled_runs( - transpiled_runs, - compile_time_ms=compile_time_ms, - model_source=model_source, - weight_bindings=weight_bindings, - prepared_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), - ) - - if handwritten_runs and transpiled_runs: - handwritten_latest = handwritten_runs[-1].response.strip() - transpiled_latest = transpiled_runs[-1].response.strip() - summary["comparison"] = { - "end_to_end_speedup_vs_handwritten": ( - summary["handwritten"]["total_time_ms"]["mean"] - / max(summary["transpiled"]["total_ms"]["mean"], 1e-9) - ), - "responses_match": handwritten_latest == transpiled_latest, - "handwritten_latest_response": handwritten_latest, - "transpiled_latest_response": transpiled_latest, - } - - if artifact_dir is not None: - summary_path = artifact_dir / "summary.json" - summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") - print(f"saved_summary={summary_path}") - - _print_summary(summary) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python/examples/compare_parakeet_performance.py b/python/examples/compare_parakeet_performance.py deleted file mode 100644 index 621f1e1f5..000000000 --- a/python/examples/compare_parakeet_performance.py +++ /dev/null @@ -1,793 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -from collections import defaultdict -from dataclasses import asdict -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import numpy as np -import torch -from scipy.io import wavfile - -EXAMPLES_DIR = Path(__file__).resolve().parent -PYTHON_ROOT = EXAMPLES_DIR.parent -PROJECT_ROOT = PYTHON_ROOT.parent - -sys.path.insert(0, str(PYTHON_ROOT)) -sys.path.insert(0, str(EXAMPLES_DIR)) - -from src.cactus import cactus_destroy -from src.cactus import cactus_init -from src.cactus import cactus_transcribe -from src.transpile.capture_pytorch import capture_model -from src.transpile.canonicalize.cleanup import canonicalize_exported_graph -from src.transpile.model_adapters import canonicalize_model_interface -from src.transpile.optimize_graph import FusionConfig -from src.transpile.optimize_graph import optimize_graph -from transpile_hf_model import TranspileWrapper -from transpile_hf_model import _ctc_greedy_decode_token_ids -from transpile_hf_model import _decode_token_ids -from transpile_hf_model import _infer_task_from_config -from transpile_hf_model import _load_optional_json -from transpile_hf_model import _load_optional_tokenizer -from transpile_hf_model import _load_transformers_bundle -from transpile_hf_model import _lower_preoptimized_ir -from transpile_hf_model import _parse_dtype -from transpile_hf_model import _prepare_audio_inputs -from transpile_hf_model import _validate_weights_dir - - -@dataclass -class NumericSummary: - mean: float - min: float - max: float - median: float - - -@dataclass -class ProfileEntry: - op: str - time_ms: float - shape: str - raw_line: str - - -@dataclass -class ProfileSection: - entries: list[ProfileEntry] - total_ms: float - - -@dataclass -class HandwrittenRun: - wall_ms: float - total_time_ms: float - time_to_first_token_ms: float - prefill_tps: float - decode_tps: float - decode_tokens: int - prefill_tokens: int - total_tokens: int - confidence: float - transcript: str - response_json: dict[str, Any] - - -@dataclass -class TranspiledRun: - preprocess_ms: float - execute_ms: float - decode_ms: float - total_ms: float - graph_only_ms: float - output_frames: int - output_vocab: int - decode_tokens: int - transcript: str | None - input_shapes: dict[str, list[int]] - - -def _numeric_summary(values: list[float]) -> NumericSummary: - arr = np.asarray(values, dtype=np.float64) - return NumericSummary( - mean=float(np.mean(arr)), - min=float(np.min(arr)), - max=float(np.max(arr)), - median=float(np.median(arr)), - ) - - -def _resolve_default_weights_dir(model_id: str) -> Path | None: - candidate = PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower() - if (candidate / "config.txt").exists(): - return candidate - return None - - -def _audio_duration_seconds(audio_file: str) -> float: - sample_rate, samples = wavfile.read(audio_file) - if sample_rate <= 0: - raise ValueError(f"invalid sample rate in {audio_file}: {sample_rate}") - return float(len(samples)) / float(sample_rate) - - -def _profile_env(profile_file: Path | None): - class _ProfileContext: - def __enter__(self_inner): - self_inner.old_profile = os.environ.get("CACTUS_PROFILE") - self_inner.old_profile_file = os.environ.get("CACTUS_PROFILE_FILE") - if profile_file is None: - os.environ.pop("CACTUS_PROFILE", None) - os.environ.pop("CACTUS_PROFILE_FILE", None) - return self_inner - profile_file.parent.mkdir(parents=True, exist_ok=True) - if profile_file.exists(): - profile_file.unlink() - os.environ.pop("CACTUS_PROFILE", None) - os.environ["CACTUS_PROFILE_FILE"] = str(profile_file) - return self_inner - - def __exit__(self_inner, exc_type, exc, tb): - if self_inner.old_profile is None: - os.environ.pop("CACTUS_PROFILE", None) - else: - os.environ["CACTUS_PROFILE"] = self_inner.old_profile - - if self_inner.old_profile_file is None: - os.environ.pop("CACTUS_PROFILE_FILE", None) - else: - os.environ["CACTUS_PROFILE_FILE"] = self_inner.old_profile_file - return False - - return _ProfileContext() - - -def _parse_profile_file(path: Path) -> list[ProfileSection]: - if not path.exists(): - return [] - - lines = path.read_text().splitlines() - sections: list[ProfileSection] = [] - idx = 0 - while idx < len(lines): - if lines[idx].strip() != "=== Graph Execution Profile ===": - idx += 1 - continue - - entries: list[ProfileEntry] = [] - total_ms = 0.0 - idx += 1 - while idx < len(lines): - line = lines[idx].rstrip() - stripped = line.strip() - if stripped.startswith("Total execution time:"): - try: - total_ms = float(stripped.split(":", 1)[1].strip().split()[0]) - except Exception: - total_ms = 0.0 - elif stripped == "================================": - break - elif ( - stripped - and not stripped.startswith("Operation") - and not set(stripped).issubset({"-"}) - and not stripped.startswith("Total execution time:") - ): - parts = stripped.split() - if len(parts) >= 3: - op = parts[0] - try: - time_ms = float(parts[1]) - except ValueError: - time_ms = 0.0 - shape = parts[2] if parts[2].startswith("[") else "" - entries.append(ProfileEntry(op=op, time_ms=time_ms, shape=shape, raw_line=stripped)) - idx += 1 - - sections.append(ProfileSection(entries=entries, total_ms=total_ms)) - idx += 1 - - return sections - - -def _summarize_profile_sections(sections: list[ProfileSection]) -> dict[str, Any]: - op_total_ms: dict[str, float] = defaultdict(float) - op_count: dict[str, int] = defaultdict(int) - all_entries: list[ProfileEntry] = [] - total_ms_values: list[float] = [] - - for section in sections: - total_ms_values.append(section.total_ms) - all_entries.extend(section.entries) - for entry in section.entries: - op_total_ms[entry.op] += entry.time_ms - op_count[entry.op] += 1 - - by_op = [] - for op, total_ms in sorted(op_total_ms.items(), key=lambda item: item[1], reverse=True): - count = op_count[op] - by_op.append( - { - "op": op, - "count": count, - "total_ms": round(total_ms, 6), - "mean_ms": round(total_ms / max(count, 1), 6), - } - ) - - slowest_nodes = [ - { - "op": entry.op, - "time_ms": round(entry.time_ms, 6), - "shape": entry.shape, - "raw_line": entry.raw_line, - } - for entry in sorted(all_entries, key=lambda item: item.time_ms, reverse=True)[:50] - ] - - return { - "runs_profiled": len(sections), - "graph_total_ms": asdict(_numeric_summary(total_ms_values)) if total_ms_values else None, - "entries_per_run": [len(section.entries) for section in sections], - "by_op": by_op, - "slowest_nodes": slowest_nodes, - } - - -def _run_handwritten_once( - *, - model_handle: Any, - audio_file: str, -) -> HandwrittenRun: - options_json = json.dumps( - { - "use_vad": False, - "max_tokens": 4096, - } - ) - start = time.perf_counter() - response = cactus_transcribe( - model_handle, - audio_file, - "", - options_json, - None, - None, - ) - end = time.perf_counter() - - payload = json.loads(response) - return HandwrittenRun( - wall_ms=(end - start) * 1000.0, - total_time_ms=float(payload.get("total_time_ms", 0.0)), - time_to_first_token_ms=float(payload.get("time_to_first_token_ms", 0.0)), - prefill_tps=float(payload.get("prefill_tps", 0.0)), - decode_tps=float(payload.get("decode_tps", 0.0)), - decode_tokens=int(payload.get("decode_tokens", 0)), - prefill_tokens=int(payload.get("prefill_tokens", 0)), - total_tokens=int(payload.get("total_tokens", 0)), - confidence=float(payload.get("confidence", 0.0)), - transcript=str(payload.get("response", "")), - response_json=payload, - ) - - -def _run_transpiled_once( - *, - tg: Any, - processor: object | None, - input_names: tuple[str, ...], - model_config: dict[str, object], - preprocessor_config: dict[str, object], - model: torch.nn.Module, - torch_dtype: torch.dtype, - audio_file: str, - tokenizer: object | None, - blank_token_id: int | None, -) -> TranspiledRun: - start_pre = time.perf_counter() - prepared = _prepare_audio_inputs( - processor, - input_names=input_names, - config=model_config, - preprocessor_config=preprocessor_config, - model=model, - task="ctc_logits", - audio_file=audio_file, - torch_dtype=torch_dtype, - ) - end_pre = time.perf_counter() - - input_arrays = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] - - start_exec = time.perf_counter() - tg.set_inputs(input_arrays) - outputs = tg.execute() - end_exec = time.perf_counter() - - start_decode = time.perf_counter() - logits = outputs[0].numpy().astype(np.float32) - token_ids = _ctc_greedy_decode_token_ids(logits, blank_token_id=blank_token_id) - transcript = _decode_token_ids(tokenizer, token_ids) if tokenizer is not None else None - end_decode = time.perf_counter() - - return TranspiledRun( - preprocess_ms=(end_pre - start_pre) * 1000.0, - execute_ms=(end_exec - start_exec) * 1000.0, - decode_ms=(end_decode - start_decode) * 1000.0, - total_ms=(end_decode - start_pre) * 1000.0, - graph_only_ms=(end_exec - start_exec) * 1000.0, - output_frames=int(logits.shape[1]), - output_vocab=int(logits.shape[2]), - decode_tokens=len(token_ids), - transcript=transcript, - input_shapes={ - name: list(tensor.shape) - for name, tensor in zip(prepared.names, prepared.tensors) - }, - ) - - -def _run_transpiled_graph_only_once( - *, - tg: Any, - cached_inputs: list[np.ndarray], -) -> float: - start = time.perf_counter() - tg.set_inputs(cached_inputs) - tg.execute() - end = time.perf_counter() - return (end - start) * 1000.0 - - -def _warmup(fn, count: int) -> None: - for _ in range(max(count, 0)): - fn() - - -def _roundtrip_jsonable(obj: Any) -> Any: - if isinstance(obj, dict): - return {str(key): _roundtrip_jsonable(value) for key, value in obj.items()} - if isinstance(obj, list): - return [_roundtrip_jsonable(value) for value in obj] - if isinstance(obj, tuple): - return [_roundtrip_jsonable(value) for value in obj] - if isinstance(obj, Path): - return str(obj) - if isinstance(obj, NumericSummary): - return asdict(obj) - if hasattr(obj, "__dataclass_fields__"): - return asdict(obj) - return obj - - -def _build_transpiled_graph( - *, - model_id: str, - audio_file: str, - weights_dir: str | None, - torch_dtype: torch.dtype, - token: str | None, - trust_remote_code: bool, - local_files_only: bool, - fusion_config: FusionConfig, -): - task = _infer_task_from_config(model_id) - if task != "ctc_logits": - raise RuntimeError(f"expected a CTC model, got task={task}") - - model_source, processor, model, model_config = _load_transformers_bundle( - model_id=model_id, - task=task, - torch_dtype=torch_dtype, - token=token, - trust_remote_code=trust_remote_code, - local_files_only=local_files_only, - ) - - preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") - if not preprocessor_config: - preprocessor_config = _load_optional_json(model_id, "preprocessor_config.json") - - tokenizer = _load_optional_tokenizer( - model_id=model_id, - model_source=model_source, - token=token, - trust_remote_code=trust_remote_code, - local_files_only=local_files_only, - ) - - canonical = canonicalize_model_interface(model, task=task) - prepared = _prepare_audio_inputs( - processor, - input_names=canonical.input_names, - config=model_config, - preprocessor_config=preprocessor_config, - model=model, - task=task, - audio_file=audio_file, - torch_dtype=torch_dtype, - ) - canonical = canonicalize_model_interface(model, task=task, input_names=prepared.names) - - wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() - - capture_started = time.perf_counter() - captured = capture_model(wrapper, prepared.tensors) - canonicalize_exported_graph(captured.ir_graph) - optimize_graph(captured.ir_graph, config=fusion_config) - tg = _lower_preoptimized_ir(captured.ir_graph) - capture_finished = time.perf_counter() - - cached_inputs = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] - blank_token_id = getattr(getattr(model, "config", None), "pad_token_id", None) - - return { - "tg": tg, - "model": model, - "processor": processor, - "model_config": model_config, - "preprocessor_config": preprocessor_config, - "tokenizer": tokenizer, - "blank_token_id": int(blank_token_id) if blank_token_id is not None else None, - "compile_time_ms": (capture_finished - capture_started) * 1000.0, - "cached_inputs": cached_inputs, - "input_names": tuple(prepared.names), - "prepared_input_shapes": { - name: list(tensor.shape) - for name, tensor in zip(prepared.names, prepared.tensors) - }, - "model_source": model_source, - "weight_bindings": sum( - 1 - for value in captured.ir_graph.values.values() - if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) - ), - } - - -def _benchmark_handwritten( - *, - weights_dir: Path, - audio_file: str, - warmup: int, - repeats: int, - profile_repeats: int, - artifact_dir: Path | None, -) -> dict[str, Any]: - model_handle = cactus_init(str(weights_dir), None, False) - try: - _warmup(lambda: _run_handwritten_once(model_handle=model_handle, audio_file=audio_file), warmup) - - runs: list[HandwrittenRun] = [] - for _ in range(max(repeats, 1)): - runs.append(_run_handwritten_once(model_handle=model_handle, audio_file=audio_file)) - - profile_path = ( - artifact_dir / "handwritten_profile.txt" - if artifact_dir is not None and profile_repeats > 0 - else None - ) - with _profile_env(profile_path): - for _ in range(max(profile_repeats, 0)): - _run_handwritten_once(model_handle=model_handle, audio_file=audio_file) - - profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] - profile_summary = _summarize_profile_sections(profile_sections) - - return { - "weights_dir": str(weights_dir), - "runs": [_roundtrip_jsonable(run) for run in runs], - "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), - "internal_total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), - "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), - "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), - "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), - "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), - "confidence": asdict(_numeric_summary([run.confidence for run in runs])), - "latest_transcript": runs[-1].transcript if runs else "", - "profile_file": str(profile_path) if profile_path is not None else None, - "profile_summary": profile_summary, - } - finally: - cactus_destroy(model_handle) - - -def _benchmark_transpiled( - *, - model_id: str, - audio_file: str, - weights_dir: str | None, - warmup: int, - repeats: int, - profile_repeats: int, - torch_dtype: torch.dtype, - token: str | None, - trust_remote_code: bool, - local_files_only: bool, - artifact_dir: Path | None, - fusion_config: FusionConfig, -) -> dict[str, Any]: - built = _build_transpiled_graph( - model_id=model_id, - audio_file=audio_file, - weights_dir=weights_dir, - torch_dtype=torch_dtype, - token=token, - trust_remote_code=trust_remote_code, - local_files_only=local_files_only, - fusion_config=fusion_config, - ) - - tg = built["tg"] - model = built["model"] - processor = built["processor"] - input_names = built["input_names"] - model_config = built["model_config"] - preprocessor_config = built["preprocessor_config"] - tokenizer = built["tokenizer"] - blank_token_id = built["blank_token_id"] - cached_inputs = built["cached_inputs"] - - _warmup( - lambda: _run_transpiled_once( - tg=tg, - processor=processor, - input_names=input_names, - model_config=model_config, - preprocessor_config=preprocessor_config, - model=model, - torch_dtype=torch_dtype, - audio_file=audio_file, - tokenizer=tokenizer, - blank_token_id=blank_token_id, - ), - warmup, - ) - _warmup(lambda: _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs), warmup) - - runs: list[TranspiledRun] = [] - graph_only_ms: list[float] = [] - for _ in range(max(repeats, 1)): - runs.append( - _run_transpiled_once( - tg=tg, - processor=processor, - input_names=input_names, - model_config=model_config, - preprocessor_config=preprocessor_config, - model=model, - torch_dtype=torch_dtype, - audio_file=audio_file, - tokenizer=tokenizer, - blank_token_id=blank_token_id, - ) - ) - graph_only_ms.append(_run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs)) - - profile_path = ( - artifact_dir / "transpiled_profile.txt" - if artifact_dir is not None and profile_repeats > 0 - else None - ) - with _profile_env(profile_path): - for _ in range(max(profile_repeats, 0)): - _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs) - - profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] - profile_summary = _summarize_profile_sections(profile_sections) - - return { - "model_source": built["model_source"], - "compile_time_ms": built["compile_time_ms"], - "weight_bindings": built["weight_bindings"], - "prepared_input_shapes": built["prepared_input_shapes"], - "runs": [_roundtrip_jsonable(run) for run in runs], - "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), - "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), - "decode_ms": asdict(_numeric_summary([run.decode_ms for run in runs])), - "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), - "graph_only_ms": asdict(_numeric_summary(graph_only_ms)), - "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), - "output_frames": asdict(_numeric_summary([float(run.output_frames) for run in runs])), - "latest_transcript": runs[-1].transcript if runs else "", - "profile_file": str(profile_path) if profile_path is not None else None, - "profile_summary": profile_summary, - } - - -def _print_summary( - *, - audio_duration_sec: float, - handwritten: dict[str, Any] | None, - transpiled: dict[str, Any] | None, -) -> None: - print() - print(f"audio_duration_sec={audio_duration_sec:.3f}") - - if handwritten is not None: - hw_total = handwritten["internal_total_time_ms"]["mean"] - hw_graph = None - profile_total = handwritten.get("profile_summary", {}).get("graph_total_ms") - if isinstance(profile_total, dict): - hw_graph = profile_total.get("mean") - hw_tok = handwritten["decode_tokens"]["mean"] - hw_rtf = hw_total / max(audio_duration_sec * 1000.0, 1e-9) - hw_xrt = (audio_duration_sec * 1000.0) / max(hw_total, 1e-9) - hw_tok_s = hw_tok * 1000.0 / max(hw_total, 1e-9) - print("handwritten:") - print(f" total_ms mean={hw_total:.3f}") - print(f" graph_profile_ms mean={hw_graph:.3f}" if hw_graph is not None else " graph_profile_ms unavailable") - print(f" decode_tokens mean={hw_tok:.3f}") - print(f" output_tokens_per_sec={hw_tok_s:.3f}") - print(f" real_time_factor={hw_rtf:.3f}") - print(f" audio_x_realtime={hw_xrt:.3f}") - - if transpiled is not None: - tr_total = transpiled["total_ms"]["mean"] - tr_graph = transpiled["graph_only_ms"]["mean"] - tr_tok = transpiled["decode_tokens"]["mean"] - tr_rtf = tr_total / max(audio_duration_sec * 1000.0, 1e-9) - tr_xrt = (audio_duration_sec * 1000.0) / max(tr_total, 1e-9) - tr_tok_s = tr_tok * 1000.0 / max(tr_total, 1e-9) - tr_graph_tok_s = tr_tok * 1000.0 / max(tr_graph, 1e-9) - print("transpiled:") - print(f" total_ms mean={tr_total:.3f}") - print(f" graph_only_ms mean={tr_graph:.3f}") - print(f" decode_tokens mean={tr_tok:.3f}") - print(f" output_tokens_per_sec={tr_tok_s:.3f}") - print(f" graph_only_tokens_per_sec={tr_graph_tok_s:.3f}") - print(f" real_time_factor={tr_rtf:.3f}") - print(f" audio_x_realtime={tr_xrt:.3f}") - - -def main() -> int: - parser = argparse.ArgumentParser( - description=( - "Compare handwritten Cactus Parakeet performance against the transpiled " - "Parakeet graph on the same WAV input, including per-op graph profiles." - ) - ) - parser.add_argument("--model-id", default="nvidia/parakeet-ctc-1.1b") - parser.add_argument("--audio-file", required=True) - parser.add_argument( - "--weights-dir", - default="", - help="Converted Cactus weights directory. Reused for handwritten runtime and transpiled mmap bindings.", - ) - parser.add_argument("--artifact-dir", default="", help="Optional directory for JSON summaries and raw profiles.") - parser.add_argument("--warmup", type=int, default=1) - parser.add_argument("--repeats", type=int, default=5) - parser.add_argument("--profile-repeats", type=int, default=1) - parser.add_argument("--torch-dtype", default="float16") - parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) - parser.add_argument("--trust-remote-code", action="store_true") - parser.add_argument("--local-files-only", action="store_true") - parser.add_argument("--skip-handwritten", action="store_true") - parser.add_argument("--skip-transpiled", action="store_true") - parser.add_argument("--no-fuse-gated-deltanet", action="store_true") - parser.add_argument("--no-fuse-rms-norm", action="store_true") - parser.add_argument("--no-fuse-rope", action="store_true") - parser.add_argument("--no-fuse-attention", action="store_true") - parser.add_argument("--no-fuse-attention-block", action="store_true") - parser.add_argument("--no-fuse-add-clipped", action="store_true") - args = parser.parse_args() - - audio_file = str(Path(args.audio_file).resolve()) - if not Path(audio_file).exists(): - raise FileNotFoundError(f"audio file does not exist: {audio_file}") - - torch_dtype = _parse_dtype(args.torch_dtype) - artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None - if artifact_dir is not None: - artifact_dir.mkdir(parents=True, exist_ok=True) - - weights_dir_path = None - if args.weights_dir.strip(): - weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) - else: - weights_dir_path = _resolve_default_weights_dir(args.model_id) - - if not args.skip_handwritten and weights_dir_path is None: - raise RuntimeError( - "handwritten comparison requires a converted weights directory.\n" - "\n" - f"Create one with:\n" - f" cactus convert {args.model_id} {PROJECT_ROOT / 'weights' / args.model_id.split('/')[-1].lower()}\n" - "\n" - "Or rerun with --skip-handwritten." - ) - - fusion_config = FusionConfig( - enable_gated_deltanet=not args.no_fuse_gated_deltanet, - enable_rms_norm=not args.no_fuse_rms_norm, - enable_rope=not args.no_fuse_rope, - enable_attention=not args.no_fuse_attention, - enable_attention_block=not args.no_fuse_attention_block, - enable_add_clipped=not args.no_fuse_add_clipped, - ) - - summary: dict[str, Any] = { - "model_id": args.model_id, - "audio_file": audio_file, - "audio_duration_sec": _audio_duration_seconds(audio_file), - "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, - "warmup": args.warmup, - "repeats": args.repeats, - "profile_repeats": args.profile_repeats, - "torch_dtype": args.torch_dtype, - "trust_remote_code": bool(args.trust_remote_code), - "local_files_only": bool(args.local_files_only), - } - - handwritten_summary = None - if not args.skip_handwritten and weights_dir_path is not None: - handwritten_summary = _benchmark_handwritten( - weights_dir=weights_dir_path, - audio_file=audio_file, - warmup=args.warmup, - repeats=args.repeats, - profile_repeats=args.profile_repeats, - artifact_dir=artifact_dir, - ) - summary["handwritten"] = handwritten_summary - - transpiled_summary = None - if not args.skip_transpiled: - transpiled_summary = _benchmark_transpiled( - model_id=args.model_id, - audio_file=audio_file, - weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, - warmup=args.warmup, - repeats=args.repeats, - profile_repeats=args.profile_repeats, - torch_dtype=torch_dtype, - token=args.token, - trust_remote_code=args.trust_remote_code, - local_files_only=args.local_files_only, - artifact_dir=artifact_dir, - fusion_config=fusion_config, - ) - summary["transpiled"] = transpiled_summary - - if handwritten_summary is not None and transpiled_summary is not None: - hw_total = handwritten_summary["internal_total_time_ms"]["mean"] - tr_total = transpiled_summary["total_ms"]["mean"] - hw_graph = handwritten_summary.get("profile_summary", {}).get("graph_total_ms") - tr_graph = transpiled_summary["graph_only_ms"]["mean"] - summary["comparison"] = { - "end_to_end_speedup_vs_handwritten": hw_total / max(tr_total, 1e-9), - "graph_only_speedup_vs_handwritten": ( - float(hw_graph["mean"]) / max(tr_graph, 1e-9) - if isinstance(hw_graph, dict) and "mean" in hw_graph - else None - ), - } - - if artifact_dir is not None: - summary_path = artifact_dir / "summary.json" - summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") - print(f"saved_summary={summary_path}") - if handwritten_summary is not None and handwritten_summary.get("profile_file"): - print(f"saved_handwritten_profile={handwritten_summary['profile_file']}") - if transpiled_summary is not None and transpiled_summary.get("profile_file"): - print(f"saved_transpiled_profile={transpiled_summary['profile_file']}") - - _print_summary( - audio_duration_sec=float(summary["audio_duration_sec"]), - handwritten=handwritten_summary, - transpiled=transpiled_summary, - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python/examples/inspect_gemma4_cleanup.py b/python/examples/inspect_gemma4_cleanup.py deleted file mode 100644 index ec9e78d05..000000000 --- a/python/examples/inspect_gemma4_cleanup.py +++ /dev/null @@ -1,241 +0,0 @@ -from __future__ import annotations - -import argparse -import os -from pathlib import Path - -import torch -from transformers import AutoModelForCausalLM - -from src.transpile.capture_pytorch import capture_model -from src.transpile.canonicalize.cleanup import canonicalize_exported_graph -from src.transpile.graph_ir import IRGraph -from src.transpile.model_adapters import canonicalize_model_interface - - -class Gemma4FullModelWrapper(torch.nn.Module): - def __init__(self, model: torch.nn.Module): - super().__init__() - self.model = canonicalize_model_interface(model, task="causal_lm_logits").module - - def forward(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model(input_ids) - - -class Gemma4FirstBlockCheckpointWrapper(torch.nn.Module): - def __init__(self, model: torch.nn.Module, checkpoint_name: str): - super().__init__() - adapter = canonicalize_model_interface(model, task="causal_lm_logits") - if adapter.family != "gemma4": - raise ValueError(f"expected gemma4 adapter, got family={adapter.family}") - if not hasattr(adapter.module, "debug_first_block"): - raise ValueError("Gemma4 adapter does not expose debug_first_block()") - self.adapter = adapter.module - self.checkpoint_name = checkpoint_name - - def forward(self, input_ids: torch.Tensor) -> torch.Tensor: - checkpoints = self.adapter.debug_first_block(input_ids) - if self.checkpoint_name not in checkpoints: - available = ", ".join(sorted(checkpoints.keys())) - raise KeyError(f"unknown checkpoint {self.checkpoint_name!r}; available: {available}") - return checkpoints[self.checkpoint_name] - - -def format_ir_graph(graph: IRGraph, *, max_nodes: int | None = None) -> str: - lines: list[str] = [] - lines.append("IR Summary") - lines.append(f" inputs={graph.inputs}") - lines.append(f" outputs={graph.outputs}") - lines.append(f" nodes={len(graph.order)}") - if graph.meta: - lines.append(f" meta={graph.meta}") - lines.append("") - lines.append("IR Nodes") - - node_ids = graph.order if max_nodes is None else graph.order[:max_nodes] - for index, node_id in enumerate(node_ids): - node = graph.nodes[node_id] - lines.append(f"[{index}] {node.id} op={node.op}") - lines.append(f" inputs={node.inputs}") - lines.append(f" outputs={node.outputs}") - if node.attrs: - lines.append(f" attrs={node.attrs}") - if node.kind != "generic": - lines.append(f" kind={node.kind}") - if node.meta: - lines.append(f" meta={node.meta}") - for output_id in node.outputs: - value = graph.values.get(output_id) - if value is None: - continue - lines.append( - f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" - ) - - if max_nodes is not None and len(graph.order) > max_nodes: - lines.append(f"... ({len(graph.order) - max_nodes} more nodes omitted)") - - if graph.constants: - lines.append("") - lines.append("IR Constants") - constant_ids = sorted(graph.constants.keys()) - preview_ids = constant_ids if max_nodes is None else constant_ids[: max(8, min(len(constant_ids), max_nodes))] - for value_id in preview_ids: - value = graph.constants[value_id] - value_meta = graph.values.get(value_id) - shape = None if value_meta is None else value_meta.shape - dtype = None if value_meta is None else value_meta.dtype - lines.append(f"- {value_id}: type={type(value).__name__} shape={shape} dtype={dtype}") - if len(preview_ids) < len(constant_ids): - lines.append(f"... ({len(constant_ids) - len(preview_ids)} more constants omitted)") - - return "\n".join(lines) - - -def format_unsupported_ops(graph: IRGraph) -> str: - counts = graph.meta.get("canonical_unsupported_op_counts", {}) - if not isinstance(counts, dict): - counts = {} - - examples: dict[str, list[str]] = {} - for node_id in graph.order: - node = graph.nodes[node_id] - print(f"node {node_id} op={node.op}") - if node.op not in counts: - continue - bucket = examples.setdefault(node.op, []) - if len(bucket) < 5: - bucket.append(node.id) - - total = sum(int(count) for count in counts.values()) - lines = ["Unsupported Ops", f" unique={len(counts)}", f" total_nodes={total}"] - if not counts: - lines.append(" none") - return "\n".join(lines) - - for op_name, count in counts.items(): - sample_nodes = ", ".join(examples.get(op_name, ())) - lines.append(f"- {op_name}: {count}") - if sample_nodes: - lines.append(f" examples={sample_nodes}") - return "\n".join(lines) - - -def resolve_local_model_path(model_id: str) -> str: - model_path = Path(model_id) - if model_path.exists(): - return str(model_path) - - cache_root = Path.home() / ".cache" / "huggingface" / "hub" / ("models--" + model_id.replace("/", "--")) - snapshots_dir = cache_root / "snapshots" - if not snapshots_dir.exists(): - raise FileNotFoundError( - f"could not find local snapshot for {model_id!r} under {snapshots_dir}" - ) - snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) - if not snapshots: - raise FileNotFoundError(f"no snapshots found for {model_id!r} under {snapshots_dir}") - return str(snapshots[-1]) - - -def load_model(model_id: str) -> torch.nn.Module: - token = os.environ.get("HF_TOKEN") - common_kwargs: dict[str, object] = { - "local_files_only": True, - } - if token: - common_kwargs["token"] = token - - local_model_path = resolve_local_model_path(model_id) - model = AutoModelForCausalLM.from_pretrained( - local_model_path, - torch_dtype=torch.float16, - device_map=None, - low_cpu_mem_usage=True, - **common_kwargs, - ).eval() - return model - - -def build_module( - model: torch.nn.Module, - *, - mode: str, - checkpoint: str, -) -> torch.nn.Module: - if mode == "full": - return Gemma4FullModelWrapper(model).eval() - if mode == "first_block": - return Gemma4FirstBlockCheckpointWrapper(model, checkpoint).eval() - raise ValueError(f"unknown mode: {mode}") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Capture Gemma4, run canonicalize/cleanup.py, and report unsupported ops.", - ) - parser.add_argument( - "--model-id", - default=os.environ.get("CACTUS_GEMMA_HF_MODEL_ID", "google/gemma-4-E2B"), - help="Hugging Face model id. Uses local cache only.", - ) - parser.add_argument( - "--mode", - choices=("first_block", "full"), - default="first_block", - help="Capture a first-block checkpoint or the full Gemma4 adapter graph.", - ) - parser.add_argument( - "--checkpoint", - default="after_ffn_residual", - help="First-block checkpoint name when --mode=first_block.", - ) - parser.add_argument( - "--input-ids", - default="2,818,5279,529,7001,563", - help="Comma-separated integer token ids to capture.", - ) - parser.add_argument( - "--max-nodes", - type=int, - default=120, - help="Maximum IR nodes to print when --print-ir is enabled.", - ) - parser.add_argument( - "--print-ir", - action="store_true", - help="Also print the IR before and after canonical cleanup.", - ) - args = parser.parse_args() - - model = load_model(args.model_id) - module = build_module(model, mode=args.mode, checkpoint=args.checkpoint) - token_ids = [int(token.strip()) for token in args.input_ids.split(",") if token.strip()] - input_ids = torch.tensor([token_ids], dtype=torch.long) - - print(f"Model: {args.model_id}") - print(f"Mode: {args.mode}") - print(f"Checkpoint: {args.checkpoint if args.mode == 'first_block' else '(full model)'}") - print(f"Input IDs shape: {tuple(input_ids.shape)}") - print("") - - captured = capture_model(module, (input_ids,)) - - if args.print_ir: - print("=== IR Before Canonical Cleanup ===") - print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) - print("") - - canonicalize_exported_graph(captured.ir_graph) - - print("=== Unsupported Ops After Canonical Cleanup ===") - print(format_unsupported_ops(captured.ir_graph)) - - if args.print_ir: - print("") - print("=== IR After Canonical Cleanup ===") - print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) - - -if __name__ == "__main__": - main() diff --git a/python/examples/inspect_transpile_cleanup.py b/python/examples/inspect_transpile_cleanup.py deleted file mode 100644 index 366765f62..000000000 --- a/python/examples/inspect_transpile_cleanup.py +++ /dev/null @@ -1,138 +0,0 @@ -from __future__ import annotations - -import argparse - -import torch -import torch.nn as nn - -from src.transpile.capture_pytorch import capture_model -from src.transpile.capture_pytorch import dump_graph -from src.transpile.canonicalize.cleanup import canonicalize_exported_graph -from src.transpile.graph_ir import IRGraph - - -class Toy(nn.Module): - def forward(self, x, y): - z = x + y - z = z * 0.5 - z = torch.nn.functional.gelu(z) - z = torch.softmax(z, dim=-1) - return z - - -class AttentionBlockToy(nn.Module): - def __init__(self, hidden_size: int = 8): - super().__init__() - self.out_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=torch.float16) - - def forward(self, q, k, v, gate): - attn = torch.nn.functional.scaled_dot_product_attention(q, k, v, is_causal=True, scale=1.0) - attn = attn.transpose(1, 2).reshape(q.shape[0], q.shape[2], -1) - attn = attn * torch.sigmoid(gate) - attn = attn.to(self.out_proj.weight.dtype) - return self.out_proj(attn) - - -def build_case(name: str) -> tuple[nn.Module, tuple[torch.Tensor, ...]]: - if name == "toy": - model = Toy().eval() - args = ( - torch.randn(2, 4, dtype=torch.float16), - torch.randn(2, 4, dtype=torch.float16), - ) - return model, args - - if name == "attention_block": - model = AttentionBlockToy().eval() - args = ( - torch.randn(1, 2, 3, 4, dtype=torch.float16), - torch.randn(1, 2, 3, 4, dtype=torch.float16), - torch.randn(1, 2, 3, 4, dtype=torch.float16), - torch.randn(1, 3, 8, dtype=torch.float16), - ) - return model, args - - raise ValueError(f"unknown case: {name}") - - -def format_ir_graph(graph: IRGraph) -> str: - lines: list[str] = [] - lines.append("IR Summary") - lines.append(f" inputs={graph.inputs}") - lines.append(f" outputs={graph.outputs}") - lines.append(f" nodes={len(graph.order)}") - if graph.meta: - lines.append(f" meta={graph.meta}") - lines.append("") - lines.append("IR Nodes") - - for index, node_id in enumerate(graph.order): - node = graph.nodes[node_id] - lines.append(f"[{index}] {node.id} op={node.op}") - lines.append(f" inputs={node.inputs}") - lines.append(f" outputs={node.outputs}") - if node.attrs: - lines.append(f" attrs={node.attrs}") - if node.kind != "generic": - lines.append(f" kind={node.kind}") - if node.meta: - lines.append(f" meta={node.meta}") - for output_id in node.outputs: - value = graph.values.get(output_id) - if value is None: - continue - lines.append( - f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" - ) - - if graph.constants: - lines.append("") - lines.append("IR Constants") - for value_id, value in graph.constants.items(): - value_meta = graph.values.get(value_id) - shape = None if value_meta is None else value_meta.shape - dtype = None if value_meta is None else value_meta.dtype - typename = type(value).__name__ - lines.append(f"- {value_id}: type={typename} shape={shape} dtype={dtype}") - - return "\n".join(lines) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Capture a small PyTorch model, dump the exported graph, and print IR before/after cleanup.", - ) - parser.add_argument( - "--case", - choices=("toy", "attention_block"), - default="toy", - help="Which built-in model to inspect.", - ) - parser.add_argument( - "--no-meta", - action="store_true", - help="Hide exported graph metadata to keep the dump shorter.", - ) - args = parser.parse_args() - - model, example_args = build_case(args.case) - captured = capture_model(model, example_args) - - print(f"Case: {args.case}") - print(f"Model: {type(model).__name__}") - print("") - print("=== Exported Graph ===") - print(dump_graph(captured, include_meta=not args.no_meta)) - print("") - print("=== IR Before Cleanup ===") - print(format_ir_graph(captured.ir_graph)) - print("") - - canonicalize_exported_graph(captured.ir_graph) - - print("=== IR After Cleanup ===") - print(format_ir_graph(captured.ir_graph)) - - -if __name__ == "__main__": - main() diff --git a/python/examples/test_torch_capture.py b/python/examples/test_torch_capture.py deleted file mode 100644 index c2ff748b4..000000000 --- a/python/examples/test_torch_capture.py +++ /dev/null @@ -1,70 +0,0 @@ -import torch -import torch.nn as nn - -from src.transpile.capture_pytorch import ( - capture_model_with_fallback, - dump_graph, - get_dtype, - get_shape, -) -from src.transpile.lower import transpile_captured - - -class TinyBlock(nn.Module): - def __init__(self): - super().__init__() - self.fc1 = nn.Linear(8, 16) - self.fc2 = nn.Linear(16, 4) - - def forward(self, x): - h = self.fc1(x) - h = torch.relu(h) - y = self.fc2(h) - return y - - -def main(): - model = TinyBlock().eval() - x = torch.randn(2, 8) - - captured = capture_model_with_fallback(model, args=(x,)) - - print("strict:", captured.strict) - print("graph_module type:", type(captured.graph_module).__name__) - print("state_dict keys:", list(captured.state_dict.keys())) - print() - - print("=== FX Graph ===") - print(captured.graph) - print() - - print("=== Node Walk ===") - for i, node in enumerate(captured.graph.nodes): - print(f"[{i}] op={node.op} name={node.name} target={node.target}") - print(f" args={node.args}") - print(f" kwargs={node.kwargs}") - print(f" shape={get_shape(node)} dtype={get_dtype(node)}") - print() - - print("=== Full Dump ===") - print(dump_graph(captured)) - print() - - print("=== Lower To Cactus ===") - try: - transpiled = transpile_captured(captured) - print("lowering succeeded") - print("lowered graph type:", type(transpiled.graph).__name__) - print("runtime inputs:", transpiled.runtime_inputs) - print("bound constants:", transpiled.bound_constants) - print("outputs:", transpiled.outputs) - print() - print("binding runtime input 0 from example tensor") - transpiled.set_input(0, x) - except NotImplementedError as exc: - print("lowering stopped on unimplemented case:") - print(f" {exc}") - - -if __name__ == "__main__": - main() diff --git a/python/examples/transpile_hf_causal_lm.py b/python/examples/transpile_hf_causal_lm.py deleted file mode 100644 index 05e70b8c9..000000000 --- a/python/examples/transpile_hf_causal_lm.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -import runpy -import sys -from pathlib import Path - - -def main() -> int: - if "--task" not in sys.argv[1:]: - sys.argv[1:1] = ["--task", "causal_lm_logits"] - script_path = Path(__file__).with_name("transpile_hf_model.py") - runpy.run_path(str(script_path), run_name="__main__") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python/examples/transpile_hf_model.py b/python/examples/transpile_hf_model.py deleted file mode 100644 index 8a86c7d7d..000000000 --- a/python/examples/transpile_hf_model.py +++ /dev/null @@ -1,1734 +0,0 @@ -from __future__ import annotations - -import argparse -import builtins -import copy -import gc -import importlib.util -import json -import os -import re -import sys -from collections import Counter -from dataclasses import dataclass -from dataclasses import fields -from functools import lru_cache -from pathlib import Path -from typing import Any - -import numpy as np -import torch -from scipy.io import wavfile -from scipy.signal import resample_poly - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from src.graph import Graph -from src.transpile.capture_pytorch import capture_model -from src.transpile.canonicalize.cleanup import canonicalize_exported_graph -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import verify_ir -from src.transpile.lower import TranspiledGraph -from src.transpile.lower import _lower_constant_value -from src.transpile.lower import _lower_input_value -from src.transpile.lower import _lower_ir_node -from src.transpile.model_adapters import canonicalize_model_interface -from src.transpile.optimize_graph import FusionConfig -from src.transpile.optimize_graph import optimize_graph - - -@dataclass -class PreparedInputs: - names: tuple[str, ...] - tensors: tuple[torch.Tensor, ...] - metadata: dict[str, object] - - -def _transformers_supports_model_module(module_name: str) -> bool: - try: - return importlib.util.find_spec(module_name) is not None - except Exception: - return False - - -def _candidate_external_site_packages() -> list[Path]: - candidates: list[Path] = [] - major_minor = f"python{sys.version_info.major}.{sys.version_info.minor}" - pyenv_versions = Path.home() / ".pyenv" / "versions" - if pyenv_versions.exists(): - for version_dir in sorted(pyenv_versions.iterdir(), reverse=True): - site_packages = version_dir / "lib" / major_minor / "site-packages" - if site_packages.exists(): - candidates.append(site_packages) - return candidates - - -def _ensure_transformers_supports_model_type(model_type: str) -> str | None: - normalized = str(model_type or "").strip().lower() - target_module = { - "gemma4": "transformers.models.gemma4.modeling_gemma4", - }.get(normalized) - if not target_module: - return None - if _transformers_supports_model_module(target_module): - return None - - for site_packages in _candidate_external_site_packages(): - candidate = site_packages / Path(*target_module.split(".")) - if not candidate.with_suffix(".py").exists() and not candidate.is_dir(): - continue - sys.path.insert(0, str(site_packages)) - for module_name in list(sys.modules): - root_name = module_name.split(".", 1)[0] - if root_name in {"transformers", "huggingface_hub", "tokenizers"}: - del sys.modules[module_name] - if _transformers_supports_model_module(target_module): - return str(site_packages) - try: - sys.path.remove(str(site_packages)) - except ValueError: - pass - return None - - -def _resolve_local_snapshot(model_id_or_path: str) -> str | None: - explicit = Path(model_id_or_path) - if explicit.exists(): - return str(explicit) - - snapshots_dir = ( - Path.home() - / ".cache" - / "huggingface" - / "hub" - / ("models--" + model_id_or_path.replace("/", "--")) - / "snapshots" - ) - if not snapshots_dir.exists(): - return None - snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) - if not snapshots: - return None - return str(snapshots[-1]) - - -def _snapshot_has_model_weights(path: str | Path) -> bool: - root = Path(path) - if not root.exists() or not root.is_dir(): - return False - candidates = ( - "model.safetensors", - "model.safetensors.index.json", - "pytorch_model.bin", - "pytorch_model.bin.index.json", - ) - return any((root / name).exists() for name in candidates) - - -def _validate_weights_dir(weights_dir: str | None, *, model_id: str) -> Path | None: - if not weights_dir: - return None - - root = Path(weights_dir).resolve() - if not root.exists(): - raise RuntimeError( - f"weights_dir does not exist: {root}\n" - "\n" - f"Create the folder first with:\n" - f" cactus convert {model_id} {root}\n" - ) - - manifest_path = root / "weights_manifest.json" - if not manifest_path.exists(): - raise RuntimeError( - f"weights_dir is missing weights_manifest.json: {manifest_path}\n" - "\n" - f"Re-convert with the current converter:\n" - f" cactus convert {model_id} {root}\n" - ) - - return root - - -def _serialize_json_compatible(value: Any) -> Any: - if isinstance(value, dict): - return {str(key): _serialize_json_compatible(inner) for key, inner in value.items()} - if isinstance(value, (list, tuple)): - return [_serialize_json_compatible(inner) for inner in value] - if isinstance(value, Path): - return str(value) - if isinstance(value, (str, int, float, bool)) or value is None: - return value - if isinstance(value, torch.dtype): - return str(value) - if isinstance(value, torch.Tensor): - return { - "type": "torch.Tensor", - "dtype": str(value.dtype), - "shape": list(value.shape), - } - if isinstance(value, np.ndarray): - return { - "type": "numpy.ndarray", - "dtype": str(value.dtype), - "shape": list(value.shape), - } - if hasattr(value, "__dataclass_fields__"): - return { - field.name: _serialize_json_compatible(getattr(value, field.name)) - for field in fields(value) - } - try: - return repr(value) - except Exception: - return f"<{type(value).__module__}.{type(value).__name__}>" - - -def _graph_to_dict(graph) -> dict[str, object]: - return { - "meta": _serialize_json_compatible(graph.meta), - "inputs": list(graph.inputs), - "outputs": list(graph.outputs), - "constants": { - value_id: _serialize_json_compatible(constant) - for value_id, constant in graph.constants.items() - }, - "values": { - value_id: _serialize_json_compatible(value) - for value_id, value in graph.values.items() - }, - "nodes": [ - _serialize_json_compatible(graph.nodes[node_id]) - for node_id in graph.order - ], - } - - -def _write_json(path: Path, payload: dict[str, object]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - - -def _parse_dtype(name: str) -> torch.dtype: - normalized = name.strip().lower() - mapping = { - "float16": torch.float16, - "fp16": torch.float16, - "float32": torch.float32, - "fp32": torch.float32, - "bfloat16": torch.bfloat16, - "bf16": torch.bfloat16, - } - if normalized not in mapping: - raise ValueError(f"unsupported torch dtype: {name}") - return mapping[normalized] - - -class TranspileWrapper(torch.nn.Module): - def __init__(self, adapter_module: torch.nn.Module, *, weights_dir: str | None = None): - super().__init__() - self.adapter = adapter_module - self.weights_dir = weights_dir - - def forward(self, *bound_inputs: torch.Tensor) -> torch.Tensor: - return self.adapter(*bound_inputs) - - def get_transpile_metadata(self) -> dict[str, object]: - metadata: dict[str, object] = {} - provider = getattr(self.adapter, "get_transpile_metadata", None) - if callable(provider): - provided = provider() - if isinstance(provided, dict): - metadata.update(provided) - graph_meta: dict[str, object] = {} - base_graph = metadata.get("graph", {}) - if isinstance(base_graph, dict): - graph_meta.update(base_graph) - if self.weights_dir: - graph_meta["weights_dir"] = self.weights_dir - metadata["graph"] = graph_meta - return metadata - - -def _patch_transformers_torchvision_probe() -> str | None: - has_torchvision = importlib.util.find_spec("torchvision") is not None - has_lzma = importlib.util.find_spec("_lzma") is not None - - if not has_torchvision or has_lzma: - return None - - class _InterpolationModeStub: - NEAREST = "nearest" - BILINEAR = "bilinear" - BICUBIC = "bicubic" - LANCZOS = "lanczos" - - class _TorchvisionFunctionalStub: - InterpolationMode = _InterpolationModeStub - - def __getattr__(self, name: str): - raise RuntimeError( - "torchvision functionality is unavailable because this Python build is missing _lzma; " - f"attempted to access torchvision.transforms.functional.{name}" - ) - - # Some Transformers multimodal image/video modules reference `F.InterpolationMode` - # or `tvF.InterpolationMode` in annotations even when torchvision probing is disabled. - # Installing a tiny builtins-level stub lets those modules import so the PIL fallback - # classes can be selected instead. - builtins.F = _TorchvisionFunctionalStub() - builtins.tvF = builtins.F - - import transformers.utils as tf_utils # type: ignore - import transformers.utils.import_utils as tf_import_utils # type: ignore - - @lru_cache - def _disabled() -> bool: - return False - - tf_import_utils.is_torchvision_available = _disabled - tf_import_utils.is_torchvision_v2_available = _disabled - tf_utils.is_torchvision_available = _disabled - tf_utils.is_torchvision_v2_available = _disabled - return "disabled torchvision import checks because this Python build is missing _lzma" - - -def _patch_torch_flex_attention_compat() -> str | None: - try: - import torch.nn.attention.flex_attention as flex_attention # type: ignore - except Exception: - return None - - if hasattr(flex_attention, "AuxRequest"): - return None - - class _AuxRequest: - def __init__(self, **kwargs): - for key, value in kwargs.items(): - setattr(self, key, value) - - flex_attention.AuxRequest = _AuxRequest # type: ignore[attr-defined] - return "installed torch flex_attention AuxRequest compatibility stub" - - -def _load_local_torch_state_dict(model_source: str) -> dict[str, torch.Tensor] | None: - root = Path(model_source) - if not root.exists() or not root.is_dir(): - return None - - safetensors_path = root / "model.safetensors" - if safetensors_path.exists(): - try: - from safetensors.torch import load_file # type: ignore - - return load_file(str(safetensors_path)) - except Exception: - return None - - pytorch_path = root / "pytorch_model.bin" - if pytorch_path.exists(): - try: - loaded = torch.load(str(pytorch_path), map_location="cpu", weights_only=True) - except TypeError: - loaded = torch.load(str(pytorch_path), map_location="cpu") - return loaded if isinstance(loaded, dict) else None - - return None - - -def _remap_gemma4_checkpoint_state_dict(state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - remapped: dict[str, torch.Tensor] = {} - for key, value in state_dict.items(): - new_key = key - if "audio_tower" in new_key: - new_key = re.sub(r"subsample_conv_projection\.layer(\d+)\.", r"subsample_conv_projection.conv_\1.", new_key) - new_key = new_key.replace("audio_tower.layers.", "audio_tower.conformer.") - new_key = new_key.replace(".feed_forward1.", ".ffw_layer_start.") - new_key = new_key.replace(".feed_forward2.", ".ffw_layer_end.") - new_key = re.sub(r"\.self_attn\.(q_proj|k_proj|v_proj)\.", r".attention.attn.\1.", new_key) - new_key = new_key.replace(".self_attn.per_dim_scale", ".attention.attn.per_dim_scale") - new_key = new_key.replace(".self_attn.relative_k_proj.", ".attention.attn.relative_position_embedding.pos_proj.") - new_key = new_key.replace(".self_attn.post.", ".attention.post.") - new_key = new_key.replace(".norm_pre_attn.", ".attention.pre_attn_norm.") - new_key = new_key.replace(".norm_post_attn.", ".attention.post_norm.") - new_key = new_key.replace(".norm_out.", ".norm.") - new_key = new_key.replace(".linear.weight", ".weight") - remapped[new_key] = value - if new_key.endswith(".attention.attn.per_dim_scale"): - remapped[new_key.replace(".per_dim_scale", ".per_dim_key_scale")] = value - if "lm_head.weight" not in remapped: - tied_embedding = remapped.get("model.language_model.embed_tokens.weight") - if tied_embedding is not None: - remapped["lm_head.weight"] = tied_embedding - return remapped - - -def _repair_gemma4_checkpoint_weights(model: torch.nn.Module, model_source: str) -> dict[str, object]: - raw_state_dict = _load_local_torch_state_dict(model_source) - if raw_state_dict is None: - return {"applied": False, "reason": "no local checkpoint state_dict"} - - remapped_state_dict = _remap_gemma4_checkpoint_state_dict(raw_state_dict) - load_result = model.load_state_dict(remapped_state_dict, strict=False) - return { - "applied": True, - "missing_keys": list(load_result.missing_keys), - "unexpected_keys": list(load_result.unexpected_keys), - } - - -def _load_config_json(model_id_or_path: str) -> dict[str, object]: - local_snapshot = _resolve_local_snapshot(model_id_or_path) - config_source = Path(local_snapshot) / "config.json" if local_snapshot else None - if config_source is not None and config_source.exists(): - return json.loads(config_source.read_text()) - explicit = Path(model_id_or_path) / "config.json" - if explicit.exists(): - return json.loads(explicit.read_text()) - return {} - - -def _load_optional_json(model_id_or_path: str, filename: str) -> dict[str, object]: - local_snapshot = _resolve_local_snapshot(model_id_or_path) - candidate = Path(local_snapshot) / filename if local_snapshot else None - if candidate is not None and candidate.exists(): - return json.loads(candidate.read_text()) - explicit = Path(model_id_or_path) / filename - if explicit.exists(): - return json.loads(explicit.read_text()) - return {} - - -def _load_gemma4_tokenizer_fallback(source_candidates: list[str]) -> object | None: - try: - from transformers import PreTrainedTokenizerFast # type: ignore - except Exception: - return None - - for source in source_candidates: - root = Path(source) - if not root.exists() or not root.is_dir(): - continue - tokenizer_json = root / "tokenizer.json" - tokenizer_config_path = root / "tokenizer_config.json" - if not tokenizer_json.exists() or not tokenizer_config_path.exists(): - continue - - tokenizer_config = json.loads(tokenizer_config_path.read_text()) - tokenizer = PreTrainedTokenizerFast( - tokenizer_file=str(tokenizer_json), - bos_token=tokenizer_config.get("bos_token"), - eos_token=tokenizer_config.get("eos_token"), - unk_token=tokenizer_config.get("unk_token"), - pad_token=tokenizer_config.get("pad_token"), - mask_token=tokenizer_config.get("mask_token"), - padding_side=str(tokenizer_config.get("padding_side", "right")), - additional_special_tokens=list(tokenizer_config.get("extra_special_tokens", []) or []), - ) - model_max_length = tokenizer_config.get("model_max_length") - if isinstance(model_max_length, int): - tokenizer.model_max_length = model_max_length - - chat_template_path = root / "chat_template.jinja" - if chat_template_path.exists(): - tokenizer.chat_template = chat_template_path.read_text() - - for token_attr in ( - "image_token", - "audio_token", - "boi_token", - "eoi_token", - "boa_token", - "eoa_token", - ): - token_value = tokenizer_config.get(token_attr) - if isinstance(token_value, str): - setattr(tokenizer, token_attr, token_value) - setattr(tokenizer, f"{token_attr}_id", tokenizer.convert_tokens_to_ids(token_value)) - return tokenizer - - return None - - -def _load_gemma4_processor_fallback( - *, - source_candidates: list[str], - common_kwargs: dict[str, object], -) -> object | None: - try: - from transformers.models.gemma4.feature_extraction_gemma4 import Gemma4AudioFeatureExtractor # type: ignore - from transformers.models.gemma4.image_processing_gemma4 import Gemma4ImageProcessor # type: ignore - from transformers.models.gemma4.processing_gemma4 import Gemma4Processor # type: ignore - except Exception: - return None - - tokenizer = None - processor_config: dict[str, object] = {} - for source in source_candidates: - try: - try: - from transformers import AutoTokenizer # type: ignore - - tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) - except Exception: - tokenizer = _load_gemma4_tokenizer_fallback([source]) - processor_config = _load_optional_json(source, "processor_config.json") - if tokenizer is not None and processor_config: - break - except Exception: - continue - if tokenizer is None or not processor_config: - return None - - feature_config = dict(processor_config.get("feature_extractor", {}) or {}) - image_config = dict(processor_config.get("image_processor", {}) or {}) - feature_config.pop("feature_extractor_type", None) - image_config.pop("image_processor_type", None) - - feature_extractor = Gemma4AudioFeatureExtractor(**feature_config) - image_processor = Gemma4ImageProcessor(**image_config) - - processor_kwargs = {} - for key in ("image_seq_length", "audio_seq_length", "audio_ms_per_token"): - if key in processor_config: - processor_kwargs[key] = processor_config[key] - - return Gemma4Processor( - feature_extractor=feature_extractor, - image_processor=image_processor, - tokenizer=tokenizer, - **processor_kwargs, - ) - - -def _infer_task_from_config(model_id_or_path: str) -> str: - config = _load_config_json(model_id_or_path) - architectures = [str(value) for value in config.get("architectures", []) if isinstance(value, str)] - model_type = str(config.get("model_type", "") or "").lower() - - if any("CausalLM" in value for value in architectures): - return "causal_lm_logits" - if any("CTC" in value for value in architectures): - return "ctc_logits" - if model_type == "whisper": - return "encoder_hidden_states" - if any("ConditionalGeneration" in value and "Whisper" in value for value in architectures): - return "encoder_hidden_states" - - lowered_id = model_id_or_path.lower() - if "whisper" in lowered_id: - return "encoder_hidden_states" - if "ctc" in lowered_id: - return "ctc_logits" - if any(token in lowered_id for token in ("qwen", "gemma", "llama", "mistral")): - return "causal_lm_logits" - - raise RuntimeError( - f"Could not infer transpile task for {model_id_or_path}.\n" - "\n" - "Pass one explicitly with --task, for example:\n" - " --task causal_lm_logits\n" - " --task ctc_logits\n" - " --task encoder_hidden_states\n" - ) - - -def _normalize_audio_samples(samples: np.ndarray) -> np.ndarray: - if samples.dtype == np.uint8: - return ((samples.astype(np.float32) - 128.0) / 128.0).astype(np.float32) - if np.issubdtype(samples.dtype, np.integer): - info = np.iinfo(samples.dtype) - scale = float(max(abs(info.min), info.max)) - if scale <= 0: - scale = 1.0 - return (samples.astype(np.float32) / scale).astype(np.float32) - return samples.astype(np.float32) - - -def _hz_to_mel(hz: np.ndarray) -> np.ndarray: - return 2595.0 * np.log10(1.0 + hz / 700.0) - - -def _mel_to_hz(mel: np.ndarray) -> np.ndarray: - return 700.0 * (np.power(10.0, mel / 2595.0) - 1.0) - - -@lru_cache(maxsize=16) -def _mel_filter_bank(num_mels: int, sample_rate: int, n_fft: int) -> np.ndarray: - try: - from transformers.audio_utils import mel_filter_bank as transformers_mel_filter_bank # type: ignore - - return transformers_mel_filter_bank( - num_frequency_bins=n_fft // 2 + 1, - num_mel_filters=num_mels, - min_frequency=0.0, - max_frequency=sample_rate / 2.0, - sampling_rate=sample_rate, - norm="slaney", - mel_scale="slaney", - ).astype(np.float32) - except Exception: - pass - - num_freq_bins = n_fft // 2 + 1 - mel_points = np.linspace( - _hz_to_mel(np.array([0.0], dtype=np.float32))[0], - _hz_to_mel(np.array([sample_rate / 2.0], dtype=np.float32))[0], - num_mels + 2, - ) - hz_points = _mel_to_hz(mel_points) - bins = np.floor((n_fft + 1) * hz_points / sample_rate).astype(np.int64) - - filters = np.zeros((num_mels, num_freq_bins), dtype=np.float32) - for idx in range(1, num_mels + 1): - left = max(0, min(num_freq_bins - 1, int(bins[idx - 1]))) - center = max(left + 1, min(num_freq_bins - 1, int(bins[idx]))) - right = max(center + 1, min(num_freq_bins, int(bins[idx + 1]))) - for freq in range(left, center): - filters[idx - 1, freq] = float(freq - left) / float(max(center - left, 1)) - for freq in range(center, right): - filters[idx - 1, freq] = float(right - freq) / float(max(right - center, 1)) - return filters - - -def _load_audio_waveform(audio_file: str | Path, *, target_sample_rate: int) -> np.ndarray: - sample_rate, waveform = wavfile.read(str(audio_file)) - if waveform.ndim > 1: - waveform = waveform.mean(axis=1) - waveform = _normalize_audio_samples(np.asarray(waveform)) - - if int(sample_rate) != int(target_sample_rate): - gcd = int(np.gcd(int(sample_rate), int(target_sample_rate))) - waveform = resample_poly( - waveform, - int(target_sample_rate) // gcd, - int(sample_rate) // gcd, - ).astype(np.float32) - - return waveform.astype(np.float32) - - -def _generic_log_mel_features( - waveform: np.ndarray, - *, - sample_rate: int, - num_mels: int, - n_fft: int, - hop_length: int, - frame_length: int, - preemphasis: float | None = None, -) -> tuple[np.ndarray, int]: - waveform_tensor = torch.from_numpy(waveform).to(torch.float32).unsqueeze(0) - if preemphasis is not None and abs(preemphasis) > 0.0: - waveform_tensor = torch.cat( - [ - waveform_tensor[:, :1], - waveform_tensor[:, 1:] - float(preemphasis) * waveform_tensor[:, :-1], - ], - dim=1, - ) - - window = torch.hann_window(frame_length, periodic=False, dtype=torch.float32) - stft = torch.stft( - waveform_tensor, - n_fft=n_fft, - hop_length=hop_length, - win_length=frame_length, - window=window, - return_complex=True, - pad_mode="constant", - ) - magnitudes = torch.view_as_real(stft) - magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1)) - power = magnitudes.pow(2) - - mel_filters = torch.from_numpy(_mel_filter_bank(num_mels, sample_rate, n_fft)).to(torch.float32) - mel_spec = power.transpose(1, 2) @ mel_filters - mel_spec = torch.log(mel_spec + np.float32(2**-24)) - - feature_lengths = max(1, int(waveform.shape[0] // hop_length)) - attention_mask = ( - torch.arange(mel_spec.shape[1], dtype=torch.long, device=mel_spec.device).unsqueeze(0) < feature_lengths - ) - mask = attention_mask.unsqueeze(-1).to(torch.float32) - masked = mel_spec * mask - mean = masked.sum(dim=1) / float(feature_lengths) - centered = masked - mean.unsqueeze(1) - denom = float(max(1, feature_lengths - 1)) - variance = (centered.pow(2) * mask).sum(dim=1) / denom - mel_spec = (mel_spec - mean.unsqueeze(1)) / (torch.sqrt(variance).unsqueeze(1) + np.float32(1e-5)) - mel_spec = mel_spec * mask - return mel_spec[0].cpu().numpy().astype(np.float32), feature_lengths - - -def _resolve_audio_sample_rate(processor: object) -> int: - for attr_name in ("feature_extractor", "tokenizer"): - child = getattr(processor, attr_name, None) - sample_rate = getattr(child, "sampling_rate", None) - if isinstance(sample_rate, int) and sample_rate > 0: - return sample_rate - sample_rate = getattr(processor, "sampling_rate", None) - if isinstance(sample_rate, int) and sample_rate > 0: - return sample_rate - return 16000 - - -def _infer_fallback_audio_input_names(config: dict[str, object], task: str) -> tuple[str, ...]: - model_type = str(config.get("model_type", "") or "").lower() - if model_type == "whisper": - return ("input_features",) - if task == "encoder_hidden_states": - return ("input_features",) - audio_cfg = config.get("audio_config") - if isinstance(audio_cfg, dict) and any(key in audio_cfg for key in ("features", "input_feat_size", "num_mel_bins")): - return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) - encoder_cfg = config.get("encoder") - if isinstance(encoder_cfg, dict) and any(key in encoder_cfg for key in ("feat_in", "num_mel_bins")): - return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) - return ("input_values", "attention_mask") if task == "ctc_logits" else ("input_values",) - - -def _resolve_encoder_module(model: torch.nn.Module) -> torch.nn.Module | None: - get_encoder = getattr(model, "get_encoder", None) - if callable(get_encoder): - encoder = get_encoder() - if isinstance(encoder, torch.nn.Module): - return encoder - encoder = getattr(model, "encoder", None) - if isinstance(encoder, torch.nn.Module): - return encoder - model_attr = getattr(model, "model", None) - if model_attr is not None: - encoder = getattr(model_attr, "encoder", None) - if isinstance(encoder, torch.nn.Module): - return encoder - return None - - -def _infer_expected_input_feature_frames(model: torch.nn.Module) -> int | None: - config = getattr(model, "config", None) - max_source_positions = getattr(config, "max_source_positions", None) - if not isinstance(max_source_positions, int) or max_source_positions <= 0: - return None - - encoder = _resolve_encoder_module(model) - if encoder is None: - return None - - stride_product = 1 - found_conv = False - for child in encoder.children(): - if isinstance(child, torch.nn.Conv1d): - stride = child.stride[0] if isinstance(child.stride, tuple) else child.stride - stride_product *= int(stride) - found_conv = True - - if not found_conv: - return None - return int(max_source_positions) * stride_product - - -def _prepare_fallback_audio_inputs( - *, - input_names: tuple[str, ...], - config: dict[str, object], - preprocessor_config: dict[str, object], - model: torch.nn.Module, - task: str, - audio_file: str, - torch_dtype: torch.dtype, -) -> PreparedInputs: - if not input_names: - input_names = _infer_fallback_audio_input_names(config, task) - sample_rate = int(preprocessor_config.get("sampling_rate", config.get("sampling_rate", 16000)) or 16000) - num_mels = int(preprocessor_config.get("feature_size", config.get("num_mel_bins", config.get("feature_size", 80))) or 80) - encoder_cfg = config.get("encoder_config") - if isinstance(encoder_cfg, dict): - num_mels = int(encoder_cfg.get("num_mel_bins", encoder_cfg.get("feat_in", num_mels)) or num_mels) - hop_length = int(preprocessor_config.get("hop_length", config.get("hop_length", 160)) or 160) - n_fft = int(preprocessor_config.get("n_fft", config.get("n_fft", 400)) or 400) - frame_length = int(preprocessor_config.get("win_length", preprocessor_config.get("frame_length", config.get("frame_length", n_fft))) or n_fft) - preemphasis = preprocessor_config.get("preemphasis") - if preemphasis is not None: - preemphasis = float(preemphasis) - waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) - - tensors: list[torch.Tensor] = [] - if input_names and input_names[0] == "input_features": - features, feature_length = _generic_log_mel_features( - waveform, - sample_rate=sample_rate, - num_mels=num_mels, - n_fft=n_fft, - hop_length=hop_length, - frame_length=frame_length, - preemphasis=preemphasis, - ) - expected_frames = _infer_expected_input_feature_frames(model) - attention_mask: np.ndarray | None = None - target_frames = feature_length - if isinstance(expected_frames, int) and expected_frames > 0: - target_frames = expected_frames - - active_frames = min(feature_length, target_frames) - features = features[:active_frames, :] - if target_frames > active_frames: - pad_width = target_frames - active_frames - features = np.pad(features, ((0, pad_width), (0, 0)), mode="constant") - attention_mask = np.zeros((target_frames,), dtype=np.bool_) - attention_mask[:active_frames] = True - - tensors.append(torch.from_numpy(features).unsqueeze(0).to(dtype=torch_dtype)) - if attention_mask is not None and len(input_names) > 1 and input_names[1] == "attention_mask": - tensors.append(torch.from_numpy(attention_mask).unsqueeze(0)) - else: - input_values = torch.from_numpy(waveform).unsqueeze(0).to(dtype=torch_dtype) - tensors.append(input_values) - if len(input_names) > 1 and input_names[1] == "attention_mask": - tensors.append(torch.ones_like(input_values, dtype=torch.float32)) - - return PreparedInputs( - names=input_names[: len(tensors)], - tensors=tuple(tensors), - metadata={ - "audio_file": str(Path(audio_file).resolve()), - "sample_rate": sample_rate, - "fallback_audio_preprocessor": True, - "input_shapes": { - name: list(tensor.shape) - for name, tensor in zip(input_names, tensors) - }, - }, - ) - - -def _prepare_audio_inputs( - processor: object | None, - *, - input_names: tuple[str, ...], - config: dict[str, object], - preprocessor_config: dict[str, object], - model: torch.nn.Module, - task: str, - audio_file: str, - torch_dtype: torch.dtype, -) -> PreparedInputs: - if processor is None: - return _prepare_fallback_audio_inputs( - input_names=input_names, - config=config, - preprocessor_config=preprocessor_config, - model=model, - task=task, - audio_file=audio_file, - torch_dtype=torch_dtype, - ) - - sample_rate = _resolve_audio_sample_rate(processor) - waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) - batch = processor( - waveform, - sampling_rate=sample_rate, - return_tensors="pt", - ) - - preferred_keys = tuple(input_names) + tuple( - key for key in ("input_features", "input_values", "attention_mask") if key not in input_names - ) - tensor_keys = [key for key, value in batch.items() if isinstance(value, torch.Tensor)] - ordered_keys = [key for key in preferred_keys if key in tensor_keys] - ordered_keys.extend(key for key in tensor_keys if key not in ordered_keys) - if not ordered_keys: - raise RuntimeError(f"processor did not return tensor inputs for audio file: {audio_file}") - - tensors: list[torch.Tensor] = [] - for key in ordered_keys: - value = batch[key] - if not isinstance(value, torch.Tensor): - continue - if torch.is_floating_point(value): - value = value.to(dtype=torch_dtype) - tensors.append(value) - - return PreparedInputs( - names=tuple(ordered_keys[: len(tensors)]), - tensors=tuple(tensors), - metadata={ - "audio_file": str(Path(audio_file).resolve()), - "sample_rate": sample_rate, - "fallback_audio_preprocessor": False, - "input_shapes": { - name: list(tensor.shape) - for name, tensor in zip(ordered_keys, tensors) - }, - }, - ) - - -def _prepare_text_inputs(tokenizer: object, *, prompt: str, input_ids_text: str | None) -> PreparedInputs: - if input_ids_text: - token_ids = [int(part.strip()) for part in input_ids_text.split(",") if part.strip()] - if not token_ids: - raise ValueError("--input-ids was provided but no ids were parsed") - input_ids = torch.tensor([token_ids], dtype=torch.long) - else: - input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"] - return PreparedInputs( - names=("input_ids",), - tensors=(input_ids,), - metadata={ - "prompt": prompt, - "input_ids": input_ids.tolist(), - }, - ) - - -_GEMMA4_MULTIMODAL_INPUT_ORDER = ( - "input_ids", - "attention_mask", - "token_type_ids", - "pixel_values", - "pixel_position_ids", - "input_features", - "input_features_mask", -) - - -def _normalize_multimodal_prompt( - prompt: str, - *, - image_token: str | None, - num_images: int, - audio_token: str | None, - num_audio_segments: int, -) -> str: - normalized = prompt.strip() - prefixes: list[str] = [] - - if image_token and num_images > 0: - image_count = normalized.count(image_token) - if image_count < num_images: - prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) - if audio_token and num_audio_segments > 0: - audio_count = normalized.count(audio_token) - if audio_count < num_audio_segments: - prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) - - if prefixes: - prefix = "\n".join(part for part in prefixes if part) - if normalized: - return f"{prefix}\n{normalized}" - return prefix - return normalized - - -def _build_gemma4_chat_prompt( - *, - prompt: str, - image_token: str | None, - num_images: int, - audio_token: str | None, - num_audio_segments: int, - system_prompt: str = "", - enable_thinking_if_supported: bool = False, -) -> str: - result = "" - normalized_system = system_prompt.strip() - if enable_thinking_if_supported or normalized_system: - result += "<|turn>system\n" - if enable_thinking_if_supported: - result += "<|think|>" - result += normalized_system - result += "\n" - - result += "<|turn>user\n" - if image_token and num_images > 0: - for _ in range(num_images): - result += f"\n\n{image_token}\n\n" - result += prompt - if audio_token and num_audio_segments > 0: - result += "".join(audio_token for _ in range(num_audio_segments)) - result += "\n" - result += "<|turn>model\n" - return result - - -def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: - if not image_files: - return [] - try: - from PIL import Image # type: ignore - except Exception as exc: # pragma: no cover - raise RuntimeError(f"Pillow is required for --image-file: {exc}") from exc - - images: list[object] = [] - for image_file in image_files: - path = Path(image_file).resolve() - if not path.exists(): - raise RuntimeError(f"image_file does not exist: {path}") - with Image.open(path) as image: - images.append(image.convert("RGB").copy()) - return images - - -def _prepare_gemma4_multimodal_inputs( - processor: object | None, - *, - prompt: str, - image_files: tuple[str, ...], - audio_file: str | None, - torch_dtype: torch.dtype, - system_prompt: str = "", - enable_thinking_if_supported: bool = False, - use_gemma4_chat_template: bool = False, -) -> PreparedInputs: - if processor is None: - raise RuntimeError("multimodal Gemma4 transpile requires an AutoProcessor with image and audio support") - - images = _load_image_inputs(image_files) - audio_waveforms: list[np.ndarray] = [] - sample_rate: int | None = None - if audio_file: - sample_rate = _resolve_audio_sample_rate(processor) - audio_waveforms.append(_load_audio_waveform(audio_file, target_sample_rate=sample_rate)) - - image_token = getattr(processor, "image_token", None) - audio_token = getattr(processor, "audio_token", None) - processor_prompt = _normalize_multimodal_prompt( - prompt, - image_token=image_token if isinstance(image_token, str) else None, - num_images=len(images), - audio_token=audio_token if isinstance(audio_token, str) else None, - num_audio_segments=len(audio_waveforms), - ) - normalized_prompt = processor_prompt - if use_gemma4_chat_template: - normalized_prompt = prompt.strip() - processor_prompt = _build_gemma4_chat_prompt( - prompt=normalized_prompt, - image_token=image_token if isinstance(image_token, str) else None, - num_images=len(images), - audio_token=audio_token if isinstance(audio_token, str) else None, - num_audio_segments=len(audio_waveforms), - system_prompt=system_prompt, - enable_thinking_if_supported=enable_thinking_if_supported, - ) - - batch = processor( - text=processor_prompt, - images=images or None, - audio=audio_waveforms or None, - return_tensors="pt", - ) - - ordered_keys = [ - key - for key in _GEMMA4_MULTIMODAL_INPUT_ORDER - if isinstance(batch.get(key), torch.Tensor) - ] - if not ordered_keys: - raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") - - tensors: list[torch.Tensor] = [] - for key in ordered_keys: - value = batch[key] - if not isinstance(value, torch.Tensor): - continue - if torch.is_floating_point(value): - value = value.to(dtype=torch_dtype) - tensors.append(value) - - metadata: dict[str, object] = { - "prompt": normalized_prompt, - "processor_prompt": processor_prompt, - "image_files": [str(Path(path).resolve()) for path in image_files], - "input_shapes": { - name: list(tensor.shape) - for name, tensor in zip(ordered_keys, tensors) - }, - } - if audio_file: - metadata["audio_file"] = str(Path(audio_file).resolve()) - if sample_rate is not None: - metadata["sample_rate"] = sample_rate - - return PreparedInputs( - names=tuple(ordered_keys[: len(tensors)]), - tensors=tuple(tensors), - metadata=metadata, - ) - - -def _load_model_source(model_id: str, *, local_files_only: bool) -> str: - local_snapshot = _resolve_local_snapshot(model_id) - if local_snapshot and _snapshot_has_model_weights(local_snapshot): - return local_snapshot - if local_snapshot and local_files_only: - raise RuntimeError( - f"Found local snapshot for {model_id}, but it is incomplete and has no model weights:\n" - f" {local_snapshot}\n" - "\n" - "Re-run without --local-files-only to let transformers download the missing weights." - ) - return model_id - - -def _load_transformers_bundle( - *, - model_id: str, - task: str, - torch_dtype: torch.dtype, - token: str | None, - trust_remote_code: bool, - local_files_only: bool, -): - config = _load_config_json(model_id) - config_model_type = str(config.get("model_type", "") or "").lower() - external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) - patch_note = _patch_transformers_torchvision_probe() - if patch_note: - print(f"note={patch_note}") - flex_patch_note = _patch_torch_flex_attention_compat() - if flex_patch_note: - print(f"note={flex_patch_note}") - if external_transformers_site_packages: - print(f"note=using external transformers install for {config_model_type}: {external_transformers_site_packages}") - - try: - from transformers import AutoFeatureExtractor # type: ignore - from transformers import AutoModel # type: ignore - from transformers import AutoModelForCTC # type: ignore - from transformers import AutoModelForCausalLM # type: ignore - from transformers import AutoModelForSeq2SeqLM # type: ignore - from transformers import AutoModelForSpeechSeq2Seq # type: ignore - from transformers import AutoProcessor # type: ignore - from transformers import AutoTokenizer # type: ignore - except Exception as exc: # pragma: no cover - raise RuntimeError(f"transformers is not available: {exc}") from exc - - model_source = _load_model_source(model_id, local_files_only=local_files_only) - source_candidates = [] - for candidate in (model_source, model_id): - if candidate not in source_candidates: - source_candidates.append(candidate) - common_kwargs: dict[str, object] = { - "trust_remote_code": trust_remote_code, - "local_files_only": local_files_only, - } - if token: - common_kwargs["token"] = token - - if task == "causal_lm_logits": - tokenizer = None - tokenizer_errors: list[str] = [] - for source in source_candidates: - try: - tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) - break - except Exception as exc: - tokenizer_errors.append(f"{source}: {exc}") - if tokenizer is None: - raise RuntimeError( - f"Could not load tokenizer for {model_id}:\n" - + "\n".join(tokenizer_errors) - ) - model = AutoModelForCausalLM.from_pretrained( - model_source, - dtype=torch_dtype, - device_map=None, - low_cpu_mem_usage=True, - **common_kwargs, - ).eval() - return model_source, tokenizer, model, config - if task == "multimodal_causal_lm_logits": - processor = None - processor_errors: list[str] = [] - for source in source_candidates: - try: - processor = AutoProcessor.from_pretrained(source, **common_kwargs) - break - except Exception as exc: - processor_errors.append(f"{source}: {exc}") - if processor is None and config_model_type == "gemma4": - processor = _load_gemma4_processor_fallback( - source_candidates=source_candidates, - common_kwargs=common_kwargs, - ) - if processor is not None: - print("note=using manual gemma4 processor fallback") - if processor is None: - processor_config_hint = "" - if config_model_type == "gemma4": - processor_config_hint = ( - "\n" - "Gemma4 multimodal transpile needs a processor bundle, not just tokenizer/model weights.\n" - "Your local snapshot may be missing files such as `processor_config.json` or modality-specific\n" - "preprocessor configs. Use an official Gemma4 snapshot that includes the processor, or let\n" - "transformers download one by re-running without `--local-files-only`.\n" - ) - raise RuntimeError( - f"Could not load processor for {model_id}:\n" - + "\n".join(processor_errors) - + processor_config_hint - ) - - model = AutoModelForCausalLM.from_pretrained( - model_source, - dtype=torch_dtype, - device_map=None, - low_cpu_mem_usage=True, - **common_kwargs, - ).eval() - if config_model_type == "gemma4": - repair_result = _repair_gemma4_checkpoint_weights(model, model_source) - if repair_result.get("applied"): - missing = repair_result.get("missing_keys", []) - unexpected = repair_result.get("unexpected_keys", []) - print( - "note=applied gemma4 checkpoint key remap" - f" missing_after={len(missing)} unexpected_after={len(unexpected)}" - ) - return model_source, processor, model, config - - processor = None - processor_errors: list[str] = [] - missing_optional_audio_dep: str | None = None - for source in source_candidates: - for loader in (AutoProcessor, AutoFeatureExtractor): - try: - processor = loader.from_pretrained(source, **common_kwargs) - break - except Exception as exc: - processor_errors.append(f"{loader.__name__}@{source}: {exc}") - if isinstance(exc, ImportError) and "requires the librosa library" in str(exc): - missing_optional_audio_dep = "librosa" - break - if processor is not None: - break - if missing_optional_audio_dep is not None: - break - if processor is None: - if missing_optional_audio_dep == "librosa": - print("note=falling back to built-in audio preprocessing because the HF feature extractor requires librosa") - else: - print("note=falling back to built-in audio preprocessing because no HF processor/feature extractor was available") - - if task == "ctc_logits": - model_loaders = (AutoModelForCTC, AutoModel) - elif task == "encoder_hidden_states": - model_loaders = (AutoModelForSpeechSeq2Seq, AutoModelForSeq2SeqLM, AutoModel) - else: - raise NotImplementedError(f"unsupported generic HF task: {task}") - - load_errors: list[str] = [] - for loader in model_loaders: - try: - model = loader.from_pretrained( - model_source, - dtype=torch_dtype, - device_map=None, - low_cpu_mem_usage=True, - **common_kwargs, - ).eval() - return model_source, processor, model, config - except Exception as exc: - load_errors.append(f"{loader.__name__}: {exc}") - - raise RuntimeError( - f"Could not load model for task={task} from {model_source}.\n" - "\n".join(load_errors) - ) - - -def _load_optional_tokenizer( - *, - model_id: str, - model_source: str, - token: str | None, - trust_remote_code: bool, - local_files_only: bool, -): - try: - from transformers import AutoTokenizer # type: ignore - except Exception: - return None - - source_candidates = [] - for candidate in (model_source, model_id): - if candidate not in source_candidates: - source_candidates.append(candidate) - - common_kwargs: dict[str, object] = { - "trust_remote_code": trust_remote_code, - "local_files_only": local_files_only, - } - if token: - common_kwargs["token"] = token - - for source in source_candidates: - try: - return AutoTokenizer.from_pretrained(source, **common_kwargs) - except Exception: - continue - return None - - -def _ctc_greedy_decode_token_ids(logits: np.ndarray, *, blank_token_id: int | None) -> list[int]: - if logits.ndim != 3 or logits.shape[0] < 1: - raise ValueError(f"expected CTC logits with shape [batch, time, vocab], got {list(logits.shape)}") - - raw_ids = np.argmax(logits[0], axis=-1).tolist() - collapsed: list[int] = [] - previous: int | None = None - for token_id in raw_ids: - if token_id != previous: - collapsed.append(int(token_id)) - previous = int(token_id) - - if blank_token_id is None: - return collapsed - return [token_id for token_id in collapsed if int(token_id) != int(blank_token_id)] - - -def _decode_token_ids(tokenizer: object, token_ids: list[int]) -> str: - decode = getattr(tokenizer, "decode", None) - if not callable(decode): - raise TypeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") - try: - return str(decode(token_ids, skip_special_tokens=True)) - except TypeError: - return str(decode(token_ids)) - - -def _count_weight_bindings(ir_graph: IRGraph) -> int: - count = 0 - for value in ir_graph.values.values(): - if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str): - count += 1 - return count - - -def _lower_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: - verify_ir(ir) - graph = Graph() - env: dict[str, Any] = {} - runtime_inputs = [] - bound_constants = [] - - for value_id in ir.inputs: - value = ir.values[value_id] - tensor = _lower_input_value(graph, value) - env[value_id] = tensor - runtime_inputs.append(tensor) - - for value_id, const in ir.constants.items(): - value = ir.values[value_id] - lowered_const = _lower_constant_value(graph, value, const) - env[value_id] = lowered_const - if hasattr(lowered_const, "g") and hasattr(lowered_const, "id"): - bound_constants.append(lowered_const) - - for node_id in ir.order: - node = ir.nodes[node_id] - outputs = _lower_ir_node(graph, node, env, ir) - if len(outputs) != len(node.outputs): - raise ValueError( - f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" - ) - for output_id, tensor in zip(node.outputs, outputs): - env[output_id] = tensor - - outputs = [env[value_id] for value_id in ir.outputs] - return TranspiledGraph( - graph=graph, - runtime_inputs=runtime_inputs, - bound_constants=bound_constants, - outputs=outputs, - ) - - -def main() -> int: - parser = argparse.ArgumentParser( - description=( - "Load a Hugging Face model, canonicalize it into a generic transpile task, " - "capture it with the Cactus transpiler, lower it to a Cactus Graph, and " - "optionally save artifacts or run the lowered graph." - ) - ) - parser.add_argument( - "--model-id", - required=True, - help="Hugging Face model id or local snapshot path.", - ) - parser.add_argument( - "--task", - default="auto", - choices=("auto", "causal_lm_logits", "multimodal_causal_lm_logits", "ctc_logits", "encoder_hidden_states"), - help="Transpile task. Use auto to infer from config/model id.", - ) - parser.add_argument( - "--prompt", - default="The capital of France is", - help="Prompt used for causal_lm_logits or multimodal_causal_lm_logits when --input-ids is not set.", - ) - parser.add_argument( - "--input-ids", - default="", - help="Optional comma-separated token ids for causal_lm_logits.", - ) - parser.add_argument( - "--audio-file", - default="", - help="Path to a WAV file for audio or multimodal tasks.", - ) - parser.add_argument( - "--image-file", - action="append", - default=[], - help="Path to an image file for multimodal tasks. Repeat to pass multiple images.", - ) - parser.add_argument( - "--torch-dtype", - default="float16", - help="Torch dtype for model loading: float16, float32, or bfloat16.", - ) - parser.add_argument( - "--token", - default=os.environ.get("HF_TOKEN"), - help="Optional Hugging Face token. Defaults to HF_TOKEN.", - ) - parser.add_argument( - "--trust-remote-code", - action="store_true", - help="Pass trust_remote_code=True to transformers loaders.", - ) - parser.add_argument( - "--local-files-only", - action="store_true", - help="Require the model/processor to already exist locally.", - ) - parser.add_argument( - "--weights-dir", - default="", - help="Optional converted Cactus weights directory for mmap weight binding.", - ) - parser.add_argument( - "--artifact-dir", - default="", - help="Optional directory where raw_ir.json, optimized_ir.json, graph.cactus, and result.json are saved.", - ) - parser.add_argument( - "--graph-filename", - default="graph.cactus", - help="Filename to use for Graph.save() inside --artifact-dir.", - ) - parser.add_argument("--no-fuse-gated-deltanet", action="store_true") - parser.add_argument("--no-fuse-rms-norm", action="store_true") - parser.add_argument("--no-fuse-rope", action="store_true") - parser.add_argument("--no-fuse-attention", action="store_true") - parser.add_argument("--no-fuse-attention-block", action="store_true") - parser.add_argument("--no-fuse-add-clipped", action="store_true") - parser.add_argument( - "--skip-execute", - action="store_true", - help="Stop after lowering instead of running the transpiled graph.", - ) - parser.add_argument( - "--skip-reference-compare", - action="store_true", - help="Run the transpiled graph but skip the follow-up PyTorch reference pass.", - ) - args = parser.parse_args() - - image_files = tuple(str(path) for path in args.image_file if str(path).strip()) - if args.task == "auto": - inferred_task = _infer_task_from_config(args.model_id) - config_for_auto = _load_config_json(args.model_id) - model_type_for_auto = str(config_for_auto.get("model_type", "") or "").lower() - has_multimodal_config = ( - model_type_for_auto == "gemma4" - and ( - isinstance(config_for_auto.get("vision_config"), dict) - or isinstance(config_for_auto.get("audio_config"), dict) - ) - ) - if image_files or (args.audio_file and has_multimodal_config and inferred_task == "causal_lm_logits"): - task = "multimodal_causal_lm_logits" - else: - task = inferred_task - else: - task = args.task - torch_dtype = _parse_dtype(args.torch_dtype) - validated_weights_dir = _validate_weights_dir(args.weights_dir.strip() or None, model_id=args.model_id) - weights_dir = str(validated_weights_dir) if validated_weights_dir is not None else None - artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None - - model_source, processor_or_tokenizer, model, model_config = _load_transformers_bundle( - model_id=args.model_id, - task=task, - torch_dtype=torch_dtype, - token=args.token, - trust_remote_code=args.trust_remote_code, - local_files_only=args.local_files_only, - ) - preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") - if not preprocessor_config: - preprocessor_config = _load_optional_json(args.model_id, "preprocessor_config.json") - auxiliary_tokenizer = None - if task == "ctc_logits": - auxiliary_tokenizer = _load_optional_tokenizer( - model_id=args.model_id, - model_source=model_source, - token=args.token, - trust_remote_code=args.trust_remote_code, - local_files_only=args.local_files_only, - ) - - if task == "causal_lm_logits": - prepared = _prepare_text_inputs( - processor_or_tokenizer, - prompt=args.prompt, - input_ids_text=args.input_ids.strip() or None, - ) - canonical = canonicalize_model_interface( - model, - task=task, - input_names=prepared.names, - ) - elif task == "multimodal_causal_lm_logits": - prepared = _prepare_gemma4_multimodal_inputs( - processor_or_tokenizer, - prompt=args.prompt, - image_files=image_files, - audio_file=args.audio_file.strip() or None, - torch_dtype=torch_dtype, - ) - canonical = canonicalize_model_interface( - model, - task=task, - input_names=prepared.names, - ) - else: - if not args.audio_file: - raise RuntimeError(f"--audio-file is required for task={task}") - canonical = canonicalize_model_interface(model, task=task) - prepared = _prepare_audio_inputs( - processor_or_tokenizer, - input_names=canonical.input_names, - config=model_config, - preprocessor_config=preprocessor_config, - model=model, - task=task, - audio_file=args.audio_file, - torch_dtype=torch_dtype, - ) - canonical = canonicalize_model_interface( - model, - task=task, - input_names=prepared.names, - ) - wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() - - print(f"model_id={args.model_id}") - print(f"model_source={model_source}") - print(f"task={task}") - print(f"adapter_family={canonical.family}") - print(f"adapter_module={type(canonical.module).__name__}") - print(f"input_names={','.join(prepared.names)}") - for name, tensor in zip(prepared.names, prepared.tensors): - print(f"input_{name}_shape={list(tensor.shape)}") - if weights_dir: - print(f"weights_dir={weights_dir}") - - captured = capture_model(wrapper, prepared.tensors) - raw_ir_graph = copy.deepcopy(captured.ir_graph) - - fusion_config = FusionConfig( - enable_gated_deltanet=not args.no_fuse_gated_deltanet, - enable_rms_norm=not args.no_fuse_rms_norm, - enable_rope=not args.no_fuse_rope, - enable_attention=not args.no_fuse_attention, - enable_attention_block=not args.no_fuse_attention_block, - enable_add_clipped=not args.no_fuse_add_clipped, - ) - - canonicalize_exported_graph(captured.ir_graph) - optimize_graph(captured.ir_graph, config=fusion_config) - tg = _lower_preoptimized_ir(captured.ir_graph) - - optimized_ir_graph = copy.deepcopy(captured.ir_graph) - binding_count = _count_weight_bindings(optimized_ir_graph) - op_counts = Counter(optimized_ir_graph.nodes[node_id].op for node_id in optimized_ir_graph.order) - - print(f"raw_ir_nodes={len(raw_ir_graph.order)}") - print(f"optimized_ir_nodes={len(optimized_ir_graph.order)}") - print(f"weight_bindings={binding_count}") - print( - "ops=" - f"attention:{op_counts.get('attention', 0)} " - f"conv1d:{op_counts.get('conv1d', 0)} " - f"conv2d:{op_counts.get('conv2d', 0)} " - f"batch_norm:{op_counts.get('batch_norm', 0)} " - f"layer_norm:{op_counts.get('layer_norm', 0)} " - f"rms_norm:{op_counts.get('rms_norm', 0)} " - f"rope:{op_counts.get('rope', 0)} " - f"linear:{op_counts.get('linear', 0)}" - ) - - if weights_dir and binding_count == 0: - raise RuntimeError( - f"No weight bindings were resolved from {weights_dir}\n" - "\n" - "The weights folder exists, but none of the captured constants matched entries in weights_manifest.json.\n" - "\n" - f"Recommended fix:\n" - f" cactus convert {args.model_id} {weights_dir}\n" - ) - - if artifact_dir is not None: - artifact_dir.mkdir(parents=True, exist_ok=True) - _write_json( - artifact_dir / "raw_ir.json", - { - "model_id": args.model_id, - "model_source": model_source, - "task": task, - "family": canonical.family, - "inputs": _serialize_json_compatible(prepared.metadata), - "graph": _graph_to_dict(raw_ir_graph), - }, - ) - _write_json( - artifact_dir / "optimized_ir.json", - { - "model_id": args.model_id, - "model_source": model_source, - "task": task, - "family": canonical.family, - "inputs": _serialize_json_compatible(prepared.metadata), - "graph": _graph_to_dict(optimized_ir_graph), - }, - ) - graph_path = artifact_dir / args.graph_filename - tg.graph.save(graph_path) - print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") - print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") - print(f"saved_graph={graph_path}") - - if args.skip_execute: - return 0 - - if args.skip_reference_compare: - del model - del wrapper - gc.collect() - - tg.set_inputs([tensor.cpu().numpy() for tensor in prepared.tensors]) - print("execute_begin=true") - transpiled_output = tg.execute()[0].numpy().astype(np.float32) - print("execute_done=true") - - if args.skip_reference_compare: - result_payload = { - "model_id": args.model_id, - "model_source": model_source, - "task": task, - "family": canonical.family, - "inputs": _serialize_json_compatible(prepared.metadata), - "output_shape": list(transpiled_output.shape), - "raw_ir_nodes": len(raw_ir_graph.order), - "optimized_ir_nodes": len(optimized_ir_graph.order), - "weight_bindings": binding_count, - "reference_compare_skipped": True, - } - print(f"output_shape={list(transpiled_output.shape)}") - if task in {"causal_lm_logits", "multimodal_causal_lm_logits"}: - tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) - transpiled_next = int(np.argmax(transpiled_output[0, -1])) - print(f"transpiled_next_token_id={transpiled_next}") - result_payload["transpiled_next_token_id"] = transpiled_next - if hasattr(tokenizer_like, "decode"): - transpiled_token = tokenizer_like.decode([transpiled_next]) - print(f"transpiled_next_token={transpiled_token!r}") - result_payload["transpiled_next_token"] = transpiled_token - if artifact_dir is not None: - _write_json(artifact_dir / "result.json", result_payload) - print(f"saved_result={artifact_dir / 'result.json'}") - return 0 - - print("reference_begin=true") - with torch.no_grad(): - reference_output = wrapper(*prepared.tensors).detach().float().cpu().numpy() - print("reference_done=true") - - max_abs_diff = float(np.max(np.abs(reference_output - transpiled_output))) - mean_abs_diff = float(np.mean(np.abs(reference_output - transpiled_output))) - result_payload: dict[str, object] = { - "model_id": args.model_id, - "model_source": model_source, - "task": task, - "family": canonical.family, - "inputs": _serialize_json_compatible(prepared.metadata), - "output_shape": list(reference_output.shape), - "raw_ir_nodes": len(raw_ir_graph.order), - "optimized_ir_nodes": len(optimized_ir_graph.order), - "weight_bindings": binding_count, - "max_abs_diff": max_abs_diff, - "mean_abs_diff": mean_abs_diff, - } - - if task == "causal_lm_logits": - tokenizer = processor_or_tokenizer - hf_next = int(np.argmax(reference_output[0, -1])) - transpiled_next = int(np.argmax(transpiled_output[0, -1])) - print(f"hf_next_token_id={hf_next}") - print(f"transpiled_next_token_id={transpiled_next}") - print(f"logits_max_abs_diff={max_abs_diff:.6f}") - print(f"logits_mean_abs_diff={mean_abs_diff:.6f}") - print(f"hf_next_token={tokenizer.decode([hf_next])!r}") - print(f"transpiled_next_token={tokenizer.decode([transpiled_next])!r}") - result_payload.update( - { - "hf_next_token_id": hf_next, - "transpiled_next_token_id": transpiled_next, - "hf_next_token": tokenizer.decode([hf_next]), - "transpiled_next_token": tokenizer.decode([transpiled_next]), - } - ) - elif task == "ctc_logits": - blank_token_id = getattr(model.config, "pad_token_id", None) - if blank_token_id is None and auxiliary_tokenizer is not None: - blank_token_id = getattr(auxiliary_tokenizer, "pad_token_id", None) - print(f"output_shape={list(reference_output.shape)}") - print(f"output_max_abs_diff={max_abs_diff:.6f}") - print(f"output_mean_abs_diff={mean_abs_diff:.6f}") - if auxiliary_tokenizer is not None: - hf_token_ids = _ctc_greedy_decode_token_ids(reference_output, blank_token_id=blank_token_id) - transpiled_token_ids = _ctc_greedy_decode_token_ids(transpiled_output, blank_token_id=blank_token_id) - hf_transcript = _decode_token_ids(auxiliary_tokenizer, hf_token_ids) - transpiled_transcript = _decode_token_ids(auxiliary_tokenizer, transpiled_token_ids) - print(f"hf_transcript={hf_transcript!r}") - print(f"transpiled_transcript={transpiled_transcript!r}") - result_payload.update( - { - "blank_token_id": None if blank_token_id is None else int(blank_token_id), - "hf_transcript_token_ids": hf_token_ids, - "transpiled_transcript_token_ids": transpiled_token_ids, - "hf_transcript": hf_transcript, - "transpiled_transcript": transpiled_transcript, - } - ) - else: - print(f"output_shape={list(reference_output.shape)}") - print(f"output_max_abs_diff={max_abs_diff:.6f}") - print(f"output_mean_abs_diff={mean_abs_diff:.6f}") - - if artifact_dir is not None: - _write_json(artifact_dir / "result.json", result_payload) - print(f"saved_result={artifact_dir / 'result.json'}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python/src/transpile/canonicalize/cleanup.py b/python/src/transpile/canonicalize/cleanup.py deleted file mode 100644 index d72da0b3c..000000000 --- a/python/src/transpile/canonicalize/cleanup.py +++ /dev/null @@ -1,629 +0,0 @@ -from __future__ import annotations - -from collections import Counter - -import torch - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.graph_ir import IRValue -from src.transpile.graph_ir import verify_ir -from src.transpile.ops import canonicalize_op -from src.transpile.ops import get_op -from src.transpile.ops import has_op -from src.transpile.canonicalize.utils import bypass_unary_node -from src.transpile.canonicalize.utils import dce -from src.transpile.canonicalize.utils import ir_dtype_to_torch -from src.transpile.canonicalize.utils import materialize_constant_output -from src.transpile.canonicalize.utils import normalize_dim -from src.transpile.canonicalize.utils import normalize_dtype_name -from src.transpile.canonicalize.utils import numel -from src.transpile.canonicalize.utils import rebuild_graph - - -COMPILER_SUPPORTED_OPS = { - "arange", - "ones", -} - -CANONICAL_UNSUPPORTED_RENAMES = { - "aten.__and__.Tensor": "logical_and", - "aten.__or__.Tensor": "logical_or", - "aten.cumsum.default": "cumsum", - "aten.eq.Scalar": "equal", - "aten.eq.Tensor": "equal", - "aten.gt.Tensor": "greater", - "aten.index.Tensor": "advanced_index", - "aten.le.Tensor": "less_equal", -} - -FP32_SUPPORTED_ALL_INPUT_OPS = { - "precision_cast", - "view", - "slice", - "index", - "gather", - "layer_norm", - "batch_norm", - "glu", -} - -FP32_SUPPORTED_INPUT_INDICES = { - "embedding": {1}, -} - -FP16_ONLY_OUTPUT_OPS = { - "add", - "subtract", - "multiply", - "divide", - "scalar_add", - "scalar_subtract", - "scalar_subtract_reverse", - "scalar_multiply", - "scalar_divide", - "scalar_divide_reverse", - "not_equal", - "equal", - "greater", - "less_equal", - "logical_and", - "logical_or", - "logical_not", - "one_hot", - "scalar_not_equal", - "scalar_equal", - "abs", - "negate", - "pow", - "scalar_exp", - "scalar_sqrt", - "scalar_log", - "scalar_cos", - "scalar_sin", - "permute", - "matmul", - "cat", - "softmax", - "sum", - "mean", - "variance", - "min", - "max", - "rms_norm", - "embedding", - "group_norm", - "relu", - "silu", - "gelu", - "gelu_erf", - "sigmoid", - "tanh", - "linear", - "addmm", - "attention", - "attention_block", - "conv_module", - "rope", -} - - -def canonicalize_exported_graph(graph: IRGraph, *, max_passes: int = 8) -> IRGraph: - verify_ir(graph) - - for _ in range(max_passes): - changed = False - - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - if _canonicalize_node(graph, node): - changed = True - - rebuild_graph(graph) - - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - if _simplify_node(graph, node): - changed = True - - rebuild_graph(graph) - - if _legalize_precisions(graph): - changed = True - rebuild_graph(graph) - - dce(graph) - - if not changed: - break - - unsupported_counts = summarize_unsupported_ops(graph) - graph.meta = dict(graph.meta) - graph.meta["canonical_unsupported_ops"] = tuple(sorted(unsupported_counts)) - graph.meta["canonical_unsupported_op_counts"] = unsupported_counts - verify_ir(graph) - return graph - - -def summarize_unsupported_ops(graph: IRGraph) -> dict[str, int]: - counts: Counter[str] = Counter() - for node_id in graph.order: - node = graph.nodes[node_id] - if not _is_supported_without_backend_kernel(node.op): - counts[node.op] += 1 - return dict(sorted(counts.items())) - - -def _canonicalize_node(graph: IRGraph, node: IRNode) -> bool: - original_op = node.op - - if original_op == "aten.new_ones.default": - return _materialize_new_ones_constant(graph, node) - - if original_op in {"aten.new_zeros.default", "new_zeros"}: - return _materialize_new_zeros_constant(graph, node) - - if original_op == "arange": - return _materialize_arange_constant(graph, node) - - if original_op in {"identity", "contiguous"}: - return bypass_unary_node(graph, node) - - if original_op in CANONICAL_UNSUPPORTED_RENAMES: - node.op = CANONICAL_UNSUPPORTED_RENAMES[original_op] - return True - - if original_op == "multiply_inplace": - node.op = "multiply" - return True - - if original_op == "view" and "shape" not in node.attrs: - return _rewrite_view_from_output_shape(graph, node) - - if original_op in {"reshape", "flatten", "unsqueeze", "squeeze"}: - return _rewrite_view_from_output_shape(graph, node) - - if original_op == "expand": - return _rewrite_expand_if_view_equivalent(graph, node) - - if original_op == "transpose": - return _rewrite_transpose_as_permute(graph, node) - - if original_op == "movedim": - return _rewrite_movedim_as_permute(graph, node) - - if original_op == "type_as": - return _rewrite_type_as(graph, node) - - if original_op == "concat": - node.op = "cat" - if "dim" in node.attrs and "axis" not in node.attrs: - node.attrs = {**node.attrs, "axis": int(node.attrs["dim"])} - return True - - if original_op == "scaled_dot_product_attention": - node.op = "attention" - node.attrs = { - "scale": float(node.attrs.get("scale", 1.0)), - "is_causal": bool(node.attrs.get("is_causal", False)), - "window_size": int(node.attrs.get("window_size", 0)), - **({"mask": node.attrs["mask"]} if "mask" in node.attrs else {}), - **({"additive_mask": bool(node.attrs.get("additive_mask", False))} if "additive_mask" in node.attrs else {}), - } - return True - - canonical_op = canonicalize_op(original_op) - if canonical_op != original_op: - node.op = canonical_op - _normalize_attr_spellings(node) - if has_op(node.op): - _filter_attrs_to_schema(node) - return True - - if has_op(node.op): - _normalize_attr_spellings(node) - _filter_attrs_to_schema(node) - return True - - return False - - -def _simplify_node(graph: IRGraph, node: IRNode) -> bool: - if _fold_constant_node(graph, node): - return True - - if node.op == "precision_cast": - return _remove_noop_precision_cast(graph, node) - - if node.op == "view": - if _remove_noop_view(graph, node): - return True - return _collapse_view_chain(graph, node) - - if node.op == "slice": - return _remove_noop_slice(graph, node) - - if node.op == "scalar_add": - return _remove_noop_scalar(graph, node, neutral=0.0) - - if node.op == "scalar_subtract": - return _remove_noop_scalar(graph, node, neutral=0.0) - - if node.op == "scalar_multiply": - return _remove_noop_scalar(graph, node, neutral=1.0) - - if node.op == "scalar_divide": - return _remove_noop_scalar(graph, node, neutral=1.0) - - return False - - -def _fold_constant_node(graph: IRGraph, node: IRNode) -> bool: - if len(node.outputs) != 1: - return False - - output_value = graph.values.get(node.outputs[0]) - if output_value is None: - return False - - if node.op in FP16_ONLY_OUTPUT_OPS: - output_value.dtype = "fp16" - node.meta["dtype"] = "fp16" - - output_torch_dtype = ir_dtype_to_torch(output_value.dtype) if output_value.dtype is not None else None - - constant_inputs = [graph.constants.get(value_id) for value_id in node.inputs] - if node.op == "view" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): - if output_value.shape is None: - return False - return materialize_constant_output( - graph, - node, - constant_inputs[0].reshape(tuple(int(dim) for dim in output_value.shape)).to(dtype=output_torch_dtype), - ) - - if node.op == "expand" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): - if output_value.shape is None: - return False - return materialize_constant_output( - graph, - node, - constant_inputs[0].expand(tuple(int(dim) for dim in output_value.shape)).clone().to(dtype=output_torch_dtype), - ) - - if node.op == "precision_cast" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): - dtype = normalize_dtype_name(node.attrs.get("dtype")) - if dtype is None: - return False - return materialize_constant_output(graph, node, constant_inputs[0].to(dtype=ir_dtype_to_torch(dtype))) - - if node.op in {"cos", "scalar_cos"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): - return materialize_constant_output(graph, node, torch.cos(constant_inputs[0])) - - if node.op in {"sin", "scalar_sin"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): - return materialize_constant_output(graph, node, torch.sin(constant_inputs[0])) - - if node.op in {"scalar_add", "scalar_subtract", "scalar_multiply", "scalar_divide"}: - if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): - return False - scalar = float(node.attrs["value"]) - if node.op == "scalar_add": - result = constant_inputs[0] + scalar - elif node.op == "scalar_subtract": - result = constant_inputs[0] - scalar - elif node.op == "scalar_multiply": - result = constant_inputs[0] * scalar - else: - result = constant_inputs[0] / scalar - return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) - - if node.op in {"scalar_not_equal", "scalar_equal"}: - if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): - return False - scalar = float(node.attrs["value"]) - if node.op == "scalar_not_equal": - result = constant_inputs[0] != scalar - else: - result = constant_inputs[0] == scalar - return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) - - if node.op in {"not_equal", "equal", "greater", "less_equal", "logical_and", "logical_or"}: - if len(constant_inputs) != 2 or not all(isinstance(value, torch.Tensor) for value in constant_inputs): - return False - lhs, rhs = constant_inputs - if node.op == "not_equal": - result = lhs != rhs - elif node.op == "equal": - result = lhs == rhs - elif node.op == "greater": - result = lhs > rhs - elif node.op == "less_equal": - result = lhs <= rhs - elif node.op == "logical_and": - result = torch.logical_and(lhs != 0, rhs != 0) - else: - result = torch.logical_or(lhs != 0, rhs != 0) - return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) - - if node.op == "logical_not": - if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): - return False - result = torch.logical_not(constant_inputs[0] != 0) - return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) - - return False - - -def _legalize_precisions(graph: IRGraph) -> bool: - changed = False - counter = 0 - - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - - if node.op in FP32_SUPPORTED_ALL_INPUT_OPS: - continue - - inserted: list[IRNode] = [] - allowed_indices = FP32_SUPPORTED_INPUT_INDICES.get(node.op, set()) - for input_index, value_id in enumerate(list(node.inputs)): - if input_index in allowed_indices: - continue - value = graph.values.get(value_id) - if value is None or normalize_dtype_name(value.dtype) != "fp32": - continue - - counter += 1 - cast_node_id = f"{node.id}_legalize_fp16_{counter}" - cast_value_id = f"{value_id}_legalize_fp16_{counter}" - cast_node = IRNode( - id=cast_node_id, - op="precision_cast", - inputs=[value_id], - outputs=[cast_value_id], - attrs={"dtype": "fp16"}, - meta={"inserted_by": "canonicalize_exported_graph"}, - ) - graph.values[cast_value_id] = IRValue( - id=cast_value_id, - shape=value.shape, - dtype="fp16", - producer=cast_node_id, - users=[], - ) - node.inputs[input_index] = cast_value_id - inserted.append(cast_node) - changed = True - - if inserted: - insert_at = graph.order.index(node_id) - for offset, cast_node in enumerate(inserted): - graph.nodes[cast_node.id] = cast_node - graph.order.insert(insert_at + offset, cast_node.id) - - if node.op in FP16_ONLY_OUTPUT_OPS: - for output_id in node.outputs: - output_value = graph.values.get(output_id) - if output_value is not None: - output_value.dtype = "fp16" - node.meta["dtype"] = "fp16" - - return changed - - -def _rewrite_view_from_output_shape(graph: IRGraph, node: IRNode) -> bool: - if not node.outputs: - return False - output_value = graph.values.get(node.outputs[0]) - if output_value is None or output_value.shape is None: - return False - node.op = "view" - node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} - return True - - -def _rewrite_expand_if_view_equivalent(graph: IRGraph, node: IRNode) -> bool: - if not node.inputs or not node.outputs: - return False - input_value = graph.values.get(node.inputs[0]) - output_value = graph.values.get(node.outputs[0]) - if input_value is None or output_value is None: - return False - if input_value.shape is None or output_value.shape is None: - return False - if numel(input_value.shape) != numel(output_value.shape): - node.meta["canonicalization_blocked"] = "expand_requires_broadcast" - return False - node.op = "view" - node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} - return True - - -def _rewrite_transpose_as_permute(graph: IRGraph, node: IRNode) -> bool: - if not node.inputs: - return False - input_value = graph.values.get(node.inputs[0]) - if input_value is None or input_value.shape is None: - return False - rank = len(input_value.shape) - dim0 = normalize_dim(int(node.attrs["dim0"]), rank) - dim1 = normalize_dim(int(node.attrs["dim1"]), rank) - permutation = list(range(rank)) - permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] - node.op = "permute" - node.attrs = {"permutation": tuple(permutation)} - return True - - -def _rewrite_movedim_as_permute(graph: IRGraph, node: IRNode) -> bool: - if not node.inputs: - return False - input_value = graph.values.get(node.inputs[0]) - if input_value is None or input_value.shape is None: - return False - rank = len(input_value.shape) - source = [normalize_dim(int(dim), rank) for dim in node.attrs["source"]] - destination = [normalize_dim(int(dim), rank) for dim in node.attrs["destination"]] - if len(source) != len(destination): - return False - - remaining = [dim for dim in range(rank) if dim not in source] - permutation = list(remaining) - for dst, src in sorted(zip(destination, source)): - permutation.insert(dst, src) - - node.op = "permute" - node.attrs = {"permutation": tuple(permutation)} - return True - - -def _rewrite_type_as(graph: IRGraph, node: IRNode) -> bool: - if not node.inputs: - return False - target_dtype = None - if node.outputs: - output_value = graph.values.get(node.outputs[0]) - if output_value is not None: - target_dtype = output_value.dtype - if target_dtype is None and len(node.inputs) > 1: - target_value = graph.values.get(node.inputs[1]) - if target_value is not None: - target_dtype = target_value.dtype - if target_dtype is None: - return False - node.op = "precision_cast" - node.inputs = [node.inputs[0]] - node.attrs = {"dtype": normalize_dtype_name(target_dtype)} - return True - - -def _normalize_attr_spellings(node: IRNode) -> None: - attrs = dict(node.attrs) - if "dim" in attrs and "axis" not in attrs and node.op in {"softmax", "sum", "mean", "variance", "min", "max", "cat", "slice", "index"}: - attrs["axis"] = attrs.pop("dim") - if node.op == "permute" and "dims" in attrs and "permutation" not in attrs: - attrs["permutation"] = tuple(int(dim) for dim in attrs.pop("dims")) - if node.op == "slice": - attrs.setdefault("step", 1) - if node.op == "precision_cast" and "dtype" in attrs: - attrs["dtype"] = normalize_dtype_name(attrs["dtype"]) - node.attrs = attrs - - -def _filter_attrs_to_schema(node: IRNode) -> None: - schema = get_op(node.op) - if not schema.attrs: - node.attrs = {} - return - node.attrs = {name: node.attrs[name] for name in schema.attrs if name in node.attrs} - - -def _remove_noop_precision_cast(graph: IRGraph, node: IRNode) -> bool: - if len(node.inputs) != 1 or len(node.outputs) != 1: - return False - input_value = graph.values.get(node.inputs[0]) - output_value = graph.values.get(node.outputs[0]) - target_dtype = normalize_dtype_name(node.attrs.get("dtype")) - input_dtype = None if input_value is None else normalize_dtype_name(input_value.dtype) - output_dtype = None if output_value is None else normalize_dtype_name(output_value.dtype) - if target_dtype is None or target_dtype == input_dtype or (input_dtype is not None and output_dtype == input_dtype): - return bypass_unary_node(graph, node) - return False - - -def _remove_noop_view(graph: IRGraph, node: IRNode) -> bool: - if len(node.inputs) != 1 or len(node.outputs) != 1: - return False - input_value = graph.values.get(node.inputs[0]) - output_value = graph.values.get(node.outputs[0]) - if input_value is None or output_value is None: - return False - if input_value.shape is None or output_value.shape is None: - return False - if tuple(input_value.shape) == tuple(output_value.shape): - return bypass_unary_node(graph, node) - return False - - -def _collapse_view_chain(graph: IRGraph, node: IRNode) -> bool: - if len(node.inputs) != 1: - return False - input_value = graph.values.get(node.inputs[0]) - if input_value is None or input_value.producer is None: - return False - producer = graph.nodes.get(input_value.producer) - if producer is None or producer.op != "view" or len(producer.inputs) != 1: - return False - node.inputs = [producer.inputs[0]] - return True - - -def _remove_noop_slice(graph: IRGraph, node: IRNode) -> bool: - if len(node.inputs) != 1 or len(node.outputs) != 1: - return False - input_value = graph.values.get(node.inputs[0]) - output_value = graph.values.get(node.outputs[0]) - if input_value is None or output_value is None: - return False - if input_value.shape is None or output_value.shape is None: - return False - if tuple(input_value.shape) == tuple(output_value.shape): - return bypass_unary_node(graph, node) - return False - - -def _remove_noop_scalar(graph: IRGraph, node: IRNode, *, neutral: float) -> bool: - value = node.attrs.get("value") - if value is None: - return False - if float(value) != float(neutral): - return False - return bypass_unary_node(graph, node) - - -def _materialize_new_ones_constant(graph: IRGraph, node: IRNode) -> bool: - if len(node.outputs) != 1: - return False - output_id = node.outputs[0] - output_value = graph.values.get(output_id) - if output_value is None or output_value.shape is None or output_value.dtype is None: - return False - torch_dtype = ir_dtype_to_torch(output_value.dtype) - const = torch.ones(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) - return materialize_constant_output(graph, node, const) - - -def _materialize_new_zeros_constant(graph: IRGraph, node: IRNode) -> bool: - if len(node.outputs) != 1: - return False - output_id = node.outputs[0] - output_value = graph.values.get(output_id) - if output_value is None or output_value.shape is None or output_value.dtype is None: - return False - torch_dtype = ir_dtype_to_torch(output_value.dtype) - const = torch.zeros(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) - return materialize_constant_output(graph, node, const) - - -def _materialize_arange_constant(graph: IRGraph, node: IRNode) -> bool: - if len(node.outputs) != 1: - return False - start = int(node.attrs.get("start", 0)) - end = node.attrs.get("end") - if end is None: - return False - step = int(node.attrs.get("step", 1)) - dtype = normalize_dtype_name(node.attrs.get("dtype", "fp32")) - torch_dtype = ir_dtype_to_torch(dtype) - const = torch.arange(start, int(end), step=step, dtype=torch_dtype) - return materialize_constant_output(graph, node, const) - - -def _is_supported_without_backend_kernel(op_name: str) -> bool: - return has_op(op_name) or op_name in COMPILER_SUPPORTED_OPS diff --git a/python/src/transpile/canonicalize/utils.py b/python/src/transpile/canonicalize/utils.py deleted file mode 100644 index 30c4da76f..000000000 --- a/python/src/transpile/canonicalize/utils.py +++ /dev/null @@ -1,228 +0,0 @@ -from __future__ import annotations - -from typing import Any - -import torch - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.graph_ir import IRValue -from src.transpile.graph_ir import verify_ir - - -def normalize_dtype_name(dtype: Any) -> str | None: - if dtype is None: - return None - if not isinstance(dtype, str): - return str(dtype) - if dtype.startswith("torch."): - torch_name = dtype.split(".", 1)[1] - mapping = { - "bool": "bool", - "int8": "int8", - "int16": "int16", - "int32": "int32", - "int64": "int64", - "float16": "fp16", - "bfloat16": "bf16", - "float32": "fp32", - "float64": "fp64", - } - return mapping.get(torch_name, dtype) - return dtype - - -def ir_dtype_to_torch(dtype: str) -> torch.dtype: - normalized = normalize_dtype_name(dtype) - if normalized == "bool": - return torch.bool - if normalized == "int8": - return torch.int8 - if normalized == "int16": - return torch.int16 - if normalized == "int32": - return torch.int32 - if normalized == "int64": - return torch.int64 - if normalized == "fp16": - return torch.float16 - if normalized == "bf16": - return torch.bfloat16 - if normalized == "fp32": - return torch.float32 - if normalized == "fp64": - return torch.float64 - raise NotImplementedError(f"unsupported IR dtype for constant materialization: {dtype}") - - -def normalize_dim(dim: int, rank: int) -> int: - if dim < 0: - dim += rank - return dim - - -def numel(shape: tuple[int, ...]) -> int: - total = 1 - for dim in shape: - total *= int(dim) - return total - - -def refresh_value_users(graph: IRGraph) -> None: - for value in graph.values.values(): - value.users = [] - for node_id in graph.order: - node = graph.nodes[node_id] - for input_id in node.inputs: - value = graph.values.get(input_id) - if value is not None: - value.users.append(node_id) - - -def rebuild_graph(graph: IRGraph) -> None: - new_values: dict[str, IRValue] = {} - - for value_id in graph.inputs: - old = graph.values.get(value_id, IRValue(id=value_id)) - new_values[value_id] = IRValue( - id=value_id, - shape=old.shape, - dtype=old.dtype, - producer=None, - users=[], - meta=dict(old.meta), - ) - - for value_id in graph.constants.keys(): - old = graph.values.get(value_id, IRValue(id=value_id)) - new_values[value_id] = IRValue( - id=value_id, - shape=old.shape, - dtype=old.dtype, - producer=None, - users=[], - meta=dict(old.meta), - ) - - for node_id in graph.order: - node = graph.nodes[node_id] - for output_id in node.outputs: - old = graph.values.get(output_id, IRValue(id=output_id)) - new_values[output_id] = IRValue( - id=output_id, - shape=old.shape, - dtype=old.dtype, - producer=node_id, - users=[], - meta=dict(old.meta), - ) - - for node_id in graph.order: - node = graph.nodes[node_id] - for input_id in node.inputs: - if input_id not in new_values: - old = graph.values.get(input_id, IRValue(id=input_id)) - new_values[input_id] = IRValue( - id=input_id, - shape=old.shape, - dtype=old.dtype, - producer=old.producer, - users=[], - meta=dict(old.meta), - ) - new_values[input_id].users.append(node_id) - - for output_id in graph.outputs: - if output_id not in new_values: - old = graph.values.get(output_id, IRValue(id=output_id)) - new_values[output_id] = IRValue( - id=output_id, - shape=old.shape, - dtype=old.dtype, - producer=old.producer, - users=[], - meta=dict(old.meta), - ) - - graph.values = new_values - verify_ir(graph) - - -def dce(graph: IRGraph) -> None: - live_values = set(graph.outputs) - live_nodes: set[str] = set() - - changed = True - while changed: - changed = False - for node_id in reversed(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - if not any(output in live_values for output in node.outputs): - continue - if node_id not in live_nodes: - live_nodes.add(node_id) - changed = True - for input_id in node.inputs: - if input_id not in live_values: - live_values.add(input_id) - changed = True - - graph.order = [node_id for node_id in graph.order if node_id in live_nodes] - graph.nodes = {node_id: node for node_id, node in graph.nodes.items() if node_id in live_nodes} - - live_value_ids = set(graph.inputs) | set(graph.outputs) - for node in graph.nodes.values(): - live_value_ids.update(node.inputs) - live_value_ids.update(node.outputs) - - graph.values = {value_id: value for value_id, value in graph.values.items() if value_id in live_value_ids} - graph.constants = {value_id: value for value_id, value in graph.constants.items() if value_id in live_value_ids} - rebuild_graph(graph) - - -def remove_node(graph: IRGraph, node_id: str) -> None: - if node_id in graph.nodes: - del graph.nodes[node_id] - graph.order = [existing_id for existing_id in graph.order if existing_id != node_id] - - -def drop_value_if_unused(graph: IRGraph, value_id: str) -> None: - if value_id in graph.inputs or value_id in graph.outputs or value_id in graph.constants: - return - value = graph.values.get(value_id) - if value is None: - return - if value.producer is None and not value.users: - del graph.values[value_id] - - -def bypass_unary_node(graph: IRGraph, node: IRNode) -> bool: - if len(node.inputs) != 1 or len(node.outputs) != 1: - return False - input_id = node.inputs[0] - output_id = node.outputs[0] - - for other in graph.nodes.values(): - other.inputs = [input_id if value_id == output_id else value_id for value_id in other.inputs] - graph.outputs = [input_id if value_id == output_id else value_id for value_id in graph.outputs] - - remove_node(graph, node.id) - if output_id in graph.values: - del graph.values[output_id] - drop_value_if_unused(graph, input_id) - return True - - -def materialize_constant_output(graph: IRGraph, node: IRNode, value: Any) -> bool: - if len(node.outputs) != 1: - return False - output_id = node.outputs[0] - output_value = graph.values.get(output_id) - if output_value is None: - return False - graph.constants[output_id] = value - output_value.producer = None - remove_node(graph, node.id) - return True diff --git a/python/src/transpile/capture/capture_pytorch.py b/python/src/transpile/capture/capture_pytorch.py deleted file mode 100644 index 43448936f..000000000 --- a/python/src/transpile/capture/capture_pytorch.py +++ /dev/null @@ -1,302 +0,0 @@ -from dataclasses import dataclass -import inspect -from typing import Any -from typing import Iterable - -import torch -from torch.export import export - -from src.transpile.capture.graph_ir import IRGraph - - -class CapturePhaseError(RuntimeError): - def __init__(self, phase: str, message: str, *, cause: Exception | None = None): - super().__init__(f"[capture:{phase}] {message}") - self.phase = phase - self.cause = cause - - -@dataclass -class TensorMetadata: - shape: tuple[Any, ...] - dtype: Any - stride: tuple[Any, ...] | None = None - device: Any | None = None - - -@dataclass -class CapturedModel: - exported_program: Any - graph_module: Any - graph: Any - state_dict: dict[str, Any] - ir_graph: IRGraph - source_module: Any - example_args: tuple[Any, ...] - example_kwargs: dict[str, Any] - strict: bool - transpile_metadata: dict[str, Any] - - def named_parameters(self): - return self.exported_program.named_parameters() - - def named_buffers(self): - return self.exported_program.named_buffers() - - def parameters_dict(self) -> dict[str, Any]: - return dict(self.named_parameters()) - - def buffers_dict(self) -> dict[str, Any]: - return dict(self.named_buffers()) - - def placeholders(self) -> list[Any]: - return [node for node in self.graph.nodes if node.op == "placeholder"] - - def outputs(self) -> list[Any]: - return [node for node in self.graph.nodes if node.op == "output"] - - -def _normalize_args(args: Any) -> tuple[Any, ...]: - if args is None: - return () - if isinstance(args, tuple): - return args - if isinstance(args, list): - return tuple(args) - return (args,) - - -def _normalize_kwargs(kwargs: dict[str, Any] | None) -> dict[str, Any]: - return {} if kwargs is None else dict(kwargs) - - -def _clone_example_value(value: Any) -> Any: - if isinstance(value, torch.Tensor): - return value.detach().clone() - if isinstance(value, tuple): - return tuple(_clone_example_value(v) for v in value) - if isinstance(value, list): - return [_clone_example_value(v) for v in value] - if isinstance(value, dict): - return {k: _clone_example_value(v) for k, v in value.items()} - return value - - -def _clone_examples(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[tuple[Any, ...], dict[str, Any]]: - return ( - tuple(_clone_example_value(arg) for arg in args), - {k: _clone_example_value(v) for k, v in kwargs.items()}, - ) - - -def _inject_export_safe_kwargs(model: torch.nn.Module, kwargs: dict[str, Any]) -> dict[str, Any]: - try: - signature = inspect.signature(model.forward) - except (TypeError, ValueError): - return kwargs - - updated = dict(kwargs) - parameters = signature.parameters - - # HF causal LM forwards often default to returning cache objects and ModelOutput - # containers, which torch.export cannot serialize cleanly. Keep the capture - # boundary tensor-only unless the caller explicitly asked otherwise. - if "use_cache" in parameters and "use_cache" not in updated: - updated["use_cache"] = False - if "return_dict" in parameters and "return_dict" not in updated: - updated["return_dict"] = False - return updated - - -def _call_transpile_metadata_provider(module: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: - for attr_name in ("get_transpile_metadata", "transpile_metadata"): - provider = getattr(module, attr_name, None) - if not callable(provider): - continue - try: - metadata = provider(args=args, kwargs=kwargs) - except TypeError: - metadata = provider() - if metadata is None: - return {} - if not isinstance(metadata, dict): - raise TypeError(f"{type(module).__name__}.{attr_name} must return a dict or None") - return metadata - return {} - - -def _collect_transpile_metadata(model: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: - graph_meta: dict[str, Any] = {} - provider_graph_meta: dict[str, dict[str, Any]] = {} - import_hints: list[dict[str, Any]] = [] - seen_modules: set[int] = set() - - for module_path, module in model.named_modules(): - module_id = id(module) - if module_id in seen_modules: - continue - seen_modules.add(module_id) - - metadata = _call_transpile_metadata_provider(module, args, kwargs) - if not metadata: - continue - - provider_key = module_path or type(module).__name__ - provider_meta = metadata.get("graph", {}) - if isinstance(provider_meta, dict) and provider_meta: - provider_graph_meta[provider_key] = dict(provider_meta) - if not module_path: - graph_meta.update(provider_meta) - - raw_hints = metadata.get("import_hints", ()) - for raw_hint in raw_hints: - if not isinstance(raw_hint, dict): - raise TypeError(f"{type(module).__name__} import hints must be dictionaries") - hint = dict(raw_hint) - hint.setdefault("provider", provider_key) - import_hints.append(hint) - - if provider_graph_meta: - graph_meta.setdefault("transpile_metadata_providers", provider_graph_meta) - if import_hints: - graph_meta["import_hint_count"] = len(import_hints) - - if not graph_meta and not import_hints: - return {} - return { - "graph": graph_meta, - "import_hints": import_hints, - } - -def capture_model(model, args, kwargs=None, *, strict=True) -> CapturedModel: - if not isinstance(model, torch.nn.Module): - raise TypeError("model must be a torch.nn.Module") - - normalized_args = _normalize_args(args) - normalized_kwargs = _inject_export_safe_kwargs(model, _normalize_kwargs(kwargs)) - - if not normalized_args and not normalized_kwargs: - raise ValueError("capture_model requires example args or kwargs") - - model = model.eval() - example_args, example_kwargs = _clone_examples(normalized_args, normalized_kwargs) - transpile_metadata = _collect_transpile_metadata(model, example_args, example_kwargs) - - try: - ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) - except Exception as exc: - raise CapturePhaseError( - "export", - f"torch.export failed for model={type(model).__name__} strict={strict}: {exc}", - cause=exc, - ) from exc - from src.transpile.capture.import_ir import import_captured_to_ir - - raw_captured = CapturedModel( - exported_program=ep, - graph_module=ep.graph_module, - graph=ep.graph, - state_dict=dict(ep.state_dict), - ir_graph=IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[]), - source_module=model, - example_args=example_args, - example_kwargs=example_kwargs, - strict=strict, - transpile_metadata=transpile_metadata, - ) - try: - ir_graph = import_captured_to_ir(raw_captured, strict=strict) - except Exception as exc: - raise CapturePhaseError( - "import", - f"failed to import exported graph to IR for model={type(model).__name__} strict={strict}: {exc}", - cause=exc, - ) from exc - - return CapturedModel( - exported_program=ep, - graph_module=ep.graph_module, - graph=ep.graph, - state_dict=dict(ep.state_dict), - ir_graph=ir_graph, - source_module=model, - example_args=example_args, - example_kwargs=example_kwargs, - strict=strict, - transpile_metadata=transpile_metadata, - ) - - -def capture_model_with_fallback(model, args, kwargs=None) -> CapturedModel: - try: - return capture_model(model, args, kwargs=kwargs, strict=True) - except Exception: - return capture_model(model, args, kwargs=kwargs, strict=False) - - -def resolve_attr(root: Any, target: str) -> Any: - obj = root - for atom in target.split("."): - obj = getattr(obj, atom) - return obj - - -def format_target(node: Any) -> str: - return str(node.target) - -def get_tensor_metadata(node: Any) -> TensorMetadata | None: - meta = getattr(node, "meta", {}) or {} - value = meta.get("val") - if value is None: - return None - - if isinstance(value, torch.Tensor): - stride = tuple(int(s) for s in value.stride()) if value.layout == torch.strided else None - return TensorMetadata( - shape=tuple(value.shape), - dtype=value.dtype, - stride=stride, - device=value.device, - ) - - shape = tuple(getattr(value, "shape", ())) - dtype = getattr(value, "dtype", None) - stride_value = getattr(value, "stride", None) - stride = tuple(stride_value) if isinstance(stride_value, Iterable) else None - device = getattr(value, "device", None) - if not shape and dtype is None and stride is None and device is None: - return None - return TensorMetadata(shape=shape, dtype=dtype, stride=stride, device=device) - - -def get_shape(node: Any) -> tuple[Any, ...] | None: - metadata = get_tensor_metadata(node) - return None if metadata is None else metadata.shape - - -def get_dtype(node: Any) -> Any | None: - metadata = get_tensor_metadata(node) - return None if metadata is None else metadata.dtype - - -def dump_graph(captured: CapturedModel, *, include_meta: bool = True) -> str: - lines: list[str] = [] - for index, node in enumerate(captured.graph.nodes): - lines.append(f"[{index}] {node.op} {node.name} target={node.target}") - lines.append(f" args={node.args}") - lines.append(f" kwargs={node.kwargs}") - if include_meta: - tensor_meta = get_tensor_metadata(node) - if tensor_meta is not None: - lines.append( - " tensor_meta=" - f"shape={tensor_meta.shape} dtype={tensor_meta.dtype} " - f"stride={tensor_meta.stride} device={tensor_meta.device}" - ) - else: - lines.append(f" meta={getattr(node, 'meta', {})}") - return "\n".join(lines) - - -def print_graph(captured: CapturedModel, *, include_meta: bool = True) -> None: - print(dump_graph(captured, include_meta=include_meta)) diff --git a/python/src/transpile/capture/graph_ir.py b/python/src/transpile/capture/graph_ir.py deleted file mode 100644 index 56d9dcd18..000000000 --- a/python/src/transpile/capture/graph_ir.py +++ /dev/null @@ -1,125 +0,0 @@ -from dataclasses import dataclass -from dataclasses import field - - -@dataclass -class IRValue: - id: str - shape: tuple[int, ...] | None = None - dtype: str | None = None - producer: str | None = None - users: list[str] = field(default_factory=list) - meta: dict[str, object] = field(default_factory=dict) - -@dataclass -class IRNode: - id: str - op: str - inputs: list[str] - outputs: list[str] - attrs: dict[str, object] = field(default_factory=dict) - meta: dict[str, object] = field(default_factory=dict) - kind: str = "generic" - -@dataclass -class IRGraph: - values: dict[str, IRValue] - nodes: dict[str, IRNode] - order: list[str] - inputs: list[str] - outputs: list[str] - constants: dict[str, object] = field(default_factory=dict) - meta: dict[str, object] = field(default_factory=dict) - - def add_node(self, node: IRNode) -> None: - if node.id in self.nodes: - raise ValueError(f"duplicate IR node id: {node.id}") - self.nodes[node.id] = node - for output in node.outputs: - if output in self.values: - raise ValueError(f"duplicate IR value id: {output}") - self.values[output] = IRValue(id=output, producer=node.id) - - def add_value(self, value: IRValue) -> None: - if value.id in self.values: - raise ValueError(f"duplicate IR value id: {value.id}") - self.values[value.id] = value - - -def verify_ir(graph: IRGraph) -> None: - order_set = set(graph.order) - node_keys = set(graph.nodes.keys()) - if len(order_set) != len(graph.order): - raise ValueError("IR order contains duplicate node ids") - if order_set != node_keys: - missing_in_order = node_keys - order_set - missing_in_nodes = order_set - node_keys - raise ValueError( - "IR order/nodes mismatch: " - f"missing_in_order={sorted(missing_in_order)} " - f"missing_in_nodes={sorted(missing_in_nodes)}" - ) - - seen_nodes: set[str] = set() - seen_outputs: set[str] = set() - expected_users: dict[str, list[str]] = {} - - for value_id in graph.inputs: - if value_id not in graph.values: - raise ValueError(f"IR input missing value entry: {value_id}") - expected_users.setdefault(value_id, []) - - for value_id in graph.constants: - if value_id not in graph.values: - raise ValueError(f"IR constant missing value entry: {value_id}") - expected_users.setdefault(value_id, []) - - for node_id in graph.order: - if node_id in seen_nodes: - raise ValueError(f"duplicate node in IR order: {node_id}") - seen_nodes.add(node_id) - - node = graph.nodes[node_id] - if not node.outputs: - raise ValueError(f"IR node has no outputs: {node_id}") - - for input_id in node.inputs: - if input_id not in graph.values: - raise ValueError(f"IR node {node_id} references missing input value {input_id}") - expected_users.setdefault(input_id, []).append(node_id) - producer = graph.values[input_id].producer - if producer is not None and producer not in graph.nodes: - raise ValueError( - f"IR value {input_id} for node {node_id} has missing producer {producer}" - ) - - for output_id in node.outputs: - if output_id in seen_outputs: - raise ValueError(f"IR value produced more than once: {output_id}") - seen_outputs.add(output_id) - if output_id not in graph.values: - raise ValueError(f"IR node {node_id} missing output value entry {output_id}") - value = graph.values[output_id] - if value.producer != node_id: - raise ValueError( - f"IR value {output_id} producer mismatch: expected {node_id}, got {value.producer}" - ) - expected_users.setdefault(output_id, []) - - for output_id in graph.outputs: - if output_id not in graph.values: - raise ValueError(f"IR graph output missing value entry: {output_id}") - - for value_id, value in graph.values.items(): - expected = expected_users.get(value_id, []) - if value.users != expected: - raise ValueError( - f"IR value {value_id} user list mismatch: expected {expected}, got {value.users}" - ) - if value.producer is None and value_id not in graph.inputs and value_id not in graph.constants: - if value_id in seen_outputs: - raise ValueError(f"IR value {value_id} is produced but has no producer") - if value.producer is not None and value.producer not in graph.nodes: - raise ValueError( - f"IR value {value_id} references unknown producer {value.producer}" - ) diff --git a/python/src/transpile/capture/import_ir.py b/python/src/transpile/capture/import_ir.py deleted file mode 100644 index d1d02ea9e..000000000 --- a/python/src/transpile/capture/import_ir.py +++ /dev/null @@ -1,266 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any - -import torch -from torch.export.graph_signature import ConstantArgument -from torch.export.graph_signature import InputKind - -from src.transpile.capture.capture_pytorch import format_target -from src.transpile.capture.capture_pytorch import get_dtype -from src.transpile.capture.capture_pytorch import get_shape -from src.transpile.capture.capture_pytorch import resolve_attr -from src.transpile.capture.graph_ir import IRGraph -from src.transpile.capture.graph_ir import verify_ir -from src.transpile.capture.importers import extract_literals -from src.transpile.capture.importers import ImportContext -from src.transpile.capture.importers import import_call_function -from src.transpile.capture.importers import import_get_attr -from src.transpile.capture.importers import import_output -from src.transpile.capture.importers import import_placeholder -from src.transpile.capture.importers import value_id -from src.transpile.capture.normalize import dtype_to_ir -from src.transpile.capture.import_semantics import apply_import_semantics -from src.transpile.capture.weight_binding import resolve_transpile_weights_dir -from src.transpile.capture.weight_binding import resolve_weight_binding - - -def import_captured_to_ir(captured: Any, *, strict: bool = True) -> IRGraph: - transpile_metadata = getattr(captured, "transpile_metadata", {}) or {} - graph_meta = transpile_metadata.get("graph", {}) - ir = IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[], constants={}, meta=dict(graph_meta)) - ctx = ImportContext(strict=strict, transpile_metadata=transpile_metadata) - weights_dir = resolve_transpile_weights_dir(ir.meta) - if weights_dir: - ir.meta["weights_dir"] = weights_dir - placeholder_specs: dict[str, Any] = {} - inline_counter = 0 - - def _try_register_weight_binding(value_id_str: str, target: str, value: Any) -> None: - if not isinstance(value, torch.Tensor): - return - binding = resolve_weight_binding( - weights_dir=weights_dir, - source_name=target, - ) - if binding is None: - return - bindings = ir.meta.setdefault("weight_bindings", {}) - if isinstance(bindings, dict): - bindings[value_id_str] = { - "path": binding.path, - "kind": binding.kind, - "source_name": binding.source_name, - } - if value_id_str in ir.values: - ir.values[value_id_str].meta.update( - { - "path": binding.path, - "kind": binding.kind, - "source_name": binding.source_name, - } - ) - - def _make_inlined_node(node: Any, *, inline_index: int) -> Any: - return SimpleNamespace( - name=f"inl_{inline_index}_{node.name}", - op=node.op, - target=node.target, - args=node.args, - kwargs=getattr(node, "kwargs", {}), - meta=getattr(node, "meta", {}), - ) - - def _resolve_target(target: str) -> Any: - candidates: list[str] = [] - parts = target.split(".") - for i in range(len(parts)): - candidate = ".".join(parts[i:]) - if candidate and candidate not in candidates: - candidates.append(candidate) - - exported_program = getattr(captured, "exported_program", None) - if exported_program is not None: - tensor_constants = getattr(exported_program, "tensor_constants", None) - if tensor_constants is not None: - for candidate in candidates: - if candidate in tensor_constants: - return tensor_constants[candidate] - constants = getattr(exported_program, "constants", None) - if constants is not None: - for candidate in candidates: - if candidate in constants: - return constants[candidate] - - for candidate in candidates: - try: - return resolve_attr(captured.graph_module, candidate) - except AttributeError: - pass - - for candidate in candidates: - if candidate in captured.state_dict: - return captured.state_dict[candidate] - - source_module = getattr(captured, "source_module", None) - if source_module is not None: - for candidate in candidates: - try: - return resolve_attr(source_module, candidate) - except AttributeError: - pass - - raise NotImplementedError(f"could not resolve get_attr target: {target}") from None - - def _collect_output_ids(arg: Any, local_alias: dict[str, str]) -> list[str]: - if hasattr(arg, "name") and hasattr(arg, "op"): - name = arg.name - if name in local_alias: - return [local_alias[name]] - return [ctx.resolve_value_id(arg)] - if isinstance(arg, (tuple, list)): - out: list[str] = [] - for item in arg: - out.extend(_collect_output_ids(item, local_alias)) - return out - return [] - - def _inline_wrap_with_set_grad_enabled(node: Any, shape: Any, dtype: Any) -> None: - nonlocal inline_counter - subgraph_ref = node.args[1] - if not hasattr(subgraph_ref, "op") or subgraph_ref.op != "get_attr": - raise NotImplementedError("wrap_with_set_grad_enabled without get_attr submodule is not supported") - - subgraph_module = _resolve_target(subgraph_ref.target) - if not hasattr(subgraph_module, "graph"): - raise NotImplementedError("wrap_with_set_grad_enabled target is not a GraphModule") - - inline_counter += 1 - inline_index = inline_counter - local_alias: dict[str, str] = {} - subgraph_inputs = list(node.args[2:]) - input_index = 0 - - ctx.push_local_aliases(local_alias) - try: - for sub_node in subgraph_module.graph.nodes: - sub_shape = get_shape(sub_node) - sub_dtype = dtype_to_ir(get_dtype(sub_node)) - sub_desc = ( - f"inlined:{node.name}:{sub_node.op}:{getattr(sub_node, 'name', '')}:" - f"{format_target(sub_node) if sub_node.op == 'call_function' else getattr(sub_node, 'target', '')}" - ) - ctx.push_node(sub_desc) - try: - if sub_node.op == "placeholder": - if input_index >= len(subgraph_inputs): - raise NotImplementedError("subgraph placeholder/input arity mismatch during inlining") - local_alias[sub_node.name] = ctx.resolve_value_id(subgraph_inputs[input_index]) - input_index += 1 - continue - - if sub_node.op == "get_attr": - prefixed = _make_inlined_node(sub_node, inline_index=inline_index) - value = _resolve_target(sub_node.target) - import_get_attr(ir, prefixed, ctx, value, shape=sub_shape, dtype=sub_dtype, source_name=str(sub_node.target)) - _try_register_weight_binding(f"v_{prefixed.name}", str(sub_node.target), value) - local_alias[sub_node.name] = f"v_{prefixed.name}" - continue - - if sub_node.op == "call_function": - prefixed = _make_inlined_node(sub_node, inline_index=inline_index) - import_call_function( - ir, - prefixed, - ctx, - shape=sub_shape, - dtype=sub_dtype, - torch_op=format_target(sub_node), - ) - local_alias[sub_node.name] = f"v_{prefixed.name}" - continue - - if sub_node.op == "output": - ctx.tuple_aliases[node.name] = _collect_output_ids(sub_node.args[0], local_alias) - return - - raise NotImplementedError(f"unsupported inlined subgraph node op: {sub_node.op}") - finally: - ctx.pop_node() - finally: - ctx.pop_local_aliases() - - raise NotImplementedError("inlined subgraph had no output node") - - for spec in captured.exported_program.graph_signature.input_specs: - arg = getattr(spec, "arg", None) - name = getattr(arg, "name", None) - if name is not None: - placeholder_specs[name] = spec - - for node in captured.graph.nodes: - shape = get_shape(node) - dtype = dtype_to_ir(get_dtype(node)) - node_desc = f"top:{node.op}:{getattr(node, 'name', '')}:{getattr(node, 'target', '')}" - ctx.push_node(node_desc) - try: - if node.op == "placeholder": - spec = placeholder_specs.get(node.name) - if spec is not None and isinstance(getattr(spec, "arg", None), ConstantArgument): - import_get_attr( - ir, - node, - ctx, - spec.arg.value, - shape=shape, - dtype=dtype, - source_name=node.name, - ) - continue - if spec is not None and spec.kind in {InputKind.PARAMETER, InputKind.BUFFER, InputKind.CONSTANT_TENSOR}: - target = getattr(spec, "target", None) - if target is None: - raise NotImplementedError(f"placeholder {node.name} has no target for lifted constant input") - value = _resolve_target(target) - import_get_attr(ir, node, ctx, value, shape=shape, dtype=dtype, source_name=str(target)) - _try_register_weight_binding(value_id(node, ctx), str(target), value) - continue - import_placeholder(ir, node, ctx, shape=shape, dtype=dtype) - continue - - if node.op == "get_attr": - if str(node.target).startswith("submod_"): - continue - value = _resolve_target(node.target) - import_get_attr(ir, node, ctx, value, shape=shape, dtype=dtype, source_name=str(node.target)) - _try_register_weight_binding(value_id(node, ctx), str(node.target), value) - continue - - if node.op == "call_function": - torch_op = format_target(node) - if "wrap_with_set_grad_enabled" in torch_op: - _inline_wrap_with_set_grad_enabled(node, shape, dtype) - continue - if torch_op == "": - source = node.args[0] - if hasattr(source, "name") and source.name in ctx.tuple_aliases: - index_value = extract_literals(node.args[1]) - if not isinstance(index_value, int): - raise NotImplementedError(f"unsupported tuple getitem index: {node.args!r}") - ctx.alias_values[node.name] = ctx.tuple_aliases[source.name][index_value] - continue - import_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) - continue - - if node.op == "output": - import_output(ir, node, ctx) - continue - - raise NotImplementedError(f"unsupported FX node op: {node.op}") - finally: - ctx.pop_node() - - apply_import_semantics(ir) - verify_ir(ir) - return ir diff --git a/python/src/transpile/capture/import_semantics.py b/python/src/transpile/capture/import_semantics.py deleted file mode 100644 index b97666f2b..000000000 --- a/python/src/transpile/capture/import_semantics.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -import torch - -from src.transpile.capture.graph_ir import IRGraph -from src.transpile.capture.graph_ir import IRNode -from src.transpile.capture.graph_ir import verify_ir - -def apply_import_semantics(graph: IRGraph) -> IRGraph: - return graph diff --git a/python/src/transpile/capture/importers.py b/python/src/transpile/capture/importers.py deleted file mode 100644 index 4b3eccd9d..000000000 --- a/python/src/transpile/capture/importers.py +++ /dev/null @@ -1,1236 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from dataclasses import field -from types import SimpleNamespace -from typing import Any - -import torch - -from src.transpile.capture.graph_ir import IRGraph -from src.transpile.capture.graph_ir import IRNode -from src.transpile.capture.graph_ir import IRValue -from src.transpile.capture.normalize import dtype_to_ir -from src.transpile.capture.normalize import normalize_target - - -class UnsupportedImportError(NotImplementedError): - pass - - -@dataclass -class ImportContext: - strict: bool = True - alias_values: dict[str, str] = field(default_factory=dict) - tuple_aliases: dict[str, list[str]] = field(default_factory=dict) - local_alias_stack: list[dict[str, str]] = field(default_factory=list) - node_stack: list[str] = field(default_factory=list) - transpile_metadata: dict[str, object] = field(default_factory=dict) - - def push_local_aliases(self, aliases: dict[str, str]) -> None: - self.local_alias_stack.append(aliases) - - def pop_local_aliases(self) -> None: - if not self.local_alias_stack: - raise RuntimeError("local alias stack underflow") - self.local_alias_stack.pop() - - def resolve_value_id(self, node: Any) -> str: - if is_fx_node(node): - name = getattr(node, "name", "") - for aliases in reversed(self.local_alias_stack): - if name in aliases: - return aliases[name] - resolved = self.alias_values.get(name) - if resolved is not None: - return resolved - return f"v_{name}" - raise TypeError(f"cannot resolve value id for non-FX node: {type(node).__name__}") - - def fail(self, message: str) -> None: - if self.node_stack: - message = f"{message} [context: {' > '.join(self.node_stack)}]" - if self.strict: - raise UnsupportedImportError(message) - raise UnsupportedImportError(f"{message} [non-strict import has no generic fallback]") - - def push_node(self, description: str) -> None: - self.node_stack.append(description) - - def pop_node(self) -> None: - if self.node_stack: - self.node_stack.pop() - - def _node_module_paths(self, node: Any) -> tuple[str, ...]: - meta = getattr(node, "meta", {}) or {} - raw_stack = meta.get("nn_module_stack") - if not isinstance(raw_stack, dict): - return () - paths: list[str] = [] - for entry in raw_stack.values(): - if not isinstance(entry, (tuple, list)) or not entry: - continue - module_path = entry[0] - if isinstance(module_path, str) and module_path: - paths.append(module_path) - # Prefer the deepest module path first. - paths.reverse() - deduped: list[str] = [] - for path in paths: - if path not in deduped: - deduped.append(path) - return tuple(deduped) - - def lookup_import_hint(self, node: Any, *, torch_op: str, op_name: str) -> dict[str, object]: - hints = self.transpile_metadata.get("import_hints", ()) - if not isinstance(hints, list): - return {} - - module_paths = self._node_module_paths(node) - matched_attrs: dict[str, object] = {} - matched_meta: dict[str, object] = {} - matched_provider: list[str] = [] - - for hint in hints: - if not isinstance(hint, dict): - continue - hint_op = hint.get("op") - if hint_op is not None and hint_op != op_name: - continue - hint_torch_op = hint.get("torch_op") - if hint_torch_op is not None and hint_torch_op != torch_op: - continue - - module_path = hint.get("module_path") - if module_path is not None and module_path not in module_paths: - continue - module_path_suffix = hint.get("module_path_suffix") - if module_path_suffix is not None and not any( - path == module_path_suffix or path.endswith(f".{module_path_suffix}") for path in module_paths - ): - continue - - attrs = hint.get("attrs", {}) - if isinstance(attrs, dict): - matched_attrs.update(attrs) - meta = hint.get("meta", {}) - if isinstance(meta, dict): - matched_meta.update(meta) - provider = hint.get("provider") - if isinstance(provider, str) and provider not in matched_provider: - matched_provider.append(provider) - - if not matched_attrs and not matched_meta and not matched_provider: - return {} - - if matched_provider: - matched_meta.setdefault("import_hint_providers", tuple(matched_provider)) - matched_meta.setdefault("import_hint_source", "capture_metadata") - return {"attrs": matched_attrs, "meta": matched_meta} - - -def value_id(node: Any, ctx: ImportContext) -> str: - return ctx.resolve_value_id(node) - - -def node_id(node: Any) -> str: - return f"n_{node.name}" - - -def is_fx_node(value: Any) -> bool: - return hasattr(value, "name") and hasattr(value, "op") - - -def extract_input_ids(arg: Any, ctx: ImportContext) -> list[str]: - if is_fx_node(arg): - return [value_id(arg, ctx)] - if isinstance(arg, (tuple, list)): - ids: list[str] = [] - for item in arg: - ids.extend(extract_input_ids(item, ctx)) - return ids - if isinstance(arg, dict): - ids: list[str] = [] - for item in arg.values(): - ids.extend(extract_input_ids(item, ctx)) - return ids - return [] - - -def extract_literals(value: Any) -> Any: - if is_fx_node(value): - return None - if isinstance(value, (tuple, list)): - return type(value)(extract_literals(v) for v in value) - if isinstance(value, dict): - return {k: extract_literals(v) for k, v in value.items()} - if isinstance(value, torch.dtype): - return str(value) - if isinstance(value, (int, float, bool, str)) or value is None: - return value - return None - - -def make_prefixed_node(node: Any, prefix: str) -> Any: - return SimpleNamespace( - name=f"{prefix}{node.name}", - op=node.op, - target=node.target, - args=node.args, - kwargs=getattr(node, "kwargs", {}), - meta=getattr(node, "meta", {}), - ) - - -def add_value_if_missing(ir: IRGraph, value: IRValue) -> None: - if value.id not in ir.values: - ir.add_value(value) - - -def register_node( - ir: IRGraph, - ir_node: IRNode, - *, - shape: tuple[int, ...] | None, - dtype: str | None, -) -> None: - ir.add_node(ir_node) - ir.order.append(ir_node.id) - for output_id in ir_node.outputs: - ir.values[output_id].shape = shape - ir.values[output_id].dtype = dtype - for input_id in ir_node.inputs: - if input_id in ir.values: - ir.values[input_id].users.append(ir_node.id) - - -def _extract_numeric_literal(value: Any) -> float | int | None: - literal = extract_literals(value) - if isinstance(literal, bool): - return int(literal) - if isinstance(literal, (int, float)): - return literal - return None - - -def _base_meta(shape: tuple[int, ...] | None, dtype: str | None, torch_op: str, node: Any) -> dict[str, object]: - return { - "shape": shape, - "dtype": dtype, - "torch_op": torch_op, - "torch_name": node.name, - } - - -def _import_scalar_binary( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, - op_name: str, -) -> None: - lhs, rhs = node.args[0], node.args[1] - lhs_is_node = is_fx_node(lhs) - rhs_is_node = is_fx_node(rhs) - lhs_literal = _extract_numeric_literal(lhs) - rhs_literal = _extract_numeric_literal(rhs) - - if lhs_is_node and rhs_is_node: - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return - - if op_name == "add": - if lhs_is_node and rhs_literal is not None: - scalar_op = "scalar_add" - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - scalar_op = "scalar_add" - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - scalar_op = None - elif op_name == "multiply": - if lhs_is_node and rhs_literal is not None: - scalar_op = "scalar_multiply" - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - scalar_op = "scalar_multiply" - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - scalar_op = None - elif op_name == "subtract": - if lhs_is_node and rhs_literal is not None: - scalar_op = "scalar_subtract" - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - scalar_op = "scalar_subtract_reverse" - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - scalar_op = None - elif op_name == "divide": - if lhs_is_node and rhs_literal is not None: - scalar_op = "scalar_divide" - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - scalar_op = "scalar_divide_reverse" - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - scalar_op = None - else: - scalar_op = None - - if scalar_op is None: - ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") - - ir_node = IRNode( - id=node_id(node), - op=scalar_op, - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_placeholder( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, -) -> None: - add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) - ir.inputs.append(value_id(node, ctx)) - - -def import_get_attr( - ir: IRGraph, - node: Any, - ctx: ImportContext, - value: Any, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - source_name: str | None = None, -) -> None: - add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) - ir.constants[value_id(node, ctx)] = value.detach().cpu() if isinstance(value, torch.Tensor) else value - ir.values[value_id(node, ctx)].meta["source_name"] = source_name - if source_name is not None: - bindings = ir.meta.setdefault("weight_bindings", {}) - if isinstance(bindings, dict): - bindings.setdefault(value_id(node, ctx), {})["source_name"] = source_name - - -def import_output(ir: IRGraph, node: Any, ctx: ImportContext) -> None: - ir.outputs.extend(extract_input_ids(node.args[0], ctx)) - - -def import_call_function( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, -) -> None: - op = normalize_target(torch_op) - import_hint = ctx.lookup_import_hint(node, torch_op=torch_op, op_name=op) - importer = OP_IMPORTERS.get(op) - if importer is None: - import_opaque_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name=op) - else: - importer(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) - _apply_import_hint(ir, node, import_hint) - - -def _apply_import_hint(ir: IRGraph, node: Any, hint: dict[str, object]) -> None: - if not hint: - return - ir_node = ir.nodes.get(node_id(node)) - if ir_node is None: - return - attrs = hint.get("attrs", {}) - if isinstance(attrs, dict): - ir_node.attrs.update(attrs) - meta = hint.get("meta", {}) - if isinstance(meta, dict): - ir_node.meta.update(meta) - - -def import_opaque_call_function( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, - op_name: str, -) -> None: - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=extract_input_ids(getattr(node, "args", ()), ctx) + extract_input_ids(getattr(node, "kwargs", {}), ctx), - outputs=[value_id(node, ctx)], - attrs={ - "opaque": True, - "torch_op": torch_op, - "args": extract_literals(getattr(node, "args", ())), - "kwargs": extract_literals(getattr(node, "kwargs", {})), - }, - meta=_base_meta(shape, dtype, torch_op, node), - kind="opaque", - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_add(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="add") - - -def import_subtract(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="subtract") - - -def import_multiply(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") - - -def import_multiply_inplace(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply_inplace") - - -def import_divide(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="divide") - - -def import_not_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - lhs, rhs = node.args[0], node.args[1] - lhs_is_node = is_fx_node(lhs) - rhs_is_node = is_fx_node(rhs) - lhs_literal = _extract_numeric_literal(lhs) - rhs_literal = _extract_numeric_literal(rhs) - - if lhs_is_node and rhs_is_node: - ir_node = IRNode( - id=node_id(node), - op="not_equal", - inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return - - if lhs_is_node and rhs_literal is not None: - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") - - ir_node = IRNode( - id=node_id(node), - op="scalar_not_equal", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_negate(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="scalar_multiply", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"value": -1.0}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_precision_cast(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_dtype = extract_literals(node.args[1]) if len(node.args) > 1 else None - ir_node = IRNode( - id=node_id(node), - op="precision_cast", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"dtype": target_dtype}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_arange(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - attrs: dict[str, object] = {} - if len(node.args) == 1: - attrs["start"] = 0 - attrs["end"] = int(node.args[0]) - elif len(node.args) >= 2: - attrs["start"] = int(node.args[0]) - attrs["end"] = int(node.args[1]) - else: - ctx.fail(f"unsupported arange signature for {torch_op}: {node.args!r}") - - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - attrs["step"] = int(node.args[2]) - - target_dtype = None - if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None: - target_dtype = extract_literals(node.kwargs["dtype"]) - elif dtype is not None: - target_dtype = dtype - if target_dtype is not None: - attrs["dtype"] = target_dtype - - ir_node = IRNode( - id=node_id(node), - op="arange", - inputs=[], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_type_as(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="type_as", - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_identity(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="identity", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_reshape(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_shape = tuple(int(v) for v in node.args[1]) - ir_node = IRNode( - id=node_id(node), - op="reshape", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"shape": target_shape}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_flatten(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="flatten", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"start_dim": int(node.args[1]), "end_dim": int(node.args[2])}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_unsqueeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - dim = int(node.args[1]) - attrs: dict[str, object] = {"dim": dim} - if shape is not None: - attrs["shape"] = tuple(int(v) for v in shape) - ir_node = IRNode( - id=node_id(node), - op="unsqueeze", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_expand(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_shape = tuple(int(v) for v in node.args[1]) - ir_node = IRNode( - id=node_id(node), - op="expand", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"shape": target_shape}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_transpose(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - attrs = {"dim0": 0, "dim1": 1} if torch_op == "aten.t.default" else {"dim0": int(node.args[1]), "dim1": int(node.args[2])} - ir_node = IRNode( - id=node_id(node), - op="transpose", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_permute(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - permutation = tuple(int(v) for v in node.args[1]) - ir_node = IRNode( - id=node_id(node), - op="permute", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"permutation": permutation}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_matmul(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="matmul", - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_linear(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] - attrs = {"has_bias": False} - if len(node.args) > 2 and node.args[2] is not None: - inputs.append(value_id(node.args[2], ctx)) - attrs["has_bias"] = True - ir_node = IRNode( - id=node_id(node), - op="linear", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_addmm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="addmm", - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_unary(op_name: str): - def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return _importer - - -def import_pow_rsqrt(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="pow", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"exponent": -0.5}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_softmax(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="softmax", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"axis": int(node.args[1])}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - -def import_scaled_dot_product_attention( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, -) -> None: - attrs: dict[str, object] = {} - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)] - if len(node.args) > 3: - mask_literal = extract_literals(node.args[3]) - if mask_literal is not None: - attrs["mask"] = mask_literal - elif is_fx_node(node.args[3]): - inputs.append(value_id(node.args[3], ctx)) - if len(node.args) > 4 and extract_literals(node.args[4]) is not None: - attrs["dropout_p"] = float(node.args[4]) - if len(node.args) > 5 and extract_literals(node.args[5]) is not None: - attrs["is_causal"] = bool(node.args[5]) - if "scale" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["scale"]) is not None: - attrs["scale"] = float(node.kwargs["scale"]) - if "enable_gqa" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["enable_gqa"]) is not None: - attrs["enable_gqa"] = bool(node.kwargs["enable_gqa"]) - ir_node = IRNode( - id=node_id(node), - op="scaled_dot_product_attention", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_reduce(op_name: str): - def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - attrs: dict[str, object] = {} - if len(node.args) > 1 and extract_literals(node.args[1]) is not None: - attrs["axis"] = extract_literals(node.args[1]) - keepdim = None - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - keepdim = bool(extract_literals(node.args[2])) - elif "keepdim" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["keepdim"]) is not None: - keepdim = bool(extract_literals(node.kwargs["keepdim"])) - if keepdim is not None: - attrs["keepdim"] = keepdim - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return _importer - - -def import_cat(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - tensors_arg = node.args[0] - axis = int(node.args[1]) if len(node.args) > 1 else 0 - ir_node = IRNode( - id=node_id(node), - op="cat", - inputs=extract_input_ids(tensors_arg, ctx), - outputs=[value_id(node, ctx)], - attrs={"axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_split_with_sizes(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported split_with_sizes signature for {torch_op}: {node.args!r}") - sizes = extract_literals(node.args[1]) - if not isinstance(sizes, (tuple, list)) or not all(isinstance(v, int) for v in sizes): - ctx.fail(f"unsupported split sizes for {torch_op}: {node.args!r}") - axis = -1 - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - axis = int(extract_literals(node.args[2])) - ir_node = IRNode( - id=node_id(node), - op="split_with_sizes", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"sizes": tuple(int(v) for v in sizes), "axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_chunk(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported chunk signature for {torch_op}: {node.args!r}") - chunks = extract_literals(node.args[1]) - if not isinstance(chunks, int): - ctx.fail(f"unsupported chunk count for {torch_op}: {node.args!r}") - axis = 0 - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - axis = int(extract_literals(node.args[2])) - ir_node = IRNode( - id=node_id(node), - op="chunk", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"chunks": int(chunks), "axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_pow(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx)] - attrs: dict[str, object] = {} - exponent_literal = extract_literals(node.args[1]) if len(node.args) > 1 else None - if exponent_literal is None and len(node.args) > 1 and is_fx_node(node.args[1]): - inputs.append(value_id(node.args[1], ctx)) - else: - attrs["exponent"] = exponent_literal - ir_node = IRNode( - id=node_id(node), - op="pow", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_slice(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - attrs = {"axis": int(node.args[1]), "start": int(node.args[2]), "end": int(node.args[3])} - if len(node.args) > 4 and extract_literals(node.args[4]) is not None: - attrs["step"] = int(node.args[4]) - ir_node = IRNode( - id=node_id(node), - op="slice", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = list(getattr(node, "args", ())) - if not inputs: - ctx.fail(f"unsupported diff signature for {torch_op}: missing input tensor") - - source = inputs[0] - if not is_fx_node(source): - ctx.fail(f"unsupported diff signature for {torch_op}: source is not FX node") - - n_value = 1 - if len(inputs) > 1 and extract_literals(inputs[1]) is not None: - n_value = int(extract_literals(inputs[1])) - elif "n" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["n"]) is not None: - n_value = int(extract_literals(node.kwargs["n"])) - if n_value != 1: - ctx.fail(f"unsupported diff order for {torch_op}: n={n_value}") - - dim_value = None - if len(inputs) > 2 and extract_literals(inputs[2]) is not None: - dim_value = int(extract_literals(inputs[2])) - elif "dim" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dim"]) is not None: - dim_value = int(extract_literals(node.kwargs["dim"])) - if dim_value is None: - ctx.fail(f"unsupported diff signature for {torch_op}: missing integer dim") - - prepend_value = None - append_value = None - if len(inputs) > 3: - prepend_value = extract_literals(inputs[3]) - if len(inputs) > 4: - append_value = extract_literals(inputs[4]) - if "prepend" in getattr(node, "kwargs", {}): - prepend_value = extract_literals(node.kwargs["prepend"]) - if "append" in getattr(node, "kwargs", {}): - append_value = extract_literals(node.kwargs["append"]) - if prepend_value is not None or append_value is not None: - ctx.fail(f"unsupported diff signature for {torch_op}: prepend/append are not supported") - - source_value_id = value_id(source, ctx) - source_shape = None - if source_value_id in ir.values: - source_shape = ir.values[source_value_id].shape - if source_shape is None: - source_shape = shape - if source_shape is None: - ctx.fail(f"unsupported diff import for {torch_op}: missing source shape") - - rank = len(source_shape) - normalized_dim = dim_value if dim_value >= 0 else dim_value + rank - if normalized_dim < 0 or normalized_dim >= rank: - ctx.fail(f"unsupported diff dimension for {torch_op}: dim={dim_value} rank={rank}") - - dim_extent = int(source_shape[normalized_dim]) - if dim_extent < 1: - ctx.fail(f"unsupported diff input extent for {torch_op}: dim size {dim_extent}") - - left_shape = list(source_shape) - left_shape[normalized_dim] = max(0, dim_extent - 1) - left_shape_tuple = tuple(int(v) for v in left_shape) - - left_node = IRNode( - id=f"{node_id(node)}__diff_left", - op="slice", - inputs=[source_value_id], - outputs=[f"{value_id(node, ctx)}__diff_left"], - attrs={"axis": normalized_dim, "start": 0, "end": dim_extent - 1, "step": 1}, - meta=_base_meta(left_shape_tuple, dtype, torch_op, node), - ) - register_node(ir, left_node, shape=left_shape_tuple, dtype=dtype) - - right_node = IRNode( - id=f"{node_id(node)}__diff_right", - op="slice", - inputs=[source_value_id], - outputs=[f"{value_id(node, ctx)}__diff_right"], - attrs={"axis": normalized_dim, "start": 1, "end": dim_extent, "step": 1}, - meta=_base_meta(left_shape_tuple, dtype, torch_op, node), - ) - register_node(ir, right_node, shape=left_shape_tuple, dtype=dtype) - - diff_node = IRNode( - id=node_id(node), - op="subtract", - inputs=[right_node.outputs[0], left_node.outputs[0]], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, diff_node, shape=shape, dtype=dtype) - - -def import_index(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if torch_op.startswith("aten.select"): - axis = int(node.args[1]) - index_value = int(node.args[2]) - else: - index_value = int(node.args[1]) - axis = int(node.args[2]) if len(node.args) > 2 else 0 - ir_node = IRNode( - id=node_id(node), - op="index", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"index_value": index_value, "axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_gather(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="gather", - inputs=[value_id(node.args[0], ctx), value_id(node.args[2], ctx)], - outputs=[value_id(node, ctx)], - attrs={"axis": int(node.args[1])}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_embedding(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] - attrs: dict[str, object] = {} - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - attrs["padding_idx"] = int(node.args[2]) - if len(node.args) > 3 and extract_literals(node.args[3]) is not None: - attrs["scale_grad_by_freq"] = bool(node.args[3]) - if len(node.args) > 4 and extract_literals(node.args[4]) is not None: - attrs["sparse"] = bool(node.args[4]) - ir_node = IRNode( - id=node_id(node), - op="embedding", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def _extract_int_list_or_scalar(value: Any, *, default: int) -> int: - literal = extract_literals(value) - if literal is None: - return default - if isinstance(literal, int): - return int(literal) - if isinstance(literal, (tuple, list)) and literal: - first = literal[0] - if isinstance(first, int): - return int(first) - raise UnsupportedImportError(f"unsupported integer/list literal: {value!r}") - - -def import_conv1d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported conv1d signature for {torch_op}: {node.args!r}") - - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] - if len(node.args) > 2 and is_fx_node(node.args[2]): - inputs.append(value_id(node.args[2], ctx)) - - stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 - padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 - dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 - groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 - - ir_node = IRNode( - id=node_id(node), - op="conv1d", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_pad(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported pad signature for {torch_op}: {node.args!r}") - - pads = extract_literals(node.args[1]) - if not isinstance(pads, (tuple, list)) or not all(isinstance(v, int) for v in pads): - ctx.fail(f"unsupported pads for {torch_op}: {node.args!r}") - - mode = "constant" - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - mode = str(extract_literals(node.args[2])) - if "mode" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["mode"]) is not None: - mode = str(extract_literals(node.kwargs["mode"])) - - value = 0.0 - if len(node.args) > 3 and extract_literals(node.args[3]) is not None: - value = float(extract_literals(node.args[3])) - if "value" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["value"]) is not None: - value = float(extract_literals(node.kwargs["value"])) - - ir_node = IRNode( - id=node_id(node), - op="pad", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"pads": tuple(int(v) for v in pads), "mode": mode, "value": value}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if not node.args: - ctx.fail(f"unsupported ones signature for {torch_op}: {node.args!r}") - - shape_literal = extract_literals(node.args[0]) - if isinstance(shape_literal, int): - output_shape = (int(shape_literal),) - elif isinstance(shape_literal, (tuple, list)) and all(isinstance(v, int) for v in shape_literal): - output_shape = tuple(int(v) for v in shape_literal) - else: - ctx.fail(f"unsupported ones shape for {torch_op}: {node.args!r}") - - node_dtype = dtype - if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None: - node_dtype = dtype_to_ir(extract_literals(node.kwargs["dtype"])) - elif len(node.args) > 1 and extract_literals(node.args[1]) is not None: - node_dtype = dtype_to_ir(extract_literals(node.args[1])) - if node_dtype is None: - node_dtype = "fp32" - - ir_node = IRNode( - id=node_id(node), - op="ones", - inputs=[], - outputs=[value_id(node, ctx)], - attrs={"shape": output_shape, "dtype": node_dtype}, - meta=_base_meta(shape or output_shape, node_dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape or output_shape, dtype=node_dtype) - - -def import_layer_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx), value_id(node.args[3], ctx)] - attrs = {"normalized_shape": tuple(int(v) for v in node.args[1]), "eps": float(node.args[4])} - ir_node = IRNode( - id=node_id(node), - op="layer_norm", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_rms_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx)] - attrs = {"normalized_shape": tuple(int(v) for v in node.args[1]), "eps": float(node.args[3])} - ir_node = IRNode( - id=node_id(node), - op="rms_norm", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_group_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx), value_id(node.args[3], ctx)] - attrs = {"num_groups": int(node.args[1]), "eps": float(node.args[4])} - ir_node = IRNode( - id=node_id(node), - op="group_norm", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_batch_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [ - value_id(node.args[0], ctx), - value_id(node.args[1], ctx), - value_id(node.args[2], ctx), - value_id(node.args[3], ctx), - value_id(node.args[4], ctx), - ] - attrs = {"training": bool(node.args[5]), "momentum": float(node.args[6]), "eps": float(node.args[7])} - ir_node = IRNode( - id=node_id(node), - op="batch_norm", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_contiguous(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="contiguous", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - index_value = extract_literals(node.args[1]) - if not isinstance(index_value, int): - ctx.fail(f"unsupported getitem index for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="getitem", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"index": index_value}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -OP_IMPORTERS = { - "arange": import_arange, - "identity": import_identity, - "add": import_add, - "subtract": import_subtract, - "multiply": import_multiply, - "multiply_inplace": import_multiply_inplace, - "divide": import_divide, - "precision_cast": import_precision_cast, - "type_as": import_type_as, - "negate": import_negate, - "abs": import_unary("abs"), - "not_equal": import_not_equal, - "cos": import_unary("cos"), - "sin": import_unary("sin"), - "scalar_exp": import_unary("scalar_exp"), - "scalar_sqrt": import_unary("scalar_sqrt"), - "scalar_log": import_unary("scalar_log"), - "rsqrt": import_pow_rsqrt, - "reshape": import_reshape, - "flatten": import_flatten, - "unsqueeze": import_unsqueeze, - "expand": import_expand, - "transpose": import_transpose, - "permute": import_permute, - "matmul": import_matmul, - "linear": import_linear, - "addmm": import_addmm, - "relu": import_unary("relu"), - "silu": import_unary("silu"), - "gelu": import_unary("gelu"), - "gelu_erf": import_unary("gelu_erf"), - "sigmoid": import_unary("sigmoid"), - "softplus": import_unary("softplus"), - "tanh": import_unary("tanh"), - "softmax": import_softmax, - "scaled_dot_product_attention": import_scaled_dot_product_attention, - "sum": import_reduce("sum"), - "mean": import_reduce("mean"), - "variance": import_reduce("variance"), - "min": import_reduce("min"), - "max": import_reduce("max"), - "cat": import_cat, - "split_with_sizes": import_split_with_sizes, - "chunk": import_chunk, - "ones": import_ones, - "pad": import_pad, - "pow": import_pow, - "aten.diff.default": import_diff, - "slice": import_slice, - "index": import_index, - "gather": import_gather, - "embedding": import_embedding, - "conv1d": import_conv1d, - "layer_norm": import_layer_norm, - "rms_norm": import_rms_norm, - "group_norm": import_group_norm, - "batch_norm": import_batch_norm, - "contiguous": import_contiguous, - "getitem": import_getitem, -} diff --git a/python/src/transpile/capture/normalize.py b/python/src/transpile/capture/normalize.py deleted file mode 100644 index f9d23b546..000000000 --- a/python/src/transpile/capture/normalize.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import annotations - -from typing import Any - -import torch - - -def normalize_target(target: str) -> str: - if target.startswith("aten.mul_"): - return "multiply_inplace" - if target.startswith("aten._assert_tensor_metadata"): - return "identity" - if target.startswith("aten.lift_fresh_copy"): - return "identity" - if target.startswith("aten.clone"): - return "identity" - if target.startswith("aten.alias"): - return "identity" - if target.startswith("aten.detach"): - return "identity" - if target.startswith("aten.arange.start"): - return "arange" - if target.startswith("aten.arange"): - return "arange" - if target.startswith("aten.add"): - return "add" - if target.startswith("aten.sub"): - return "subtract" - if target.startswith("aten.mul"): - return "multiply" - if target.startswith("aten.div"): - return "divide" - if target.startswith("aten.to.dtype"): - return "precision_cast" - if target.startswith("aten.type_as"): - return "type_as" - if target.startswith("aten.neg"): - return "negate" - if target.startswith("aten.ne.") or target == "aten.ne": - return "not_equal" - if target.startswith("aten.abs"): - return "abs" - if target.startswith("aten.expand"): - return "expand" - if target.startswith("aten.exp"): - return "scalar_exp" - if target.startswith("aten.sqrt"): - return "scalar_sqrt" - if target.startswith("aten.log"): - return "scalar_log" - if target.startswith("aten.rsqrt"): - return "rsqrt" - if target.startswith("aten.cos"): - return "cos" - if target.startswith("aten.sin"): - return "sin" - if target in ("aten.view.default", "aten.reshape.default", "aten._unsafe_view.default"): - return "reshape" - if target == "aten.flatten.using_ints": - return "flatten" - if target.startswith("aten.unsqueeze"): - return "unsqueeze" - if target == "aten.t.default" or target.startswith("aten.transpose"): - return "transpose" - if target.startswith("aten.permute"): - return "permute" - if target.startswith("aten.contiguous"): - return "contiguous" - if target.startswith("aten.mm") or target.startswith("aten.matmul") or target.startswith("aten.bmm"): - return "matmul" - if target.startswith("aten.linear"): - return "linear" - if target.startswith("aten.addmm"): - return "addmm" - if target.startswith("aten.relu"): - return "relu" - if target.startswith("aten.silu"): - return "silu" - if target == "aten.gelu.erf": - return "gelu_erf" - if target.startswith("aten.gelu"): - return "gelu" - if target.startswith("aten.sigmoid"): - return "sigmoid" - if target.startswith("aten.softplus"): - return "softplus" - if target.startswith("aten.tanh"): - return "tanh" - if target.startswith("aten.softmax"): - return "softmax" - if target.startswith("aten.scaled_dot_product_attention"): - return "scaled_dot_product_attention" - if target.startswith("aten.sum"): - return "sum" - if target.startswith("aten.mean"): - return "mean" - if target.startswith("aten.var"): - return "variance" - if target.startswith("aten.min") or target.startswith("aten.amin"): - return "min" - if target.startswith("aten.max") or target.startswith("aten.amax"): - return "max" - if target.startswith("aten.cat"): - return "cat" - if target.startswith("aten.split_with_sizes"): - return "split_with_sizes" - if target.startswith("aten.chunk"): - return "chunk" - if target.startswith("aten.ones"): - return "ones" - if target.startswith("aten.slice"): - return "slice" - if target.startswith("aten.select"): - return "index" - if target.startswith("aten.gather"): - return "gather" - if target.startswith("aten.pad"): - return "pad" - if target.startswith("aten.embedding"): - return "embedding" - if target.startswith("aten.conv1d"): - return "conv1d" - if target.startswith("aten.pow"): - return "pow" - if target.startswith("aten.layer_norm"): - return "layer_norm" - if target.startswith("aten.native_layer_norm"): - return "layer_norm" - if target.startswith("aten.group_norm"): - return "group_norm" - if target.startswith("aten.batch_norm") or target.startswith("aten._native_batch_norm_legit_no_training"): - return "batch_norm" - if target.startswith("aten.rms_norm"): - return "rms_norm" - if target == "": - return "getitem" - return target - - -def dtype_to_ir(dtype: Any | None) -> str | None: - if dtype is None: - return None - - dtype_map = { - torch.float16: "fp16", - torch.float32: "fp32", - torch.float64: "fp64", - torch.bfloat16: "bf16", - torch.int8: "int8", - torch.int16: "int16", - torch.int32: "int32", - torch.int64: "int64", - torch.bool: "bool", - } - return dtype_map.get(dtype, str(dtype)) diff --git a/python/src/transpile/capture/ops.py b/python/src/transpile/capture/ops.py deleted file mode 100644 index b0340e4ed..000000000 --- a/python/src/transpile/capture/ops.py +++ /dev/null @@ -1,159 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class OpSchema: - name: str - num_inputs: int | None = None - attrs: tuple[str, ...] = () - backend_op: str | None = None - is_layout: bool = False - is_reduction: bool = False - is_fusible: bool = False - - -# Small canonical op registry for the transpiler. -# This is metadata for canonical IR ops, not another IR layer. - -OPS: tuple[OpSchema, ...] = ( - OpSchema("add", num_inputs=2, backend_op="add", is_fusible=True), - OpSchema("add_clipped", num_inputs=2, backend_op="add_clipped", is_fusible=True), - OpSchema("subtract", num_inputs=2, backend_op="subtract", is_fusible=True), - OpSchema("multiply", num_inputs=2, backend_op="multiply", is_fusible=True), - OpSchema("divide", num_inputs=2, backend_op="divide", is_fusible=True), - OpSchema("not_equal", num_inputs=2, backend_op="not_equal"), - OpSchema("abs", num_inputs=1, backend_op="abs", is_fusible=True), - OpSchema("pow", num_inputs=1, attrs=("exponent",), backend_op="pow"), - OpSchema("negate", num_inputs=1, is_fusible=True), - OpSchema("scalar_add", num_inputs=1, attrs=("value",), backend_op="scalar_add", is_fusible=True), - OpSchema("scalar_subtract", num_inputs=1, attrs=("value",), backend_op="scalar_subtract", is_fusible=True), - OpSchema("scalar_subtract_reverse", num_inputs=1, attrs=("value",)), - OpSchema("scalar_multiply", num_inputs=1, attrs=("value",), backend_op="scalar_multiply", is_fusible=True), - OpSchema("scalar_divide", num_inputs=1, attrs=("value",), backend_op="scalar_divide", is_fusible=True), - OpSchema("scalar_divide_reverse", num_inputs=1, attrs=("value",)), - OpSchema("scalar_not_equal", num_inputs=1, attrs=("value",), backend_op="scalar_not_equal"), - OpSchema("scalar_exp", num_inputs=1, backend_op="scalar_exp", is_fusible=True), - OpSchema("scalar_sqrt", num_inputs=1, backend_op="scalar_sqrt", is_fusible=True), - OpSchema("scalar_log", num_inputs=1, backend_op="scalar_log", is_fusible=True), - OpSchema("scalar_cos", num_inputs=1, backend_op="scalar_cos"), - OpSchema("scalar_sin", num_inputs=1, backend_op="scalar_sin"), - OpSchema("precision_cast", num_inputs=1, attrs=("dtype",), backend_op="precision_cast"), - OpSchema("view", num_inputs=1, attrs=("shape",), backend_op="reshape", is_layout=True), - OpSchema("permute", num_inputs=1, attrs=("permutation",), backend_op="permute", is_layout=True), - OpSchema("slice", num_inputs=1, attrs=("axis", "start", "end", "step"), backend_op="slice"), - OpSchema("split_with_sizes", num_inputs=1, attrs=("sizes", "axis")), - OpSchema("chunk", num_inputs=1, attrs=("chunks", "axis")), - OpSchema("ones", num_inputs=0, attrs=("shape", "dtype")), - OpSchema("index", num_inputs=1, attrs=("axis", "index_value"), backend_op="index"), - OpSchema("cat", num_inputs=None, attrs=("axis",), backend_op="cat"), - OpSchema("matmul", num_inputs=2, backend_op="matmul"), - OpSchema("linear", num_inputs=None, attrs=("has_bias",), backend_op="matmul"), - OpSchema("addmm", num_inputs=3), - OpSchema("gather", num_inputs=2, backend_op="gather"), - OpSchema("embedding", num_inputs=2, backend_op="embedding_from_tensor"), - OpSchema("sum", num_inputs=1, attrs=("axis", "keepdim"), backend_op="sum", is_reduction=True), - OpSchema("mean", num_inputs=1, attrs=("axis", "keepdim"), backend_op="mean", is_reduction=True), - OpSchema("variance", num_inputs=1, attrs=("axis", "keepdim"), backend_op="variance", is_reduction=True), - OpSchema("min", num_inputs=1, attrs=("axis", "keepdim"), backend_op="min", is_reduction=True), - OpSchema("max", num_inputs=1, attrs=("axis", "keepdim"), backend_op="max", is_reduction=True), - OpSchema("relu", num_inputs=1, backend_op="relu", is_fusible=True), - OpSchema("silu", num_inputs=1, backend_op="silu", is_fusible=True), - OpSchema("gelu", num_inputs=1, backend_op="gelu", is_fusible=True), - OpSchema("gelu_erf", num_inputs=1, backend_op="gelu_erf", is_fusible=True), - OpSchema("sigmoid", num_inputs=1, backend_op="sigmoid", is_fusible=True), - OpSchema("tanh", num_inputs=1, backend_op="tanh", is_fusible=True), - OpSchema("softplus", num_inputs=1), - OpSchema("softmax", num_inputs=1, attrs=("axis",), backend_op="softmax"), - OpSchema("layer_norm", num_inputs=None, attrs=("eps",), backend_op="layer_norm"), - OpSchema("group_norm", num_inputs=3, attrs=("num_groups", "eps"), backend_op="group_norm"), - OpSchema("batch_norm", num_inputs=5, attrs=("axis", "eps"), backend_op="batch_norm"), - OpSchema("conv1d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv1d"), - OpSchema("pad", num_inputs=1, attrs=("pads", "value", "mode")), - - # CUSTOM / FUSED OPS - OpSchema("rms_norm", num_inputs=2, attrs=("eps",), backend_op="rms_norm"), - OpSchema("rope", num_inputs=1, attrs=("theta", "position_offset"), backend_op="rope"), - OpSchema( - "attention", - num_inputs=3, - attrs=("scale", "is_causal", "window_size", "mask", "additive_mask"), - backend_op="attention", - ), - OpSchema( - "attention_block", - num_inputs=None, - attrs=("scale", "is_causal", "window_size", "has_gate", "has_bias", "attention_output_shape"), - is_fusible=True, - ), - OpSchema( - "gated_deltanet_prefill", - num_inputs=None, - attrs=( - "num_k_heads", - "num_v_heads", - "key_dim", - "value_dim", - "eps", - "chunk_size", - "has_z", - "has_dt_bias", - "has_a_log", - "has_conv", - ), - is_fusible=True, - ), - OpSchema( - "gated_deltanet_decode", - num_inputs=None, - attrs=( - "num_k_heads", - "num_v_heads", - "key_dim", - "value_dim", - "eps", - "has_z", - "has_dt_bias", - "has_a_log", - "has_conv", - ), - is_fusible=True, - ), -) - - -ALL_OPS: dict[str, OpSchema] = {op.name: op for op in OPS} - - -# Import/capture spellings reduced into the canonical registry above. -ALIASES: dict[str, str] = { - "reshape": "view", - "flatten": "view", - "unsqueeze": "view", - "expand": "view", - "contiguous": "view", - "identity": "view", - "transpose": "permute", - "concat": "cat", - "type_as": "precision_cast", - "scaled_dot_product_attention": "attention", - "cos": "scalar_cos", - "sin": "scalar_sin", -} - - -PRECISION_OP = "precision_cast" -SUPPORTED_PRECISIONS: tuple[str, ...] = ("int4", "int8", "fp16", "fp32") - - -def canonicalize_op(name: str) -> str: - return ALIASES.get(name, name) - - -def get_op(name: str) -> OpSchema: - return ALL_OPS[canonicalize_op(name)] - - -def has_op(name: str) -> bool: - return canonicalize_op(name) in ALL_OPS diff --git a/python/src/transpile/capture/weight_binding.py b/python/src/transpile/capture/weight_binding.py deleted file mode 100644 index 29c07dc81..000000000 --- a/python/src/transpile/capture/weight_binding.py +++ /dev/null @@ -1,185 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import json -import os -from pathlib import Path -import re - - -@dataclass(frozen=True) -class WeightBinding: - path: str - kind: str # "weight" | "embedding" - source_name: str - - -_PROJECT_ROOT = Path(__file__).resolve().parents[3] - - -def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: - candidates: list[str] = [] - raw = model_name_or_path.strip() - if not raw: - return candidates - - def _add(name: str) -> None: - name = name.strip().lower() - if name and name not in candidates: - candidates.append(name) - - _add(raw.split("/")[-1]) - - path = Path(raw) - parts = path.parts - for part in parts: - if part.startswith("models--"): - _add(part[len("models--"):].split("--")[-1]) - break - - return candidates - - -def _default_weights_dir_for_model_name(model_name_or_path: str) -> str | None: - if not model_name_or_path: - return None - for model_dir_name in _candidate_model_dir_names(model_name_or_path): - candidate = _PROJECT_ROOT / "weights" / model_dir_name - if candidate.exists(): - return str(candidate) - return None - - -def _normalized_source_candidates(source_name: str) -> list[str]: - candidates: list[str] = [] - - def _add(name: str) -> None: - if name and name not in candidates: - candidates.append(name) - - raw = source_name.strip() - _add(raw) - - for prefix in ("p_", "b_", "c_"): - if raw.startswith(prefix): - _add(raw[len(prefix):]) - - stripped = candidates[-1] - - if stripped.startswith("module.backbone."): - tail = stripped[len("module.backbone."):] - _add(f"model.{tail}") - _add(f"model.language_model.{tail}") - if tail.startswith("layers."): - _add(f"model.{tail}") - _add(f"model.language_model.{tail}") - - layer_match = re.match( - r"^(?:module_)?backbone_layers_slice_none__\d+__none____modules__(\d+)___(.+)$", - stripped, - ) - if layer_match: - layer_index = int(layer_match.group(1)) - tail = layer_match.group(2) - tail_map = { - "self_attn_q_proj_weight": "self_attn.q_proj.weight", - "self_attn_k_proj_weight": "self_attn.k_proj.weight", - "self_attn_v_proj_weight": "self_attn.v_proj.weight", - "self_attn_o_proj_weight": "self_attn.o_proj.weight", - "self_attn_q_norm_weight": "self_attn.q_norm.weight", - "self_attn_k_norm_weight": "self_attn.k_norm.weight", - "mlp_gate_proj_weight": "mlp.gate_proj.weight", - "mlp_up_proj_weight": "mlp.up_proj.weight", - "mlp_down_proj_weight": "mlp.down_proj.weight", - "input_layernorm_weight": "input_layernorm.weight", - "post_attention_layernorm_weight": "post_attention_layernorm.weight", - "linear_attn_in_proj_qkv_weight": "linear_attn.in_proj_qkv.weight", - "linear_attn_conv1d_weight": "linear_attn.conv1d.weight", - "linear_attn_norm_weight": "linear_attn.norm.weight", - "linear_attn_dt_bias": "linear_attn.dt_bias", - "linear_attn_A_log": "linear_attn.A_log", - } - dotted_tail = tail_map.get(tail) - if dotted_tail is not None: - _add(f"model.layers.{layer_index}.{dotted_tail}") - _add(f"model.language_model.layers.{layer_index}.{dotted_tail}") - - embed_map = { - "module_backbone_embed_tokens_weight": [ - "model.embed_tokens.weight", - "model.language_model.embed_tokens.weight", - ], - "module_backbone_embed_tokens_per_layer_weight": [ - "model.embed_tokens_per_layer.weight", - "model.language_model.embed_tokens_per_layer.weight", - ], - "module_backbone_per_layer_model_projection_weight": [ - "model.per_layer_model_projection.weight", - "model.language_model.per_layer_model_projection.weight", - ], - "module_backbone_norm_weight": [ - "model.norm.weight", - "model.language_model.norm.weight", - ], - "module_model_lm_head_weight": [ - "lm_head.weight", - ], - } - for key, mapped in embed_map.items(): - if stripped == key: - for item in mapped: - _add(item) - - return candidates - - -def resolve_transpile_weights_dir(graph_meta: dict[str, object]) -> str | None: - explicit = graph_meta.get("weights_dir") - if isinstance(explicit, str) and explicit: - return explicit - - family = str(graph_meta.get("adapter_family", "")).upper() - family_env = f"CACTUS_TRANSPILER_WEIGHTS_DIR_{family}" - if family and family_env in os.environ and os.environ[family_env]: - return os.environ[family_env] - - generic = os.environ.get("CACTUS_TRANSPILER_WEIGHTS_DIR") - if generic: - return generic - - model_name_or_path = graph_meta.get("model_name_or_path") - if isinstance(model_name_or_path, str) and model_name_or_path: - default_dir = _default_weights_dir_for_model_name(model_name_or_path) - if default_dir: - return default_dir - return None - - -def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> WeightBinding | None: - if not weights_dir: - return None - root = Path(weights_dir) - if not root.exists(): - return None - manifest_path = root / "weights_manifest.json" - if not manifest_path.exists(): - return None - try: - manifest = json.loads(manifest_path.read_text()) - except Exception: - return None - if not isinstance(manifest, dict): - return None - for candidate_name in _normalized_source_candidates(source_name): - entry = manifest.get(candidate_name) - if not isinstance(entry, dict): - continue - filename = entry.get("filename") - kind = entry.get("kind", "weight") - if not isinstance(filename, str) or not isinstance(kind, str): - continue - candidate = root / filename - if not candidate.exists(): - continue - return WeightBinding(path=str(candidate), kind=kind, source_name=candidate_name) - return None diff --git a/python/src/transpile/capture_pytorch.py b/python/src/transpile/capture_pytorch.py deleted file mode 100644 index 7a9bb3cd8..000000000 --- a/python/src/transpile/capture_pytorch.py +++ /dev/null @@ -1,302 +0,0 @@ -from dataclasses import dataclass -import inspect -from typing import Any -from typing import Iterable - -import torch -from torch.export import export - -from src.transpile.graph_ir import IRGraph - - -class CapturePhaseError(RuntimeError): - def __init__(self, phase: str, message: str, *, cause: Exception | None = None): - super().__init__(f"[capture:{phase}] {message}") - self.phase = phase - self.cause = cause - - -@dataclass -class TensorMetadata: - shape: tuple[Any, ...] - dtype: Any - stride: tuple[Any, ...] | None = None - device: Any | None = None - - -@dataclass -class CapturedModel: - exported_program: Any - graph_module: Any - graph: Any - state_dict: dict[str, Any] - ir_graph: IRGraph - source_module: Any - example_args: tuple[Any, ...] - example_kwargs: dict[str, Any] - strict: bool - transpile_metadata: dict[str, Any] - - def named_parameters(self): - return self.exported_program.named_parameters() - - def named_buffers(self): - return self.exported_program.named_buffers() - - def parameters_dict(self) -> dict[str, Any]: - return dict(self.named_parameters()) - - def buffers_dict(self) -> dict[str, Any]: - return dict(self.named_buffers()) - - def placeholders(self) -> list[Any]: - return [node for node in self.graph.nodes if node.op == "placeholder"] - - def outputs(self) -> list[Any]: - return [node for node in self.graph.nodes if node.op == "output"] - - -def _normalize_args(args: Any) -> tuple[Any, ...]: - if args is None: - return () - if isinstance(args, tuple): - return args - if isinstance(args, list): - return tuple(args) - return (args,) - - -def _normalize_kwargs(kwargs: dict[str, Any] | None) -> dict[str, Any]: - return {} if kwargs is None else dict(kwargs) - - -def _clone_example_value(value: Any) -> Any: - if isinstance(value, torch.Tensor): - return value.detach().clone() - if isinstance(value, tuple): - return tuple(_clone_example_value(v) for v in value) - if isinstance(value, list): - return [_clone_example_value(v) for v in value] - if isinstance(value, dict): - return {k: _clone_example_value(v) for k, v in value.items()} - return value - - -def _clone_examples(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[tuple[Any, ...], dict[str, Any]]: - return ( - tuple(_clone_example_value(arg) for arg in args), - {k: _clone_example_value(v) for k, v in kwargs.items()}, - ) - - -def _inject_export_safe_kwargs(model: torch.nn.Module, kwargs: dict[str, Any]) -> dict[str, Any]: - try: - signature = inspect.signature(model.forward) - except (TypeError, ValueError): - return kwargs - - updated = dict(kwargs) - parameters = signature.parameters - - # HF causal LM forwards often default to returning cache objects and ModelOutput - # containers, which torch.export cannot serialize cleanly. Keep the capture - # boundary tensor-only unless the caller explicitly asked otherwise. - if "use_cache" in parameters and "use_cache" not in updated: - updated["use_cache"] = False - if "return_dict" in parameters and "return_dict" not in updated: - updated["return_dict"] = False - return updated - - -def _call_transpile_metadata_provider(module: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: - for attr_name in ("get_transpile_metadata", "transpile_metadata"): - provider = getattr(module, attr_name, None) - if not callable(provider): - continue - try: - metadata = provider(args=args, kwargs=kwargs) - except TypeError: - metadata = provider() - if metadata is None: - return {} - if not isinstance(metadata, dict): - raise TypeError(f"{type(module).__name__}.{attr_name} must return a dict or None") - return metadata - return {} - - -def _collect_transpile_metadata(model: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: - graph_meta: dict[str, Any] = {} - provider_graph_meta: dict[str, dict[str, Any]] = {} - import_hints: list[dict[str, Any]] = [] - seen_modules: set[int] = set() - - for module_path, module in model.named_modules(): - module_id = id(module) - if module_id in seen_modules: - continue - seen_modules.add(module_id) - - metadata = _call_transpile_metadata_provider(module, args, kwargs) - if not metadata: - continue - - provider_key = module_path or type(module).__name__ - provider_meta = metadata.get("graph", {}) - if isinstance(provider_meta, dict) and provider_meta: - provider_graph_meta[provider_key] = dict(provider_meta) - if not module_path: - graph_meta.update(provider_meta) - - raw_hints = metadata.get("import_hints", ()) - for raw_hint in raw_hints: - if not isinstance(raw_hint, dict): - raise TypeError(f"{type(module).__name__} import hints must be dictionaries") - hint = dict(raw_hint) - hint.setdefault("provider", provider_key) - import_hints.append(hint) - - if provider_graph_meta: - graph_meta.setdefault("transpile_metadata_providers", provider_graph_meta) - if import_hints: - graph_meta["import_hint_count"] = len(import_hints) - - if not graph_meta and not import_hints: - return {} - return { - "graph": graph_meta, - "import_hints": import_hints, - } - -def capture_model(model, args, kwargs=None, *, strict=True) -> CapturedModel: - if not isinstance(model, torch.nn.Module): - raise TypeError("model must be a torch.nn.Module") - - normalized_args = _normalize_args(args) - normalized_kwargs = _inject_export_safe_kwargs(model, _normalize_kwargs(kwargs)) - - if not normalized_args and not normalized_kwargs: - raise ValueError("capture_model requires example args or kwargs") - - model = model.eval() - example_args, example_kwargs = _clone_examples(normalized_args, normalized_kwargs) - transpile_metadata = _collect_transpile_metadata(model, example_args, example_kwargs) - - try: - ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) - except Exception as exc: - raise CapturePhaseError( - "export", - f"torch.export failed for model={type(model).__name__} strict={strict}: {exc}", - cause=exc, - ) from exc - from src.transpile.import_ir import import_captured_to_ir - - raw_captured = CapturedModel( - exported_program=ep, - graph_module=ep.graph_module, - graph=ep.graph, - state_dict=dict(ep.state_dict), - ir_graph=IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[]), - source_module=model, - example_args=example_args, - example_kwargs=example_kwargs, - strict=strict, - transpile_metadata=transpile_metadata, - ) - try: - ir_graph = import_captured_to_ir(raw_captured, strict=strict) - except Exception as exc: - raise CapturePhaseError( - "import", - f"failed to import exported graph to IR for model={type(model).__name__} strict={strict}: {exc}", - cause=exc, - ) from exc - - return CapturedModel( - exported_program=ep, - graph_module=ep.graph_module, - graph=ep.graph, - state_dict=dict(ep.state_dict), - ir_graph=ir_graph, - source_module=model, - example_args=example_args, - example_kwargs=example_kwargs, - strict=strict, - transpile_metadata=transpile_metadata, - ) - - -def capture_model_with_fallback(model, args, kwargs=None) -> CapturedModel: - try: - return capture_model(model, args, kwargs=kwargs, strict=True) - except Exception: - return capture_model(model, args, kwargs=kwargs, strict=False) - - -def resolve_attr(root: Any, target: str) -> Any: - obj = root - for atom in target.split("."): - obj = getattr(obj, atom) - return obj - - -def format_target(node: Any) -> str: - return str(node.target) - -def get_tensor_metadata(node: Any) -> TensorMetadata | None: - meta = getattr(node, "meta", {}) or {} - value = meta.get("val") - if value is None: - return None - - if isinstance(value, torch.Tensor): - stride = tuple(int(s) for s in value.stride()) if value.layout == torch.strided else None - return TensorMetadata( - shape=tuple(value.shape), - dtype=value.dtype, - stride=stride, - device=value.device, - ) - - shape = tuple(getattr(value, "shape", ())) - dtype = getattr(value, "dtype", None) - stride_value = getattr(value, "stride", None) - stride = tuple(stride_value) if isinstance(stride_value, Iterable) else None - device = getattr(value, "device", None) - if not shape and dtype is None and stride is None and device is None: - return None - return TensorMetadata(shape=shape, dtype=dtype, stride=stride, device=device) - - -def get_shape(node: Any) -> tuple[Any, ...] | None: - metadata = get_tensor_metadata(node) - return None if metadata is None else metadata.shape - - -def get_dtype(node: Any) -> Any | None: - metadata = get_tensor_metadata(node) - return None if metadata is None else metadata.dtype - - -def dump_graph(captured: CapturedModel, *, include_meta: bool = True) -> str: - lines: list[str] = [] - for index, node in enumerate(captured.graph.nodes): - lines.append(f"[{index}] {node.op} {node.name} target={node.target}") - lines.append(f" args={node.args}") - lines.append(f" kwargs={node.kwargs}") - if include_meta: - tensor_meta = get_tensor_metadata(node) - if tensor_meta is not None: - lines.append( - " tensor_meta=" - f"shape={tensor_meta.shape} dtype={tensor_meta.dtype} " - f"stride={tensor_meta.stride} device={tensor_meta.device}" - ) - else: - lines.append(f" meta={getattr(node, 'meta', {})}") - return "\n".join(lines) - - -def print_graph(captured: CapturedModel, *, include_meta: bool = True) -> None: - print(dump_graph(captured, include_meta=include_meta)) diff --git a/python/src/transpile/fusion/__init__.py b/python/src/transpile/fusion/__init__.py deleted file mode 100644 index 9a8dc325a..000000000 --- a/python/src/transpile/fusion/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -from src.transpile.fusion.conv import ConvModuleMatch -from src.transpile.fusion.conv import match_conv_module -from src.transpile.fusion.attention import AttentionBlockMatch -from src.transpile.fusion.attention import AttentionMatch -from src.transpile.fusion.attention import SelfAttentionBlockMatch -from src.transpile.fusion.attention import match_attention -from src.transpile.fusion.attention import match_attention_block -from src.transpile.fusion.attention import match_self_attention_block -from src.transpile.fusion.deltanet import GatedDeltaNetMatch -from src.transpile.fusion.deltanet import match_gated_deltanet -from src.transpile.fusion.linear import LinearMatch -from src.transpile.fusion.linear import match_linear -from src.transpile.fusion.mlp import GatedMLPMatch -from src.transpile.fusion.mlp import match_gated_mlp -from src.transpile.fusion.rel_pos_bias import RelPosBiasMatch -from src.transpile.fusion.rel_pos_bias import match_rel_pos_bias -from src.transpile.fusion.rms_norm import RMSNormMatch -from src.transpile.fusion.rms_norm import match_rms_norm -from src.transpile.fusion.rope import RoPEMatch -from src.transpile.fusion.rope import match_rope - -__all__ = [ - "ConvModuleMatch", - "AttentionBlockMatch", - "AttentionMatch", - "SelfAttentionBlockMatch", - "GatedDeltaNetMatch", - "GatedMLPMatch", - "LinearMatch", - "RelPosBiasMatch", - "RMSNormMatch", - "RoPEMatch", - "match_attention", - "match_attention_block", - "match_self_attention_block", - "match_conv_module", - "match_gated_deltanet", - "match_gated_mlp", - "match_linear", - "match_rel_pos_bias", - "match_rms_norm", - "match_rope", -] diff --git a/python/src/transpile/fusion/attention.py b/python/src/transpile/fusion/attention.py deleted file mode 100644 index eac7ee9d3..000000000 --- a/python/src/transpile/fusion/attention.py +++ /dev/null @@ -1,550 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.fusion.common import collect_node_ids -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_passthrough -from src.transpile.fusion.linear import match_linear -from src.transpile.fusion.rms_norm import match_rms_norm -from src.transpile.fusion.rope import match_rope - - -@dataclass(frozen=True) -class AttentionMatch: - query_value_id: str - key_value_id: str - value_value_id: str - source_input_value_ids: tuple[str, str, str] - weight_value_ids: tuple[str | None, str | None, str | None] - has_rope: bool - has_qk_norm: bool - has_gqa_repeat: bool - is_causal: bool - scale: float - window_size: int - node_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class AttentionBlockMatch: - attention_node_id: str - query_value_id: str - key_value_id: str - value_value_id: str - mask_value_id: str | None - gate_value_id: str | None - output_projection_weight_value_id: str - output_projection_bias_value_id: str | None - attention_output_shape: tuple[int, ...] - is_causal: bool - additive_mask: bool - scale: float - window_size: int - node_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class SelfAttentionBlockMatch: - attention_node_id: str - hidden_value_id: str - query_weight_value_id: str - query_projection_bias_value_id: str | None - query_add_value_id: str | None - rel_query_add_value_id: str | None - key_weight_value_id: str - key_projection_bias_value_id: str | None - value_weight_value_id: str - value_projection_bias_value_id: str | None - mask_value_id: str | None - relative_key_input_value_id: str | None - relative_key_weight_value_id: str | None - relative_key_projection_bias_value_id: str | None - gate_value_id: str | None - output_projection_weight_value_id: str - output_projection_bias_value_id: str | None - query_shape: tuple[int, ...] - key_shape: tuple[int, ...] - value_shape: tuple[int, ...] - relative_key_shape: tuple[int, ...] | None - attention_output_shape: tuple[int, ...] - is_causal: bool - additive_mask: bool - scale: float - rel_pos_scale: float | None - window_size: int - node_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class _ProjectedAttentionInputMatch: - hidden_value_id: str - weight_value_id: str - projection_bias_value_id: str | None - projected_shape: tuple[int, ...] - add_value_id: str | None - node_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class _RelativeKeyProjectionMatch: - input_value_id: str - weight_value_id: str - projection_bias_value_id: str | None - projected_shape: tuple[int, ...] - node_ids: tuple[str, ...] - - -def match_attention(graph: IRGraph, node: IRNode) -> AttentionMatch | None: - if node.op not in {"attention", "scaled_dot_product_attention"} or len(node.inputs) < 3: - return None - - q_info = _extract_attention_input(graph, node.inputs[0], role="q") - k_info = _extract_attention_input(graph, node.inputs[1], role="k") - v_info = _extract_attention_input(graph, node.inputs[2], role="v") - if q_info is None or k_info is None or v_info is None: - return None - - has_rope = bool(q_info["has_rope"] and k_info["has_rope"]) - has_qk_norm = bool(q_info["has_rms_norm"] and k_info["has_rms_norm"]) - has_gqa = bool(k_info["has_gqa_repeat"] or v_info["has_gqa_repeat"] or node.attrs.get("enable_gqa", False)) - node_ids = {node.id, *q_info["node_ids"], *k_info["node_ids"], *v_info["node_ids"]} - - return AttentionMatch( - query_value_id=node.inputs[0], - key_value_id=node.inputs[1], - value_value_id=node.inputs[2], - source_input_value_ids=(q_info["source_input"], k_info["source_input"], v_info["source_input"]), - weight_value_ids=(q_info.get("weight_value_id"), k_info.get("weight_value_id"), v_info.get("weight_value_id")), - has_rope=has_rope, - has_qk_norm=has_qk_norm, - has_gqa_repeat=has_gqa, - is_causal=bool(node.attrs.get("is_causal", False)), - scale=float(node.attrs.get("scale", 1.0)), - window_size=int(node.attrs.get("window_size", 0)), - node_ids=tuple(sorted(node_ids)), - ) - - -def match_self_attention_block(graph: IRGraph, node: IRNode) -> SelfAttentionBlockMatch | None: - projection = match_linear(graph, node) - if projection is None: - return None - - output_path = _extract_attention_output_path(graph, projection.input_value_id) - if output_path is None: - return None - attention_node = graph.nodes.get(output_path["attention_node_id"]) - if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: - return None - - query_match = _extract_projected_attention_input(graph, attention_node.inputs[0], role="q") - key_match = _extract_projected_attention_input(graph, attention_node.inputs[1], role="k") - value_match = _extract_projected_attention_input(graph, attention_node.inputs[2], role="v") - if query_match is None or key_match is None or value_match is None: - return None - - if not ( - query_match.hidden_value_id == key_match.hidden_value_id == value_match.hidden_value_id - ): - return None - - mask_value_id = attention_node.inputs[3] if len(attention_node.inputs) > 3 else None - rel_query_add_value_id: str | None = None - relative_key_input_value_id: str | None = None - relative_key_weight_value_id: str | None = None - relative_key_projection_bias_value_id: str | None = None - relative_key_shape: tuple[int, ...] | None = None - rel_pos_scale: float | None = None - rel_pos_node_ids: tuple[str, ...] = () - - if mask_value_id is not None: - rel_pos_node = producer(graph, strip_passthrough(graph, mask_value_id)) - if rel_pos_node is not None and rel_pos_node.op == "rel_pos_bias" and len(rel_pos_node.inputs) == 2: - rel_query_match = _extract_projected_attention_input(graph, rel_pos_node.inputs[0], role="q") - relative_key_match = _extract_relative_key_projection(graph, rel_pos_node.inputs[1]) - if rel_query_match is None or relative_key_match is None: - return None - if not ( - rel_query_match.hidden_value_id == query_match.hidden_value_id - and rel_query_match.weight_value_id == query_match.weight_value_id - and rel_query_match.projection_bias_value_id == query_match.projection_bias_value_id - and rel_query_match.projected_shape == query_match.projected_shape - ): - return None - rel_query_add_value_id = rel_query_match.add_value_id - relative_key_input_value_id = relative_key_match.input_value_id - relative_key_weight_value_id = relative_key_match.weight_value_id - relative_key_projection_bias_value_id = relative_key_match.projection_bias_value_id - relative_key_shape = relative_key_match.projected_shape - rel_pos_scale = float(rel_pos_node.attrs.get("scale", 1.0)) - rel_pos_node_ids = collect_node_ids( - rel_pos_node.id, - rel_pos_node.meta.get("rel_pos_bias_nodes", ()), - rel_query_match.node_ids, - relative_key_match.node_ids, - ) - mask_value_id = None - - return SelfAttentionBlockMatch( - attention_node_id=attention_node.id, - hidden_value_id=query_match.hidden_value_id, - query_weight_value_id=query_match.weight_value_id, - query_projection_bias_value_id=query_match.projection_bias_value_id, - query_add_value_id=query_match.add_value_id, - rel_query_add_value_id=rel_query_add_value_id, - key_weight_value_id=key_match.weight_value_id, - key_projection_bias_value_id=key_match.projection_bias_value_id, - value_weight_value_id=value_match.weight_value_id, - value_projection_bias_value_id=value_match.projection_bias_value_id, - mask_value_id=mask_value_id, - relative_key_input_value_id=relative_key_input_value_id, - relative_key_weight_value_id=relative_key_weight_value_id, - relative_key_projection_bias_value_id=relative_key_projection_bias_value_id, - gate_value_id=output_path["gate_value_id"], - output_projection_weight_value_id=projection.weight_value_id, - output_projection_bias_value_id=projection.bias_value_id, - query_shape=query_match.projected_shape, - key_shape=key_match.projected_shape, - value_shape=value_match.projected_shape, - relative_key_shape=relative_key_shape, - attention_output_shape=output_path["attention_output_shape"], - is_causal=bool(attention_node.attrs.get("is_causal", True)), - additive_mask=bool(attention_node.attrs.get("additive_mask", False)), - scale=float(attention_node.attrs.get("scale", 1.0)), - rel_pos_scale=rel_pos_scale, - window_size=int(attention_node.attrs.get("window_size", 0)), - node_ids=collect_node_ids( - attention_node.id, - projection.node_ids, - output_path["node_ids"], - query_match.node_ids, - key_match.node_ids, - value_match.node_ids, - rel_pos_node_ids, - ), - ) - - -def match_attention_block(graph: IRGraph, node: IRNode) -> AttentionBlockMatch | None: - projection = match_linear(graph, node) - if projection is None: - return None - - output_path = _extract_attention_output_path(graph, projection.input_value_id) - if output_path is None: - return None - attention_node = graph.nodes.get(output_path["attention_node_id"]) - if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: - return None - - return AttentionBlockMatch( - attention_node_id=attention_node.id, - query_value_id=attention_node.inputs[0], - key_value_id=attention_node.inputs[1], - value_value_id=attention_node.inputs[2], - mask_value_id=attention_node.inputs[3] if len(attention_node.inputs) > 3 else None, - gate_value_id=output_path["gate_value_id"], - output_projection_weight_value_id=projection.weight_value_id, - output_projection_bias_value_id=projection.bias_value_id, - attention_output_shape=output_path["attention_output_shape"], - is_causal=bool(attention_node.attrs.get("is_causal", True)), - additive_mask=bool(attention_node.attrs.get("additive_mask", False)), - scale=float(attention_node.attrs.get("scale", 1.0)), - window_size=int(attention_node.attrs.get("window_size", 0)), - node_ids=tuple(sorted({attention_node.id, *projection.node_ids, *output_path["node_ids"]})), - ) - - -def _extract_attention_output_path(graph: IRGraph, value_id: str) -> dict[str, object] | None: - current = value_id - gate_value_id: str | None = None - - while True: - node = producer(graph, current) - if node is None: - return None - if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: - current = node.inputs[0] - continue - if node.op == "type_as" and len(node.inputs) >= 1: - current = node.inputs[0] - continue - break - - node = producer(graph, current) - if node is not None and node.op == "multiply" and len(node.inputs) == 2: - for attn_candidate, gate_candidate in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - attn_match = _extract_attention_output_path(graph, attn_candidate) - if attn_match is None: - continue - attn_match["gate_value_id"] = gate_candidate - attn_match["node_ids"] = tuple(sorted({node.id, *attn_match["node_ids"]})) - return attn_match - - reshape_node = producer(graph, current) - if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: - return None - - attention_output_shape = tuple(int(v) for v in reshape_node.attrs.get("shape", ())) - current = reshape_node.inputs[0] - transpose_node = producer(graph, current) - if transpose_node is None: - return None - - if transpose_node.op == "transpose": - if int(transpose_node.attrs.get("dim0", -1)) != 1 or int(transpose_node.attrs.get("dim1", -1)) != 2: - return None - current = transpose_node.inputs[0] - elif transpose_node.op == "permute": - permutation = tuple(int(v) for v in transpose_node.attrs.get("permutation", ())) - if permutation != (0, 2, 1, 3): - return None - current = transpose_node.inputs[0] - else: - return None - - attention_node = producer(graph, current) - if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"}: - return None - - return { - "attention_node_id": attention_node.id, - "gate_value_id": gate_value_id, - "attention_output_shape": attention_output_shape, - "node_ids": tuple(sorted({reshape_node.id, transpose_node.id})), - } - - -def _extract_projected_attention_input(graph: IRGraph, value_id: str, *, role: str) -> _ProjectedAttentionInputMatch | None: - current = strip_passthrough(graph, value_id) - add_node = producer(graph, current) - if add_node is not None and add_node.op == "add" and len(add_node.inputs) == 2: - for projected_candidate, add_candidate in ((add_node.inputs[0], add_node.inputs[1]), (add_node.inputs[1], add_node.inputs[0])): - projected = _extract_transposed_projection(graph, projected_candidate) - if projected is None: - continue - if not _looks_like_attention_addend(graph, add_candidate, projected.projected_shape): - continue - return _ProjectedAttentionInputMatch( - hidden_value_id=projected.hidden_value_id, - weight_value_id=projected.weight_value_id, - projection_bias_value_id=projected.projection_bias_value_id, - projected_shape=projected.projected_shape, - add_value_id=strip_passthrough(graph, add_candidate), - node_ids=collect_node_ids(add_node.id, projected.node_ids), - ) - - if role == "q": - return None - - return _extract_transposed_projection(graph, current) - - -def _extract_transposed_projection(graph: IRGraph, value_id: str) -> _ProjectedAttentionInputMatch | None: - current = strip_passthrough(graph, value_id) - layout_node = producer(graph, current) - if layout_node is None or len(layout_node.inputs) != 1: - return None - - if layout_node.op == "permute": - permutation = tuple(int(v) for v in layout_node.attrs.get("permutation", ())) - if permutation != (0, 2, 1, 3): - return None - elif layout_node.op == "transpose": - if int(layout_node.attrs.get("dim0", -1)) != 1 or int(layout_node.attrs.get("dim1", -1)) != 2: - return None - else: - return None - - projected_value_id = strip_passthrough(graph, layout_node.inputs[0]) - projected_value = graph.values.get(projected_value_id) - if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: - return None - - reshape_node = producer(graph, projected_value_id) - if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: - return None - - linear_node = producer(graph, reshape_node.inputs[0]) - if linear_node is None: - return None - linear = match_linear(graph, linear_node) - if linear is None: - return None - - return _ProjectedAttentionInputMatch( - hidden_value_id=linear.input_value_id, - weight_value_id=linear.weight_value_id, - projection_bias_value_id=linear.bias_value_id, - projected_shape=tuple(int(v) for v in projected_value.shape), - add_value_id=None, - node_ids=collect_node_ids(layout_node.id, reshape_node.id, linear.node_ids), - ) - - -def _extract_relative_key_projection(graph: IRGraph, value_id: str) -> _RelativeKeyProjectionMatch | None: - current = strip_passthrough(graph, value_id) - projected_value = graph.values.get(current) - if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: - return None - - reshape_node = producer(graph, current) - if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: - return None - - linear_node = producer(graph, reshape_node.inputs[0]) - if linear_node is None: - return None - linear = match_linear(graph, linear_node) - if linear is None: - return None - - return _RelativeKeyProjectionMatch( - input_value_id=linear.input_value_id, - weight_value_id=linear.weight_value_id, - projection_bias_value_id=linear.bias_value_id, - projected_shape=tuple(int(v) for v in projected_value.shape), - node_ids=collect_node_ids(reshape_node.id, linear.node_ids), - ) - - -def _looks_like_attention_addend(graph: IRGraph, value_id: str, target_shape: tuple[int, ...]) -> bool: - value = graph.values.get(strip_passthrough(graph, value_id)) - if value is None or value.shape is None: - return False - - shape = tuple(int(v) for v in value.shape) - if len(shape) != 4 or len(target_shape) != 4: - return False - - batch, seq_len, heads, head_dim = target_shape - if shape[0] not in (1, batch) or shape[3] != head_dim: - return False - - # Allow either BHSD-style tensors like [B, H, 1, D] or BSHD-style tensors like [B, 1, H, D]. - if shape[1] == heads and shape[2] in (1, seq_len): - return True - if shape[1] in (1, seq_len) and shape[2] == heads: - return True - return False - - -def _extract_attention_input(graph: IRGraph, value_id: str, *, role: str) -> dict[str, object] | None: - node_ids: set[str] = set() - current = strip_passthrough(graph, value_id) - has_gqa_repeat = False - - while True: - node = producer(graph, current) - if node is None: - return { - "source_input": current, - "weight_value_id": None, - "has_rope": False, - "has_rms_norm": False, - "has_gqa_repeat": has_gqa_repeat, - "node_ids": tuple(sorted(node_ids)), - } - node_ids.add(node.id) - - if node.op in {"reshape", "view", "transpose", "permute"}: - current = node.inputs[0] - continue - - rope = match_rope(graph, current) - if rope is not None: - rope_info = _extract_attention_input(graph, rope.input_value_id, role=role) - if rope_info is None: - return None - rope_info["has_rope"] = True - rope_info["node_ids"] = tuple(sorted(set(rope_info["node_ids"]) | node_ids | set(rope.node_ids))) - rope_info["has_gqa_repeat"] = bool(rope_info["has_gqa_repeat"] or has_gqa_repeat) - return rope_info - - rms = match_rms_norm(graph, node) - if rms is not None: - if role == "v": - return None - rms_info = _extract_attention_input(graph, rms.input_value_id, role=role) - if rms_info is None: - return None - rms_info["has_rms_norm"] = True - rms_info["node_ids"] = tuple(sorted(set(rms_info["node_ids"]) | node_ids | set(rms.node_ids))) - rms_info["has_gqa_repeat"] = bool(rms_info["has_gqa_repeat"] or has_gqa_repeat) - return rms_info - - if _looks_like_gqa_repeat(graph, current): - has_gqa_repeat = True - current = _unwrap_gqa_repeat(graph, current) - if current is None: - return None - continue - - linear_node = producer(graph, current) - if linear_node is None: - return { - "source_input": strip_passthrough(graph, current), - "weight_value_id": None, - "has_rope": False, - "has_rms_norm": False, - "has_gqa_repeat": has_gqa_repeat, - "node_ids": tuple(sorted(node_ids)), - } - linear = match_linear(graph, linear_node) - if linear is None: - return { - "source_input": strip_passthrough(graph, current), - "weight_value_id": None, - "has_rope": False, - "has_rms_norm": False, - "has_gqa_repeat": has_gqa_repeat, - "node_ids": tuple(sorted(node_ids)), - } - return { - "source_input": strip_passthrough(graph, linear.input_value_id), - "weight_value_id": linear.weight_value_id, - "has_rope": False, - "has_rms_norm": False, - "has_gqa_repeat": has_gqa_repeat, - "node_ids": tuple(sorted(set(node_ids) | set(linear.node_ids))), - } - - -def _looks_like_gqa_repeat(graph: IRGraph, value_id: str) -> bool: - return _unwrap_gqa_repeat(graph, value_id) is not None - - -def _unwrap_gqa_repeat(graph: IRGraph, value_id: str) -> str | None: - current = strip_passthrough(graph, value_id) - reshape_node = producer(graph, current) - if reshape_node is None or reshape_node.op not in {"reshape", "view"}: - return None - current = reshape_node.inputs[0] - expand_node = producer(graph, current) - if expand_node is None or expand_node.op != "expand": - return None - current = expand_node.inputs[0] - while True: - node = producer(graph, current) - if node is None: - return None - if node.op == "slice": - current = node.inputs[0] - continue - if node.op != "unsqueeze": - return None - if int(node.attrs.get("dim", -999)) != 2: - return None - break - base_value_id = strip_passthrough(graph, node.inputs[0]) - base_shape = graph.values.get(base_value_id).shape if graph.values.get(base_value_id) is not None else None - target_shape = tuple(int(v) for v in expand_node.attrs.get("shape", ())) - if base_shape is None or len(base_shape) != 4 or len(target_shape) != 5: - return None - expected = (int(base_shape[0]), int(base_shape[1]), int(target_shape[2]), int(base_shape[2]), int(base_shape[3])) - if expected != target_shape: - return None - return base_value_id diff --git a/python/src/transpile/fusion/common.py b/python/src/transpile/fusion/common.py deleted file mode 100644 index 828783b7e..000000000 --- a/python/src/transpile/fusion/common.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode - - -@dataclass(frozen=True) -class FusionMatch: - kind: str - anchor_node_id: str - node_ids: tuple[str, ...] - value_ids: tuple[str, ...] - attrs: dict[str, object] - - -def producer(graph: IRGraph, value_id: str) -> IRNode | None: - value = graph.values.get(value_id) - if value is None or value.producer is None: - return None - return graph.nodes.get(value.producer) - - -def strip_passthrough(graph: IRGraph, value_id: str) -> str: - current = value_id - while True: - node = producer(graph, current) - if node is None: - return current - if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: - current = node.inputs[0] - continue - if node.op == "type_as" and len(node.inputs) >= 1: - current = node.inputs[0] - continue - return current - - -def strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: - current = value_id - while True: - node = producer(graph, current) - if node is None: - return current - if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: - current = node.inputs[0] - continue - if node.op == "type_as" and len(node.inputs) >= 1: - current = node.inputs[0] - continue - return current - - -def collect_node_ids(*parts: Any) -> tuple[str, ...]: - node_ids: set[str] = set() - for part in parts: - if part is None: - continue - if isinstance(part, str): - node_ids.add(part) - continue - if isinstance(part, IRNode): - node_ids.add(part.id) - continue - if isinstance(part, (list, tuple, set)): - for item in part: - if isinstance(item, str): - node_ids.add(item) - elif isinstance(item, IRNode): - node_ids.add(item.id) - return tuple(sorted(node_ids)) diff --git a/python/src/transpile/fusion/conv.py b/python/src/transpile/fusion/conv.py deleted file mode 100644 index 1f75d6e79..000000000 --- a/python/src/transpile/fusion/conv.py +++ /dev/null @@ -1,123 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from src.transpile.fusion.common import collect_node_ids -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_passthrough -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode - - -@dataclass(frozen=True) -class ConvModuleMatch: - input_value_id: str - pointwise1_weight_value_id: str - pointwise1_bias_value_id: str | None - depthwise_weight_value_id: str - depthwise_bias_value_id: str | None - batch_norm_weight_value_id: str - batch_norm_bias_value_id: str - batch_norm_running_mean_value_id: str - batch_norm_running_var_value_id: str - pointwise2_weight_value_id: str - pointwise2_bias_value_id: str | None - eps: float - depthwise_kernel_size: int - depthwise_padding: int - node_ids: tuple[str, ...] - - -def match_conv_module(graph: IRGraph, node: IRNode) -> ConvModuleMatch | None: - if node.op != "permute" or len(node.inputs) != 1 or len(node.outputs) != 1: - return None - if tuple(node.attrs.get("permutation", ())) != (0, 2, 1): - return None - - pointwise2 = producer(graph, strip_passthrough(graph, node.inputs[0])) - if not _is_pointwise_conv1d(pointwise2): - return None - - silu = producer(graph, strip_passthrough(graph, pointwise2.inputs[0])) - if silu is None or silu.op != "silu" or len(silu.inputs) != 1: - return None - - batch_norm = producer(graph, strip_passthrough(graph, silu.inputs[0])) - if batch_norm is None or batch_norm.op != "batch_norm" or len(batch_norm.inputs) != 5: - return None - - depthwise = producer(graph, strip_passthrough(graph, batch_norm.inputs[0])) - if not _is_same_depthwise_conv1d(depthwise): - return None - - glu = producer(graph, strip_passthrough(graph, depthwise.inputs[0])) - if glu is None or glu.op != "glu" or len(glu.inputs) != 1: - return None - if int(glu.attrs.get("axis", -1)) not in {1, -2}: - return None - - pointwise1 = producer(graph, strip_passthrough(graph, glu.inputs[0])) - if not _is_pointwise_conv1d(pointwise1): - return None - - input_permute = producer(graph, strip_passthrough(graph, pointwise1.inputs[0])) - if input_permute is None or input_permute.op != "permute" or len(input_permute.inputs) != 1: - return None - if tuple(input_permute.attrs.get("permutation", ())) != (0, 2, 1): - return None - - depthwise_weight_shape = graph.values.get(depthwise.inputs[1]).shape if depthwise.inputs[1] in graph.values else None - if depthwise_weight_shape is None or len(depthwise_weight_shape) != 3: - return None - depthwise_kernel_size = int(depthwise_weight_shape[2]) - - return ConvModuleMatch( - input_value_id=strip_passthrough(graph, input_permute.inputs[0]), - pointwise1_weight_value_id=pointwise1.inputs[1], - pointwise1_bias_value_id=pointwise1.inputs[2] if len(pointwise1.inputs) > 2 else None, - depthwise_weight_value_id=depthwise.inputs[1], - depthwise_bias_value_id=depthwise.inputs[2] if len(depthwise.inputs) > 2 else None, - batch_norm_weight_value_id=batch_norm.inputs[1], - batch_norm_bias_value_id=batch_norm.inputs[2], - batch_norm_running_mean_value_id=batch_norm.inputs[3], - batch_norm_running_var_value_id=batch_norm.inputs[4], - pointwise2_weight_value_id=pointwise2.inputs[1], - pointwise2_bias_value_id=pointwise2.inputs[2] if len(pointwise2.inputs) > 2 else None, - eps=float(batch_norm.attrs.get("eps", 1e-5)), - depthwise_kernel_size=depthwise_kernel_size, - depthwise_padding=int(depthwise.attrs.get("padding", 0)), - node_ids=collect_node_ids( - node, - pointwise2, - silu, - batch_norm, - depthwise, - glu, - pointwise1, - input_permute, - ), - ) - - -def _is_pointwise_conv1d(node: IRNode | None) -> bool: - return ( - node is not None - and node.op == "conv1d" - and len(node.inputs) >= 2 - and int(node.attrs.get("stride", 1)) == 1 - and int(node.attrs.get("padding", 0)) == 0 - and int(node.attrs.get("dilation", 1)) == 1 - and int(node.attrs.get("groups", 1)) == 1 - ) - - -def _is_same_depthwise_conv1d(node: IRNode | None) -> bool: - if node is None or node.op != "conv1d" or len(node.inputs) < 2: - return False - groups = int(node.attrs.get("groups", 1)) - padding = int(node.attrs.get("padding", 0)) - stride = int(node.attrs.get("stride", 1)) - dilation = int(node.attrs.get("dilation", 1)) - if stride != 1 or dilation != 1: - return False - return groups > 1 and padding >= 0 diff --git a/python/src/transpile/fusion/deltanet.py b/python/src/transpile/fusion/deltanet.py deleted file mode 100644 index cef656a5d..000000000 --- a/python/src/transpile/fusion/deltanet.py +++ /dev/null @@ -1,357 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_passthrough -from src.transpile.fusion.linear import match_linear -from src.transpile.fusion.rms_norm import match_rms_norm -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode - - -@dataclass(frozen=True) -class GatedDeltaNetMatch: - output_value_id: str - normalized_input_value_id: str - qkv_weight_value_id: str - a_weight_value_id: str - b_weight_value_id: str - norm_weight_value_id: str - z_weight_value_id: str | None - dt_bias_value_id: str | None - a_log_value_id: str | None - conv_weight_value_id: str | None - num_k_heads: int - num_v_heads: int - key_dim: int - value_dim: int - eps: float - chunk_size: int - mode: str - node_ids: tuple[str, ...] - - -def match_gated_deltanet(graph: IRGraph, node: IRNode) -> GatedDeltaNetMatch | None: - if node.op not in {"view", "reshape"} or len(node.outputs) != 1: - return None - output_value = graph.values.get(node.outputs[0]) - if output_value is None or output_value.shape is None or len(output_value.shape) != 3: - return None - - body_value_id = strip_passthrough(graph, node.inputs[0]) - body_node = producer(graph, body_value_id) - if body_node is None: - return None - - z_weight_value_id: str | None = None - rms_match = None - normalized_input_value_id: str | None = None - node_ids: set[str] = {node.id} - - if body_node.op == "multiply" and len(body_node.inputs) == 2: - for rms_value_id, z_value_id in ((body_node.inputs[0], body_node.inputs[1]), (body_node.inputs[1], body_node.inputs[0])): - rms_match = _match_rms_from_value(graph, rms_value_id) - if rms_match is None: - continue - z_weight_value_id, normalized_input_value_id, z_node_ids = _extract_z_branch(graph, z_value_id) - if normalized_input_value_id is None: - continue - node_ids.update(z_node_ids) - node_ids.add(body_node.id) - break - if rms_match is None or normalized_input_value_id is None: - return None - else: - rms_match = _match_rms_from_value(graph, body_value_id) - if rms_match is None: - return None - - y2d_value_id = rms_match.input_value_id - if rms_match.weight_value_id is None: - return None - norm_weight_value_id = rms_match.weight_value_id - eps = float(rms_match.eps) - node_ids.update(rms_match.node_ids) - - if normalized_input_value_id is None: - normalized_input_value_id = _infer_normalized_input_from_y_branch(graph, y2d_value_id) - if normalized_input_value_id is None: - return None - - qkv_info = _find_qkv_linear(graph, normalized_input_value_id) - if qkv_info is None: - return None - qkv_weight_value_id, conv_weight_value_id, num_k_heads, key_dim, num_v_heads, value_dim, qkv_node_ids = qkv_info - node_ids.update(qkv_node_ids) - - a_info = _find_a_linear(graph, normalized_input_value_id) - b_info = _find_b_linear(graph, normalized_input_value_id) - if a_info is None or b_info is None: - return None - a_weight_value_id, dt_bias_value_id, a_log_value_id, a_node_ids = a_info - b_weight_value_id, b_node_ids = b_info - node_ids.update(a_node_ids) - node_ids.update(b_node_ids) - - mode, chunk_size = _classify_mode(graph, y2d_value_id, match_node_ids=node_ids) - - return GatedDeltaNetMatch( - output_value_id=node.outputs[0], - normalized_input_value_id=normalized_input_value_id, - qkv_weight_value_id=qkv_weight_value_id, - a_weight_value_id=a_weight_value_id, - b_weight_value_id=b_weight_value_id, - norm_weight_value_id=norm_weight_value_id, - z_weight_value_id=z_weight_value_id, - dt_bias_value_id=dt_bias_value_id, - a_log_value_id=a_log_value_id, - conv_weight_value_id=conv_weight_value_id, - num_k_heads=num_k_heads, - num_v_heads=num_v_heads, - key_dim=key_dim, - value_dim=value_dim, - eps=eps, - chunk_size=chunk_size, - mode=mode, - node_ids=tuple(sorted(node_ids)), - ) - - -def _match_rms_from_value(graph: IRGraph, value_id: str): - node = producer(graph, strip_passthrough(graph, value_id)) - if node is None: - return None - return match_rms_norm(graph, node) - - -def _extract_z_branch(graph: IRGraph, value_id: str) -> tuple[str | None, str | None, tuple[str, ...]]: - current = strip_passthrough(graph, value_id) - silu_node = producer(graph, current) - if silu_node is None or silu_node.op != "silu" or len(silu_node.inputs) != 1: - return None, None, () - linear_value_id = strip_passthrough(graph, silu_node.inputs[0]) - linear_node = producer(graph, linear_value_id) - while linear_node is not None and linear_node.op in {"view", "reshape"} and len(linear_node.inputs) == 1: - linear_value_id = strip_passthrough(graph, linear_node.inputs[0]) - linear_node = producer(graph, linear_value_id) - if linear_node is None: - return None, None, () - linear_match = match_linear(graph, linear_node) - if linear_match is None: - return None, None, () - return ( - linear_match.weight_value_id, - linear_match.input_value_id, - tuple(sorted({silu_node.id, *linear_match.node_ids})), - ) - - -def _infer_normalized_input_from_y_branch(graph: IRGraph, value_id: str) -> str | None: - current = strip_passthrough(graph, value_id) - node = producer(graph, current) - visited: set[str] = set() - while node is not None and node.id not in visited: - visited.add(node.id) - for input_value_id in node.inputs: - candidate = producer(graph, strip_passthrough(graph, input_value_id)) - if candidate is None: - continue - linear_match = match_linear(graph, candidate) - if linear_match is not None: - return linear_match.input_value_id - if len(node.inputs) != 1: - return None - node = producer(graph, strip_passthrough(graph, node.inputs[0])) - return None - - -def _find_qkv_linear( - graph: IRGraph, - normalized_input_value_id: str, -) -> tuple[str, str | None, int, int, int, int, tuple[str, ...]] | None: - for node_id in graph.order: - node = graph.nodes[node_id] - linear_match = match_linear(graph, node) - if linear_match is None or linear_match.input_value_id != normalized_input_value_id: - continue - split_node = _find_descendant_op(graph, linear_match.output_value_id, {"split_with_sizes"}, max_depth=8) - if split_node is None: - continue - split_sizes = tuple(int(v) for v in split_node.attrs.get("sizes", ())) - if len(split_sizes) != 3: - continue - - conv_node = _find_descendant_op(graph, linear_match.output_value_id, {"conv1d"}, max_depth=6) - conv_weight_value_id = conv_node.inputs[1] if conv_node is not None and len(conv_node.inputs) >= 2 else None - - q_shape = None - v_shape = None - node_ids = {node.id, split_node.id} - if conv_node is not None: - node_ids.add(conv_node.id) - - split_value = graph.values.get(split_node.outputs[0]) - if split_value is None: - continue - for getitem_node_id in split_value.users: - getitem_node = graph.nodes.get(getitem_node_id) - if getitem_node is None or getitem_node.op != "getitem" or not getitem_node.outputs: - continue - shape = _find_rank4_shape(graph, getitem_node.outputs[0], max_depth=5) - if shape is None: - continue - branch_has_sum = _find_descendant_op(graph, getitem_node.outputs[0], {"sum"}, max_depth=8) is not None - node_ids.add(getitem_node.id) - if branch_has_sum and q_shape is None: - q_shape = shape - elif not branch_has_sum and v_shape is None: - v_shape = shape - - if q_shape is None or v_shape is None: - continue - - num_k_heads = int(q_shape[2]) - key_dim = int(q_shape[3]) - num_v_heads = int(v_shape[2]) - value_dim = int(v_shape[3]) - if num_k_heads <= 0 or key_dim <= 0 or num_v_heads <= 0 or value_dim <= 0: - continue - - return ( - linear_match.weight_value_id, - conv_weight_value_id, - num_k_heads, - key_dim, - num_v_heads, - value_dim, - tuple(sorted(node_ids | set(linear_match.node_ids))), - ) - return None - - -def _find_a_linear( - graph: IRGraph, - normalized_input_value_id: str, -) -> tuple[str, str | None, str | None, tuple[str, ...]] | None: - for node_id in graph.order: - node = graph.nodes[node_id] - linear_match = match_linear(graph, node) - if linear_match is None or linear_match.input_value_id != normalized_input_value_id: - continue - softplus_node = _find_descendant_op(graph, linear_match.output_value_id, {"softplus"}, max_depth=6) - if softplus_node is None: - continue - dt_bias_value_id = None - softplus_input_node = producer(graph, strip_passthrough(graph, softplus_node.inputs[0])) - if softplus_input_node is not None and softplus_input_node.op == "add": - for input_value_id in softplus_input_node.inputs: - value = graph.values.get(input_value_id) - if value is not None and value.producer is None and input_value_id in graph.constants: - dt_bias_value_id = input_value_id - break - a_log_value_id = None - multiply_node = _find_descendant_op(graph, softplus_node.outputs[0], {"multiply"}, max_depth=6) - if multiply_node is not None: - for input_value_id in multiply_node.inputs: - a_log_value_id = _extract_exp_constant_input(graph, input_value_id) - if a_log_value_id is not None: - break - node_ids = {softplus_node.id, *linear_match.node_ids} - if softplus_input_node is not None: - node_ids.add(softplus_input_node.id) - if multiply_node is not None: - node_ids.add(multiply_node.id) - return linear_match.weight_value_id, dt_bias_value_id, a_log_value_id, tuple(sorted(node_ids)) - return None - - -def _find_b_linear(graph: IRGraph, normalized_input_value_id: str) -> tuple[str, tuple[str, ...]] | None: - for node_id in graph.order: - node = graph.nodes[node_id] - linear_match = match_linear(graph, node) - if linear_match is None or linear_match.input_value_id != normalized_input_value_id: - continue - sigmoid_node = _find_descendant_op(graph, linear_match.output_value_id, {"sigmoid"}, max_depth=4) - if sigmoid_node is None: - continue - return linear_match.weight_value_id, tuple(sorted({sigmoid_node.id, *linear_match.node_ids})) - return None - - -def _extract_exp_constant_input(graph: IRGraph, value_id: str) -> str | None: - node = producer(graph, strip_passthrough(graph, value_id)) - if node is None: - value = graph.values.get(value_id) - if value is not None and value.producer is None and value_id in graph.constants: - return value_id - return None - if node.op == "scalar_multiply": - return _extract_exp_constant_input(graph, node.inputs[0]) - if node.op == "scalar_exp": - source = strip_passthrough(graph, node.inputs[0]) - value = graph.values.get(source) - if value is not None and value.producer is None and source in graph.constants: - return source - return None - - -def _find_rank4_shape(graph: IRGraph, value_id: str, *, max_depth: int) -> tuple[int, int, int, int] | None: - frontier = [(value_id, 0)] - seen: set[str] = set() - while frontier: - current, depth = frontier.pop(0) - if current in seen or depth > max_depth: - continue - seen.add(current) - value = graph.values.get(current) - if value is not None and value.shape is not None and len(value.shape) == 4: - shape = tuple(int(dim) for dim in value.shape) - return shape # type: ignore[return-value] - for user_id in graph.values.get(current, None).users if graph.values.get(current, None) is not None else (): - user = graph.nodes.get(user_id) - if user is None or not user.outputs: - continue - frontier.extend((output_id, depth + 1) for output_id in user.outputs) - return None - - -def _find_descendant_op(graph: IRGraph, value_id: str, targets: set[str], *, max_depth: int) -> IRNode | None: - frontier = [(value_id, 0)] - seen: set[str] = set() - while frontier: - current, depth = frontier.pop(0) - if current in seen or depth > max_depth: - continue - seen.add(current) - value = graph.values.get(current) - if value is None: - continue - for user_id in value.users: - user = graph.nodes.get(user_id) - if user is None or not user.outputs: - continue - if user.op in targets: - return user - frontier.extend((output_id, depth + 1) for output_id in user.outputs) - return None - - -def _classify_mode(graph: IRGraph, y2d_value_id: str, *, match_node_ids: set[str]) -> tuple[str, int]: - del y2d_value_id - chunk_size = _infer_prefill_chunk_size(graph, match_node_ids) - # Current exported graphs are uncached/full-sequence captures, so fuse the - # linear-attention block to the prefill kernel by default. Decode should be - # emitted only once the capture path models cache/state explicitly. - return "prefill", chunk_size - - -def _infer_prefill_chunk_size(graph: IRGraph, match_node_ids: set[str]) -> int: - for node_id in match_node_ids: - node = graph.nodes.get(node_id) - if node is None or node.op != "aten.triu.default" or not node.inputs: - continue - ones_value = graph.values.get(node.inputs[0]) - if ones_value is not None and ones_value.shape and len(ones_value.shape) >= 1: - return int(ones_value.shape[-1]) - return 64 diff --git a/python/src/transpile/fusion/linear.py b/python/src/transpile/fusion/linear.py deleted file mode 100644 index df5346730..000000000 --- a/python/src/transpile/fusion/linear.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_passthrough - - -@dataclass(frozen=True) -class LinearMatch: - input_value_id: str - weight_value_id: str - bias_value_id: str | None - output_value_id: str - node_ids: tuple[str, ...] - kind: str - - -def match_linear(graph: IRGraph, node: IRNode) -> LinearMatch | None: - if node.op == "linear" and len(node.inputs) >= 2 and len(node.outputs) == 1: - return LinearMatch( - input_value_id=strip_passthrough(graph, node.inputs[0]), - weight_value_id=node.inputs[1], - bias_value_id=node.inputs[2] if bool(node.attrs.get("has_bias", False)) and len(node.inputs) > 2 else None, - output_value_id=node.outputs[0], - node_ids=(node.id,), - kind="linear", - ) - - if node.op == "addmm" and len(node.inputs) == 3 and len(node.outputs) == 1: - return LinearMatch( - input_value_id=strip_passthrough(graph, node.inputs[1]), - weight_value_id=node.inputs[2], - bias_value_id=node.inputs[0], - output_value_id=node.outputs[0], - node_ids=(node.id,), - kind="addmm", - ) - - if node.op != "add" or len(node.inputs) != 2 or len(node.outputs) != 1: - return None - - for matmul_value_id, bias_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - matmul_node = producer(graph, strip_passthrough(graph, matmul_value_id)) - if matmul_node is None or matmul_node.op != "matmul" or len(matmul_node.inputs) != 2: - continue - return LinearMatch( - input_value_id=strip_passthrough(graph, matmul_node.inputs[0]), - weight_value_id=matmul_node.inputs[1], - bias_value_id=bias_value_id, - output_value_id=node.outputs[0], - node_ids=tuple(sorted({node.id, matmul_node.id})), - kind="matmul_add", - ) - - return None diff --git a/python/src/transpile/fusion/mlp.py b/python/src/transpile/fusion/mlp.py deleted file mode 100644 index 1c3e0ab34..000000000 --- a/python/src/transpile/fusion/mlp.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.fusion.common import strip_passthrough -from src.transpile.fusion.linear import match_linear - - -@dataclass(frozen=True) -class GatedMLPMatch: - input_value_id: str - gate_weight_value_id: str - up_weight_value_id: str - down_weight_value_id: str - activation: str - bias_value_ids: tuple[str | None, str | None, str | None] - node_ids: tuple[str, ...] - - -def match_gated_mlp(graph: IRGraph, node: IRNode) -> GatedMLPMatch | None: - down = match_linear(graph, node) - if down is None: - return None - - mul_node = graph.nodes.get(graph.values[down.input_value_id].producer) if graph.values.get(down.input_value_id) is not None else None - if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: - return None - - for activated_value_id, up_value_id in ((mul_node.inputs[0], mul_node.inputs[1]), (mul_node.inputs[1], mul_node.inputs[0])): - activated_node = graph.nodes.get(graph.values[strip_passthrough(graph, activated_value_id)].producer) if graph.values.get(strip_passthrough(graph, activated_value_id)) is not None else None - if activated_node is None or activated_node.op not in {"gelu", "silu"}: - continue - - gate = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer]) if graph.values.get(strip_passthrough(graph, activated_node.inputs[0])) is not None and graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer in graph.nodes else None - up = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, up_value_id)].producer]) if graph.values.get(strip_passthrough(graph, up_value_id)) is not None and graph.values[strip_passthrough(graph, up_value_id)].producer in graph.nodes else None - if gate is None or up is None: - continue - if strip_passthrough(graph, gate.input_value_id) != strip_passthrough(graph, up.input_value_id): - continue - - return GatedMLPMatch( - input_value_id=strip_passthrough(graph, gate.input_value_id), - gate_weight_value_id=gate.weight_value_id, - up_weight_value_id=up.weight_value_id, - down_weight_value_id=down.weight_value_id, - activation=activated_node.op, - bias_value_ids=(gate.bias_value_id, up.bias_value_id, down.bias_value_id), - node_ids=tuple(sorted({*gate.node_ids, activated_node.id, *up.node_ids, mul_node.id, *down.node_ids})), - ) - - return None diff --git a/python/src/transpile/fusion/rel_pos_bias.py b/python/src/transpile/fusion/rel_pos_bias.py deleted file mode 100644 index d9b5a3def..000000000 --- a/python/src/transpile/fusion/rel_pos_bias.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_passthrough -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode - - -@dataclass(frozen=True) -class RelPosBiasMatch: - query_value_id: str - relative_key_value_id: str - scale: float - node_ids: tuple[str, ...] - - -def match_rel_pos_bias(graph: IRGraph, node: IRNode) -> RelPosBiasMatch | None: - if node.op != "scalar_multiply" or len(node.inputs) != 1: - return None - - final_slice = producer(graph, strip_passthrough(graph, node.inputs[0])) - if final_slice is None or final_slice.op != "slice": - return None - if int(final_slice.attrs.get("axis", -1)) != 3 or int(final_slice.attrs.get("start", -1)) != 0: - return None - - reshape_after_shift = producer(graph, strip_passthrough(graph, final_slice.inputs[0])) - if reshape_after_shift is None or reshape_after_shift.op not in {"reshape", "view"} or len(reshape_after_shift.inputs) != 1: - return None - - shift_slice = producer(graph, strip_passthrough(graph, reshape_after_shift.inputs[0])) - if shift_slice is None or shift_slice.op != "slice": - return None - if int(shift_slice.attrs.get("axis", -1)) != 2 or int(shift_slice.attrs.get("start", -1)) != 1: - return None - - reshape_for_shift = producer(graph, strip_passthrough(graph, shift_slice.inputs[0])) - if reshape_for_shift is None or reshape_for_shift.op not in {"reshape", "view"} or len(reshape_for_shift.inputs) != 1: - return None - - pad = producer(graph, strip_passthrough(graph, reshape_for_shift.inputs[0])) - if pad is None or pad.op != "pad" or len(pad.inputs) != 1: - return None - if tuple(int(v) for v in pad.attrs.get("pads", ())) != (1, 0): - return None - if str(pad.attrs.get("mode", "constant")) != "constant": - return None - if float(pad.attrs.get("value", 0.0)) != 0.0: - return None - - matmul = producer(graph, strip_passthrough(graph, pad.inputs[0])) - if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: - return None - - rhs_permute = producer(graph, strip_passthrough(graph, matmul.inputs[1])) - if rhs_permute is None or rhs_permute.op != "permute" or len(rhs_permute.inputs) != 1: - return None - if tuple(int(v) for v in rhs_permute.attrs.get("permutation", ())) != (0, 2, 3, 1): - return None - - query_value_id = strip_passthrough(graph, matmul.inputs[0]) - relative_key_value_id = strip_passthrough(graph, rhs_permute.inputs[0]) - - query_shape = graph.values.get(query_value_id).shape if query_value_id in graph.values else None - relative_key_shape = graph.values.get(relative_key_value_id).shape if relative_key_value_id in graph.values else None - output_shape = graph.values.get(node.outputs[0]).shape if node.outputs and node.outputs[0] in graph.values else None - if query_shape is None or relative_key_shape is None or output_shape is None: - return None - if len(query_shape) != 4 or len(relative_key_shape) != 4 or len(output_shape) != 4: - return None - - batch, heads, seq_len, head_dim = (int(v) for v in query_shape) - rel_batch, rel_len, rel_heads, rel_dim = (int(v) for v in relative_key_shape) - out_batch, out_heads, out_t, out_s = (int(v) for v in output_shape) - if rel_batch not in {1, batch}: - return None - if rel_heads != heads or rel_dim != head_dim: - return None - if rel_len < 2 * seq_len - 1: - return None - if (out_batch, out_heads, out_t, out_s) != (batch, heads, seq_len, seq_len): - return None - - return RelPosBiasMatch( - query_value_id=query_value_id, - relative_key_value_id=relative_key_value_id, - scale=float(node.attrs.get("value", 1.0)), - node_ids=tuple( - sorted( - { - node.id, - final_slice.id, - reshape_after_shift.id, - shift_slice.id, - reshape_for_shift.id, - pad.id, - matmul.id, - rhs_permute.id, - } - ) - ), - ) diff --git a/python/src/transpile/fusion/rms_norm.py b/python/src/transpile/fusion/rms_norm.py deleted file mode 100644 index 793f1aff2..000000000 --- a/python/src/transpile/fusion/rms_norm.py +++ /dev/null @@ -1,112 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -import torch - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_passthrough - - -@dataclass(frozen=True) -class RMSNormMatch: - input_value_id: str - weight_value_id: str | None - weight_offset: float - eps: float - node_ids: tuple[str, ...] - - -def match_rms_norm(graph: IRGraph, node: IRNode) -> RMSNormMatch | None: - anchor_value_id = node.inputs[0] if node.op == "type_as" and node.inputs else node.outputs[0] - anchor_value_id = strip_passthrough(graph, anchor_value_id) - anchor_node = producer(graph, anchor_value_id) - if anchor_node is None or anchor_node.op != "multiply" or len(anchor_node.inputs) != 2: - return None - - for normed_value_id, scale_value_id in ( - (anchor_node.inputs[0], anchor_node.inputs[1]), - (anchor_node.inputs[1], anchor_node.inputs[0]), - ): - scale = _extract_scale(graph, scale_value_id) - norm = _extract_normed_value(graph, normed_value_id) - if scale is None or norm is None: - continue - if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): - continue - return RMSNormMatch( - input_value_id=norm["input_value_id"], - weight_value_id=scale["weight_value_id"], - weight_offset=float(scale["offset"]), - eps=float(norm["eps"]), - node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"], *scale["node_ids"]})), - ) - - norm = _extract_normed_value(graph, anchor_node.outputs[0]) - if norm is None: - return None - if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): - return None - return RMSNormMatch( - input_value_id=norm["input_value_id"], - weight_value_id=None, - weight_offset=0.0, - eps=float(norm["eps"]), - node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"]})), - ) - - -def _extract_scale(graph: IRGraph, value_id: str) -> dict[str, object] | None: - current = value_id - offset = 0.0 - node_ids: set[str] = set() - producer_node = producer(graph, current) - if producer_node is not None and producer_node.op == "scalar_add" and float(producer_node.attrs.get("value", 0.0)) == 1.0: - offset = 1.0 - node_ids.add(producer_node.id) - current = producer_node.inputs[0] - current = strip_passthrough(graph, current) - if current not in graph.constants or not isinstance(graph.constants[current], torch.Tensor): - return None - return {"weight_value_id": current, "offset": offset, "node_ids": tuple(sorted(node_ids))} - - -def _extract_normed_value(graph: IRGraph, value_id: str) -> dict[str, object] | None: - mul = producer(graph, strip_passthrough(graph, value_id)) - if mul is None or mul.op != "multiply" or len(mul.inputs) != 2: - return None - for x_candidate, rsqrt_candidate in ((mul.inputs[0], mul.inputs[1]), (mul.inputs[1], mul.inputs[0])): - rsqrt = _extract_rsqrt_chain(graph, rsqrt_candidate) - if rsqrt is None: - continue - if strip_passthrough(graph, x_candidate) != strip_passthrough(graph, rsqrt["pow_input_value_id"]): - continue - return { - "input_value_id": strip_passthrough(graph, x_candidate), - "pow_input_value_id": rsqrt["pow_input_value_id"], - "eps": rsqrt["eps"], - "node_ids": tuple(sorted({mul.id, *rsqrt["node_ids"]})), - } - return None - - -def _extract_rsqrt_chain(graph: IRGraph, value_id: str) -> dict[str, object] | None: - pow_node = producer(graph, strip_passthrough(graph, value_id)) - if pow_node is None or pow_node.op != "pow" or float(pow_node.attrs.get("exponent", 0.0)) != -0.5: - return None - add_node = producer(graph, pow_node.inputs[0]) - if add_node is None or add_node.op != "scalar_add": - return None - mean_node = producer(graph, add_node.inputs[0]) - if mean_node is None or mean_node.op != "mean" or not bool(mean_node.attrs.get("keepdim", False)): - return None - pow2_node = producer(graph, mean_node.inputs[0]) - if pow2_node is None or pow2_node.op != "pow" or float(pow2_node.attrs.get("exponent", 0.0)) != 2.0: - return None - return { - "pow_input_value_id": pow2_node.inputs[0], - "eps": float(add_node.attrs.get("value", 0.0)), - "node_ids": (pow_node.id, add_node.id, mean_node.id, pow2_node.id), - } diff --git a/python/src/transpile/fusion/rope.py b/python/src/transpile/fusion/rope.py deleted file mode 100644 index 592ee2933..000000000 --- a/python/src/transpile/fusion/rope.py +++ /dev/null @@ -1,252 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -import torch - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_layout_passthrough -from src.transpile.fusion.common import strip_passthrough - - -@dataclass(frozen=True) -class RoPEMatch: - input_value_id: str - theta: float - position_offset: int - partial: bool - node_ids: tuple[str, ...] - - -def match_rope(graph: IRGraph, value_id: str) -> RoPEMatch | None: - desc = _extract_rope_descriptor(graph, value_id) - if desc is None: - return None - return RoPEMatch( - input_value_id=strip_passthrough(graph, desc["input_value_id"]), - theta=float(desc["theta"]), - position_offset=int(desc["position_offset"]), - partial=bool(desc.get("partial_rope", False)), - node_ids=tuple(sorted(set(desc.get("node_ids", ())))), - ) - - -def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: - current = strip_layout_passthrough(graph, value_id) - node = producer(graph, current) - if node is None: - return None - if node.op == "rope": - return { - "input_value_id": node.inputs[0], - "theta": float(node.attrs.get("theta", 0.0)), - "position_offset": int(node.attrs.get("position_offset", 0)), - "partial_rope": False, - "node_ids": (node.id,), - } - classic = _extract_classic_rope_descriptor(graph, current) - if classic is not None: - classic["partial_rope"] = False - return classic - if node.op == "cat" and len(node.inputs) == 2: - for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) - if rope_branch is None: - continue - passthrough_node = producer(graph, strip_layout_passthrough(graph, passthrough_value_id)) - if passthrough_node is None or passthrough_node.op != "slice": - continue - if strip_passthrough(graph, passthrough_node.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): - continue - rope_branch["partial_rope"] = True - rope_branch["node_ids"] = tuple(sorted({node.id, passthrough_node.id, *rope_branch["node_ids"]})) - return rope_branch - return None - - -def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: - node = producer(graph, strip_layout_passthrough(graph, value_id)) - if node is None or node.op != "add" or len(node.inputs) != 2: - return None - for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - direct_mult = producer(graph, direct_mult_id) - rotated_mult = producer(graph, rotated_mult_id) - if direct_mult is None or direct_mult.op != "multiply" or rotated_mult is None or rotated_mult.op != "multiply": - continue - direct_match = _extract_direct_rope_branch(graph, direct_mult) - rotated_match = _extract_rotated_rope_branch(graph, rotated_mult) - if direct_match is None or rotated_match is None: - continue - input_value_id = strip_passthrough(graph, direct_match["input_value_id"]) - if input_value_id != strip_passthrough(graph, rotated_match["input_value_id"]): - continue - cos_info = _extract_rope_trig(graph, direct_match["trig_value_id"], expected="scalar_cos") - sin_info = _extract_rope_trig(graph, rotated_match["trig_value_id"], expected="scalar_sin") - if cos_info is None or sin_info is None: - continue - if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: - continue - if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): - continue - return { - "input_value_id": input_value_id, - "theta": cos_info["theta"], - "position_offset": cos_info["position_offset"], - "node_ids": tuple(sorted({node.id, *direct_match["node_ids"], *rotated_match["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), - } - return None - - -def _extract_direct_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: - for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - if _unwrap_trig_value(graph, trig_value_id, expected="scalar_cos") is not None: - return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} - return None - - -def _extract_rotated_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: - for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - input_value_id = _extract_rotate_half_source(graph, rotated_value_id) - if input_value_id is None: - continue - if _unwrap_trig_value(graph, trig_value_id, expected="scalar_sin") is None: - continue - return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} - return None - - -def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: - cat_node = producer(graph, strip_passthrough(graph, value_id)) - if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: - return None - neg_node = producer(graph, cat_node.inputs[0]) - right_slice = producer(graph, cat_node.inputs[1]) - if neg_node is None or neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: - return None - left_slice = producer(graph, neg_node.inputs[0]) - if left_slice is None or right_slice is None or left_slice.op != "slice" or right_slice.op != "slice": - return None - if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): - return None - return left_slice.inputs[0] - - -def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: - trig_info = _unwrap_trig_value(graph, value_id, expected=expected) - if trig_info is None: - return None - trig_node = trig_info["trig_node"] - cat_node = producer(graph, trig_node.inputs[0]) - if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: - return None - angle_info = _extract_rope_angle_source(graph, cat_node.inputs[0]) - if angle_info is None: - return None - matmul_node = angle_info["matmul_node"] - inv_freq_const_id = None - arange_node = None - for input_id in matmul_node.inputs: - if inv_freq_const_id is None: - inv_freq_const_id = _find_constant_ancestor(graph, input_id) - if arange_node is None: - arange_node = _find_arange_ancestor(graph, input_id) - if inv_freq_const_id is None or arange_node is None: - return None - theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) - if theta is None: - return None - return { - "theta": theta, - "position_offset": int(arange_node.attrs.get("start", 0)), - "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), - } - - -def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: - current = value_id - while True: - node = producer(graph, current) - if node is None: - return None - if node.op in {"unsqueeze", "precision_cast", "reshape", "view", "expand"}: - current = node.inputs[0] - continue - if node.op == "type_as": - current = node.inputs[0] - continue - if node.op == "scalar_multiply": - current = node.inputs[0] - continue - if node.op != expected: - return None - return {"trig_node": node} - - -def _extract_rope_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: - current = value_id - while True: - current = strip_layout_passthrough(graph, current) - node = producer(graph, current) - if node is None: - return None - if node.op == "permute": - matmul_node = producer(graph, node.inputs[0]) - if matmul_node is None or matmul_node.op != "matmul": - return None - return {"matmul_node": matmul_node} - if node.op == "matmul": - return {"matmul_node": node} - if node.op in {"index", "slice"} and len(node.inputs) == 1: - current = node.inputs[0] - continue - return None - - -def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: - current = value_id - visited: set[str] = set() - while current not in visited: - visited.add(current) - current = strip_layout_passthrough(graph, current) - if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): - return current - node = producer(graph, current) - if node is None or len(node.inputs) != 1: - return None - current = node.inputs[0] - return None - - -def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: - current = value_id - visited: set[str] = set() - while current not in visited: - visited.add(current) - current = strip_layout_passthrough(graph, current) - node = producer(graph, current) - if node is None: - return None - if node.op == "arange": - return node - if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): - return None - if len(node.inputs) != 1: - return None - current = node.inputs[0] - return None - - -def _infer_rope_theta(value: Any) -> float | None: - if not isinstance(value, torch.Tensor): - return None - flat = value.detach().cpu().float().reshape(-1) - if flat.numel() < 2: - return None - second = float(flat[1].item()) - if second <= 0.0: - return None - inv_count = flat.numel() - return float((1.0 / second) ** inv_count) diff --git a/python/src/transpile/graph_ir.py b/python/src/transpile/graph_ir.py deleted file mode 100644 index 632585c57..000000000 --- a/python/src/transpile/graph_ir.py +++ /dev/null @@ -1,201 +0,0 @@ -import copy -from dataclasses import dataclass -from dataclasses import field - -import torch - - -def _clone_ir_payload(value, memo): - if isinstance(value, (str, int, float, bool)) or value is None: - return value - if isinstance(value, torch.Tensor): - # IR constants may be real tensors or export-time tensor wrappers. - # They are treated as immutable graph payloads, so preserve them by - # reference instead of triggering Tensor.__deepcopy__. - return value - if isinstance(value, tuple): - return tuple(_clone_ir_payload(item, memo) for item in value) - if isinstance(value, list): - return [_clone_ir_payload(item, memo) for item in value] - if isinstance(value, dict): - return { - copy.deepcopy(key, memo): _clone_ir_payload(inner, memo) - for key, inner in value.items() - } - try: - return copy.deepcopy(value, memo) - except Exception: - return value - - -@dataclass -class IRValue: - id: str - shape: tuple[int, ...] | None = None - dtype: str | None = None - producer: str | None = None - users: list[str] = field(default_factory=list) - meta: dict[str, object] = field(default_factory=dict) - - def __deepcopy__(self, memo): - existing = memo.get(id(self)) - if existing is not None: - return existing - cloned = IRValue( - id=self.id, - shape=_clone_ir_payload(self.shape, memo), - dtype=self.dtype, - producer=self.producer, - users=list(self.users), - meta=_clone_ir_payload(self.meta, memo), - ) - memo[id(self)] = cloned - return cloned - -@dataclass -class IRNode: - id: str - op: str - inputs: list[str] - outputs: list[str] - attrs: dict[str, object] = field(default_factory=dict) - meta: dict[str, object] = field(default_factory=dict) - kind: str = "generic" - - def __deepcopy__(self, memo): - existing = memo.get(id(self)) - if existing is not None: - return existing - cloned = IRNode( - id=self.id, - op=self.op, - inputs=list(self.inputs), - outputs=list(self.outputs), - attrs=_clone_ir_payload(self.attrs, memo), - meta=_clone_ir_payload(self.meta, memo), - kind=self.kind, - ) - memo[id(self)] = cloned - return cloned - -@dataclass -class IRGraph: - values: dict[str, IRValue] - nodes: dict[str, IRNode] - order: list[str] - inputs: list[str] - outputs: list[str] - constants: dict[str, object] = field(default_factory=dict) - meta: dict[str, object] = field(default_factory=dict) - - def __deepcopy__(self, memo): - existing = memo.get(id(self)) - if existing is not None: - return existing - cloned = IRGraph( - values={value_id: copy.deepcopy(value, memo) for value_id, value in self.values.items()}, - nodes={node_id: copy.deepcopy(node, memo) for node_id, node in self.nodes.items()}, - order=list(self.order), - inputs=list(self.inputs), - outputs=list(self.outputs), - constants={ - value_id: _clone_ir_payload(constant, memo) - for value_id, constant in self.constants.items() - }, - meta=_clone_ir_payload(self.meta, memo), - ) - memo[id(self)] = cloned - return cloned - - def add_node(self, node: IRNode) -> None: - if node.id in self.nodes: - raise ValueError(f"duplicate IR node id: {node.id}") - self.nodes[node.id] = node - for output in node.outputs: - if output in self.values: - raise ValueError(f"duplicate IR value id: {output}") - self.values[output] = IRValue(id=output, producer=node.id) - - def add_value(self, value: IRValue) -> None: - if value.id in self.values: - raise ValueError(f"duplicate IR value id: {value.id}") - self.values[value.id] = value - - -def verify_ir(graph: IRGraph) -> None: - order_set = set(graph.order) - node_keys = set(graph.nodes.keys()) - if len(order_set) != len(graph.order): - raise ValueError("IR order contains duplicate node ids") - if order_set != node_keys: - missing_in_order = node_keys - order_set - missing_in_nodes = order_set - node_keys - raise ValueError( - "IR order/nodes mismatch: " - f"missing_in_order={sorted(missing_in_order)} " - f"missing_in_nodes={sorted(missing_in_nodes)}" - ) - - seen_nodes: set[str] = set() - seen_outputs: set[str] = set() - expected_users: dict[str, list[str]] = {} - - for value_id in graph.inputs: - if value_id not in graph.values: - raise ValueError(f"IR input missing value entry: {value_id}") - expected_users.setdefault(value_id, []) - - for value_id in graph.constants: - if value_id not in graph.values: - raise ValueError(f"IR constant missing value entry: {value_id}") - expected_users.setdefault(value_id, []) - - for node_id in graph.order: - if node_id in seen_nodes: - raise ValueError(f"duplicate node in IR order: {node_id}") - seen_nodes.add(node_id) - - node = graph.nodes[node_id] - if not node.outputs: - raise ValueError(f"IR node has no outputs: {node_id}") - - for input_id in node.inputs: - if input_id not in graph.values: - raise ValueError(f"IR node {node_id} references missing input value {input_id}") - expected_users.setdefault(input_id, []).append(node_id) - producer = graph.values[input_id].producer - if producer is not None and producer not in graph.nodes: - raise ValueError( - f"IR value {input_id} for node {node_id} has missing producer {producer}" - ) - - for output_id in node.outputs: - if output_id in seen_outputs: - raise ValueError(f"IR value produced more than once: {output_id}") - seen_outputs.add(output_id) - if output_id not in graph.values: - raise ValueError(f"IR node {node_id} missing output value entry {output_id}") - value = graph.values[output_id] - if value.producer != node_id: - raise ValueError( - f"IR value {output_id} producer mismatch: expected {node_id}, got {value.producer}" - ) - expected_users.setdefault(output_id, []) - - for output_id in graph.outputs: - if output_id not in graph.values: - raise ValueError(f"IR graph output missing value entry: {output_id}") - - for value_id, value in graph.values.items(): - expected = expected_users.get(value_id, []) - if value.users != expected: - raise ValueError( - f"IR value {value_id} user list mismatch: expected {expected}, got {value.users}" - ) - if value.producer is None and value_id not in graph.inputs and value_id not in graph.constants: - if value_id in seen_outputs: - raise ValueError(f"IR value {value_id} is produced but has no producer") - if value.producer is not None and value.producer not in graph.nodes: - raise ValueError( - f"IR value {value_id} references unknown producer {value.producer}" - ) diff --git a/python/src/transpile/import_ir.py b/python/src/transpile/import_ir.py deleted file mode 100644 index 1052b464e..000000000 --- a/python/src/transpile/import_ir.py +++ /dev/null @@ -1,266 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any - -import torch -from torch.export.graph_signature import ConstantArgument -from torch.export.graph_signature import InputKind - -from src.transpile.capture_pytorch import format_target -from src.transpile.capture_pytorch import get_dtype -from src.transpile.capture_pytorch import get_shape -from src.transpile.capture_pytorch import resolve_attr -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import verify_ir -from src.transpile.importers import extract_literals -from src.transpile.importers import ImportContext -from src.transpile.importers import import_call_function -from src.transpile.importers import import_get_attr -from src.transpile.importers import import_output -from src.transpile.importers import import_placeholder -from src.transpile.importers import value_id -from src.transpile.normalize import dtype_to_ir -from src.transpile.import_semantics import apply_import_semantics -from src.transpile.weight_binding import resolve_transpile_weights_dir -from src.transpile.weight_binding import resolve_weight_binding - - -def import_captured_to_ir(captured: Any, *, strict: bool = True) -> IRGraph: - transpile_metadata = getattr(captured, "transpile_metadata", {}) or {} - graph_meta = transpile_metadata.get("graph", {}) - ir = IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[], constants={}, meta=dict(graph_meta)) - ctx = ImportContext(strict=strict, transpile_metadata=transpile_metadata) - weights_dir = resolve_transpile_weights_dir(ir.meta) - if weights_dir: - ir.meta["weights_dir"] = weights_dir - placeholder_specs: dict[str, Any] = {} - inline_counter = 0 - - def _try_register_weight_binding(value_id_str: str, target: str, value: Any) -> None: - if not isinstance(value, torch.Tensor): - return - binding = resolve_weight_binding( - weights_dir=weights_dir, - source_name=target, - ) - if binding is None: - return - bindings = ir.meta.setdefault("weight_bindings", {}) - if isinstance(bindings, dict): - bindings[value_id_str] = { - "path": binding.path, - "kind": binding.kind, - "source_name": binding.source_name, - } - if value_id_str in ir.values: - ir.values[value_id_str].meta.update( - { - "path": binding.path, - "kind": binding.kind, - "source_name": binding.source_name, - } - ) - - def _make_inlined_node(node: Any, *, inline_index: int) -> Any: - return SimpleNamespace( - name=f"inl_{inline_index}_{node.name}", - op=node.op, - target=node.target, - args=node.args, - kwargs=getattr(node, "kwargs", {}), - meta=getattr(node, "meta", {}), - ) - - def _resolve_target(target: str) -> Any: - candidates: list[str] = [] - parts = target.split(".") - for i in range(len(parts)): - candidate = ".".join(parts[i:]) - if candidate and candidate not in candidates: - candidates.append(candidate) - - exported_program = getattr(captured, "exported_program", None) - if exported_program is not None: - tensor_constants = getattr(exported_program, "tensor_constants", None) - if tensor_constants is not None: - for candidate in candidates: - if candidate in tensor_constants: - return tensor_constants[candidate] - constants = getattr(exported_program, "constants", None) - if constants is not None: - for candidate in candidates: - if candidate in constants: - return constants[candidate] - - for candidate in candidates: - try: - return resolve_attr(captured.graph_module, candidate) - except AttributeError: - pass - - for candidate in candidates: - if candidate in captured.state_dict: - return captured.state_dict[candidate] - - source_module = getattr(captured, "source_module", None) - if source_module is not None: - for candidate in candidates: - try: - return resolve_attr(source_module, candidate) - except AttributeError: - pass - - raise NotImplementedError(f"could not resolve get_attr target: {target}") from None - - def _collect_output_ids(arg: Any, local_alias: dict[str, str]) -> list[str]: - if hasattr(arg, "name") and hasattr(arg, "op"): - name = arg.name - if name in local_alias: - return [local_alias[name]] - return [ctx.resolve_value_id(arg)] - if isinstance(arg, (tuple, list)): - out: list[str] = [] - for item in arg: - out.extend(_collect_output_ids(item, local_alias)) - return out - return [] - - def _inline_wrap_with_set_grad_enabled(node: Any, shape: Any, dtype: Any) -> None: - nonlocal inline_counter - subgraph_ref = node.args[1] - if not hasattr(subgraph_ref, "op") or subgraph_ref.op != "get_attr": - raise NotImplementedError("wrap_with_set_grad_enabled without get_attr submodule is not supported") - - subgraph_module = _resolve_target(subgraph_ref.target) - if not hasattr(subgraph_module, "graph"): - raise NotImplementedError("wrap_with_set_grad_enabled target is not a GraphModule") - - inline_counter += 1 - inline_index = inline_counter - local_alias: dict[str, str] = {} - subgraph_inputs = list(node.args[2:]) - input_index = 0 - - ctx.push_local_aliases(local_alias) - try: - for sub_node in subgraph_module.graph.nodes: - sub_shape = get_shape(sub_node) - sub_dtype = dtype_to_ir(get_dtype(sub_node)) - sub_desc = ( - f"inlined:{node.name}:{sub_node.op}:{getattr(sub_node, 'name', '')}:" - f"{format_target(sub_node) if sub_node.op == 'call_function' else getattr(sub_node, 'target', '')}" - ) - ctx.push_node(sub_desc) - try: - if sub_node.op == "placeholder": - if input_index >= len(subgraph_inputs): - raise NotImplementedError("subgraph placeholder/input arity mismatch during inlining") - local_alias[sub_node.name] = ctx.resolve_value_id(subgraph_inputs[input_index]) - input_index += 1 - continue - - if sub_node.op == "get_attr": - prefixed = _make_inlined_node(sub_node, inline_index=inline_index) - value = _resolve_target(sub_node.target) - import_get_attr(ir, prefixed, ctx, value, shape=sub_shape, dtype=sub_dtype, source_name=str(sub_node.target)) - _try_register_weight_binding(f"v_{prefixed.name}", str(sub_node.target), value) - local_alias[sub_node.name] = f"v_{prefixed.name}" - continue - - if sub_node.op == "call_function": - prefixed = _make_inlined_node(sub_node, inline_index=inline_index) - import_call_function( - ir, - prefixed, - ctx, - shape=sub_shape, - dtype=sub_dtype, - torch_op=format_target(sub_node), - ) - local_alias[sub_node.name] = f"v_{prefixed.name}" - continue - - if sub_node.op == "output": - ctx.tuple_aliases[node.name] = _collect_output_ids(sub_node.args[0], local_alias) - return - - raise NotImplementedError(f"unsupported inlined subgraph node op: {sub_node.op}") - finally: - ctx.pop_node() - finally: - ctx.pop_local_aliases() - - raise NotImplementedError("inlined subgraph had no output node") - - for spec in captured.exported_program.graph_signature.input_specs: - arg = getattr(spec, "arg", None) - name = getattr(arg, "name", None) - if name is not None: - placeholder_specs[name] = spec - - for node in captured.graph.nodes: - shape = get_shape(node) - dtype = dtype_to_ir(get_dtype(node)) - node_desc = f"top:{node.op}:{getattr(node, 'name', '')}:{getattr(node, 'target', '')}" - ctx.push_node(node_desc) - try: - if node.op == "placeholder": - spec = placeholder_specs.get(node.name) - if spec is not None and isinstance(getattr(spec, "arg", None), ConstantArgument): - import_get_attr( - ir, - node, - ctx, - spec.arg.value, - shape=shape, - dtype=dtype, - source_name=node.name, - ) - continue - if spec is not None and spec.kind in {InputKind.PARAMETER, InputKind.BUFFER, InputKind.CONSTANT_TENSOR}: - target = getattr(spec, "target", None) - if target is None: - raise NotImplementedError(f"placeholder {node.name} has no target for lifted constant input") - value = _resolve_target(target) - import_get_attr(ir, node, ctx, value, shape=shape, dtype=dtype, source_name=str(target)) - _try_register_weight_binding(value_id(node, ctx), str(target), value) - continue - import_placeholder(ir, node, ctx, shape=shape, dtype=dtype) - continue - - if node.op == "get_attr": - if str(node.target).startswith("submod_"): - continue - value = _resolve_target(node.target) - import_get_attr(ir, node, ctx, value, shape=shape, dtype=dtype, source_name=str(node.target)) - _try_register_weight_binding(value_id(node, ctx), str(node.target), value) - continue - - if node.op == "call_function": - torch_op = format_target(node) - if "wrap_with_set_grad_enabled" in torch_op: - _inline_wrap_with_set_grad_enabled(node, shape, dtype) - continue - if torch_op == "": - source = node.args[0] - if hasattr(source, "name") and source.name in ctx.tuple_aliases: - index_value = extract_literals(node.args[1]) - if not isinstance(index_value, int): - raise NotImplementedError(f"unsupported tuple getitem index: {node.args!r}") - ctx.alias_values[node.name] = ctx.tuple_aliases[source.name][index_value] - continue - import_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) - continue - - if node.op == "output": - import_output(ir, node, ctx) - continue - - raise NotImplementedError(f"unsupported FX node op: {node.op}") - finally: - ctx.pop_node() - - apply_import_semantics(ir) - verify_ir(ir) - return ir diff --git a/python/src/transpile/import_semantics.py b/python/src/transpile/import_semantics.py deleted file mode 100644 index 7ef37ed1f..000000000 --- a/python/src/transpile/import_semantics.py +++ /dev/null @@ -1,337 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -import torch - -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_passthrough -from src.transpile.canonicalize.utils import rebuild_graph -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.graph_ir import verify_ir - - -@dataclass(frozen=True) -class EarlyRoPEMatch: - anchor_node_id: str - input_value_id: str - theta: float - position_offset: int - partial: bool - node_ids: tuple[str, ...] - - -def apply_import_semantics(graph: IRGraph) -> IRGraph: - _rewrite_explicit_attention(graph) - _tag_explicit_linear(graph) - _rewrite_early_rope(graph) - rebuild_graph(graph) - verify_ir(graph) - return graph - - -def _rewrite_explicit_attention(graph: IRGraph) -> None: - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None or node.op != "scaled_dot_product_attention": - continue - additive_mask = bool(node.attrs.get("additive_mask", False)) - if not additive_mask and len(node.inputs) > 3: - mask_value = graph.values.get(node.inputs[3]) - if mask_value is not None and mask_value.dtype not in (None, "bool"): - additive_mask = True - node.op = "attention" - node.kind = "semantic" - node.meta["semantic_source"] = "import" - node.attrs = { - "scale": float(node.attrs.get("scale", 1.0)), - "is_causal": bool(node.attrs.get("is_causal", False)), - "window_size": int(node.attrs.get("window_size", 0)), - **({"enable_gqa": bool(node.attrs.get("enable_gqa", False))} if "enable_gqa" in node.attrs else {}), - **({"additive_mask": True} if additive_mask else {}), - } - - -def _tag_explicit_linear(graph: IRGraph) -> None: - for node in graph.nodes.values(): - if node.op not in {"linear", "addmm"}: - continue - node.meta.setdefault("semantic_source", "import") - - -def _rewrite_early_rope(graph: IRGraph) -> None: - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None or not node.outputs: - continue - match = match_early_rope(graph, node.outputs[0]) - if match is None or match.anchor_node_id != node.id: - continue - if match.partial: - node.meta["semantic_hint"] = "partial_rope" - node.meta["rope_theta_hint"] = float(match.theta) - node.meta["rope_position_offset_hint"] = int(match.position_offset) - continue - node.op = "rope" - node.inputs = [match.input_value_id] - node.attrs = { - "theta": float(match.theta), - "position_offset": int(match.position_offset), - } - node.kind = "semantic" - node.meta["semantic_source"] = "import" - - -def match_early_rope(graph: IRGraph, value_id: str) -> EarlyRoPEMatch | None: - desc = _extract_rope_descriptor(graph, value_id) - if desc is None: - return None - return EarlyRoPEMatch( - anchor_node_id=str(desc["anchor_node_id"]), - input_value_id=str(desc["input_value_id"]), - theta=float(desc["theta"]), - position_offset=int(desc["position_offset"]), - partial=bool(desc.get("partial_rope", False)), - node_ids=tuple(sorted(set(desc.get("node_ids", ())))), - ) - - -def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: - current = _strip_layout_passthrough(graph, value_id) - node = producer(graph, current) - if node is None: - return None - if node.op == "rope": - return { - "anchor_node_id": node.id, - "input_value_id": node.inputs[0], - "theta": float(node.attrs.get("theta", 0.0)), - "position_offset": int(node.attrs.get("position_offset", 0)), - "partial_rope": False, - "node_ids": (node.id,), - } - classic = _extract_classic_rope_descriptor(graph, current) - if classic is not None: - classic["partial_rope"] = False - return classic - if node.op == "cat" and len(node.inputs) == 2: - for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) - if rope_branch is None: - continue - passthrough = producer(graph, _strip_layout_passthrough(graph, passthrough_value_id)) - if passthrough is None or passthrough.op != "slice": - continue - if strip_passthrough(graph, passthrough.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): - continue - rope_branch["anchor_node_id"] = node.id - rope_branch["partial_rope"] = True - rope_branch["node_ids"] = tuple(sorted({node.id, passthrough.id, *rope_branch["node_ids"]})) - return rope_branch - return None - - -def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: - node = producer(graph, _strip_layout_passthrough(graph, value_id)) - if node is None or node.op != "add" or len(node.inputs) != 2: - return None - for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - direct_mult = producer(graph, direct_mult_id) - rotated_mult = producer(graph, rotated_mult_id) - if direct_mult is None or rotated_mult is None: - continue - if direct_mult.op != "multiply" or rotated_mult.op != "multiply": - continue - direct = _extract_direct_branch(graph, direct_mult) - rotated = _extract_rotated_branch(graph, rotated_mult) - if direct is None or rotated is None: - continue - input_value_id = strip_passthrough(graph, direct["input_value_id"]) - if input_value_id != strip_passthrough(graph, rotated["input_value_id"]): - continue - cos_info = _extract_rope_trig(graph, direct["trig_value_id"], expected=("cos", "scalar_cos")) - sin_info = _extract_rope_trig(graph, rotated["trig_value_id"], expected=("sin", "scalar_sin")) - if cos_info is None or sin_info is None: - continue - if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: - continue - if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): - continue - return { - "anchor_node_id": node.id, - "input_value_id": input_value_id, - "theta": float(cos_info["theta"]), - "position_offset": int(cos_info["position_offset"]), - "node_ids": tuple(sorted({node.id, *direct["node_ids"], *rotated["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), - } - return None - - -def _extract_direct_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: - for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - if _unwrap_trig_value(graph, trig_value_id, expected=("cos", "scalar_cos")) is not None: - return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} - return None - - -def _extract_rotated_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: - for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - input_value_id = _extract_rotate_half_source(graph, rotated_value_id) - if input_value_id is None: - continue - if _unwrap_trig_value(graph, trig_value_id, expected=("sin", "scalar_sin")) is None: - continue - return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} - return None - - -def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: - cat_node = producer(graph, strip_passthrough(graph, value_id)) - if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: - return None - neg_node = producer(graph, cat_node.inputs[0]) - right_slice = producer(graph, cat_node.inputs[1]) - if neg_node is None or right_slice is None: - return None - if neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: - return None - left_slice = producer(graph, neg_node.inputs[0]) - if left_slice is None or left_slice.op != "slice" or right_slice.op != "slice": - return None - if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): - return None - return left_slice.inputs[0] - - -def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: - trig = _unwrap_trig_value(graph, value_id, expected=expected) - if trig is None: - return None - trig_node = trig["trig_node"] - cat_node = producer(graph, trig_node.inputs[0]) - if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: - return None - angle_info = _extract_angle_source(graph, cat_node.inputs[0]) - if angle_info is None: - return None - matmul_node = angle_info["matmul_node"] - inv_freq_const_id = None - arange_node = None - for input_id in matmul_node.inputs: - if inv_freq_const_id is None: - inv_freq_const_id = _find_constant_ancestor(graph, input_id) - if arange_node is None: - arange_node = _find_arange_ancestor(graph, input_id) - if inv_freq_const_id is None or arange_node is None: - return None - theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) - if theta is None: - return None - return { - "theta": theta, - "position_offset": int(arange_node.attrs.get("start", 0)), - "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), - } - - -def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: - current = value_id - while True: - node = producer(graph, current) - if node is None: - return None - if node.op in {"unsqueeze", "reshape", "view", "expand", "precision_cast"}: - current = node.inputs[0] - continue - if node.op == "type_as": - current = node.inputs[0] - continue - if node.op == "scalar_multiply": - current = node.inputs[0] - continue - if node.op not in expected: - return None - return {"trig_node": node} - - -def _extract_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: - current = value_id - while True: - current = _strip_layout_passthrough(graph, current) - node = producer(graph, current) - if node is None: - return None - if node.op in {"transpose", "permute"}: - matmul_node = producer(graph, node.inputs[0]) - if matmul_node is None or matmul_node.op != "matmul": - return None - return {"matmul_node": matmul_node} - if node.op == "matmul": - return {"matmul_node": node} - if node.op in {"index", "slice"} and len(node.inputs) == 1: - current = node.inputs[0] - continue - return None - - -def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: - current = value_id - visited: set[str] = set() - while current not in visited: - visited.add(current) - current = _strip_layout_passthrough(graph, current) - if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): - return current - node = producer(graph, current) - if node is None or len(node.inputs) != 1: - return None - current = node.inputs[0] - return None - - -def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: - current = value_id - visited: set[str] = set() - while current not in visited: - visited.add(current) - current = _strip_layout_passthrough(graph, current) - node = producer(graph, current) - if node is None: - return None - if node.op == "arange": - return node - if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): - return None - if len(node.inputs) != 1: - return None - current = node.inputs[0] - return None - - -def _strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: - current = value_id - while True: - node = producer(graph, current) - if node is None: - return current - if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: - current = node.inputs[0] - continue - if node.op == "type_as" and len(node.inputs) >= 1: - current = node.inputs[0] - continue - return current - - -def _infer_rope_theta(value: Any) -> float | None: - if not isinstance(value, torch.Tensor): - return None - flat = value.detach().cpu().float().reshape(-1) - if flat.numel() < 2: - return None - second = float(flat[1].item()) - if second <= 0.0: - return None - return float((1.0 / second) ** flat.numel()) diff --git a/python/src/transpile/importers.py b/python/src/transpile/importers.py deleted file mode 100644 index f63abb433..000000000 --- a/python/src/transpile/importers.py +++ /dev/null @@ -1,1754 +0,0 @@ -from __future__ import annotations - -import math -from dataclasses import dataclass -from dataclasses import field -from types import SimpleNamespace -from typing import Any - -import torch - -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.graph_ir import IRValue -from src.transpile.normalize import dtype_to_ir -from src.transpile.normalize import normalize_target - - -class UnsupportedImportError(NotImplementedError): - pass - - -@dataclass -class ImportContext: - strict: bool = True - alias_values: dict[str, str] = field(default_factory=dict) - tuple_aliases: dict[str, list[str]] = field(default_factory=dict) - local_alias_stack: list[dict[str, str]] = field(default_factory=list) - node_stack: list[str] = field(default_factory=list) - transpile_metadata: dict[str, object] = field(default_factory=dict) - - def push_local_aliases(self, aliases: dict[str, str]) -> None: - self.local_alias_stack.append(aliases) - - def pop_local_aliases(self) -> None: - if not self.local_alias_stack: - raise RuntimeError("local alias stack underflow") - self.local_alias_stack.pop() - - def resolve_value_id(self, node: Any) -> str: - if is_fx_node(node): - name = getattr(node, "name", "") - for aliases in reversed(self.local_alias_stack): - if name in aliases: - return aliases[name] - resolved = self.alias_values.get(name) - if resolved is not None: - return resolved - return f"v_{name}" - raise TypeError(f"cannot resolve value id for non-FX node: {type(node).__name__}") - - def fail(self, message: str) -> None: - if self.node_stack: - message = f"{message} [context: {' > '.join(self.node_stack)}]" - if self.strict: - raise UnsupportedImportError(message) - raise UnsupportedImportError(f"{message} [non-strict import has no generic fallback]") - - def push_node(self, description: str) -> None: - self.node_stack.append(description) - - def pop_node(self) -> None: - if self.node_stack: - self.node_stack.pop() - - def _node_module_paths(self, node: Any) -> tuple[str, ...]: - meta = getattr(node, "meta", {}) or {} - raw_stack = meta.get("nn_module_stack") - if not isinstance(raw_stack, dict): - return () - paths: list[str] = [] - for entry in raw_stack.values(): - if not isinstance(entry, (tuple, list)) or not entry: - continue - module_path = entry[0] - if isinstance(module_path, str) and module_path: - paths.append(module_path) - # Prefer the deepest module path first. - paths.reverse() - deduped: list[str] = [] - for path in paths: - if path not in deduped: - deduped.append(path) - return tuple(deduped) - - def lookup_import_hint(self, node: Any, *, torch_op: str, op_name: str) -> dict[str, object]: - hints = self.transpile_metadata.get("import_hints", ()) - if not isinstance(hints, list): - return {} - - module_paths = self._node_module_paths(node) - matched_attrs: dict[str, object] = {} - matched_meta: dict[str, object] = {} - matched_provider: list[str] = [] - - for hint in hints: - if not isinstance(hint, dict): - continue - hint_op = hint.get("op") - if hint_op is not None and hint_op != op_name: - continue - hint_torch_op = hint.get("torch_op") - if hint_torch_op is not None and hint_torch_op != torch_op: - continue - - module_path = hint.get("module_path") - if module_path is not None and module_path not in module_paths: - continue - module_path_suffix = hint.get("module_path_suffix") - if module_path_suffix is not None and not any( - path == module_path_suffix or path.endswith(f".{module_path_suffix}") for path in module_paths - ): - continue - - attrs = hint.get("attrs", {}) - if isinstance(attrs, dict): - matched_attrs.update(attrs) - meta = hint.get("meta", {}) - if isinstance(meta, dict): - matched_meta.update(meta) - provider = hint.get("provider") - if isinstance(provider, str) and provider not in matched_provider: - matched_provider.append(provider) - - if not matched_attrs and not matched_meta and not matched_provider: - return {} - - if matched_provider: - matched_meta.setdefault("import_hint_providers", tuple(matched_provider)) - matched_meta.setdefault("import_hint_source", "capture_metadata") - return {"attrs": matched_attrs, "meta": matched_meta} - - -def value_id(node: Any, ctx: ImportContext) -> str: - return ctx.resolve_value_id(node) - - -def node_id(node: Any) -> str: - return f"n_{node.name}" - - -def is_fx_node(value: Any) -> bool: - return hasattr(value, "name") and hasattr(value, "op") - - -def extract_input_ids(arg: Any, ctx: ImportContext) -> list[str]: - if is_fx_node(arg): - return [value_id(arg, ctx)] - if isinstance(arg, (tuple, list)): - ids: list[str] = [] - for item in arg: - ids.extend(extract_input_ids(item, ctx)) - return ids - if isinstance(arg, dict): - ids: list[str] = [] - for item in arg.values(): - ids.extend(extract_input_ids(item, ctx)) - return ids - return [] - - -def extract_literals(value: Any) -> Any: - if is_fx_node(value): - meta = getattr(value, "meta", {}) or {} - meta_value = meta.get("val") - if meta_value is not None and not isinstance(meta_value, torch.Tensor): - return extract_literals(meta_value) - return None - if isinstance(value, torch.Size): - return tuple(extract_literals(v) for v in value) - if isinstance(value, (tuple, list)): - return type(value)(extract_literals(v) for v in value) - if isinstance(value, dict): - return {k: extract_literals(v) for k, v in value.items()} - if isinstance(value, torch.dtype): - return str(value) - if isinstance(value, (int, float, bool, str)) or value is None: - return value - return None - - -def make_prefixed_node(node: Any, prefix: str) -> Any: - return SimpleNamespace( - name=f"{prefix}{node.name}", - op=node.op, - target=node.target, - args=node.args, - kwargs=getattr(node, "kwargs", {}), - meta=getattr(node, "meta", {}), - ) - - -def add_value_if_missing(ir: IRGraph, value: IRValue) -> None: - if value.id not in ir.values: - ir.add_value(value) - - -def register_node( - ir: IRGraph, - ir_node: IRNode, - *, - shape: tuple[int, ...] | None, - dtype: str | None, -) -> None: - ir.add_node(ir_node) - ir.order.append(ir_node.id) - for output_id in ir_node.outputs: - ir.values[output_id].shape = shape - ir.values[output_id].dtype = dtype - for input_id in ir_node.inputs: - if input_id in ir.values: - ir.values[input_id].users.append(ir_node.id) - - -def _extract_numeric_literal(value: Any) -> float | int | None: - literal = extract_literals(value) - if isinstance(literal, bool): - return int(literal) - if isinstance(literal, (int, float)): - return literal - return None - - -def _extract_static_int(value: Any) -> int | None: - if is_fx_node(value): - meta = getattr(value, "meta", {}) or {} - meta_value = meta.get("val") - if meta_value is not None and not isinstance(meta_value, torch.Tensor): - return _extract_static_int(meta_value) - return None - - literal = extract_literals(value) - if isinstance(literal, bool): - return int(literal) - if isinstance(literal, int): - return int(literal) - if isinstance(literal, float) and float(literal).is_integer(): - return int(literal) - - try: - return int(value) - except Exception: - return None - - -def _extract_int_sequence_literal(value: Any) -> tuple[int, ...] | None: - if is_fx_node(value): - meta = getattr(value, "meta", {}) or {} - meta_value = meta.get("val") - if meta_value is not None and not isinstance(meta_value, torch.Tensor): - return _extract_int_sequence_literal(meta_value) - return None - - if isinstance(value, torch.Size): - value = tuple(value) - - if isinstance(value, (tuple, list)): - result: list[int] = [] - for item in value: - item_value = _extract_static_int(item) - if item_value is None: - return None - result.append(item_value) - return tuple(result) - - scalar_value = _extract_static_int(value) - if scalar_value is not None: - return (scalar_value,) - return None - - -def _extract_fx_tensor_shape(value: Any) -> tuple[Any, ...] | None: - if not is_fx_node(value): - return None - meta = getattr(value, "meta", {}) or {} - meta_value = meta.get("val") - if isinstance(meta_value, torch.Tensor): - return tuple(meta_value.shape) - tensor_meta = meta.get("tensor_meta") - tensor_shape = getattr(tensor_meta, "shape", None) - if tensor_shape is not None: - return tuple(tensor_shape) - return None - - -def _base_meta(shape: tuple[int, ...] | None, dtype: str | None, torch_op: str, node: Any) -> dict[str, object]: - return { - "shape": shape, - "dtype": dtype, - "torch_op": torch_op, - "torch_name": node.name, - } - - -def _extract_fx_torch_dtype(node: Any) -> torch.dtype | None: - if not is_fx_node(node): - return None - meta = getattr(node, "meta", {}) or {} - value = meta.get("val") - dtype = getattr(value, "dtype", None) - if isinstance(dtype, torch.dtype): - return dtype - tensor_meta = meta.get("tensor_meta") - dtype = getattr(tensor_meta, "dtype", None) - if isinstance(dtype, torch.dtype): - return dtype - return None - - -def _import_scalar_binary( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, - op_name: str, -) -> None: - lhs, rhs = node.args[0], node.args[1] - lhs_is_node = is_fx_node(lhs) - rhs_is_node = is_fx_node(rhs) - lhs_literal = _extract_numeric_literal(lhs) - rhs_literal = _extract_numeric_literal(rhs) - - if lhs_is_node and rhs_is_node: - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return - - if op_name == "add": - if lhs_is_node and rhs_literal is not None: - scalar_op = "scalar_add" - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - scalar_op = "scalar_add" - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - scalar_op = None - elif op_name == "multiply": - if lhs_is_node and rhs_literal is not None: - scalar_op = "scalar_multiply" - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - scalar_op = "scalar_multiply" - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - scalar_op = None - elif op_name == "subtract": - if lhs_is_node and rhs_literal is not None: - scalar_op = "scalar_subtract" - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - scalar_op = "scalar_subtract_reverse" - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - scalar_op = None - elif op_name == "divide": - if lhs_is_node and rhs_literal is not None: - scalar_op = "scalar_divide" - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - scalar_op = "scalar_divide_reverse" - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - scalar_op = None - else: - scalar_op = None - - if scalar_op is None: - ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") - - ir_node = IRNode( - id=node_id(node), - op=scalar_op, - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_placeholder( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, -) -> None: - add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) - ir.inputs.append(value_id(node, ctx)) - - -def import_get_attr( - ir: IRGraph, - node: Any, - ctx: ImportContext, - value: Any, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - source_name: str | None = None, -) -> None: - add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) - ir.constants[value_id(node, ctx)] = value.detach().cpu() if isinstance(value, torch.Tensor) else value - ir.values[value_id(node, ctx)].meta["source_name"] = source_name - if source_name is not None: - bindings = ir.meta.setdefault("weight_bindings", {}) - if isinstance(bindings, dict): - bindings.setdefault(value_id(node, ctx), {})["source_name"] = source_name - - -def import_output(ir: IRGraph, node: Any, ctx: ImportContext) -> None: - ir.outputs.extend(extract_input_ids(node.args[0], ctx)) - - -def import_call_function( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, -) -> None: - op = normalize_target(torch_op) - import_hint = ctx.lookup_import_hint(node, torch_op=torch_op, op_name=op) - importer = OP_IMPORTERS.get(op) - if importer is None: - import_opaque_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name=op) - else: - importer(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) - _apply_import_hint(ir, node, import_hint) - - -def _apply_import_hint(ir: IRGraph, node: Any, hint: dict[str, object]) -> None: - if not hint: - return - ir_node = ir.nodes.get(node_id(node)) - if ir_node is None: - return - attrs = hint.get("attrs", {}) - if isinstance(attrs, dict): - ir_node.attrs.update(attrs) - meta = hint.get("meta", {}) - if isinstance(meta, dict): - ir_node.meta.update(meta) - - -def import_opaque_call_function( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, - op_name: str, -) -> None: - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=extract_input_ids(getattr(node, "args", ()), ctx) + extract_input_ids(getattr(node, "kwargs", {}), ctx), - outputs=[value_id(node, ctx)], - attrs={ - "opaque": True, - "torch_op": torch_op, - "args": extract_literals(getattr(node, "args", ())), - "kwargs": extract_literals(getattr(node, "kwargs", {})), - }, - meta=_base_meta(shape, dtype, torch_op, node), - kind="opaque", - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_add(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="add") - - -def import_subtract(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="subtract") - - -def import_multiply(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") - - -def import_multiply_inplace(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") - - -def import_divide(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="divide") - - -def import_not_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - lhs, rhs = node.args[0], node.args[1] - lhs_is_node = is_fx_node(lhs) - rhs_is_node = is_fx_node(rhs) - lhs_literal = _extract_numeric_literal(lhs) - rhs_literal = _extract_numeric_literal(rhs) - - if lhs_is_node and rhs_is_node: - ir_node = IRNode( - id=node_id(node), - op="not_equal", - inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return - - if lhs_is_node and rhs_literal is not None: - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") - - ir_node = IRNode( - id=node_id(node), - op="scalar_not_equal", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - lhs, rhs = node.args[0], node.args[1] - lhs_is_node = is_fx_node(lhs) - rhs_is_node = is_fx_node(rhs) - lhs_literal = _extract_numeric_literal(lhs) - rhs_literal = _extract_numeric_literal(rhs) - - if lhs_is_node and rhs_is_node: - ir_node = IRNode( - id=node_id(node), - op="equal", - inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return - - if lhs_is_node and rhs_literal is not None: - inputs = [value_id(lhs, ctx)] - attrs = {"value": rhs_literal} - elif rhs_is_node and lhs_literal is not None: - inputs = [value_id(rhs, ctx)] - attrs = {"value": lhs_literal} - else: - ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") - - ir_node = IRNode( - id=node_id(node), - op="scalar_equal", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_binary(op_name: str): - def _importer( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, - ) -> None: - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - return _importer - - -def import_where( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, -) -> None: - condition = node.args[0] - true_value = node.args[1] - false_value = node.args[2] - - inputs = [value_id(condition, ctx)] - attrs: dict[str, object] = {} - - if is_fx_node(true_value): - inputs.append(value_id(true_value, ctx)) - attrs["true_is_scalar"] = False - else: - literal = _extract_numeric_literal(true_value) - if literal is None: - ctx.fail(f"unsupported true branch for {torch_op}: {node.args!r}") - attrs["true_is_scalar"] = True - attrs["true_value"] = float(literal) - - if is_fx_node(false_value): - inputs.append(value_id(false_value, ctx)) - attrs["false_is_scalar"] = False - else: - literal = _extract_numeric_literal(false_value) - if literal is None: - ctx.fail(f"unsupported false branch for {torch_op}: {node.args!r}") - attrs["false_is_scalar"] = True - attrs["false_value"] = float(literal) - - ir_node = IRNode( - id=node_id(node), - op="where", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_negate(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="scalar_multiply", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"value": -1.0}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_precision_cast(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_dtype = extract_literals(node.args[1]) if len(node.args) > 1 else None - ir_node = IRNode( - id=node_id(node), - op="precision_cast", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"dtype": target_dtype}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_arange(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - attrs: dict[str, object] = {} - if len(node.args) == 1: - attrs["start"] = 0 - attrs["end"] = int(node.args[0]) - elif len(node.args) >= 2: - attrs["start"] = int(node.args[0]) - attrs["end"] = int(node.args[1]) - else: - ctx.fail(f"unsupported arange signature for {torch_op}: {node.args!r}") - - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - attrs["step"] = int(node.args[2]) - - target_dtype = None - if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None: - target_dtype = extract_literals(node.kwargs["dtype"]) - elif dtype is not None: - target_dtype = dtype - if target_dtype is not None: - attrs["dtype"] = target_dtype - - ir_node = IRNode( - id=node_id(node), - op="arange", - inputs=[], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_type_as(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="type_as", - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_identity(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="identity", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_reshape(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_shape = _extract_int_sequence_literal(node.args[1]) - if target_shape is None: - ctx.fail(f"unsupported reshape shape for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="reshape", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"shape": target_shape}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_flatten(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="flatten", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"start_dim": int(node.args[1]), "end_dim": int(node.args[2])}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_unsqueeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - dim = int(node.args[1]) - attrs: dict[str, object] = {"dim": dim} - if shape is not None: - attrs["shape"] = tuple(int(v) for v in shape) - ir_node = IRNode( - id=node_id(node), - op="unsqueeze", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_squeeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - dim = int(node.args[1]) if len(node.args) > 1 and node.args[1] is not None else 0 - attrs: dict[str, object] = {"dim": dim} - if shape is not None: - attrs["shape"] = tuple(int(v) for v in shape) - ir_node = IRNode( - id=node_id(node), - op="squeeze", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_masked_scatter( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, -) -> None: - ir_node = IRNode( - id=node_id(node), - op="masked_scatter", - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_masked_fill( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, -) -> None: - value = _extract_numeric_literal(node.args[2]) - if value is None: - ctx.fail(f"unsupported masked_fill value for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="masked_fill", - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], - outputs=[value_id(node, ctx)], - attrs={"value": float(value)}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_expand(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - target_shape = _extract_int_sequence_literal(node.args[1]) - if target_shape is None and shape is not None: - target_shape = _extract_int_sequence_literal(shape) - if target_shape is None: - ctx.fail(f"unsupported expand shape for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="expand", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"shape": target_shape}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_one_hot(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - num_classes = _extract_static_int(node.args[1]) if len(node.args) > 1 else None - if (num_classes is None or num_classes <= 0) and shape is not None and shape: - num_classes = int(shape[-1]) - if num_classes is None or num_classes <= 0: - ctx.fail(f"unsupported one_hot num_classes for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="one_hot", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"num_classes": int(num_classes)}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_tril(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - diagonal = int(extract_literals(node.args[1])) if len(node.args) > 1 and extract_literals(node.args[1]) is not None else 0 - ir_node = IRNode( - id=node_id(node), - op="tril", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"diagonal": diagonal}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_unfold(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 4: - ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") - dimension = extract_literals(node.args[1]) - size = extract_literals(node.args[2]) - step = extract_literals(node.args[3]) - if not isinstance(dimension, int) or not isinstance(size, int) or not isinstance(step, int): - ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="unfold", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"dimension": dimension, "size": size, "step": step}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_transpose(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - attrs = {"dim0": 0, "dim1": 1} if torch_op == "aten.t.default" else {"dim0": int(node.args[1]), "dim1": int(node.args[2])} - ir_node = IRNode( - id=node_id(node), - op="transpose", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_permute(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - permutation = _extract_int_sequence_literal(node.args[1]) - if permutation is None: - ctx.fail(f"unsupported permute dims for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="permute", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"permutation": permutation}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_numpy_t(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - input_shape = _extract_fx_tensor_shape(node.args[0]) - rank_source = input_shape if input_shape is not None else shape - if rank_source is None: - ctx.fail(f"unsupported numpy_T rank inference for {torch_op}: {node.args!r}") - permutation = tuple(range(len(rank_source) - 1, -1, -1)) - ir_node = IRNode( - id=node_id(node), - op="permute", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"permutation": permutation}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_movedim(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - source = _extract_int_sequence_literal(node.args[1]) - destination = _extract_int_sequence_literal(node.args[2]) - if source is None or destination is None: - ctx.fail(f"unsupported movedim dims for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="movedim", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"source": source, "destination": destination}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_matmul(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="matmul", - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_linear(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] - attrs = {"has_bias": False} - if len(node.args) > 2 and node.args[2] is not None: - inputs.append(value_id(node.args[2], ctx)) - attrs["has_bias"] = True - ir_node = IRNode( - id=node_id(node), - op="linear", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_addmm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="addmm", - inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_unary(op_name: str): - def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return _importer - - -def import_clamp(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - min_value = extract_literals(node.args[1]) if len(node.args) > 1 else None - max_value = extract_literals(node.args[2]) if len(node.args) > 2 else None - attrs: dict[str, object] = {} - if isinstance(min_value, (int, float)): - attrs["min"] = float(min_value) - if isinstance(max_value, (int, float)): - attrs["max"] = float(max_value) - ir_node = IRNode( - id=node_id(node), - op="clamp", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_pow_rsqrt(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="pow", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"exponent": -0.5}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_reciprocal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="pow", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"exponent": -1.0}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_floor_divide( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, -) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported floor_divide signature for {torch_op}: {node.args!r}") - divisor = _extract_numeric_literal(node.args[1]) - if divisor is None: - ctx.fail(f"unsupported floor_divide divisor for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="scalar_floor_divide", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"value": float(divisor)}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_softmax(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="softmax", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"axis": int(node.args[1])}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_glu(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - axis = -1 - if len(node.args) > 1 and extract_literals(node.args[1]) is not None: - axis = int(node.args[1]) - ir_node = IRNode( - id=node_id(node), - op="glu", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_scaled_dot_product_attention( - ir: IRGraph, - node: Any, - ctx: ImportContext, - *, - shape: tuple[int, ...] | None, - dtype: str | None, - torch_op: str, -) -> None: - attrs: dict[str, object] = {} - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)] - if len(node.args) > 3: - mask_literal = extract_literals(node.args[3]) - if mask_literal is not None: - attrs["mask"] = mask_literal - elif is_fx_node(node.args[3]): - inputs.append(value_id(node.args[3], ctx)) - mask_dtype = _extract_fx_torch_dtype(node.args[3]) - if mask_dtype is not None and mask_dtype != torch.bool: - attrs["additive_mask"] = True - if len(node.args) > 4 and extract_literals(node.args[4]) is not None: - attrs["dropout_p"] = float(node.args[4]) - if len(node.args) > 5 and extract_literals(node.args[5]) is not None: - attrs["is_causal"] = bool(node.args[5]) - if "scale" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["scale"]) is not None: - attrs["scale"] = float(node.kwargs["scale"]) - if "enable_gqa" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["enable_gqa"]) is not None: - attrs["enable_gqa"] = bool(node.kwargs["enable_gqa"]) - ir_node = IRNode( - id=node_id(node), - op="scaled_dot_product_attention", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_reduce(op_name: str): - def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - attrs: dict[str, object] = {} - if len(node.args) > 1 and extract_literals(node.args[1]) is not None: - attrs["axis"] = extract_literals(node.args[1]) - keepdim = None - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - keepdim = bool(extract_literals(node.args[2])) - elif "keepdim" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["keepdim"]) is not None: - keepdim = bool(extract_literals(node.kwargs["keepdim"])) - if keepdim is not None: - attrs["keepdim"] = keepdim - ir_node = IRNode( - id=node_id(node), - op=op_name, - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - return _importer - - -def import_cat(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - tensors_arg = node.args[0] - axis = int(node.args[1]) if len(node.args) > 1 else 0 - ir_node = IRNode( - id=node_id(node), - op="cat", - inputs=extract_input_ids(tensors_arg, ctx), - outputs=[value_id(node, ctx)], - attrs={"axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_split_with_sizes(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported split_with_sizes signature for {torch_op}: {node.args!r}") - sizes = _extract_int_sequence_literal(node.args[1]) - if sizes is None: - ctx.fail(f"unsupported split sizes for {torch_op}: {node.args!r}") - axis = -1 - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - axis = int(extract_literals(node.args[2])) - ir_node = IRNode( - id=node_id(node), - op="split_with_sizes", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"sizes": sizes, "axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_chunk(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported chunk signature for {torch_op}: {node.args!r}") - chunks = extract_literals(node.args[1]) - if not isinstance(chunks, int): - ctx.fail(f"unsupported chunk count for {torch_op}: {node.args!r}") - axis = 0 - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - axis = int(extract_literals(node.args[2])) - ir_node = IRNode( - id=node_id(node), - op="chunk", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"chunks": int(chunks), "axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_pow(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - base_literal = _extract_numeric_literal(node.args[0]) if len(node.args) > 0 else None - attrs: dict[str, object] = {} - exponent_literal = extract_literals(node.args[1]) if len(node.args) > 1 else None - - if base_literal is not None and isinstance(exponent_literal, (int, float)): - add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) - ir.constants[value_id(node, ctx)] = base_literal**exponent_literal - return - - if base_literal is not None: - if not (len(node.args) > 1 and is_fx_node(node.args[1])): - ctx.fail(f"unsupported pow signature for {torch_op}: scalar base with non-literal exponent {node.args!r}") - if float(base_literal) <= 0.0: - ctx.fail(f"unsupported pow signature for {torch_op}: non-positive scalar base {base_literal!r}") - - exponent_value_id = value_id(node.args[1], ctx) - mul_output_id = f"{value_id(node, ctx)}__pow_scalar_base_log_mul" - mul_node = IRNode( - id=f"{node_id(node)}__pow_scalar_base_log_mul", - op="scalar_multiply", - inputs=[exponent_value_id], - outputs=[mul_output_id], - attrs={"value": float(math.log(float(base_literal)))}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, mul_node, shape=shape, dtype=dtype) - - exp_node = IRNode( - id=node_id(node), - op="scalar_exp", - inputs=[mul_output_id], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, exp_node, shape=shape, dtype=dtype) - return - - inputs = [value_id(node.args[0], ctx)] - if exponent_literal is None and len(node.args) > 1 and is_fx_node(node.args[1]): - inputs.append(value_id(node.args[1], ctx)) - else: - attrs["exponent"] = exponent_literal - ir_node = IRNode( - id=node_id(node), - op="pow", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_slice(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - attrs = {"axis": int(node.args[1]), "start": int(node.args[2]), "end": int(node.args[3])} - if len(node.args) > 4 and extract_literals(node.args[4]) is not None: - attrs["step"] = int(node.args[4]) - ir_node = IRNode( - id=node_id(node), - op="slice", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = list(getattr(node, "args", ())) - if not inputs: - ctx.fail(f"unsupported diff signature for {torch_op}: missing input tensor") - - source = inputs[0] - if not is_fx_node(source): - ctx.fail(f"unsupported diff signature for {torch_op}: source is not FX node") - - n_value = 1 - if len(inputs) > 1 and extract_literals(inputs[1]) is not None: - n_value = int(extract_literals(inputs[1])) - elif "n" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["n"]) is not None: - n_value = int(extract_literals(node.kwargs["n"])) - if n_value != 1: - ctx.fail(f"unsupported diff order for {torch_op}: n={n_value}") - - dim_value = None - if len(inputs) > 2 and extract_literals(inputs[2]) is not None: - dim_value = int(extract_literals(inputs[2])) - elif "dim" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dim"]) is not None: - dim_value = int(extract_literals(node.kwargs["dim"])) - if dim_value is None: - ctx.fail(f"unsupported diff signature for {torch_op}: missing integer dim") - - prepend_value = None - append_value = None - if len(inputs) > 3: - prepend_value = extract_literals(inputs[3]) - if len(inputs) > 4: - append_value = extract_literals(inputs[4]) - if "prepend" in getattr(node, "kwargs", {}): - prepend_value = extract_literals(node.kwargs["prepend"]) - if "append" in getattr(node, "kwargs", {}): - append_value = extract_literals(node.kwargs["append"]) - if prepend_value is not None or append_value is not None: - ctx.fail(f"unsupported diff signature for {torch_op}: prepend/append are not supported") - - source_value_id = value_id(source, ctx) - source_shape = None - if source_value_id in ir.values: - source_shape = ir.values[source_value_id].shape - if source_shape is None: - source_shape = shape - if source_shape is None: - ctx.fail(f"unsupported diff import for {torch_op}: missing source shape") - - rank = len(source_shape) - normalized_dim = dim_value if dim_value >= 0 else dim_value + rank - if normalized_dim < 0 or normalized_dim >= rank: - ctx.fail(f"unsupported diff dimension for {torch_op}: dim={dim_value} rank={rank}") - - dim_extent = int(source_shape[normalized_dim]) - if dim_extent < 1: - ctx.fail(f"unsupported diff input extent for {torch_op}: dim size {dim_extent}") - - left_shape = list(source_shape) - left_shape[normalized_dim] = max(0, dim_extent - 1) - left_shape_tuple = tuple(int(v) for v in left_shape) - - left_node = IRNode( - id=f"{node_id(node)}__diff_left", - op="slice", - inputs=[source_value_id], - outputs=[f"{value_id(node, ctx)}__diff_left"], - attrs={"axis": normalized_dim, "start": 0, "end": dim_extent - 1, "step": 1}, - meta=_base_meta(left_shape_tuple, dtype, torch_op, node), - ) - register_node(ir, left_node, shape=left_shape_tuple, dtype=dtype) - - right_node = IRNode( - id=f"{node_id(node)}__diff_right", - op="slice", - inputs=[source_value_id], - outputs=[f"{value_id(node, ctx)}__diff_right"], - attrs={"axis": normalized_dim, "start": 1, "end": dim_extent, "step": 1}, - meta=_base_meta(left_shape_tuple, dtype, torch_op, node), - ) - register_node(ir, right_node, shape=left_shape_tuple, dtype=dtype) - - diff_node = IRNode( - id=node_id(node), - op="subtract", - inputs=[right_node.outputs[0], left_node.outputs[0]], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, diff_node, shape=shape, dtype=dtype) - - -def import_index(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if torch_op.startswith("aten.select"): - axis = int(node.args[1]) - index_value = int(node.args[2]) - else: - index_value = int(node.args[1]) - axis = int(node.args[2]) if len(node.args) > 2 else 0 - ir_node = IRNode( - id=node_id(node), - op="index", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"index_value": index_value, "axis": axis}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_gather(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="gather", - inputs=[value_id(node.args[0], ctx), value_id(node.args[2], ctx)], - outputs=[value_id(node, ctx)], - attrs={"axis": int(node.args[1])}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_embedding(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] - attrs: dict[str, object] = {} - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - attrs["padding_idx"] = int(node.args[2]) - if len(node.args) > 3 and extract_literals(node.args[3]) is not None: - attrs["scale_grad_by_freq"] = bool(node.args[3]) - if len(node.args) > 4 and extract_literals(node.args[4]) is not None: - attrs["sparse"] = bool(node.args[4]) - ir_node = IRNode( - id=node_id(node), - op="embedding", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def _extract_int_list_or_scalar(value: Any, *, default: int) -> int: - literal = extract_literals(value) - if literal is None: - return default - if isinstance(literal, int): - return int(literal) - if isinstance(literal, (tuple, list)) and literal: - first = literal[0] - if isinstance(first, int): - return int(first) - raise UnsupportedImportError(f"unsupported integer/list literal: {value!r}") - - -def import_conv1d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported conv1d signature for {torch_op}: {node.args!r}") - - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] - if len(node.args) > 2 and is_fx_node(node.args[2]): - inputs.append(value_id(node.args[2], ctx)) - - stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 - padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 - dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 - groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 - - ir_node = IRNode( - id=node_id(node), - op="conv1d", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_conv2d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported conv2d signature for {torch_op}: {node.args!r}") - - inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] - if len(node.args) > 2 and is_fx_node(node.args[2]): - inputs.append(value_id(node.args[2], ctx)) - - stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 - padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 - dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 - groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 - - ir_node = IRNode( - id=node_id(node), - op="conv2d", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_pad(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if len(node.args) < 2: - ctx.fail(f"unsupported pad signature for {torch_op}: {node.args!r}") - - pads = extract_literals(node.args[1]) - if not isinstance(pads, (tuple, list)) or not all(isinstance(v, int) for v in pads): - ctx.fail(f"unsupported pads for {torch_op}: {node.args!r}") - - mode = "constant" - if len(node.args) > 2 and extract_literals(node.args[2]) is not None: - mode = str(extract_literals(node.args[2])) - if "mode" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["mode"]) is not None: - mode = str(extract_literals(node.kwargs["mode"])) - - value = 0.0 - if len(node.args) > 3 and extract_literals(node.args[3]) is not None: - value = float(extract_literals(node.args[3])) - if "value" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["value"]) is not None: - value = float(extract_literals(node.kwargs["value"])) - - ir_node = IRNode( - id=node_id(node), - op="pad", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"pads": tuple(int(v) for v in pads), "mode": mode, "value": value}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if not node.args: - ctx.fail(f"unsupported ones signature for {torch_op}: {node.args!r}") - - output_shape = _extract_int_sequence_literal(node.args[0]) - if output_shape is None: - ctx.fail(f"unsupported ones shape for {torch_op}: {node.args!r}") - - node_dtype = dtype - if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None: - node_dtype = dtype_to_ir(extract_literals(node.kwargs["dtype"])) - elif len(node.args) > 1 and extract_literals(node.args[1]) is not None: - node_dtype = dtype_to_ir(extract_literals(node.args[1])) - if node_dtype is None: - node_dtype = "fp32" - - ir_node = IRNode( - id=node_id(node), - op="ones", - inputs=[], - outputs=[value_id(node, ctx)], - attrs={"shape": output_shape, "dtype": node_dtype}, - meta=_base_meta(shape or output_shape, node_dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape or output_shape, dtype=node_dtype) - - -def import_layer_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - if not node.args: - ctx.fail(f"unsupported layer_norm signature for {torch_op}: {node.args!r}") - - normalized_shape_value = None - if len(node.args) > 1: - normalized_shape_value = _extract_int_sequence_literal(node.args[1]) - if normalized_shape_value is None: - normalized_shape_value = _extract_int_sequence_literal(getattr(node, "kwargs", {}).get("normalized_shape")) - if normalized_shape_value is None: - ctx.fail(f"unsupported normalized_shape for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") - - eps_value = None - if len(node.args) > 4: - eps_value = _extract_numeric_literal(node.args[4]) - if eps_value is None: - eps_value = _extract_numeric_literal(getattr(node, "kwargs", {}).get("eps")) - if eps_value is None: - eps_value = 1e-5 - - inputs = [value_id(node.args[0], ctx)] - weight = node.args[2] if len(node.args) > 2 else getattr(node, "kwargs", {}).get("weight") - bias = node.args[3] if len(node.args) > 3 else getattr(node, "kwargs", {}).get("bias") - if not is_fx_node(weight): - ctx.fail(f"unsupported layer_norm weight form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") - inputs.append(value_id(weight, ctx)) - if bias is None: - pass - elif is_fx_node(bias): - inputs.append(value_id(bias, ctx)) - else: - ctx.fail(f"unsupported layer_norm bias form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") - - attrs = {"normalized_shape": tuple(int(v) for v in normalized_shape_value), "eps": float(eps_value)} - ir_node = IRNode( - id=node_id(node), - op="layer_norm", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_rms_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx)] - attrs = {"normalized_shape": tuple(int(v) for v in node.args[1]), "eps": float(node.args[3])} - ir_node = IRNode( - id=node_id(node), - op="rms_norm", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_group_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx), value_id(node.args[3], ctx)] - attrs = {"num_groups": int(node.args[1]), "eps": float(node.args[4])} - ir_node = IRNode( - id=node_id(node), - op="group_norm", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_batch_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - inputs = [ - value_id(node.args[0], ctx), - value_id(node.args[1], ctx), - value_id(node.args[2], ctx), - value_id(node.args[3], ctx), - value_id(node.args[4], ctx), - ] - attrs = {"training": bool(node.args[5]), "momentum": float(node.args[6]), "eps": float(node.args[7])} - ir_node = IRNode( - id=node_id(node), - op="batch_norm", - inputs=inputs, - outputs=[value_id(node, ctx)], - attrs=attrs, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_contiguous(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - ir_node = IRNode( - id=node_id(node), - op="contiguous", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: - index_value = extract_literals(node.args[1]) - if not isinstance(index_value, int): - ctx.fail(f"unsupported getitem index for {torch_op}: {node.args!r}") - ir_node = IRNode( - id=node_id(node), - op="getitem", - inputs=[value_id(node.args[0], ctx)], - outputs=[value_id(node, ctx)], - attrs={"index": index_value}, - meta=_base_meta(shape, dtype, torch_op, node), - ) - register_node(ir, ir_node, shape=shape, dtype=dtype) - - -OP_IMPORTERS = { - "arange": import_arange, - "identity": import_identity, - "add": import_add, - "subtract": import_subtract, - "multiply": import_multiply, - "multiply_inplace": import_multiply_inplace, - "divide": import_divide, - "precision_cast": import_precision_cast, - "type_as": import_type_as, - "negate": import_negate, - "abs": import_unary("abs"), - "clamp": import_clamp, - "not_equal": import_not_equal, - "equal": import_equal, - "logical_and": import_binary("logical_and"), - "logical_or": import_binary("logical_or"), - "logical_not": import_unary("logical_not"), - "where": import_where, - "masked_scatter": import_masked_scatter, - "masked_fill": import_masked_fill, - "cos": import_unary("cos"), - "sin": import_unary("sin"), - "floor_divide": import_floor_divide, - "scalar_exp": import_unary("scalar_exp"), - "scalar_sqrt": import_unary("scalar_sqrt"), - "reciprocal": import_reciprocal, - "scalar_log": import_unary("scalar_log"), - "rsqrt": import_pow_rsqrt, - "reshape": import_reshape, - "flatten": import_flatten, - "unsqueeze": import_unsqueeze, - "squeeze": import_squeeze, - "expand": import_expand, - "one_hot": import_one_hot, - "tril": import_tril, - "unfold": import_unfold, - "numpy_T": import_numpy_t, - "transpose": import_transpose, - "permute": import_permute, - "movedim": import_movedim, - "matmul": import_matmul, - "linear": import_linear, - "addmm": import_addmm, - "relu": import_unary("relu"), - "silu": import_unary("silu"), - "gelu": import_unary("gelu"), - "gelu_erf": import_unary("gelu_erf"), - "sigmoid": import_unary("sigmoid"), - "glu": import_glu, - "softplus": import_unary("softplus"), - "tanh": import_unary("tanh"), - "softmax": import_softmax, - "scaled_dot_product_attention": import_scaled_dot_product_attention, - "sum": import_reduce("sum"), - "mean": import_reduce("mean"), - "variance": import_reduce("variance"), - "min": import_reduce("min"), - "max": import_reduce("max"), - "cat": import_cat, - "split_with_sizes": import_split_with_sizes, - "chunk": import_chunk, - "ones": import_ones, - "pad": import_pad, - "pow": import_pow, - "aten.diff.default": import_diff, - "slice": import_slice, - "index": import_index, - "gather": import_gather, - "embedding": import_embedding, - "conv1d": import_conv1d, - "conv2d": import_conv2d, - "layer_norm": import_layer_norm, - "rms_norm": import_rms_norm, - "group_norm": import_group_norm, - "batch_norm": import_batch_norm, - "contiguous": import_contiguous, - "getitem": import_getitem, -} diff --git a/python/src/transpile/lower.py b/python/src/transpile/lower.py deleted file mode 100644 index 5784e4cac..000000000 --- a/python/src/transpile/lower.py +++ /dev/null @@ -1,2142 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import os -from typing import Any - -import torch - -from src.graph import Graph -from src.graph import Tensor -from src.transpile.capture_pytorch import CapturedModel -from src.transpile.canonicalize.cleanup import canonicalize_exported_graph -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.graph_ir import IRValue -from src.transpile.graph_ir import verify_ir -from src.transpile.optimize_graph import optimize_graph -from src.transpile.weight_binding import WeightBinding - - -@dataclass -class TranspiledGraph: - graph: Graph - runtime_inputs: list[Tensor] - bound_constants: list[Tensor] - outputs: list[Tensor] - - def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: - if index < 0 or index >= len(self.runtime_inputs): - raise IndexError( - f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" - ) - self.graph.set_input(self.runtime_inputs[index], data, dtype=dtype) - - def set_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> None: - if len(inputs) != len(self.runtime_inputs): - raise ValueError( - f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" - ) - for index, value in enumerate(inputs): - self.set_input(index, value) - - def execute(self) -> list[Tensor]: - self.graph.execute() - return self.outputs - - -@dataclass -class BroadcastAlias: - tensor: Tensor - logical_shape: tuple[int, ...] - kind: str - - -def transpile_captured(captured: CapturedModel) -> TranspiledGraph: - return transpile_ir(captured.ir_graph) - - -def transpile_ir(ir: IRGraph) -> TranspiledGraph: - verify_ir(ir) - canonicalize_exported_graph(ir) - optimize_graph(ir) - g = Graph() - env: dict[str, Any] = {} - runtime_inputs: list[Tensor] = [] - bound_constants: list[Tensor] = [] - - for value_id in ir.inputs: - value = ir.values[value_id] - tensor = _lower_input_value(g, value) - env[value_id] = tensor - runtime_inputs.append(tensor) - - for value_id, const in ir.constants.items(): - value = ir.values[value_id] - lowered_const = _lower_constant_value(g, value, const) - env[value_id] = lowered_const - if isinstance(lowered_const, Tensor): - bound_constants.append(lowered_const) - - for node_id in ir.order: - node = ir.nodes[node_id] - outputs = _lower_ir_node(g, node, env, ir) - if len(outputs) != len(node.outputs): - raise ValueError( - f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" - ) - for output_id, tensor in zip(node.outputs, outputs): - env[output_id] = tensor - - outputs = [env[value_id] for value_id in ir.outputs] - return TranspiledGraph( - graph=g, - runtime_inputs=runtime_inputs, - bound_constants=bound_constants, - outputs=outputs, - ) - - -def _lower_input_value(g: Graph, value: IRValue) -> Tensor: - if value.shape is None or value.dtype is None: - raise ValueError(f"IR input missing shape or dtype: {value.id}") - return g.input(shape=value.shape, dtype=_map_ir_dtype(value.dtype)) - -#Need to work on this for mmmaps and constants that can be bound as tensors vs materialized as literals -- Fuck this shit I fucking hate my life -def _lower_constant_value(g: Graph, value: IRValue, const: Any) -> Any: - binding = _lookup_weight_binding(value) - if binding is not None: - _debug_mmap_binding(value.id, binding) - if binding.kind == "embedding": - print("generating embedding tensor") # Observe which shit actually has embeddings generated - return g.mmap_embeddings(binding.path) - return g.mmap_weights(binding.path) - - _debug_constant_fallback(value, const) - - if isinstance(const, torch.nn.Parameter): - const = const.detach() - if isinstance(const, torch.Tensor): - tensor_value = const.detach().cpu() - else: - raise NotImplementedError( - f"unsupported IR constant type for {value.id}: {type(const).__name__}" - ) - - if tensor_value.numel() == 1: - return tensor_value.item() - - return _materialize_constant_tensor(g, tensor_value) - - -def _lookup_weight_binding(value: IRValue) -> WeightBinding | None: - meta = getattr(value, "meta", None) - if isinstance(meta, dict): - path = meta.get("path") - kind = meta.get("kind") - source_name = meta.get("source_name") - if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): - return WeightBinding(path=path, kind=kind, source_name=source_name) - return None - - -def _debug_mmap_binding(value_id: str, binding: WeightBinding) -> None: - if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": - return - print( - "[transpile:mmap] " - f"value={value_id} " - f"kind={binding.kind} " - f"source={binding.source_name} " - f"path={binding.path}" - ) - - -def _debug_constant_fallback(value: IRValue, const: Any) -> None: - if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": - return - const_type = type(const).__name__ - shape = None - dtype = None - if isinstance(const, torch.Tensor): - shape = tuple(const.shape) - dtype = str(const.dtype) - source_name = None - if isinstance(value.meta, dict): - source_name = value.meta.get("source_name") - print( - "[transpile:fallback] " - f"value={value.id} " - f"source={source_name} " - f"const_type={const_type} " - f"shape={shape} " - f"dtype={dtype}" - ) - - -def _debug_embedding_lowering(node: IRNode, embedding_tensor: Tensor, indices_tensor: Tensor) -> None: - if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": - return - print( - "[transpile:embedding] " - f"node={node.id} " - f"embedding_id={embedding_tensor.id} " - f"embedding_shape={tuple(embedding_tensor.shape)} " - f"embedding_dtype={embedding_tensor.dtype} " - f"indices_id={indices_tensor.id} " - f"indices_shape={tuple(indices_tensor.shape)} " - f"indices_dtype={indices_tensor.dtype}" - ) - - -def _matmul_with_quantized_rhs_legalization( - g: Graph, - lhs: Tensor, - rhs: Tensor, - *, - pretransposed_rhs: bool = False, -) -> Tensor: - if rhs.dtype in (Graph.INT8, Graph.INT4) and lhs.dtype == Graph.FP32: - lhs = g.precision_cast(lhs, Graph.FP16) - return g.matmul(lhs, rhs, pretransposed_rhs=pretransposed_rhs) - - -def _legalize_for_transpose(g: Graph, x: Tensor) -> Tensor: - if x.dtype == Graph.FP32: - return g.precision_cast(x, Graph.FP16) - return x - - -def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> list[Any]: - op = node.op - - if op == "arange": - start = int(node.attrs.get("start", 0)) - end = int(node.attrs["end"]) - step = int(node.attrs.get("step", 1)) - dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) - tensor_value = torch.arange(start, end, step=step, dtype=dtype) - return [_materialize_constant_tensor(g, tensor_value)] - - if op == "add": - return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "add")] - - if op == "add_clipped": - lhs, rhs = _legalize_elementwise_binary_inputs(g, _tensor(env, node.inputs[0]), _tensor(env, node.inputs[1])) - return [g.add_clipped(lhs, rhs)] - - if op == "subtract": - return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "subtract")] - - if op == "multiply": - return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] - - if op == "multiply_inplace": - return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] - - if op == "divide": - return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "divide")] - - if op == "not_equal": - return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "not_equal")] - - if op == "equal": - return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "equal")] - - if op == "greater": - return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "greater")] - - if op == "less_equal": - return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "less_equal")] - - if op == "logical_and": - lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") - rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") - return [_lower_binary_op(g, lhs, rhs, "multiply")] - - if op == "logical_or": - lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") - rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") - return [_lower_compare_op(g, _lower_binary_op(g, lhs, rhs, "add"), 0.0, "not_equal")] - - if op == "logical_not": - return [_lower_compare_op(g, env[node.inputs[0]], 0.0, "equal")] - - if op == "where": - return [_lower_where_op(g, node, env)] - - if op == "scalar_add": - return [g.scalar_add(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] - - if op == "scalar_subtract": - return [g.scalar_subtract(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] - - if op == "scalar_subtract_reverse": - x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) - return [g.scalar_add(g.scalar_multiply(x, -1.0), float(node.attrs["value"]))] - - if op == "scalar_multiply": - return [g.scalar_multiply(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] - - if op == "scalar_divide": - return [g.scalar_divide(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] - - if op == "scalar_floor_divide": - source = _tensor(env, node.inputs[0]) - x = _ensure_fp16_tensor(g, source) - y = g.scalar_floor_divide(x, float(node.attrs["value"])) - output_dtype = ir.values[node.outputs[0]].dtype - if output_dtype is None: - return [y] - target_dtype = _map_ir_dtype(output_dtype) - if y.dtype == target_dtype: - return [y] - return [g.precision_cast(y, target_dtype)] - - if op == "scalar_not_equal": - return [g.scalar_not_equal(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"]))] - - if op == "scalar_equal": - not_equal = g.scalar_not_equal(_ensure_fp16_tensor(g, _tensor(env, node.inputs[0])), float(node.attrs["value"])) - return [g.scalar_not_equal(not_equal, 1.0)] - - if op == "scalar_divide_reverse": - raise NotImplementedError("scalar_divide_reverse is not directly supported by Cactus graph ops") - - if op == "precision_cast": - target_dtype = node.attrs.get("dtype") - if target_dtype is None: - return [_tensor(env, node.inputs[0])] - return [g.precision_cast(_tensor(env, node.inputs[0]), _map_ir_or_torch_dtype(target_dtype))] - - if op == "type_as": - source = _tensor(env, node.inputs[0]) - target = _tensor(env, node.inputs[1]) - return [g.precision_cast(source, target.dtype)] - - if op == "abs": - return [g.abs(_tensor(env, node.inputs[0]))] - - if op == "clamp": - x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) - min_value = node.attrs.get("min") - max_value = node.attrs.get("max") - if min_value is not None: - x = g.scalar_add(g.relu(g.scalar_subtract(x, float(min_value))), float(min_value)) - if max_value is not None: - upper_delta = g.relu(g.scalar_add(g.scalar_multiply(x, -1.0), float(max_value))) - x = g.scalar_add(g.scalar_multiply(upper_delta, -1.0), float(max_value)) - return [x] - - if op == "negate": - return [g.scalar_multiply(_tensor(env, node.inputs[0]), -1.0)] - - if op == "pow": - exponent = node.attrs.get("exponent") - if exponent is None: - raise NotImplementedError("tensor-tensor pow is not supported by Cactus graph ops") - return [g.pow(_tensor(env, node.inputs[0]), float(exponent))] - - if op == "scalar_exp": - return [g.scalar_exp(_tensor(env, node.inputs[0]))] - - if op == "scalar_sqrt": - return [g.scalar_sqrt(_tensor(env, node.inputs[0]))] - - if op == "scalar_log": - return [g.scalar_log(_tensor(env, node.inputs[0]))] - - if op == "scalar_cos": - return [g.scalar_cos(_tensor(env, node.inputs[0]))] - - if op == "scalar_sin": - return [g.scalar_sin(_tensor(env, node.inputs[0]))] - - if op in {"reshape", "view"}: - source = env[node.inputs[0]] - if isinstance(source, BroadcastAlias): - input_shape = tuple(source.logical_shape) - else: - input_shape = tuple(_tensor(env, node.inputs[0]).shape) - target_shape = _resolve_reshape_shape(input_shape, tuple(node.attrs["shape"])) - if isinstance(source, BroadcastAlias): - if source.kind != "gqa_repeat_kv": - raise NotImplementedError(f"unsupported broadcast alias kind in reshape: {source.kind}") - base_shape = tuple(source.tensor.shape) - if len(base_shape) != 4: - raise NotImplementedError(f"gqa_repeat_kv alias requires 4D base tensor, got {base_shape}") - if len(target_shape) != 4: - raise NotImplementedError(f"gqa_repeat_kv reshape requires 4D target shape, got {target_shape}") - batch, kv_heads, seq_len, head_dim = base_shape - if target_shape[0] != batch or target_shape[2] != seq_len or target_shape[3] != head_dim: - raise NotImplementedError( - f"gqa_repeat_kv reshape mismatch: base {base_shape}, target {target_shape}" - ) - if target_shape[1] % max(kv_heads, 1) != 0: - raise NotImplementedError( - f"gqa_repeat_kv target head count must be a multiple of kv heads: {base_shape} -> {target_shape}" - ) - return [BroadcastAlias(source.tensor, target_shape, source.kind)] - return [g.reshape(_tensor(env, node.inputs[0]), target_shape)] - - if op == "flatten": - return [ - g.flatten( - _tensor(env, node.inputs[0]), - start_dim=int(node.attrs["start_dim"]), - end_dim=int(node.attrs["end_dim"]), - ) - ] - - if op == "unsqueeze": - x = _tensor(env, node.inputs[0]) - target_shape = node.attrs.get("shape") - if target_shape is None: - dim = _normalize_dim(int(node.attrs["dim"]), len(x.shape) + 1) - shape_list = list(x.shape) - shape_list.insert(dim, 1) - target_shape = tuple(shape_list) - return [g.reshape(x, tuple(target_shape))] - - if op == "expand": - matched_base = _match_gqa_expand_alias(ir, node) - if matched_base is not None: - return [BroadcastAlias(_tensor(env, matched_base), tuple(node.attrs["shape"]), "gqa_repeat_kv")] - x = _tensor(env, node.inputs[0]) - target_shape = _resolve_expand_shape(tuple(x.shape), tuple(node.attrs["shape"])) - if target_shape == tuple(x.shape): - return [x] - return [g.expand(x, target_shape)] - - if op == "one_hot": - indices = _tensor(env, node.inputs[0]) - num_classes = int(node.attrs["num_classes"]) - eye = torch.eye(num_classes, dtype=torch.float16) - embedding = _materialize_constant_tensor(g, eye) - return [g.embedding_from_tensor(embedding, indices)] - - if op == "transpose": - x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) - dim0 = _normalize_dim(int(node.attrs["dim0"]), len(x.shape)) - dim1 = _normalize_dim(int(node.attrs["dim1"]), len(x.shape)) - rank = len(x.shape) - permutation = list(range(rank)) - permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] - if rank == 2 and permutation == [1, 0]: - return [g.transpose(x)] - return [g.permute(x, permutation)] - - if op == "permute": - x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) - permutation = tuple(_normalize_dim(int(dim), len(x.shape)) for dim in node.attrs["permutation"]) - if len(permutation) == 2 and permutation == (1, 0): - return [g.transpose(x)] - return [g.permute(x, permutation)] - - if op == "matmul": - lhs = _tensor(env, node.inputs[0]) - rhs = _tensor(env, node.inputs[1]) - if len(lhs.shape) == 2 and len(rhs.shape) == 2: - return [_matmul_with_quantized_rhs_legalization(g, lhs, rhs)] - legalized = _legalize_matmul_inputs(g, lhs, rhs, node) - if legalized is None: - static_batched = _lower_static_batched_matmul(g, lhs, rhs) - if static_batched is not None: - return [static_batched] - return [_matmul_with_quantized_rhs_legalization(g, lhs, rhs)] - lhs_2d, rhs_2d, output_shape = legalized - out = _matmul_with_quantized_rhs_legalization(g, lhs_2d, rhs_2d) - return [g.reshape(out, output_shape)] - - if op == "linear": - x = _tensor(env, node.inputs[0]) - weight = _tensor(env, node.inputs[1]) - reshape_back: tuple[int, ...] | None = None - if len(x.shape) > 2: - x = _flatten_to_2d_for_linear(g, x) - output_value = ir.values.get(node.outputs[0]) - output_shape = output_value.shape if output_value is not None else None - if output_shape is None: - output_shape = node.meta.get("shape") - if not isinstance(output_shape, tuple): - raise NotImplementedError(f"linear missing output shape metadata for node {node.id}") - reshape_back = tuple(int(v) for v in output_shape) - - out = _matmul_with_quantized_rhs_legalization(g, x, weight, pretransposed_rhs=True) - if node.attrs.get("has_bias"): - out = g.add(out, _tensor(env, node.inputs[2])) - if reshape_back is not None: - out = g.reshape(out, reshape_back) - return [out] - - if op == "addmm": - bias = _tensor(env, node.inputs[0]) - lhs = _tensor(env, node.inputs[1]) - rhs = _tensor(env, node.inputs[2]) - return [g.add(bias, _matmul_with_quantized_rhs_legalization(g, lhs, rhs))] - - if op == "relu": - return [g.relu(_tensor(env, node.inputs[0]))] - - if op == "silu": - return [g.silu(_tensor(env, node.inputs[0]))] - - if op == "gelu": - return [g.gelu(_tensor(env, node.inputs[0]))] - - if op == "gelu_erf": - return [g.gelu_erf(_tensor(env, node.inputs[0]))] - - if op == "sigmoid": - return [g.sigmoid(_tensor(env, node.inputs[0]))] - - if op == "glu": - return [g.glu(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] - - if op == "softplus": - x = _tensor(env, node.inputs[0]) - return [_lower_softplus(g, x)] - - if op == "tanh": - return [g.tanh(_tensor(env, node.inputs[0]))] - - if op == "softmax": - return [g.softmax(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] - - if op in {"scaled_dot_product_attention", "attention"}: - mask = node.attrs.get("mask") - mask_tensor: Tensor | None = None - additive_mask = bool(node.attrs.get("additive_mask", False)) - if len(node.inputs) > 3: - mask_tensor = _tensor(env, node.inputs[3]) - elif mask is not None: - raise NotImplementedError(f"{op} with literal mask is not supported yet") - if op == "scaled_dot_product_attention": - dropout_p = float(node.attrs.get("dropout_p", 0.0)) - if dropout_p != 0.0: - raise NotImplementedError("scaled_dot_product_attention with dropout is not supported yet") - - # PyTorch SDPA exports tensors as [batch, heads, seq, dim], while Cactus attention - # expects [batch, seq, heads, dim]. Convert into Cactus layout, call attention, then - # convert back so downstream exported ops still see PyTorch layout. - query = g.permute(_attention_tensor(env, node.inputs[0]), (0, 2, 1, 3)) - key = g.permute(_attention_tensor(env, node.inputs[1]), (0, 2, 1, 3)) - value = g.permute(_attention_tensor(env, node.inputs[2]), (0, 2, 1, 3)) - out = g.attention( - query, - key, - value, - scale=float(node.attrs.get("scale", 1.0)), - is_causal=bool(node.attrs.get("is_causal", False)), - window_size=int(node.attrs.get("window_size", 0)), - mask=mask_tensor, - additive_mask=additive_mask, - ) - return [g.permute(out, (0, 2, 1, 3))] - - if op == "attention_block": - has_mask = bool(node.attrs.get("has_mask", False)) - has_gate = bool(node.attrs.get("has_gate", False)) - has_bias = bool(node.attrs.get("has_bias", False)) - input_index = 0 - query = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) - input_index += 1 - key = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) - input_index += 1 - value = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) - input_index += 1 - mask_tensor: Tensor | None = None - if has_mask: - mask_tensor = _tensor(env, node.inputs[input_index]) - input_index += 1 - - attn_out = g.attention( - query, - key, - value, - scale=float(node.attrs.get("scale", 1.0)), - is_causal=bool(node.attrs.get("is_causal", False)), - window_size=int(node.attrs.get("window_size", 0)), - mask=mask_tensor, - additive_mask=bool(node.attrs.get("additive_mask", False)), - ) - flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) - flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () - if flat_shape: - attn_out = g.reshape(attn_out, flat_shape) - - if has_gate: - gate = _tensor(env, node.inputs[input_index]) - input_index += 1 - attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) - attn_out = g.multiply(attn_out, gate) - - weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - reshape_back: tuple[int, ...] | None = None - linear_input = attn_out - if len(linear_input.shape) > 2: - linear_input = _flatten_to_2d_for_linear(g, linear_input) - output_value = ir.values.get(node.outputs[0]) - output_shape = output_value.shape if output_value is not None else None - if output_shape is None: - output_shape = node.meta.get("shape") - if not isinstance(output_shape, tuple): - raise NotImplementedError(f"attention_block missing output shape metadata for node {node.id}") - reshape_back = tuple(int(v) for v in output_shape) - - out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) - if has_bias: - out = g.add(out, _tensor(env, node.inputs[input_index])) - if reshape_back is not None: - out = g.reshape(out, reshape_back) - return [out] - - if op == "self_attention_block": - has_mask = bool(node.attrs.get("has_mask", False)) - has_gate = bool(node.attrs.get("has_gate", False)) - has_bias = bool(node.attrs.get("has_bias", False)) - has_query_projection_bias = bool(node.attrs.get("has_query_projection_bias", False)) - has_query_add = bool(node.attrs.get("has_query_add", False)) - has_rel_query_add = bool(node.attrs.get("has_rel_query_add", False)) - has_key_projection_bias = bool(node.attrs.get("has_key_projection_bias", False)) - has_value_projection_bias = bool(node.attrs.get("has_value_projection_bias", False)) - has_rel_pos_bias = bool(node.attrs.get("has_rel_pos_bias", False)) - has_relative_key_projection_bias = bool(node.attrs.get("has_relative_key_projection_bias", False)) - - input_index = 0 - hidden = _tensor(env, node.inputs[input_index]) - input_index += 1 - - query_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - query_projection_bias = None - if has_query_projection_bias: - query_projection_bias = _tensor(env, node.inputs[input_index]) - input_index += 1 - query_add = None - if has_query_add: - query_add = _tensor(env, node.inputs[input_index]) - input_index += 1 - rel_query_add = None - if has_rel_query_add: - rel_query_add = _tensor(env, node.inputs[input_index]) - input_index += 1 - - key_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - key_projection_bias = None - if has_key_projection_bias: - key_projection_bias = _tensor(env, node.inputs[input_index]) - input_index += 1 - - value_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - value_projection_bias = None - if has_value_projection_bias: - value_projection_bias = _tensor(env, node.inputs[input_index]) - input_index += 1 - - mask_tensor: Tensor | None = None - if has_mask: - mask_tensor = _tensor(env, node.inputs[input_index]) - input_index += 1 - - relative_key_input = None - relative_key_weight = None - relative_key_projection_bias = None - if has_rel_pos_bias: - relative_key_input = _tensor(env, node.inputs[input_index]) - input_index += 1 - relative_key_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - if has_relative_key_projection_bias: - relative_key_projection_bias = _tensor(env, node.inputs[input_index]) - input_index += 1 - - gate = None - if has_gate: - gate = _tensor(env, node.inputs[input_index]) - input_index += 1 - - output_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - output_bias = None - if has_bias: - output_bias = _tensor(env, node.inputs[input_index]) - - query_shape = tuple(int(v) for v in node.attrs.get("query_shape", ())) - key_shape = tuple(int(v) for v in node.attrs.get("key_shape", ())) - value_shape = tuple(int(v) for v in node.attrs.get("value_shape", ())) - relative_key_shape = tuple(int(v) for v in node.attrs.get("relative_key_shape", ())) - - query_base = _lower_projected_attention_tensor(g, hidden, query_weight, query_projection_bias, query_shape) - query = query_base - if query_add is not None: - query_add = _normalize_attention_add_tensor(g, query_add, query_shape) - query, query_add = _legalize_elementwise_binary_inputs(g, query, query_add) - query = g.add(query, query_add) - - key = _lower_projected_attention_tensor(g, hidden, key_weight, key_projection_bias, key_shape) - value = _lower_projected_attention_tensor(g, hidden, value_weight, value_projection_bias, value_shape) - - if has_rel_pos_bias: - if relative_key_input is None or relative_key_weight is None or not relative_key_shape: - raise NotImplementedError(f"self_attention_block missing relative position bias inputs for node {node.id}") - rel_query = query_base - if rel_query_add is not None: - rel_query_add = _normalize_attention_add_tensor(g, rel_query_add, query_shape) - rel_query, rel_query_add = _legalize_elementwise_binary_inputs(g, rel_query, rel_query_add) - rel_query = g.add(rel_query, rel_query_add) - relative_key = _lower_projected_attention_tensor( - g, - relative_key_input, - relative_key_weight, - relative_key_projection_bias, - relative_key_shape, - ) - rel_mask = g.rel_pos_bias(rel_query, relative_key, float(node.attrs.get("rel_pos_scale", 1.0))) - if mask_tensor is None: - mask_tensor = rel_mask - else: - mask_tensor, rel_mask = _legalize_elementwise_binary_inputs(g, mask_tensor, rel_mask) - mask_tensor = g.add(mask_tensor, rel_mask) - - attn_out = g.attention( - query, - key, - value, - scale=float(node.attrs.get("scale", 1.0)), - is_causal=bool(node.attrs.get("is_causal", False)), - window_size=int(node.attrs.get("window_size", 0)), - mask=mask_tensor, - additive_mask=bool(node.attrs.get("additive_mask", False)), - ) - flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) - flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () - if flat_shape: - attn_out = g.reshape(attn_out, flat_shape) - - if gate is not None: - attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) - attn_out = g.multiply(attn_out, gate) - - linear_input = attn_out - reshape_back: tuple[int, ...] | None = None - if len(linear_input.shape) > 2: - linear_input = _flatten_to_2d_for_linear(g, linear_input) - output_value = ir.values.get(node.outputs[0]) - output_shape = output_value.shape if output_value is not None else None - if output_shape is None: - output_shape = node.meta.get("shape") - if not isinstance(output_shape, tuple): - raise NotImplementedError(f"self_attention_block missing output shape metadata for node {node.id}") - reshape_back = tuple(int(v) for v in output_shape) - - out = _matmul_with_quantized_rhs_legalization(g, linear_input, output_weight, pretransposed_rhs=True) - if output_bias is not None: - out = g.add(out, output_bias) - if reshape_back is not None: - out = g.reshape(out, reshape_back) - return [out] - - if op in ("sum", "mean", "variance", "min", "max"): - axis = node.attrs.get("axis") - if isinstance(axis, (list, tuple)) and len(axis) == 1 and isinstance(axis[0], int): - axis = int(axis[0]) - if not isinstance(axis, int): - raise NotImplementedError(f"{op} currently requires a single integer axis") - fn = getattr(g, op) - x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) - out = fn(x, axis) - if bool(node.attrs.get("keepdim", False)): - reduced_axis = _normalize_dim(axis, len(x.shape)) - new_shape = list(x.shape) - new_shape[reduced_axis] = 1 - out = g.reshape(out, tuple(new_shape)) - return [out] - - if op == "cat": - tensors = [_tensor(env, value_id) for value_id in node.inputs] - output_dtype: int | None = None - if node.outputs: - output_value = ir.values.get(node.outputs[0]) - if output_value is not None and output_value.dtype is not None: - output_dtype = _map_ir_or_torch_dtype(output_value.dtype) - return [_cat_with_legalized_dtype(g, tensors, axis=int(node.attrs.get("axis", 0)), dtype=output_dtype)] - - if op in {"stack", "aten.stack.default"}: - tensors = [_tensor(env, value_id) for value_id in node.inputs] - if not tensors: - raise NotImplementedError("stack requires at least one input tensor") - - raw_axis = node.attrs.get("axis") - if not isinstance(raw_axis, int): - args = node.attrs.get("args", ()) - if isinstance(args, (list, tuple)) and len(args) > 1 and isinstance(args[1], int): - raw_axis = int(args[1]) - else: - raw_axis = 0 - - base_shape = tuple(int(dim) for dim in tensors[0].shape) - axis = _normalize_dim(int(raw_axis), len(base_shape) + 1) - reshaped: list[Tensor] = [] - for tensor in tensors: - if tuple(int(dim) for dim in tensor.shape) != base_shape: - raise NotImplementedError( - f"stack requires equal input shapes, got {[tuple(int(dim) for dim in t.shape) for t in tensors]}" - ) - expanded_shape = list(base_shape) - expanded_shape.insert(axis, 1) - reshaped.append(g.reshape(tensor, tuple(expanded_shape))) - return [g.cat(reshaped, axis=axis)] - - if op == "slice": - x = _tensor(env, node.inputs[0]) - axis = _normalize_dim(int(node.attrs["axis"]), len(x.shape)) - start = int(node.attrs["start"]) - end = int(node.attrs["end"]) - step = int(node.attrs.get("step", 1)) - dim_size = x.shape[axis] - if step == 0: - raise NotImplementedError("slice with step == 0 is invalid") - if step != 1: - indices = list(range(*slice(start, end, step).indices(dim_size))) - if not indices: - return [g.slice(x, axis=axis, start=0, length=0)] - expanded_shape = list(int(dim) for dim in x.shape) - expanded_shape[axis] = 1 - pieces: list[Tensor] = [] - for idx in indices: - piece = g.index(x, index_value=idx, axis=axis) - piece = g.reshape(piece, tuple(expanded_shape)) - pieces.append(piece) - if len(pieces) == 1: - return [pieces[0]] - return [g.cat(pieces, axis=axis)] - start = _normalize_index(start, dim_size) - end = _normalize_slice_end(end, dim_size) - length = max(0, end - start) - return [g.slice(x, axis=axis, start=start, length=length)] - - if op == "split_with_sizes": - x = _tensor(env, node.inputs[0]) - axis = _normalize_dim(int(node.attrs.get("axis", -1)), len(x.shape)) - sizes = tuple(int(v) for v in node.attrs["sizes"]) - start = 0 - outputs: list[Tensor] = [] - for size in sizes: - outputs.append(g.slice(x, axis=axis, start=start, length=int(size))) - start += int(size) - return [outputs] - - if op == "chunk": - x = _tensor(env, node.inputs[0]) - axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) - chunks = int(node.attrs["chunks"]) - if chunks <= 0: - raise NotImplementedError(f"chunk requires a positive chunk count, got {chunks}") - dim_size = int(x.shape[axis]) - if dim_size <= 0: - return [[g.slice(x, axis=axis, start=0, length=0)]] - chunk_size = (dim_size + chunks - 1) // chunks - outputs: list[Tensor] = [] - start = 0 - while start < dim_size: - length = min(chunk_size, dim_size - start) - outputs.append(g.slice(x, axis=axis, start=start, length=length)) - start += length - return [outputs] - - if op == "ones": - shape = tuple(int(v) for v in node.attrs.get("shape", ())) - if not shape: - raise NotImplementedError(f"ones requires a static shape, got {shape}") - torch_dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) - return [_materialize_constant_tensor(g, torch.ones(shape, dtype=torch_dtype))] - - if op == "tril": - x = _tensor(env, node.inputs[0]) - shape = tuple(int(dim) for dim in x.shape) - if len(shape) < 2: - raise NotImplementedError(f"tril expects rank >= 2, got shape {shape}") - diagonal = int(node.attrs.get("diagonal", 0)) - mask_dtype = _torch_dtype_for_graph_dtype(x.dtype) - mask = torch.ones(shape, dtype=mask_dtype).tril(diagonal=diagonal) - mask_tensor = _materialize_constant_tensor(g, mask) - return [_lower_binary_op(g, x, mask_tensor, "multiply")] - - if op == "unfold": - x = _tensor(env, node.inputs[0]) - rank = len(x.shape) - axis = _normalize_dim(int(node.attrs["dimension"]), rank) - size = int(node.attrs["size"]) - step = int(node.attrs["step"]) - if size <= 0: - raise NotImplementedError(f"unfold requires a positive size, got {size}") - if step <= 0: - raise NotImplementedError(f"unfold requires a positive step, got {step}") - dim_size = int(x.shape[axis]) - if size > dim_size: - raise NotImplementedError( - f"unfold size {size} exceeds dimension size {dim_size} for axis {axis}" - ) - - window_starts = list(range(0, dim_size - size + 1, step)) - if not window_starts: - raise NotImplementedError( - f"unfold produced no windows for axis={axis}, size={size}, step={step}, dim_size={dim_size}" - ) - - base_shape = [int(dim) for dim in x.shape] - trailing_shape = [base_shape[idx] for idx in range(rank) if idx != axis] + [size] - expanded_shape = tuple(trailing_shape[:axis] + [1] + trailing_shape[axis:]) - permutation = tuple(idx for idx in range(rank) if idx != axis) + (axis,) - - windows: list[Tensor] = [] - for start in window_starts: - piece = g.slice(x, axis=axis, start=start, length=size) - if axis != rank - 1: - piece = g.permute(piece, permutation=permutation) - piece = g.reshape(piece, expanded_shape) - windows.append(piece) - if len(windows) == 1: - return [windows[0]] - return [g.cat(windows, axis=axis)] - - if op == "pad": - x = _tensor(env, node.inputs[0]) - mode = str(node.attrs.get("mode", "constant")) - if mode != "constant": - raise NotImplementedError(f"pad mode is unsupported: {mode}") - pads = tuple(int(v) for v in node.attrs.get("pads", ())) - if len(pads) % 2 != 0: - raise NotImplementedError(f"pad expects an even-length pads tuple, got {pads}") - value = float(node.attrs.get("value", 0.0)) - - current = x - current_shape = list(int(dim) for dim in x.shape) - pad_dims = len(pads) // 2 - if pad_dims > len(current_shape): - raise NotImplementedError( - f"pad rank mismatch: pads={pads} for input shape {tuple(current_shape)}" - ) - - for pad_index in range(pad_dims): - before = pads[2 * pad_index] - after = pads[2 * pad_index + 1] - axis = len(current_shape) - 1 - pad_index - if before < 0 or after < 0: - raise NotImplementedError(f"negative pad is unsupported: {pads}") - pieces = [] - torch_dtype = _torch_dtype_for_graph_dtype(current.dtype) - if before > 0: - left_shape = list(current_shape) - left_shape[axis] = before - left = _materialize_constant_tensor( - g, - torch.full(tuple(left_shape), value, dtype=torch_dtype), - ) - pieces.append(left) - pieces.append(current) - if after > 0: - right_shape = list(current_shape) - right_shape[axis] = after - right = _materialize_constant_tensor( - g, - torch.full(tuple(right_shape), value, dtype=torch_dtype), - ) - pieces.append(right) - if len(pieces) > 1: - current = g.cat(pieces, axis=axis) - current_shape[axis] += before + after - return [current] - - if op == "index": - x = _tensor(env, node.inputs[0]) - axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) - index_value = _normalize_index(int(node.attrs["index_value"]), x.shape[axis]) - return [g.index(x, index_value=index_value, axis=axis)] - - if op == "gather": - return [g.gather(_tensor(env, node.inputs[0]), _tensor(env, node.inputs[1]))] - - if op == "embedding": - embedding_tensor = _tensor(env, node.inputs[0]) - indices_tensor = _tensor(env, node.inputs[1]) - _debug_embedding_lowering(node, embedding_tensor, indices_tensor) - return [g.embedding_from_tensor(embedding_tensor, indices_tensor)] - - if op == "conv_module": - input_index = 0 - x_nlc = _tensor(env, node.inputs[input_index]) - input_index += 1 - - pointwise1_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - pointwise1_bias = None - if bool(node.attrs.get("has_pointwise1_bias", False)): - pointwise1_bias = _tensor(env, node.inputs[input_index]) - input_index += 1 - - depthwise_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - depthwise_bias = None - if bool(node.attrs.get("has_depthwise_bias", False)): - depthwise_bias = _tensor(env, node.inputs[input_index]) - input_index += 1 - - batch_norm_weight = _tensor(env, node.inputs[input_index]) - batch_norm_bias = _tensor(env, node.inputs[input_index + 1]) - batch_norm_running_mean = _tensor(env, node.inputs[input_index + 2]) - batch_norm_running_var = _tensor(env, node.inputs[input_index + 3]) - input_index += 4 - - pointwise2_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - pointwise2_bias = None - if bool(node.attrs.get("has_pointwise2_bias", False)): - pointwise2_bias = _tensor(env, node.inputs[input_index]) - - if len(x_nlc.shape) != 3: - raise NotImplementedError(f"conv_module expects rank-3 NLC input, got {x_nlc.shape}") - kernel_size = int(node.attrs.get("depthwise_kernel_size", 0)) - padding = int(node.attrs.get("depthwise_padding", 0)) - if len(depthwise_weight.shape) != 3 or kernel_size != 9 or padding != 4: - raise NotImplementedError( - f"conv_module currently supports same depthwise kernel-9 lowering, got " - f"kernel_size={kernel_size} padding={padding} weight_shape={depthwise_weight.shape}" - ) - - current = g.conv1d_pointwise(x_nlc, pointwise1_weight, bias=pointwise1_bias) - current = g.glu(current, axis=-1) - current = g.conv1d_same_depthwise_k9(current, depthwise_weight, bias=depthwise_bias) - current = g.batch_norm( - current, - batch_norm_weight, - batch_norm_bias, - batch_norm_running_mean, - batch_norm_running_var, - axis=len(current.shape) - 1, - eps=float(node.attrs.get("eps", 1e-5)), - ) - current = g.silu(current) - current = g.conv1d_pointwise(current, pointwise2_weight, bias=pointwise2_bias) - return [current] - - if op == "conv1d": - x = _tensor(env, node.inputs[0]) - weight = _tensor(env, node.inputs[1]) - bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None - stride = int(node.attrs.get("stride", 1)) - padding = int(node.attrs.get("padding", 0)) - dilation = int(node.attrs.get("dilation", 1)) - groups = int(node.attrs.get("groups", 1)) - - if ( - len(x.shape) == 3 - and len(weight.shape) == 3 - and weight.shape[2] == 1 - and stride == 1 - and padding == 0 - and dilation == 1 - and groups == 1 - ): - x_nlc = g.permute(x, (0, 2, 1)) - out_nlc = g.conv1d_pointwise(x_nlc, weight, bias=bias) - return [g.permute(out_nlc, (0, 2, 1))] - - if ( - len(x.shape) == 3 - and len(weight.shape) == 3 - and groups == x.shape[1] == weight.shape[0] - and weight.shape[1] == 1 - and weight.shape[2] == 9 - and stride == 1 - and padding == 4 - and dilation == 1 - ): - x_nlc = g.permute(x, (0, 2, 1)) - out_nlc = g.conv1d_same_depthwise_k9(x_nlc, weight, bias=bias) - return [g.permute(out_nlc, (0, 2, 1))] - - if ( - len(x.shape) == 3 - and len(weight.shape) == 3 - and groups == x.shape[1] == weight.shape[0] - and weight.shape[1] == 1 - and stride == 1 - and padding == dilation * max(weight.shape[2] - 1, 0) - ): - x_nlc = g.permute(x, (0, 2, 1)) - out_nlc = g.conv1d_causal(x_nlc, weight, kernel_size=weight.shape[2], dilation=dilation) - if bias is not None: - bias_reshaped = g.reshape(bias, (1, 1, int(weight.shape[0]))) - out_nlc, bias_reshaped = _legalize_elementwise_binary_inputs(g, out_nlc, bias_reshaped) - out_nlc = g.add(out_nlc, bias_reshaped) - return [g.permute(out_nlc, (0, 2, 1))] - - if dilation != 1: - raise NotImplementedError(f"conv1d with dilation != 1 is unsupported by generic lowering: {dilation}") - if padding != 0: - if len(x.shape) != 3: - raise NotImplementedError(f"generic conv1d padding expects rank-3 input, got {x.shape}") - batch_size, channels, _ = (int(dim) for dim in x.shape) - pad_dtype = _torch_dtype_for_graph_dtype(x.dtype) - pieces: list[Tensor] = [] - if padding > 0: - left = _materialize_constant_tensor( - g, - torch.zeros((batch_size, channels, padding), dtype=pad_dtype), - ) - pieces.append(left) - pieces.append(x) - if padding > 0: - right = _materialize_constant_tensor( - g, - torch.zeros((batch_size, channels, padding), dtype=pad_dtype), - ) - pieces.append(right) - x = g.cat(pieces, axis=2) - if groups != 1: - if len(x.shape) != 3 or len(weight.shape) != 3: - raise NotImplementedError( - f"grouped conv1d lowering expects rank-3 input and weight, got {x.shape} and {weight.shape}" - ) - input_channels = int(x.shape[1]) - if input_channels % groups != 0: - raise NotImplementedError( - f"grouped conv1d requires input channels divisible by groups, got C_in={input_channels}, groups={groups}" - ) - output_channels = int(weight.shape[0]) - if output_channels % groups != 0: - raise NotImplementedError( - f"grouped conv1d requires output channels divisible by groups, got C_out={output_channels}, groups={groups}" - ) - input_channels_per_group = input_channels // groups - output_channels_per_group = output_channels // groups - if int(weight.shape[1]) != input_channels_per_group: - raise NotImplementedError( - "grouped conv1d weight has incompatible per-group input channels: " - f"weight C_in/group={int(weight.shape[1])}, expected {input_channels_per_group}" - ) - outputs: list[Tensor] = [] - for group_index in range(groups): - x_group = g.slice( - x, - axis=1, - start=group_index * input_channels_per_group, - length=input_channels_per_group, - ) - weight_group = g.slice( - weight, - axis=0, - start=group_index * output_channels_per_group, - length=output_channels_per_group, - ) - bias_group = None - if bias is not None: - bias_group = g.slice( - bias, - axis=0, - start=group_index * output_channels_per_group, - length=output_channels_per_group, - ) - outputs.append(g.conv1d(x_group, weight_group, bias=bias_group, stride=stride)) - if len(outputs) == 1: - return [outputs[0]] - return [g.cat(outputs, axis=1)] - return [g.conv1d(x, weight, bias=bias, stride=stride)] - - if op == "conv2d": - x = _tensor(env, node.inputs[0]) - weight = _tensor(env, node.inputs[1]) - bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None - stride = int(node.attrs.get("stride", 1)) - padding = int(node.attrs.get("padding", 0)) - dilation = int(node.attrs.get("dilation", 1)) - groups = int(node.attrs.get("groups", 1)) - - if len(x.shape) != 4 or len(weight.shape) != 4: - raise NotImplementedError(f"conv2d lowering expects rank-4 tensors, got {x.shape} and {weight.shape}") - if dilation != 1: - raise NotImplementedError(f"conv2d with dilation != 1 is unsupported: {dilation}") - - if stride == 2 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): - return [g.conv2d_k3s2p1(x, weight, bias=bias)] - - if ( - stride == 2 - and padding == 1 - and groups == x.shape[1] == weight.shape[0] - and weight.shape[1] == 1 - and weight.shape[2:] == (3, 3) - ): - return [g.conv2d_depthwise_k3s2p1(x, weight, bias=bias)] - - if ( - stride == 1 - and padding == 0 - and groups == 1 - and weight.shape[2:] == (1, 1) - ): - return [g.conv2d_pointwise_1x1(x, weight, bias=bias)] - - if stride == 1 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): - return [g.conv2d_k3s1p1(x, weight, bias=bias)] - - return [g.conv2d(x, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups)] - - if op == "layer_norm": - x = _tensor(env, node.inputs[0]) - weight = _tensor(env, node.inputs[1]) - bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None - return [g.layer_norm(x, weight, bias=bias, eps=float(node.attrs["eps"]))] - - if op == "rms_norm": - x = _tensor(env, node.inputs[0]) - weight = _tensor(env, node.inputs[1]) - reshape_back: tuple[int, ...] | None = None - if len(x.shape) > 2: - reshape_back = tuple(int(dim) for dim in x.shape) - x = _flatten_to_2d_for_linear(g, x) - out = g.rms_norm(x, weight, eps=float(node.attrs["eps"])) - if reshape_back is not None: - out = g.reshape(out, reshape_back) - return [out] - - if op == "rope": - rope_input = _tensor(env, node.inputs[0]) - if _rope_input_is_bhsd(ir, node.inputs[0]): - rope_input = g.permute(rope_input, (0, 2, 1, 3)) - rope_out = g.rope( - rope_input, - float(node.attrs["theta"]), - position_offset=int(node.attrs.get("position_offset", 0)), - ) - return [g.permute(rope_out, (0, 2, 1, 3))] - return [ - g.rope( - rope_input, - float(node.attrs["theta"]), - position_offset=int(node.attrs.get("position_offset", 0)), - ) - ] - - if op == "rel_pos_bias": - query = _attention_tensor(env, node.inputs[0]) - if len(query.shape) != 4: - raise NotImplementedError(f"rel_pos_bias expects rank-4 query input, got {query.shape}") - query = g.permute(query, (0, 2, 1, 3)) - relative_key = _tensor(env, node.inputs[1]) - return [g.rel_pos_bias(query, relative_key, float(node.attrs.get("scale", 1.0)))] - - if op in {"gated_deltanet_prefill", "gated_deltanet_decode"}: - x = _tensor(env, node.inputs[0]) - qkv_weight = _tensor(env, node.inputs[1]) - a_weight = _tensor(env, node.inputs[2]) - b_weight = _tensor(env, node.inputs[3]) - norm_weight = _tensor(env, node.inputs[4]) - - input_index = 5 - z_weight = None - if bool(node.attrs.get("has_z", False)): - z_weight = _tensor(env, node.inputs[input_index]) - input_index += 1 - dt_bias = None - if bool(node.attrs.get("has_dt_bias", False)): - dt_bias = _tensor(env, node.inputs[input_index]) - input_index += 1 - a_log = None - if bool(node.attrs.get("has_a_log", False)): - a_log = _tensor(env, node.inputs[input_index]) - input_index += 1 - conv_weight = None - if bool(node.attrs.get("has_conv", False)): - conv_weight = _tensor(env, node.inputs[input_index]) - - if len(x.shape) != 3: - raise NotImplementedError(f"{op} currently expects rank-3 normalized input, got {x.shape}") - - batch_size, seq_len, hidden_dim = (int(dim) for dim in x.shape) - if batch_size != 1: - raise NotImplementedError(f"{op} currently supports batch size 1, got {x.shape}") - - num_k_heads = int(node.attrs["num_k_heads"]) - num_v_heads = int(node.attrs["num_v_heads"]) - key_dim = int(node.attrs["key_dim"]) - value_dim = int(node.attrs["value_dim"]) - eps = float(node.attrs.get("eps", 1e-6)) - chunk_size = int(node.attrs.get("chunk_size", 64)) - - mixed_qkv = _matmul_with_quantized_rhs_legalization( - g, - _flatten_to_2d_for_linear(g, x), - qkv_weight, - pretransposed_rhs=True, - ) - mixed_qkv_dim = int(qkv_weight.shape[0]) - mixed_qkv = g.reshape(mixed_qkv, (batch_size, seq_len, mixed_qkv_dim)) - if conv_weight is not None: - kernel_size = int(conv_weight.shape[2]) - mixed_qkv = g.conv1d_causal(mixed_qkv, conv_weight, kernel_size=kernel_size, dilation=1) - mixed_qkv = g.silu(mixed_qkv) - - q_proj_dim = num_k_heads * key_dim - v_proj_dim = num_v_heads * value_dim - k_proj_dim = num_k_heads * key_dim - q_proj = g.slice(mixed_qkv, axis=2, start=0, length=q_proj_dim) - k_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim, length=k_proj_dim) - v_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim + k_proj_dim, length=v_proj_dim) - - q_4d = g.reshape(q_proj, (batch_size, seq_len, num_k_heads, key_dim)) - k_4d = g.reshape(k_proj, (batch_size, seq_len, num_k_heads, key_dim)) - v_4d = g.reshape(v_proj, (batch_size, seq_len, num_v_heads, value_dim)) - - q_norm = g.sum(g.multiply(q_4d, q_4d), axis=3) - q_norm = g.scalar_sqrt(g.scalar_add(q_norm, eps)) - q_norm = g.reshape(q_norm, (batch_size, seq_len, num_k_heads, 1)) - q_4d = g.divide(q_4d, q_norm) - - k_norm = g.sum(g.multiply(k_4d, k_4d), axis=3) - k_norm = g.scalar_sqrt(g.scalar_add(k_norm, eps)) - k_norm = g.reshape(k_norm, (batch_size, seq_len, num_k_heads, 1)) - k_4d = g.divide(k_4d, k_norm) - - a_logits = _matmul_with_quantized_rhs_legalization( - g, - _flatten_to_2d_for_linear(g, x), - a_weight, - pretransposed_rhs=True, - ) - a_logits = g.reshape(a_logits, (batch_size, seq_len, int(a_weight.shape[0]))) - b_logits = _matmul_with_quantized_rhs_legalization( - g, - _flatten_to_2d_for_linear(g, x), - b_weight, - pretransposed_rhs=True, - ) - b_logits = g.reshape(b_logits, (batch_size, seq_len, int(b_weight.shape[0]))) - - if dt_bias is not None: - dt_bias_2d = g.reshape(dt_bias, (1, int(dt_bias.shape[0]))) - a_logits, dt_bias_2d = _legalize_elementwise_binary_inputs(g, a_logits, dt_bias_2d) - a_logits = g.add(a_logits, dt_bias_2d) - a_softplus = _lower_softplus(g, a_logits) - - if a_log is not None: - a_log_2d = g.reshape(a_log, (1, int(a_log.shape[0]))) - neg_exp_a = g.scalar_multiply(g.scalar_exp(a_log_2d), -1.0) - neg_exp_a, a_softplus = _legalize_elementwise_binary_inputs(g, neg_exp_a, a_softplus) - gate_log = g.multiply(neg_exp_a, a_softplus) - else: - gate_log = g.scalar_multiply(a_softplus, -1.0) - beta = g.sigmoid(b_logits) - - initial_state = _materialize_constant_tensor( - g, - torch.zeros((batch_size, key_dim, num_v_heads, value_dim), dtype=torch.float16), - ) - - if op == "gated_deltanet_decode": - deltanet_out = g.gated_deltanet_decode(q_4d, k_4d, v_4d, gate_log, beta, initial_state, 0.0) - else: - deltanet_out = g.gated_deltanet_prefill(q_4d, k_4d, v_4d, gate_log, beta, initial_state, chunk_size, 0.0) - - y_4d = g.slice(deltanet_out, axis=1, start=0, length=seq_len) - y_2d = g.reshape(y_4d, (seq_len * num_v_heads, value_dim)) - - if z_weight is not None: - z_proj = _matmul_with_quantized_rhs_legalization( - g, - _flatten_to_2d_for_linear(g, x), - z_weight, - pretransposed_rhs=True, - ) - z_proj = g.reshape(z_proj, (seq_len * num_v_heads, value_dim)) - y_2d = g.multiply(g.rms_norm(y_2d, norm_weight, eps=eps), g.silu(z_proj)) - - return [g.reshape(y_2d, (batch_size, seq_len, num_v_heads * value_dim))] - - if op == "group_norm": - return [ - g.group_norm( - _tensor(env, node.inputs[0]), - _tensor(env, node.inputs[1]), - _tensor(env, node.inputs[2]), - num_groups=int(node.attrs["num_groups"]), - eps=float(node.attrs["eps"]), - ) - ] - - if op == "batch_norm": - axis = int(node.attrs.get("axis", 1)) - return [ - g.batch_norm( - _tensor(env, node.inputs[0]), - _tensor(env, node.inputs[1]), - _tensor(env, node.inputs[2]), - _tensor(env, node.inputs[3]), - _tensor(env, node.inputs[4]), - axis=axis, - eps=float(node.attrs["eps"]), - ) - ] - - if op == "identity": - return [env[node.inputs[0]]] - - if op == "contiguous": - return [_tensor(env, node.inputs[0])] - - if op == "advanced_index": - lowered = _try_lower_advanced_index(g, node, env, ir) - if lowered is not None: - return [lowered] - raise NotImplementedError(f"unsupported advanced_index pattern for node {node.id}") - - if op in {"masked_scatter", "aten.masked_scatter.default"}: - base = _tensor(env, node.inputs[0]) - mask = _tensor(env, node.inputs[1]) - source = _tensor(env, node.inputs[2]) - if source.dtype != base.dtype: - source = _ensure_tensor_dtype(g, source, base.dtype) - return [g.masked_scatter(base, mask, source)] - - if op == "masked_fill": - base = _tensor(env, node.inputs[0]) - mask = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") - inverse_mask = g.scalar_add(g.scalar_multiply(mask, -1.0), 1.0) - fill_term = g.scalar_multiply(_ensure_tensor_dtype(g, mask, base.dtype), float(node.attrs["value"])) - base_term = _lower_binary_op(g, base, _ensure_tensor_dtype(g, inverse_mask, base.dtype), "multiply") - return [_lower_binary_op(g, fill_term, base_term, "add")] - - if op == "getitem": - source = env[node.inputs[0]] - index = int(node.attrs["index"]) - if isinstance(source, (tuple, list)): - return [source[index]] - if isinstance(source, Tensor): - normalized_index = _normalize_index(index, int(source.shape[0])) - return [g.index(source, normalized_index, axis=0)] - raise NotImplementedError(f"getitem source is not tuple/list for node {node.id}") - - raise NotImplementedError(f"unsupported IR op in lowering: {op}") - - -def _tensor(env: dict[str, Any], value_id: str) -> Tensor: - try: - value = env[value_id] - except KeyError as exc: - raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc - if not isinstance(value, Tensor): - raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") - return value - - -def _attention_tensor(env: dict[str, Any], value_id: str) -> Tensor: - try: - value = env[value_id] - except KeyError as exc: - raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc - if isinstance(value, BroadcastAlias): - if value.kind != "gqa_repeat_kv": - raise TypeError(f"unsupported broadcast alias for attention input {value_id}: {value.kind}") - return value.tensor - if not isinstance(value, Tensor): - raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") - return value - - -def _lower_projected_attention_tensor( - g: Graph, - hidden: Tensor, - weight: Tensor, - bias: Tensor | None, - target_shape: tuple[int, ...], -) -> Tensor: - if len(target_shape) != 4: - raise NotImplementedError(f"projected attention tensor expects rank-4 target shape, got {target_shape}") - - linear_input = _flatten_to_2d_for_linear(g, hidden) - out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) - if bias is not None: - out = g.add(out, bias) - return g.reshape(out, target_shape) - - -def _normalize_attention_add_tensor(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: - if tuple(tensor.shape) == target_shape: - return tensor - - if len(tensor.shape) != 4 or len(target_shape) != 4: - raise NotImplementedError( - f"unsupported attention add tensor shape {tuple(tensor.shape)} for target {target_shape}" - ) - - batch, seq_len, heads, head_dim = target_shape - shape = tuple(int(v) for v in tensor.shape) - - if shape[0] in (1, batch) and shape[1] == heads and shape[2] in (1, seq_len) and shape[3] == head_dim: - return g.permute(tensor, (0, 2, 1, 3)) - if shape[0] in (1, batch) and shape[1] in (1, seq_len) and shape[2] == heads and shape[3] == head_dim: - return tensor - - raise NotImplementedError( - f"unsupported attention add tensor shape {shape} for target {target_shape}" - ) - - -def _rope_input_is_bhsd(ir: IRGraph, value_id: str) -> bool: - value = ir.values.get(value_id) - if value is None or value.producer is None: - return False - node = ir.nodes.get(value.producer) - if node is None: - return False - if node.op == "permute": - permutation = tuple(int(dim) for dim in node.attrs.get("permutation", ())) - return permutation == (0, 2, 1, 3) - if node.op == "transpose": - return int(node.attrs.get("dim0", -1)) == 1 and int(node.attrs.get("dim1", -1)) == 2 - return False - - -def _lower_softplus(g: Graph, x: Tensor) -> Tensor: - # Stable fp16 softplus: relu(x) + log(1 + exp(-abs(x))). - abs_x = g.abs(x) - neg_abs_x = g.scalar_multiply(abs_x, -1.0) - exp_term = g.scalar_exp(neg_abs_x) - log_term = g.scalar_log(g.scalar_add(exp_term, 1.0)) - return g.add(g.relu(x), log_term) - - -def _normalize_dim(dim: int, rank: int) -> int: - if dim < 0: - dim += rank - return dim - - -def _normalize_index(index: int, dim_size: int) -> int: - if index < 0: - index += dim_size - return index - - -def _static_shape(shape: Any) -> tuple[int, ...] | None: - if not isinstance(shape, tuple): - return None - dims: list[int] = [] - for dim in shape: - try: - dims.append(int(dim)) - except Exception: - return None - return tuple(dims) - - -def _constant_tensor_from_ir(ir: IRGraph, value_id: str) -> torch.Tensor | None: - const = ir.constants.get(value_id) - if isinstance(const, torch.nn.Parameter): - const = const.detach() - if isinstance(const, torch.Tensor): - return const.detach().cpu() - return None - - -def _try_lower_identity_broadcast_advanced_index( - g: Graph, - node: IRNode, - env: dict[str, Any], - ir: IRGraph, -) -> Tensor | None: - if len(node.inputs) != 3: - return None - - source = _tensor(env, node.inputs[0]) - source_shape = tuple(int(dim) for dim in source.shape) - if len(source_shape) != 2 or int(source_shape[0]) != 1: - return None - - output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) - if output_shape is None: - return None - - batch_index = _constant_tensor_from_ir(ir, node.inputs[1]) - token_index = _constant_tensor_from_ir(ir, node.inputs[2]) - if batch_index is None or token_index is None: - return None - - try: - batch_index = torch.broadcast_to(batch_index.to(torch.int64), output_shape) - token_index = torch.broadcast_to(token_index.to(torch.int64), output_shape) - except RuntimeError: - return None - - if torch.count_nonzero(batch_index).item() != 0: - return None - - flat_index = token_index.reshape(-1) - expected = torch.arange(int(source_shape[1]), dtype=torch.int64) - if flat_index.numel() != expected.numel() or not torch.equal(flat_index, expected): - return None - - return g.reshape(source, output_shape) - - -def _try_lower_prefix_mask_advanced_index( - g: Graph, - node: IRNode, - env: dict[str, Any], -) -> Tensor | None: - if len(node.inputs) != 2: - return None - - source = _tensor(env, node.inputs[0]) - mask = _tensor(env, node.inputs[1]) - source_shape = tuple(int(dim) for dim in source.shape) - mask_shape = tuple(int(dim) for dim in mask.shape) - if not mask_shape or len(mask_shape) > len(source_shape): - return None - if source_shape[: len(mask_shape)] != mask_shape: - return None - return g.masked_select_prefix(source, mask) - - -def _try_lower_advanced_index( - g: Graph, - node: IRNode, - env: dict[str, Any], - ir: IRGraph, -) -> Tensor | None: - lowered = _try_lower_identity_broadcast_advanced_index(g, node, env, ir) - if lowered is not None: - return lowered - return _try_lower_prefix_mask_advanced_index(g, node, env) - - -def _normalize_slice_end(end: int, dim_size: int) -> int: - if end < 0: - end += dim_size - return max(0, min(end, dim_size)) - - -def _legalize_elementwise_binary_inputs(g: Graph, lhs: Tensor, rhs: Tensor) -> tuple[Tensor, Tensor]: - if lhs.shape == rhs.shape: - return lhs, rhs - - lhs_rank = len(lhs.shape) - rhs_rank = len(rhs.shape) - - if lhs_rank > rhs_rank: - rhs = _reshape_for_trailing_broadcast(g, rhs, lhs.shape) - elif rhs_rank > lhs_rank: - lhs = _reshape_for_trailing_broadcast(g, lhs, rhs.shape) - - if lhs.shape == rhs.shape: - return lhs, rhs - - return lhs, rhs - - -def _is_scalar_like(value: Any) -> bool: - return isinstance(value, (int, float, bool)) - - -def _lower_binary_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: - if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): - lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) - target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 - lhs = _ensure_tensor_dtype(g, lhs, target_dtype) - rhs = _ensure_tensor_dtype(g, rhs, target_dtype) - if op == "add": - return g.add(lhs, rhs) - if op == "subtract": - return g.subtract(lhs, rhs) - if op == "multiply": - return g.multiply(lhs, rhs) - if op == "divide": - return g.divide(lhs, rhs) - raise NotImplementedError(f"unsupported binary op: {op}") - - if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): - lhs_value = _ensure_tensor_dtype( - g, - lhs_value, - Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, - ) - scalar = float(rhs_value) - if op == "add": - return g.scalar_add(lhs_value, scalar) - if op == "subtract": - return g.scalar_subtract(lhs_value, scalar) - if op == "multiply": - return g.scalar_multiply(lhs_value, scalar) - if op == "divide": - return g.scalar_divide(lhs_value, scalar) - raise NotImplementedError(f"unsupported binary op: {op}") - - if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): - rhs_value = _ensure_tensor_dtype( - g, - rhs_value, - Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, - ) - scalar = float(lhs_value) - if op == "add": - return g.scalar_add(rhs_value, scalar) - if op == "subtract": - return g.scalar_add(g.scalar_multiply(rhs_value, -1.0), scalar) - if op == "multiply": - return g.scalar_multiply(rhs_value, scalar) - if op == "divide": - raise NotImplementedError("scalar/tensor divide is not directly supported by Cactus graph ops") - raise NotImplementedError(f"unsupported binary op: {op}") - - raise TypeError( - f"unsupported lowered operand types for {op}: " - f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" - ) - - -def _lower_compare_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: - if op == "equal": - not_equal = _lower_compare_op(g, lhs_value, rhs_value, "not_equal") - return g.scalar_not_equal(not_equal, 1.0) - - if op == "greater": - delta = _lower_binary_op(g, lhs_value, rhs_value, "subtract") - delta = _ensure_fp16_tensor(g, delta) - return g.scalar_not_equal(g.relu(delta), 0.0) - - if op == "less_equal": - greater = _lower_compare_op(g, lhs_value, rhs_value, "greater") - return g.scalar_not_equal(greater, 1.0) - - if op != "not_equal": - raise NotImplementedError(f"unsupported compare op: {op}") - - if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): - lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) - target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 - lhs = _ensure_tensor_dtype(g, lhs, target_dtype) - rhs = _ensure_tensor_dtype(g, rhs, target_dtype) - return g.not_equal(lhs, rhs) - - if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): - lhs_value = _ensure_tensor_dtype( - g, - lhs_value, - Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, - ) - return g.scalar_not_equal(lhs_value, float(rhs_value)) - - if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): - rhs_value = _ensure_tensor_dtype( - g, - rhs_value, - Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, - ) - return g.scalar_not_equal(rhs_value, float(lhs_value)) - - raise TypeError( - f"unsupported lowered operand types for {op}: " - f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" - ) - - -def _reshape_for_trailing_broadcast(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: - tensor_shape = tuple(tensor.shape) - target_rank = len(target_shape) - tensor_rank = len(tensor_shape) - - if tensor_rank > target_rank: - return tensor - - padded_shape = (1,) * (target_rank - tensor_rank) + tensor_shape - - # Only legalize cases that are valid trailing broadcasts, e.g.: - # (H,) -> (1, H), (1, 1, H), etc. - for src_dim, tgt_dim in zip(padded_shape, target_shape): - if src_dim != 1 and src_dim != tgt_dim: - return tensor - - if padded_shape == tensor_shape: - return tensor - - return g.reshape(tensor, padded_shape) - - -def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: - if tensor.dtype == Graph.FP16: - return tensor - return g.precision_cast(tensor, Graph.FP16) - - -def _ensure_tensor_dtype(g: Graph, tensor: Tensor, dtype: int) -> Tensor: - if tensor.dtype == dtype: - return tensor - return g.precision_cast(tensor, dtype) - - -def _cat_with_legalized_dtype(g: Graph, tensors: list[Tensor], *, axis: int, dtype: int | None) -> Tensor: - if not tensors: - raise NotImplementedError("cat requires at least one tensor") - if dtype is None: - dtypes = {tensor.dtype for tensor in tensors} - dtype = tensors[0].dtype if len(dtypes) == 1 else Graph.FP16 - legalized = [_ensure_tensor_dtype(g, tensor, dtype) for tensor in tensors] - return g.cat(legalized, axis=axis) - - -def _lower_where_branch_term(g: Graph, branch_value: Any, mask: Tensor, *, dtype: int | None) -> Tensor: - if dtype is not None: - mask = _ensure_tensor_dtype(g, mask, dtype) - if isinstance(branch_value, Tensor): - branch_mask = mask if dtype is None else _ensure_tensor_dtype(g, mask, branch_value.dtype) - return _lower_binary_op(g, branch_value, branch_mask, "multiply") - return g.scalar_multiply(mask, float(branch_value)) - - -def _lower_where_op(g: Graph, node: IRNode, env: dict[str, Any]) -> Tensor: - condition = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") - false_mask = g.scalar_add(g.scalar_multiply(condition, -1.0), 1.0) - - input_index = 1 - if bool(node.attrs.get("true_is_scalar", False)): - true_value: Any = float(node.attrs["true_value"]) - else: - true_value = env[node.inputs[input_index]] - input_index += 1 - - if bool(node.attrs.get("false_is_scalar", False)): - false_value: Any = float(node.attrs["false_value"]) - else: - false_value = env[node.inputs[input_index]] - - result_dtype: int | None = None - if isinstance(true_value, Tensor): - result_dtype = true_value.dtype - elif isinstance(false_value, Tensor): - result_dtype = false_value.dtype - - true_term = _lower_where_branch_term(g, true_value, condition, dtype=result_dtype) - false_term = _lower_where_branch_term(g, false_value, false_mask, dtype=result_dtype) - return _lower_binary_op(g, true_term, false_term, "add") - - -def _flatten_to_2d_for_linear(g: Graph, tensor: Tensor) -> Tensor: - shape = tuple(tensor.shape) - if len(shape) <= 2: - return tensor - leading = 1 - for dim in shape[:-1]: - leading *= int(dim) - return g.reshape(tensor, (leading, int(shape[-1]))) - - -def _broadcast_batch_shape(lhs_batch: tuple[int, ...], rhs_batch: tuple[int, ...]) -> tuple[int, ...] | None: - rank = max(len(lhs_batch), len(rhs_batch)) - lhs_padded = (1,) * (rank - len(lhs_batch)) + lhs_batch - rhs_padded = (1,) * (rank - len(rhs_batch)) + rhs_batch - result: list[int] = [] - for lhs_dim, rhs_dim in zip(lhs_padded, rhs_padded): - if lhs_dim == rhs_dim: - result.append(int(lhs_dim)) - elif lhs_dim == 1: - result.append(int(rhs_dim)) - elif rhs_dim == 1: - result.append(int(lhs_dim)) - else: - return None - return tuple(result) - - -def _project_broadcast_coord(coord: tuple[int, ...], operand_batch: tuple[int, ...]) -> tuple[int, ...]: - if not operand_batch: - return () - offset = len(coord) - len(operand_batch) - projected: list[int] = [] - for axis, dim in enumerate(operand_batch): - value = coord[offset + axis] - projected.append(0 if int(dim) == 1 else int(value)) - return tuple(projected) - - -def _index_batch_prefix(g: Graph, tensor: Tensor, coord: tuple[int, ...]) -> Tensor: - for index_value in coord: - tensor = g.index(tensor, index_value, axis=0) - return tensor - - -def _lower_static_batched_matmul(g: Graph, lhs: Tensor, rhs: Tensor) -> Tensor | None: - lhs_shape = tuple(int(dim) for dim in lhs.shape) - rhs_shape = tuple(int(dim) for dim in rhs.shape) - if len(lhs_shape) < 3 or len(rhs_shape) < 3: - return None - if lhs_shape[-1] != rhs_shape[-2]: - return None - - lhs_batch = lhs_shape[:-2] - rhs_batch = rhs_shape[:-2] - batch_shape = _broadcast_batch_shape(lhs_batch, rhs_batch) - if batch_shape is None: - return None - - output_matrix_shape = (int(lhs_shape[-2]), int(rhs_shape[-1])) - rank = len(batch_shape) - - def _build(dim: int, coord_prefix: tuple[int, ...]) -> Tensor: - if dim == rank: - lhs_coord = _project_broadcast_coord(coord_prefix, lhs_batch) - rhs_coord = _project_broadcast_coord(coord_prefix, rhs_batch) - lhs_slice = _index_batch_prefix(g, lhs, lhs_coord) - rhs_slice = _index_batch_prefix(g, rhs, rhs_coord) - out = _matmul_with_quantized_rhs_legalization(g, lhs_slice, rhs_slice) - return g.reshape(out, (1,) * rank + output_matrix_shape) - - pieces = [_build(dim + 1, coord_prefix + (index_value,)) for index_value in range(int(batch_shape[dim]))] - if len(pieces) == 1: - return pieces[0] - return g.cat(pieces, axis=dim) - - return _build(0, ()) - - -def _legalize_matmul_inputs( - g: Graph, - lhs: Tensor, - rhs: Tensor, - node: IRNode, -) -> tuple[Tensor, Tensor, tuple[int, ...]] | None: - lhs_shape = tuple(lhs.shape) - rhs_shape = tuple(rhs.shape) - if len(lhs_shape) <= 2 and len(rhs_shape) <= 2: - return None - - if len(lhs_shape) > 2 and len(rhs_shape) == 2 and lhs_shape[-1] == rhs_shape[0]: - leading = 1 - for dim in lhs_shape[:-1]: - leading *= int(dim) - lhs_2d = g.reshape(lhs, (leading, int(lhs_shape[-1]))) - output_shape = lhs_shape[:-1] + (int(rhs_shape[1]),) - return lhs_2d, rhs, output_shape - - # Cactus matmul is 2D-only. Legalize the narrow case where both operands have - # only singleton leading dims, e.g. rotary helper matmuls: - # (1, M, K) @ (1, K, N) -> reshape to (M, K) @ (K, N) -> reshape back. - if any(dim != 1 for dim in lhs_shape[:-2]): - return None - if any(dim != 1 for dim in rhs_shape[:-2]): - return None - - lhs_2d_shape = lhs_shape[-2:] - rhs_2d_shape = rhs_shape[-2:] - if lhs_2d_shape[-1] != rhs_2d_shape[0]: - return None - - output_shape = node.meta.get("shape") - if not isinstance(output_shape, tuple): - output_shape = lhs_shape[:-2] + (lhs_2d_shape[0], rhs_2d_shape[1]) - output_shape = tuple(int(v) for v in output_shape) - - lhs_2d = g.reshape(lhs, lhs_2d_shape) - rhs_2d = g.reshape(rhs, rhs_2d_shape) - return lhs_2d, rhs_2d, output_shape - - -def _resolve_expand_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: - if len(requested_shape) < len(input_shape): - raise NotImplementedError(f"expand cannot reduce rank: {input_shape} -> {requested_shape}") - - padded_input = (1,) * (len(requested_shape) - len(input_shape)) + input_shape - resolved: list[int] = [] - for in_dim, req_dim in zip(padded_input, requested_shape): - if req_dim == -1: - resolved.append(in_dim) - continue - if req_dim < -1: - raise NotImplementedError(f"invalid expand dimension: {req_dim}") - resolved.append(int(req_dim)) - return tuple(resolved) - - -def _resolve_reshape_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: - resolved = [int(v) for v in requested_shape] - unknown_indices = [idx for idx, dim in enumerate(resolved) if dim == -1] - if len(unknown_indices) > 1: - raise NotImplementedError(f"reshape with multiple inferred dimensions is unsupported: {requested_shape}") - if not unknown_indices: - return tuple(resolved) - - input_elements = 1 - for dim in input_shape: - input_elements *= int(dim) - - known_elements = 1 - for dim in resolved: - if dim != -1: - known_elements *= int(dim) - - if known_elements == 0 or input_elements % known_elements != 0: - raise NotImplementedError(f"cannot infer reshape target {requested_shape} from input shape {input_shape}") - - resolved[unknown_indices[0]] = input_elements // known_elements - return tuple(resolved) - - -def _match_gqa_expand_alias(ir: IRGraph, expand_node: IRNode) -> str | None: - target_shape = tuple(int(v) for v in expand_node.attrs["shape"]) - if len(target_shape) != 5: - return None - - current_value_id = expand_node.inputs[0] - base_value_id: str | None = None - - while True: - producer_id = ir.values[current_value_id].producer - if producer_id is None: - return None - producer = ir.nodes[producer_id] - if producer.op == "slice": - axis = int(producer.attrs.get("axis", -1)) - input_shape = ir.values[producer.inputs[0]].shape - if input_shape is None: - return None - normalized_axis = _normalize_dim(axis, len(input_shape)) - start = int(producer.attrs.get("start", 0)) - end = int(producer.attrs.get("end", input_shape[normalized_axis])) - if start != 0 or end < input_shape[normalized_axis]: - return None - current_value_id = producer.inputs[0] - continue - if producer.op == "unsqueeze": - base_value_id = producer.inputs[0] - base_shape = ir.values[base_value_id].shape - if base_shape is None: - return None - unsqueezed_dim = _normalize_dim(int(producer.attrs.get("dim", 0)), len(base_shape) + 1) - if unsqueezed_dim != 2: - return None - break - if producer.op in {"reshape", "view"}: - base_value_id = producer.inputs[0] - base_shape = ir.values[base_value_id].shape - view_shape = ir.values[current_value_id].shape - if base_shape is None or view_shape is None: - return None - if len(base_shape) != 4 or len(view_shape) != 5: - return None - if tuple(int(v) for v in view_shape) != ( - int(base_shape[0]), - int(base_shape[1]), - 1, - int(base_shape[2]), - int(base_shape[3]), - ): - return None - break - return None - - if base_value_id is None: - return None - base_shape = ir.values[base_value_id].shape - if base_shape is None or len(base_shape) != 4: - return None - - expected_target = ( - int(base_shape[0]), - int(base_shape[1]), - int(target_shape[2]), - int(base_shape[2]), - int(base_shape[3]), - ) - if target_shape != expected_target: - return None - if int(target_shape[2]) <= 1: - return None - - return base_value_id - - -def _map_ir_dtype(dtype: str) -> int: - if dtype == "bf16": - return Graph.FP32 - if dtype == "fp16": - return Graph.FP16 - if dtype in ("fp32", "fp64"): - return Graph.FP32 - if dtype == "int8": - return Graph.INT8 - if dtype in ("int32", "int64", "bool"): - return Graph.FP32 - - raise NotImplementedError(f"unsupported IR dtype: {dtype}") - - -def _map_ir_or_torch_dtype(dtype: Any) -> int: - if dtype is None: - raise NotImplementedError("missing dtype for precision_cast") - if isinstance(dtype, str): - if dtype.startswith("torch."): - return _map_torch_dtype(getattr(torch, dtype.split(".", 1)[1])) - return _map_ir_dtype(dtype) - return _map_torch_dtype(dtype) - - -def _map_torch_dtype(dtype: Any) -> int: - if dtype == torch.bfloat16: - return Graph.FP32 - if dtype == torch.float16: - return Graph.FP16 - if dtype == torch.float32: - return Graph.FP32 - if dtype == torch.float64: - return Graph.FP32 - if dtype == torch.int8: - return Graph.INT8 - if dtype in (torch.int16, torch.int32, torch.int64, torch.bool): - return Graph.FP32 - raise NotImplementedError(f"unsupported torch dtype: {dtype}") - - -def _materialize_constant_tensor(g: Graph, tensor_value: torch.Tensor) -> Tensor: - graph_dtype = _map_torch_dtype(tensor_value.dtype) - materialized = tensor_value.detach().cpu() - if graph_dtype == Graph.FP32 and materialized.dtype not in (torch.float32, torch.float64): - materialized = materialized.to(torch.float32) - elif graph_dtype == Graph.FP16 and materialized.dtype != torch.float16: - materialized = materialized.to(torch.float16) - elif graph_dtype == Graph.INT8 and materialized.dtype != torch.int8: - materialized = materialized.to(torch.int8) - - tensor = g.input(shape=tuple(materialized.shape), dtype=graph_dtype) - g.set_input(tensor, materialized, dtype=graph_dtype) - return tensor - - -def _materialize_constant_torch_dtype(dtype: Any) -> torch.dtype: - if dtype is None: - return torch.float32 - if isinstance(dtype, str): - if dtype.startswith("torch."): - return getattr(torch, dtype.split(".", 1)[1]) - if dtype in ("fp16", "bf16"): - return torch.float16 - if dtype in ("fp32", "fp64", "int16", "int32", "int64", "bool"): - return torch.float32 - if dtype == "int8": - return torch.int8 - if isinstance(dtype, torch.dtype): - return dtype - raise NotImplementedError(f"unsupported dtype for constant materialization: {dtype}") - - -def _torch_dtype_for_graph_dtype(dtype: int) -> torch.dtype: - if dtype == Graph.FP16: - return torch.float16 - if dtype == Graph.FP32: - return torch.float32 - if dtype == Graph.INT8: - return torch.int8 - raise NotImplementedError(f"unsupported graph dtype for constant materialization: {dtype}") diff --git a/python/src/transpile/model_adapters.py b/python/src/transpile/model_adapters.py deleted file mode 100644 index 46f215895..000000000 --- a/python/src/transpile/model_adapters.py +++ /dev/null @@ -1,803 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import inspect -from typing import Callable - -import torch - - -@dataclass -class CanonicalizedModel: - module: torch.nn.Module - task: str - family: str - input_names: tuple[str, ...] = () - - -def _model_name_or_path(model: torch.nn.Module) -> str: - value = getattr(model, "name_or_path", None) - if isinstance(value, str) and value: - return value - config = getattr(model, "config", None) - value = getattr(config, "_name_or_path", None) - if isinstance(value, str) and value: - return value - return "" - - -def _transpile_graph_meta(model: torch.nn.Module, *, adapter_family: str, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: - return { - "adapter_family": adapter_family, - "adapter_type": adapter_type, - "model_name_or_path": _model_name_or_path(model), - "input_names": input_names, - } - - -def _extract_tensor_output(output: object, *, preferred_field: str | None = None) -> torch.Tensor: - if preferred_field is not None: - value = getattr(output, preferred_field, None) - if isinstance(value, torch.Tensor): - return value - - if isinstance(output, torch.Tensor): - return output - if isinstance(output, tuple) and output and isinstance(output[0], torch.Tensor): - return output[0] - - for field_name in ("last_hidden_state", "logits"): - value = getattr(output, field_name, None) - if isinstance(value, torch.Tensor): - return value - - raise TypeError(f"could not extract tensor output from {type(output).__name__}") - - -def _infer_input_names(module: torch.nn.Module, *, preferred: tuple[str, ...]) -> tuple[str, ...]: - try: - signature = inspect.signature(module.forward) - except (TypeError, ValueError): - return preferred[:1] - - control_names = { - "self", - "return_dict", - "use_cache", - "past_key_values", - "cache_position", - "position_ids", - "labels", - "decoder_input_ids", - "decoder_attention_mask", - "output_attentions", - "output_hidden_states", - } - available = [ - name - for name, parameter in signature.parameters.items() - if name not in control_names - and parameter.kind in ( - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.KEYWORD_ONLY, - ) - ] - matched = [name for name in preferred if name in available] - if matched: - return tuple(matched) - if available: - return tuple(available[: min(2, len(available))]) - return preferred[:1] - - -class BoundInputAdapter(torch.nn.Module): - def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str, metadata_task: str): - super().__init__() - self.model = model - self.input_names = tuple(input_names) - self.family = family - self.metadata_task = metadata_task - - def _kwargs_from_bound_inputs(self, *bound_inputs: torch.Tensor | None) -> dict[str, torch.Tensor]: - provided = tuple(bound_inputs) - if len(self.input_names) > len(provided): - raise ValueError( - f"adapter expected at most {len(provided)} bound inputs, got {len(self.input_names)} names" - ) - kwargs: dict[str, torch.Tensor] = {} - for index, name in enumerate(self.input_names): - value = provided[index] - if value is None: - raise ValueError(f"missing required bound input {index} for {name}") - kwargs[name] = value - return kwargs - - def get_transpile_metadata(self): - return { - "graph": { - **_transpile_graph_meta( - self.model, - adapter_family=self.family, - adapter_type=type(self).__name__, - input_names=self.input_names, - ), - "task": self.metadata_task, - } - } - - -def _gemma4_import_hints(backbone: torch.nn.Module, *, module_path_suffix_prefix: str) -> list[dict[str, object]]: - sliding_window = getattr(backbone.config, "sliding_window", None) - layer_types = list(getattr(backbone.config, "layer_types", [])) - import_hints: list[dict[str, object]] = [] - for layer_index, layer_type in enumerate(layer_types): - attrs: dict[str, object] = {} - if layer_type == "sliding_attention" and sliding_window is not None: - attrs["window_size"] = int(sliding_window) - import_hints.append( - { - "module_path_suffix": f"{module_path_suffix_prefix}.layers.{layer_index}.self_attn", - "op": "scaled_dot_product_attention", - "attrs": attrs, - "meta": { - "attention_layer_type": layer_type, - "attention_layer_index": layer_index, - }, - } - ) - return import_hints - - -class CausalLMLogitsAdapter(torch.nn.Module): - def __init__(self, model: torch.nn.Module): - super().__init__() - self.model = model - - def forward(self, input_ids: torch.Tensor): - return self.model( - input_ids=input_ids, - use_cache=False, - return_dict=False, - )[0] - - def get_transpile_metadata(self): - return { - "graph": { - **_transpile_graph_meta( - self.model, - adapter_family="generic", - adapter_type=type(self).__name__, - input_names=("input_ids",), - ), - } - } - - -class GemmaCausalLMLogitsAdapter(torch.nn.Module): - def __init__(self, model: torch.nn.Module): - super().__init__() - self.model = model - self.backbone = model.model - from transformers.models.gemma.modeling_gemma import create_causal_mask # type: ignore - - self._create_causal_mask = create_causal_mask - - def forward(self, input_ids: torch.Tensor): - return self.debug_forward(input_ids)[0] - - def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: - inputs_embeds = self.backbone.embed_tokens(input_ids) - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) - causal_mask = self._create_causal_mask( - config=self.backbone.config, - inputs_embeds=inputs_embeds, - attention_mask=None, - past_key_values=None, - position_ids=position_ids, - ) - - hidden_states = inputs_embeds - position_embeddings = self.backbone.rotary_emb(hidden_states, position_ids=position_ids) - checkpoints: list[torch.Tensor] = [] - - for decoder_layer in self.backbone.layers[: self.backbone.config.num_hidden_layers]: - hidden_states = decoder_layer( - hidden_states, - attention_mask=causal_mask, - position_ids=position_ids, - past_key_values=None, - use_cache=False, - position_embeddings=position_embeddings, - ) - checkpoints.append(hidden_states) - - hidden_states = self.backbone.norm(hidden_states) - checkpoints.append(hidden_states) - return self.model.lm_head(hidden_states), checkpoints - - def get_transpile_metadata(self): - return { - "graph": { - **_transpile_graph_meta( - self.model, - adapter_family="gemma", - adapter_type=type(self).__name__, - input_names=("input_ids",), - ), - "num_hidden_layers": int(self.backbone.config.num_hidden_layers), - } - } - - -class Gemma3CausalLMLogitsAdapter(torch.nn.Module): - def __init__(self, model: torch.nn.Module): - super().__init__() - self.model = model - self.backbone = model.model - from transformers.models.gemma3.modeling_gemma3 import create_causal_mask # type: ignore - from transformers.models.gemma3.modeling_gemma3 import create_sliding_window_causal_mask # type: ignore - - self._create_causal_mask = create_causal_mask - self._create_sliding_window_causal_mask = create_sliding_window_causal_mask - - def forward(self, input_ids: torch.Tensor): - return self.debug_forward(input_ids)[0] - - def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: - inputs_embeds = self.backbone.embed_tokens(input_ids) - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) - - mask_kwargs = { - "config": self.backbone.config, - "inputs_embeds": inputs_embeds, - "attention_mask": None, - "past_key_values": None, - "position_ids": position_ids, - } - causal_mask_mapping = { - "full_attention": self._create_causal_mask(**mask_kwargs), - "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), - } - - hidden_states = inputs_embeds - checkpoints: list[torch.Tensor] = [] - position_embeddings = { - layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) - for layer_type in self.backbone.config.layer_types - } - - for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - hidden_states = decoder_layer( - hidden_states, - attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], - position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], - position_ids=position_ids, - past_key_values=None, - ) - checkpoints.append(hidden_states) - - hidden_states = self.backbone.norm(hidden_states) - checkpoints.append(hidden_states) - return self.model.lm_head(hidden_states), checkpoints - - def get_transpile_metadata(self): - sliding_window = getattr(self.backbone.config, "sliding_window", None) - layer_types = list(getattr(self.backbone.config, "layer_types", [])) - return { - "graph": { - **_transpile_graph_meta( - self.model, - adapter_family="gemma3", - adapter_type=type(self).__name__, - input_names=("input_ids",), - ), - "num_hidden_layers": int(self.backbone.config.num_hidden_layers), - "layer_types": tuple(layer_types), - "sliding_window": None if sliding_window is None else int(sliding_window), - }, - "import_hints": _gemma4_import_hints(self.backbone, module_path_suffix_prefix="backbone"), - } - - -class Gemma4CausalLMLogitsAdapter(torch.nn.Module): - def __init__(self, model: torch.nn.Module): - super().__init__() - self.model = model - model_backbone = model.model - self.backbone = getattr(model_backbone, "language_model", model_backbone) - from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore - from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore - - self._create_causal_mask = create_causal_mask - self._create_sliding_window_causal_mask = create_sliding_window_causal_mask - - def forward(self, input_ids: torch.Tensor): - return self.debug_forward(input_ids)[0] - - def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: - inputs_embeds = self.backbone.embed_tokens(input_ids) - per_layer_inputs = None - if self.backbone.hidden_size_per_layer_input: - per_layer_inputs = self.backbone.get_per_layer_inputs(input_ids, inputs_embeds) - per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) - - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) - mask_kwargs = { - "config": self.backbone.config, - "inputs_embeds": inputs_embeds, - "attention_mask": None, - "past_key_values": None, - "position_ids": position_ids, - } - causal_mask_mapping = { - "full_attention": self._create_causal_mask(**mask_kwargs), - "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), - } - - hidden_states = inputs_embeds - checkpoints: list[torch.Tensor] = [] - position_embeddings = { - layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) - for layer_type in self.backbone.unique_layer_types - } - shared_kv_states: dict[str, torch.Tensor] = {} - - for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - per_layer_input = per_layer_inputs[:, :, i, :] if per_layer_inputs is not None else None - hidden_states = decoder_layer( - hidden_states, - per_layer_input, - shared_kv_states=shared_kv_states, - position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], - attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], - position_ids=position_ids, - past_key_values=None, - ) - checkpoints.append(hidden_states) - - hidden_states = self.backbone.norm(hidden_states) - checkpoints.append(hidden_states) - return self.model.lm_head(hidden_states), checkpoints - - def debug_first_block(self, input_ids: torch.Tensor) -> dict[str, torch.Tensor]: - inputs_embeds = self.backbone.embed_tokens(input_ids) - per_layer_inputs = None - if self.backbone.hidden_size_per_layer_input: - per_layer_inputs = self.backbone.get_per_layer_inputs(input_ids, inputs_embeds) - per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) - - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) - mask_kwargs = { - "config": self.backbone.config, - "inputs_embeds": inputs_embeds, - "attention_mask": None, - "past_key_values": None, - "position_ids": position_ids, - } - causal_mask_mapping = { - "full_attention": self._create_causal_mask(**mask_kwargs), - "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), - } - - hidden_states = inputs_embeds - position_embeddings = { - layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) - for layer_type in self.backbone.unique_layer_types - } - shared_kv_states: dict[str, torch.Tensor] = {} - layer = self.backbone.layers[0] - layer_type = self.backbone.config.layer_types[0] - per_layer_input = per_layer_inputs[:, :, 0, :] if per_layer_inputs is not None else None - - checkpoints: dict[str, torch.Tensor] = {} - - residual = hidden_states - normed = layer.input_layernorm(hidden_states) - checkpoints["pre_attn_norm"] = normed - - attn_out = layer.self_attn( - normed, - position_embeddings=position_embeddings[layer_type], - attention_mask=causal_mask_mapping[layer_type], - position_ids=position_ids, - past_key_values=None, - shared_kv_states=shared_kv_states, - ) - if isinstance(attn_out, tuple): - attn_out = attn_out[0] - checkpoints["attn_o_proj"] = attn_out - - post_attn_norm = layer.post_attention_layernorm(attn_out) - checkpoints["post_attn_norm"] = post_attn_norm - - after_attention = residual + post_attn_norm - checkpoints["after_attention_residual"] = after_attention - - pre_ffn_norm = layer.pre_feedforward_layernorm(after_attention) - checkpoints["pre_ffn_norm"] = pre_ffn_norm - - mlp_out = layer.mlp(pre_ffn_norm) - checkpoints["mlp_down"] = mlp_out - - post_ffn_norm = layer.post_feedforward_layernorm(mlp_out) - checkpoints["post_ffn_norm"] = post_ffn_norm - - after_ffn = after_attention + post_ffn_norm - checkpoints["after_ffn_residual"] = after_ffn - - if per_layer_input is not None: - gated = layer.per_layer_input_gate(after_ffn) - gated = layer.act_fn(gated) - projected = gated * per_layer_input - per_layer_proj = layer.per_layer_projection(projected) - checkpoints["per_layer_input_proj"] = per_layer_proj - post_per_layer_input_norm = layer.post_per_layer_input_norm(per_layer_proj) - checkpoints["post_per_layer_input_norm"] = post_per_layer_input_norm - after_ffn = after_ffn + post_per_layer_input_norm - - layer_scalar = getattr(layer, "layer_scalar", None) - if layer_scalar is not None: - after_ffn = after_ffn * layer_scalar - checkpoints["layer_scalar_out"] = after_ffn - return checkpoints - - def get_transpile_metadata(self): - sliding_window = getattr(self.backbone.config, "sliding_window", None) - layer_types = list(getattr(self.backbone.config, "layer_types", [])) - return { - "graph": { - **_transpile_graph_meta( - self.model, - adapter_family="gemma4", - adapter_type=type(self).__name__, - input_names=("input_ids",), - ), - "num_hidden_layers": int(self.backbone.config.num_hidden_layers), - "layer_types": tuple(layer_types), - "sliding_window": None if sliding_window is None else int(sliding_window), - }, - "import_hints": _gemma4_import_hints(self.backbone, module_path_suffix_prefix="backbone"), - } - - -class Gemma4MultimodalCausalLMLogitsAdapter(BoundInputAdapter): - def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...]): - super().__init__( - model, - input_names=input_names, - family="gemma4", - metadata_task="multimodal_causal_lm_logits", - ) - model_backbone = model.model - self.multimodal_backbone = model_backbone - self.backbone = getattr(model_backbone, "language_model", model_backbone) - self._create_causal_mask_mapping = None - self._create_masks_for_generate = None - try: - from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping # type: ignore - from transformers.models.gemma4.modeling_gemma4 import create_masks_for_generate # type: ignore - - self._create_causal_mask_mapping = create_causal_mask_mapping - self._create_masks_for_generate = create_masks_for_generate - except Exception: - pass - - def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: - kwargs = self._kwargs_from_bound_inputs(*bound_inputs) - input_ids = kwargs["input_ids"] - attention_mask = kwargs.get("attention_mask") - token_type_ids = kwargs.get("token_type_ids") - pixel_values = kwargs.get("pixel_values") - pixel_position_ids = kwargs.get("pixel_position_ids") - input_features = kwargs.get("input_features") - input_features_mask = kwargs.get("input_features_mask") - - multimodal_backbone = self.multimodal_backbone - get_placeholder_mask = getattr(multimodal_backbone, "get_placeholder_mask", None) - get_image_features = getattr(multimodal_backbone, "get_image_features", None) - get_audio_features = getattr(multimodal_backbone, "get_audio_features", None) - lm_head = getattr(self.model, "lm_head", None) - if ( - not callable(get_placeholder_mask) - or not callable(get_image_features) - or not callable(get_audio_features) - or not callable(self._create_causal_mask_mapping) - or not callable(self._create_masks_for_generate) - or not isinstance(lm_head, torch.nn.Module) - ): - outputs = self.model( - return_dict=True, - use_cache=False, - **kwargs, - ) - return _extract_tensor_output(outputs, preferred_field="logits") - - inputs_embeds = self.model.get_input_embeddings()(input_ids) - text_mask, image_mask, audio_mask = get_placeholder_mask( - token_type_ids=token_type_ids, - input_ids=input_ids, - inputs_embeds=inputs_embeds, - ) - - per_layer_inputs = None - text_config = self.multimodal_backbone.config.get_text_config() - if getattr(text_config, "hidden_size_per_layer_input", None): - per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype) - per_layer_inputs = self.backbone.get_per_layer_inputs(per_layer_inputs_tokens) - - if pixel_values is not None: - image_features = get_image_features( - pixel_values, - pixel_position_ids, - None, - return_dict=True, - ).pooler_output - image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype) - image_mask_expanded = image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) - inputs_embeds = inputs_embeds.masked_scatter(image_mask_expanded, image_features) - - if input_features is not None and input_features_mask is not None: - audio_output = get_audio_features(input_features, ~input_features_mask, return_dict=True) - audio_features = audio_output.pooler_output.to(inputs_embeds.device, inputs_embeds.dtype) - audio_mask_expanded = audio_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device) - inputs_embeds = inputs_embeds.masked_scatter(audio_mask_expanded, audio_features) - - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) - if getattr(text_config, "use_bidirectional_attention", None) == "vision": - causal_mask_mapping = self._create_causal_mask_mapping( - self.multimodal_backbone.config, - inputs_embeds, - attention_mask, - None, - position_ids, - token_type_ids, - pixel_values, - is_training=self.training, - ) - else: - causal_mask_mapping = self._create_masks_for_generate( - self.multimodal_backbone.config, - inputs_embeds, - attention_mask, - None, - position_ids, - ) - - hidden_states = self.backbone( - input_ids=None, - per_layer_inputs=per_layer_inputs, - attention_mask=causal_mask_mapping, - position_ids=position_ids, - past_key_values=None, - inputs_embeds=inputs_embeds, - use_cache=False, - return_dict=True, - ).last_hidden_state - return lm_head(hidden_states) - - def get_transpile_metadata(self): - sliding_window = getattr(self.backbone.config, "sliding_window", None) - layer_types = list(getattr(self.backbone.config, "layer_types", [])) - return { - "graph": { - **_transpile_graph_meta( - self.model, - adapter_family="gemma4", - adapter_type=type(self).__name__, - input_names=self.input_names, - ), - "task": self.metadata_task, - "num_hidden_layers": int(self.backbone.config.num_hidden_layers), - "layer_types": tuple(layer_types), - "sliding_window": None if sliding_window is None else int(sliding_window), - }, - "import_hints": _gemma4_import_hints( - self.backbone, - module_path_suffix_prefix="model.model.language_model", - ), - } - - -class Qwen35CausalLMLogitsAdapter(torch.nn.Module): - def __init__(self, model: torch.nn.Module): - super().__init__() - self.model = model - self.backbone = model.model - from transformers.models.qwen3_5.modeling_qwen3_5 import create_causal_mask # type: ignore - - self._create_causal_mask = create_causal_mask - - def forward(self, input_ids: torch.Tensor): - return self.debug_forward(input_ids)[0] - - def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: - inputs_embeds = self.backbone.embed_tokens(input_ids) - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) - position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1) - text_position_ids = position_ids[0] - multimodal_position_ids = position_ids[1:] - - causal_mask = self._create_causal_mask( - config=self.backbone.config, - inputs_embeds=inputs_embeds, - attention_mask=None, - past_key_values=None, - position_ids=text_position_ids, - ) - linear_attn_mask = self.backbone._update_linear_attn_mask(None, None) - - hidden_states = inputs_embeds - checkpoints: list[torch.Tensor] = [] - position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) - - for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask - hidden_states = decoder_layer( - hidden_states, - position_embeddings=position_embeddings, - attention_mask=layer_mask, - position_ids=text_position_ids, - past_key_values=None, - use_cache=False, - ) - checkpoints.append(hidden_states) - - hidden_states = self.backbone.norm(hidden_states) - checkpoints.append(hidden_states) - return self.model.lm_head(hidden_states), checkpoints - - def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) - import_hints: list[dict[str, object]] = [] - for layer_index, layer_type in enumerate(layer_types): - import_hints.append( - { - "module_path_suffix": f"backbone.layers.{layer_index}.self_attn", - "meta": { - "attention_layer_type": layer_type, - "attention_layer_index": layer_index, - }, - } - ) - return { - "graph": { - **_transpile_graph_meta( - self.model, - adapter_family="qwen3_5", - adapter_type=type(self).__name__, - input_names=("input_ids",), - ), - "num_hidden_layers": int(self.backbone.config.num_hidden_layers), - "layer_types": layer_types, - }, - "import_hints": import_hints, - } - - -class CTCLogitsAdapter(BoundInputAdapter): - def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): - super().__init__(model, input_names=input_names, family=family, metadata_task="ctc_logits") - - def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: - outputs = self.model(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) - return _extract_tensor_output(outputs, preferred_field="logits") - - -class EncoderHiddenStatesAdapter(BoundInputAdapter): - def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): - encoder = None - get_encoder = getattr(model, "get_encoder", None) - if callable(get_encoder): - encoder = get_encoder() - if encoder is None: - encoder = getattr(model, "encoder", None) - if encoder is None: - model_attr = getattr(model, "model", None) - if model_attr is not None: - encoder = getattr(model_attr, "encoder", None) - if not isinstance(encoder, torch.nn.Module): - raise NotImplementedError(f"{type(model).__name__} does not expose an encoder module") - super().__init__(model, input_names=input_names, family=family, metadata_task="encoder_hidden_states") - self.encoder = encoder - - def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: - outputs = self.encoder(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) - return _extract_tensor_output(outputs, preferred_field="last_hidden_state") - - -def _family_key(model: torch.nn.Module) -> str: - module_name = type(model).__module__ - if module_name.startswith("transformers.models.whisper."): - return "whisper" - if module_name.startswith("transformers.models.gemma4."): - return "gemma4" - if module_name.startswith("transformers.models.gemma3."): - return "gemma3" - if module_name.startswith("transformers.models.gemma."): - return "gemma" - if module_name.startswith("transformers.models.qwen3_5."): - return "qwen3_5" - return "generic" - - -def canonicalize_model_interface( - model: torch.nn.Module, - task: str = "causal_lm_logits", - *, - input_names: tuple[str, ...] | None = None, -) -> CanonicalizedModel: - family = _family_key(model) - adapter_factory: Callable[[torch.nn.Module], torch.nn.Module] - resolved_input_names = tuple(input_names or ()) - - if task == "causal_lm_logits": - if family == "gemma": - adapter_factory = GemmaCausalLMLogitsAdapter - elif family == "gemma4": - adapter_factory = Gemma4CausalLMLogitsAdapter - elif family == "gemma3": - adapter_factory = Gemma3CausalLMLogitsAdapter - elif family == "qwen3_5": - adapter_factory = Qwen35CausalLMLogitsAdapter - else: - adapter_factory = CausalLMLogitsAdapter - resolved_input_names = ("input_ids",) - elif task == "multimodal_causal_lm_logits": - if family != "gemma4": - raise NotImplementedError(f"{type(model).__name__} does not support task={task}") - if not resolved_input_names: - resolved_input_names = ( - "input_ids", - "attention_mask", - "token_type_ids", - "pixel_values", - "pixel_position_ids", - "input_features", - "input_features_mask", - ) - adapter_factory = lambda inner_model: Gemma4MultimodalCausalLMLogitsAdapter( # type: ignore[assignment] - inner_model, - input_names=resolved_input_names, - ) - elif task == "ctc_logits": - if not resolved_input_names: - resolved_input_names = _infer_input_names( - model, - preferred=("input_values", "input_features", "attention_mask"), - ) - adapter_factory = lambda inner_model: CTCLogitsAdapter( # type: ignore[assignment] - inner_model, - input_names=resolved_input_names, - family=family, - ) - elif task == "encoder_hidden_states": - encoder_module = None - get_encoder = getattr(model, "get_encoder", None) - if callable(get_encoder): - encoder_module = get_encoder() - if encoder_module is None: - encoder_module = getattr(model, "encoder", None) - if encoder_module is None and getattr(model, "model", None) is not None: - encoder_module = getattr(model.model, "encoder", None) - if not isinstance(encoder_module, torch.nn.Module): - raise NotImplementedError(f"{type(model).__name__} does not support task={task}") - if not resolved_input_names: - resolved_input_names = _infer_input_names( - encoder_module, - preferred=("input_features", "input_values", "attention_mask"), - ) - adapter_factory = lambda inner_model: EncoderHiddenStatesAdapter( # type: ignore[assignment] - inner_model, - input_names=resolved_input_names, - family=family, - ) - else: - raise NotImplementedError(f"unsupported canonicalization task: {task}") - - return CanonicalizedModel( - module=adapter_factory(model).eval(), - task=task, - family=family, - input_names=resolved_input_names, - ) diff --git a/python/src/transpile/model_patterns.py b/python/src/transpile/model_patterns.py deleted file mode 100644 index 22be81bda..000000000 --- a/python/src/transpile/model_patterns.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class GoldPattern: - name: str - source_models: tuple[str, ...] - semantic_ops: tuple[str, ...] - description: str - - -GOLD_PATTERNS: tuple[GoldPattern, ...] = ( - GoldPattern( - name="decoder_attention_gqa", - source_models=("model_gemma.cpp", "model_qwen.cpp", "model_lfm2.cpp", "model_gemma4.cpp"), - semantic_ops=("rms_norm", "rope", "attention", "attention_int8_hybrid"), - description=( - "Q/K/V projections from the same normalized hidden state; Q and K optionally get per-head RMSNorm, " - "Q/K are rotary-embedded, then Cactus attention consumes native Q-head and KV-head counts." - ), - ), - GoldPattern( - name="gated_mlp_gelu", - source_models=("model_gemma.cpp", "model_gemma4.cpp"), - semantic_ops=("matmul", "gelu", "multiply"), - description="Two projection branches from the same normalized input: GELU(gate) * up, then a down projection.", - ), - GoldPattern( - name="gated_mlp_silu", - source_models=("model_qwen.cpp", "model_lfm2.cpp"), - semantic_ops=("matmul", "silu", "multiply"), - description="Two projection branches from the same normalized input: SiLU(gate) * up, then a down projection.", - ), - GoldPattern( - name="decoder_block_post_attn_norm", - source_models=("model_gemma.cpp", "model_gemma4.cpp", "model_gemma3n.cpp"), - semantic_ops=("rms_norm", "attention", "add_clipped"), - description=( - "Pre-norm attention block with post-attention RMSNorm before the residual add, followed by a second " - "pre-FFN RMSNorm and a post-FFN RMSNorm before the final residual add." - ), - ), - GoldPattern( - name="decoder_block_simple_residual", - source_models=("model_qwen.cpp", "model_lfm2.cpp"), - semantic_ops=("rms_norm", "attention", "add"), - description=( - "Pre-norm attention block with a direct residual add after attention, followed by a second RMSNorm and " - "a gated MLP branch with another direct residual add." - ), - ), - GoldPattern( - name="gemma4_partial_rope_attention", - source_models=("model_gemma4.cpp",), - semantic_ops=("rms_norm", "rope", "concat", "attention"), - description=( - "Gemma4 attention where only a prefix of the head dimension is rotary-embedded, optionally with shared " - "K/V heads and sliding-window attention." - ), - ), - GoldPattern( - name="sliding_window_attention_mask", - source_models=("model_gemma.cpp", "model_gemma4.cpp"), - semantic_ops=("attention",), - description=( - "Exported PyTorch may build a boolean mask with diff/cumsum/comparisons, but the handwritten Cactus " - "models encode the same behavior directly as an attention `window_size`." - ), - ), -) - - -def gold_patterns_by_name() -> dict[str, GoldPattern]: - return {pattern.name: pattern for pattern in GOLD_PATTERNS} diff --git a/python/src/transpile/normalize.py b/python/src/transpile/normalize.py deleted file mode 100644 index b2d3f2de3..000000000 --- a/python/src/transpile/normalize.py +++ /dev/null @@ -1,199 +0,0 @@ -from __future__ import annotations - -from typing import Any - -import torch - - -def normalize_target(target: str) -> str: - if target.startswith("aten.mul_"): - return "multiply_inplace" - if target.startswith("aten._assert_tensor_metadata"): - return "identity" - if target.startswith("aten.lift_fresh_copy"): - return "identity" - if target.startswith("aten.clone"): - return "identity" - if target.startswith("aten.alias"): - return "identity" - if target.startswith("aten.detach"): - return "identity" - if target.startswith("aten.dropout"): - return "identity" - if target.startswith("aten.native_dropout"): - return "identity" - if target.startswith("aten.arange.start"): - return "arange" - if target.startswith("aten.arange"): - return "arange" - if target.startswith("aten.add"): - return "add" - if target.startswith("aten.sub"): - return "subtract" - if target.startswith("aten.mul"): - return "multiply" - if target.startswith("aten.floor_divide"): - return "floor_divide" - if target.startswith("aten.div"): - return "divide" - if target.startswith("aten.to.dtype"): - return "precision_cast" - if target.startswith("aten.to.device"): - return "identity" - if target.startswith("aten.type_as"): - return "type_as" - if target.startswith("aten.neg"): - return "negate" - if target.startswith("aten.ne.") or target == "aten.ne": - return "not_equal" - if target.startswith("aten.eq.") or target == "aten.eq": - return "equal" - if target.startswith("aten.logical_or") or target.startswith("aten.__or__"): - return "logical_or" - if target.startswith("aten.logical_and") or target.startswith("aten.__and__"): - return "logical_and" - if target.startswith("aten.logical_not") or target.startswith("aten.bitwise_not"): - return "logical_not" - if target.startswith("aten.abs"): - return "abs" - if target.startswith("aten.clamp"): - return "clamp" - if target.startswith("aten.where"): - return "where" - if target.startswith("aten.masked_scatter"): - return "masked_scatter" - if target.startswith("aten.masked_fill"): - return "masked_fill" - if target.startswith("aten.expand"): - return "expand" - if target.startswith("aten.exp"): - return "scalar_exp" - if target.startswith("aten.sqrt"): - return "scalar_sqrt" - if target.startswith("aten.reciprocal"): - return "reciprocal" - if target.startswith("aten.log"): - return "scalar_log" - if target.startswith("aten.rsqrt"): - return "rsqrt" - if target.startswith("aten.cos"): - return "cos" - if target.startswith("aten.sin"): - return "sin" - if target in ("aten.view.default", "aten.reshape.default", "aten._unsafe_view.default"): - return "reshape" - if target == "aten.flatten.using_ints": - return "flatten" - if target.startswith("aten.squeeze"): - return "squeeze" - if target.startswith("aten.unsqueeze"): - return "unsqueeze" - if target == "aten.t.default" or target.startswith("aten.transpose"): - return "transpose" - if target.startswith("aten.permute"): - return "permute" - if target.startswith("aten.numpy_T"): - return "numpy_T" - if target.startswith("aten.movedim"): - return "movedim" - if target.startswith("aten.contiguous"): - return "contiguous" - if target.startswith("aten.mm") or target.startswith("aten.matmul") or target.startswith("aten.bmm"): - return "matmul" - if target.startswith("aten.linear"): - return "linear" - if target.startswith("aten.addmm"): - return "addmm" - if target.startswith("aten.relu"): - return "relu" - if target.startswith("aten.silu"): - return "silu" - if target == "aten.gelu.erf": - return "gelu_erf" - if target.startswith("aten.gelu"): - return "gelu" - if target.startswith("aten.sigmoid"): - return "sigmoid" - if target.startswith("aten.glu"): - return "glu" - if target.startswith("aten.softplus"): - return "softplus" - if target.startswith("aten.tanh"): - return "tanh" - if target.startswith("aten.softmax"): - return "softmax" - if target.startswith("aten.scaled_dot_product_attention"): - return "scaled_dot_product_attention" - if target.startswith("aten.sum"): - return "sum" - if target.startswith("aten.mean"): - return "mean" - if target.startswith("aten.all"): - return "min" - if target.startswith("aten.var"): - return "variance" - if target.startswith("aten.min") or target.startswith("aten.amin"): - return "min" - if target.startswith("aten.max") or target.startswith("aten.amax"): - return "max" - if target.startswith("aten.cat"): - return "cat" - if target.startswith("aten.split_with_sizes"): - return "split_with_sizes" - if target.startswith("aten.chunk"): - return "chunk" - if target.startswith("aten.ones"): - return "ones" - if target.startswith("aten.slice"): - return "slice" - if target.startswith("aten.select"): - return "index" - if target.startswith("aten.gather"): - return "gather" - if target.startswith("aten.pad"): - return "pad" - if target.startswith("aten.one_hot"): - return "one_hot" - if target.startswith("aten.tril"): - return "tril" - if target.startswith("aten.unfold"): - return "unfold" - if target.startswith("aten.embedding"): - return "embedding" - if target.startswith("aten.conv1d"): - return "conv1d" - if target.startswith("aten.conv2d"): - return "conv2d" - if target.startswith("aten.pow"): - return "pow" - if target.startswith("aten.layer_norm"): - return "layer_norm" - if target.startswith("aten.native_layer_norm"): - return "layer_norm" - if target.startswith("aten.group_norm"): - return "group_norm" - if target.startswith("aten.batch_norm") or target.startswith("aten._native_batch_norm_legit_no_training"): - return "batch_norm" - if target.startswith("aten.rms_norm"): - return "rms_norm" - if target == "": - return "getitem" - return target - - -def dtype_to_ir(dtype: Any | None) -> str | None: - if dtype is None: - return None - - dtype_map = { - torch.float16: "fp16", - torch.float32: "fp32", - torch.float64: "fp64", - torch.bfloat16: "bf16", - torch.int8: "int8", - torch.int16: "int16", - torch.int32: "int32", - torch.int64: "int64", - torch.bool: "bool", - } - return dtype_map.get(dtype, str(dtype)) diff --git a/python/src/transpile/ops.py b/python/src/transpile/ops.py deleted file mode 100644 index e4441acb1..000000000 --- a/python/src/transpile/ops.py +++ /dev/null @@ -1,234 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class OpSchema: - name: str - num_inputs: int | None = None - attrs: tuple[str, ...] = () - backend_op: str | None = None - is_layout: bool = False - is_reduction: bool = False - is_fusible: bool = False - - -# Small canonical op registry for the transpiler. -# This is metadata for canonical IR ops, not another IR layer. - -OPS: tuple[OpSchema, ...] = ( - OpSchema("add", num_inputs=2, backend_op="add", is_fusible=True), - OpSchema("add_clipped", num_inputs=2, backend_op="add_clipped", is_fusible=True), - OpSchema("subtract", num_inputs=2, backend_op="subtract", is_fusible=True), - OpSchema("multiply", num_inputs=2, backend_op="multiply", is_fusible=True), - OpSchema("divide", num_inputs=2, backend_op="divide", is_fusible=True), - OpSchema("not_equal", num_inputs=2, backend_op="not_equal"), - OpSchema("equal", num_inputs=2), - OpSchema("greater", num_inputs=2), - OpSchema("less_equal", num_inputs=2), - OpSchema("logical_and", num_inputs=2), - OpSchema("logical_or", num_inputs=2), - OpSchema("logical_not", num_inputs=1), - OpSchema("abs", num_inputs=1, backend_op="abs", is_fusible=True), - OpSchema("clamp", num_inputs=1, attrs=("min", "max")), - OpSchema("pow", num_inputs=1, attrs=("exponent",), backend_op="pow"), - OpSchema("negate", num_inputs=1, is_fusible=True), - OpSchema("scalar_add", num_inputs=1, attrs=("value",), backend_op="scalar_add", is_fusible=True), - OpSchema("scalar_subtract", num_inputs=1, attrs=("value",), backend_op="scalar_subtract", is_fusible=True), - OpSchema("scalar_subtract_reverse", num_inputs=1, attrs=("value",)), - OpSchema("scalar_multiply", num_inputs=1, attrs=("value",), backend_op="scalar_multiply", is_fusible=True), - OpSchema("scalar_divide", num_inputs=1, attrs=("value",), backend_op="scalar_divide", is_fusible=True), - OpSchema("scalar_floor_divide", num_inputs=1, attrs=("value",), backend_op="scalar_floor_divide"), - OpSchema("scalar_divide_reverse", num_inputs=1, attrs=("value",)), - OpSchema("scalar_not_equal", num_inputs=1, attrs=("value",), backend_op="scalar_not_equal"), - OpSchema("scalar_equal", num_inputs=1, attrs=("value",)), - OpSchema("scalar_exp", num_inputs=1, backend_op="scalar_exp", is_fusible=True), - OpSchema("scalar_sqrt", num_inputs=1, backend_op="scalar_sqrt", is_fusible=True), - OpSchema("scalar_log", num_inputs=1, backend_op="scalar_log", is_fusible=True), - OpSchema("scalar_cos", num_inputs=1, backend_op="scalar_cos"), - OpSchema("scalar_sin", num_inputs=1, backend_op="scalar_sin"), - OpSchema("precision_cast", num_inputs=1, attrs=("dtype",), backend_op="precision_cast"), - OpSchema("view", num_inputs=1, attrs=("shape",), backend_op="reshape", is_layout=True), - OpSchema("permute", num_inputs=1, attrs=("permutation",), backend_op="permute", is_layout=True), - OpSchema("squeeze", num_inputs=1, attrs=("dim",), is_layout=True), - OpSchema("movedim", num_inputs=1, attrs=("source", "destination"), is_layout=True), - OpSchema("slice", num_inputs=1, attrs=("axis", "start", "end", "step"), backend_op="slice"), - OpSchema("split_with_sizes", num_inputs=1, attrs=("sizes", "axis")), - OpSchema("chunk", num_inputs=1, attrs=("chunks", "axis")), - OpSchema("ones", num_inputs=0, attrs=("shape", "dtype")), - OpSchema("one_hot", num_inputs=1, attrs=("num_classes",)), - OpSchema("tril", num_inputs=1, attrs=("diagonal",)), - OpSchema("unfold", num_inputs=1, attrs=("dimension", "size", "step")), - OpSchema("index", num_inputs=1, attrs=("axis", "index_value"), backend_op="index"), - OpSchema("advanced_index", num_inputs=None), - OpSchema( - "where", - num_inputs=None, - attrs=("true_is_scalar", "true_value", "false_is_scalar", "false_value"), - ), - OpSchema("masked_fill", num_inputs=2, attrs=("value",)), - OpSchema("masked_scatter", num_inputs=3), - OpSchema("getitem", num_inputs=1, attrs=("index",)), - OpSchema("cat", num_inputs=None, attrs=("axis",), backend_op="cat"), - OpSchema("matmul", num_inputs=2, backend_op="matmul"), - OpSchema("linear", num_inputs=None, attrs=("has_bias",), backend_op="matmul"), - OpSchema("addmm", num_inputs=3), - OpSchema("gather", num_inputs=2, backend_op="gather"), - OpSchema("embedding", num_inputs=2, backend_op="embedding_from_tensor"), - OpSchema("sum", num_inputs=1, attrs=("axis", "keepdim"), backend_op="sum", is_reduction=True), - OpSchema("mean", num_inputs=1, attrs=("axis", "keepdim"), backend_op="mean", is_reduction=True), - OpSchema("variance", num_inputs=1, attrs=("axis", "keepdim"), backend_op="variance", is_reduction=True), - OpSchema("min", num_inputs=1, attrs=("axis", "keepdim"), backend_op="min", is_reduction=True), - OpSchema("max", num_inputs=1, attrs=("axis", "keepdim"), backend_op="max", is_reduction=True), - OpSchema("relu", num_inputs=1, backend_op="relu", is_fusible=True), - OpSchema("silu", num_inputs=1, backend_op="silu", is_fusible=True), - OpSchema("gelu", num_inputs=1, backend_op="gelu", is_fusible=True), - OpSchema("gelu_erf", num_inputs=1, backend_op="gelu_erf", is_fusible=True), - OpSchema("sigmoid", num_inputs=1, backend_op="sigmoid", is_fusible=True), - OpSchema("glu", num_inputs=1, attrs=("axis",), backend_op="glu"), - OpSchema("tanh", num_inputs=1, backend_op="tanh", is_fusible=True), - OpSchema("softplus", num_inputs=1), - OpSchema("softmax", num_inputs=1, attrs=("axis",), backend_op="softmax"), - OpSchema("layer_norm", num_inputs=None, attrs=("eps",), backend_op="layer_norm"), - OpSchema("group_norm", num_inputs=3, attrs=("num_groups", "eps"), backend_op="group_norm"), - OpSchema("batch_norm", num_inputs=5, attrs=("axis", "eps"), backend_op="batch_norm"), - OpSchema("conv1d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv1d"), - OpSchema("conv2d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv2d"), - OpSchema("pad", num_inputs=1, attrs=("pads", "value", "mode")), - - # CUSTOM / FUSED OPS - OpSchema("rms_norm", num_inputs=2, attrs=("eps",), backend_op="rms_norm"), - OpSchema("rope", num_inputs=1, attrs=("theta", "position_offset"), backend_op="rope"), - OpSchema("rel_pos_bias", num_inputs=2, attrs=("scale",), backend_op="rel_pos_bias"), - OpSchema( - "attention", - num_inputs=3, - attrs=("scale", "is_causal", "window_size", "mask", "additive_mask"), - backend_op="attention", - ), - OpSchema( - "attention_block", - num_inputs=None, - attrs=( - "scale", - "is_causal", - "window_size", - "has_mask", - "additive_mask", - "has_gate", - "has_bias", - "attention_output_shape", - ), - is_fusible=True, - ), - OpSchema( - "self_attention_block", - num_inputs=None, - attrs=( - "scale", - "is_causal", - "window_size", - "has_mask", - "additive_mask", - "has_gate", - "has_bias", - "has_query_projection_bias", - "has_query_add", - "has_rel_query_add", - "has_key_projection_bias", - "has_value_projection_bias", - "has_rel_pos_bias", - "has_relative_key_projection_bias", - "query_shape", - "key_shape", - "value_shape", - "relative_key_shape", - "rel_pos_scale", - "attention_output_shape", - ), - is_fusible=True, - ), - OpSchema( - "conv_module", - num_inputs=None, - attrs=( - "eps", - "has_pointwise1_bias", - "has_depthwise_bias", - "has_pointwise2_bias", - "depthwise_kernel_size", - "depthwise_padding", - ), - is_fusible=True, - ), - OpSchema( - "gated_deltanet_prefill", - num_inputs=None, - attrs=( - "num_k_heads", - "num_v_heads", - "key_dim", - "value_dim", - "eps", - "chunk_size", - "has_z", - "has_dt_bias", - "has_a_log", - "has_conv", - ), - is_fusible=True, - ), - OpSchema( - "gated_deltanet_decode", - num_inputs=None, - attrs=( - "num_k_heads", - "num_v_heads", - "key_dim", - "value_dim", - "eps", - "has_z", - "has_dt_bias", - "has_a_log", - "has_conv", - ), - is_fusible=True, - ), -) - - -ALL_OPS: dict[str, OpSchema] = {op.name: op for op in OPS} - - -# Import/capture spellings reduced into the canonical registry above. -ALIASES: dict[str, str] = { - "reshape": "view", - "flatten": "view", - "unsqueeze": "view", - "expand": "view", - "contiguous": "view", - "identity": "view", - "transpose": "permute", - "concat": "cat", - "type_as": "precision_cast", - "scaled_dot_product_attention": "attention", - "cos": "scalar_cos", - "sin": "scalar_sin", -} - - -PRECISION_OP = "precision_cast" -SUPPORTED_PRECISIONS: tuple[str, ...] = ("int4", "int8", "fp16", "fp32") - - -def canonicalize_op(name: str) -> str: - return ALIASES.get(name, name) - - -def get_op(name: str) -> OpSchema: - return ALL_OPS[canonicalize_op(name)] - - -def has_op(name: str) -> bool: - return canonicalize_op(name) in ALL_OPS diff --git a/python/src/transpile/optimize_graph.py b/python/src/transpile/optimize_graph.py deleted file mode 100644 index 474c1439d..000000000 --- a/python/src/transpile/optimize_graph.py +++ /dev/null @@ -1,737 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -import torch - -from src.transpile.canonicalize.cleanup import canonicalize_exported_graph -from src.transpile.canonicalize.utils import rebuild_graph -from src.transpile.fusion import match_attention -from src.transpile.fusion import match_attention_block -from src.transpile.fusion import match_conv_module -from src.transpile.fusion import match_gated_deltanet -from src.transpile.fusion import match_gated_mlp -from src.transpile.fusion import match_rel_pos_bias -from src.transpile.fusion import match_rms_norm -from src.transpile.fusion import match_rope -from src.transpile.fusion import match_self_attention_block -from src.transpile.fusion.common import producer -from src.transpile.fusion.common import strip_passthrough -from src.transpile.graph_ir import IRGraph -from src.transpile.graph_ir import IRNode -from src.transpile.graph_ir import IRValue -from src.transpile.graph_ir import verify_ir -from src.transpile.model_patterns import GOLD_PATTERNS -from src.transpile.normalize import dtype_to_ir - - -@dataclass(frozen=True) -class DetectedPattern: - name: str - anchor_node_id: str - node_ids: tuple[str, ...] - value_ids: tuple[str, ...] - details: dict[str, object] - - -@dataclass(frozen=True) -class FusionConfig: - enable_gated_deltanet: bool = True - enable_rms_norm: bool = True - enable_rope: bool = True - enable_rel_pos_bias: bool = True - enable_attention: bool = True - enable_self_attention_block: bool = True - enable_attention_block: bool = True - enable_conv_module: bool = True - enable_add_clipped: bool = True - - -def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: - config = config or FusionConfig() - verify_ir(graph) - canonicalize_exported_graph(graph) - - for _ in range(max_passes): - changed = False - if config.enable_gated_deltanet and fuse_gated_deltanet(graph): - changed = True - if config.enable_rms_norm and fuse_rms_norm(graph): - changed = True - if config.enable_rope and fuse_rope(graph): - changed = True - if config.enable_rel_pos_bias and fuse_rel_pos_bias(graph): - changed = True - if config.enable_attention and fuse_attention(graph): - changed = True - if config.enable_self_attention_block and fuse_self_attention_blocks(graph): - changed = True - if config.enable_attention_block and fuse_attention_blocks(graph): - changed = True - if config.enable_conv_module and fuse_conv_modules(graph): - changed = True - if config.enable_add_clipped and fuse_add_clipped(graph): - changed = True - if not changed: - break - canonicalize_exported_graph(graph) - - annotate_gold_patterns(graph) - verify_ir(graph) - return graph - - -def fuse_gated_deltanet(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - - match = match_gated_deltanet(graph, node) - if match is None: - continue - - inputs = [ - match.normalized_input_value_id, - match.qkv_weight_value_id, - match.a_weight_value_id, - match.b_weight_value_id, - match.norm_weight_value_id, - ] - if match.z_weight_value_id is not None: - inputs.append(match.z_weight_value_id) - if match.dt_bias_value_id is not None: - inputs.append(match.dt_bias_value_id) - if match.a_log_value_id is not None: - inputs.append(match.a_log_value_id) - if match.conv_weight_value_id is not None: - inputs.append(match.conv_weight_value_id) - - node.op = f"gated_deltanet_{match.mode}" - node.inputs = inputs - node.attrs = { - "num_k_heads": int(match.num_k_heads), - "num_v_heads": int(match.num_v_heads), - "key_dim": int(match.key_dim), - "value_dim": int(match.value_dim), - "eps": float(match.eps), - "chunk_size": int(match.chunk_size), - "has_z": bool(match.z_weight_value_id is not None), - "has_dt_bias": bool(match.dt_bias_value_id is not None), - "has_a_log": bool(match.a_log_value_id is not None), - "has_conv": bool(match.conv_weight_value_id is not None), - } - node.kind = "semantic" - node.meta["gated_deltanet_mode"] = match.mode - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def annotate_gold_patterns(graph: IRGraph) -> list[DetectedPattern]: - _clear_gold_pattern_annotations(graph) - patterns = [ - *_detect_gated_mlps(graph), - *_detect_decoder_attentions(graph), - *_detect_transformer_blocks(graph), - ] - - for pattern in patterns: - for node_id in pattern.node_ids: - node = graph.nodes.get(node_id) - if node is None: - continue - node.meta.setdefault("gold_patterns", []) - node.meta["gold_patterns"].append(pattern.name) - - anchor = graph.nodes.get(pattern.anchor_node_id) - if anchor is not None: - anchor.meta["gold_pattern_anchor"] = True - anchor.meta["gold_pattern_details"] = pattern.details - - graph.meta["gold_patterns_catalog"] = tuple(pattern.name for pattern in GOLD_PATTERNS) - graph.meta["detected_gold_patterns"] = patterns - return patterns - - -def summarize_detected_gold_patterns(graph: IRGraph) -> dict[str, int]: - summary: dict[str, int] = {} - for pattern in annotate_gold_patterns(graph): - summary[pattern.name] = summary.get(pattern.name, 0) + 1 - return summary - - -def fuse_rms_norm(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None or node.op not in {"multiply", "type_as", "precision_cast"}: - continue - - match = match_rms_norm(graph, node) - if match is None: - continue - - weight_value_id = match.weight_value_id - if weight_value_id is None: - input_value = graph.values.get(match.input_value_id) - if input_value is None or input_value.shape is None or not input_value.shape: - continue - hidden_dim = input_value.shape[-1] - if not isinstance(hidden_dim, int) or hidden_dim <= 0: - continue - weight_value_id = _materialize_ones_constant( - graph, - hidden_dim, - dtype=input_value.dtype, - suffix="rms_norm_ones", - ) - if float(match.weight_offset) != 0.0: - weight_value_id = _materialize_shifted_constant( - graph, - weight_value_id, - float(match.weight_offset), - suffix="rms_norm_scale", - ) - - node.op = "rms_norm" - node.inputs = [match.input_value_id, weight_value_id] - node.attrs = {"eps": float(match.eps)} - node.kind = "semantic" - node.meta["rms_weight_offset"] = float(match.weight_offset) - node.meta["rms_input_value_id"] = match.input_value_id - node.meta["rms_weight_value_id"] = weight_value_id - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def fuse_rope(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None or not node.outputs: - continue - - match = match_rope(graph, node.outputs[0]) - if match is None or match.partial: - continue - - node.op = "rope" - node.inputs = [match.input_value_id] - node.attrs = { - "theta": float(match.theta), - "position_offset": int(match.position_offset), - } - node.kind = "semantic" - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def fuse_attention(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: - continue - - match = match_attention(graph, node) - if match is None: - continue - - window_size = int(node.attrs.get("window_size", 0)) - if len(node.inputs) > 3: - mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) - if mask_info is not None: - node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) - if window_size == 0: - window_size = int(mask_info["window_size"]) - node.meta["window_size_source"] = "mask_pattern" - - if window_size != 0 and "window_size_source" not in node.meta: - node.meta["window_size_source"] = "import_attr" - - semantic_attrs = { - key: value - for key, value in node.attrs.items() - if key not in {"mask", "dropout_p", "scale", "is_causal"} - } - semantic_attrs.update( - { - "scale": float(node.attrs.get("scale", 1.0)), - "is_causal": bool(node.attrs.get("is_causal", True)), - "window_size": window_size, - } - ) - - if ( - node.op == "attention" - and node.inputs[:3] == [match.query_value_id, match.key_value_id, match.value_value_id] - and node.attrs == semantic_attrs - ): - continue - - node.op = "attention" - node.inputs = [match.query_value_id, match.key_value_id, match.value_value_id] - node.attrs = semantic_attrs - node.kind = "semantic" - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def fuse_rel_pos_bias(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - - match = match_rel_pos_bias(graph, node) - if match is None: - continue - - node.op = "rel_pos_bias" - node.inputs = [match.query_value_id, match.relative_key_value_id] - node.attrs = {"scale": float(match.scale)} - node.kind = "semantic" - node.meta["rel_pos_bias_nodes"] = match.node_ids - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def fuse_attention_blocks(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - - match = match_attention_block(graph, node) - if match is None: - continue - - inputs = [match.query_value_id, match.key_value_id, match.value_value_id] - if match.mask_value_id is not None: - inputs.append(match.mask_value_id) - if match.gate_value_id is not None: - inputs.append(match.gate_value_id) - inputs.append(match.output_projection_weight_value_id) - if match.output_projection_bias_value_id is not None: - inputs.append(match.output_projection_bias_value_id) - - node.op = "attention_block" - node.inputs = inputs - node.attrs = { - "scale": float(match.scale), - "is_causal": bool(match.is_causal), - "window_size": int(match.window_size), - "has_mask": bool(match.mask_value_id is not None), - "additive_mask": bool(match.additive_mask), - "has_gate": bool(match.gate_value_id is not None), - "has_bias": bool(match.output_projection_bias_value_id is not None), - "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), - } - node.kind = "semantic" - node.meta["attention_block_source"] = match.attention_node_id - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def fuse_self_attention_blocks(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - - match = match_self_attention_block(graph, node) - if match is None: - continue - - inputs = [match.hidden_value_id, match.query_weight_value_id] - if match.query_projection_bias_value_id is not None: - inputs.append(match.query_projection_bias_value_id) - if match.query_add_value_id is not None: - inputs.append(match.query_add_value_id) - if match.rel_query_add_value_id is not None: - inputs.append(match.rel_query_add_value_id) - inputs.append(match.key_weight_value_id) - if match.key_projection_bias_value_id is not None: - inputs.append(match.key_projection_bias_value_id) - inputs.append(match.value_weight_value_id) - if match.value_projection_bias_value_id is not None: - inputs.append(match.value_projection_bias_value_id) - if match.mask_value_id is not None: - inputs.append(match.mask_value_id) - if match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None: - inputs.append(match.relative_key_input_value_id) - inputs.append(match.relative_key_weight_value_id) - if match.relative_key_projection_bias_value_id is not None: - inputs.append(match.relative_key_projection_bias_value_id) - if match.gate_value_id is not None: - inputs.append(match.gate_value_id) - inputs.append(match.output_projection_weight_value_id) - if match.output_projection_bias_value_id is not None: - inputs.append(match.output_projection_bias_value_id) - - node.op = "self_attention_block" - node.inputs = inputs - node.attrs = { - "scale": float(match.scale), - "is_causal": bool(match.is_causal), - "window_size": int(match.window_size), - "has_mask": bool(match.mask_value_id is not None), - "additive_mask": bool(match.additive_mask), - "has_gate": bool(match.gate_value_id is not None), - "has_bias": bool(match.output_projection_bias_value_id is not None), - "has_query_projection_bias": bool(match.query_projection_bias_value_id is not None), - "has_query_add": bool(match.query_add_value_id is not None), - "has_rel_query_add": bool(match.rel_query_add_value_id is not None), - "has_key_projection_bias": bool(match.key_projection_bias_value_id is not None), - "has_value_projection_bias": bool(match.value_projection_bias_value_id is not None), - "has_rel_pos_bias": bool(match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None), - "has_relative_key_projection_bias": bool(match.relative_key_projection_bias_value_id is not None), - "query_shape": tuple(int(dim) for dim in match.query_shape), - "key_shape": tuple(int(dim) for dim in match.key_shape), - "value_shape": tuple(int(dim) for dim in match.value_shape), - "relative_key_shape": tuple(int(dim) for dim in match.relative_key_shape) if match.relative_key_shape is not None else (), - "rel_pos_scale": float(match.rel_pos_scale) if match.rel_pos_scale is not None else 1.0, - "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), - } - node.kind = "semantic" - node.meta["self_attention_block_source"] = match.attention_node_id - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def fuse_add_clipped(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None or node.op != "add" or len(node.inputs) != 2: - continue - - lhs = strip_passthrough(graph, node.inputs[0]) - rhs = strip_passthrough(graph, node.inputs[1]) - if not (_looks_like_gemma_residual_add(graph, lhs, rhs) or _looks_like_gemma_residual_add(graph, rhs, lhs)): - continue - - node.op = "add_clipped" - node.kind = "semantic" - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def fuse_conv_modules(graph: IRGraph) -> bool: - changed = False - for node_id in list(graph.order): - node = graph.nodes.get(node_id) - if node is None: - continue - - match = match_conv_module(graph, node) - if match is None: - continue - - inputs = [ - match.input_value_id, - match.pointwise1_weight_value_id, - ] - if match.pointwise1_bias_value_id is not None: - inputs.append(match.pointwise1_bias_value_id) - inputs.append(match.depthwise_weight_value_id) - if match.depthwise_bias_value_id is not None: - inputs.append(match.depthwise_bias_value_id) - inputs.extend( - [ - match.batch_norm_weight_value_id, - match.batch_norm_bias_value_id, - match.batch_norm_running_mean_value_id, - match.batch_norm_running_var_value_id, - match.pointwise2_weight_value_id, - ] - ) - if match.pointwise2_bias_value_id is not None: - inputs.append(match.pointwise2_bias_value_id) - - node.op = "conv_module" - node.inputs = inputs - node.attrs = { - "eps": float(match.eps), - "has_pointwise1_bias": bool(match.pointwise1_bias_value_id is not None), - "has_depthwise_bias": bool(match.depthwise_bias_value_id is not None), - "has_pointwise2_bias": bool(match.pointwise2_bias_value_id is not None), - "depthwise_kernel_size": int(match.depthwise_kernel_size), - "depthwise_padding": int(match.depthwise_padding), - } - node.kind = "semantic" - node.meta["conv_module_nodes"] = match.node_ids - changed = True - - if changed: - rebuild_graph(graph) - return changed - - -def _materialize_shifted_constant(graph: IRGraph, value_id: str, offset: float, *, suffix: str) -> str: - base = graph.constants[value_id] - if not isinstance(base, torch.Tensor): - raise NotImplementedError(f"expected tensor constant for {value_id}, got {type(base).__name__}") - - new_value_id = f"{value_id}_{suffix}" - if new_value_id in graph.constants: - return new_value_id - - shifted = base.detach().cpu() + offset - graph.constants[new_value_id] = shifted - graph.values[new_value_id] = IRValue( - id=new_value_id, - shape=tuple(shifted.shape), - dtype=dtype_to_ir(shifted.dtype), - producer=None, - users=[], - ) - return new_value_id - - -def _materialize_ones_constant(graph: IRGraph, size: int, *, dtype: str | None, suffix: str) -> str: - dtype_map = { - "fp16": torch.float16, - "bf16": torch.bfloat16, - "fp32": torch.float32, - "fp64": torch.float64, - } - torch_dtype = dtype_map.get(dtype, torch.float32) - new_value_id = f"c_{suffix}_{size}_{dtype or 'fp32'}" - if new_value_id in graph.constants: - return new_value_id - - value = torch.ones((size,), dtype=torch_dtype) - graph.constants[new_value_id] = value - graph.values[new_value_id] = IRValue( - id=new_value_id, - shape=(size,), - dtype=dtype_to_ir(value.dtype), - producer=None, - users=[], - ) - return new_value_id - - -def _clear_gold_pattern_annotations(graph: IRGraph) -> None: - for node in graph.nodes.values(): - node.meta.pop("gold_patterns", None) - node.meta.pop("gold_pattern_anchor", None) - node.meta.pop("gold_pattern_details", None) - - -def _detect_gated_mlps(graph: IRGraph) -> list[DetectedPattern]: - patterns: list[DetectedPattern] = [] - for node_id in graph.order: - node = graph.nodes.get(node_id) - if node is None: - continue - - match = match_gated_mlp(graph, node) - if match is None: - continue - - pattern_name = "gated_mlp_gelu" if match.activation == "gelu" else "gated_mlp_silu" - patterns.append( - DetectedPattern( - name=pattern_name, - anchor_node_id=node.id, - node_ids=match.node_ids, - value_ids=(match.input_value_id,), - details={ - "activation": match.activation, - "input_value_id": match.input_value_id, - "gate_weight_value_id": match.gate_weight_value_id, - "up_weight_value_id": match.up_weight_value_id, - "down_weight_value_id": match.down_weight_value_id, - }, - ) - ) - return patterns - - -def _detect_decoder_attentions(graph: IRGraph) -> list[DetectedPattern]: - patterns: list[DetectedPattern] = [] - for node_id in graph.order: - node = graph.nodes.get(node_id) - if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: - continue - - match = match_attention(graph, node) - if match is None: - continue - - q_input, k_input, v_input = match.source_input_value_ids - pattern_name = "decoder_attention_gqa" if match.has_gqa_repeat else "decoder_attention" - - node_ids = set(match.node_ids) - details = { - "input_value_ids": (q_input, k_input, v_input), - "q_linear_weight_value_id": match.weight_value_ids[0], - "k_linear_weight_value_id": match.weight_value_ids[1], - "v_linear_weight_value_id": match.weight_value_ids[2], - "has_rope": bool(match.has_rope), - "has_qk_rms_norm": bool(match.has_qk_norm), - "has_gqa_repeat": bool(match.has_gqa_repeat), - "is_causal": bool(match.is_causal), - "scale": float(match.scale), - "window_size_hint": int(match.window_size), - } - - if len(node.inputs) > 3: - details["mask_value_id"] = node.inputs[3] - mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) - if mask_info is not None: - details["mask_pattern"] = "sliding_window_attention_mask" - details["window_size_hint"] = int(mask_info["window_size"]) - node_ids.update(mask_info["node_ids"]) - - patterns.append( - DetectedPattern( - name=pattern_name, - anchor_node_id=node.id, - node_ids=tuple(sorted(node_ids)), - value_ids=(q_input, k_input, v_input), - details=details, - ) - ) - return patterns - - -def _detect_transformer_blocks(graph: IRGraph) -> list[DetectedPattern]: - patterns: list[DetectedPattern] = [] - for node_id in graph.order: - node = graph.nodes.get(node_id) - if node is None or node.op not in {"add", "add_clipped"} or len(node.inputs) != 2: - continue - - rhs_pattern = _find_anchor_pattern(graph, node.inputs[1], {"gated_mlp_gelu", "gated_mlp_silu"}) - lhs_pattern = _find_anchor_pattern(graph, node.inputs[0], {"gated_mlp_gelu", "gated_mlp_silu"}) - mlp_pattern = rhs_pattern or lhs_pattern - if mlp_pattern is None: - continue - - residual_value_id = node.inputs[0] if rhs_pattern is not None else node.inputs[1] - attn_pattern = _find_anchor_pattern( - graph, - residual_value_id, - {"decoder_attention_gqa", "decoder_attention", "gemma4_partial_rope_attention"}, - ) - if attn_pattern is None: - continue - - node_ids = {node.id} - for candidate in graph.nodes.values(): - gold_patterns = candidate.meta.get("gold_patterns", []) - if attn_pattern in gold_patterns or mlp_pattern in gold_patterns: - node_ids.add(candidate.id) - - patterns.append( - DetectedPattern( - name="decoder_block_post_attn_norm" if node.op == "add_clipped" else "decoder_block_simple_residual", - anchor_node_id=node.id, - node_ids=tuple(sorted(node_ids)), - value_ids=(residual_value_id,), - details={ - "residual_value_id": residual_value_id, - "attention_pattern": attn_pattern, - "mlp_pattern": mlp_pattern, - "residual_op": node.op, - }, - ) - ) - return patterns - - -def _looks_like_gemma_residual_add(graph: IRGraph, residual_value_id: str, branch_value_id: str) -> bool: - branch_node = producer(graph, strip_passthrough(graph, branch_value_id)) - if branch_node is None or branch_node.op != "rms_norm": - return False - return strip_passthrough(graph, residual_value_id) != strip_passthrough(graph, branch_node.inputs[0]) - - -def _find_anchor_pattern(graph: IRGraph, value_id: str, names: set[str]) -> str | None: - current = value_id - visited: set[str] = set() - while current not in visited: - visited.add(current) - current = strip_passthrough(graph, current) - node = producer(graph, current) - if node is None: - return None - - for name in node.meta.get("gold_patterns", []): - if name in names and node.meta.get("gold_pattern_anchor"): - return name - - if len(node.inputs) != 1: - return None - current = node.inputs[0] - return None - - -def _extract_sliding_window_mask(graph: IRGraph, value_id: str) -> dict[str, object] | None: - node_ids: set[str] = set() - top_and = producer(graph, strip_passthrough(graph, value_id)) - if top_and is None or top_and.op not in {"aten.__and__.Tensor", "logical_and"}: - return None - node_ids.add(top_and.id) - - stack = list(top_and.inputs) - saw_diff = False - saw_cumsum = False - window_candidates: list[int] = [] - - while stack: - current = strip_passthrough(graph, stack.pop()) - node = producer(graph, current) - if node is None or node.id in node_ids: - continue - node_ids.add(node.id) - - if node.op in {"aten.diff.default", "diff"}: - saw_diff = True - elif node.op in {"aten.cumsum.default", "cumsum"}: - saw_cumsum = True - elif node.op == "scalar_subtract": - maybe_window = int(node.attrs.get("value", 0)) - if maybe_window > 0: - window_candidates.append(maybe_window) - - stack.extend(node.inputs) - - if not saw_diff or not saw_cumsum: - return None - - return { - "window_size": max((candidate for candidate in window_candidates if candidate > 1), default=0), - "node_ids": tuple(sorted(node_ids)), - } diff --git a/python/src/transpile/test_capture.py b/python/src/transpile/test_capture.py deleted file mode 100644 index 539d6f437..000000000 --- a/python/src/transpile/test_capture.py +++ /dev/null @@ -1,187 +0,0 @@ -from __future__ import annotations - -import json -import re -from dataclasses import asdict -from pathlib import Path -from typing import Any - -import torch -from transformers import AutoModelForCausalLM -from transformers import AutoTokenizer - -from src.transpile.capture.capture_pytorch import capture_model_with_fallback -from src.transpile.capture.capture_pytorch import dump_graph -from src.transpile.capture.graph_ir import verify_ir - - -MODELS = ( - "google/gemma-4-E2B", - "google/gemma-3-270m-it", - "google/gemma-2b-it", - "google/gemma-2-2b", - "Qwen/Qwen3.5-2B", - "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "HuggingFaceTB/SmolLM2-360M-Instruct", -) - -PROMPT = "The capital of France is" -ARTIFACTS_DIR = Path(__file__).resolve().parents[3] / "artifacts" / "python_capture_tests" - - -def slugify(value: str) -> str: - return re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("._-") or "model" - - -def resolve_local_snapshot(model_id: str) -> str | None: - explicit = Path(model_id) - if explicit.exists(): - return str(explicit) - - snapshots_dir = ( - Path.home() - / ".cache" - / "huggingface" - / "hub" - / ("models--" + model_id.replace("/", "--")) - / "snapshots" - ) - if not snapshots_dir.exists(): - return None - - snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) - if not snapshots: - return None - return str(snapshots[-1]) - - -def jsonify(value: Any) -> Any: - if isinstance(value, dict): - return {str(key): jsonify(inner) for key, inner in value.items()} - if isinstance(value, (list, tuple)): - return [jsonify(inner) for inner in value] - if isinstance(value, Path): - return str(value) - if isinstance(value, torch.dtype): - return str(value) - if isinstance(value, torch.Tensor): - return { - "type": "torch.Tensor", - "dtype": str(value.dtype), - "shape": list(value.shape), - } - if hasattr(value, "__dataclass_fields__"): - return jsonify(asdict(value)) - if isinstance(value, (str, int, float, bool)) or value is None: - return value - return repr(value) - - -def graph_to_dict(graph) -> dict[str, Any]: - return { - "meta": jsonify(graph.meta), - "inputs": list(graph.inputs), - "outputs": list(graph.outputs), - "constants": {value_id: jsonify(constant) for value_id, constant in graph.constants.items()}, - "values": {value_id: jsonify(asdict(value)) for value_id, value in graph.values.items()}, - "nodes": [jsonify(asdict(graph.nodes[node_id])) for node_id in graph.order], - } - - -def write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - - -class CausalLMCaptureWrapper(torch.nn.Module): - def __init__(self, model: torch.nn.Module): - super().__init__() - self.model = model.eval() - - def forward(self, input_ids: torch.Tensor) -> torch.Tensor: - outputs = self.model( - input_ids=input_ids, - use_cache=False, - return_dict=False, - ) - if isinstance(outputs, torch.Tensor): - return outputs - if isinstance(outputs, (tuple, list)) and outputs and isinstance(outputs[0], torch.Tensor): - return outputs[0] - logits = getattr(outputs, "logits", None) - if isinstance(logits, torch.Tensor): - return logits - raise TypeError(f"unsupported model output type: {type(outputs).__name__}") - - -def capture_one_model(model_id: str) -> dict[str, Any]: - snapshot = resolve_local_snapshot(model_id) - if snapshot is None: - raise FileNotFoundError(f"no local snapshot found for {model_id}") - - artifact_dir = ARTIFACTS_DIR / slugify(model_id) - tokenizer = AutoTokenizer.from_pretrained(snapshot, local_files_only=True) - encoded = tokenizer(PROMPT, return_tensors="pt") - input_ids = encoded["input_ids"] - - model = AutoModelForCausalLM.from_pretrained( - snapshot, - torch_dtype="auto", - device_map=None, - low_cpu_mem_usage=True, - local_files_only=True, - ).eval() - - captured = capture_model_with_fallback( - CausalLMCaptureWrapper(model), - (input_ids,), - ) - verify_ir(captured.ir_graph) - - write_json( - artifact_dir / "graph_ir.json", - { - "model_id": model_id, - "snapshot": snapshot, - "prompt": PROMPT, - "strict": captured.strict, - "graph": graph_to_dict(captured.ir_graph), - }, - ) - (artifact_dir / "fx_graph.txt").write_text(dump_graph(captured) + "\n") - - return { - "model_id": model_id, - "snapshot": snapshot, - "artifact_dir": str(artifact_dir), - "strict": captured.strict, - "ir_nodes": len(captured.ir_graph.order), - "ir_values": len(captured.ir_graph.values), - } - - -def main() -> None: - results: list[dict[str, Any]] = [] - for model_id in MODELS: - print(f"capturing {model_id}") - try: - result = capture_one_model(model_id) - except Exception as exc: - result = { - "model_id": model_id, - "error": str(exc), - } - results.append(result) - print(json.dumps(result, indent=2)) - - write_json( - ARTIFACTS_DIR / "capture_manifest.json", - { - "prompt": PROMPT, - "results": results, - }, - ) - - -if __name__ == "__main__": - main() diff --git a/python/src/transpile/weight_binding.py b/python/src/transpile/weight_binding.py deleted file mode 100644 index f0a8221c9..000000000 --- a/python/src/transpile/weight_binding.py +++ /dev/null @@ -1,330 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import json -import os -from pathlib import Path -import re - - -@dataclass(frozen=True) -class WeightBinding: - path: str - kind: str # "weight" | "embedding" - source_name: str - - -_PROJECT_ROOT = Path(__file__).resolve().parents[3] - -_PARAKEET_GLOBAL_FILENAMES: dict[str, str] = { - "ctc_head.weight": "ctc_head_weight.weights", - "ctc_head.bias": "ctc_head_bias.bias", - "encoder.subsampling.layers.0.weight": "subsampling_conv0_weight.weights", - "encoder.subsampling.layers.0.bias": "subsampling_conv0_bias.bias", - "encoder.subsampling.layers.2.weight": "subsampling_depthwise1_weight.weights", - "encoder.subsampling.layers.2.bias": "subsampling_depthwise1_bias.bias", - "encoder.subsampling.layers.3.weight": "subsampling_pointwise1_weight.weights", - "encoder.subsampling.layers.3.bias": "subsampling_pointwise1_bias.bias", - "encoder.subsampling.layers.5.weight": "subsampling_depthwise2_weight.weights", - "encoder.subsampling.layers.5.bias": "subsampling_depthwise2_bias.bias", - "encoder.subsampling.layers.6.weight": "subsampling_pointwise2_weight.weights", - "encoder.subsampling.layers.6.bias": "subsampling_pointwise2_bias.bias", - "encoder.subsampling.linear.weight": "subsampling_linear_weight.weights", - "encoder.subsampling.linear.bias": "subsampling_linear_bias.bias", - "encoder.pre_encode.conv.0.weight": "subsampling_conv0_weight.weights", - "encoder.pre_encode.conv.0.bias": "subsampling_conv0_bias.bias", - "encoder.pre_encode.conv.2.weight": "subsampling_depthwise1_weight.weights", - "encoder.pre_encode.conv.2.bias": "subsampling_depthwise1_bias.bias", - "encoder.pre_encode.conv.3.weight": "subsampling_pointwise1_weight.weights", - "encoder.pre_encode.conv.3.bias": "subsampling_pointwise1_bias.bias", - "encoder.pre_encode.conv.5.weight": "subsampling_depthwise2_weight.weights", - "encoder.pre_encode.conv.5.bias": "subsampling_depthwise2_bias.bias", - "encoder.pre_encode.conv.6.weight": "subsampling_pointwise2_weight.weights", - "encoder.pre_encode.conv.6.bias": "subsampling_pointwise2_bias.bias", - "encoder.pre_encode.out.weight": "subsampling_linear_weight.weights", - "encoder.pre_encode.out.bias": "subsampling_linear_bias.bias", -} - -_PARAKEET_LAYER_FILENAMES: dict[str, str] = { - "feed_forward1.linear1.weight": "ff1_linear1.weights", - "feed_forward1.linear1.bias": "ff1_linear1.bias", - "feed_forward1.linear2.weight": "ff1_linear2.weights", - "feed_forward1.linear2.bias": "ff1_linear2.bias", - "feed_forward2.linear1.weight": "ff2_linear1.weights", - "feed_forward2.linear1.bias": "ff2_linear1.bias", - "feed_forward2.linear2.weight": "ff2_linear2.weights", - "feed_forward2.linear2.bias": "ff2_linear2.bias", - "self_attn.q_proj.weight": "self_attn_q.weights", - "self_attn.q_proj.bias": "self_attn_q.bias", - "self_attn.k_proj.weight": "self_attn_k.weights", - "self_attn.k_proj.bias": "self_attn_k.bias", - "self_attn.v_proj.weight": "self_attn_v.weights", - "self_attn.v_proj.bias": "self_attn_v.bias", - "self_attn.o_proj.weight": "self_attn_output.weights", - "self_attn.o_proj.bias": "self_attn_output.bias", - "self_attn.relative_k_proj.weight": "self_attn_relative_k.weights", - "self_attn.linear_q.weight": "self_attn_q.weights", - "self_attn.linear_q.bias": "self_attn_q.bias", - "self_attn.linear_k.weight": "self_attn_k.weights", - "self_attn.linear_k.bias": "self_attn_k.bias", - "self_attn.linear_v.weight": "self_attn_v.weights", - "self_attn.linear_v.bias": "self_attn_v.bias", - "self_attn.linear_out.weight": "self_attn_output.weights", - "self_attn.linear_out.bias": "self_attn_output.bias", - "self_attn.linear_pos.weight": "self_attn_relative_k.weights", - "self_attn.bias_u": "self_attn_bias_u.weights", - "self_attn.bias_v": "self_attn_bias_v.weights", - "self_attn.pos_bias_u": "self_attn_bias_u.weights", - "self_attn.pos_bias_v": "self_attn_bias_v.weights", - "conv.pointwise_conv1.weight": "conv_pointwise1.weights", - "conv.pointwise_conv1.bias": "conv_pointwise1.bias", - "conv.depthwise_conv.weight": "conv_depthwise.weights", - "conv.depthwise_conv.bias": "conv_depthwise.bias", - "conv.pointwise_conv2.weight": "conv_pointwise2.weights", - "conv.pointwise_conv2.bias": "conv_pointwise2.bias", - "conv.norm.weight": "conv_batchnorm_weight.weights", - "conv.norm.bias": "conv_batchnorm_bias.bias", - "conv.norm.running_mean": "conv_batchnorm_running_mean.weights", - "conv.norm.running_var": "conv_batchnorm_running_var.weights", - "conv.batch_norm.weight": "conv_batchnorm_weight.weights", - "conv.batch_norm.bias": "conv_batchnorm_bias.bias", - "conv.batch_norm.running_mean": "conv_batchnorm_running_mean.weights", - "conv.batch_norm.running_var": "conv_batchnorm_running_var.weights", - "norm_feed_forward1.weight": "norm_ff1.weights", - "norm_feed_forward1.bias": "norm_ff1.bias", - "norm_self_att.weight": "norm_self_attn.weights", - "norm_self_att.bias": "norm_self_attn.bias", - "norm_conv.weight": "norm_conv.weights", - "norm_conv.bias": "norm_conv.bias", - "norm_feed_forward2.weight": "norm_ff2.weights", - "norm_feed_forward2.bias": "norm_ff2.bias", - "norm_out.weight": "norm_out.weights", - "norm_out.bias": "norm_out.bias", -} - - -def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: - candidates: list[str] = [] - raw = model_name_or_path.strip() - if not raw: - return candidates - - def _add(name: str) -> None: - name = name.strip().lower() - if name and name not in candidates: - candidates.append(name) - - _add(raw.split("/")[-1]) - - path = Path(raw) - parts = path.parts - for part in parts: - if part.startswith("models--"): - _add(part[len("models--"):].split("--")[-1]) - break - - return candidates - - -def _default_weights_dir_for_model_name(model_name_or_path: str) -> str | None: - if not model_name_or_path: - return None - for model_dir_name in _candidate_model_dir_names(model_name_or_path): - candidate = _PROJECT_ROOT / "weights" / model_dir_name - if candidate.exists(): - return str(candidate) - return None - - -def _normalized_source_candidates(source_name: str) -> list[str]: - candidates: list[str] = [] - - def _add(name: str) -> None: - if name and name not in candidates: - candidates.append(name) - - raw = source_name.strip() - _add(raw) - - for prefix in ("p_", "b_", "c_"): - if raw.startswith(prefix): - _add(raw[len(prefix):]) - - for prefix in ("adapter.model.", "adapter.module.", "module."): - if raw.startswith(prefix): - _add(raw[len(prefix):]) - - stripped = candidates[-1] - - for prefix in ( - "module.backbone.", - "adapter.backbone.", - "adapter.model.model.", - "adapter.model.language_model.", - ): - if stripped.startswith(prefix): - tail = stripped[len(prefix):] - _add(f"model.{tail}") - _add(f"model.language_model.{tail}") - if tail.startswith("layers."): - _add(f"model.{tail}") - _add(f"model.language_model.{tail}") - break - - if stripped == "adapter.model.lm_head.weight": - _add("lm_head.weight") - - layer_match = re.match( - r"^(?:(?:module|adapter)_)?backbone_layers_slice_none__\d+__none____modules__(\d+)___(.+)$", - stripped, - ) - if layer_match: - layer_index = int(layer_match.group(1)) - tail = layer_match.group(2) - tail_map = { - "self_attn_q_proj_weight": "self_attn.q_proj.weight", - "self_attn_k_proj_weight": "self_attn.k_proj.weight", - "self_attn_v_proj_weight": "self_attn.v_proj.weight", - "self_attn_o_proj_weight": "self_attn.o_proj.weight", - "self_attn_q_norm_weight": "self_attn.q_norm.weight", - "self_attn_k_norm_weight": "self_attn.k_norm.weight", - "mlp_gate_proj_weight": "mlp.gate_proj.weight", - "mlp_up_proj_weight": "mlp.up_proj.weight", - "mlp_down_proj_weight": "mlp.down_proj.weight", - "input_layernorm_weight": "input_layernorm.weight", - "post_attention_layernorm_weight": "post_attention_layernorm.weight", - "linear_attn_in_proj_qkv_weight": "linear_attn.in_proj_qkv.weight", - "linear_attn_conv1d_weight": "linear_attn.conv1d.weight", - "linear_attn_norm_weight": "linear_attn.norm.weight", - "linear_attn_dt_bias": "linear_attn.dt_bias", - "linear_attn_A_log": "linear_attn.A_log", - } - dotted_tail = tail_map.get(tail) - if dotted_tail is not None: - _add(f"model.layers.{layer_index}.{dotted_tail}") - _add(f"model.language_model.layers.{layer_index}.{dotted_tail}") - - embed_map = { - "module_backbone_embed_tokens_weight": [ - "model.embed_tokens.weight", - "model.language_model.embed_tokens.weight", - ], - "adapter_backbone_embed_tokens_weight": [ - "model.embed_tokens.weight", - "model.language_model.embed_tokens.weight", - ], - "module_backbone_embed_tokens_per_layer_weight": [ - "model.embed_tokens_per_layer.weight", - "model.language_model.embed_tokens_per_layer.weight", - ], - "adapter_backbone_embed_tokens_per_layer_weight": [ - "model.embed_tokens_per_layer.weight", - "model.language_model.embed_tokens_per_layer.weight", - ], - "module_backbone_per_layer_model_projection_weight": [ - "model.per_layer_model_projection.weight", - "model.language_model.per_layer_model_projection.weight", - ], - "adapter_backbone_per_layer_model_projection_weight": [ - "model.per_layer_model_projection.weight", - "model.language_model.per_layer_model_projection.weight", - ], - "module_backbone_norm_weight": [ - "model.norm.weight", - "model.language_model.norm.weight", - ], - "adapter_backbone_norm_weight": [ - "model.norm.weight", - "model.language_model.norm.weight", - ], - "module_model_lm_head_weight": [ - "lm_head.weight", - ], - "adapter_model_lm_head_weight": [ - "lm_head.weight", - ], - } - for key, mapped in embed_map.items(): - if stripped == key: - for item in mapped: - _add(item) - - return candidates - - -def _fallback_filename_candidates(source_name: str) -> list[str]: - filenames: list[str] = [] - - def _add(filename: str | None) -> None: - if filename and filename not in filenames: - filenames.append(filename) - - for candidate in _normalized_source_candidates(source_name): - _add(_PARAKEET_GLOBAL_FILENAMES.get(candidate)) - - layer_match = re.match(r"^encoder\.layers\.(\d+)\.(.+)$", candidate) - if layer_match: - layer_index = int(layer_match.group(1)) - suffix = layer_match.group(2) - mapped = _PARAKEET_LAYER_FILENAMES.get(suffix) - if mapped is not None: - _add(f"layer_{layer_index}_{mapped}") - - return filenames - - -def resolve_transpile_weights_dir(graph_meta: dict[str, object]) -> str | None: - explicit = graph_meta.get("weights_dir") - if isinstance(explicit, str) and explicit: - return explicit - - family = str(graph_meta.get("adapter_family", "")).upper() - family_env = f"CACTUS_TRANSPILER_WEIGHTS_DIR_{family}" - if family and family_env in os.environ and os.environ[family_env]: - return os.environ[family_env] - - generic = os.environ.get("CACTUS_TRANSPILER_WEIGHTS_DIR") - if generic: - return generic - - model_name_or_path = graph_meta.get("model_name_or_path") - if isinstance(model_name_or_path, str) and model_name_or_path: - default_dir = _default_weights_dir_for_model_name(model_name_or_path) - if default_dir: - return default_dir - return None - - -def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> WeightBinding | None: - if not weights_dir: - return None - root = Path(weights_dir) - if not root.exists(): - return None - manifest_path = root / "weights_manifest.json" - if not manifest_path.exists(): - return None - try: - manifest = json.loads(manifest_path.read_text()) - except Exception: - return None - if not isinstance(manifest, dict): - return None - for candidate_name in _normalized_source_candidates(source_name): - entry = manifest.get(candidate_name) - if not isinstance(entry, dict): - continue - filename = entry.get("filename") - kind = entry.get("kind", "weight") - if not isinstance(filename, str) or not isinstance(kind, str): - continue - candidate = root / filename - if not candidate.exists(): - continue - return WeightBinding(path=str(candidate), kind=kind, source_name=candidate_name) - - for filename in _fallback_filename_candidates(source_name): - candidate = root / filename - if candidate.exists(): - return WeightBinding(path=str(candidate), kind="weight", source_name=source_name) - return None From d1ac103801729e1a2a548e7926f7edc5ed702342 Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Tue, 12 May 2026 08:59:40 -0700 Subject: [PATCH 04/11] Transpiler --- python/src/transpile/audio_preprocess.py | 298 ++ python/src/transpile/canonicalize/cleanup.py | 657 +++++ python/src/transpile/canonicalize/utils.py | 274 ++ python/src/transpile/capture_pytorch.py | 344 +++ .../src/transpile/component_bundle_runtime.py | 835 ++++++ python/src/transpile/component_partition.py | 414 +++ python/src/transpile/component_pipeline.py | 193 ++ python/src/transpile/fusion/__init__.py | 47 + python/src/transpile/fusion/attention.py | 550 ++++ python/src/transpile/fusion/common.py | 73 + python/src/transpile/fusion/conv.py | 123 + python/src/transpile/fusion/deltanet.py | 357 +++ python/src/transpile/fusion/linear.py | 58 + python/src/transpile/fusion/lstm.py | 216 ++ python/src/transpile/fusion/mlp.py | 53 + python/src/transpile/fusion/rel_pos_bias.py | 179 ++ python/src/transpile/fusion/rms_norm.py | 114 + python/src/transpile/fusion/rope.py | 252 ++ python/src/transpile/graph_ir.py | 201 ++ python/src/transpile/import_ir.py | 271 ++ python/src/transpile/import_semantics.py | 337 +++ python/src/transpile/importers.py | 1858 ++++++++++++ python/src/transpile/lower.py | 2424 ++++++++++++++++ python/src/transpile/model_adapters.py | 2507 +++++++++++++++++ python/src/transpile/model_patterns.py | 76 + python/src/transpile/normalize.py | 199 ++ python/src/transpile/ops.py | 236 ++ python/src/transpile/optimize_graph.py | 1135 ++++++++ python/src/transpile/parakeet_tdt_local.py | 756 +++++ python/src/transpile/weight_binding.py | 461 +++ 30 files changed, 15498 insertions(+) create mode 100644 python/src/transpile/audio_preprocess.py create mode 100644 python/src/transpile/canonicalize/cleanup.py create mode 100644 python/src/transpile/canonicalize/utils.py create mode 100644 python/src/transpile/capture_pytorch.py create mode 100644 python/src/transpile/component_bundle_runtime.py create mode 100644 python/src/transpile/component_partition.py create mode 100644 python/src/transpile/component_pipeline.py create mode 100644 python/src/transpile/fusion/__init__.py create mode 100644 python/src/transpile/fusion/attention.py create mode 100644 python/src/transpile/fusion/common.py create mode 100644 python/src/transpile/fusion/conv.py create mode 100644 python/src/transpile/fusion/deltanet.py create mode 100644 python/src/transpile/fusion/linear.py create mode 100644 python/src/transpile/fusion/lstm.py create mode 100644 python/src/transpile/fusion/mlp.py create mode 100644 python/src/transpile/fusion/rel_pos_bias.py create mode 100644 python/src/transpile/fusion/rms_norm.py create mode 100644 python/src/transpile/fusion/rope.py create mode 100644 python/src/transpile/graph_ir.py create mode 100644 python/src/transpile/import_ir.py create mode 100644 python/src/transpile/import_semantics.py create mode 100644 python/src/transpile/importers.py create mode 100644 python/src/transpile/lower.py create mode 100644 python/src/transpile/model_adapters.py create mode 100644 python/src/transpile/model_patterns.py create mode 100644 python/src/transpile/normalize.py create mode 100644 python/src/transpile/ops.py create mode 100644 python/src/transpile/optimize_graph.py create mode 100644 python/src/transpile/parakeet_tdt_local.py create mode 100644 python/src/transpile/weight_binding.py diff --git a/python/src/transpile/audio_preprocess.py b/python/src/transpile/audio_preprocess.py new file mode 100644 index 000000000..59d71862b --- /dev/null +++ b/python/src/transpile/audio_preprocess.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +import numpy as np +import torch +from scipy.io import wavfile +from scipy.signal import resample_poly + + +_PARAKEET_SAMPLE_RATE = 16000 +_PARAKEET_NUM_MELS = 128 +_PARAKEET_N_FFT = 512 +_PARAKEET_FRAME_LENGTH = 400 +_PARAKEET_HOP_LENGTH = 160 +_PARAKEET_PREEMPHASIS = 0.97 +_PARAKEET_LOG_FLOOR = np.float32(2**-24) + + +def normalize_audio_samples(samples: np.ndarray) -> np.ndarray: + if samples.dtype == np.uint8: + return ((samples.astype(np.float32) - 128.0) / 128.0).astype(np.float32) + if np.issubdtype(samples.dtype, np.integer): + info = np.iinfo(samples.dtype) + scale = float(max(abs(info.min), info.max)) + if scale <= 0: + scale = 1.0 + return (samples.astype(np.float32) / scale).astype(np.float32) + return samples.astype(np.float32) + + +def load_audio_waveform(audio_file: str | Path, *, target_sample_rate: int) -> np.ndarray: + sample_rate, waveform = wavfile.read(str(audio_file)) + if waveform.ndim > 1: + waveform = waveform.mean(axis=1) + waveform = normalize_audio_samples(np.asarray(waveform)) + if int(sample_rate) != int(target_sample_rate): + gcd = int(np.gcd(int(sample_rate), int(target_sample_rate))) + waveform = resample_poly( + waveform, + int(target_sample_rate) // gcd, + int(sample_rate) // gcd, + ).astype(np.float32) + return waveform.astype(np.float32) + + +def _hz_to_mel(hz: np.ndarray) -> np.ndarray: + return 2595.0 * np.log10(1.0 + hz / 700.0) + + +def _mel_to_hz(mel: np.ndarray) -> np.ndarray: + return 700.0 * (np.power(10.0, mel / 2595.0) - 1.0) + + +@lru_cache(maxsize=16) +def _mel_filter_bank(num_mels: int, sample_rate: int, n_fft: int) -> np.ndarray: + try: + from transformers.audio_utils import mel_filter_bank as transformers_mel_filter_bank # type: ignore + + return transformers_mel_filter_bank( + num_frequency_bins=n_fft // 2 + 1, + num_mel_filters=num_mels, + min_frequency=0.0, + max_frequency=sample_rate / 2.0, + sampling_rate=sample_rate, + norm="slaney", + mel_scale="slaney", + ).astype(np.float32) + except Exception: + pass + + num_freq_bins = n_fft // 2 + 1 + mel_points = np.linspace( + _hz_to_mel(np.array([0.0], dtype=np.float32))[0], + _hz_to_mel(np.array([sample_rate / 2.0], dtype=np.float32))[0], + num_mels + 2, + ) + hz_points = _mel_to_hz(mel_points) + bins = np.floor((n_fft + 1) * hz_points / sample_rate).astype(np.int64) + + filters = np.zeros((num_mels, num_freq_bins), dtype=np.float32) + for index in range(1, num_mels + 1): + left = max(0, min(num_freq_bins - 1, int(bins[index - 1]))) + center = max(left + 1, min(num_freq_bins - 1, int(bins[index]))) + right = max(center + 1, min(num_freq_bins, int(bins[index + 1]))) + for freq in range(left, center): + filters[index - 1, freq] = float(freq - left) / float(max(center - left, 1)) + for freq in range(center, right): + filters[index - 1, freq] = float(right - freq) / float(max(right - center, 1)) + return filters + + +def generic_log_mel_features( + waveform: np.ndarray, + *, + sample_rate: int, + num_mels: int, + n_fft: int, + hop_length: int, + frame_length: int, + preemphasis: float | None = None, + mel_floor: np.float32 = _PARAKEET_LOG_FLOOR, + normalize_active_frames_only: bool = True, +) -> tuple[np.ndarray, int]: + waveform_tensor = torch.from_numpy(waveform).to(torch.float32).unsqueeze(0) + if preemphasis is not None and abs(preemphasis) > 0.0: + waveform_tensor = torch.cat( + [ + waveform_tensor[:, :1], + waveform_tensor[:, 1:] - float(preemphasis) * waveform_tensor[:, :-1], + ], + dim=1, + ) + + window = torch.hann_window(frame_length, periodic=False, dtype=torch.float32) + stft = torch.stft( + waveform_tensor, + n_fft=n_fft, + hop_length=hop_length, + win_length=frame_length, + window=window, + return_complex=True, + center=True, + pad_mode="constant", + ) + magnitudes = torch.view_as_real(stft) + magnitudes = torch.sqrt(magnitudes.pow(2).sum(-1)) + power = magnitudes.pow(2) + + feature_length = max(1, int(waveform.shape[0] // hop_length)) + mel_filters = torch.from_numpy(_mel_filter_bank(num_mels, sample_rate, n_fft)).to(torch.float32) + mel_spec = power.transpose(1, 2) @ mel_filters + mel_spec = torch.log(torch.clamp(mel_spec, min=float(mel_floor))) + + if normalize_active_frames_only: + attention_mask = ( + torch.arange(mel_spec.shape[1], dtype=torch.long, device=mel_spec.device).unsqueeze(0) + < feature_length + ) + mask = attention_mask.unsqueeze(-1).to(torch.float32) + masked = mel_spec * mask + mean = masked.sum(dim=1) / float(feature_length) + centered = masked - mean.unsqueeze(1) + denom = float(max(1, feature_length - 1)) + variance = (centered.pow(2) * mask).sum(dim=1) / denom + mel_spec = (mel_spec - mean.unsqueeze(1)) / torch.sqrt(variance.unsqueeze(1) + np.float32(1e-5)) + mel_spec = mel_spec * mask + else: + # Native Parakeet normalizes across the full centered-STFT frame count, + # then trims to waveform_samples // hop_length. Keep that order for parity. + frame_count = int(mel_spec.shape[1]) + mean = mel_spec.mean(dim=1) + centered = mel_spec - mean.unsqueeze(1) + denom = float(max(1, frame_count - 1)) + variance = centered.pow(2).sum(dim=1) / denom + mel_spec = centered / torch.sqrt(variance.unsqueeze(1) + np.float32(1e-5)) + return mel_spec[0].cpu().numpy().astype(np.float32), feature_length + + +def prepare_cactus_audio_features( + audio_file: str | Path, + *, + model_type: str, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, + layout: str = "frames_mels", +) -> tuple[torch.Tensor, int]: + from src.cactus import cactus_preprocess_audio_features + + capacity_frames = int(expected_frames) if isinstance(expected_frames, int) and expected_frames > 0 else 0 + if capacity_frames <= 0: + lowered_model_type = model_type.lower() + if "whisper" in lowered_model_type: + capacity_frames = 3000 + else: + sample_rate, waveform = wavfile.read(str(audio_file)) + samples = int(waveform.shape[0]) if hasattr(waveform, "shape") else len(waveform) + if sample_rate <= 0: + sample_rate = _PARAKEET_SAMPLE_RATE + capacity_frames = max( + 1, + int(np.ceil(samples * (_PARAKEET_SAMPLE_RATE / float(sample_rate)) / _PARAKEET_HOP_LENGTH)) + 8, + ) + if "whisper" in model_type.lower(): + capacity_frames = max(capacity_frames, 3000) + capacity = max(1, capacity_frames * int(expected_mels)) + values, mel_bins, frames = cactus_preprocess_audio_features( + str(Path(audio_file).expanduser().resolve()), + model_type, + int(expected_mels), + capacity, + ) + if int(mel_bins) != int(expected_mels): + raise ValueError(f"Cactus audio frontend returned {mel_bins} mel bins, expected {expected_mels}") + feature_array = np.asarray(values, dtype=np.float32).reshape(int(mel_bins), int(frames)) + active_frames = int(frames) + if isinstance(expected_frames, int) and expected_frames > 0: + active_frames = min(active_frames, expected_frames) + if layout == "frames_mels": + feature_array = feature_array[:, :active_frames].T + if isinstance(expected_frames, int) and expected_frames > active_frames: + feature_array = np.pad(feature_array, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + elif layout == "mels_frames": + feature_array = feature_array[:, :active_frames] + if isinstance(expected_frames, int) and expected_frames > active_frames: + feature_array = np.pad(feature_array, ((0, 0), (0, expected_frames - active_frames)), mode="constant") + else: + raise ValueError(f"unsupported Cactus audio feature layout: {layout!r}") + tensor = torch.from_numpy(np.ascontiguousarray(feature_array)).unsqueeze(0).to(dtype=torch_dtype) + return tensor, active_frames + + +def prepare_native_parakeet_audio_features( + audio_file: str | Path, + *, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, +) -> tuple[torch.Tensor, int]: + try: + return prepare_cactus_audio_features( + audio_file, + model_type="parakeet", + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + except Exception: + pass + + waveform = load_audio_waveform(audio_file, target_sample_rate=_PARAKEET_SAMPLE_RATE) + features, feature_length = generic_log_mel_features( + waveform, + sample_rate=_PARAKEET_SAMPLE_RATE, + num_mels=expected_mels or _PARAKEET_NUM_MELS, + n_fft=_PARAKEET_N_FFT, + hop_length=_PARAKEET_HOP_LENGTH, + frame_length=_PARAKEET_FRAME_LENGTH, + preemphasis=_PARAKEET_PREEMPHASIS, + mel_floor=_PARAKEET_LOG_FLOOR, + normalize_active_frames_only=False, + ) + active_frames = feature_length + if isinstance(expected_frames, int) and expected_frames > 0: + active_frames = min(active_frames, expected_frames) + features = features[:active_frames, :] + if features.shape[1] != expected_mels: + raise ValueError( + f"feature mel dimension mismatch: expected {expected_mels}, got {features.shape[1]}" + ) + if isinstance(expected_frames, int) and expected_frames > active_frames: + features = np.pad(features, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + tensor = torch.from_numpy(np.ascontiguousarray(features)).unsqueeze(0).to(dtype=torch_dtype) + return tensor, active_frames + + +def prepare_native_gemma4_audio_features( + audio_file: str | Path, + *, + expected_mels: int, + torch_dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor, int]: + waveform = load_audio_waveform(audio_file, target_sample_rate=_PARAKEET_SAMPLE_RATE) + # Native Gemma4 pads to the nearest 320 samples, adds 160 samples of + # semicausal left padding, and then runs a 321-point analysis frame. + estimated_frames = int(np.ceil((len(waveform) + 640) / 160.0)) + 8 + capacity = max(1, estimated_frames * int(expected_mels)) + values, mel_bins, frames = prepare_cactus_audio_features_raw( + audio_file, + model_type="gemma4", + mel_bins=int(expected_mels), + capacity=capacity, + ) + if int(mel_bins) != int(expected_mels): + raise ValueError(f"Cactus Gemma4 audio frontend returned {mel_bins} mel bins, expected {expected_mels}") + features = np.asarray(values, dtype=np.float32).reshape(int(frames), int(mel_bins)) + feature_tensor = torch.from_numpy(np.ascontiguousarray(features)).unsqueeze(0).to(dtype=torch_dtype) + mask = torch.ones((1, int(frames)), dtype=torch.bool) + return feature_tensor, mask, int(frames) + + +def prepare_cactus_audio_features_raw( + audio_file: str | Path, + *, + model_type: str, + mel_bins: int, + capacity: int, +) -> tuple[list[float], int, int]: + from src.cactus import cactus_preprocess_audio_features + + return cactus_preprocess_audio_features( + str(Path(audio_file).expanduser().resolve()), + model_type, + int(mel_bins), + int(capacity), + ) diff --git a/python/src/transpile/canonicalize/cleanup.py b/python/src/transpile/canonicalize/cleanup.py new file mode 100644 index 000000000..fe801cb84 --- /dev/null +++ b/python/src/transpile/canonicalize/cleanup.py @@ -0,0 +1,657 @@ +from __future__ import annotations + +from collections import Counter + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.ops import canonicalize_op +from src.transpile.ops import get_op +from src.transpile.ops import has_op +from src.transpile.canonicalize.utils import bypass_unary_node +from src.transpile.canonicalize.utils import dce +from src.transpile.canonicalize.utils import ir_dtype_to_torch +from src.transpile.canonicalize.utils import materialize_constant_output +from src.transpile.canonicalize.utils import normalize_dim +from src.transpile.canonicalize.utils import normalize_dtype_name +from src.transpile.canonicalize.utils import numel +from src.transpile.canonicalize.utils import rebuild_graph + + +COMPILER_SUPPORTED_OPS = { + "arange", + "ones", +} + +CANONICAL_UNSUPPORTED_RENAMES = { + "aten.__and__.Tensor": "logical_and", + "aten.__or__.Tensor": "logical_or", + "aten.cumsum.default": "cumsum", + "aten.eq.Scalar": "equal", + "aten.eq.Tensor": "equal", + "aten.gt.Tensor": "greater", + "aten.index.Tensor": "advanced_index", + "aten.le.Tensor": "less_equal", +} + +FP32_SUPPORTED_ALL_INPUT_OPS = { + "add", + "subtract", + "multiply", + "divide", + "precision_cast", + "view", + "permute", + "transpose", + "slice", + "index", + "advanced_index", + "gather", + "matmul", + "linear", + "addmm", + "layer_norm", + "batch_norm", + "glu", + "sum", + "mean", + "variance", + "min", + "max", + "scalar_add", + "scalar_subtract", + "scalar_subtract_reverse", + "scalar_multiply", + "scalar_divide", + "scalar_floor_divide", + "scalar_not_equal", + "scalar_equal", + "abs", + "negate", + "pow", + "scalar_exp", + "scalar_sqrt", + "scalar_log", + "scalar_cos", + "scalar_sin", +} + +FP32_SUPPORTED_INPUT_INDICES = { + "embedding": {1}, +} + +FP16_ONLY_OUTPUT_OPS = { + "scalar_divide_reverse", + "not_equal", + "equal", + "greater", + "less_equal", + "logical_and", + "logical_or", + "logical_not", + "one_hot", + "scalar_not_equal", + "scalar_equal", + "cat", + "softmax", + "rms_norm", + "embedding", + "group_norm", + "lstm_cell", + "relu", + "silu", + "gelu", + "gelu_erf", + "sigmoid", + "tanh", + "attention", + "attention_block", + "conv_module", + "rope", +} + + +def canonicalize_exported_graph(graph: IRGraph, *, max_passes: int = 8) -> IRGraph: + verify_ir(graph) + + for _ in range(max_passes): + changed = False + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if _canonicalize_node(graph, node): + changed = True + + rebuild_graph(graph) + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if _simplify_node(graph, node): + changed = True + + rebuild_graph(graph) + + if _legalize_precisions(graph): + changed = True + rebuild_graph(graph) + + dce(graph) + + if not changed: + break + + unsupported_counts = summarize_unsupported_ops(graph) + graph.meta = dict(graph.meta) + graph.meta["canonical_unsupported_ops"] = tuple(sorted(unsupported_counts)) + graph.meta["canonical_unsupported_op_counts"] = unsupported_counts + verify_ir(graph) + return graph + + +def summarize_unsupported_ops(graph: IRGraph) -> dict[str, int]: + counts: Counter[str] = Counter() + for node_id in graph.order: + node = graph.nodes[node_id] + if not _is_supported_without_backend_kernel(node.op): + counts[node.op] += 1 + return dict(sorted(counts.items())) + + +def _canonicalize_node(graph: IRGraph, node: IRNode) -> bool: + original_op = node.op + + if original_op in {"aten.new_ones.default", "aten.ones.default"}: + return _materialize_new_ones_constant(graph, node) + + if original_op in {"aten.new_zeros.default", "aten.zeros.default", "new_zeros"}: + return _materialize_new_zeros_constant(graph, node) + + if original_op == "arange": + return _materialize_arange_constant(graph, node) + + if original_op in {"identity", "contiguous"}: + return bypass_unary_node(graph, node) + + if original_op in CANONICAL_UNSUPPORTED_RENAMES: + node.op = CANONICAL_UNSUPPORTED_RENAMES[original_op] + return True + + if original_op == "multiply_inplace": + node.op = "multiply" + return True + + if original_op == "view" and "shape" not in node.attrs: + return _rewrite_view_from_output_shape(graph, node) + + if original_op in {"reshape", "flatten", "unsqueeze", "squeeze"}: + return _rewrite_view_from_output_shape(graph, node) + + if original_op == "expand": + return _rewrite_expand_if_view_equivalent(graph, node) + + if original_op == "transpose": + return _rewrite_transpose_as_permute(graph, node) + + if original_op == "movedim": + return _rewrite_movedim_as_permute(graph, node) + + if original_op == "type_as": + return _rewrite_type_as(graph, node) + + if original_op == "concat": + node.op = "cat" + if "dim" in node.attrs and "axis" not in node.attrs: + node.attrs = {**node.attrs, "axis": int(node.attrs["dim"])} + return True + + if original_op == "scaled_dot_product_attention": + node.op = "attention" + node.attrs = { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", False)), + "window_size": int(node.attrs.get("window_size", 0)), + **({"mask": node.attrs["mask"]} if "mask" in node.attrs else {}), + **({"additive_mask": bool(node.attrs.get("additive_mask", False))} if "additive_mask" in node.attrs else {}), + } + return True + + canonical_op = canonicalize_op(original_op) + if canonical_op != original_op: + node.op = canonical_op + _normalize_attr_spellings(node) + if has_op(node.op): + _filter_attrs_to_schema(node) + return True + + if has_op(node.op): + _normalize_attr_spellings(node) + _filter_attrs_to_schema(node) + return True + + return False + + +def _simplify_node(graph: IRGraph, node: IRNode) -> bool: + if _fold_constant_node(graph, node): + return True + + if node.op == "precision_cast": + return _remove_noop_precision_cast(graph, node) + + if node.op == "view": + if _remove_noop_view(graph, node): + return True + return _collapse_view_chain(graph, node) + + if node.op == "slice": + return _remove_noop_slice(graph, node) + + if node.op == "scalar_add": + return _remove_noop_scalar(graph, node, neutral=0.0) + + if node.op == "scalar_subtract": + return _remove_noop_scalar(graph, node, neutral=0.0) + + if node.op == "scalar_multiply": + return _remove_noop_scalar(graph, node, neutral=1.0) + + if node.op == "scalar_divide": + return _remove_noop_scalar(graph, node, neutral=1.0) + + return False + + +def _fold_constant_node(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + + output_value = graph.values.get(node.outputs[0]) + if output_value is None: + return False + + if node.op in FP16_ONLY_OUTPUT_OPS: + output_value.dtype = "fp16" + node.meta["dtype"] = "fp16" + + output_torch_dtype = ir_dtype_to_torch(output_value.dtype) if output_value.dtype is not None else None + + constant_inputs = [graph.constants.get(value_id) for value_id in node.inputs] + if node.op == "view" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + if output_value.shape is None: + return False + return materialize_constant_output( + graph, + node, + constant_inputs[0].reshape(tuple(int(dim) for dim in output_value.shape)).to(dtype=output_torch_dtype), + ) + + if node.op == "expand" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + if output_value.shape is None: + return False + return materialize_constant_output( + graph, + node, + constant_inputs[0].expand(tuple(int(dim) for dim in output_value.shape)).clone().to(dtype=output_torch_dtype), + ) + + if node.op == "precision_cast" and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + dtype = normalize_dtype_name(node.attrs.get("dtype")) + if dtype is None: + return False + return materialize_constant_output(graph, node, constant_inputs[0].to(dtype=ir_dtype_to_torch(dtype))) + + if node.op in {"cos", "scalar_cos"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + return materialize_constant_output(graph, node, torch.cos(constant_inputs[0])) + + if node.op in {"sin", "scalar_sin"} and len(constant_inputs) == 1 and isinstance(constant_inputs[0], torch.Tensor): + return materialize_constant_output(graph, node, torch.sin(constant_inputs[0])) + + if node.op in {"scalar_add", "scalar_subtract", "scalar_multiply", "scalar_divide"}: + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + scalar = float(node.attrs["value"]) + if node.op == "scalar_add": + result = constant_inputs[0] + scalar + elif node.op == "scalar_subtract": + result = constant_inputs[0] - scalar + elif node.op == "scalar_multiply": + result = constant_inputs[0] * scalar + else: + result = constant_inputs[0] / scalar + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op in {"scalar_not_equal", "scalar_equal"}: + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + scalar = float(node.attrs["value"]) + if node.op == "scalar_not_equal": + result = constant_inputs[0] != scalar + else: + result = constant_inputs[0] == scalar + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op in {"not_equal", "equal", "greater", "less_equal", "logical_and", "logical_or"}: + if len(constant_inputs) != 2 or not all(isinstance(value, torch.Tensor) for value in constant_inputs): + return False + lhs, rhs = constant_inputs + if node.op == "not_equal": + result = lhs != rhs + elif node.op == "equal": + result = lhs == rhs + elif node.op == "greater": + result = lhs > rhs + elif node.op == "less_equal": + result = lhs <= rhs + elif node.op == "logical_and": + result = torch.logical_and(lhs != 0, rhs != 0) + else: + result = torch.logical_or(lhs != 0, rhs != 0) + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + if node.op == "logical_not": + if len(constant_inputs) != 1 or not isinstance(constant_inputs[0], torch.Tensor): + return False + result = torch.logical_not(constant_inputs[0] != 0) + return materialize_constant_output(graph, node, result.to(dtype=output_torch_dtype)) + + return False + + +def _legalize_precisions(graph: IRGraph) -> bool: + changed = False + counter = 0 + + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + if node.op in FP32_SUPPORTED_ALL_INPUT_OPS: + continue + + inserted: list[IRNode] = [] + allowed_indices = FP32_SUPPORTED_INPUT_INDICES.get(node.op, set()) + for input_index, value_id in enumerate(list(node.inputs)): + if input_index in allowed_indices: + continue + value = graph.values.get(value_id) + if value is None or normalize_dtype_name(value.dtype) != "fp32": + continue + + counter += 1 + cast_node_id = _unique_ir_id( + f"{node.id}_legalize_fp16_{counter}", + graph.nodes.keys(), + graph.order, + ) + cast_value_id = _unique_ir_id( + f"{value_id}_legalize_fp16_{counter}", + graph.values.keys(), + graph.constants.keys(), + ) + cast_node = IRNode( + id=cast_node_id, + op="precision_cast", + inputs=[value_id], + outputs=[cast_value_id], + attrs={"dtype": "fp16"}, + meta={"inserted_by": "canonicalize_exported_graph"}, + ) + graph.values[cast_value_id] = IRValue( + id=cast_value_id, + shape=value.shape, + dtype="fp16", + producer=cast_node_id, + users=[], + ) + node.inputs[input_index] = cast_value_id + inserted.append(cast_node) + changed = True + + if inserted: + insert_at = graph.order.index(node_id) + for offset, cast_node in enumerate(inserted): + graph.nodes[cast_node.id] = cast_node + graph.order.insert(insert_at + offset, cast_node.id) + + if node.op in FP16_ONLY_OUTPUT_OPS: + for output_id in node.outputs: + output_value = graph.values.get(output_id) + if output_value is not None: + output_value.dtype = "fp16" + node.meta["dtype"] = "fp16" + + return changed + + +def _unique_ir_id(base: str, *existing_groups) -> str: + existing: set[str] = set() + for group in existing_groups: + existing.update(str(item) for item in group) + if base not in existing: + return base + suffix = 1 + while True: + candidate = f"{base}_{suffix}" + if candidate not in existing: + return candidate + suffix += 1 + + +def _rewrite_view_from_output_shape(graph: IRGraph, node: IRNode) -> bool: + if not node.outputs: + return False + output_value = graph.values.get(node.outputs[0]) + if output_value is None or output_value.shape is None: + return False + node.op = "view" + node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} + return True + + +def _rewrite_expand_if_view_equivalent(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs or not node.outputs: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if numel(input_value.shape) != numel(output_value.shape): + node.meta["canonicalization_blocked"] = "expand_requires_broadcast" + return False + node.op = "view" + node.attrs = {"shape": tuple(int(dim) for dim in output_value.shape)} + return True + + +def _rewrite_transpose_as_permute(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.shape is None: + return False + rank = len(input_value.shape) + dim0 = normalize_dim(int(node.attrs["dim0"]), rank) + dim1 = normalize_dim(int(node.attrs["dim1"]), rank) + permutation = list(range(rank)) + permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] + node.op = "permute" + node.attrs = {"permutation": tuple(permutation)} + return True + + +def _rewrite_movedim_as_permute(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.shape is None: + return False + rank = len(input_value.shape) + source = [normalize_dim(int(dim), rank) for dim in node.attrs["source"]] + destination = [normalize_dim(int(dim), rank) for dim in node.attrs["destination"]] + if len(source) != len(destination): + return False + + remaining = [dim for dim in range(rank) if dim not in source] + permutation = list(remaining) + for dst, src in sorted(zip(destination, source)): + permutation.insert(dst, src) + + node.op = "permute" + node.attrs = {"permutation": tuple(permutation)} + return True + + +def _rewrite_type_as(graph: IRGraph, node: IRNode) -> bool: + if not node.inputs: + return False + target_dtype = None + if node.outputs: + output_value = graph.values.get(node.outputs[0]) + if output_value is not None: + target_dtype = output_value.dtype + if target_dtype is None and len(node.inputs) > 1: + target_value = graph.values.get(node.inputs[1]) + if target_value is not None: + target_dtype = target_value.dtype + if target_dtype is None: + return False + node.op = "precision_cast" + node.inputs = [node.inputs[0]] + node.attrs = {"dtype": normalize_dtype_name(target_dtype)} + return True + + +def _normalize_attr_spellings(node: IRNode) -> None: + attrs = dict(node.attrs) + if "dim" in attrs and "axis" not in attrs and node.op in {"softmax", "sum", "mean", "variance", "min", "max", "cat", "slice", "index"}: + attrs["axis"] = attrs.pop("dim") + if node.op == "permute" and "dims" in attrs and "permutation" not in attrs: + attrs["permutation"] = tuple(int(dim) for dim in attrs.pop("dims")) + if node.op == "slice": + attrs.setdefault("step", 1) + if node.op == "precision_cast" and "dtype" in attrs: + attrs["dtype"] = normalize_dtype_name(attrs["dtype"]) + node.attrs = attrs + + +def _filter_attrs_to_schema(node: IRNode) -> None: + schema = get_op(node.op) + if not schema.attrs: + node.attrs = {} + return + node.attrs = {name: node.attrs[name] for name in schema.attrs if name in node.attrs} + + +def _remove_noop_precision_cast(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + target_dtype = normalize_dtype_name(node.attrs.get("dtype")) + input_dtype = None if input_value is None else normalize_dtype_name(input_value.dtype) + output_dtype = None if output_value is None else normalize_dtype_name(output_value.dtype) + if target_dtype is None or target_dtype == input_dtype or (input_dtype is not None and output_dtype == input_dtype): + return bypass_unary_node(graph, node) + return False + + +def _remove_noop_view(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if tuple(input_value.shape) == tuple(output_value.shape): + return bypass_unary_node(graph, node) + return False + + +def _collapse_view_chain(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + if input_value is None or input_value.producer is None: + return False + producer = graph.nodes.get(input_value.producer) + if producer is None or producer.op != "view" or len(producer.inputs) != 1: + return False + node.inputs = [producer.inputs[0]] + return True + + +def _remove_noop_slice(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None: + return False + if input_value.shape is None or output_value.shape is None: + return False + if tuple(input_value.shape) == tuple(output_value.shape): + return bypass_unary_node(graph, node) + return False + + +def _remove_noop_scalar(graph: IRGraph, node: IRNode, *, neutral: float) -> bool: + value = node.attrs.get("value") + if value is None: + return False + if float(value) != float(neutral): + return False + return bypass_unary_node(graph, node) + + +def _materialize_new_ones_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None or output_value.dtype is None: + return False + torch_dtype = ir_dtype_to_torch(output_value.dtype) + const = torch.ones(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _materialize_new_zeros_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None or output_value.dtype is None: + return False + torch_dtype = ir_dtype_to_torch(output_value.dtype) + const = torch.zeros(tuple(int(dim) for dim in output_value.shape), dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _materialize_arange_constant(graph: IRGraph, node: IRNode) -> bool: + if len(node.outputs) != 1: + return False + start = int(node.attrs.get("start", 0)) + end = node.attrs.get("end") + if end is None: + return False + step = int(node.attrs.get("step", 1)) + dtype = normalize_dtype_name(node.attrs.get("dtype", "fp32")) + torch_dtype = ir_dtype_to_torch(dtype) + const = torch.arange(start, int(end), step=step, dtype=torch_dtype) + return materialize_constant_output(graph, node, const) + + +def _is_supported_without_backend_kernel(op_name: str) -> bool: + return has_op(op_name) or op_name in COMPILER_SUPPORTED_OPS diff --git a/python/src/transpile/canonicalize/utils.py b/python/src/transpile/canonicalize/utils.py new file mode 100644 index 000000000..aa2bb7297 --- /dev/null +++ b/python/src/transpile/canonicalize/utils.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +from typing import Any + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir + + +def normalize_dtype_name(dtype: Any) -> str | None: + if dtype is None: + return None + if not isinstance(dtype, str): + return str(dtype) + if dtype.startswith("torch."): + torch_name = dtype.split(".", 1)[1] + mapping = { + "bool": "bool", + "int8": "int8", + "int16": "int16", + "int32": "int32", + "int64": "int64", + "float16": "fp16", + "bfloat16": "bf16", + "float32": "fp32", + "float64": "fp64", + } + return mapping.get(torch_name, dtype) + return dtype + + +def ir_dtype_to_torch(dtype: str) -> torch.dtype: + normalized = normalize_dtype_name(dtype) + if normalized == "bool": + return torch.bool + if normalized == "int8": + return torch.int8 + if normalized == "int16": + return torch.int16 + if normalized == "int32": + return torch.int32 + if normalized == "int64": + return torch.int64 + if normalized == "fp16": + return torch.float16 + if normalized == "bf16": + return torch.bfloat16 + if normalized == "fp32": + return torch.float32 + if normalized == "fp64": + return torch.float64 + raise NotImplementedError(f"unsupported IR dtype for constant materialization: {dtype}") + + +def normalize_dim(dim: int, rank: int) -> int: + if dim < 0: + dim += rank + return dim + + +def numel(shape: tuple[int, ...]) -> int: + total = 1 + for dim in shape: + total *= int(dim) + return total + + +def refresh_value_users(graph: IRGraph) -> None: + for value in graph.values.values(): + value.users = [] + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + value = graph.values.get(input_id) + if value is not None: + value.users.append(node_id) + + +def rebuild_graph(graph: IRGraph) -> None: + new_values: dict[str, IRValue] = {} + + for value_id in graph.inputs: + old = graph.values.get(value_id, IRValue(id=value_id)) + new_values[value_id] = IRValue( + id=value_id, + shape=old.shape, + dtype=old.dtype, + producer=None, + users=[], + meta=dict(old.meta), + ) + + for value_id in graph.constants.keys(): + old = graph.values.get(value_id, IRValue(id=value_id)) + new_values[value_id] = IRValue( + id=value_id, + shape=old.shape, + dtype=old.dtype, + producer=None, + users=[], + meta=dict(old.meta), + ) + + for node_id in graph.order: + node = graph.nodes[node_id] + for output_id in node.outputs: + old = graph.values.get(output_id, IRValue(id=output_id)) + new_values[output_id] = IRValue( + id=output_id, + shape=old.shape, + dtype=old.dtype, + producer=node_id, + users=[], + meta=dict(old.meta), + ) + + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + if input_id not in new_values: + old = graph.values.get(input_id, IRValue(id=input_id)) + new_values[input_id] = IRValue( + id=input_id, + shape=old.shape, + dtype=old.dtype, + producer=old.producer, + users=[], + meta=dict(old.meta), + ) + new_values[input_id].users.append(node_id) + + for output_id in graph.outputs: + if output_id not in new_values: + old = graph.values.get(output_id, IRValue(id=output_id)) + new_values[output_id] = IRValue( + id=output_id, + shape=old.shape, + dtype=old.dtype, + producer=old.producer, + users=[], + meta=dict(old.meta), + ) + + graph.values = new_values + verify_ir(graph) + + +def dce(graph: IRGraph) -> None: + live_values = set(graph.outputs) + live_nodes: set[str] = set() + + changed = True + while changed: + changed = False + for node_id in reversed(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + if not any(output in live_values for output in node.outputs): + continue + if node_id not in live_nodes: + live_nodes.add(node_id) + changed = True + for input_id in node.inputs: + if input_id not in live_values: + live_values.add(input_id) + changed = True + + graph.order = [node_id for node_id in graph.order if node_id in live_nodes] + graph.nodes = {node_id: node for node_id, node in graph.nodes.items() if node_id in live_nodes} + + live_value_ids = set(graph.inputs) | set(graph.outputs) + for node in graph.nodes.values(): + live_value_ids.update(node.inputs) + live_value_ids.update(node.outputs) + + graph.values = {value_id: value for value_id, value in graph.values.items() if value_id in live_value_ids} + graph.constants = {value_id: value for value_id, value in graph.constants.items() if value_id in live_value_ids} + rebuild_graph(graph) + + +def remove_node(graph: IRGraph, node_id: str) -> None: + if node_id in graph.nodes: + del graph.nodes[node_id] + graph.order = [existing_id for existing_id in graph.order if existing_id != node_id] + + +def drop_value_if_unused(graph: IRGraph, value_id: str) -> None: + if value_id in graph.inputs or value_id in graph.outputs or value_id in graph.constants: + return + value = graph.values.get(value_id) + if value is None: + return + if value.producer is None and not value.users: + del graph.values[value_id] + + +def bypass_unary_node(graph: IRGraph, node: IRNode) -> bool: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return False + input_id = node.inputs[0] + output_id = node.outputs[0] + + for other in graph.nodes.values(): + other.inputs = [input_id if value_id == output_id else value_id for value_id in other.inputs] + graph.outputs = [input_id if value_id == output_id else value_id for value_id in graph.outputs] + + remove_node(graph, node.id) + if output_id in graph.values: + del graph.values[output_id] + drop_value_if_unused(graph, input_id) + return True + + +def _is_gemma4_graph(graph: IRGraph) -> bool: + return str(graph.meta.get("adapter_family") or graph.meta.get("family") or "").lower() == "gemma4" + + +def _weight_binding_meta(value: IRValue | None) -> dict[str, object] | None: + if value is None: + return None + path = value.meta.get("path") + kind = value.meta.get("kind") + source_name = value.meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + return dict(value.meta) + return None + + +def _gemma4_materialized_binding_meta(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + if not _is_gemma4_graph(graph) or len(node.inputs) != 1: + return None + if node.op == "precision_cast": + pass + elif node.op == "scalar_add" and float(node.attrs.get("value", 0.0)) in (0.0, 1.0): + pass + else: + return None + + value_id = node.inputs[0] + for _ in range(4): + value = graph.values.get(value_id) + binding_meta = _weight_binding_meta(value) + if binding_meta is not None: + binding_meta["materialized_from_value_id"] = value_id + binding_meta["materialized_by_op"] = node.op + return binding_meta + producer_id = value.producer if value is not None else None + producer_node = graph.nodes.get(producer_id) if producer_id is not None else None + if producer_node is None or producer_node.op not in {"precision_cast", "type_as"}: + break + if len(producer_node.inputs) != 1: + break + value_id = producer_node.inputs[0] + return None + + +def materialize_constant_output(graph: IRGraph, node: IRNode, value: Any) -> bool: + if len(node.outputs) != 1: + return False + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None: + return False + binding_meta = _gemma4_materialized_binding_meta(graph, node) + if binding_meta is not None: + output_value.meta.update(binding_meta) + graph.constants[output_id] = value + output_value.producer = None + remove_node(graph, node.id) + return True diff --git a/python/src/transpile/capture_pytorch.py b/python/src/transpile/capture_pytorch.py new file mode 100644 index 000000000..e21f1c712 --- /dev/null +++ b/python/src/transpile/capture_pytorch.py @@ -0,0 +1,344 @@ +from dataclasses import dataclass +import inspect +from typing import Any +from typing import Iterable + +import torch +from torch.export import export +from torch.fx.passes.shape_prop import ShapeProp + +from src.transpile.graph_ir import IRGraph + + +class CapturePhaseError(RuntimeError): + def __init__(self, phase: str, message: str, *, cause: Exception | None = None): + super().__init__(f"[capture:{phase}] {message}") + self.phase = phase + self.cause = cause + + +@dataclass +class TensorMetadata: + shape: tuple[Any, ...] + dtype: Any + stride: tuple[Any, ...] | None = None + device: Any | None = None + + +@dataclass +class CapturedModel: + exported_program: Any + graph_module: Any + graph: Any + state_dict: dict[str, Any] + ir_graph: IRGraph + source_module: Any + example_args: tuple[Any, ...] + example_kwargs: dict[str, Any] + strict: bool + transpile_metadata: dict[str, Any] + + def named_parameters(self): + return self.exported_program.named_parameters() + + def named_buffers(self): + return self.exported_program.named_buffers() + + def parameters_dict(self) -> dict[str, Any]: + return dict(self.named_parameters()) + + def buffers_dict(self) -> dict[str, Any]: + return dict(self.named_buffers()) + + def placeholders(self) -> list[Any]: + return [node for node in self.graph.nodes if node.op == "placeholder"] + + def outputs(self) -> list[Any]: + return [node for node in self.graph.nodes if node.op == "output"] + + +def _normalize_args(args: Any) -> tuple[Any, ...]: + if args is None: + return () + if isinstance(args, tuple): + return args + if isinstance(args, list): + return tuple(args) + return (args,) + + +def _normalize_kwargs(kwargs: dict[str, Any] | None) -> dict[str, Any]: + return {} if kwargs is None else dict(kwargs) + + +def _clone_example_value(value: Any) -> Any: + if isinstance(value, torch.Tensor): + return value.detach().clone() + if isinstance(value, tuple): + return tuple(_clone_example_value(v) for v in value) + if isinstance(value, list): + return [_clone_example_value(v) for v in value] + if isinstance(value, dict): + return {k: _clone_example_value(v) for k, v in value.items()} + return value + + +def _clone_examples(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[tuple[Any, ...], dict[str, Any]]: + return ( + tuple(_clone_example_value(arg) for arg in args), + {k: _clone_example_value(v) for k, v in kwargs.items()}, + ) + + +def _inject_export_safe_kwargs(model: torch.nn.Module, kwargs: dict[str, Any]) -> dict[str, Any]: + try: + signature = inspect.signature(model.forward) + except (TypeError, ValueError): + return kwargs + + updated = dict(kwargs) + parameters = signature.parameters + + # HF causal LM forwards often default to returning cache objects and ModelOutput + # containers, which torch.export cannot serialize cleanly. Keep the capture + # boundary tensor-only unless the caller explicitly asked otherwise. + if "use_cache" in parameters and "use_cache" not in updated: + updated["use_cache"] = False + if "return_dict" in parameters and "return_dict" not in updated: + updated["return_dict"] = False + return updated + + +def _call_transpile_metadata_provider(module: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: + for attr_name in ("get_transpile_metadata", "transpile_metadata"): + provider = getattr(module, attr_name, None) + if not callable(provider): + continue + try: + metadata = provider(args=args, kwargs=kwargs) + except TypeError: + metadata = provider() + if metadata is None: + return {} + if not isinstance(metadata, dict): + raise TypeError(f"{type(module).__name__}.{attr_name} must return a dict or None") + return metadata + return {} + + +def _collect_transpile_metadata(model: torch.nn.Module, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: + graph_meta: dict[str, Any] = {} + provider_graph_meta: dict[str, dict[str, Any]] = {} + import_hints: list[dict[str, Any]] = [] + seen_modules: set[int] = set() + + for module_path, module in model.named_modules(): + module_id = id(module) + if module_id in seen_modules: + continue + seen_modules.add(module_id) + + metadata = _call_transpile_metadata_provider(module, args, kwargs) + if not metadata: + continue + + provider_key = module_path or type(module).__name__ + provider_meta = metadata.get("graph", {}) + if isinstance(provider_meta, dict) and provider_meta: + provider_graph_meta[provider_key] = dict(provider_meta) + if not module_path: + graph_meta.update(provider_meta) + + raw_hints = metadata.get("import_hints", ()) + for raw_hint in raw_hints: + if not isinstance(raw_hint, dict): + raise TypeError(f"{type(module).__name__} import hints must be dictionaries") + hint = dict(raw_hint) + hint.setdefault("provider", provider_key) + import_hints.append(hint) + + if provider_graph_meta: + graph_meta.setdefault("transpile_metadata_providers", provider_graph_meta) + if import_hints: + graph_meta["import_hint_count"] = len(import_hints) + + if not graph_meta and not import_hints: + return {} + return { + "graph": graph_meta, + "import_hints": import_hints, + } + + +def _prepare_import_graph_module(ep: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + graph_module = ep.graph_module + if kwargs: + return graph_module + + try: + graph_module = ep.module() + ShapeProp(graph_module).propagate(*args) + except Exception: + return ep.graph_module + return graph_module + +def capture_model(model, args, kwargs=None, *, strict=True) -> CapturedModel: + if not isinstance(model, torch.nn.Module): + raise TypeError("model must be a torch.nn.Module") + + normalized_args = _normalize_args(args) + normalized_kwargs = _inject_export_safe_kwargs(model, _normalize_kwargs(kwargs)) + + if not normalized_args and not normalized_kwargs: + raise ValueError("capture_model requires example args or kwargs") + + model = model.eval() + example_args, example_kwargs = _clone_examples(normalized_args, normalized_kwargs) + transpile_metadata = _collect_transpile_metadata(model, example_args, example_kwargs) + + try: + ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) + except Exception as exc: + raise CapturePhaseError( + "export", + f"torch.export failed for model={type(model).__name__} strict={strict}: {exc}", + cause=exc, + ) from exc + from src.transpile.import_ir import import_captured_to_ir + + import_graph_module = _prepare_import_graph_module(ep, example_args, example_kwargs) + + raw_captured = CapturedModel( + exported_program=ep, + graph_module=import_graph_module, + graph=import_graph_module.graph, + state_dict=dict(ep.state_dict), + ir_graph=IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[]), + source_module=model, + example_args=example_args, + example_kwargs=example_kwargs, + strict=strict, + transpile_metadata=transpile_metadata, + ) + try: + ir_graph = import_captured_to_ir(raw_captured, strict=strict) + except Exception as exc: + raise CapturePhaseError( + "import", + f"failed to import exported graph to IR for model={type(model).__name__} strict={strict}: {exc}", + cause=exc, + ) from exc + + return CapturedModel( + exported_program=ep, + graph_module=ep.graph_module, + graph=ep.graph, + state_dict=dict(ep.state_dict), + ir_graph=ir_graph, + source_module=model, + example_args=example_args, + example_kwargs=example_kwargs, + strict=strict, + transpile_metadata=transpile_metadata, + ) + + +def capture_model_with_fallback(model, args, kwargs=None) -> CapturedModel: + try: + return capture_model(model, args, kwargs=kwargs, strict=True) + except Exception: + return capture_model(model, args, kwargs=kwargs, strict=False) + + +def resolve_attr(root: Any, target: str) -> Any: + obj = root + for atom in target.split("."): + obj = getattr(obj, atom) + return obj + + +def format_target(node: Any) -> str: + return str(node.target) + + +def _try_materialize_int_tuple(values: Iterable[Any]) -> tuple[int, ...] | None: + materialized: list[int] = [] + try: + for value in values: + materialized.append(int(value)) + except Exception: + return None + return tuple(materialized) + + +def get_tensor_metadata(node: Any) -> TensorMetadata | None: + meta = getattr(node, "meta", {}) or {} + tensor_meta = meta.get("tensor_meta") + if tensor_meta is not None: + shape = tuple(getattr(tensor_meta, "shape", ())) + dtype = getattr(tensor_meta, "dtype", None) + stride_value = getattr(tensor_meta, "stride", None) + stride = _try_materialize_int_tuple(stride_value) if isinstance(stride_value, Iterable) else None + device = getattr(tensor_meta, "device", None) + if shape or dtype is not None or stride is not None or device is not None: + return TensorMetadata( + shape=shape, + dtype=dtype, + stride=stride, + device=device, + ) + + value = meta.get("val") + if value is None: + return None + + if isinstance(value, torch.Tensor): + stride = _try_materialize_int_tuple(value.stride()) if value.layout == torch.strided else None + return TensorMetadata( + shape=tuple(value.shape), + dtype=value.dtype, + stride=stride, + device=value.device, + ) + + shape = tuple(getattr(value, "shape", ())) + dtype = getattr(value, "dtype", None) + stride_value = getattr(value, "stride", None) + stride = tuple(stride_value) if isinstance(stride_value, Iterable) else None + device = getattr(value, "device", None) + if not shape and dtype is None and stride is None and device is None: + return None + return TensorMetadata(shape=shape, dtype=dtype, stride=stride, device=device) + + +def get_shape(node: Any) -> tuple[Any, ...] | None: + metadata = get_tensor_metadata(node) + return None if metadata is None else metadata.shape + + +def get_dtype(node: Any) -> Any | None: + metadata = get_tensor_metadata(node) + return None if metadata is None else metadata.dtype + + +def dump_graph(captured: CapturedModel, *, include_meta: bool = True) -> str: + lines: list[str] = [] + for index, node in enumerate(captured.graph.nodes): + lines.append(f"[{index}] {node.op} {node.name} target={node.target}") + lines.append(f" args={node.args}") + lines.append(f" kwargs={node.kwargs}") + if include_meta: + tensor_meta = get_tensor_metadata(node) + if tensor_meta is not None: + lines.append( + " tensor_meta=" + f"shape={tensor_meta.shape} dtype={tensor_meta.dtype} " + f"stride={tensor_meta.stride} device={tensor_meta.device}" + ) + else: + lines.append(f" meta={getattr(node, 'meta', {})}") + return "\n".join(lines) + + +def print_graph(captured: CapturedModel, *, include_meta: bool = True) -> None: + print(dump_graph(captured, include_meta=include_meta)) diff --git a/python/src/transpile/component_bundle_runtime.py b/python/src/transpile/component_bundle_runtime.py new file mode 100644 index 000000000..901b9eeb9 --- /dev/null +++ b/python/src/transpile/component_bundle_runtime.py @@ -0,0 +1,835 @@ +from __future__ import annotations + +from dataclasses import dataclass +import ctypes +import json +from collections.abc import Mapping +from pathlib import Path +import re +import struct +import time +from typing import Any + +import numpy as np +import torch + +from src.cactus import _lib +from src.cactus import cactus_node_t +from src.graph import Graph +from src.graph import Tensor +from src.tensor_io import CACTUS_MAGIC +from src.tensor_io import FLAG_INTERLEAVED +from src.tensor_io import align_offset +from src.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features +from src.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform +from src.transpile.audio_preprocess import prepare_cactus_audio_features +from src.transpile.parakeet_tdt_local import greedy_decode_parakeet_tdt_token_ids +from src.transpile.parakeet_tdt_local import load_parakeet_tdt_config +from src.transpile.parakeet_tdt_local import prepare_parakeet_tdt_audio_features + + +_HEADER_SIZE = 84 +_PRECISION_TO_DTYPE = { + Graph.INT8: np.int8, + Graph.FP16: np.float16, + Graph.FP32: np.float32, + Graph.INT4: np.uint8, +} + + +@dataclass +class LoadedTensorFile: + path: Path + precision: int + shape: tuple[int, ...] + data: np.memmap + scales: np.memmap | None + group_size: int + num_groups: int + is_interleaved: bool + original_n: int + + +@dataclass +class LoadedComponentGraph: + component: str + graph: Graph + runtime_inputs: list[Tensor] + outputs: list[Tensor] + bound_constant_bindings: list[dict[str, object]] + bound_tensor_files: list[object] + + def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: + if index < 0 or index >= len(self.runtime_inputs): + raise IndexError( + f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" + ) + self.graph.set_input(self.runtime_inputs[index], data, dtype=dtype) + + def set_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> None: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" + ) + for index, value in enumerate(inputs): + self.set_input(index, value) + + def execute(self) -> list[Tensor]: + self.graph.execute() + return self.outputs + + +def load_component_bundle_manifest(bundle_dir_or_manifest: str | Path) -> tuple[Path, dict[str, object]]: + path = Path(bundle_dir_or_manifest).expanduser().resolve() + if path.is_dir(): + candidate = path / "manifest.json" if path.name == "components" else path / "components" / "manifest.json" + if not candidate.exists(): + candidate = path / "manifest.json" + path = candidate + if not path.exists(): + raise FileNotFoundError(f"component bundle manifest not found: {path}") + manifest = json.loads(path.read_text()) + bundle_root = path.parent.parent if path.parent.name == "components" else path.parent + return bundle_root, manifest + + +def load_saved_component_graph( + *, + bundle_root: str | Path, + component_entry: dict[str, object], + weights_dir: str | Path | None = None, +) -> LoadedComponentGraph: + root = Path(bundle_root).expanduser().resolve() + graph_relpath = component_entry.get("graph") + if not isinstance(graph_relpath, str) or not graph_relpath: + raise ValueError(f"component entry is missing graph path: {component_entry}") + + graph_path = (root / graph_relpath).resolve() + graph = Graph.load(graph_path) + + runtime_inputs = [ + graph._tensor_from_node(int(node_id)) + for node_id in component_entry.get("runtime_input_node_ids", []) + ] + outputs = [ + graph._tensor_from_node(int(node_id)) + for node_id in component_entry.get("output_node_ids", []) + ] + bound_constant_bindings = list(component_entry.get("bound_constant_bindings") or []) + bound_tensor_files = _rebind_bound_constants( + graph=graph, + bundle_root=root, + bindings=bound_constant_bindings, + weights_dir=weights_dir, + ) + return LoadedComponentGraph( + component=str(component_entry.get("component", "unknown")), + graph=graph, + runtime_inputs=runtime_inputs, + outputs=outputs, + bound_constant_bindings=bound_constant_bindings, + bound_tensor_files=bound_tensor_files, + ) + + +def load_saved_component_graphs( + bundle_dir_or_manifest: str | Path, + *, + weights_dir: str | Path | None = None, +) -> tuple[dict[str, LoadedComponentGraph], dict[str, object]]: + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + loaded: dict[str, LoadedComponentGraph] = {} + for component_entry in manifest.get("components", []): + if not isinstance(component_entry, dict): + continue + component_name = str(component_entry.get("component", "")).strip() + if not component_name: + continue + loaded[component_name] = load_saved_component_graph( + bundle_root=bundle_root, + component_entry=component_entry, + weights_dir=weights_dir, + ) + return loaded, manifest + + +def run_transpiled_bundle( + bundle_dir_or_manifest: str | Path, + *, + audio_file: str | Path | None = None, + prompt: str | None = None, + input_ids: str | list[int] | tuple[int, ...] | None = None, + weights_dir: str | Path | None = None, + torch_dtype: torch.dtype = torch.float16, +) -> dict[str, object]: + component_graphs, manifest = load_saved_component_graphs( + bundle_dir_or_manifest, + weights_dir=weights_dir, + ) + family = str(manifest.get("family", "") or "") + task = str(manifest.get("task", "") or "") + if family == "parakeet_tdt" and task == "tdt_transcription": + if audio_file is None: + raise ValueError("audio_file is required for Parakeet TDT component bundles") + return _run_parakeet_tdt_bundle( + component_graphs=component_graphs, + manifest=manifest, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + if task in {"causal_lm_logits", "multimodal_causal_lm_logits"}: + return _run_causal_lm_logits_bundle( + component_graphs=component_graphs, + manifest=manifest, + prompt=prompt, + input_ids=input_ids, + ) + if task == "encoder_hidden_states": + if audio_file is None: + inputs_meta = manifest.get("inputs") + if isinstance(inputs_meta, dict): + stored_audio = inputs_meta.get("audio_file") + if isinstance(stored_audio, str) and stored_audio: + audio_file = stored_audio + if audio_file is None: + raise ValueError("audio_file is required for encoder_hidden_states bundles") + return _run_encoder_hidden_states_bundle( + component_graphs=component_graphs, + manifest=manifest, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + raise NotImplementedError( + f"saved transpiled bundle execution is not implemented for family={family!r} task={task!r}" + ) + + +def execute_loaded_component_pipeline( + components: list[LoadedComponentGraph], + *, + initial_store: dict[str, Any], +) -> tuple[dict[str, np.ndarray], dict[str, list[np.ndarray]]]: + store: dict[str, np.ndarray] = {} + for key, value in initial_store.items(): + store[key] = _to_numpy(value) + + outputs_by_component: dict[str, list[np.ndarray]] = {} + for component in components: + runtime_inputs = [] + input_names = component_input_names(component) + for input_name in input_names: + if input_name not in store: + raise KeyError( + f"component {component.component} is missing pipeline input {input_name!r}" + ) + runtime_inputs.append(store[input_name]) + component.set_inputs(runtime_inputs) + raw_outputs = component.execute() + numpy_outputs = [output.numpy().copy() for output in raw_outputs] + output_names = component_output_names(component) + if len(numpy_outputs) != len(output_names): + raise ValueError( + f"component {component.component} produced {len(numpy_outputs)} outputs, " + f"expected {len(output_names)}" + ) + for output_name, value in zip(output_names, numpy_outputs, strict=True): + store[output_name] = value + outputs_by_component[component.component] = numpy_outputs + return store, outputs_by_component + + +def component_input_names(component: LoadedComponentGraph) -> tuple[str, ...]: + return tuple(str(value) for value in getattr(component, "_input_names", ())) + + +def component_output_names(component: LoadedComponentGraph) -> tuple[str, ...]: + return tuple(str(value) for value in getattr(component, "_output_names", ())) + + +def _run_parakeet_tdt_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + audio_file: str | Path, + torch_dtype: torch.dtype, +) -> dict[str, object]: + if "audio_encoder" not in component_graphs or "decoder" not in component_graphs: + raise ValueError("Parakeet TDT component bundle must include audio_encoder and decoder graphs") + + inputs_meta = manifest.get("inputs") or {} + input_shapes = inputs_meta.get("input_shapes") if isinstance(inputs_meta, dict) else {} + if not isinstance(input_shapes, dict): + input_shapes = {} + expected_shape = input_shapes.get("input_features") + if not (isinstance(expected_shape, list) and len(expected_shape) == 3): + raise ValueError("Parakeet TDT bundle manifest is missing inputs.input_shapes.input_features") + + model_source = str(manifest.get("model_source", "") or "") + config = load_parakeet_tdt_config(model_source) + preprocess_start = time.perf_counter() + input_features, active_frames = prepare_parakeet_tdt_audio_features( + audio_file=audio_file, + expected_frames=int(expected_shape[1]), + expected_mels=int(expected_shape[2]), + torch_dtype=torch_dtype, + ) + preprocess_end = time.perf_counter() + + _attach_component_io_names(manifest, component_graphs) + encoder_start = time.perf_counter() + store, _ = execute_loaded_component_pipeline( + [component_graphs["audio_encoder"]], + initial_store={"input_features": input_features}, + ) + encoder_end = time.perf_counter() + encoder_hidden_states = np.asarray(store["encoder_hidden_states"]) + batch_size = int(encoder_hidden_states.shape[0]) + if batch_size != 1: + raise ValueError("Parakeet TDT saved bundle runtime currently expects batch size 1") + + state_dtype = np.float16 if torch_dtype == torch.float16 else np.float32 + initial_states: list[np.ndarray] = [] + for _ in range(config.predictor_num_layers): + state_shape = (batch_size, config.predictor_hidden_dim) + initial_states.append(np.zeros(state_shape, dtype=state_dtype)) + initial_states.append(np.zeros(state_shape, dtype=state_dtype)) + + decoder_component = component_graphs["decoder"] + decoder_steps = 0 + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + nonlocal decoder_steps + decoder_steps += 1 + input_store: dict[str, object] = { + "encoder_frame": np.ascontiguousarray(frame), + "token_ids": np.full((batch_size,), token_id, dtype=np.int64), + } + for index in range(config.predictor_num_layers): + input_store[f"state_h_{index}"] = np.ascontiguousarray(state_values[index * 2]) + input_store[f"state_c_{index}"] = np.ascontiguousarray(state_values[index * 2 + 1]) + + runtime_inputs = [input_store[name] for name in component_input_names(decoder_component)] + decoder_component.set_inputs(runtime_inputs) + outputs = decoder_component.execute() + logits = outputs[0].numpy().astype(np.float32, copy=False) + next_states = tuple(output.numpy() for output in outputs[1:]) + return logits, next_states + + decoder_start = time.perf_counter() + emitted = greedy_decode_parakeet_tdt_token_ids( + config=config, + encoder_hidden_states=encoder_hidden_states, + initial_states=tuple(initial_states), + step=_step, + ) + decoder_end = time.perf_counter() + total_end = decoder_end + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "audio_file": str(Path(audio_file).expanduser().resolve()), + "preprocess_ms": (preprocess_end - preprocess_start) * 1000.0, + "encoder_ms": (encoder_end - encoder_start) * 1000.0, + "decoder_ms": (decoder_end - decoder_start) * 1000.0, + "total_ms": (total_end - preprocess_start) * 1000.0, + "decoder_steps": decoder_steps, + "active_feature_frames": active_frames, + "token_ids": emitted, + "transcript": _decode_parakeet_tdt_token_ids(config.vocabulary, emitted), + "encoder_hidden_shape": list(encoder_hidden_states.shape), + "component_order": list(manifest.get("component_order", [])), + } + + +def _run_causal_lm_logits_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + prompt: str | None, + input_ids: str | list[int] | tuple[int, ...] | None, +) -> dict[str, object]: + if "decoder" not in component_graphs: + raise ValueError("causal LM component bundle must include a decoder graph") + + token_ids, tokenizer = _resolve_causal_lm_input_ids( + manifest=manifest, + prompt=prompt, + input_ids=input_ids, + ) + if not token_ids: + raise ValueError("causal LM bundle input token ids are empty") + + _attach_component_io_names(manifest, component_graphs) + decoder = component_graphs["decoder"] + runtime_inputs = component_input_names(decoder) + if runtime_inputs and runtime_inputs != ("input_ids",): + raise ValueError( + "causal LM bundle runner currently expects decoder logical input ('input_ids',), " + f"got {runtime_inputs!r}" + ) + + input_array = np.asarray([token_ids], dtype=np.int64) + start = time.perf_counter() + decoder.set_inputs([input_array]) + outputs = decoder.execute() + end = time.perf_counter() + if not outputs: + raise RuntimeError("causal LM decoder graph produced no outputs") + + logits = outputs[0].numpy() + if logits.ndim != 3: + raise RuntimeError(f"expected logits with shape [batch, seq, vocab], got {list(logits.shape)}") + next_token_id = int(np.argmax(logits[0, -1])) + decoded = None + if tokenizer is None: + try: + tokenizer = _load_bundle_tokenizer(manifest) + except Exception: + tokenizer = None + if tokenizer is not None and hasattr(tokenizer, "decode"): + decoded = tokenizer.decode([next_token_id]) + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "component_order": list(manifest.get("component_order", [])), + "input_ids": token_ids, + "input_shape": list(input_array.shape), + "output_shape": list(logits.shape), + "decoder_ms": (end - start) * 1000.0, + "total_ms": (end - start) * 1000.0, + "next_token_id": next_token_id, + "next_token": decoded, + } + + +def _run_encoder_hidden_states_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + audio_file: str | Path, + torch_dtype: torch.dtype, +) -> dict[str, object]: + component_name = "audio_encoder" if "audio_encoder" in component_graphs else "encoder" + if component_name not in component_graphs: + raise ValueError("encoder_hidden_states bundle must include an audio_encoder or encoder graph") + + inputs_meta = manifest.get("inputs") or {} + input_shapes = inputs_meta.get("input_shapes") if isinstance(inputs_meta, dict) else {} + if not isinstance(input_shapes, dict): + input_shapes = {} + expected_shape = input_shapes.get("input_features") + if not (isinstance(expected_shape, list) and len(expected_shape) == 3): + raise ValueError("encoder bundle manifest is missing inputs.input_shapes.input_features") + + preprocess_start = time.perf_counter() + input_features, active_frames = _prepare_generic_audio_encoder_features( + audio_file=audio_file, + manifest=manifest, + expected_shape=expected_shape, + torch_dtype=torch_dtype, + ) + preprocess_end = time.perf_counter() + + _attach_component_io_names(manifest, component_graphs) + encoder = component_graphs[component_name] + encoder_start = time.perf_counter() + encoder.set_inputs([input_features]) + outputs = encoder.execute() + encoder_end = time.perf_counter() + if not outputs: + raise RuntimeError("encoder graph produced no outputs") + hidden = outputs[0].numpy() + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "audio_file": str(Path(audio_file).expanduser().resolve()), + "component_order": list(manifest.get("component_order", [])), + "active_feature_frames": active_frames, + "input_shape": list(input_features.shape), + "encoder_hidden_shape": list(hidden.shape), + "preprocess_ms": (preprocess_end - preprocess_start) * 1000.0, + "encoder_ms": (encoder_end - encoder_start) * 1000.0, + "total_ms": (encoder_end - preprocess_start) * 1000.0, + } + + +def _prepare_generic_audio_encoder_features( + *, + audio_file: str | Path, + manifest: dict[str, object], + expected_shape: list[object], + torch_dtype: torch.dtype, +) -> tuple[np.ndarray, int]: + family = str(manifest.get("family", "") or "") + inputs_meta = manifest.get("inputs") if isinstance(manifest.get("inputs"), dict) else {} + sample_rate = int(inputs_meta.get("sample_rate", 16000) if isinstance(inputs_meta, dict) else 16000) + batch = int(expected_shape[0]) + if batch != 1: + raise ValueError("saved audio encoder bundle runtime currently expects batch size 1") + + if family == "whisper": + expected_mels = int(expected_shape[1]) + expected_frames = int(expected_shape[2]) + try: + features, active_frames = prepare_cactus_audio_features( + audio_file, + model_type="whisper", + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + layout="mels_frames", + ) + return np.ascontiguousarray(features.detach().cpu().numpy()), active_frames + except Exception: + pass + else: + expected_frames = int(expected_shape[1]) + expected_mels = int(expected_shape[2]) + + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + features, feature_length = _generic_log_mel_features( + waveform, + sample_rate=sample_rate, + num_mels=expected_mels, + n_fft=400, + hop_length=160, + frame_length=400, + preemphasis=None, + ) + active_frames = min(feature_length, expected_frames) + features = features[:active_frames, :] + if expected_frames > active_frames: + features = np.pad(features, ((0, expected_frames - active_frames), (0, 0)), mode="constant") + if family == "whisper": + features = np.ascontiguousarray(features.T) + features = np.ascontiguousarray(features, dtype=np.float16 if torch_dtype == torch.float16 else np.float32) + return np.expand_dims(features, axis=0), active_frames + + +def _resolve_causal_lm_input_ids( + *, + manifest: dict[str, object], + prompt: str | None, + input_ids: str | list[int] | tuple[int, ...] | None, +) -> tuple[list[int], object | None]: + if input_ids is not None: + return _parse_input_ids(input_ids), None + + if prompt is None: + inputs_meta = manifest.get("inputs") + if isinstance(inputs_meta, dict): + stored_ids = inputs_meta.get("input_ids") + parsed = _parse_nested_manifest_input_ids(stored_ids) + if parsed: + return parsed, None + stored_prompt = inputs_meta.get("prompt") + if isinstance(stored_prompt, str) and stored_prompt: + prompt = stored_prompt + if not prompt: + raise ValueError("provide --input-ids or --prompt for causal LM component bundles") + + tokenizer = _load_bundle_tokenizer(manifest) + token_ids = _tokenize_bundle_prompt(tokenizer, prompt) + return token_ids, tokenizer + + +def _parse_input_ids(input_ids: str | list[int] | tuple[int, ...]) -> list[int]: + if isinstance(input_ids, str): + parsed = [int(part.strip()) for part in input_ids.split(",") if part.strip()] + else: + parsed = [int(value) for value in input_ids] + if not parsed: + raise ValueError("input_ids was provided but no token ids were parsed") + return parsed + + +def _parse_nested_manifest_input_ids(value: object) -> list[int] | None: + if isinstance(value, list) and value: + if all(isinstance(item, int) for item in value): + return [int(item) for item in value] + first = value[0] + if isinstance(first, list) and all(isinstance(item, int) for item in first): + return [int(item) for item in first] + return None + + +def _patch_missing_lzma_backport() -> str | None: + try: + import importlib.util + import sys + + if importlib.util.find_spec("_lzma") is not None: + return None + if importlib.util.find_spec("backports.lzma") is None: + return None + import backports.lzma as backports_lzma # type: ignore + + sys.modules.setdefault("lzma", backports_lzma) + return "using backports.lzma because this Python build is missing _lzma" + except Exception: + return None + + +def _load_bundle_tokenizer(manifest: dict[str, object]): + _patch_missing_lzma_backport() + try: + from transformers import AutoTokenizer # type: ignore + except Exception as exc: + raise RuntimeError(f"transformers is required to tokenize --prompt: {exc}") from exc + + model_source = str(manifest.get("model_source", "") or manifest.get("model_id", "") or "") + if not model_source: + raise ValueError("bundle manifest is missing model_source/model_id; provide --input-ids instead") + return AutoTokenizer.from_pretrained( + model_source, + local_files_only=True, + trust_remote_code=True, + ) + + +def _tokenize_bundle_prompt(tokenizer: object, prompt: str) -> list[int]: + apply_chat_template = getattr(tokenizer, "apply_chat_template", None) + if callable(apply_chat_template): + try: + encoded = apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=True, + add_generation_prompt=True, + return_dict=True, + ) + ids = encoded["input_ids"] if isinstance(encoded, Mapping) else encoded + if ids and isinstance(ids[0], list): + ids = ids[0] + return [int(value) for value in ids] + except Exception: + pass + + encoded = tokenizer(prompt, return_tensors=None) # type: ignore[operator] + ids = encoded["input_ids"] if isinstance(encoded, Mapping) else encoded + if ids and isinstance(ids[0], list): + ids = ids[0] + return [int(value) for value in ids] + + +def _attach_component_io_names( + manifest: dict[str, object], + component_graphs: dict[str, LoadedComponentGraph], +) -> None: + family = str(manifest.get("family", "") or "") + task = str(manifest.get("task", "") or "") + config = None + if family == "parakeet_tdt" and task == "tdt_transcription": + config = load_parakeet_tdt_config(str(manifest.get("model_source", "") or "")) + + for component_entry in manifest.get("components", []): + if not isinstance(component_entry, dict): + continue + name = str(component_entry.get("component", "")).strip() + if not name or name not in component_graphs: + continue + component = component_graphs[name] + logical_inputs = tuple(str(value) for value in component_entry.get("logical_inputs", [])) + logical_outputs = tuple(str(value) for value in component_entry.get("logical_outputs", [])) + if not logical_inputs or not logical_outputs: + logical_inputs, logical_outputs = _infer_legacy_component_io_names( + family=family, + task=task, + component_name=name, + config=config, + ) + component._input_names = logical_inputs + component._output_names = logical_outputs + + +def _infer_legacy_component_io_names( + *, + family: str, + task: str, + component_name: str, + config, +) -> tuple[tuple[str, ...], tuple[str, ...]]: + if task in {"causal_lm_logits", "multimodal_causal_lm_logits"} and component_name == "decoder": + return ("input_ids",), ("logits",) + if task == "encoder_hidden_states" and component_name in {"audio_encoder", "encoder"}: + return ("input_features",), ("encoder_hidden_states",) + if family == "parakeet_tdt" and task == "tdt_transcription": + if component_name == "audio_encoder": + return ("input_features",), ("encoder_hidden_states",) + if component_name == "decoder": + predictor_layers = 0 if config is None else int(config.predictor_num_layers) + state_inputs: list[str] = [] + state_outputs: list[str] = [] + for index in range(predictor_layers): + state_inputs.extend((f"state_h_{index}", f"state_c_{index}")) + state_outputs.extend((f"state_h_{index}", f"state_c_{index}")) + return ( + ("encoder_frame", "token_ids", *state_inputs), + ("step_logits", *state_outputs), + ) + raise ValueError( + f"component bundle manifest is missing logical IO names for family={family!r} task={task!r} component={component_name!r}" + ) + + +def _rebind_bound_constants( + *, + graph: Graph, + bundle_root: Path, + bindings: list[dict[str, object]], + weights_dir: str | Path | None, +) -> list[object]: + loaded: list[object] = [] + for binding in bindings: + node_id = int(binding["node_id"]) + raw_path = str(binding["path"]) + tensor_path = _resolve_bound_tensor_path( + raw_path, + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + tensor = graph._tensor_from_node(node_id) + binding_format = str(binding.get("format", "tensor_io") or "tensor_io") + if binding_format == "npy": + constant_array = np.load(tensor_path, mmap_mode="r") + precision = int(binding.get("precision", tensor.dtype)) + graph.set_external_input(tensor, int(constant_array.ctypes.data), dtype=precision) + loaded.append(constant_array) + continue + + tensor_file = _open_cactus_tensor_file(tensor_path) + graph.set_external_input(tensor, int(tensor_file.data.ctypes.data), dtype=tensor_file.precision) + if tensor_file.scales is not None and tensor_file.group_size > 0: + rc = _lib.cactus_graph_set_grouped_scales( + graph.h, + cactus_node_t(tensor.id), + int(tensor_file.group_size), + int(tensor_file.num_groups), + tensor_file.scales.ctypes.data_as(ctypes.c_void_p), + ) + if rc != 0: + raise RuntimeError("graph_set_grouped_scales failed") + if tensor_file.is_interleaved: + graph.set_interleaved(tensor, True, tensor_file.original_n) + loaded.append(tensor_file) + return loaded + + +def _resolve_bound_tensor_path( + raw_path: str, + *, + bundle_root: Path, + weights_dir: str | Path | None, +) -> Path: + candidates: list[Path] = [] + explicit = Path(raw_path).expanduser() + if explicit.is_absolute(): + candidates.append(explicit) + else: + candidates.append((bundle_root / explicit).resolve()) + candidates.append(explicit.resolve()) + + if weights_dir is not None: + weights_root = Path(weights_dir).expanduser().resolve() + raw_parts = Path(raw_path).parts + for index in range(len(raw_parts)): + candidates.append(weights_root.joinpath(*raw_parts[index:])) + candidates.append(weights_root / Path(raw_path).name) + + for candidate in candidates: + if candidate.exists(): + return candidate.resolve() + raise FileNotFoundError( + f"could not resolve bound tensor file {raw_path!r} from bundle_root={bundle_root}" + + ("" if weights_dir is None else f" weights_dir={Path(weights_dir).expanduser().resolve()}") + ) + + +def _open_cactus_tensor_file(path: str | Path) -> LoadedTensorFile: + tensor_path = Path(path).expanduser().resolve() + with tensor_path.open("rb") as handle: + header = handle.read(_HEADER_SIZE) + if len(header) < _HEADER_SIZE: + raise RuntimeError(f"tensor file is too small for a Cactus header: {tensor_path}") + + magic = header[:4] + if magic != CACTUS_MAGIC: + raise RuntimeError(f"tensor file is missing the CACT header: {tensor_path}") + + flags = struct.unpack_from(" 0) + precision = int(struct.unpack_from(" 0 else 0 + data_offset = ( + align_offset(scales_offset + scales_bytes, alignment) + if scales_bytes > 0 + else aligned_header + ) + + data_element_count = byte_size // np.dtype(dtype).itemsize + data = np.memmap(tensor_path, mode="r", dtype=dtype, offset=data_offset, shape=(data_element_count,)) + scales = None + if scales_bytes > 0: + scales = np.memmap( + tensor_path, + mode="r", + dtype=np.float16, + offset=scales_offset, + shape=(scales_bytes // np.dtype(np.float16).itemsize,), + ) + return LoadedTensorFile( + path=tensor_path, + precision=precision, + shape=shape, + data=data, + scales=scales, + group_size=group_size, + num_groups=num_groups, + is_interleaved=bool(flags & FLAG_INTERLEAVED), + original_n=original_n, + ) + + +def _decode_parakeet_tdt_token_ids(vocabulary: tuple[str, ...], token_ids: list[int]) -> str: + pieces: list[str] = [] + for token_id in token_ids: + if token_id < 0 or token_id >= len(vocabulary): + continue + piece = vocabulary[token_id] + if piece.startswith("<|") and piece.endswith("|>"): + continue + pieces.append(piece) + text = "".join(pieces).replace("▁", " ") + return re.sub(r"\s+", " ", text).strip() + + +def _to_numpy(value: Any) -> np.ndarray: + if isinstance(value, np.ndarray): + return np.ascontiguousarray(value) + if isinstance(value, torch.Tensor): + return np.ascontiguousarray(value.detach().cpu().numpy()) + raise TypeError(f"unsupported runtime value type: {type(value).__name__}") + diff --git a/python/src/transpile/component_partition.py b/python/src/transpile/component_partition.py new file mode 100644 index 000000000..baf4c738e --- /dev/null +++ b/python/src/transpile/component_partition.py @@ -0,0 +1,414 @@ +from __future__ import annotations + +import copy +from collections import Counter + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir + + +COMPONENT_AUDIO_ENCODER = "audio_encoder" +COMPONENT_VISION_ENCODER = "vision_encoder" +COMPONENT_LM_ENCODER = "lm_encoder" +COMPONENT_DECODER = "decoder" +COMPONENT_UNSPECIFIED = "unspecified" + +COMPONENT_ORDER = ( + COMPONENT_AUDIO_ENCODER, + COMPONENT_VISION_ENCODER, + COMPONENT_LM_ENCODER, + COMPONENT_DECODER, +) + + +def _normalized_name_candidates( + *, + value_id: str | None = None, + source_name: str | None = None, + meta: dict[str, object] | None = None, +) -> tuple[str, ...]: + names: list[str] = [] + + def _add(value: object) -> None: + if not isinstance(value, str): + return + normalized = value.strip().lower() + if normalized and normalized not in names: + names.append(normalized) + + if value_id is not None: + _add(value_id) + if source_name is not None: + _add(source_name) + if isinstance(meta, dict): + _add(meta.get("torch_name")) + for key in ("module_paths", "import_hint_providers"): + raw_paths = meta.get(key) + if isinstance(raw_paths, (tuple, list)): + for path in raw_paths: + _add(path) + return tuple(names) + + +def _contains_any(name: str, needles: tuple[str, ...]) -> bool: + return any(needle in name for needle in needles) + + +def classify_component_name_candidates( + candidates: tuple[str, ...], + *, + family: str, + task: str, +) -> str | None: + if not candidates: + return None + + for name in candidates: + if _contains_any( + name, + ( + "pixel_values", + "image_features", + "vision_tower", + "vision_model", + "vision_encoder", + "vision_backbone", + "embed_vision", + "image_tower", + "visual_encoder", + "visual.", + ), + ): + return COMPONENT_VISION_ENCODER + + for name in candidates: + if _contains_any( + name, + ( + "input_features", + "audio_features", + "speech_features", + "mel_features", + "audio_tower", + "audio_encoder", + "speech_encoder", + "speech_model.encoder", + "acoustic_encoder", + "feature_encoder", + "encoder.pre_encode", + "encoder.subsampling", + "encoder.layers", + "encoder.conformer", + "conformer.", + "subsample_conv_projection", + ), + ): + return COMPONENT_AUDIO_ENCODER + + for name in candidates: + if _contains_any( + name, + ( + "lm_head", + "transformer.h", + "decoder.layers", + "decoder.layer", + "decoder.block", + "language_model.layers", + "language_model.model.layers", + "model.language_model.layers", + "text_model.layers", + "model.layers", + "backbone.layers", + "adapter_backbone_layers_slice", + "decoder.prediction", + "dec_rnn", + "joint.", + "jointnet", + "ctc_head", + "classifier", + "output_projection", + "output_layer", + ), + ): + return COMPONENT_DECODER + + for name in candidates: + if task == "multimodal_causal_lm_logits" and _contains_any( + name, + ( + "input_ids", + "inputs_embeds", + "text_embeds", + "position_ids", + "get_input_embeddings", + "embed_tokens", + "embed_text", + "embed_tokens_per_layer", + "per_layer_model_projection", + "per_layer_input", + "placeholder", + "multimodal_projector", + "multi_modal_projector", + "connector", + "merge", + "masked_scatter", + "create_causal_mask", + "create_masks_for_generate", + "token_type", + "attention_mask", + "pixel_position", + "multimodal_backbone", + "adapter_model_model_language_model_embed_tokens", + "adapter_backbone_embed_tokens_per_layer", + "adapter_backbone_per_layer_model_projection", + ), + ): + return COMPONENT_LM_ENCODER + + for name in candidates: + if task in {"ctc_logits", "encoder_hidden_states"} and name.startswith("encoder."): + return COMPONENT_AUDIO_ENCODER + + if family == "gemma4" and task == "multimodal_causal_lm_logits": + for name in candidates: + if _contains_any( + name, + ( + "input_ids", + "token_type_ids", + "attention_mask", + "input_features_mask", + ), + ): + return COMPONENT_LM_ENCODER + + if family == "parakeet_tdt": + for name in candidates: + if _contains_any( + name, + ( + "decoder.prediction", + "dec_rnn", + "joint.", + "jointnet", + "predictor_", + "tdt_joint", + ), + ): + return COMPONENT_DECODER + if name.startswith("encoder."): + return COMPONENT_AUDIO_ENCODER + + return None + + +def classify_value_component(value_id: str, value: IRValue, *, family: str, task: str) -> str | None: + return classify_component_name_candidates( + _normalized_name_candidates( + value_id=value_id, + source_name=value.meta.get("source_name") if isinstance(value.meta, dict) else None, + meta=value.meta, + ), + family=family, + task=task, + ) + + +def classify_node_component(node: IRNode, *, family: str, task: str) -> str | None: + return classify_component_name_candidates( + _normalized_name_candidates(meta=node.meta), + family=family, + task=task, + ) + + +def _task_default_component(*, family: str, task: str) -> str: + if task == "multimodal_causal_lm_logits": + return COMPONENT_DECODER + if task == "causal_lm_logits": + return COMPONENT_DECODER + if task in {"ctc_logits", "encoder_hidden_states"}: + return COMPONENT_AUDIO_ENCODER + if family == "parakeet_tdt": + return COMPONENT_AUDIO_ENCODER + return COMPONENT_UNSPECIFIED + + +def _rebuild_users(graph: IRGraph) -> None: + for value in graph.values.values(): + value.users = [] + for value_id in graph.inputs: + graph.values[value_id].producer = None + for value_id in graph.constants: + graph.values[value_id].producer = None + for node_id in graph.order: + node = graph.nodes[node_id] + for output_id in node.outputs: + graph.values[output_id].producer = node_id + for input_id in node.inputs: + graph.values[input_id].users.append(node_id) + + +def annotate_ir_components(graph: IRGraph) -> None: + family = str(graph.meta.get("adapter_family", "") or "").lower() + task = str(graph.meta.get("task", "") or "").lower() + default_component = _task_default_component(family=family, task=task) + + for value_id, value in graph.values.items(): + component = classify_value_component(value_id, value, family=family, task=task) + if component is not None: + value.meta["component"] = component + + for node_id in graph.order: + node = graph.nodes[node_id] + component = classify_node_component(node, family=family, task=task) + if component is not None: + node.meta["component"] = component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = component + + for _ in range(8): + changed = False + for node_id in graph.order: + node = graph.nodes[node_id] + component = node.meta.get("component") + if component not in COMPONENT_ORDER: + input_components = { + graph.values[input_id].meta.get("component") + for input_id in node.inputs + if graph.values[input_id].meta.get("component") in COMPONENT_ORDER + } + if len(input_components) == 1: + component = next(iter(input_components)) + node.meta["component"] = component + changed = True + component = node.meta.get("component") + if component in COMPONENT_ORDER: + for output_id in node.outputs: + if graph.values[output_id].meta.get("component") != component: + graph.values[output_id].meta["component"] = component + changed = True + + for node_id in reversed(graph.order): + node = graph.nodes[node_id] + component = node.meta.get("component") + if component in COMPONENT_ORDER: + continue + output_components: set[str] = set() + for output_id in node.outputs: + value = graph.values[output_id] + for user_id in value.users: + user_component = graph.nodes[user_id].meta.get("component") + if user_component in COMPONENT_ORDER: + output_components.add(user_component) + if len(output_components) == 1: + component = next(iter(output_components)) + node.meta["component"] = component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = component + changed = True + + if not changed: + break + + for value in graph.values.values(): + component = value.meta.get("component") + if component not in COMPONENT_ORDER and default_component != COMPONENT_UNSPECIFIED: + value.meta["component"] = default_component + + for node_id in graph.order: + node = graph.nodes[node_id] + component = node.meta.get("component") + if component not in COMPONENT_ORDER and default_component != COMPONENT_UNSPECIFIED: + node.meta["component"] = default_component + for output_id in node.outputs: + graph.values[output_id].meta["component"] = default_component + + counts = Counter( + str(graph.nodes[node_id].meta.get("component", COMPONENT_UNSPECIFIED)) + for node_id in graph.order + ) + graph.meta["component_counts"] = dict(sorted(counts.items())) + + +def summarize_ir_components(graph: IRGraph) -> dict[str, int]: + annotate_ir_components(graph) + counts = graph.meta.get("component_counts", {}) + if isinstance(counts, dict): + return {str(key): int(value) for key, value in counts.items()} + return {} + + +def _clone_value_for_subgraph(value: IRValue, *, as_input: bool) -> IRValue: + cloned = copy.deepcopy(value) + if as_input: + cloned.producer = None + cloned.users = [] + return cloned + + +def extract_component_subgraphs(graph: IRGraph) -> dict[str, IRGraph]: + annotate_ir_components(graph) + component_graphs: dict[str, IRGraph] = {} + + for component in COMPONENT_ORDER: + node_ids = [ + node_id + for node_id in graph.order + if graph.nodes[node_id].meta.get("component") == component + ] + if not node_ids: + continue + + node_id_set = set(node_ids) + inputs: list[str] = [] + outputs: list[str] = [] + constants: dict[str, object] = {} + values: dict[str, IRValue] = {} + nodes: dict[str, IRNode] = {} + + for node_id in node_ids: + node = copy.deepcopy(graph.nodes[node_id]) + nodes[node_id] = node + for input_id in node.inputs: + if input_id in graph.constants: + constants[input_id] = graph.constants[input_id] + if input_id not in values: + values[input_id] = _clone_value_for_subgraph(graph.values[input_id], as_input=True) + continue + producer = graph.values[input_id].producer + if producer not in node_id_set and input_id not in inputs: + inputs.append(input_id) + values[input_id] = _clone_value_for_subgraph(graph.values[input_id], as_input=True) + for output_id in node.outputs: + if output_id not in values: + values[output_id] = _clone_value_for_subgraph(graph.values[output_id], as_input=False) + + for node_id in node_ids: + node = graph.nodes[node_id] + for output_id in node.outputs: + value = graph.values[output_id] + if output_id in graph.outputs or any(user_id not in node_id_set for user_id in value.users): + if output_id not in outputs: + outputs.append(output_id) + + component_graph = IRGraph( + values=values, + nodes=nodes, + order=list(node_ids), + inputs=inputs, + outputs=outputs, + constants=constants, + meta={ + **copy.deepcopy(graph.meta), + "component": component, + }, + ) + _rebuild_users(component_graph) + verify_ir(component_graph) + component_graphs[component] = component_graph + + return component_graphs diff --git a/python/src/transpile/component_pipeline.py b/python/src/transpile/component_pipeline.py new file mode 100644 index 000000000..e9e7c5568 --- /dev/null +++ b/python/src/transpile/component_pipeline.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import copy +from dataclasses import dataclass +from dataclasses import field +import re +from typing import Any + +import numpy as np +import torch + +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.lower import TranspiledGraph +from src.transpile.lower import transpile_preoptimized_ir +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph + + +@dataclass +class ComponentModuleSpec: + component: str + module: torch.nn.Module + example_inputs: tuple[torch.Tensor, ...] + input_keys: tuple[str, ...] + output_keys: tuple[str, ...] + graph_meta: dict[str, object] = field(default_factory=dict) + metadata: dict[str, object] = field(default_factory=dict) + + +@dataclass +class CapturedComponent: + component: str + input_keys: tuple[str, ...] + output_keys: tuple[str, ...] + raw_ir_graph: IRGraph + optimized_ir_graph: IRGraph + transpiled_graph: TranspiledGraph + metadata: dict[str, object] = field(default_factory=dict) + + +_GRAPH_ARG_INPUT_RE = re.compile(r"^v_args_(\d+)$") + + +def _align_input_keys_to_graph_inputs( + declared_input_keys: tuple[str, ...], + graph_inputs: tuple[str, ...] | list[str], +) -> tuple[str, ...]: + aligned: list[str] = [] + for value_id in graph_inputs: + match = _GRAPH_ARG_INPUT_RE.match(str(value_id)) + if match is None: + return declared_input_keys + input_index = int(match.group(1)) + if input_index < 0 or input_index >= len(declared_input_keys): + return declared_input_keys + aligned.append(declared_input_keys[input_index]) + return tuple(aligned) + + +def _call_optional_capture_hook(spec: ComponentModuleSpec, attr_name: str) -> None: + hook = getattr(spec.module, attr_name, None) + if not callable(hook): + return + try: + hook( + component=spec.component, + example_inputs=spec.example_inputs, + input_keys=spec.input_keys, + output_keys=spec.output_keys, + ) + except TypeError: + hook() + + +def capture_component_spec( + spec: ComponentModuleSpec, + *, + fusion_config: FusionConfig | None = None, +) -> CapturedComponent: + wrapped = _ComponentCaptureWrapper(spec.module, graph_meta=spec.graph_meta).eval() + capture_error: Exception | None = None + _call_optional_capture_hook(spec, "prepare_for_capture") + try: + captured = capture_model(wrapped, spec.example_inputs) + except Exception as exc: + capture_error = exc + raise + finally: + try: + _call_optional_capture_hook(spec, "restore_after_capture") + except Exception: + if capture_error is None: + raise + raw_ir_graph = copy.deepcopy(captured.ir_graph) + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + canonicalize_exported_graph(optimized_ir_graph) + optimize_graph(optimized_ir_graph, config=fusion_config) + transpiled_graph = transpile_preoptimized_ir(copy.deepcopy(optimized_ir_graph)) + if len(spec.output_keys) != len(transpiled_graph.outputs): + raise ValueError( + f"component {spec.component} declared {len(spec.output_keys)} output keys but lowered " + f"{len(transpiled_graph.outputs)} graph outputs" + ) + aligned_input_keys = _align_input_keys_to_graph_inputs(spec.input_keys, optimized_ir_graph.inputs) + return CapturedComponent( + component=spec.component, + input_keys=aligned_input_keys, + output_keys=spec.output_keys, + raw_ir_graph=raw_ir_graph, + optimized_ir_graph=optimized_ir_graph, + transpiled_graph=transpiled_graph, + metadata=dict(spec.metadata), + ) + + +def execute_component_pipeline( + components: list[CapturedComponent], + *, + initial_store: dict[str, Any], +) -> tuple[dict[str, np.ndarray], dict[str, list[np.ndarray]]]: + store: dict[str, np.ndarray] = {} + for key, value in initial_store.items(): + store[key] = _to_numpy(value) + + outputs_by_component: dict[str, list[np.ndarray]] = {} + for component in components: + runtime_inputs = [] + for input_key in component.input_keys: + if input_key not in store: + raise KeyError( + f"component {component.component} is missing pipeline input {input_key!r}" + ) + runtime_inputs.append(store[input_key]) + component.transpiled_graph.set_inputs(runtime_inputs) + raw_outputs = component.transpiled_graph.execute() + numpy_outputs = [output.numpy().copy() for output in raw_outputs] + if len(numpy_outputs) != len(component.output_keys): + raise ValueError( + f"component {component.component} produced {len(numpy_outputs)} outputs, " + f"expected {len(component.output_keys)}" + ) + for output_key, value in zip(component.output_keys, numpy_outputs, strict=True): + store[output_key] = value + outputs_by_component[component.component] = numpy_outputs + + return store, outputs_by_component + + +def initial_store_from_named_tensors( + keys: tuple[str, ...] | list[str], + tensors: tuple[torch.Tensor, ...] | list[torch.Tensor], +) -> dict[str, np.ndarray]: + if len(keys) != len(tensors): + raise ValueError(f"expected {len(keys)} tensors, got {len(tensors)}") + return { + str(key): _to_numpy(tensor) + for key, tensor in zip(keys, tensors, strict=True) + } + + +def _to_numpy(value: Any) -> np.ndarray: + if isinstance(value, np.ndarray): + return np.ascontiguousarray(value) + if isinstance(value, torch.Tensor): + return np.ascontiguousarray(value.detach().cpu().numpy()) + raise TypeError(f"unsupported pipeline value type: {type(value).__name__}") + + +class _ComponentCaptureWrapper(torch.nn.Module): + def __init__(self, module: torch.nn.Module, *, graph_meta: dict[str, object]): + super().__init__() + self.module = module + self.graph_meta = dict(graph_meta) + + def forward(self, *args: torch.Tensor) -> Any: + return self.module(*args) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.module, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta = {} + existing = metadata.get("graph", {}) + if isinstance(existing, dict): + graph_meta.update(existing) + graph_meta.update(self.graph_meta) + metadata["graph"] = graph_meta + return metadata diff --git a/python/src/transpile/fusion/__init__.py b/python/src/transpile/fusion/__init__.py new file mode 100644 index 000000000..da61cf10e --- /dev/null +++ b/python/src/transpile/fusion/__init__.py @@ -0,0 +1,47 @@ +from src.transpile.fusion.conv import ConvModuleMatch +from src.transpile.fusion.conv import match_conv_module +from src.transpile.fusion.attention import AttentionBlockMatch +from src.transpile.fusion.attention import AttentionMatch +from src.transpile.fusion.attention import SelfAttentionBlockMatch +from src.transpile.fusion.attention import match_attention +from src.transpile.fusion.attention import match_attention_block +from src.transpile.fusion.attention import match_self_attention_block +from src.transpile.fusion.deltanet import GatedDeltaNetMatch +from src.transpile.fusion.deltanet import match_gated_deltanet +from src.transpile.fusion.linear import LinearMatch +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.lstm import LSTMCellMatch +from src.transpile.fusion.lstm import match_lstm_cell +from src.transpile.fusion.mlp import GatedMLPMatch +from src.transpile.fusion.mlp import match_gated_mlp +from src.transpile.fusion.rel_pos_bias import RelPosBiasMatch +from src.transpile.fusion.rel_pos_bias import match_rel_pos_bias +from src.transpile.fusion.rms_norm import RMSNormMatch +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.fusion.rope import RoPEMatch +from src.transpile.fusion.rope import match_rope + +__all__ = [ + "ConvModuleMatch", + "AttentionBlockMatch", + "AttentionMatch", + "SelfAttentionBlockMatch", + "GatedDeltaNetMatch", + "GatedMLPMatch", + "LinearMatch", + "LSTMCellMatch", + "RelPosBiasMatch", + "RMSNormMatch", + "RoPEMatch", + "match_attention", + "match_attention_block", + "match_self_attention_block", + "match_conv_module", + "match_gated_deltanet", + "match_gated_mlp", + "match_linear", + "match_lstm_cell", + "match_rel_pos_bias", + "match_rms_norm", + "match_rope", +] diff --git a/python/src/transpile/fusion/attention.py b/python/src/transpile/fusion/attention.py new file mode 100644 index 000000000..88017bca8 --- /dev/null +++ b/python/src/transpile/fusion/attention.py @@ -0,0 +1,550 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import collect_node_ids +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.fusion.rope import match_rope + + +@dataclass(frozen=True) +class AttentionMatch: + query_value_id: str + key_value_id: str + value_value_id: str + source_input_value_ids: tuple[str, str, str] + weight_value_ids: tuple[str | None, str | None, str | None] + has_rope: bool + has_qk_norm: bool + has_gqa_repeat: bool + is_causal: bool + scale: float + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class AttentionBlockMatch: + attention_node_id: str + query_value_id: str + key_value_id: str + value_value_id: str + mask_value_id: str | None + gate_value_id: str | None + output_projection_weight_value_id: str + output_projection_bias_value_id: str | None + attention_output_shape: tuple[int, ...] + is_causal: bool + additive_mask: bool + scale: float + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class SelfAttentionBlockMatch: + attention_node_id: str + hidden_value_id: str + query_weight_value_id: str + query_projection_bias_value_id: str | None + query_add_value_id: str | None + rel_query_add_value_id: str | None + key_weight_value_id: str + key_projection_bias_value_id: str | None + value_weight_value_id: str + value_projection_bias_value_id: str | None + mask_value_id: str | None + relative_key_input_value_id: str | None + relative_key_weight_value_id: str | None + relative_key_projection_bias_value_id: str | None + gate_value_id: str | None + output_projection_weight_value_id: str + output_projection_bias_value_id: str | None + query_shape: tuple[int, ...] + key_shape: tuple[int, ...] + value_shape: tuple[int, ...] + relative_key_shape: tuple[int, ...] | None + attention_output_shape: tuple[int, ...] + is_causal: bool + additive_mask: bool + scale: float + rel_pos_scale: float | None + window_size: int + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _ProjectedAttentionInputMatch: + hidden_value_id: str + weight_value_id: str + projection_bias_value_id: str | None + projected_shape: tuple[int, ...] + add_value_id: str | None + node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class _RelativeKeyProjectionMatch: + input_value_id: str + weight_value_id: str + projection_bias_value_id: str | None + projected_shape: tuple[int, ...] + node_ids: tuple[str, ...] + + +def match_attention(graph: IRGraph, node: IRNode) -> AttentionMatch | None: + if node.op not in {"attention", "scaled_dot_product_attention"} or len(node.inputs) < 3: + return None + + q_info = _extract_attention_input(graph, node.inputs[0], role="q") + k_info = _extract_attention_input(graph, node.inputs[1], role="k") + v_info = _extract_attention_input(graph, node.inputs[2], role="v") + if q_info is None or k_info is None or v_info is None: + return None + + has_rope = bool(q_info["has_rope"] and k_info["has_rope"]) + has_qk_norm = bool(q_info["has_rms_norm"] and k_info["has_rms_norm"]) + has_gqa = bool(k_info["has_gqa_repeat"] or v_info["has_gqa_repeat"] or node.attrs.get("enable_gqa", False)) + node_ids = {node.id, *q_info["node_ids"], *k_info["node_ids"], *v_info["node_ids"]} + + return AttentionMatch( + query_value_id=node.inputs[0], + key_value_id=node.inputs[1], + value_value_id=node.inputs[2], + source_input_value_ids=(q_info["source_input"], k_info["source_input"], v_info["source_input"]), + weight_value_ids=(q_info.get("weight_value_id"), k_info.get("weight_value_id"), v_info.get("weight_value_id")), + has_rope=has_rope, + has_qk_norm=has_qk_norm, + has_gqa_repeat=has_gqa, + is_causal=bool(node.attrs.get("is_causal", False)), + scale=float(node.attrs.get("scale", 0.0)), + window_size=int(node.attrs.get("window_size", 0)), + node_ids=tuple(sorted(node_ids)), + ) + + +def match_self_attention_block(graph: IRGraph, node: IRNode) -> SelfAttentionBlockMatch | None: + projection = match_linear(graph, node) + if projection is None: + return None + + output_path = _extract_attention_output_path(graph, projection.input_value_id) + if output_path is None: + return None + attention_node = graph.nodes.get(output_path["attention_node_id"]) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: + return None + + query_match = _extract_projected_attention_input(graph, attention_node.inputs[0], role="q") + key_match = _extract_projected_attention_input(graph, attention_node.inputs[1], role="k") + value_match = _extract_projected_attention_input(graph, attention_node.inputs[2], role="v") + if query_match is None or key_match is None or value_match is None: + return None + + if not ( + query_match.hidden_value_id == key_match.hidden_value_id == value_match.hidden_value_id + ): + return None + + mask_value_id = attention_node.inputs[3] if len(attention_node.inputs) > 3 else None + rel_query_add_value_id: str | None = None + relative_key_input_value_id: str | None = None + relative_key_weight_value_id: str | None = None + relative_key_projection_bias_value_id: str | None = None + relative_key_shape: tuple[int, ...] | None = None + rel_pos_scale: float | None = None + rel_pos_node_ids: tuple[str, ...] = () + + if mask_value_id is not None: + rel_pos_node = producer(graph, strip_passthrough(graph, mask_value_id)) + if rel_pos_node is not None and rel_pos_node.op == "rel_pos_bias" and len(rel_pos_node.inputs) == 2: + rel_query_match = _extract_projected_attention_input(graph, rel_pos_node.inputs[0], role="q") + relative_key_match = _extract_relative_key_projection(graph, rel_pos_node.inputs[1]) + if rel_query_match is None or relative_key_match is None: + return None + if not ( + rel_query_match.hidden_value_id == query_match.hidden_value_id + and rel_query_match.weight_value_id == query_match.weight_value_id + and rel_query_match.projection_bias_value_id == query_match.projection_bias_value_id + and rel_query_match.projected_shape == query_match.projected_shape + ): + return None + rel_query_add_value_id = rel_query_match.add_value_id + relative_key_input_value_id = relative_key_match.input_value_id + relative_key_weight_value_id = relative_key_match.weight_value_id + relative_key_projection_bias_value_id = relative_key_match.projection_bias_value_id + relative_key_shape = relative_key_match.projected_shape + rel_pos_scale = float(rel_pos_node.attrs.get("scale", 1.0)) + rel_pos_node_ids = collect_node_ids( + rel_pos_node.id, + rel_pos_node.meta.get("rel_pos_bias_nodes", ()), + rel_query_match.node_ids, + relative_key_match.node_ids, + ) + mask_value_id = None + + return SelfAttentionBlockMatch( + attention_node_id=attention_node.id, + hidden_value_id=query_match.hidden_value_id, + query_weight_value_id=query_match.weight_value_id, + query_projection_bias_value_id=query_match.projection_bias_value_id, + query_add_value_id=query_match.add_value_id, + rel_query_add_value_id=rel_query_add_value_id, + key_weight_value_id=key_match.weight_value_id, + key_projection_bias_value_id=key_match.projection_bias_value_id, + value_weight_value_id=value_match.weight_value_id, + value_projection_bias_value_id=value_match.projection_bias_value_id, + mask_value_id=mask_value_id, + relative_key_input_value_id=relative_key_input_value_id, + relative_key_weight_value_id=relative_key_weight_value_id, + relative_key_projection_bias_value_id=relative_key_projection_bias_value_id, + gate_value_id=output_path["gate_value_id"], + output_projection_weight_value_id=projection.weight_value_id, + output_projection_bias_value_id=projection.bias_value_id, + query_shape=query_match.projected_shape, + key_shape=key_match.projected_shape, + value_shape=value_match.projected_shape, + relative_key_shape=relative_key_shape, + attention_output_shape=output_path["attention_output_shape"], + is_causal=bool(attention_node.attrs.get("is_causal", True)), + additive_mask=bool(attention_node.attrs.get("additive_mask", False)), + scale=float(attention_node.attrs.get("scale", 0.0)), + rel_pos_scale=rel_pos_scale, + window_size=int(attention_node.attrs.get("window_size", 0)), + node_ids=collect_node_ids( + attention_node.id, + projection.node_ids, + output_path["node_ids"], + query_match.node_ids, + key_match.node_ids, + value_match.node_ids, + rel_pos_node_ids, + ), + ) + + +def match_attention_block(graph: IRGraph, node: IRNode) -> AttentionBlockMatch | None: + projection = match_linear(graph, node) + if projection is None: + return None + + output_path = _extract_attention_output_path(graph, projection.input_value_id) + if output_path is None: + return None + attention_node = graph.nodes.get(output_path["attention_node_id"]) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"} or len(attention_node.inputs) < 3: + return None + + return AttentionBlockMatch( + attention_node_id=attention_node.id, + query_value_id=attention_node.inputs[0], + key_value_id=attention_node.inputs[1], + value_value_id=attention_node.inputs[2], + mask_value_id=attention_node.inputs[3] if len(attention_node.inputs) > 3 else None, + gate_value_id=output_path["gate_value_id"], + output_projection_weight_value_id=projection.weight_value_id, + output_projection_bias_value_id=projection.bias_value_id, + attention_output_shape=output_path["attention_output_shape"], + is_causal=bool(attention_node.attrs.get("is_causal", True)), + additive_mask=bool(attention_node.attrs.get("additive_mask", False)), + scale=float(attention_node.attrs.get("scale", 0.0)), + window_size=int(attention_node.attrs.get("window_size", 0)), + node_ids=tuple(sorted({attention_node.id, *projection.node_ids, *output_path["node_ids"]})), + ) + + +def _extract_attention_output_path(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + gate_value_id: str | None = None + + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + break + + node = producer(graph, current) + if node is not None and node.op == "multiply" and len(node.inputs) == 2: + for attn_candidate, gate_candidate in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + attn_match = _extract_attention_output_path(graph, attn_candidate) + if attn_match is None: + continue + attn_match["gate_value_id"] = gate_candidate + attn_match["node_ids"] = tuple(sorted({node.id, *attn_match["node_ids"]})) + return attn_match + + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + attention_output_shape = tuple(int(v) for v in reshape_node.attrs.get("shape", ())) + current = reshape_node.inputs[0] + transpose_node = producer(graph, current) + if transpose_node is None: + return None + + if transpose_node.op == "transpose": + if int(transpose_node.attrs.get("dim0", -1)) != 1 or int(transpose_node.attrs.get("dim1", -1)) != 2: + return None + current = transpose_node.inputs[0] + elif transpose_node.op == "permute": + permutation = tuple(int(v) for v in transpose_node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + current = transpose_node.inputs[0] + else: + return None + + attention_node = producer(graph, current) + if attention_node is None or attention_node.op not in {"attention", "scaled_dot_product_attention"}: + return None + + return { + "attention_node_id": attention_node.id, + "gate_value_id": gate_value_id, + "attention_output_shape": attention_output_shape, + "node_ids": tuple(sorted({reshape_node.id, transpose_node.id})), + } + + +def _extract_projected_attention_input(graph: IRGraph, value_id: str, *, role: str) -> _ProjectedAttentionInputMatch | None: + current = strip_passthrough(graph, value_id) + add_node = producer(graph, current) + if add_node is not None and add_node.op == "add" and len(add_node.inputs) == 2: + for projected_candidate, add_candidate in ((add_node.inputs[0], add_node.inputs[1]), (add_node.inputs[1], add_node.inputs[0])): + projected = _extract_transposed_projection(graph, projected_candidate) + if projected is None: + continue + if not _looks_like_attention_addend(graph, add_candidate, projected.projected_shape): + continue + return _ProjectedAttentionInputMatch( + hidden_value_id=projected.hidden_value_id, + weight_value_id=projected.weight_value_id, + projection_bias_value_id=projected.projection_bias_value_id, + projected_shape=projected.projected_shape, + add_value_id=strip_passthrough(graph, add_candidate), + node_ids=collect_node_ids(add_node.id, projected.node_ids), + ) + + if role == "q": + return None + + return _extract_transposed_projection(graph, current) + + +def _extract_transposed_projection(graph: IRGraph, value_id: str) -> _ProjectedAttentionInputMatch | None: + current = strip_passthrough(graph, value_id) + layout_node = producer(graph, current) + if layout_node is None or len(layout_node.inputs) != 1: + return None + + if layout_node.op == "permute": + permutation = tuple(int(v) for v in layout_node.attrs.get("permutation", ())) + if permutation != (0, 2, 1, 3): + return None + elif layout_node.op == "transpose": + if int(layout_node.attrs.get("dim0", -1)) != 1 or int(layout_node.attrs.get("dim1", -1)) != 2: + return None + else: + return None + + projected_value_id = strip_passthrough(graph, layout_node.inputs[0]) + projected_value = graph.values.get(projected_value_id) + if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: + return None + + reshape_node = producer(graph, projected_value_id) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + linear_node = producer(graph, reshape_node.inputs[0]) + if linear_node is None: + return None + linear = match_linear(graph, linear_node) + if linear is None: + return None + + return _ProjectedAttentionInputMatch( + hidden_value_id=linear.input_value_id, + weight_value_id=linear.weight_value_id, + projection_bias_value_id=linear.bias_value_id, + projected_shape=tuple(int(v) for v in projected_value.shape), + add_value_id=None, + node_ids=collect_node_ids(layout_node.id, reshape_node.id, linear.node_ids), + ) + + +def _extract_relative_key_projection(graph: IRGraph, value_id: str) -> _RelativeKeyProjectionMatch | None: + current = strip_passthrough(graph, value_id) + projected_value = graph.values.get(current) + if projected_value is None or projected_value.shape is None or len(projected_value.shape) != 4: + return None + + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"} or len(reshape_node.inputs) != 1: + return None + + linear_node = producer(graph, reshape_node.inputs[0]) + if linear_node is None: + return None + linear = match_linear(graph, linear_node) + if linear is None: + return None + + return _RelativeKeyProjectionMatch( + input_value_id=linear.input_value_id, + weight_value_id=linear.weight_value_id, + projection_bias_value_id=linear.bias_value_id, + projected_shape=tuple(int(v) for v in projected_value.shape), + node_ids=collect_node_ids(reshape_node.id, linear.node_ids), + ) + + +def _looks_like_attention_addend(graph: IRGraph, value_id: str, target_shape: tuple[int, ...]) -> bool: + value = graph.values.get(strip_passthrough(graph, value_id)) + if value is None or value.shape is None: + return False + + shape = tuple(int(v) for v in value.shape) + if len(shape) != 4 or len(target_shape) != 4: + return False + + batch, seq_len, heads, head_dim = target_shape + if shape[0] not in (1, batch) or shape[3] != head_dim: + return False + + # Allow either BHSD-style tensors like [B, H, 1, D] or BSHD-style tensors like [B, 1, H, D]. + if shape[1] == heads and shape[2] in (1, seq_len): + return True + if shape[1] in (1, seq_len) and shape[2] == heads: + return True + return False + + +def _extract_attention_input(graph: IRGraph, value_id: str, *, role: str) -> dict[str, object] | None: + node_ids: set[str] = set() + current = strip_passthrough(graph, value_id) + has_gqa_repeat = False + + while True: + node = producer(graph, current) + if node is None: + return { + "source_input": current, + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + node_ids.add(node.id) + + if node.op in {"reshape", "view", "transpose", "permute"}: + current = node.inputs[0] + continue + + rope = match_rope(graph, current) + if rope is not None: + rope_info = _extract_attention_input(graph, rope.input_value_id, role=role) + if rope_info is None: + return None + rope_info["has_rope"] = True + rope_info["node_ids"] = tuple(sorted(set(rope_info["node_ids"]) | node_ids | set(rope.node_ids))) + rope_info["has_gqa_repeat"] = bool(rope_info["has_gqa_repeat"] or has_gqa_repeat) + return rope_info + + rms = match_rms_norm(graph, node) + if rms is not None: + if role == "v": + return None + rms_info = _extract_attention_input(graph, rms.input_value_id, role=role) + if rms_info is None: + return None + rms_info["has_rms_norm"] = True + rms_info["node_ids"] = tuple(sorted(set(rms_info["node_ids"]) | node_ids | set(rms.node_ids))) + rms_info["has_gqa_repeat"] = bool(rms_info["has_gqa_repeat"] or has_gqa_repeat) + return rms_info + + if _looks_like_gqa_repeat(graph, current): + has_gqa_repeat = True + current = _unwrap_gqa_repeat(graph, current) + if current is None: + return None + continue + + linear_node = producer(graph, current) + if linear_node is None: + return { + "source_input": strip_passthrough(graph, current), + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + linear = match_linear(graph, linear_node) + if linear is None: + return { + "source_input": strip_passthrough(graph, current), + "weight_value_id": None, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(node_ids)), + } + return { + "source_input": strip_passthrough(graph, linear.input_value_id), + "weight_value_id": linear.weight_value_id, + "has_rope": False, + "has_rms_norm": False, + "has_gqa_repeat": has_gqa_repeat, + "node_ids": tuple(sorted(set(node_ids) | set(linear.node_ids))), + } + + +def _looks_like_gqa_repeat(graph: IRGraph, value_id: str) -> bool: + return _unwrap_gqa_repeat(graph, value_id) is not None + + +def _unwrap_gqa_repeat(graph: IRGraph, value_id: str) -> str | None: + current = strip_passthrough(graph, value_id) + reshape_node = producer(graph, current) + if reshape_node is None or reshape_node.op not in {"reshape", "view"}: + return None + current = reshape_node.inputs[0] + expand_node = producer(graph, current) + if expand_node is None or expand_node.op != "expand": + return None + current = expand_node.inputs[0] + while True: + node = producer(graph, current) + if node is None: + return None + if node.op == "slice": + current = node.inputs[0] + continue + if node.op != "unsqueeze": + return None + if int(node.attrs.get("dim", -999)) != 2: + return None + break + base_value_id = strip_passthrough(graph, node.inputs[0]) + base_shape = graph.values.get(base_value_id).shape if graph.values.get(base_value_id) is not None else None + target_shape = tuple(int(v) for v in expand_node.attrs.get("shape", ())) + if base_shape is None or len(base_shape) != 4 or len(target_shape) != 5: + return None + expected = (int(base_shape[0]), int(base_shape[1]), int(target_shape[2]), int(base_shape[2]), int(base_shape[3])) + if expected != target_shape: + return None + return base_value_id diff --git a/python/src/transpile/fusion/common.py b/python/src/transpile/fusion/common.py new file mode 100644 index 000000000..828783b7e --- /dev/null +++ b/python/src/transpile/fusion/common.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class FusionMatch: + kind: str + anchor_node_id: str + node_ids: tuple[str, ...] + value_ids: tuple[str, ...] + attrs: dict[str, object] + + +def producer(graph: IRGraph, value_id: str) -> IRNode | None: + value = graph.values.get(value_id) + if value is None or value.producer is None: + return None + return graph.nodes.get(value.producer) + + +def strip_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def collect_node_ids(*parts: Any) -> tuple[str, ...]: + node_ids: set[str] = set() + for part in parts: + if part is None: + continue + if isinstance(part, str): + node_ids.add(part) + continue + if isinstance(part, IRNode): + node_ids.add(part.id) + continue + if isinstance(part, (list, tuple, set)): + for item in part: + if isinstance(item, str): + node_ids.add(item) + elif isinstance(item, IRNode): + node_ids.add(item.id) + return tuple(sorted(node_ids)) diff --git a/python/src/transpile/fusion/conv.py b/python/src/transpile/fusion/conv.py new file mode 100644 index 000000000..1f75d6e79 --- /dev/null +++ b/python/src/transpile/fusion/conv.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import collect_node_ids +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class ConvModuleMatch: + input_value_id: str + pointwise1_weight_value_id: str + pointwise1_bias_value_id: str | None + depthwise_weight_value_id: str + depthwise_bias_value_id: str | None + batch_norm_weight_value_id: str + batch_norm_bias_value_id: str + batch_norm_running_mean_value_id: str + batch_norm_running_var_value_id: str + pointwise2_weight_value_id: str + pointwise2_bias_value_id: str | None + eps: float + depthwise_kernel_size: int + depthwise_padding: int + node_ids: tuple[str, ...] + + +def match_conv_module(graph: IRGraph, node: IRNode) -> ConvModuleMatch | None: + if node.op != "permute" or len(node.inputs) != 1 or len(node.outputs) != 1: + return None + if tuple(node.attrs.get("permutation", ())) != (0, 2, 1): + return None + + pointwise2 = producer(graph, strip_passthrough(graph, node.inputs[0])) + if not _is_pointwise_conv1d(pointwise2): + return None + + silu = producer(graph, strip_passthrough(graph, pointwise2.inputs[0])) + if silu is None or silu.op != "silu" or len(silu.inputs) != 1: + return None + + batch_norm = producer(graph, strip_passthrough(graph, silu.inputs[0])) + if batch_norm is None or batch_norm.op != "batch_norm" or len(batch_norm.inputs) != 5: + return None + + depthwise = producer(graph, strip_passthrough(graph, batch_norm.inputs[0])) + if not _is_same_depthwise_conv1d(depthwise): + return None + + glu = producer(graph, strip_passthrough(graph, depthwise.inputs[0])) + if glu is None or glu.op != "glu" or len(glu.inputs) != 1: + return None + if int(glu.attrs.get("axis", -1)) not in {1, -2}: + return None + + pointwise1 = producer(graph, strip_passthrough(graph, glu.inputs[0])) + if not _is_pointwise_conv1d(pointwise1): + return None + + input_permute = producer(graph, strip_passthrough(graph, pointwise1.inputs[0])) + if input_permute is None or input_permute.op != "permute" or len(input_permute.inputs) != 1: + return None + if tuple(input_permute.attrs.get("permutation", ())) != (0, 2, 1): + return None + + depthwise_weight_shape = graph.values.get(depthwise.inputs[1]).shape if depthwise.inputs[1] in graph.values else None + if depthwise_weight_shape is None or len(depthwise_weight_shape) != 3: + return None + depthwise_kernel_size = int(depthwise_weight_shape[2]) + + return ConvModuleMatch( + input_value_id=strip_passthrough(graph, input_permute.inputs[0]), + pointwise1_weight_value_id=pointwise1.inputs[1], + pointwise1_bias_value_id=pointwise1.inputs[2] if len(pointwise1.inputs) > 2 else None, + depthwise_weight_value_id=depthwise.inputs[1], + depthwise_bias_value_id=depthwise.inputs[2] if len(depthwise.inputs) > 2 else None, + batch_norm_weight_value_id=batch_norm.inputs[1], + batch_norm_bias_value_id=batch_norm.inputs[2], + batch_norm_running_mean_value_id=batch_norm.inputs[3], + batch_norm_running_var_value_id=batch_norm.inputs[4], + pointwise2_weight_value_id=pointwise2.inputs[1], + pointwise2_bias_value_id=pointwise2.inputs[2] if len(pointwise2.inputs) > 2 else None, + eps=float(batch_norm.attrs.get("eps", 1e-5)), + depthwise_kernel_size=depthwise_kernel_size, + depthwise_padding=int(depthwise.attrs.get("padding", 0)), + node_ids=collect_node_ids( + node, + pointwise2, + silu, + batch_norm, + depthwise, + glu, + pointwise1, + input_permute, + ), + ) + + +def _is_pointwise_conv1d(node: IRNode | None) -> bool: + return ( + node is not None + and node.op == "conv1d" + and len(node.inputs) >= 2 + and int(node.attrs.get("stride", 1)) == 1 + and int(node.attrs.get("padding", 0)) == 0 + and int(node.attrs.get("dilation", 1)) == 1 + and int(node.attrs.get("groups", 1)) == 1 + ) + + +def _is_same_depthwise_conv1d(node: IRNode | None) -> bool: + if node is None or node.op != "conv1d" or len(node.inputs) < 2: + return False + groups = int(node.attrs.get("groups", 1)) + padding = int(node.attrs.get("padding", 0)) + stride = int(node.attrs.get("stride", 1)) + dilation = int(node.attrs.get("dilation", 1)) + if stride != 1 or dilation != 1: + return False + return groups > 1 and padding >= 0 diff --git a/python/src/transpile/fusion/deltanet.py b/python/src/transpile/fusion/deltanet.py new file mode 100644 index 000000000..cef656a5d --- /dev/null +++ b/python/src/transpile/fusion/deltanet.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear +from src.transpile.fusion.rms_norm import match_rms_norm +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class GatedDeltaNetMatch: + output_value_id: str + normalized_input_value_id: str + qkv_weight_value_id: str + a_weight_value_id: str + b_weight_value_id: str + norm_weight_value_id: str + z_weight_value_id: str | None + dt_bias_value_id: str | None + a_log_value_id: str | None + conv_weight_value_id: str | None + num_k_heads: int + num_v_heads: int + key_dim: int + value_dim: int + eps: float + chunk_size: int + mode: str + node_ids: tuple[str, ...] + + +def match_gated_deltanet(graph: IRGraph, node: IRNode) -> GatedDeltaNetMatch | None: + if node.op not in {"view", "reshape"} or len(node.outputs) != 1: + return None + output_value = graph.values.get(node.outputs[0]) + if output_value is None or output_value.shape is None or len(output_value.shape) != 3: + return None + + body_value_id = strip_passthrough(graph, node.inputs[0]) + body_node = producer(graph, body_value_id) + if body_node is None: + return None + + z_weight_value_id: str | None = None + rms_match = None + normalized_input_value_id: str | None = None + node_ids: set[str] = {node.id} + + if body_node.op == "multiply" and len(body_node.inputs) == 2: + for rms_value_id, z_value_id in ((body_node.inputs[0], body_node.inputs[1]), (body_node.inputs[1], body_node.inputs[0])): + rms_match = _match_rms_from_value(graph, rms_value_id) + if rms_match is None: + continue + z_weight_value_id, normalized_input_value_id, z_node_ids = _extract_z_branch(graph, z_value_id) + if normalized_input_value_id is None: + continue + node_ids.update(z_node_ids) + node_ids.add(body_node.id) + break + if rms_match is None or normalized_input_value_id is None: + return None + else: + rms_match = _match_rms_from_value(graph, body_value_id) + if rms_match is None: + return None + + y2d_value_id = rms_match.input_value_id + if rms_match.weight_value_id is None: + return None + norm_weight_value_id = rms_match.weight_value_id + eps = float(rms_match.eps) + node_ids.update(rms_match.node_ids) + + if normalized_input_value_id is None: + normalized_input_value_id = _infer_normalized_input_from_y_branch(graph, y2d_value_id) + if normalized_input_value_id is None: + return None + + qkv_info = _find_qkv_linear(graph, normalized_input_value_id) + if qkv_info is None: + return None + qkv_weight_value_id, conv_weight_value_id, num_k_heads, key_dim, num_v_heads, value_dim, qkv_node_ids = qkv_info + node_ids.update(qkv_node_ids) + + a_info = _find_a_linear(graph, normalized_input_value_id) + b_info = _find_b_linear(graph, normalized_input_value_id) + if a_info is None or b_info is None: + return None + a_weight_value_id, dt_bias_value_id, a_log_value_id, a_node_ids = a_info + b_weight_value_id, b_node_ids = b_info + node_ids.update(a_node_ids) + node_ids.update(b_node_ids) + + mode, chunk_size = _classify_mode(graph, y2d_value_id, match_node_ids=node_ids) + + return GatedDeltaNetMatch( + output_value_id=node.outputs[0], + normalized_input_value_id=normalized_input_value_id, + qkv_weight_value_id=qkv_weight_value_id, + a_weight_value_id=a_weight_value_id, + b_weight_value_id=b_weight_value_id, + norm_weight_value_id=norm_weight_value_id, + z_weight_value_id=z_weight_value_id, + dt_bias_value_id=dt_bias_value_id, + a_log_value_id=a_log_value_id, + conv_weight_value_id=conv_weight_value_id, + num_k_heads=num_k_heads, + num_v_heads=num_v_heads, + key_dim=key_dim, + value_dim=value_dim, + eps=eps, + chunk_size=chunk_size, + mode=mode, + node_ids=tuple(sorted(node_ids)), + ) + + +def _match_rms_from_value(graph: IRGraph, value_id: str): + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + return None + return match_rms_norm(graph, node) + + +def _extract_z_branch(graph: IRGraph, value_id: str) -> tuple[str | None, str | None, tuple[str, ...]]: + current = strip_passthrough(graph, value_id) + silu_node = producer(graph, current) + if silu_node is None or silu_node.op != "silu" or len(silu_node.inputs) != 1: + return None, None, () + linear_value_id = strip_passthrough(graph, silu_node.inputs[0]) + linear_node = producer(graph, linear_value_id) + while linear_node is not None and linear_node.op in {"view", "reshape"} and len(linear_node.inputs) == 1: + linear_value_id = strip_passthrough(graph, linear_node.inputs[0]) + linear_node = producer(graph, linear_value_id) + if linear_node is None: + return None, None, () + linear_match = match_linear(graph, linear_node) + if linear_match is None: + return None, None, () + return ( + linear_match.weight_value_id, + linear_match.input_value_id, + tuple(sorted({silu_node.id, *linear_match.node_ids})), + ) + + +def _infer_normalized_input_from_y_branch(graph: IRGraph, value_id: str) -> str | None: + current = strip_passthrough(graph, value_id) + node = producer(graph, current) + visited: set[str] = set() + while node is not None and node.id not in visited: + visited.add(node.id) + for input_value_id in node.inputs: + candidate = producer(graph, strip_passthrough(graph, input_value_id)) + if candidate is None: + continue + linear_match = match_linear(graph, candidate) + if linear_match is not None: + return linear_match.input_value_id + if len(node.inputs) != 1: + return None + node = producer(graph, strip_passthrough(graph, node.inputs[0])) + return None + + +def _find_qkv_linear( + graph: IRGraph, + normalized_input_value_id: str, +) -> tuple[str, str | None, int, int, int, int, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + split_node = _find_descendant_op(graph, linear_match.output_value_id, {"split_with_sizes"}, max_depth=8) + if split_node is None: + continue + split_sizes = tuple(int(v) for v in split_node.attrs.get("sizes", ())) + if len(split_sizes) != 3: + continue + + conv_node = _find_descendant_op(graph, linear_match.output_value_id, {"conv1d"}, max_depth=6) + conv_weight_value_id = conv_node.inputs[1] if conv_node is not None and len(conv_node.inputs) >= 2 else None + + q_shape = None + v_shape = None + node_ids = {node.id, split_node.id} + if conv_node is not None: + node_ids.add(conv_node.id) + + split_value = graph.values.get(split_node.outputs[0]) + if split_value is None: + continue + for getitem_node_id in split_value.users: + getitem_node = graph.nodes.get(getitem_node_id) + if getitem_node is None or getitem_node.op != "getitem" or not getitem_node.outputs: + continue + shape = _find_rank4_shape(graph, getitem_node.outputs[0], max_depth=5) + if shape is None: + continue + branch_has_sum = _find_descendant_op(graph, getitem_node.outputs[0], {"sum"}, max_depth=8) is not None + node_ids.add(getitem_node.id) + if branch_has_sum and q_shape is None: + q_shape = shape + elif not branch_has_sum and v_shape is None: + v_shape = shape + + if q_shape is None or v_shape is None: + continue + + num_k_heads = int(q_shape[2]) + key_dim = int(q_shape[3]) + num_v_heads = int(v_shape[2]) + value_dim = int(v_shape[3]) + if num_k_heads <= 0 or key_dim <= 0 or num_v_heads <= 0 or value_dim <= 0: + continue + + return ( + linear_match.weight_value_id, + conv_weight_value_id, + num_k_heads, + key_dim, + num_v_heads, + value_dim, + tuple(sorted(node_ids | set(linear_match.node_ids))), + ) + return None + + +def _find_a_linear( + graph: IRGraph, + normalized_input_value_id: str, +) -> tuple[str, str | None, str | None, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + softplus_node = _find_descendant_op(graph, linear_match.output_value_id, {"softplus"}, max_depth=6) + if softplus_node is None: + continue + dt_bias_value_id = None + softplus_input_node = producer(graph, strip_passthrough(graph, softplus_node.inputs[0])) + if softplus_input_node is not None and softplus_input_node.op == "add": + for input_value_id in softplus_input_node.inputs: + value = graph.values.get(input_value_id) + if value is not None and value.producer is None and input_value_id in graph.constants: + dt_bias_value_id = input_value_id + break + a_log_value_id = None + multiply_node = _find_descendant_op(graph, softplus_node.outputs[0], {"multiply"}, max_depth=6) + if multiply_node is not None: + for input_value_id in multiply_node.inputs: + a_log_value_id = _extract_exp_constant_input(graph, input_value_id) + if a_log_value_id is not None: + break + node_ids = {softplus_node.id, *linear_match.node_ids} + if softplus_input_node is not None: + node_ids.add(softplus_input_node.id) + if multiply_node is not None: + node_ids.add(multiply_node.id) + return linear_match.weight_value_id, dt_bias_value_id, a_log_value_id, tuple(sorted(node_ids)) + return None + + +def _find_b_linear(graph: IRGraph, normalized_input_value_id: str) -> tuple[str, tuple[str, ...]] | None: + for node_id in graph.order: + node = graph.nodes[node_id] + linear_match = match_linear(graph, node) + if linear_match is None or linear_match.input_value_id != normalized_input_value_id: + continue + sigmoid_node = _find_descendant_op(graph, linear_match.output_value_id, {"sigmoid"}, max_depth=4) + if sigmoid_node is None: + continue + return linear_match.weight_value_id, tuple(sorted({sigmoid_node.id, *linear_match.node_ids})) + return None + + +def _extract_exp_constant_input(graph: IRGraph, value_id: str) -> str | None: + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + value = graph.values.get(value_id) + if value is not None and value.producer is None and value_id in graph.constants: + return value_id + return None + if node.op == "scalar_multiply": + return _extract_exp_constant_input(graph, node.inputs[0]) + if node.op == "scalar_exp": + source = strip_passthrough(graph, node.inputs[0]) + value = graph.values.get(source) + if value is not None and value.producer is None and source in graph.constants: + return source + return None + + +def _find_rank4_shape(graph: IRGraph, value_id: str, *, max_depth: int) -> tuple[int, int, int, int] | None: + frontier = [(value_id, 0)] + seen: set[str] = set() + while frontier: + current, depth = frontier.pop(0) + if current in seen or depth > max_depth: + continue + seen.add(current) + value = graph.values.get(current) + if value is not None and value.shape is not None and len(value.shape) == 4: + shape = tuple(int(dim) for dim in value.shape) + return shape # type: ignore[return-value] + for user_id in graph.values.get(current, None).users if graph.values.get(current, None) is not None else (): + user = graph.nodes.get(user_id) + if user is None or not user.outputs: + continue + frontier.extend((output_id, depth + 1) for output_id in user.outputs) + return None + + +def _find_descendant_op(graph: IRGraph, value_id: str, targets: set[str], *, max_depth: int) -> IRNode | None: + frontier = [(value_id, 0)] + seen: set[str] = set() + while frontier: + current, depth = frontier.pop(0) + if current in seen or depth > max_depth: + continue + seen.add(current) + value = graph.values.get(current) + if value is None: + continue + for user_id in value.users: + user = graph.nodes.get(user_id) + if user is None or not user.outputs: + continue + if user.op in targets: + return user + frontier.extend((output_id, depth + 1) for output_id in user.outputs) + return None + + +def _classify_mode(graph: IRGraph, y2d_value_id: str, *, match_node_ids: set[str]) -> tuple[str, int]: + del y2d_value_id + chunk_size = _infer_prefill_chunk_size(graph, match_node_ids) + # Current exported graphs are uncached/full-sequence captures, so fuse the + # linear-attention block to the prefill kernel by default. Decode should be + # emitted only once the capture path models cache/state explicitly. + return "prefill", chunk_size + + +def _infer_prefill_chunk_size(graph: IRGraph, match_node_ids: set[str]) -> int: + for node_id in match_node_ids: + node = graph.nodes.get(node_id) + if node is None or node.op != "aten.triu.default" or not node.inputs: + continue + ones_value = graph.values.get(node.inputs[0]) + if ones_value is not None and ones_value.shape and len(ones_value.shape) >= 1: + return int(ones_value.shape[-1]) + return 64 diff --git a/python/src/transpile/fusion/linear.py b/python/src/transpile/fusion/linear.py new file mode 100644 index 000000000..df5346730 --- /dev/null +++ b/python/src/transpile/fusion/linear.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class LinearMatch: + input_value_id: str + weight_value_id: str + bias_value_id: str | None + output_value_id: str + node_ids: tuple[str, ...] + kind: str + + +def match_linear(graph: IRGraph, node: IRNode) -> LinearMatch | None: + if node.op == "linear" and len(node.inputs) >= 2 and len(node.outputs) == 1: + return LinearMatch( + input_value_id=strip_passthrough(graph, node.inputs[0]), + weight_value_id=node.inputs[1], + bias_value_id=node.inputs[2] if bool(node.attrs.get("has_bias", False)) and len(node.inputs) > 2 else None, + output_value_id=node.outputs[0], + node_ids=(node.id,), + kind="linear", + ) + + if node.op == "addmm" and len(node.inputs) == 3 and len(node.outputs) == 1: + return LinearMatch( + input_value_id=strip_passthrough(graph, node.inputs[1]), + weight_value_id=node.inputs[2], + bias_value_id=node.inputs[0], + output_value_id=node.outputs[0], + node_ids=(node.id,), + kind="addmm", + ) + + if node.op != "add" or len(node.inputs) != 2 or len(node.outputs) != 1: + return None + + for matmul_value_id, bias_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + matmul_node = producer(graph, strip_passthrough(graph, matmul_value_id)) + if matmul_node is None or matmul_node.op != "matmul" or len(matmul_node.inputs) != 2: + continue + return LinearMatch( + input_value_id=strip_passthrough(graph, matmul_node.inputs[0]), + weight_value_id=matmul_node.inputs[1], + bias_value_id=bias_value_id, + output_value_id=node.outputs[0], + node_ids=tuple(sorted({node.id, matmul_node.id})), + kind="matmul_add", + ) + + return None diff --git a/python/src/transpile/fusion/lstm.py b/python/src/transpile/fusion/lstm.py new file mode 100644 index 000000000..f342071c4 --- /dev/null +++ b/python/src/transpile/fusion/lstm.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import collect_node_ids +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class LSTMCellMatch: + anchor_node_id: str + node_ids: tuple[str, ...] + x_value_id: str + h_prev_value_id: str + c_prev_value_id: str + weight_ih_value_id: str + weight_hh_value_id: str + bias_ih_value_id: str + bias_hh_value_id: str | None + h_output_value_id: str + c_output_value_id: str + + +def match_lstm_cell(graph: IRGraph, node: IRNode) -> LSTMCellMatch | None: + if node.op != "multiply" or len(node.inputs) != 2 or len(node.outputs) != 1: + return None + + for output_gate_value_id, c_tanh_value_id in ( + (node.inputs[0], node.inputs[1]), + (node.inputs[1], node.inputs[0]), + ): + output_gate = _extract_gate_activation(graph, output_gate_value_id, expected_activation="sigmoid") + if output_gate is None or output_gate["index"] != 3: + continue + + c_tanh_node = producer(graph, strip_passthrough(graph, c_tanh_value_id)) + if c_tanh_node is None or c_tanh_node.op != "tanh" or len(c_tanh_node.inputs) != 1: + continue + + c_next_value_id = strip_passthrough(graph, c_tanh_node.inputs[0]) + c_next_node = producer(graph, c_next_value_id) + if c_next_node is None or c_next_node.op != "add" or len(c_next_node.inputs) != 2 or len(c_next_node.outputs) != 1: + continue + + for forget_term_value_id, input_term_value_id in ( + (c_next_node.inputs[0], c_next_node.inputs[1]), + (c_next_node.inputs[1], c_next_node.inputs[0]), + ): + forget_term = _extract_forget_term(graph, forget_term_value_id) + input_term = _extract_input_term(graph, input_term_value_id) + if forget_term is None or input_term is None: + continue + + input_gate = input_term["input_gate"] + cell_gate = input_term["cell_gate"] + forget_gate = forget_term["forget_gate"] + chunk_node_id = output_gate["chunk_node_id"] + if ( + input_gate["chunk_node_id"] != chunk_node_id + or forget_gate["chunk_node_id"] != chunk_node_id + or cell_gate["chunk_node_id"] != chunk_node_id + ): + continue + if input_gate["index"] != 0 or forget_gate["index"] != 1 or cell_gate["index"] != 2: + continue + + chunk_node = graph.nodes.get(chunk_node_id) + if chunk_node is None or len(chunk_node.inputs) != 1: + continue + + gates = _extract_lstm_gate_sum(graph, chunk_node.inputs[0]) + if gates is None: + continue + + return LSTMCellMatch( + anchor_node_id=node.id, + node_ids=collect_node_ids( + node, + c_tanh_node, + c_next_node, + output_gate["node_ids"], + forget_gate["node_ids"], + input_gate["node_ids"], + cell_gate["node_ids"], + forget_term["node_ids"], + input_term["node_ids"], + chunk_node, + gates["node_ids"], + ), + x_value_id=gates["x_value_id"], + h_prev_value_id=gates["h_prev_value_id"], + c_prev_value_id=forget_term["c_prev_value_id"], + weight_ih_value_id=gates["weight_ih_value_id"], + weight_hh_value_id=gates["weight_hh_value_id"], + bias_ih_value_id=gates["bias_value_ids"][0], + bias_hh_value_id=gates["bias_value_ids"][1] if len(gates["bias_value_ids"]) > 1 else None, + h_output_value_id=node.outputs[0], + c_output_value_id=c_next_node.outputs[0], + ) + + return None + + +def _extract_gate_activation( + graph: IRGraph, + value_id: str, + *, + expected_activation: str, +) -> dict[str, object] | None: + activation_node = producer(graph, strip_passthrough(graph, value_id)) + if activation_node is None or activation_node.op != expected_activation or len(activation_node.inputs) != 1: + return None + + getitem_node = producer(graph, strip_passthrough(graph, activation_node.inputs[0])) + if getitem_node is None or getitem_node.op != "getitem" or len(getitem_node.inputs) != 1: + return None + + chunk_node = producer(graph, strip_passthrough(graph, getitem_node.inputs[0])) + if chunk_node is None or chunk_node.op != "chunk": + return None + if int(chunk_node.attrs.get("chunks", 0)) != 4: + return None + + return { + "chunk_node_id": chunk_node.id, + "index": int(getitem_node.attrs.get("index", -1)), + "preactivation_value_id": getitem_node.inputs[0], + "node_ids": (activation_node.id, getitem_node.id), + } + + +def _extract_forget_term(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul_node = producer(graph, strip_passthrough(graph, value_id)) + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for gate_value_id, c_prev_value_id in ( + (mul_node.inputs[0], mul_node.inputs[1]), + (mul_node.inputs[1], mul_node.inputs[0]), + ): + forget_gate = _extract_gate_activation(graph, gate_value_id, expected_activation="sigmoid") + if forget_gate is None: + continue + return { + "forget_gate": forget_gate, + "c_prev_value_id": strip_passthrough(graph, c_prev_value_id), + "node_ids": (mul_node.id,), + } + return None + + +def _extract_input_term(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul_node = producer(graph, strip_passthrough(graph, value_id)) + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for input_gate_value_id, cell_gate_value_id in ( + (mul_node.inputs[0], mul_node.inputs[1]), + (mul_node.inputs[1], mul_node.inputs[0]), + ): + input_gate = _extract_gate_activation(graph, input_gate_value_id, expected_activation="sigmoid") + if input_gate is None: + continue + cell_tanh_node = producer(graph, strip_passthrough(graph, cell_gate_value_id)) + if cell_tanh_node is None or cell_tanh_node.op != "tanh" or len(cell_tanh_node.inputs) != 1: + continue + cell_gate = _extract_gate_activation(graph, cell_gate_value_id, expected_activation="tanh") + if cell_gate is None: + continue + return { + "input_gate": input_gate, + "cell_gate": cell_gate, + "node_ids": (mul_node.id, cell_tanh_node.id), + } + return None + + +def _extract_lstm_gate_sum(graph: IRGraph, value_id: str) -> dict[str, object] | None: + terms, node_ids = _flatten_add_terms(graph, strip_passthrough(graph, value_id)) + linear_nodes: list[IRNode] = [] + bias_value_ids: list[str] = [] + for term_value_id in terms: + term_value_id = strip_passthrough(graph, term_value_id) + term_node = producer(graph, term_value_id) + if term_node is not None and term_node.op == "linear" and len(term_node.inputs) >= 2: + linear_nodes.append(term_node) + continue + if term_value_id in graph.constants: + bias_value_ids.append(term_value_id) + continue + return None + + if len(linear_nodes) != 2 or len(bias_value_ids) not in {1, 2}: + return None + + first_linear, second_linear = linear_nodes + return { + "x_value_id": first_linear.inputs[0], + "weight_ih_value_id": first_linear.inputs[1], + "h_prev_value_id": second_linear.inputs[0], + "weight_hh_value_id": second_linear.inputs[1], + "bias_value_ids": tuple(bias_value_ids), + "node_ids": (*node_ids, first_linear.id, second_linear.id), + } + + +def _flatten_add_terms(graph: IRGraph, value_id: str) -> tuple[list[str], tuple[str, ...]]: + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return [value_id], () + left_terms, left_nodes = _flatten_add_terms(graph, node.inputs[0]) + right_terms, right_nodes = _flatten_add_terms(graph, node.inputs[1]) + return [*left_terms, *right_terms], (*left_nodes, *right_nodes, node.id) diff --git a/python/src/transpile/fusion/mlp.py b/python/src/transpile/fusion/mlp.py new file mode 100644 index 000000000..1c3e0ab34 --- /dev/null +++ b/python/src/transpile/fusion/mlp.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import strip_passthrough +from src.transpile.fusion.linear import match_linear + + +@dataclass(frozen=True) +class GatedMLPMatch: + input_value_id: str + gate_weight_value_id: str + up_weight_value_id: str + down_weight_value_id: str + activation: str + bias_value_ids: tuple[str | None, str | None, str | None] + node_ids: tuple[str, ...] + + +def match_gated_mlp(graph: IRGraph, node: IRNode) -> GatedMLPMatch | None: + down = match_linear(graph, node) + if down is None: + return None + + mul_node = graph.nodes.get(graph.values[down.input_value_id].producer) if graph.values.get(down.input_value_id) is not None else None + if mul_node is None or mul_node.op != "multiply" or len(mul_node.inputs) != 2: + return None + + for activated_value_id, up_value_id in ((mul_node.inputs[0], mul_node.inputs[1]), (mul_node.inputs[1], mul_node.inputs[0])): + activated_node = graph.nodes.get(graph.values[strip_passthrough(graph, activated_value_id)].producer) if graph.values.get(strip_passthrough(graph, activated_value_id)) is not None else None + if activated_node is None or activated_node.op not in {"gelu", "silu"}: + continue + + gate = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer]) if graph.values.get(strip_passthrough(graph, activated_node.inputs[0])) is not None and graph.values[strip_passthrough(graph, activated_node.inputs[0])].producer in graph.nodes else None + up = match_linear(graph, graph.nodes[graph.values[strip_passthrough(graph, up_value_id)].producer]) if graph.values.get(strip_passthrough(graph, up_value_id)) is not None and graph.values[strip_passthrough(graph, up_value_id)].producer in graph.nodes else None + if gate is None or up is None: + continue + if strip_passthrough(graph, gate.input_value_id) != strip_passthrough(graph, up.input_value_id): + continue + + return GatedMLPMatch( + input_value_id=strip_passthrough(graph, gate.input_value_id), + gate_weight_value_id=gate.weight_value_id, + up_weight_value_id=up.weight_value_id, + down_weight_value_id=down.weight_value_id, + activation=activated_node.op, + bias_value_ids=(gate.bias_value_id, up.bias_value_id, down.bias_value_id), + node_ids=tuple(sorted({*gate.node_ids, activated_node.id, *up.node_ids, mul_node.id, *down.node_ids})), + ) + + return None diff --git a/python/src/transpile/fusion/rel_pos_bias.py b/python/src/transpile/fusion/rel_pos_bias.py new file mode 100644 index 000000000..bcb623ffd --- /dev/null +++ b/python/src/transpile/fusion/rel_pos_bias.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode + + +@dataclass(frozen=True) +class RelPosBiasMatch: + query_value_id: str + relative_key_value_id: str + scale: float + node_ids: tuple[str, ...] + + +def match_rel_pos_bias(graph: IRGraph, node: IRNode) -> RelPosBiasMatch | None: + if node.op != "scalar_multiply" or len(node.inputs) != 1: + return None + + gather_match = _match_gather_rel_pos_bias(graph, node) + if gather_match is not None: + return gather_match + + final_slice = producer(graph, strip_passthrough(graph, node.inputs[0])) + if final_slice is None or final_slice.op != "slice": + return None + if int(final_slice.attrs.get("axis", -1)) != 3 or int(final_slice.attrs.get("start", -1)) != 0: + return None + + reshape_after_shift = producer(graph, strip_passthrough(graph, final_slice.inputs[0])) + if reshape_after_shift is None or reshape_after_shift.op not in {"reshape", "view"} or len(reshape_after_shift.inputs) != 1: + return None + + shift_slice = producer(graph, strip_passthrough(graph, reshape_after_shift.inputs[0])) + if shift_slice is None or shift_slice.op != "slice": + return None + if int(shift_slice.attrs.get("axis", -1)) != 2 or int(shift_slice.attrs.get("start", -1)) != 1: + return None + + reshape_for_shift = producer(graph, strip_passthrough(graph, shift_slice.inputs[0])) + if reshape_for_shift is None or reshape_for_shift.op not in {"reshape", "view"} or len(reshape_for_shift.inputs) != 1: + return None + + pad = producer(graph, strip_passthrough(graph, reshape_for_shift.inputs[0])) + if pad is None or pad.op != "pad" or len(pad.inputs) != 1: + return None + if tuple(int(v) for v in pad.attrs.get("pads", ())) != (1, 0): + return None + if str(pad.attrs.get("mode", "constant")) != "constant": + return None + if float(pad.attrs.get("value", 0.0)) != 0.0: + return None + + matmul = producer(graph, strip_passthrough(graph, pad.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + + rhs_permute = producer(graph, strip_passthrough(graph, matmul.inputs[1])) + if rhs_permute is None or rhs_permute.op != "permute" or len(rhs_permute.inputs) != 1: + return None + if tuple(int(v) for v in rhs_permute.attrs.get("permutation", ())) != (0, 2, 3, 1): + return None + + query_value_id = strip_passthrough(graph, matmul.inputs[0]) + relative_key_value_id = strip_passthrough(graph, rhs_permute.inputs[0]) + + query_shape = graph.values.get(query_value_id).shape if query_value_id in graph.values else None + relative_key_shape = graph.values.get(relative_key_value_id).shape if relative_key_value_id in graph.values else None + output_shape = graph.values.get(node.outputs[0]).shape if node.outputs and node.outputs[0] in graph.values else None + if query_shape is None or relative_key_shape is None or output_shape is None: + return None + if len(query_shape) != 4 or len(relative_key_shape) != 4 or len(output_shape) != 4: + return None + + batch, heads, seq_len, head_dim = (int(v) for v in query_shape) + rel_batch, rel_len, rel_heads, rel_dim = (int(v) for v in relative_key_shape) + out_batch, out_heads, out_t, out_s = (int(v) for v in output_shape) + if rel_batch not in {1, batch}: + return None + if rel_heads != heads or rel_dim != head_dim: + return None + if rel_len < 2 * seq_len - 1: + return None + if (out_batch, out_heads, out_t, out_s) != (batch, heads, seq_len, seq_len): + return None + + return RelPosBiasMatch( + query_value_id=query_value_id, + relative_key_value_id=relative_key_value_id, + scale=float(node.attrs.get("value", 1.0)), + node_ids=tuple( + sorted( + { + node.id, + final_slice.id, + reshape_after_shift.id, + shift_slice.id, + reshape_for_shift.id, + pad.id, + matmul.id, + rhs_permute.id, + } + ) + ), + ) + + +def _match_gather_rel_pos_bias(graph: IRGraph, node: IRNode) -> RelPosBiasMatch | None: + gather = producer(graph, strip_passthrough(graph, node.inputs[0])) + if gather is None or gather.op != "gather" or len(gather.inputs) < 2: + return None + if int(gather.attrs.get("axis", -1)) not in {-1, 3}: + return None + + matmul = producer(graph, strip_passthrough(graph, gather.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + + query_heads = producer(graph, strip_passthrough(graph, matmul.inputs[0])) + rhs_transpose = producer(graph, strip_passthrough(graph, matmul.inputs[1])) + if query_heads is None or query_heads.op != "permute" or len(query_heads.inputs) != 1: + return None + if rhs_transpose is None or rhs_transpose.op != "permute" or len(rhs_transpose.inputs) != 1: + return None + if tuple(int(v) for v in query_heads.attrs.get("permutation", ())) != (0, 2, 1, 3): + return None + if tuple(int(v) for v in rhs_transpose.attrs.get("permutation", ())) != (0, 1, 3, 2): + return None + + relative_key_heads = producer(graph, strip_passthrough(graph, rhs_transpose.inputs[0])) + if relative_key_heads is None or relative_key_heads.op != "permute" or len(relative_key_heads.inputs) != 1: + return None + if tuple(int(v) for v in relative_key_heads.attrs.get("permutation", ())) != (0, 2, 1, 3): + return None + + query_value_id = strip_passthrough(graph, query_heads.inputs[0]) + relative_key_value_id = strip_passthrough(graph, relative_key_heads.inputs[0]) + output_value_id = node.outputs[0] if node.outputs else "" + + query_shape = graph.values.get(query_value_id).shape if query_value_id in graph.values else None + relative_key_shape = graph.values.get(relative_key_value_id).shape if relative_key_value_id in graph.values else None + output_shape = graph.values.get(output_value_id).shape if output_value_id in graph.values else None + if query_shape is None or relative_key_shape is None or output_shape is None: + return None + if len(query_shape) != 4 or len(relative_key_shape) != 4 or len(output_shape) != 4: + return None + + batch, seq_len, heads, head_dim = (int(v) for v in query_shape) + rel_batch, rel_len, rel_heads, rel_dim = (int(v) for v in relative_key_shape) + out_batch, out_heads, out_t, out_s = (int(v) for v in output_shape) + if rel_batch not in {1, batch}: + return None + if rel_heads != heads or rel_dim != head_dim: + return None + if rel_len < 2 * seq_len - 1: + return None + if (out_batch, out_heads, out_t, out_s) != (batch, heads, seq_len, seq_len): + return None + + return RelPosBiasMatch( + query_value_id=query_value_id, + relative_key_value_id=relative_key_value_id, + scale=float(node.attrs.get("value", 1.0)), + node_ids=tuple( + sorted( + { + node.id, + gather.id, + matmul.id, + query_heads.id, + rhs_transpose.id, + relative_key_heads.id, + } + ) + ), + ) diff --git a/python/src/transpile/fusion/rms_norm.py b/python/src/transpile/fusion/rms_norm.py new file mode 100644 index 000000000..b906aad96 --- /dev/null +++ b/python/src/transpile/fusion/rms_norm.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class RMSNormMatch: + input_value_id: str + weight_value_id: str | None + weight_offset: float + eps: float + node_ids: tuple[str, ...] + + +def match_rms_norm(graph: IRGraph, node: IRNode) -> RMSNormMatch | None: + anchor_value_id = node.inputs[0] if node.op == "type_as" and node.inputs else node.outputs[0] + anchor_value_id = strip_passthrough(graph, anchor_value_id) + anchor_node = producer(graph, anchor_value_id) + if anchor_node is None or anchor_node.op != "multiply" or len(anchor_node.inputs) != 2: + return None + + for normed_value_id, scale_value_id in ( + (anchor_node.inputs[0], anchor_node.inputs[1]), + (anchor_node.inputs[1], anchor_node.inputs[0]), + ): + scale = _extract_scale(graph, scale_value_id) + norm = _extract_normed_value(graph, normed_value_id) + if scale is None or norm is None: + continue + if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): + continue + return RMSNormMatch( + input_value_id=norm["input_value_id"], + weight_value_id=scale["weight_value_id"], + weight_offset=float(scale["offset"]), + eps=float(norm["eps"]), + node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"], *scale["node_ids"]})), + ) + + norm = _extract_normed_value(graph, anchor_node.outputs[0]) + if norm is None: + return None + if strip_passthrough(graph, norm["input_value_id"]) != strip_passthrough(graph, norm["pow_input_value_id"]): + return None + return RMSNormMatch( + input_value_id=norm["input_value_id"], + weight_value_id=None, + weight_offset=0.0, + eps=float(norm["eps"]), + node_ids=tuple(sorted({anchor_node.id, *norm["node_ids"]})), + ) + + +def _extract_scale(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + offset = 0.0 + node_ids: set[str] = set() + producer_node = producer(graph, current) + if producer_node is not None and producer_node.op == "scalar_add": + add_value = float(producer_node.attrs.get("value", 0.0)) + if add_value in (0.0, 1.0): + offset = add_value + node_ids.add(producer_node.id) + current = producer_node.inputs[0] + current = strip_passthrough(graph, current) + if current not in graph.constants or not isinstance(graph.constants[current], torch.Tensor): + return None + return {"weight_value_id": current, "offset": offset, "node_ids": tuple(sorted(node_ids))} + + +def _extract_normed_value(graph: IRGraph, value_id: str) -> dict[str, object] | None: + mul = producer(graph, strip_passthrough(graph, value_id)) + if mul is None or mul.op != "multiply" or len(mul.inputs) != 2: + return None + for x_candidate, rsqrt_candidate in ((mul.inputs[0], mul.inputs[1]), (mul.inputs[1], mul.inputs[0])): + rsqrt = _extract_rsqrt_chain(graph, rsqrt_candidate) + if rsqrt is None: + continue + if strip_passthrough(graph, x_candidate) != strip_passthrough(graph, rsqrt["pow_input_value_id"]): + continue + return { + "input_value_id": strip_passthrough(graph, x_candidate), + "pow_input_value_id": rsqrt["pow_input_value_id"], + "eps": rsqrt["eps"], + "node_ids": tuple(sorted({mul.id, *rsqrt["node_ids"]})), + } + return None + + +def _extract_rsqrt_chain(graph: IRGraph, value_id: str) -> dict[str, object] | None: + pow_node = producer(graph, strip_passthrough(graph, value_id)) + if pow_node is None or pow_node.op != "pow" or float(pow_node.attrs.get("exponent", 0.0)) != -0.5: + return None + add_node = producer(graph, pow_node.inputs[0]) + if add_node is None or add_node.op != "scalar_add": + return None + mean_node = producer(graph, add_node.inputs[0]) + if mean_node is None or mean_node.op != "mean" or not bool(mean_node.attrs.get("keepdim", False)): + return None + pow2_node = producer(graph, mean_node.inputs[0]) + if pow2_node is None or pow2_node.op != "pow" or float(pow2_node.attrs.get("exponent", 0.0)) != 2.0: + return None + return { + "pow_input_value_id": pow2_node.inputs[0], + "eps": float(add_node.attrs.get("value", 0.0)), + "node_ids": (pow_node.id, add_node.id, mean_node.id, pow2_node.id), + } diff --git a/python/src/transpile/fusion/rope.py b/python/src/transpile/fusion/rope.py new file mode 100644 index 000000000..592ee2933 --- /dev/null +++ b/python/src/transpile/fusion/rope.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_layout_passthrough +from src.transpile.fusion.common import strip_passthrough + + +@dataclass(frozen=True) +class RoPEMatch: + input_value_id: str + theta: float + position_offset: int + partial: bool + node_ids: tuple[str, ...] + + +def match_rope(graph: IRGraph, value_id: str) -> RoPEMatch | None: + desc = _extract_rope_descriptor(graph, value_id) + if desc is None: + return None + return RoPEMatch( + input_value_id=strip_passthrough(graph, desc["input_value_id"]), + theta=float(desc["theta"]), + position_offset=int(desc["position_offset"]), + partial=bool(desc.get("partial_rope", False)), + node_ids=tuple(sorted(set(desc.get("node_ids", ())))), + ) + + +def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = strip_layout_passthrough(graph, value_id) + node = producer(graph, current) + if node is None: + return None + if node.op == "rope": + return { + "input_value_id": node.inputs[0], + "theta": float(node.attrs.get("theta", 0.0)), + "position_offset": int(node.attrs.get("position_offset", 0)), + "partial_rope": False, + "node_ids": (node.id,), + } + classic = _extract_classic_rope_descriptor(graph, current) + if classic is not None: + classic["partial_rope"] = False + return classic + if node.op == "cat" and len(node.inputs) == 2: + for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) + if rope_branch is None: + continue + passthrough_node = producer(graph, strip_layout_passthrough(graph, passthrough_value_id)) + if passthrough_node is None or passthrough_node.op != "slice": + continue + if strip_passthrough(graph, passthrough_node.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): + continue + rope_branch["partial_rope"] = True + rope_branch["node_ids"] = tuple(sorted({node.id, passthrough_node.id, *rope_branch["node_ids"]})) + return rope_branch + return None + + +def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node = producer(graph, strip_layout_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return None + for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + direct_mult = producer(graph, direct_mult_id) + rotated_mult = producer(graph, rotated_mult_id) + if direct_mult is None or direct_mult.op != "multiply" or rotated_mult is None or rotated_mult.op != "multiply": + continue + direct_match = _extract_direct_rope_branch(graph, direct_mult) + rotated_match = _extract_rotated_rope_branch(graph, rotated_mult) + if direct_match is None or rotated_match is None: + continue + input_value_id = strip_passthrough(graph, direct_match["input_value_id"]) + if input_value_id != strip_passthrough(graph, rotated_match["input_value_id"]): + continue + cos_info = _extract_rope_trig(graph, direct_match["trig_value_id"], expected="scalar_cos") + sin_info = _extract_rope_trig(graph, rotated_match["trig_value_id"], expected="scalar_sin") + if cos_info is None or sin_info is None: + continue + if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: + continue + if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): + continue + return { + "input_value_id": input_value_id, + "theta": cos_info["theta"], + "position_offset": cos_info["position_offset"], + "node_ids": tuple(sorted({node.id, *direct_match["node_ids"], *rotated_match["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), + } + return None + + +def _extract_direct_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + if _unwrap_trig_value(graph, trig_value_id, expected="scalar_cos") is not None: + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotated_rope_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + input_value_id = _extract_rotate_half_source(graph, rotated_value_id) + if input_value_id is None: + continue + if _unwrap_trig_value(graph, trig_value_id, expected="scalar_sin") is None: + continue + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + neg_node = producer(graph, cat_node.inputs[0]) + right_slice = producer(graph, cat_node.inputs[1]) + if neg_node is None or neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: + return None + left_slice = producer(graph, neg_node.inputs[0]) + if left_slice is None or right_slice is None or left_slice.op != "slice" or right_slice.op != "slice": + return None + if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): + return None + return left_slice.inputs[0] + + +def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: + trig_info = _unwrap_trig_value(graph, value_id, expected=expected) + if trig_info is None: + return None + trig_node = trig_info["trig_node"] + cat_node = producer(graph, trig_node.inputs[0]) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: + return None + angle_info = _extract_rope_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + matmul_node = angle_info["matmul_node"] + inv_freq_const_id = None + arange_node = None + for input_id in matmul_node.inputs: + if inv_freq_const_id is None: + inv_freq_const_id = _find_constant_ancestor(graph, input_id) + if arange_node is None: + arange_node = _find_arange_ancestor(graph, input_id) + if inv_freq_const_id is None or arange_node is None: + return None + theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) + if theta is None: + return None + return { + "theta": theta, + "position_offset": int(arange_node.attrs.get("start", 0)), + "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), + } + + +def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: str) -> dict[str, object] | None: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"unsqueeze", "precision_cast", "reshape", "view", "expand"}: + current = node.inputs[0] + continue + if node.op == "type_as": + current = node.inputs[0] + continue + if node.op == "scalar_multiply": + current = node.inputs[0] + continue + if node.op != expected: + return None + return {"trig_node": node} + + +def _extract_rope_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + while True: + current = strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "permute": + matmul_node = producer(graph, node.inputs[0]) + if matmul_node is None or matmul_node.op != "matmul": + return None + return {"matmul_node": matmul_node} + if node.op == "matmul": + return {"matmul_node": node} + if node.op in {"index", "slice"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + return None + + +def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_layout_passthrough(graph, current) + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return current + node = producer(graph, current) + if node is None or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "arange": + return node + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return None + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _infer_rope_theta(value: Any) -> float | None: + if not isinstance(value, torch.Tensor): + return None + flat = value.detach().cpu().float().reshape(-1) + if flat.numel() < 2: + return None + second = float(flat[1].item()) + if second <= 0.0: + return None + inv_count = flat.numel() + return float((1.0 / second) ** inv_count) diff --git a/python/src/transpile/graph_ir.py b/python/src/transpile/graph_ir.py new file mode 100644 index 000000000..632585c57 --- /dev/null +++ b/python/src/transpile/graph_ir.py @@ -0,0 +1,201 @@ +import copy +from dataclasses import dataclass +from dataclasses import field + +import torch + + +def _clone_ir_payload(value, memo): + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.Tensor): + # IR constants may be real tensors or export-time tensor wrappers. + # They are treated as immutable graph payloads, so preserve them by + # reference instead of triggering Tensor.__deepcopy__. + return value + if isinstance(value, tuple): + return tuple(_clone_ir_payload(item, memo) for item in value) + if isinstance(value, list): + return [_clone_ir_payload(item, memo) for item in value] + if isinstance(value, dict): + return { + copy.deepcopy(key, memo): _clone_ir_payload(inner, memo) + for key, inner in value.items() + } + try: + return copy.deepcopy(value, memo) + except Exception: + return value + + +@dataclass +class IRValue: + id: str + shape: tuple[int, ...] | None = None + dtype: str | None = None + producer: str | None = None + users: list[str] = field(default_factory=list) + meta: dict[str, object] = field(default_factory=dict) + + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRValue( + id=self.id, + shape=_clone_ir_payload(self.shape, memo), + dtype=self.dtype, + producer=self.producer, + users=list(self.users), + meta=_clone_ir_payload(self.meta, memo), + ) + memo[id(self)] = cloned + return cloned + +@dataclass +class IRNode: + id: str + op: str + inputs: list[str] + outputs: list[str] + attrs: dict[str, object] = field(default_factory=dict) + meta: dict[str, object] = field(default_factory=dict) + kind: str = "generic" + + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRNode( + id=self.id, + op=self.op, + inputs=list(self.inputs), + outputs=list(self.outputs), + attrs=_clone_ir_payload(self.attrs, memo), + meta=_clone_ir_payload(self.meta, memo), + kind=self.kind, + ) + memo[id(self)] = cloned + return cloned + +@dataclass +class IRGraph: + values: dict[str, IRValue] + nodes: dict[str, IRNode] + order: list[str] + inputs: list[str] + outputs: list[str] + constants: dict[str, object] = field(default_factory=dict) + meta: dict[str, object] = field(default_factory=dict) + + def __deepcopy__(self, memo): + existing = memo.get(id(self)) + if existing is not None: + return existing + cloned = IRGraph( + values={value_id: copy.deepcopy(value, memo) for value_id, value in self.values.items()}, + nodes={node_id: copy.deepcopy(node, memo) for node_id, node in self.nodes.items()}, + order=list(self.order), + inputs=list(self.inputs), + outputs=list(self.outputs), + constants={ + value_id: _clone_ir_payload(constant, memo) + for value_id, constant in self.constants.items() + }, + meta=_clone_ir_payload(self.meta, memo), + ) + memo[id(self)] = cloned + return cloned + + def add_node(self, node: IRNode) -> None: + if node.id in self.nodes: + raise ValueError(f"duplicate IR node id: {node.id}") + self.nodes[node.id] = node + for output in node.outputs: + if output in self.values: + raise ValueError(f"duplicate IR value id: {output}") + self.values[output] = IRValue(id=output, producer=node.id) + + def add_value(self, value: IRValue) -> None: + if value.id in self.values: + raise ValueError(f"duplicate IR value id: {value.id}") + self.values[value.id] = value + + +def verify_ir(graph: IRGraph) -> None: + order_set = set(graph.order) + node_keys = set(graph.nodes.keys()) + if len(order_set) != len(graph.order): + raise ValueError("IR order contains duplicate node ids") + if order_set != node_keys: + missing_in_order = node_keys - order_set + missing_in_nodes = order_set - node_keys + raise ValueError( + "IR order/nodes mismatch: " + f"missing_in_order={sorted(missing_in_order)} " + f"missing_in_nodes={sorted(missing_in_nodes)}" + ) + + seen_nodes: set[str] = set() + seen_outputs: set[str] = set() + expected_users: dict[str, list[str]] = {} + + for value_id in graph.inputs: + if value_id not in graph.values: + raise ValueError(f"IR input missing value entry: {value_id}") + expected_users.setdefault(value_id, []) + + for value_id in graph.constants: + if value_id not in graph.values: + raise ValueError(f"IR constant missing value entry: {value_id}") + expected_users.setdefault(value_id, []) + + for node_id in graph.order: + if node_id in seen_nodes: + raise ValueError(f"duplicate node in IR order: {node_id}") + seen_nodes.add(node_id) + + node = graph.nodes[node_id] + if not node.outputs: + raise ValueError(f"IR node has no outputs: {node_id}") + + for input_id in node.inputs: + if input_id not in graph.values: + raise ValueError(f"IR node {node_id} references missing input value {input_id}") + expected_users.setdefault(input_id, []).append(node_id) + producer = graph.values[input_id].producer + if producer is not None and producer not in graph.nodes: + raise ValueError( + f"IR value {input_id} for node {node_id} has missing producer {producer}" + ) + + for output_id in node.outputs: + if output_id in seen_outputs: + raise ValueError(f"IR value produced more than once: {output_id}") + seen_outputs.add(output_id) + if output_id not in graph.values: + raise ValueError(f"IR node {node_id} missing output value entry {output_id}") + value = graph.values[output_id] + if value.producer != node_id: + raise ValueError( + f"IR value {output_id} producer mismatch: expected {node_id}, got {value.producer}" + ) + expected_users.setdefault(output_id, []) + + for output_id in graph.outputs: + if output_id not in graph.values: + raise ValueError(f"IR graph output missing value entry: {output_id}") + + for value_id, value in graph.values.items(): + expected = expected_users.get(value_id, []) + if value.users != expected: + raise ValueError( + f"IR value {value_id} user list mismatch: expected {expected}, got {value.users}" + ) + if value.producer is None and value_id not in graph.inputs and value_id not in graph.constants: + if value_id in seen_outputs: + raise ValueError(f"IR value {value_id} is produced but has no producer") + if value.producer is not None and value.producer not in graph.nodes: + raise ValueError( + f"IR value {value_id} references unknown producer {value.producer}" + ) diff --git a/python/src/transpile/import_ir.py b/python/src/transpile/import_ir.py new file mode 100644 index 000000000..291debdef --- /dev/null +++ b/python/src/transpile/import_ir.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import torch +from torch.export.graph_signature import ConstantArgument +from torch.export.graph_signature import InputKind + +from src.transpile.capture_pytorch import format_target +from src.transpile.capture_pytorch import get_dtype +from src.transpile.capture_pytorch import get_shape +from src.transpile.capture_pytorch import resolve_attr +from src.transpile.component_partition import annotate_ir_components +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import verify_ir +from src.transpile.importers import extract_literals +from src.transpile.importers import ImportContext +from src.transpile.importers import import_call_function +from src.transpile.importers import import_get_attr +from src.transpile.importers import import_output +from src.transpile.importers import import_placeholder +from src.transpile.importers import value_id +from src.transpile.normalize import dtype_to_ir +from src.transpile.import_semantics import apply_import_semantics +from src.transpile.weight_binding import resolve_transpile_weights_dir +from src.transpile.weight_binding import resolve_weight_binding + + +def import_captured_to_ir(captured: Any, *, strict: bool = True) -> IRGraph: + transpile_metadata = getattr(captured, "transpile_metadata", {}) or {} + graph_meta = transpile_metadata.get("graph", {}) + ir = IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[], constants={}, meta=dict(graph_meta)) + ctx = ImportContext(strict=strict, transpile_metadata=transpile_metadata) + weights_dir = resolve_transpile_weights_dir(ir.meta) + if weights_dir: + ir.meta["weights_dir"] = weights_dir + placeholder_specs: dict[str, Any] = {} + inline_counter = 0 + + def _try_register_weight_binding(value_id_str: str, target: str, value: Any) -> None: + if not isinstance(value, torch.Tensor): + return + binding = resolve_weight_binding( + weights_dir=weights_dir, + source_name=target, + ) + if binding is None: + return + bindings = ir.meta.setdefault("weight_bindings", {}) + if isinstance(bindings, dict): + bindings[value_id_str] = { + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + if value_id_str in ir.values: + ir.values[value_id_str].meta.update( + { + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + ) + + def _make_inlined_node(node: Any, *, inline_index: int) -> Any: + return SimpleNamespace( + name=f"inl_{inline_index}_{node.name}", + op=node.op, + target=node.target, + args=node.args, + kwargs=getattr(node, "kwargs", {}), + meta=getattr(node, "meta", {}), + ) + + def _resolve_target(target: str) -> Any: + candidates: list[str] = [] + parts = target.split(".") + for i in range(len(parts)): + candidate = ".".join(parts[i:]) + if candidate and candidate not in candidates: + candidates.append(candidate) + + exported_program = getattr(captured, "exported_program", None) + if exported_program is not None: + tensor_constants = getattr(exported_program, "tensor_constants", None) + if tensor_constants is not None: + for candidate in candidates: + if candidate in tensor_constants: + return tensor_constants[candidate] + constants = getattr(exported_program, "constants", None) + if constants is not None: + for candidate in candidates: + if candidate in constants: + return constants[candidate] + + for candidate in candidates: + try: + return resolve_attr(captured.graph_module, candidate) + except AttributeError: + pass + + for candidate in candidates: + if candidate in captured.state_dict: + return captured.state_dict[candidate] + + source_module = getattr(captured, "source_module", None) + if source_module is not None: + for candidate in candidates: + try: + return resolve_attr(source_module, candidate) + except AttributeError: + pass + + raise NotImplementedError(f"could not resolve get_attr target: {target}") from None + + def _collect_output_ids(arg: Any, local_alias: dict[str, str]) -> list[str]: + if hasattr(arg, "name") and hasattr(arg, "op"): + name = arg.name + if name in local_alias: + return [local_alias[name]] + return [ctx.resolve_value_id(arg)] + if isinstance(arg, (tuple, list)): + out: list[str] = [] + for item in arg: + out.extend(_collect_output_ids(item, local_alias)) + return out + return [] + + def _inline_wrap_with_set_grad_enabled(node: Any, shape: Any, dtype: Any) -> None: + nonlocal inline_counter + subgraph_ref = node.args[1] + if not hasattr(subgraph_ref, "op") or subgraph_ref.op != "get_attr": + raise NotImplementedError("wrap_with_set_grad_enabled without get_attr submodule is not supported") + + subgraph_module = _resolve_target(subgraph_ref.target) + if not hasattr(subgraph_module, "graph"): + raise NotImplementedError("wrap_with_set_grad_enabled target is not a GraphModule") + + inline_counter += 1 + inline_index = inline_counter + local_alias: dict[str, str] = {} + subgraph_inputs = list(node.args[2:]) + input_index = 0 + + ctx.push_local_aliases(local_alias) + try: + for sub_node in subgraph_module.graph.nodes: + sub_shape = get_shape(sub_node) + sub_dtype = dtype_to_ir(get_dtype(sub_node)) + sub_desc = ( + f"inlined:{node.name}:{sub_node.op}:{getattr(sub_node, 'name', '')}:" + f"{format_target(sub_node) if sub_node.op == 'call_function' else getattr(sub_node, 'target', '')}" + ) + ctx.push_node(sub_desc) + try: + if sub_node.op == "placeholder": + if input_index >= len(subgraph_inputs): + raise NotImplementedError("subgraph placeholder/input arity mismatch during inlining") + local_alias[sub_node.name] = ctx.resolve_value_id(subgraph_inputs[input_index]) + input_index += 1 + continue + + if sub_node.op == "get_attr": + prefixed = _make_inlined_node(sub_node, inline_index=inline_index) + value = _resolve_target(sub_node.target) + import_get_attr(ir, prefixed, ctx, value, shape=sub_shape, dtype=sub_dtype, source_name=str(sub_node.target)) + _try_register_weight_binding(f"v_{prefixed.name}", str(sub_node.target), value) + local_alias[sub_node.name] = f"v_{prefixed.name}" + continue + + if sub_node.op == "call_function": + prefixed = _make_inlined_node(sub_node, inline_index=inline_index) + import_call_function( + ir, + prefixed, + ctx, + shape=sub_shape, + dtype=sub_dtype, + torch_op=format_target(sub_node), + ) + local_alias[sub_node.name] = f"v_{prefixed.name}" + continue + + if sub_node.op == "output": + ctx.tuple_aliases[node.name] = _collect_output_ids(sub_node.args[0], local_alias) + return + + raise NotImplementedError(f"unsupported inlined subgraph node op: {sub_node.op}") + finally: + ctx.pop_node() + finally: + ctx.pop_local_aliases() + + raise NotImplementedError("inlined subgraph had no output node") + + for spec in captured.exported_program.graph_signature.input_specs: + arg = getattr(spec, "arg", None) + name = getattr(arg, "name", None) + if name is not None: + placeholder_specs[name] = spec + + for node in captured.graph.nodes: + shape = get_shape(node) + dtype = dtype_to_ir(get_dtype(node)) + node_desc = f"top:{node.op}:{getattr(node, 'name', '')}:{getattr(node, 'target', '')}" + ctx.push_node(node_desc) + try: + if node.op == "placeholder": + spec = placeholder_specs.get(node.name) + if spec is not None and isinstance(getattr(spec, "arg", None), ConstantArgument): + import_get_attr( + ir, + node, + ctx, + spec.arg.value, + shape=shape, + dtype=dtype, + source_name=node.name, + ) + continue + if spec is not None and spec.kind in {InputKind.PARAMETER, InputKind.BUFFER, InputKind.CONSTANT_TENSOR}: + target = getattr(spec, "target", None) + if target is None: + raise NotImplementedError(f"placeholder {node.name} has no target for lifted constant input") + value = _resolve_target(target) + import_get_attr(ir, node, ctx, value, shape=shape, dtype=dtype, source_name=str(target)) + _try_register_weight_binding(value_id(node, ctx), str(target), value) + continue + import_placeholder(ir, node, ctx, shape=shape, dtype=dtype) + continue + + if node.op == "get_attr": + if str(node.target).startswith("submod_"): + continue + value = _resolve_target(node.target) + import_get_attr(ir, node, ctx, value, shape=shape, dtype=dtype, source_name=str(node.target)) + _try_register_weight_binding(value_id(node, ctx), str(node.target), value) + continue + + if node.op == "call_function": + torch_op = format_target(node) + if "wrap_with_set_grad_enabled" in torch_op: + _inline_wrap_with_set_grad_enabled(node, shape, dtype) + continue + if torch_op == "": + source = node.args[0] + if hasattr(source, "name") and source.name in ctx.tuple_aliases: + index_value = extract_literals(node.args[1]) + if not isinstance(index_value, int): + raise NotImplementedError(f"unsupported tuple getitem index: {node.args!r}") + ctx.alias_values[node.name] = ctx.tuple_aliases[source.name][index_value] + continue + import_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) + continue + + if node.op == "call_module" and str(getattr(node, "target", "")) == "_guards_fn" and not node.users: + continue + + if node.op == "output": + import_output(ir, node, ctx) + continue + + raise NotImplementedError(f"unsupported FX node op: {node.op}") + finally: + ctx.pop_node() + + apply_import_semantics(ir) + annotate_ir_components(ir) + verify_ir(ir) + return ir diff --git a/python/src/transpile/import_semantics.py b/python/src/transpile/import_semantics.py new file mode 100644 index 000000000..9b9e86a32 --- /dev/null +++ b/python/src/transpile/import_semantics.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.canonicalize.utils import rebuild_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import verify_ir + + +@dataclass(frozen=True) +class EarlyRoPEMatch: + anchor_node_id: str + input_value_id: str + theta: float + position_offset: int + partial: bool + node_ids: tuple[str, ...] + + +def apply_import_semantics(graph: IRGraph) -> IRGraph: + _rewrite_explicit_attention(graph) + _tag_explicit_linear(graph) + _rewrite_early_rope(graph) + rebuild_graph(graph) + verify_ir(graph) + return graph + + +def _rewrite_explicit_attention(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "scaled_dot_product_attention": + continue + additive_mask = bool(node.attrs.get("additive_mask", False)) + if not additive_mask and len(node.inputs) > 3: + mask_value = graph.values.get(node.inputs[3]) + if mask_value is not None and mask_value.dtype not in (None, "bool"): + additive_mask = True + node.op = "attention" + node.kind = "semantic" + node.meta["semantic_source"] = "import" + node.attrs = { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", False)), + "window_size": int(node.attrs.get("window_size", 0)), + **({"enable_gqa": bool(node.attrs.get("enable_gqa", False))} if "enable_gqa" in node.attrs else {}), + **({"additive_mask": True} if additive_mask else {}), + } + + +def _tag_explicit_linear(graph: IRGraph) -> None: + for node in graph.nodes.values(): + if node.op not in {"linear", "addmm"}: + continue + node.meta.setdefault("semantic_source", "import") + + +def _rewrite_early_rope(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or not node.outputs: + continue + match = match_early_rope(graph, node.outputs[0]) + if match is None or match.anchor_node_id != node.id: + continue + if match.partial: + node.meta["semantic_hint"] = "partial_rope" + node.meta["rope_theta_hint"] = float(match.theta) + node.meta["rope_position_offset_hint"] = int(match.position_offset) + continue + node.op = "rope" + node.inputs = [match.input_value_id] + node.attrs = { + "theta": float(match.theta), + "position_offset": int(match.position_offset), + } + node.kind = "semantic" + node.meta["semantic_source"] = "import" + + +def match_early_rope(graph: IRGraph, value_id: str) -> EarlyRoPEMatch | None: + desc = _extract_rope_descriptor(graph, value_id) + if desc is None: + return None + return EarlyRoPEMatch( + anchor_node_id=str(desc["anchor_node_id"]), + input_value_id=str(desc["input_value_id"]), + theta=float(desc["theta"]), + position_offset=int(desc["position_offset"]), + partial=bool(desc.get("partial_rope", False)), + node_ids=tuple(sorted(set(desc.get("node_ids", ())))), + ) + + +def _extract_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = _strip_layout_passthrough(graph, value_id) + node = producer(graph, current) + if node is None: + return None + if node.op == "rope": + return { + "anchor_node_id": node.id, + "input_value_id": node.inputs[0], + "theta": float(node.attrs.get("theta", 0.0)), + "position_offset": int(node.attrs.get("position_offset", 0)), + "partial_rope": False, + "node_ids": (node.id,), + } + classic = _extract_classic_rope_descriptor(graph, current) + if classic is not None: + classic["partial_rope"] = False + return classic + if node.op == "cat" and len(node.inputs) == 2: + for rope_branch_value_id, passthrough_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + rope_branch = _extract_rope_descriptor(graph, rope_branch_value_id) + if rope_branch is None: + continue + passthrough = producer(graph, _strip_layout_passthrough(graph, passthrough_value_id)) + if passthrough is None or passthrough.op != "slice": + continue + if strip_passthrough(graph, passthrough.inputs[0]) != strip_passthrough(graph, rope_branch["input_value_id"]): + continue + rope_branch["anchor_node_id"] = node.id + rope_branch["partial_rope"] = True + rope_branch["node_ids"] = tuple(sorted({node.id, passthrough.id, *rope_branch["node_ids"]})) + return rope_branch + return None + + +def _extract_classic_rope_descriptor(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node = producer(graph, _strip_layout_passthrough(graph, value_id)) + if node is None or node.op != "add" or len(node.inputs) != 2: + return None + for direct_mult_id, rotated_mult_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + direct_mult = producer(graph, direct_mult_id) + rotated_mult = producer(graph, rotated_mult_id) + if direct_mult is None or rotated_mult is None: + continue + if direct_mult.op != "multiply" or rotated_mult.op != "multiply": + continue + direct = _extract_direct_branch(graph, direct_mult) + rotated = _extract_rotated_branch(graph, rotated_mult) + if direct is None or rotated is None: + continue + input_value_id = strip_passthrough(graph, direct["input_value_id"]) + if input_value_id != strip_passthrough(graph, rotated["input_value_id"]): + continue + cos_info = _extract_rope_trig(graph, direct["trig_value_id"], expected=("cos", "scalar_cos")) + sin_info = _extract_rope_trig(graph, rotated["trig_value_id"], expected=("sin", "scalar_sin")) + if cos_info is None or sin_info is None: + continue + if abs(float(cos_info["theta"]) - float(sin_info["theta"])) > 1e-2: + continue + if int(cos_info["position_offset"]) != int(sin_info["position_offset"]): + continue + return { + "anchor_node_id": node.id, + "input_value_id": input_value_id, + "theta": float(cos_info["theta"]), + "position_offset": int(cos_info["position_offset"]), + "node_ids": tuple(sorted({node.id, *direct["node_ids"], *rotated["node_ids"], *cos_info["node_ids"], *sin_info["node_ids"]})), + } + return None + + +def _extract_direct_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for input_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + if _unwrap_trig_value(graph, trig_value_id, expected=("cos", "scalar_cos")) is not None: + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotated_branch(graph: IRGraph, node: IRNode) -> dict[str, object] | None: + for rotated_value_id, trig_value_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + input_value_id = _extract_rotate_half_source(graph, rotated_value_id) + if input_value_id is None: + continue + if _unwrap_trig_value(graph, trig_value_id, expected=("sin", "scalar_sin")) is None: + continue + return {"input_value_id": input_value_id, "trig_value_id": trig_value_id, "node_ids": (node.id,)} + return None + + +def _extract_rotate_half_source(graph: IRGraph, value_id: str) -> str | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + neg_node = producer(graph, cat_node.inputs[0]) + right_slice = producer(graph, cat_node.inputs[1]) + if neg_node is None or right_slice is None: + return None + if neg_node.op != "scalar_multiply" or float(neg_node.attrs.get("value", 0.0)) != -1.0: + return None + left_slice = producer(graph, neg_node.inputs[0]) + if left_slice is None or left_slice.op != "slice" or right_slice.op != "slice": + return None + if strip_passthrough(graph, left_slice.inputs[0]) != strip_passthrough(graph, right_slice.inputs[0]): + return None + return left_slice.inputs[0] + + +def _extract_rope_trig(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: + trig = _unwrap_trig_value(graph, value_id, expected=expected) + if trig is None: + return None + trig_node = trig["trig_node"] + cat_node = producer(graph, trig_node.inputs[0]) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2 or cat_node.inputs[0] != cat_node.inputs[1]: + return None + angle_info = _extract_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + matmul_node = angle_info["matmul_node"] + inv_freq_const_id = None + arange_node = None + for input_id in matmul_node.inputs: + if inv_freq_const_id is None: + inv_freq_const_id = _find_constant_ancestor(graph, input_id) + if arange_node is None: + arange_node = _find_arange_ancestor(graph, input_id) + if inv_freq_const_id is None or arange_node is None: + return None + theta = _infer_rope_theta(graph.constants[inv_freq_const_id]) + if theta is None: + return None + return { + "theta": theta, + "position_offset": int(arange_node.attrs.get("start", 0)), + "node_ids": tuple(sorted({trig_node.id, cat_node.id, matmul_node.id})), + } + + +def _unwrap_trig_value(graph: IRGraph, value_id: str, *, expected: tuple[str, ...]) -> dict[str, object] | None: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return None + if node.op in {"unsqueeze", "reshape", "view", "expand", "precision_cast"}: + current = node.inputs[0] + continue + if node.op == "type_as": + current = node.inputs[0] + continue + if node.op == "scalar_multiply": + current = node.inputs[0] + continue + if node.op not in expected: + return None + return {"trig_node": node} + + +def _extract_angle_source(graph: IRGraph, value_id: str) -> dict[str, object] | None: + current = value_id + while True: + current = _strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op in {"transpose", "permute"}: + matmul_node = producer(graph, node.inputs[0]) + if matmul_node is None or matmul_node.op != "matmul": + return None + return {"matmul_node": matmul_node} + if node.op == "matmul": + return {"matmul_node": node} + if node.op in {"index", "slice"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + return None + + +def _find_constant_ancestor(graph: IRGraph, value_id: str) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = _strip_layout_passthrough(graph, current) + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return current + node = producer(graph, current) + if node is None or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _find_arange_ancestor(graph: IRGraph, value_id: str) -> IRNode | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = _strip_layout_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + if node.op == "arange": + return node + if current in graph.constants and isinstance(graph.constants[current], torch.Tensor): + return None + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _strip_layout_passthrough(graph: IRGraph, value_id: str) -> str: + current = value_id + while True: + node = producer(graph, current) + if node is None: + return current + if node.op in {"precision_cast", "contiguous", "reshape", "view", "unsqueeze", "expand"} and len(node.inputs) == 1: + current = node.inputs[0] + continue + if node.op == "type_as" and len(node.inputs) >= 1: + current = node.inputs[0] + continue + return current + + +def _infer_rope_theta(value: Any) -> float | None: + if not isinstance(value, torch.Tensor): + return None + flat = value.detach().cpu().float().reshape(-1) + if flat.numel() < 2: + return None + second = float(flat[1].item()) + if second <= 0.0: + return None + return float((1.0 / second) ** flat.numel()) diff --git a/python/src/transpile/importers.py b/python/src/transpile/importers.py new file mode 100644 index 000000000..79486201c --- /dev/null +++ b/python/src/transpile/importers.py @@ -0,0 +1,1858 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from dataclasses import field +from types import SimpleNamespace +from typing import Any + +import torch + +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.normalize import dtype_to_ir +from src.transpile.normalize import normalize_target + + +class UnsupportedImportError(NotImplementedError): + pass + + +@dataclass +class ImportContext: + strict: bool = True + alias_values: dict[str, str] = field(default_factory=dict) + tuple_aliases: dict[str, list[str]] = field(default_factory=dict) + local_alias_stack: list[dict[str, str]] = field(default_factory=list) + node_stack: list[str] = field(default_factory=list) + transpile_metadata: dict[str, object] = field(default_factory=dict) + + def push_local_aliases(self, aliases: dict[str, str]) -> None: + self.local_alias_stack.append(aliases) + + def pop_local_aliases(self) -> None: + if not self.local_alias_stack: + raise RuntimeError("local alias stack underflow") + self.local_alias_stack.pop() + + def resolve_value_id(self, node: Any) -> str: + if is_fx_node(node): + name = getattr(node, "name", "") + for aliases in reversed(self.local_alias_stack): + if name in aliases: + return aliases[name] + resolved = self.alias_values.get(name) + if resolved is not None: + return resolved + return f"v_{name}" + raise TypeError(f"cannot resolve value id for non-FX node: {type(node).__name__}") + + def fail(self, message: str) -> None: + if self.node_stack: + message = f"{message} [context: {' > '.join(self.node_stack)}]" + if self.strict: + raise UnsupportedImportError(message) + raise UnsupportedImportError(f"{message} [non-strict import has no generic fallback]") + + def push_node(self, description: str) -> None: + self.node_stack.append(description) + + def pop_node(self) -> None: + if self.node_stack: + self.node_stack.pop() + + def _node_module_paths(self, node: Any) -> tuple[str, ...]: + meta = getattr(node, "meta", {}) or {} + raw_stack = meta.get("nn_module_stack") + if not isinstance(raw_stack, dict): + return () + paths: list[str] = [] + for entry in raw_stack.values(): + if not isinstance(entry, (tuple, list)) or not entry: + continue + module_path = entry[0] + if isinstance(module_path, str) and module_path: + paths.append(module_path) + # Prefer the deepest module path first. + paths.reverse() + deduped: list[str] = [] + for path in paths: + if path not in deduped: + deduped.append(path) + return tuple(deduped) + + def lookup_import_hint(self, node: Any, *, torch_op: str, op_name: str) -> dict[str, object]: + hints = self.transpile_metadata.get("import_hints", ()) + if not isinstance(hints, list): + return {} + + module_paths = self._node_module_paths(node) + matched_attrs: dict[str, object] = {} + matched_meta: dict[str, object] = {} + matched_provider: list[str] = [] + + for hint in hints: + if not isinstance(hint, dict): + continue + hint_op = hint.get("op") + if hint_op is not None and hint_op != op_name: + continue + hint_torch_op = hint.get("torch_op") + if hint_torch_op is not None and hint_torch_op != torch_op: + continue + + module_path = hint.get("module_path") + if module_path is not None and module_path not in module_paths: + continue + module_path_suffix = hint.get("module_path_suffix") + if module_path_suffix is not None and not any( + path == module_path_suffix or path.endswith(f".{module_path_suffix}") for path in module_paths + ): + continue + + attrs = hint.get("attrs", {}) + if isinstance(attrs, dict): + matched_attrs.update(attrs) + meta = hint.get("meta", {}) + if isinstance(meta, dict): + matched_meta.update(meta) + provider = hint.get("provider") + if isinstance(provider, str) and provider not in matched_provider: + matched_provider.append(provider) + + if not matched_attrs and not matched_meta and not matched_provider: + return {} + + if matched_provider: + matched_meta.setdefault("import_hint_providers", tuple(matched_provider)) + matched_meta.setdefault("import_hint_source", "capture_metadata") + return {"attrs": matched_attrs, "meta": matched_meta} + + +def value_id(node: Any, ctx: ImportContext) -> str: + return ctx.resolve_value_id(node) + + +def node_id(node: Any) -> str: + return f"n_{node.name}" + + +def is_fx_node(value: Any) -> bool: + return hasattr(value, "name") and hasattr(value, "op") + + +def extract_input_ids(arg: Any, ctx: ImportContext) -> list[str]: + if is_fx_node(arg): + return [value_id(arg, ctx)] + if isinstance(arg, (tuple, list)): + ids: list[str] = [] + for item in arg: + ids.extend(extract_input_ids(item, ctx)) + return ids + if isinstance(arg, dict): + ids: list[str] = [] + for item in arg.values(): + ids.extend(extract_input_ids(item, ctx)) + return ids + return [] + + +def extract_literals(value: Any) -> Any: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return extract_literals(meta_value) + return None + if isinstance(value, torch.Size): + return tuple(extract_literals(v) for v in value) + if isinstance(value, (tuple, list)): + return type(value)(extract_literals(v) for v in value) + if isinstance(value, dict): + return {k: extract_literals(v) for k, v in value.items()} + if isinstance(value, torch.dtype): + return str(value) + if isinstance(value, (int, float, bool, str)) or value is None: + return value + return None + + +def make_prefixed_node(node: Any, prefix: str) -> Any: + return SimpleNamespace( + name=f"{prefix}{node.name}", + op=node.op, + target=node.target, + args=node.args, + kwargs=getattr(node, "kwargs", {}), + meta=getattr(node, "meta", {}), + ) + + +def add_value_if_missing(ir: IRGraph, value: IRValue) -> None: + if value.id not in ir.values: + ir.add_value(value) + + +def register_node( + ir: IRGraph, + ir_node: IRNode, + *, + shape: tuple[int, ...] | None, + dtype: str | None, +) -> None: + ir.add_node(ir_node) + ir.order.append(ir_node.id) + for output_id in ir_node.outputs: + ir.values[output_id].shape = shape + ir.values[output_id].dtype = dtype + for input_id in ir_node.inputs: + if input_id in ir.values: + ir.values[input_id].users.append(ir_node.id) + + +def _extract_numeric_literal(value: Any) -> float | int | None: + literal = extract_literals(value) + if isinstance(literal, bool): + return int(literal) + if isinstance(literal, (int, float)): + return literal + return None + + +def _extract_static_int(value: Any) -> int | None: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return _extract_static_int(meta_value) + return None + + literal = extract_literals(value) + if isinstance(literal, bool): + return int(literal) + if isinstance(literal, int): + return int(literal) + if isinstance(literal, float) and float(literal).is_integer(): + return int(literal) + + try: + return int(value) + except Exception: + return None + + +def _extract_int_sequence_literal(value: Any) -> tuple[int, ...] | None: + if is_fx_node(value): + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if meta_value is not None and not isinstance(meta_value, torch.Tensor): + return _extract_int_sequence_literal(meta_value) + return None + + if isinstance(value, torch.Size): + value = tuple(value) + + if isinstance(value, (tuple, list)): + result: list[int] = [] + for item in value: + item_value = _extract_static_int(item) + if item_value is None: + return None + result.append(item_value) + return tuple(result) + + scalar_value = _extract_static_int(value) + if scalar_value is not None: + return (scalar_value,) + return None + + +def _try_materialize_shape(shape: tuple[Any, ...] | None) -> tuple[int, ...] | None: + if shape is None: + return None + result: list[int] = [] + for dim in shape: + dim_value = _extract_static_int(dim) + if dim_value is None: + return None + result.append(dim_value) + return tuple(result) + + +def _extract_fx_tensor_shape(value: Any) -> tuple[Any, ...] | None: + if not is_fx_node(value): + return None + meta = getattr(value, "meta", {}) or {} + meta_value = meta.get("val") + if isinstance(meta_value, torch.Tensor): + return tuple(meta_value.shape) + tensor_meta = meta.get("tensor_meta") + tensor_shape = getattr(tensor_meta, "shape", None) + if tensor_shape is not None: + return tuple(tensor_shape) + return None + + +def _base_meta(shape: tuple[int, ...] | None, dtype: str | None, torch_op: str, node: Any) -> dict[str, object]: + meta = { + "shape": shape, + "dtype": dtype, + "torch_op": torch_op, + "torch_name": node.name, + } + raw_meta = getattr(node, "meta", {}) or {} + raw_stack = raw_meta.get("nn_module_stack") + if isinstance(raw_stack, dict): + module_paths: list[str] = [] + for entry in raw_stack.values(): + if not isinstance(entry, (tuple, list)) or not entry: + continue + module_path = entry[0] + if isinstance(module_path, str) and module_path and module_path not in module_paths: + module_paths.append(module_path) + if module_paths: + meta["module_paths"] = tuple(module_paths) + return meta + + +def _extract_fx_torch_dtype(node: Any) -> torch.dtype | None: + if not is_fx_node(node): + return None + meta = getattr(node, "meta", {}) or {} + value = meta.get("val") + dtype = getattr(value, "dtype", None) + if isinstance(dtype, torch.dtype): + return dtype + tensor_meta = meta.get("tensor_meta") + dtype = getattr(tensor_meta, "dtype", None) + if isinstance(dtype, torch.dtype): + return dtype + return None + + +def _import_scalar_binary( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, + op_name: str, +) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if op_name == "add": + if lhs_is_node and rhs_literal is not None: + scalar_op = "scalar_add" + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + scalar_op = "scalar_add" + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + scalar_op = None + elif op_name == "multiply": + if lhs_is_node and rhs_literal is not None: + scalar_op = "scalar_multiply" + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + scalar_op = "scalar_multiply" + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + scalar_op = None + elif op_name == "subtract": + if lhs_is_node and rhs_literal is not None: + scalar_op = "scalar_subtract" + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + scalar_op = "scalar_subtract_reverse" + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + scalar_op = None + elif op_name == "divide": + if lhs_is_node and rhs_literal is not None: + scalar_op = "scalar_divide" + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + scalar_op = "scalar_divide_reverse" + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + scalar_op = None + else: + scalar_op = None + + if scalar_op is None: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op=scalar_op, + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_placeholder( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, +) -> None: + add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) + ir.inputs.append(value_id(node, ctx)) + + +def import_get_attr( + ir: IRGraph, + node: Any, + ctx: ImportContext, + value: Any, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + source_name: str | None = None, +) -> None: + add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) + ir.constants[value_id(node, ctx)] = value.detach().cpu() if isinstance(value, torch.Tensor) else value + ir.values[value_id(node, ctx)].meta["source_name"] = source_name + if source_name is not None: + bindings = ir.meta.setdefault("weight_bindings", {}) + if isinstance(bindings, dict): + bindings.setdefault(value_id(node, ctx), {})["source_name"] = source_name + + +def import_output(ir: IRGraph, node: Any, ctx: ImportContext) -> None: + ir.outputs.extend(extract_input_ids(node.args[0], ctx)) + + +def import_call_function( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + op = normalize_target(torch_op) + import_hint = ctx.lookup_import_hint(node, torch_op=torch_op, op_name=op) + importer = OP_IMPORTERS.get(op) + if importer is None: + import_opaque_call_function(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name=op) + else: + importer(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op) + _apply_import_hint(ir, node, import_hint) + + +def _apply_import_hint(ir: IRGraph, node: Any, hint: dict[str, object]) -> None: + if not hint: + return + ir_node = ir.nodes.get(node_id(node)) + if ir_node is None: + return + attrs = hint.get("attrs", {}) + if isinstance(attrs, dict): + ir_node.attrs.update(attrs) + meta = hint.get("meta", {}) + if isinstance(meta, dict): + ir_node.meta.update(meta) + + +def import_opaque_call_function( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, + op_name: str, +) -> None: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=extract_input_ids(getattr(node, "args", ()), ctx) + extract_input_ids(getattr(node, "kwargs", {}), ctx), + outputs=[value_id(node, ctx)], + attrs={ + "opaque": True, + "torch_op": torch_op, + "args": extract_literals(getattr(node, "args", ())), + "kwargs": extract_literals(getattr(node, "kwargs", {})), + }, + meta=_base_meta(shape, dtype, torch_op, node), + kind="opaque", + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_add(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="add") + + +def import_subtract(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="subtract") + + +def import_multiply(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") + + +def import_multiply_inplace(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="multiply") + + +def import_divide(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + _import_scalar_binary(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op, op_name="divide") + + +def import_not_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op="not_equal", + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if lhs_is_node and rhs_literal is not None: + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op="scalar_not_equal", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_equal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + lhs, rhs = node.args[0], node.args[1] + lhs_is_node = is_fx_node(lhs) + rhs_is_node = is_fx_node(rhs) + lhs_literal = _extract_numeric_literal(lhs) + rhs_literal = _extract_numeric_literal(rhs) + + if lhs_is_node and rhs_is_node: + ir_node = IRNode( + id=node_id(node), + op="equal", + inputs=[value_id(lhs, ctx), value_id(rhs, ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return + + if lhs_is_node and rhs_literal is not None: + inputs = [value_id(lhs, ctx)] + attrs = {"value": rhs_literal} + elif rhs_is_node and lhs_literal is not None: + inputs = [value_id(rhs, ctx)] + attrs = {"value": lhs_literal} + else: + ctx.fail(f"unsupported scalar form for {torch_op}: {node.args!r}") + + ir_node = IRNode( + id=node_id(node), + op="scalar_equal", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_binary(op_name: str): + def _importer( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, + ) -> None: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + return _importer + + +def import_where( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + condition = node.args[0] + true_value = node.args[1] + false_value = node.args[2] + + inputs = [value_id(condition, ctx)] + attrs: dict[str, object] = {} + + if is_fx_node(true_value): + inputs.append(value_id(true_value, ctx)) + attrs["true_is_scalar"] = False + else: + literal = _extract_numeric_literal(true_value) + if literal is None: + ctx.fail(f"unsupported true branch for {torch_op}: {node.args!r}") + attrs["true_is_scalar"] = True + attrs["true_value"] = float(literal) + + if is_fx_node(false_value): + inputs.append(value_id(false_value, ctx)) + attrs["false_is_scalar"] = False + else: + literal = _extract_numeric_literal(false_value) + if literal is None: + ctx.fail(f"unsupported false branch for {torch_op}: {node.args!r}") + attrs["false_is_scalar"] = True + attrs["false_value"] = float(literal) + + ir_node = IRNode( + id=node_id(node), + op="where", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_negate(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="scalar_multiply", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": -1.0}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_precision_cast(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + target_dtype = extract_literals(node.args[1]) if len(node.args) > 1 else None + ir_node = IRNode( + id=node_id(node), + op="precision_cast", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"dtype": target_dtype}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_arange(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + attrs: dict[str, object] = {} + if len(node.args) == 1: + attrs["start"] = 0 + attrs["end"] = int(node.args[0]) + elif len(node.args) >= 2: + attrs["start"] = int(node.args[0]) + attrs["end"] = int(node.args[1]) + else: + ctx.fail(f"unsupported arange signature for {torch_op}: {node.args!r}") + + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + attrs["step"] = int(node.args[2]) + + target_dtype = None + if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None: + target_dtype = extract_literals(node.kwargs["dtype"]) + elif dtype is not None: + target_dtype = dtype + if target_dtype is not None: + attrs["dtype"] = target_dtype + + ir_node = IRNode( + id=node_id(node), + op="arange", + inputs=[], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_type_as(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="type_as", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_identity(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="identity", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_reshape(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + target_shape = _extract_int_sequence_literal(node.args[1]) + if target_shape is None: + ctx.fail(f"unsupported reshape shape for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="reshape", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"shape": target_shape}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_flatten(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="flatten", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"start_dim": int(node.args[1]), "end_dim": int(node.args[2])}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_unsqueeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + dim = int(node.args[1]) + attrs: dict[str, object] = {"dim": dim} + materialized_shape = _try_materialize_shape(shape) + if materialized_shape is not None: + attrs["shape"] = materialized_shape + ir_node = IRNode( + id=node_id(node), + op="unsqueeze", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_squeeze(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + dim = int(node.args[1]) if len(node.args) > 1 and node.args[1] is not None else 0 + attrs: dict[str, object] = {"dim": dim} + materialized_shape = _try_materialize_shape(shape) + if materialized_shape is not None: + attrs["shape"] = materialized_shape + ir_node = IRNode( + id=node_id(node), + op="squeeze", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_masked_scatter( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + ir_node = IRNode( + id=node_id(node), + op="masked_scatter", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_masked_fill( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + value = _extract_numeric_literal(node.args[2]) + if value is None: + ctx.fail(f"unsupported masked_fill value for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="masked_fill", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": float(value)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_expand(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + target_shape = _extract_int_sequence_literal(node.args[1]) + if target_shape is None and shape is not None: + target_shape = _extract_int_sequence_literal(shape) + if target_shape is None: + ctx.fail(f"unsupported expand shape for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="expand", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"shape": target_shape}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_one_hot(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + num_classes = _extract_static_int(node.args[1]) if len(node.args) > 1 else None + if (num_classes is None or num_classes <= 0) and shape is not None and shape: + num_classes = int(shape[-1]) + if num_classes is None or num_classes <= 0: + ctx.fail(f"unsupported one_hot num_classes for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="one_hot", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"num_classes": int(num_classes)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_tril(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + diagonal = int(extract_literals(node.args[1])) if len(node.args) > 1 and extract_literals(node.args[1]) is not None else 0 + ir_node = IRNode( + id=node_id(node), + op="tril", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"diagonal": diagonal}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_unfold(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 4: + ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") + dimension = extract_literals(node.args[1]) + size = extract_literals(node.args[2]) + step = extract_literals(node.args[3]) + if not isinstance(dimension, int) or not isinstance(size, int) or not isinstance(step, int): + ctx.fail(f"unsupported unfold args for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="unfold", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"dimension": dimension, "size": size, "step": step}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_transpose(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + attrs = {"dim0": 0, "dim1": 1} if torch_op == "aten.t.default" else {"dim0": int(node.args[1]), "dim1": int(node.args[2])} + ir_node = IRNode( + id=node_id(node), + op="transpose", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_permute(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + permutation = _extract_int_sequence_literal(node.args[1]) + if permutation is None: + ctx.fail(f"unsupported permute dims for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="permute", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"permutation": permutation}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_numpy_t(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + input_shape = _extract_fx_tensor_shape(node.args[0]) + rank_source = input_shape if input_shape is not None else shape + if rank_source is None: + ctx.fail(f"unsupported numpy_T rank inference for {torch_op}: {node.args!r}") + permutation = tuple(range(len(rank_source) - 1, -1, -1)) + ir_node = IRNode( + id=node_id(node), + op="permute", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"permutation": permutation}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_movedim(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + source = _extract_int_sequence_literal(node.args[1]) + destination = _extract_int_sequence_literal(node.args[2]) + if source is None or destination is None: + ctx.fail(f"unsupported movedim dims for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="movedim", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"source": source, "destination": destination}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_matmul(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="matmul", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_linear(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + attrs = {"has_bias": False} + if len(node.args) > 2 and node.args[2] is not None: + inputs.append(value_id(node.args[2], ctx)) + attrs["has_bias"] = True + ir_node = IRNode( + id=node_id(node), + op="linear", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_addmm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="addmm", + inputs=[value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_unary(op_name: str): + def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return _importer + + +def import_clamp(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + min_value = extract_literals(node.args[1]) if len(node.args) > 1 else None + max_value = extract_literals(node.args[2]) if len(node.args) > 2 else None + attrs: dict[str, object] = {} + if isinstance(min_value, (int, float)): + attrs["min"] = float(min_value) + if isinstance(max_value, (int, float)): + attrs["max"] = float(max_value) + ir_node = IRNode( + id=node_id(node), + op="clamp", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_pow_rsqrt(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="pow", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"exponent": -0.5}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_reciprocal(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="pow", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"exponent": -1.0}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_floor_divide( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported floor_divide signature for {torch_op}: {node.args!r}") + divisor = _extract_numeric_literal(node.args[1]) + if divisor is None: + ctx.fail(f"unsupported floor_divide divisor for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="scalar_floor_divide", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"value": float(divisor)}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_softmax(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="softmax", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"axis": int(node.args[1])}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_glu(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + axis = -1 + if len(node.args) > 1 and extract_literals(node.args[1]) is not None: + axis = int(node.args[1]) + ir_node = IRNode( + id=node_id(node), + op="glu", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_scaled_dot_product_attention( + ir: IRGraph, + node: Any, + ctx: ImportContext, + *, + shape: tuple[int, ...] | None, + dtype: str | None, + torch_op: str, +) -> None: + attrs: dict[str, object] = {} + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx), value_id(node.args[2], ctx)] + if len(node.args) > 3: + mask_literal = extract_literals(node.args[3]) + if mask_literal is not None: + attrs["mask"] = mask_literal + elif is_fx_node(node.args[3]): + inputs.append(value_id(node.args[3], ctx)) + mask_dtype = _extract_fx_torch_dtype(node.args[3]) + if mask_dtype is not None and mask_dtype != torch.bool: + attrs["additive_mask"] = True + if len(node.args) > 4 and extract_literals(node.args[4]) is not None: + attrs["dropout_p"] = float(node.args[4]) + if len(node.args) > 5 and extract_literals(node.args[5]) is not None: + attrs["is_causal"] = bool(node.args[5]) + if "scale" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["scale"]) is not None: + attrs["scale"] = float(node.kwargs["scale"]) + if "enable_gqa" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["enable_gqa"]) is not None: + attrs["enable_gqa"] = bool(node.kwargs["enable_gqa"]) + ir_node = IRNode( + id=node_id(node), + op="scaled_dot_product_attention", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_reduce(op_name: str): + def _importer(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + attrs: dict[str, object] = {} + if len(node.args) > 1 and extract_literals(node.args[1]) is not None: + attrs["axis"] = extract_literals(node.args[1]) + keepdim = None + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + keepdim = bool(extract_literals(node.args[2])) + elif "keepdim" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["keepdim"]) is not None: + keepdim = bool(extract_literals(node.kwargs["keepdim"])) + if keepdim is not None: + attrs["keepdim"] = keepdim + ir_node = IRNode( + id=node_id(node), + op=op_name, + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + return _importer + + +def import_cat(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + tensors_arg = node.args[0] + axis = int(node.args[1]) if len(node.args) > 1 else 0 + ir_node = IRNode( + id=node_id(node), + op="cat", + inputs=extract_input_ids(tensors_arg, ctx), + outputs=[value_id(node, ctx)], + attrs={"axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_split_with_sizes(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported split_with_sizes signature for {torch_op}: {node.args!r}") + sizes = _extract_int_sequence_literal(node.args[1]) + if sizes is None: + ctx.fail(f"unsupported split sizes for {torch_op}: {node.args!r}") + axis = -1 + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + axis = int(extract_literals(node.args[2])) + ir_node = IRNode( + id=node_id(node), + op="split_with_sizes", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"sizes": sizes, "axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_chunk(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported chunk signature for {torch_op}: {node.args!r}") + chunks = extract_literals(node.args[1]) + if not isinstance(chunks, int): + ctx.fail(f"unsupported chunk count for {torch_op}: {node.args!r}") + axis = 0 + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + axis = int(extract_literals(node.args[2])) + ir_node = IRNode( + id=node_id(node), + op="chunk", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"chunks": int(chunks), "axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_pow(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + base_literal = _extract_numeric_literal(node.args[0]) if len(node.args) > 0 else None + attrs: dict[str, object] = {} + exponent_literal = extract_literals(node.args[1]) if len(node.args) > 1 else None + + if base_literal is not None and isinstance(exponent_literal, (int, float)): + add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None)) + ir.constants[value_id(node, ctx)] = base_literal**exponent_literal + return + + if base_literal is not None: + if not (len(node.args) > 1 and is_fx_node(node.args[1])): + ctx.fail(f"unsupported pow signature for {torch_op}: scalar base with non-literal exponent {node.args!r}") + if float(base_literal) <= 0.0: + ctx.fail(f"unsupported pow signature for {torch_op}: non-positive scalar base {base_literal!r}") + + exponent_value_id = value_id(node.args[1], ctx) + mul_output_id = f"{value_id(node, ctx)}__pow_scalar_base_log_mul" + mul_node = IRNode( + id=f"{node_id(node)}__pow_scalar_base_log_mul", + op="scalar_multiply", + inputs=[exponent_value_id], + outputs=[mul_output_id], + attrs={"value": float(math.log(float(base_literal)))}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, mul_node, shape=shape, dtype=dtype) + + exp_node = IRNode( + id=node_id(node), + op="scalar_exp", + inputs=[mul_output_id], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, exp_node, shape=shape, dtype=dtype) + return + + inputs = [value_id(node.args[0], ctx)] + if exponent_literal is None and len(node.args) > 1 and is_fx_node(node.args[1]): + inputs.append(value_id(node.args[1], ctx)) + else: + attrs["exponent"] = exponent_literal + ir_node = IRNode( + id=node_id(node), + op="pow", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_slice(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + attrs = {"axis": int(node.args[1]), "start": int(node.args[2]), "end": int(node.args[3])} + if len(node.args) > 4 and extract_literals(node.args[4]) is not None: + attrs["step"] = int(node.args[4]) + ir_node = IRNode( + id=node_id(node), + op="slice", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = list(getattr(node, "args", ())) + if not inputs: + ctx.fail(f"unsupported diff signature for {torch_op}: missing input tensor") + + source = inputs[0] + if not is_fx_node(source): + ctx.fail(f"unsupported diff signature for {torch_op}: source is not FX node") + + n_value = 1 + if len(inputs) > 1 and extract_literals(inputs[1]) is not None: + n_value = int(extract_literals(inputs[1])) + elif "n" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["n"]) is not None: + n_value = int(extract_literals(node.kwargs["n"])) + if n_value != 1: + ctx.fail(f"unsupported diff order for {torch_op}: n={n_value}") + + dim_value = None + if len(inputs) > 2 and extract_literals(inputs[2]) is not None: + dim_value = int(extract_literals(inputs[2])) + elif "dim" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dim"]) is not None: + dim_value = int(extract_literals(node.kwargs["dim"])) + if dim_value is None: + ctx.fail(f"unsupported diff signature for {torch_op}: missing integer dim") + + prepend_value = None + append_value = None + prepend_node = None + append_node = None + if len(inputs) > 3: + if is_fx_node(inputs[3]): + prepend_node = inputs[3] + else: + prepend_value = extract_literals(inputs[3]) + if len(inputs) > 4: + if is_fx_node(inputs[4]): + append_node = inputs[4] + else: + append_value = extract_literals(inputs[4]) + if "prepend" in getattr(node, "kwargs", {}): + prepend_arg = node.kwargs["prepend"] + if is_fx_node(prepend_arg): + prepend_node = prepend_arg + prepend_value = None + else: + prepend_value = extract_literals(prepend_arg) + if "append" in getattr(node, "kwargs", {}): + append_arg = node.kwargs["append"] + if is_fx_node(append_arg): + append_node = append_arg + append_value = None + else: + append_value = extract_literals(append_arg) + if prepend_value is not None or append_value is not None: + ctx.fail(f"unsupported diff signature for {torch_op}: literal prepend/append are not supported") + + source_value_id = value_id(source, ctx) + source_shape = None + if source_value_id in ir.values: + source_shape = ir.values[source_value_id].shape + if source_shape is None: + source_shape = shape + if source_shape is None: + ctx.fail(f"unsupported diff import for {torch_op}: missing source shape") + + rank = len(source_shape) + normalized_dim = dim_value if dim_value >= 0 else dim_value + rank + if normalized_dim < 0 or normalized_dim >= rank: + ctx.fail(f"unsupported diff dimension for {torch_op}: dim={dim_value} rank={rank}") + + dim_extent = int(source_shape[normalized_dim]) + if dim_extent < 1: + ctx.fail(f"unsupported diff input extent for {torch_op}: dim size {dim_extent}") + + concat_inputs = [source_value_id] + concat_shapes = [tuple(int(v) for v in source_shape)] + + def _append_concat_input(arg: Any, label: str) -> None: + nonlocal concat_inputs, concat_shapes + if arg is None: + return + arg_value_id = value_id(arg, ctx) + arg_value = ir.values.get(arg_value_id) + arg_shape = arg_value.shape if arg_value is not None else None + if arg_shape is None: + ctx.fail(f"unsupported diff import for {torch_op}: missing {label} shape") + arg_shape = tuple(int(v) for v in arg_shape) + if len(arg_shape) != rank: + ctx.fail( + f"unsupported diff import for {torch_op}: {label} rank {len(arg_shape)} does not match source rank {rank}" + ) + for axis, (arg_dim, src_dim) in enumerate(zip(arg_shape, source_shape, strict=True)): + if axis == normalized_dim: + continue + if int(arg_dim) != int(src_dim): + ctx.fail( + f"unsupported diff import for {torch_op}: {label} shape {arg_shape} " + f"is incompatible with source shape {source_shape}" + ) + concat_inputs.append(arg_value_id) + concat_shapes.append(arg_shape) + + if prepend_node is not None: + concat_inputs = [] + concat_shapes = [] + _append_concat_input(prepend_node, "prepend") + concat_inputs.append(source_value_id) + concat_shapes.append(tuple(int(v) for v in source_shape)) + if append_node is not None: + _append_concat_input(append_node, "append") + elif append_node is not None: + _append_concat_input(append_node, "append") + + augmented_shape = list(source_shape) + augmented_shape[normalized_dim] = sum(int(shape_item[normalized_dim]) for shape_item in concat_shapes) + augmented_shape_tuple = tuple(int(v) for v in augmented_shape) + diff_shape = list(augmented_shape) + diff_shape[normalized_dim] = max(0, int(augmented_shape[normalized_dim]) - 1) + diff_shape_tuple = tuple(int(v) for v in diff_shape) + + diff_source_value_id = source_value_id + if len(concat_inputs) > 1: + concat_output_id = f"{value_id(node, ctx)}__diff_source" + concat_node = IRNode( + id=f"{node_id(node)}__diff_source", + op="cat", + inputs=concat_inputs, + outputs=[concat_output_id], + attrs={"axis": normalized_dim}, + meta=_base_meta(augmented_shape_tuple, dtype, torch_op, node), + ) + register_node(ir, concat_node, shape=augmented_shape_tuple, dtype=dtype) + diff_source_value_id = concat_output_id + + left_shape = list(diff_shape_tuple) + left_shape_tuple = tuple(int(v) for v in left_shape) + + left_node = IRNode( + id=f"{node_id(node)}__diff_left", + op="slice", + inputs=[diff_source_value_id], + outputs=[f"{value_id(node, ctx)}__diff_left"], + attrs={"axis": normalized_dim, "start": 0, "end": int(augmented_shape[normalized_dim]) - 1, "step": 1}, + meta=_base_meta(left_shape_tuple, dtype, torch_op, node), + ) + register_node(ir, left_node, shape=left_shape_tuple, dtype=dtype) + + right_node = IRNode( + id=f"{node_id(node)}__diff_right", + op="slice", + inputs=[diff_source_value_id], + outputs=[f"{value_id(node, ctx)}__diff_right"], + attrs={"axis": normalized_dim, "start": 1, "end": int(augmented_shape[normalized_dim]), "step": 1}, + meta=_base_meta(left_shape_tuple, dtype, torch_op, node), + ) + register_node(ir, right_node, shape=left_shape_tuple, dtype=dtype) + + diff_node = IRNode( + id=node_id(node), + op="subtract", + inputs=[right_node.outputs[0], left_node.outputs[0]], + outputs=[value_id(node, ctx)], + meta=_base_meta(diff_shape_tuple, dtype, torch_op, node), + ) + register_node(ir, diff_node, shape=diff_shape_tuple, dtype=dtype) + + +def import_index(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if torch_op.startswith("aten.select"): + axis = int(node.args[1]) + index_value = int(node.args[2]) + else: + index_value = int(node.args[1]) + axis = int(node.args[2]) if len(node.args) > 2 else 0 + ir_node = IRNode( + id=node_id(node), + op="index", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"index_value": index_value, "axis": axis}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_gather(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="gather", + inputs=[value_id(node.args[0], ctx), value_id(node.args[2], ctx)], + outputs=[value_id(node, ctx)], + attrs={"axis": int(node.args[1])}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_embedding(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + attrs: dict[str, object] = {} + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + attrs["padding_idx"] = int(node.args[2]) + if len(node.args) > 3 and extract_literals(node.args[3]) is not None: + attrs["scale_grad_by_freq"] = bool(node.args[3]) + if len(node.args) > 4 and extract_literals(node.args[4]) is not None: + attrs["sparse"] = bool(node.args[4]) + ir_node = IRNode( + id=node_id(node), + op="embedding", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def _extract_int_list_or_scalar(value: Any, *, default: int) -> int: + literal = extract_literals(value) + if literal is None: + return default + if isinstance(literal, int): + return int(literal) + if isinstance(literal, (tuple, list)) and literal: + first = literal[0] + if isinstance(first, int): + return int(first) + raise UnsupportedImportError(f"unsupported integer/list literal: {value!r}") + + +def import_conv1d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported conv1d signature for {torch_op}: {node.args!r}") + + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + if len(node.args) > 2 and is_fx_node(node.args[2]): + inputs.append(value_id(node.args[2], ctx)) + + stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 + padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 + dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 + groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 + + ir_node = IRNode( + id=node_id(node), + op="conv1d", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_conv2d(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported conv2d signature for {torch_op}: {node.args!r}") + + inputs = [value_id(node.args[0], ctx), value_id(node.args[1], ctx)] + if len(node.args) > 2 and is_fx_node(node.args[2]): + inputs.append(value_id(node.args[2], ctx)) + + stride = _extract_int_list_or_scalar(node.args[3], default=1) if len(node.args) > 3 else 1 + padding = _extract_int_list_or_scalar(node.args[4], default=0) if len(node.args) > 4 else 0 + dilation = _extract_int_list_or_scalar(node.args[5], default=1) if len(node.args) > 5 else 1 + groups = _extract_int_list_or_scalar(node.args[6], default=1) if len(node.args) > 6 else 1 + + ir_node = IRNode( + id=node_id(node), + op="conv2d", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_pad(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if len(node.args) < 2: + ctx.fail(f"unsupported pad signature for {torch_op}: {node.args!r}") + + pads = extract_literals(node.args[1]) + if not isinstance(pads, (tuple, list)) or not all(isinstance(v, int) for v in pads): + ctx.fail(f"unsupported pads for {torch_op}: {node.args!r}") + + mode = "constant" + if len(node.args) > 2 and extract_literals(node.args[2]) is not None: + mode = str(extract_literals(node.args[2])) + if "mode" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["mode"]) is not None: + mode = str(extract_literals(node.kwargs["mode"])) + + value = 0.0 + if len(node.args) > 3 and extract_literals(node.args[3]) is not None: + value = float(extract_literals(node.args[3])) + if "value" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["value"]) is not None: + value = float(extract_literals(node.kwargs["value"])) + + ir_node = IRNode( + id=node_id(node), + op="pad", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"pads": tuple(int(v) for v in pads), "mode": mode, "value": value}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if not node.args: + ctx.fail(f"unsupported ones signature for {torch_op}: {node.args!r}") + + output_shape = _extract_int_sequence_literal(node.args[0]) + if output_shape is None: + ctx.fail(f"unsupported ones shape for {torch_op}: {node.args!r}") + + node_dtype = dtype + if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None: + node_dtype = dtype_to_ir(extract_literals(node.kwargs["dtype"])) + elif len(node.args) > 1 and extract_literals(node.args[1]) is not None: + node_dtype = dtype_to_ir(extract_literals(node.args[1])) + if node_dtype is None: + node_dtype = "fp32" + + ir_node = IRNode( + id=node_id(node), + op="ones", + inputs=[], + outputs=[value_id(node, ctx)], + attrs={"shape": output_shape, "dtype": node_dtype}, + meta=_base_meta(shape or output_shape, node_dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape or output_shape, dtype=node_dtype) + + +def import_layer_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + if not node.args: + ctx.fail(f"unsupported layer_norm signature for {torch_op}: {node.args!r}") + + normalized_shape_value = None + if len(node.args) > 1: + normalized_shape_value = _extract_int_sequence_literal(node.args[1]) + if normalized_shape_value is None: + normalized_shape_value = _extract_int_sequence_literal(getattr(node, "kwargs", {}).get("normalized_shape")) + if normalized_shape_value is None: + ctx.fail(f"unsupported normalized_shape for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + + eps_value = None + if len(node.args) > 4: + eps_value = _extract_numeric_literal(node.args[4]) + if eps_value is None: + eps_value = _extract_numeric_literal(getattr(node, "kwargs", {}).get("eps")) + if eps_value is None: + eps_value = 1e-5 + + inputs = [value_id(node.args[0], ctx)] + weight = node.args[2] if len(node.args) > 2 else getattr(node, "kwargs", {}).get("weight") + bias = node.args[3] if len(node.args) > 3 else getattr(node, "kwargs", {}).get("bias") + if not is_fx_node(weight): + ctx.fail(f"unsupported layer_norm weight form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + inputs.append(value_id(weight, ctx)) + if bias is None: + pass + elif is_fx_node(bias): + inputs.append(value_id(bias, ctx)) + else: + ctx.fail(f"unsupported layer_norm bias form for {torch_op}: {node.args!r} {getattr(node, 'kwargs', {})!r}") + + attrs = {"normalized_shape": tuple(int(v) for v in normalized_shape_value), "eps": float(eps_value)} + ir_node = IRNode( + id=node_id(node), + op="layer_norm", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_rms_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx)] + attrs = {"normalized_shape": tuple(int(v) for v in node.args[1]), "eps": float(node.args[3])} + ir_node = IRNode( + id=node_id(node), + op="rms_norm", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_group_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [value_id(node.args[0], ctx), value_id(node.args[2], ctx), value_id(node.args[3], ctx)] + attrs = {"num_groups": int(node.args[1]), "eps": float(node.args[4])} + ir_node = IRNode( + id=node_id(node), + op="group_norm", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_batch_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + inputs = [ + value_id(node.args[0], ctx), + value_id(node.args[1], ctx), + value_id(node.args[2], ctx), + value_id(node.args[3], ctx), + value_id(node.args[4], ctx), + ] + attrs = {"training": bool(node.args[5]), "momentum": float(node.args[6]), "eps": float(node.args[7])} + ir_node = IRNode( + id=node_id(node), + op="batch_norm", + inputs=inputs, + outputs=[value_id(node, ctx)], + attrs=attrs, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_contiguous(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + ir_node = IRNode( + id=node_id(node), + op="contiguous", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None: + index_value = extract_literals(node.args[1]) + if not isinstance(index_value, int): + ctx.fail(f"unsupported getitem index for {torch_op}: {node.args!r}") + ir_node = IRNode( + id=node_id(node), + op="getitem", + inputs=[value_id(node.args[0], ctx)], + outputs=[value_id(node, ctx)], + attrs={"index": index_value}, + meta=_base_meta(shape, dtype, torch_op, node), + ) + register_node(ir, ir_node, shape=shape, dtype=dtype) + + +OP_IMPORTERS = { + "arange": import_arange, + "identity": import_identity, + "add": import_add, + "subtract": import_subtract, + "multiply": import_multiply, + "multiply_inplace": import_multiply_inplace, + "divide": import_divide, + "precision_cast": import_precision_cast, + "type_as": import_type_as, + "negate": import_negate, + "abs": import_unary("abs"), + "clamp": import_clamp, + "not_equal": import_not_equal, + "equal": import_equal, + "logical_and": import_binary("logical_and"), + "logical_or": import_binary("logical_or"), + "logical_not": import_unary("logical_not"), + "where": import_where, + "masked_scatter": import_masked_scatter, + "masked_fill": import_masked_fill, + "cos": import_unary("cos"), + "sin": import_unary("sin"), + "floor_divide": import_floor_divide, + "scalar_exp": import_unary("scalar_exp"), + "scalar_sqrt": import_unary("scalar_sqrt"), + "reciprocal": import_reciprocal, + "scalar_log": import_unary("scalar_log"), + "rsqrt": import_pow_rsqrt, + "reshape": import_reshape, + "flatten": import_flatten, + "unsqueeze": import_unsqueeze, + "squeeze": import_squeeze, + "expand": import_expand, + "one_hot": import_one_hot, + "tril": import_tril, + "unfold": import_unfold, + "numpy_T": import_numpy_t, + "transpose": import_transpose, + "permute": import_permute, + "movedim": import_movedim, + "matmul": import_matmul, + "linear": import_linear, + "addmm": import_addmm, + "relu": import_unary("relu"), + "silu": import_unary("silu"), + "gelu": import_unary("gelu"), + "gelu_erf": import_unary("gelu_erf"), + "sigmoid": import_unary("sigmoid"), + "glu": import_glu, + "softplus": import_unary("softplus"), + "tanh": import_unary("tanh"), + "softmax": import_softmax, + "scaled_dot_product_attention": import_scaled_dot_product_attention, + "sum": import_reduce("sum"), + "mean": import_reduce("mean"), + "variance": import_reduce("variance"), + "min": import_reduce("min"), + "max": import_reduce("max"), + "cat": import_cat, + "split_with_sizes": import_split_with_sizes, + "chunk": import_chunk, + "ones": import_ones, + "pad": import_pad, + "pow": import_pow, + "aten.diff.default": import_diff, + "slice": import_slice, + "index": import_index, + "gather": import_gather, + "embedding": import_embedding, + "conv1d": import_conv1d, + "conv2d": import_conv2d, + "layer_norm": import_layer_norm, + "rms_norm": import_rms_norm, + "group_norm": import_group_norm, + "batch_norm": import_batch_norm, + "contiguous": import_contiguous, + "getitem": import_getitem, +} diff --git a/python/src/transpile/lower.py b/python/src/transpile/lower.py new file mode 100644 index 000000000..8f20818b5 --- /dev/null +++ b/python/src/transpile/lower.py @@ -0,0 +1,2424 @@ +from __future__ import annotations + +from dataclasses import dataclass +import math +import os +from typing import Any + +import torch + +from src.graph import Graph +from src.graph import Tensor +from src.transpile.capture_pytorch import CapturedModel +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.optimize_graph import optimize_graph +from src.transpile.weight_binding import WeightBinding + + +@dataclass +class TranspiledGraph: + graph: Graph + runtime_inputs: list[Tensor] + bound_constants: list[Tensor] + bound_constant_bindings: list[dict[str, object]] + outputs: list[Tensor] + + def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: + if index < 0 or index >= len(self.runtime_inputs): + raise IndexError( + f"runtime input index out of range: {index} (have {len(self.runtime_inputs)})" + ) + self.graph.set_input(self.runtime_inputs[index], data, dtype=dtype) + + def set_inputs(self, inputs: list[Any] | tuple[Any, ...]) -> None: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"expected {len(self.runtime_inputs)} runtime inputs, got {len(inputs)}" + ) + for index, value in enumerate(inputs): + self.set_input(index, value) + + def execute(self) -> list[Tensor]: + self.graph.execute() + return self.outputs + + +@dataclass +class BroadcastAlias: + tensor: Tensor + logical_shape: tuple[int, ...] + kind: str + + +def transpile_captured(captured: CapturedModel) -> TranspiledGraph: + return transpile_ir(captured.ir_graph) + + +def transpile_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + canonicalize_exported_graph(ir) + optimize_graph(ir) + return transpile_preoptimized_ir(ir) + + +def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + g = Graph() + g._transpile_materialized_constants = [] # type: ignore[attr-defined] + env: dict[str, Any] = {} + runtime_inputs: list[Tensor] = [] + bound_constants: list[Tensor] = [] + bound_constant_bindings: list[dict[str, object]] = [] + + for value_id in ir.inputs: + value = ir.values[value_id] + tensor = _lower_input_value(g, value) + env[value_id] = tensor + runtime_inputs.append(tensor) + + for value_id, const in ir.constants.items(): + value = ir.values[value_id] + binding = _lookup_weight_binding(value) + lowered_const = _lower_constant_value(g, value, const) + env[value_id] = lowered_const + if isinstance(lowered_const, Tensor): + bound_constants.append(lowered_const) + if binding is not None: + bound_constant_bindings.append( + { + "node_id": int(lowered_const.id), + "value_id": str(value_id), + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + ) + + for node_id in ir.order: + node = ir.nodes[node_id] + outputs = _lower_ir_node(g, node, env, ir) + if len(outputs) != len(node.outputs): + raise ValueError( + f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" + ) + for output_id, tensor in zip(node.outputs, outputs): + env[output_id] = tensor + + outputs = [env[value_id] for value_id in ir.outputs] + seen_bound_constant_ids = {int(tensor.id) for tensor in bound_constants} + for tensor in getattr(g, "_transpile_materialized_constants", []): + tensor_id = int(tensor.id) + if tensor_id in seen_bound_constant_ids: + continue + bound_constants.append(tensor) + seen_bound_constant_ids.add(tensor_id) + return TranspiledGraph( + graph=g, + runtime_inputs=runtime_inputs, + bound_constants=bound_constants, + bound_constant_bindings=bound_constant_bindings, + outputs=outputs, + ) + + +def _lower_input_value(g: Graph, value: IRValue) -> Tensor: + if value.shape is None or value.dtype is None: + raise ValueError(f"IR input missing shape or dtype: {value.id}") + return g.input(shape=value.shape, dtype=_map_ir_dtype(value.dtype)) + +def _lower_constant_value(g: Graph, value: IRValue, const: Any) -> Any: + binding = _lookup_weight_binding(value) + if binding is not None: + _debug_mmap_binding(value.id, binding) + if binding.kind == "embedding": + print("generating embedding tensor") # Observe which shit actually has embeddings generated + return g.mmap_embeddings(binding.path) + return g.mmap_weights(binding.path) + + _debug_constant_fallback(value, const) + + if isinstance(const, torch.nn.Parameter): + const = const.detach() + if isinstance(const, torch.Tensor): + tensor_value = const.detach().cpu() + else: + raise NotImplementedError( + f"unsupported IR constant type for {value.id}: {type(const).__name__}" + ) + + if tensor_value.numel() == 1: + return tensor_value.item() + + return _materialize_constant_tensor(g, tensor_value) + + +def _lookup_weight_binding(value: IRValue) -> WeightBinding | None: + meta = getattr(value, "meta", None) + if isinstance(meta, dict): + path = meta.get("path") + kind = meta.get("kind") + source_name = meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + return WeightBinding(path=path, kind=kind, source_name=source_name) + return None + + +def _debug_mmap_binding(value_id: str, binding: WeightBinding) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + print( + "[transpile:mmap] " + f"value={value_id} " + f"kind={binding.kind} " + f"source={binding.source_name} " + f"path={binding.path}" + ) + + +def _debug_constant_fallback(value: IRValue, const: Any) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + const_type = type(const).__name__ + shape = None + dtype = None + if isinstance(const, torch.Tensor): + shape = tuple(const.shape) + dtype = str(const.dtype) + source_name = None + if isinstance(value.meta, dict): + source_name = value.meta.get("source_name") + print( + "[transpile:fallback] " + f"value={value.id} " + f"source={source_name} " + f"const_type={const_type} " + f"shape={shape} " + f"dtype={dtype}" + ) + + +def _debug_embedding_lowering(node: IRNode, embedding_tensor: Tensor, indices_tensor: Tensor) -> None: + if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1": + return + print( + "[transpile:embedding] " + f"node={node.id} " + f"embedding_id={embedding_tensor.id} " + f"embedding_shape={tuple(embedding_tensor.shape)} " + f"embedding_dtype={embedding_tensor.dtype} " + f"indices_id={indices_tensor.id} " + f"indices_shape={tuple(indices_tensor.shape)} " + f"indices_dtype={indices_tensor.dtype}" + ) + + +def _matmul_with_quantized_rhs_legalization( + g: Graph, + lhs: Tensor, + rhs: Tensor, + *, + pretransposed_rhs: bool = False, + output_dtype: int | None = None, +) -> Tensor: + if rhs.dtype in (Graph.INT8, Graph.INT4) and lhs.dtype == Graph.FP32: + lhs = g.precision_cast(lhs, Graph.FP16) + return g.matmul(lhs, rhs, pretransposed_rhs=pretransposed_rhs, output_dtype=output_dtype) + + +def _matmul_output_dtype( + lhs: Tensor, + rhs: Tensor, + *, + output_dtype: int | None, +) -> int | None: + if output_dtype != Graph.FP16: + return output_dtype + if rhs.dtype in (Graph.INT8, Graph.INT4): + return output_dtype + if lhs.dtype == Graph.FP32 or rhs.dtype == Graph.FP32: + return Graph.FP32 + return output_dtype + + +def _legalize_for_transpose(g: Graph, x: Tensor) -> Tensor: + return x + + +def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> list[Any]: + op = node.op + + if op == "arange": + start = int(node.attrs.get("start", 0)) + end = int(node.attrs["end"]) + step = int(node.attrs.get("step", 1)) + dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) + tensor_value = torch.arange(start, end, step=step, dtype=dtype) + return [_materialize_constant_tensor(g, tensor_value)] + + if op == "add": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "add")] + + if op == "add_clipped": + lhs, rhs = _legalize_elementwise_binary_inputs(g, _tensor(env, node.inputs[0]), _tensor(env, node.inputs[1])) + return [g.add_clipped(lhs, rhs)] + + if op == "subtract": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "subtract")] + + if op == "multiply": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] + + if op == "multiply_inplace": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "multiply")] + + if op == "divide": + return [_lower_binary_op(g, env[node.inputs[0]], env[node.inputs[1]], "divide")] + + if op == "not_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "not_equal")] + + if op == "equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "equal")] + + if op == "greater": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "greater")] + + if op == "less_equal": + return [_lower_compare_op(g, env[node.inputs[0]], env[node.inputs[1]], "less_equal")] + + if op == "logical_and": + lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + return [_lower_binary_op(g, lhs, rhs, "multiply")] + + if op == "logical_or": + lhs = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + rhs = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + return [_lower_compare_op(g, _lower_binary_op(g, lhs, rhs, "add"), 0.0, "not_equal")] + + if op == "logical_not": + return [_lower_compare_op(g, env[node.inputs[0]], 0.0, "equal")] + + if op == "where": + return [_lower_where_op(g, node, env)] + + if op == "scalar_add": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_add(x, float(node.attrs["value"]))] + + if op == "scalar_subtract": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_subtract(x, float(node.attrs["value"]))] + + if op == "scalar_subtract_reverse": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_add(g.scalar_multiply(x, -1.0), float(node.attrs["value"]))] + + if op == "scalar_multiply": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_multiply(x, float(node.attrs["value"]))] + + if op == "scalar_divide": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_divide(x, float(node.attrs["value"]))] + + if op == "scalar_floor_divide": + source = _tensor(env, node.inputs[0]) + x = _ensure_scalar_math_tensor(g, source) + y = g.scalar_floor_divide(x, float(node.attrs["value"])) + output_dtype = ir.values[node.outputs[0]].dtype + if output_dtype is None: + return [y] + target_dtype = _map_ir_dtype(output_dtype) + if y.dtype == target_dtype: + return [y] + return [g.precision_cast(y, target_dtype)] + + if op == "scalar_not_equal": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + return [g.scalar_not_equal(x, float(node.attrs["value"]))] + + if op == "scalar_equal": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + not_equal = g.scalar_not_equal(x, float(node.attrs["value"])) + return [g.scalar_not_equal(not_equal, 1.0)] + + if op == "scalar_divide_reverse": + raise NotImplementedError("scalar_divide_reverse is not directly supported by Cactus graph ops") + + if op == "precision_cast": + target_dtype = node.attrs.get("dtype") + if target_dtype is None: + return [_tensor(env, node.inputs[0])] + return [g.precision_cast(_tensor(env, node.inputs[0]), _map_ir_or_torch_dtype(target_dtype))] + + if op == "type_as": + source = _tensor(env, node.inputs[0]) + target = _tensor(env, node.inputs[1]) + return [g.precision_cast(source, target.dtype)] + + if op == "abs": + return [g.abs(_tensor(env, node.inputs[0]))] + + if op == "clamp": + x = _ensure_fp16_tensor(g, _tensor(env, node.inputs[0])) + min_value = node.attrs.get("min") + max_value = node.attrs.get("max") + if min_value is not None and math.isfinite(float(min_value)): + x = g.scalar_add(g.relu(g.scalar_subtract(x, float(min_value))), float(min_value)) + if max_value is not None and math.isfinite(float(max_value)): + upper_delta = g.relu(g.scalar_add(g.scalar_multiply(x, -1.0), float(max_value))) + x = g.scalar_add(g.scalar_multiply(upper_delta, -1.0), float(max_value)) + return [x] + + if op == "negate": + return [g.scalar_multiply(_tensor(env, node.inputs[0]), -1.0)] + + if op == "pow": + exponent = node.attrs.get("exponent") + if exponent is None: + raise NotImplementedError("tensor-tensor pow is not supported by Cactus graph ops") + return [g.pow(_tensor(env, node.inputs[0]), float(exponent))] + + if op == "scalar_exp": + return [g.scalar_exp(_tensor(env, node.inputs[0]))] + + if op == "scalar_sqrt": + return [g.scalar_sqrt(_tensor(env, node.inputs[0]))] + + if op == "scalar_log": + return [g.scalar_log(_tensor(env, node.inputs[0]))] + + if op == "scalar_cos": + return [g.scalar_cos(_tensor(env, node.inputs[0]))] + + if op == "scalar_sin": + return [g.scalar_sin(_tensor(env, node.inputs[0]))] + + if op in {"reshape", "view"}: + source = env[node.inputs[0]] + if isinstance(source, BroadcastAlias): + input_shape = tuple(source.logical_shape) + else: + input_shape = tuple(_tensor(env, node.inputs[0]).shape) + target_shape = _resolve_reshape_shape(input_shape, tuple(node.attrs["shape"])) + if isinstance(source, BroadcastAlias): + if source.kind != "gqa_repeat_kv": + raise NotImplementedError(f"unsupported broadcast alias kind in reshape: {source.kind}") + base_shape = tuple(source.tensor.shape) + if len(base_shape) != 4: + raise NotImplementedError(f"gqa_repeat_kv alias requires 4D base tensor, got {base_shape}") + if len(target_shape) != 4: + raise NotImplementedError(f"gqa_repeat_kv reshape requires 4D target shape, got {target_shape}") + batch, kv_heads, seq_len, head_dim = base_shape + if target_shape[0] != batch or target_shape[2] != seq_len or target_shape[3] != head_dim: + raise NotImplementedError( + f"gqa_repeat_kv reshape mismatch: base {base_shape}, target {target_shape}" + ) + if target_shape[1] % max(kv_heads, 1) != 0: + raise NotImplementedError( + f"gqa_repeat_kv target head count must be a multiple of kv heads: {base_shape} -> {target_shape}" + ) + return [BroadcastAlias(source.tensor, target_shape, source.kind)] + return [g.reshape(_tensor(env, node.inputs[0]), target_shape)] + + if op == "flatten": + return [ + g.flatten( + _tensor(env, node.inputs[0]), + start_dim=int(node.attrs["start_dim"]), + end_dim=int(node.attrs["end_dim"]), + ) + ] + + if op == "unsqueeze": + x = _tensor(env, node.inputs[0]) + target_shape = node.attrs.get("shape") + if target_shape is None: + dim = _normalize_dim(int(node.attrs["dim"]), len(x.shape) + 1) + shape_list = list(x.shape) + shape_list.insert(dim, 1) + target_shape = tuple(shape_list) + return [g.reshape(x, tuple(target_shape))] + + if op == "expand": + matched_base = _match_gqa_expand_alias(ir, node) + if matched_base is not None: + return [BroadcastAlias(_tensor(env, matched_base), tuple(node.attrs["shape"]), "gqa_repeat_kv")] + x = _tensor(env, node.inputs[0]) + target_shape = _resolve_expand_shape(tuple(x.shape), tuple(node.attrs["shape"])) + if target_shape == tuple(x.shape): + return [x] + return [g.expand(x, target_shape)] + + if op == "one_hot": + indices = _tensor(env, node.inputs[0]) + num_classes = int(node.attrs["num_classes"]) + eye = torch.eye(num_classes, dtype=torch.float16) + embedding = _materialize_constant_tensor(g, eye) + return [g.embedding_from_tensor(embedding, indices)] + + if op == "transpose": + x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + dim0 = _normalize_dim(int(node.attrs["dim0"]), len(x.shape)) + dim1 = _normalize_dim(int(node.attrs["dim1"]), len(x.shape)) + rank = len(x.shape) + permutation = list(range(rank)) + permutation[dim0], permutation[dim1] = permutation[dim1], permutation[dim0] + if rank == 2 and permutation == [1, 0]: + return [g.transpose(x)] + return [g.permute(x, permutation)] + + if op == "permute": + x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + permutation = tuple(_normalize_dim(int(dim), len(x.shape)) for dim in node.attrs["permutation"]) + if len(permutation) == 2 and permutation == (1, 0): + return [g.transpose(x)] + return [g.permute(x, permutation)] + + if op == "matmul": + lhs = _tensor(env, node.inputs[0]) + rhs = _tensor(env, node.inputs[1]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = ( + _map_ir_dtype(output_value.dtype) + if output_value is not None and output_value.dtype is not None + else None + ) + matmul_output_dtype = _matmul_output_dtype(lhs, rhs, output_dtype=output_dtype) + if len(lhs.shape) == 2 and len(rhs.shape) == 2: + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + legalized = _legalize_matmul_inputs(g, lhs, rhs, node) + if legalized is None: + static_batched = _lower_static_batched_matmul(g, lhs, rhs, output_dtype=matmul_output_dtype) + if static_batched is not None: + if output_dtype is not None and static_batched.dtype != output_dtype: + static_batched = g.precision_cast(static_batched, output_dtype) + return [static_batched] + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + lhs_2d, rhs_2d, output_shape = legalized + out = _matmul_with_quantized_rhs_legalization(g, lhs_2d, rhs_2d, output_dtype=matmul_output_dtype) + out = g.reshape(out, output_shape) + if output_dtype is not None and out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "linear": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else x.dtype + reshape_back: tuple[int, ...] | None = None + if len(x.shape) > 2: + x = _flatten_to_2d_for_linear(g, x) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"linear missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + matmul_output_dtype = _matmul_output_dtype(x, weight, output_dtype=output_dtype) + if node.attrs.get("has_bias"): + bias = _tensor(env, node.inputs[2]) + if ( + x.dtype == Graph.FP16 + and weight.dtype == Graph.FP16 + and output_dtype == Graph.FP16 + ): + matmul_output_dtype = Graph.FP32 + elif bias.dtype == Graph.FP32 and weight.dtype not in (Graph.INT8, Graph.INT4): + matmul_output_dtype = Graph.FP32 + out = _matmul_with_quantized_rhs_legalization( + g, + x, + weight, + pretransposed_rhs=True, + output_dtype=matmul_output_dtype, + ) + if node.attrs.get("has_bias"): + bias = _ensure_tensor_dtype(g, bias, matmul_output_dtype) + out = g.add(out, bias) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "addmm": + bias = _tensor(env, node.inputs[0]) + lhs = _tensor(env, node.inputs[1]) + rhs = _tensor(env, node.inputs[2]) + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else lhs.dtype + matmul_output_dtype = _matmul_output_dtype(lhs, rhs, output_dtype=output_dtype) + if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 and bias.dtype == Graph.FP16 and output_dtype == Graph.FP16: + matmul_output_dtype = Graph.FP32 + elif bias.dtype == Graph.FP32 and rhs.dtype not in (Graph.INT8, Graph.INT4): + matmul_output_dtype = Graph.FP32 + out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + bias = _ensure_tensor_dtype(g, bias, matmul_output_dtype) + out = g.add(bias, out) + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return [out] + + if op == "relu": + return [g.relu(_tensor(env, node.inputs[0]))] + + if op == "silu": + return [g.silu(_tensor(env, node.inputs[0]))] + + if op == "gelu": + return [g.gelu(_tensor(env, node.inputs[0]))] + + if op == "gelu_erf": + return [g.gelu_erf(_tensor(env, node.inputs[0]))] + + if op == "sigmoid": + return [g.sigmoid(_tensor(env, node.inputs[0]))] + + if op == "glu": + return [g.glu(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] + + if op == "softplus": + x = _tensor(env, node.inputs[0]) + return [_lower_softplus(g, x)] + + if op == "tanh": + return [g.tanh(_tensor(env, node.inputs[0]))] + + if op == "softmax": + return [g.softmax(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] + + if op == "lstm_cell": + x = _tensor(env, node.inputs[0]) + h_prev = _tensor(env, node.inputs[1]) + c_prev = _tensor(env, node.inputs[2]) + weight_ih = _tensor(env, node.inputs[3]) + weight_hh = _tensor(env, node.inputs[4]) + bias_ih = _tensor(env, node.inputs[5]) + bias_hh = _tensor(env, node.inputs[6]) + + lstm_out = g.lstm_cell( + _ensure_fp16_tensor(g, x), + _ensure_fp16_tensor(g, h_prev), + _ensure_fp16_tensor(g, c_prev), + _ensure_fp16_tensor(g, weight_ih), + _ensure_fp16_tensor(g, weight_hh), + _ensure_fp16_tensor(g, bias_ih), + _ensure_fp16_tensor(g, bias_hh), + ) + h_new = g.slice(lstm_out, axis=2, start=0, length=1) + c_new = g.slice(lstm_out, axis=2, start=1, length=1) + h_shape = ir.values[node.outputs[0]].shape + c_shape = ir.values[node.outputs[1]].shape + if h_shape is not None: + h_new = g.reshape(h_new, tuple(int(dim) for dim in h_shape)) + if c_shape is not None: + c_new = g.reshape(c_new, tuple(int(dim) for dim in c_shape)) + return [h_new, c_new] + + if op in {"scaled_dot_product_attention", "attention"}: + mask = node.attrs.get("mask") + mask_tensor: Tensor | None = None + additive_mask = bool(node.attrs.get("additive_mask", False)) + if len(node.inputs) > 3: + mask_tensor = _tensor(env, node.inputs[3]) + elif mask is not None: + raise NotImplementedError(f"{op} with literal mask is not supported yet") + if op == "scaled_dot_product_attention": + dropout_p = float(node.attrs.get("dropout_p", 0.0)) + if dropout_p != 0.0: + raise NotImplementedError("scaled_dot_product_attention with dropout is not supported yet") + + # PyTorch SDPA exports tensors as [batch, heads, seq, dim], while Cactus attention + # expects [batch, seq, heads, dim]. Convert into Cactus layout, call attention, then + # convert back so downstream exported ops still see PyTorch layout. + query = g.permute(_attention_tensor(env, node.inputs[0]), (0, 2, 1, 3)) + key = g.permute(_attention_tensor(env, node.inputs[1]), (0, 2, 1, 3)) + value = g.permute(_attention_tensor(env, node.inputs[2]), (0, 2, 1, 3)) + if mask_tensor is not None: + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=additive_mask, + ) + return [g.permute(out, (0, 2, 1, 3))] + + if op == "attention_block": + has_mask = bool(node.attrs.get("has_mask", False)) + has_gate = bool(node.attrs.get("has_gate", False)) + has_bias = bool(node.attrs.get("has_bias", False)) + input_index = 0 + query = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + key = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + value = g.permute(_attention_tensor(env, node.inputs[input_index]), (0, 2, 1, 3)) + input_index += 1 + mask_tensor: Tensor | None = None + if has_mask: + mask_tensor = _tensor(env, node.inputs[input_index]) + input_index += 1 + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + + attn_out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=bool(node.attrs.get("additive_mask", False)), + ) + flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) + flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () + if flat_shape: + attn_out = g.reshape(attn_out, flat_shape) + + if has_gate: + gate = _tensor(env, node.inputs[input_index]) + input_index += 1 + attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) + attn_out = g.multiply(attn_out, gate) + + weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + reshape_back: tuple[int, ...] | None = None + linear_input = attn_out + if len(linear_input.shape) > 2: + linear_input = _flatten_to_2d_for_linear(g, linear_input) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"attention_block missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) + if has_bias: + out = g.add(out, _tensor(env, node.inputs[input_index])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "self_attention_block": + has_mask = bool(node.attrs.get("has_mask", False)) + has_gate = bool(node.attrs.get("has_gate", False)) + has_bias = bool(node.attrs.get("has_bias", False)) + has_query_projection_bias = bool(node.attrs.get("has_query_projection_bias", False)) + has_query_add = bool(node.attrs.get("has_query_add", False)) + has_rel_query_add = bool(node.attrs.get("has_rel_query_add", False)) + has_key_projection_bias = bool(node.attrs.get("has_key_projection_bias", False)) + has_value_projection_bias = bool(node.attrs.get("has_value_projection_bias", False)) + has_rel_pos_bias = bool(node.attrs.get("has_rel_pos_bias", False)) + has_relative_key_projection_bias = bool(node.attrs.get("has_relative_key_projection_bias", False)) + + input_index = 0 + hidden = _tensor(env, node.inputs[input_index]) + input_index += 1 + + query_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + query_projection_bias = None + if has_query_projection_bias: + query_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + query_add = None + if has_query_add: + query_add = _tensor(env, node.inputs[input_index]) + input_index += 1 + rel_query_add = None + if has_rel_query_add: + rel_query_add = _tensor(env, node.inputs[input_index]) + input_index += 1 + + key_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + key_projection_bias = None + if has_key_projection_bias: + key_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + value_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + value_projection_bias = None + if has_value_projection_bias: + value_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + mask_tensor: Tensor | None = None + if has_mask: + mask_tensor = _tensor(env, node.inputs[input_index]) + input_index += 1 + + relative_key_input = None + relative_key_weight = None + relative_key_projection_bias = None + if has_rel_pos_bias: + relative_key_input = _tensor(env, node.inputs[input_index]) + input_index += 1 + relative_key_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + if has_relative_key_projection_bias: + relative_key_projection_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + gate = None + if has_gate: + gate = _tensor(env, node.inputs[input_index]) + input_index += 1 + + output_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + output_bias = None + if has_bias: + output_bias = _tensor(env, node.inputs[input_index]) + + query_shape = tuple(int(v) for v in node.attrs.get("query_shape", ())) + key_shape = tuple(int(v) for v in node.attrs.get("key_shape", ())) + value_shape = tuple(int(v) for v in node.attrs.get("value_shape", ())) + relative_key_shape = tuple(int(v) for v in node.attrs.get("relative_key_shape", ())) + + query_base = _lower_projected_attention_tensor(g, hidden, query_weight, query_projection_bias, query_shape) + query = query_base + if query_add is not None: + query_add = _normalize_attention_add_tensor(g, query_add, query_shape) + query, query_add = _legalize_elementwise_binary_inputs(g, query, query_add) + query = g.add(query, query_add) + + key = _lower_projected_attention_tensor(g, hidden, key_weight, key_projection_bias, key_shape) + value = _lower_projected_attention_tensor(g, hidden, value_weight, value_projection_bias, value_shape) + + if has_rel_pos_bias: + if relative_key_input is None or relative_key_weight is None or not relative_key_shape: + raise NotImplementedError(f"self_attention_block missing relative position bias inputs for node {node.id}") + rel_query = query_base + if rel_query_add is not None: + rel_query_add = _normalize_attention_add_tensor(g, rel_query_add, query_shape) + rel_query, rel_query_add = _legalize_elementwise_binary_inputs(g, rel_query, rel_query_add) + rel_query = g.add(rel_query, rel_query_add) + relative_key = _lower_projected_attention_tensor( + g, + relative_key_input, + relative_key_weight, + relative_key_projection_bias, + relative_key_shape, + ) + rel_mask = g.rel_pos_bias(rel_query, relative_key, float(node.attrs.get("rel_pos_scale", 1.0))) + if mask_tensor is None: + mask_tensor = rel_mask + else: + mask_tensor, rel_mask = _legalize_elementwise_binary_inputs(g, mask_tensor, rel_mask) + mask_tensor = g.add(mask_tensor, rel_mask) + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + elif mask_tensor is not None: + mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + + attn_out = g.attention( + query, + key, + value, + scale=_resolve_attention_scale(node, query), + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + mask=mask_tensor, + additive_mask=bool(node.attrs.get("additive_mask", False)), + ) + flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) + flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () + if flat_shape: + attn_out = g.reshape(attn_out, flat_shape) + + if gate is not None: + attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) + attn_out = g.multiply(attn_out, gate) + + linear_input = attn_out + reshape_back: tuple[int, ...] | None = None + if len(linear_input.shape) > 2: + linear_input = _flatten_to_2d_for_linear(g, linear_input) + output_value = ir.values.get(node.outputs[0]) + output_shape = output_value.shape if output_value is not None else None + if output_shape is None: + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + raise NotImplementedError(f"self_attention_block missing output shape metadata for node {node.id}") + reshape_back = tuple(int(v) for v in output_shape) + + out = _matmul_with_quantized_rhs_legalization(g, linear_input, output_weight, pretransposed_rhs=True) + if output_bias is not None: + out = g.add(out, output_bias) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op in ("sum", "mean", "variance", "min", "max"): + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + axes = _normalize_reduction_axes(node.attrs.get("axis"), len(x.shape)) + keepdim = bool(node.attrs.get("keepdim", False)) + out = _lower_reduction(g, op, x, axes=axes) + if keepdim: + new_shape = list(x.shape) + for reduced_axis in axes: + new_shape[reduced_axis] = 1 + out = g.reshape(out, tuple(new_shape)) + return [out] + + if op == "cumsum": + x = _ensure_scalar_math_tensor(g, _tensor(env, node.inputs[0])) + raw_axis = node.attrs.get("axis") + if not isinstance(raw_axis, int): + args = node.attrs.get("args", ()) + if isinstance(args, (list, tuple)) and len(args) > 1 and isinstance(args[1], int): + raw_axis = int(args[1]) + else: + raw_axis = -1 + axis = _normalize_dim(int(raw_axis), len(x.shape)) + return [g.cumsum(x, axis)] + + if op == "cat": + tensors = [_tensor(env, value_id) for value_id in node.inputs] + output_dtype: int | None = None + if node.outputs: + output_value = ir.values.get(node.outputs[0]) + if output_value is not None and output_value.dtype is not None: + output_dtype = _map_ir_or_torch_dtype(output_value.dtype) + return [_cat_with_legalized_dtype(g, tensors, axis=int(node.attrs.get("axis", 0)), dtype=output_dtype)] + + if op in {"stack", "aten.stack.default"}: + tensors = [_tensor(env, value_id) for value_id in node.inputs] + if not tensors: + raise NotImplementedError("stack requires at least one input tensor") + + raw_axis = node.attrs.get("axis") + if not isinstance(raw_axis, int): + args = node.attrs.get("args", ()) + if isinstance(args, (list, tuple)) and len(args) > 1 and isinstance(args[1], int): + raw_axis = int(args[1]) + else: + raw_axis = 0 + + base_shape = tuple(int(dim) for dim in tensors[0].shape) + axis = _normalize_dim(int(raw_axis), len(base_shape) + 1) + reshaped: list[Tensor] = [] + for tensor in tensors: + if tuple(int(dim) for dim in tensor.shape) != base_shape: + raise NotImplementedError( + f"stack requires equal input shapes, got {[tuple(int(dim) for dim in t.shape) for t in tensors]}" + ) + expanded_shape = list(base_shape) + expanded_shape.insert(axis, 1) + reshaped.append(g.reshape(tensor, tuple(expanded_shape))) + return [g.cat(reshaped, axis=axis)] + + if op == "slice": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs["axis"]), len(x.shape)) + start = int(node.attrs["start"]) + end = int(node.attrs["end"]) + step = int(node.attrs.get("step", 1)) + dim_size = x.shape[axis] + if step == 0: + raise NotImplementedError("slice with step == 0 is invalid") + if step != 1: + indices = list(range(*slice(start, end, step).indices(dim_size))) + if axis == 0 or x.dtype == Graph.FP16: + return [_lower_static_strided_slice_via_gather(g, x, axis=axis, indices=indices)] + if not indices: + return [g.slice(x, axis=axis, start=0, length=0)] + expanded_shape = list(int(dim) for dim in x.shape) + expanded_shape[axis] = 1 + pieces: list[Tensor] = [] + for idx in indices: + piece = g.index(x, index_value=idx, axis=axis) + piece = g.reshape(piece, tuple(expanded_shape)) + pieces.append(piece) + if len(pieces) == 1: + return [pieces[0]] + return [g.cat(pieces, axis=axis)] + start = _normalize_index(start, dim_size) + end = _normalize_slice_end(end, dim_size) + length = max(0, end - start) + return [g.slice(x, axis=axis, start=start, length=length)] + + if op == "split_with_sizes": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", -1)), len(x.shape)) + sizes = tuple(int(v) for v in node.attrs["sizes"]) + start = 0 + outputs: list[Tensor] = [] + for size in sizes: + outputs.append(g.slice(x, axis=axis, start=start, length=int(size))) + start += int(size) + return [outputs] + + if op == "chunk": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) + chunks = int(node.attrs["chunks"]) + if chunks <= 0: + raise NotImplementedError(f"chunk requires a positive chunk count, got {chunks}") + dim_size = int(x.shape[axis]) + if dim_size <= 0: + return [[g.slice(x, axis=axis, start=0, length=0)]] + chunk_size = (dim_size + chunks - 1) // chunks + outputs: list[Tensor] = [] + start = 0 + while start < dim_size: + length = min(chunk_size, dim_size - start) + outputs.append(g.slice(x, axis=axis, start=start, length=length)) + start += length + return [outputs] + + if op == "ones": + shape = tuple(int(v) for v in node.attrs.get("shape", ())) + if not shape: + raise NotImplementedError(f"ones requires a static shape, got {shape}") + torch_dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype")) + return [_materialize_constant_tensor(g, torch.ones(shape, dtype=torch_dtype))] + + if op == "tril": + x = _tensor(env, node.inputs[0]) + shape = tuple(int(dim) for dim in x.shape) + if len(shape) < 2: + raise NotImplementedError(f"tril expects rank >= 2, got shape {shape}") + diagonal = int(node.attrs.get("diagonal", 0)) + mask_dtype = _torch_dtype_for_graph_dtype(x.dtype) + mask = torch.ones(shape, dtype=mask_dtype).tril(diagonal=diagonal) + mask_tensor = _materialize_constant_tensor(g, mask) + return [_lower_binary_op(g, x, mask_tensor, "multiply")] + + if op == "unfold": + x = _tensor(env, node.inputs[0]) + rank = len(x.shape) + axis = _normalize_dim(int(node.attrs["dimension"]), rank) + size = int(node.attrs["size"]) + step = int(node.attrs["step"]) + if size <= 0: + raise NotImplementedError(f"unfold requires a positive size, got {size}") + if step <= 0: + raise NotImplementedError(f"unfold requires a positive step, got {step}") + dim_size = int(x.shape[axis]) + if size > dim_size: + raise NotImplementedError( + f"unfold size {size} exceeds dimension size {dim_size} for axis {axis}" + ) + + window_starts = list(range(0, dim_size - size + 1, step)) + if not window_starts: + raise NotImplementedError( + f"unfold produced no windows for axis={axis}, size={size}, step={step}, dim_size={dim_size}" + ) + + base_shape = [int(dim) for dim in x.shape] + trailing_shape = [base_shape[idx] for idx in range(rank) if idx != axis] + [size] + expanded_shape = tuple(trailing_shape[:axis] + [1] + trailing_shape[axis:]) + permutation = tuple(idx for idx in range(rank) if idx != axis) + (axis,) + + windows: list[Tensor] = [] + for start in window_starts: + piece = g.slice(x, axis=axis, start=start, length=size) + if axis != rank - 1: + piece = g.permute(piece, permutation=permutation) + piece = g.reshape(piece, expanded_shape) + windows.append(piece) + if len(windows) == 1: + return [windows[0]] + return [g.cat(windows, axis=axis)] + + if op == "pad": + x = _tensor(env, node.inputs[0]) + mode = str(node.attrs.get("mode", "constant")) + if mode != "constant": + raise NotImplementedError(f"pad mode is unsupported: {mode}") + pads = tuple(int(v) for v in node.attrs.get("pads", ())) + if len(pads) % 2 != 0: + raise NotImplementedError(f"pad expects an even-length pads tuple, got {pads}") + value = float(node.attrs.get("value", 0.0)) + + current = x + current_shape = list(int(dim) for dim in x.shape) + pad_dims = len(pads) // 2 + if pad_dims > len(current_shape): + raise NotImplementedError( + f"pad rank mismatch: pads={pads} for input shape {tuple(current_shape)}" + ) + + for pad_index in range(pad_dims): + before = pads[2 * pad_index] + after = pads[2 * pad_index + 1] + axis = len(current_shape) - 1 - pad_index + if before < 0 or after < 0: + raise NotImplementedError(f"negative pad is unsupported: {pads}") + pieces = [] + torch_dtype = _torch_dtype_for_graph_dtype(current.dtype) + if before > 0: + left_shape = list(current_shape) + left_shape[axis] = before + left = _materialize_constant_tensor( + g, + torch.full(tuple(left_shape), value, dtype=torch_dtype), + ) + pieces.append(left) + pieces.append(current) + if after > 0: + right_shape = list(current_shape) + right_shape[axis] = after + right = _materialize_constant_tensor( + g, + torch.full(tuple(right_shape), value, dtype=torch_dtype), + ) + pieces.append(right) + if len(pieces) > 1: + current = g.cat(pieces, axis=axis) + current_shape[axis] += before + after + return [current] + + if op == "index": + x = _tensor(env, node.inputs[0]) + axis = _normalize_dim(int(node.attrs.get("axis", 0)), len(x.shape)) + index_value = _normalize_index(int(node.attrs["index_value"]), x.shape[axis]) + return [g.index(x, index_value=index_value, axis=axis)] + + if op == "gather": + return [ + g.gather( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + axis=int(node.attrs.get("axis", 0)), + ) + ] + + if op == "embedding": + embedding_tensor = _tensor(env, node.inputs[0]) + indices_tensor = _tensor(env, node.inputs[1]) + _debug_embedding_lowering(node, embedding_tensor, indices_tensor) + return [g.embedding_from_tensor(embedding_tensor, indices_tensor)] + + if op == "conv_module": + input_index = 0 + x_nlc = _tensor(env, node.inputs[input_index]) + input_index += 1 + + pointwise1_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + pointwise1_bias = None + if bool(node.attrs.get("has_pointwise1_bias", False)): + pointwise1_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + depthwise_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + depthwise_bias = None + if bool(node.attrs.get("has_depthwise_bias", False)): + depthwise_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + + batch_norm_weight = _tensor(env, node.inputs[input_index]) + batch_norm_bias = _tensor(env, node.inputs[input_index + 1]) + batch_norm_running_mean = _tensor(env, node.inputs[input_index + 2]) + batch_norm_running_var = _tensor(env, node.inputs[input_index + 3]) + input_index += 4 + + pointwise2_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + pointwise2_bias = None + if bool(node.attrs.get("has_pointwise2_bias", False)): + pointwise2_bias = _tensor(env, node.inputs[input_index]) + + if len(x_nlc.shape) != 3: + raise NotImplementedError(f"conv_module expects rank-3 NLC input, got {x_nlc.shape}") + kernel_size = int(node.attrs.get("depthwise_kernel_size", 0)) + padding = int(node.attrs.get("depthwise_padding", 0)) + if len(depthwise_weight.shape) != 3 or kernel_size != 9 or padding != 4: + raise NotImplementedError( + f"conv_module currently supports same depthwise kernel-9 lowering, got " + f"kernel_size={kernel_size} padding={padding} weight_shape={depthwise_weight.shape}" + ) + + current = g.conv1d_pointwise(x_nlc, pointwise1_weight, bias=pointwise1_bias) + current = g.glu(current, axis=-1) + current = g.conv1d_same_depthwise_k9(current, depthwise_weight, bias=depthwise_bias) + current = g.batch_norm( + current, + batch_norm_weight, + batch_norm_bias, + batch_norm_running_mean, + batch_norm_running_var, + axis=len(current.shape) - 1, + eps=float(node.attrs.get("eps", 1e-5)), + ) + current = g.silu(current) + current = g.conv1d_pointwise(current, pointwise2_weight, bias=pointwise2_bias) + return [current] + + if op == "conv1d": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + stride = int(node.attrs.get("stride", 1)) + padding = int(node.attrs.get("padding", 0)) + dilation = int(node.attrs.get("dilation", 1)) + groups = int(node.attrs.get("groups", 1)) + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and weight.shape[2] == 1 + and stride == 1 + and padding == 0 + and dilation == 1 + and groups == 1 + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_pointwise(x_nlc, weight, bias=bias) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and weight.shape[2] == 9 + and stride == 1 + and padding == 4 + and dilation == 1 + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_same_depthwise_k9(x_nlc, weight, bias=bias) + return [g.permute(out_nlc, (0, 2, 1))] + + if ( + len(x.shape) == 3 + and len(weight.shape) == 3 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and stride == 1 + and padding == dilation * max(weight.shape[2] - 1, 0) + ): + x_nlc = g.permute(x, (0, 2, 1)) + out_nlc = g.conv1d_causal(x_nlc, weight, kernel_size=weight.shape[2], dilation=dilation) + if bias is not None: + bias_reshaped = g.reshape(bias, (1, 1, int(weight.shape[0]))) + out_nlc, bias_reshaped = _legalize_elementwise_binary_inputs(g, out_nlc, bias_reshaped) + out_nlc = g.add(out_nlc, bias_reshaped) + return [g.permute(out_nlc, (0, 2, 1))] + + if dilation != 1: + raise NotImplementedError(f"conv1d with dilation != 1 is unsupported by generic lowering: {dilation}") + if padding != 0: + if len(x.shape) != 3: + raise NotImplementedError(f"generic conv1d padding expects rank-3 input, got {x.shape}") + batch_size, channels, _ = (int(dim) for dim in x.shape) + pad_dtype = _torch_dtype_for_graph_dtype(x.dtype) + pieces: list[Tensor] = [] + if padding > 0: + left = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, padding), dtype=pad_dtype), + ) + pieces.append(left) + pieces.append(x) + if padding > 0: + right = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, padding), dtype=pad_dtype), + ) + pieces.append(right) + x = g.cat(pieces, axis=2) + if groups != 1: + if len(x.shape) != 3 or len(weight.shape) != 3: + raise NotImplementedError( + f"grouped conv1d lowering expects rank-3 input and weight, got {x.shape} and {weight.shape}" + ) + input_channels = int(x.shape[1]) + if input_channels % groups != 0: + raise NotImplementedError( + f"grouped conv1d requires input channels divisible by groups, got C_in={input_channels}, groups={groups}" + ) + output_channels = int(weight.shape[0]) + if output_channels % groups != 0: + raise NotImplementedError( + f"grouped conv1d requires output channels divisible by groups, got C_out={output_channels}, groups={groups}" + ) + input_channels_per_group = input_channels // groups + output_channels_per_group = output_channels // groups + if int(weight.shape[1]) != input_channels_per_group: + raise NotImplementedError( + "grouped conv1d weight has incompatible per-group input channels: " + f"weight C_in/group={int(weight.shape[1])}, expected {input_channels_per_group}" + ) + outputs: list[Tensor] = [] + for group_index in range(groups): + x_group = g.slice( + x, + axis=1, + start=group_index * input_channels_per_group, + length=input_channels_per_group, + ) + weight_group = g.slice( + weight, + axis=0, + start=group_index * output_channels_per_group, + length=output_channels_per_group, + ) + bias_group = None + if bias is not None: + bias_group = g.slice( + bias, + axis=0, + start=group_index * output_channels_per_group, + length=output_channels_per_group, + ) + outputs.append(g.conv1d(x_group, weight_group, bias=bias_group, stride=stride)) + if len(outputs) == 1: + return [outputs[0]] + return [g.cat(outputs, axis=1)] + return [g.conv1d(x, weight, bias=bias, stride=stride)] + + if op == "conv2d": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + stride = int(node.attrs.get("stride", 1)) + padding = int(node.attrs.get("padding", 0)) + dilation = int(node.attrs.get("dilation", 1)) + groups = int(node.attrs.get("groups", 1)) + + if len(x.shape) != 4 or len(weight.shape) != 4: + raise NotImplementedError(f"conv2d lowering expects rank-4 tensors, got {x.shape} and {weight.shape}") + if dilation != 1: + raise NotImplementedError(f"conv2d with dilation != 1 is unsupported: {dilation}") + + if stride == 2 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): + return [g.conv2d_k3s2p1(x, weight, bias=bias)] + + if ( + stride == 2 + and padding == 1 + and groups == x.shape[1] == weight.shape[0] + and weight.shape[1] == 1 + and weight.shape[2:] == (3, 3) + ): + return [g.conv2d_depthwise_k3s2p1(x, weight, bias=bias)] + + if ( + stride == 1 + and padding == 0 + and groups == 1 + and weight.shape[2:] == (1, 1) + ): + return [g.conv2d_pointwise_1x1(x, weight, bias=bias)] + + if stride == 1 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): + return [g.conv2d_k3s1p1(x, weight, bias=bias)] + + return [g.conv2d(x, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups)] + + if op == "layer_norm": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + return [g.layer_norm(x, weight, bias=bias, eps=float(node.attrs["eps"]))] + + if op == "rms_norm": + x = _tensor(env, node.inputs[0]) + weight = _tensor(env, node.inputs[1]) + reshape_back: tuple[int, ...] | None = None + if len(x.shape) > 2: + reshape_back = tuple(int(dim) for dim in x.shape) + x = _flatten_to_2d_for_linear(g, x) + out = g.rms_norm(x, weight, eps=float(node.attrs["eps"])) + if reshape_back is not None: + out = g.reshape(out, reshape_back) + return [out] + + if op == "rope": + rope_input = _tensor(env, node.inputs[0]) + if _rope_input_is_bhsd(ir, node.inputs[0]): + rope_input = g.permute(rope_input, (0, 2, 1, 3)) + rope_out = g.rope( + rope_input, + float(node.attrs["theta"]), + position_offset=int(node.attrs.get("position_offset", 0)), + ) + return [g.permute(rope_out, (0, 2, 1, 3))] + return [ + g.rope( + rope_input, + float(node.attrs["theta"]), + position_offset=int(node.attrs.get("position_offset", 0)), + ) + ] + + if op == "rel_pos_bias": + query = _attention_tensor(env, node.inputs[0]) + if len(query.shape) != 4: + raise NotImplementedError(f"rel_pos_bias expects rank-4 query input, got {query.shape}") + relative_key = _tensor(env, node.inputs[1]) + if len(relative_key.shape) != 4: + raise NotImplementedError(f"rel_pos_bias expects rank-4 relative_key input, got {relative_key.shape}") + if int(query.shape[2]) == int(relative_key.shape[2]) and int(query.shape[3]) == int(relative_key.shape[3]): + pass + elif int(query.shape[1]) == int(relative_key.shape[2]) and int(query.shape[3]) == int(relative_key.shape[3]): + query = g.permute(query, (0, 2, 1, 3)) + else: + raise NotImplementedError( + "rel_pos_bias query/relative_key layout mismatch: " + f"query={query.shape}, relative_key={relative_key.shape}" + ) + return [g.rel_pos_bias(query, relative_key, float(node.attrs.get("scale", 1.0)))] + + if op in {"gated_deltanet_prefill", "gated_deltanet_decode"}: + x = _tensor(env, node.inputs[0]) + qkv_weight = _tensor(env, node.inputs[1]) + a_weight = _tensor(env, node.inputs[2]) + b_weight = _tensor(env, node.inputs[3]) + norm_weight = _tensor(env, node.inputs[4]) + + input_index = 5 + z_weight = None + if bool(node.attrs.get("has_z", False)): + z_weight = _tensor(env, node.inputs[input_index]) + input_index += 1 + dt_bias = None + if bool(node.attrs.get("has_dt_bias", False)): + dt_bias = _tensor(env, node.inputs[input_index]) + input_index += 1 + a_log = None + if bool(node.attrs.get("has_a_log", False)): + a_log = _tensor(env, node.inputs[input_index]) + input_index += 1 + conv_weight = None + if bool(node.attrs.get("has_conv", False)): + conv_weight = _tensor(env, node.inputs[input_index]) + + if len(x.shape) != 3: + raise NotImplementedError(f"{op} currently expects rank-3 normalized input, got {x.shape}") + + batch_size, seq_len, hidden_dim = (int(dim) for dim in x.shape) + if batch_size != 1: + raise NotImplementedError(f"{op} currently supports batch size 1, got {x.shape}") + + num_k_heads = int(node.attrs["num_k_heads"]) + num_v_heads = int(node.attrs["num_v_heads"]) + key_dim = int(node.attrs["key_dim"]) + value_dim = int(node.attrs["value_dim"]) + eps = float(node.attrs.get("eps", 1e-6)) + chunk_size = int(node.attrs.get("chunk_size", 64)) + + mixed_qkv = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + qkv_weight, + pretransposed_rhs=True, + ) + mixed_qkv_dim = int(qkv_weight.shape[0]) + mixed_qkv = g.reshape(mixed_qkv, (batch_size, seq_len, mixed_qkv_dim)) + if conv_weight is not None: + kernel_size = int(conv_weight.shape[2]) + mixed_qkv = g.conv1d_causal(mixed_qkv, conv_weight, kernel_size=kernel_size, dilation=1) + mixed_qkv = g.silu(mixed_qkv) + + q_proj_dim = num_k_heads * key_dim + v_proj_dim = num_v_heads * value_dim + k_proj_dim = num_k_heads * key_dim + q_proj = g.slice(mixed_qkv, axis=2, start=0, length=q_proj_dim) + k_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim, length=k_proj_dim) + v_proj = g.slice(mixed_qkv, axis=2, start=q_proj_dim + k_proj_dim, length=v_proj_dim) + + q_4d = g.reshape(q_proj, (batch_size, seq_len, num_k_heads, key_dim)) + k_4d = g.reshape(k_proj, (batch_size, seq_len, num_k_heads, key_dim)) + v_4d = g.reshape(v_proj, (batch_size, seq_len, num_v_heads, value_dim)) + + q_norm = g.sum(g.multiply(q_4d, q_4d), axis=3) + q_norm = g.scalar_sqrt(g.scalar_add(q_norm, eps)) + q_norm = g.reshape(q_norm, (batch_size, seq_len, num_k_heads, 1)) + q_4d = g.divide(q_4d, q_norm) + + k_norm = g.sum(g.multiply(k_4d, k_4d), axis=3) + k_norm = g.scalar_sqrt(g.scalar_add(k_norm, eps)) + k_norm = g.reshape(k_norm, (batch_size, seq_len, num_k_heads, 1)) + k_4d = g.divide(k_4d, k_norm) + + a_logits = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + a_weight, + pretransposed_rhs=True, + ) + a_logits = g.reshape(a_logits, (batch_size, seq_len, int(a_weight.shape[0]))) + b_logits = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + b_weight, + pretransposed_rhs=True, + ) + b_logits = g.reshape(b_logits, (batch_size, seq_len, int(b_weight.shape[0]))) + + if dt_bias is not None: + dt_bias_2d = g.reshape(dt_bias, (1, int(dt_bias.shape[0]))) + a_logits, dt_bias_2d = _legalize_elementwise_binary_inputs(g, a_logits, dt_bias_2d) + a_logits = g.add(a_logits, dt_bias_2d) + a_softplus = _lower_softplus(g, a_logits) + + if a_log is not None: + a_log_2d = g.reshape(a_log, (1, int(a_log.shape[0]))) + neg_exp_a = g.scalar_multiply(g.scalar_exp(a_log_2d), -1.0) + neg_exp_a, a_softplus = _legalize_elementwise_binary_inputs(g, neg_exp_a, a_softplus) + gate_log = g.multiply(neg_exp_a, a_softplus) + else: + gate_log = g.scalar_multiply(a_softplus, -1.0) + beta = g.sigmoid(b_logits) + + initial_state = _materialize_constant_tensor( + g, + torch.zeros((batch_size, key_dim, num_v_heads, value_dim), dtype=torch.float16), + ) + + if op == "gated_deltanet_decode": + deltanet_out = g.gated_deltanet_decode(q_4d, k_4d, v_4d, gate_log, beta, initial_state, 0.0) + else: + deltanet_out = g.gated_deltanet_prefill(q_4d, k_4d, v_4d, gate_log, beta, initial_state, chunk_size, 0.0) + + y_4d = g.slice(deltanet_out, axis=1, start=0, length=seq_len) + y_2d = g.reshape(y_4d, (seq_len * num_v_heads, value_dim)) + + if z_weight is not None: + z_proj = _matmul_with_quantized_rhs_legalization( + g, + _flatten_to_2d_for_linear(g, x), + z_weight, + pretransposed_rhs=True, + ) + z_proj = g.reshape(z_proj, (seq_len * num_v_heads, value_dim)) + y_2d = g.multiply(g.rms_norm(y_2d, norm_weight, eps=eps), g.silu(z_proj)) + + return [g.reshape(y_2d, (batch_size, seq_len, num_v_heads * value_dim))] + + if op == "group_norm": + return [ + g.group_norm( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + _tensor(env, node.inputs[2]), + num_groups=int(node.attrs["num_groups"]), + eps=float(node.attrs["eps"]), + ) + ] + + if op == "batch_norm": + axis = int(node.attrs.get("axis", 1)) + return [ + g.batch_norm( + _tensor(env, node.inputs[0]), + _tensor(env, node.inputs[1]), + _tensor(env, node.inputs[2]), + _tensor(env, node.inputs[3]), + _tensor(env, node.inputs[4]), + axis=axis, + eps=float(node.attrs["eps"]), + ) + ] + + if op == "identity": + return [env[node.inputs[0]]] + + if op == "contiguous": + return [_tensor(env, node.inputs[0])] + + if op == "advanced_index": + lowered = _try_lower_advanced_index(g, node, env, ir) + if lowered is not None: + return [lowered] + raise NotImplementedError(f"unsupported advanced_index pattern for node {node.id}") + + if op in {"masked_scatter", "aten.masked_scatter.default"}: + base = _tensor(env, node.inputs[0]) + mask = _tensor(env, node.inputs[1]) + source = _tensor(env, node.inputs[2]) + if source.dtype != base.dtype: + source = _ensure_tensor_dtype(g, source, base.dtype) + return [g.masked_scatter(base, mask, source)] + + if op == "masked_fill": + base = _tensor(env, node.inputs[0]) + mask = _lower_compare_op(g, env[node.inputs[1]], 0.0, "not_equal") + inverse_mask = g.scalar_add(g.scalar_multiply(mask, -1.0), 1.0) + fill_value = _clamp_scalar_for_dtype(float(node.attrs["value"]), base.dtype) + fill_term = g.scalar_multiply(_ensure_tensor_dtype(g, mask, base.dtype), fill_value) + base_term = _lower_binary_op(g, base, _ensure_tensor_dtype(g, inverse_mask, base.dtype), "multiply") + return [_lower_binary_op(g, fill_term, base_term, "add")] + + if op == "getitem": + source = env[node.inputs[0]] + index = int(node.attrs["index"]) + if isinstance(source, (tuple, list)): + return [source[index]] + if isinstance(source, Tensor): + normalized_index = _normalize_index(index, int(source.shape[0])) + return [g.index(source, normalized_index, axis=0)] + raise NotImplementedError(f"getitem source is not tuple/list for node {node.id}") + + raise NotImplementedError(f"unsupported IR op in lowering: {op}") + + +def _tensor(env: dict[str, Any], value_id: str) -> Tensor: + try: + value = env[value_id] + except KeyError as exc: + raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc + if not isinstance(value, Tensor): + raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") + return value + + +def _attention_tensor(env: dict[str, Any], value_id: str) -> Tensor: + try: + value = env[value_id] + except KeyError as exc: + raise NotImplementedError(f"missing IR value during lowering: {value_id}") from exc + if isinstance(value, BroadcastAlias): + if value.kind != "gqa_repeat_kv": + raise TypeError(f"unsupported broadcast alias for attention input {value_id}: {value.kind}") + return value.tensor + if not isinstance(value, Tensor): + raise TypeError(f"expected lowered tensor for {value_id}, got {type(value).__name__}") + return value + + +def _resolve_attention_scale(node: IRNode, query: Tensor) -> float: + scale = float(node.attrs.get("scale", 0.0) or 0.0) + if scale > 0.0: + return scale + if len(query.shape) >= 1 and int(query.shape[-1]) > 0: + return float(int(query.shape[-1]) ** -0.5) + return 1.0 + + +def _lower_projected_attention_tensor( + g: Graph, + hidden: Tensor, + weight: Tensor, + bias: Tensor | None, + target_shape: tuple[int, ...], +) -> Tensor: + if len(target_shape) != 4: + raise NotImplementedError(f"projected attention tensor expects rank-4 target shape, got {target_shape}") + + linear_input = _flatten_to_2d_for_linear(g, hidden) + out = _matmul_with_quantized_rhs_legalization(g, linear_input, weight, pretransposed_rhs=True) + if bias is not None: + out = g.add(out, bias) + return g.reshape(out, target_shape) + + +def _normalize_attention_add_tensor(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: + if tuple(tensor.shape) == target_shape: + return tensor + + if len(tensor.shape) != 4 or len(target_shape) != 4: + raise NotImplementedError( + f"unsupported attention add tensor shape {tuple(tensor.shape)} for target {target_shape}" + ) + + batch, seq_len, heads, head_dim = target_shape + shape = tuple(int(v) for v in tensor.shape) + + if shape[0] in (1, batch) and shape[1] == heads and shape[2] in (1, seq_len) and shape[3] == head_dim: + return g.permute(tensor, (0, 2, 1, 3)) + if shape[0] in (1, batch) and shape[1] in (1, seq_len) and shape[2] == heads and shape[3] == head_dim: + return tensor + + raise NotImplementedError( + f"unsupported attention add tensor shape {shape} for target {target_shape}" + ) + + +def _rope_input_is_bhsd(ir: IRGraph, value_id: str) -> bool: + value = ir.values.get(value_id) + if value is None or value.producer is None: + return False + node = ir.nodes.get(value.producer) + if node is None: + return False + if node.op == "permute": + permutation = tuple(int(dim) for dim in node.attrs.get("permutation", ())) + return permutation == (0, 2, 1, 3) + if node.op == "transpose": + return int(node.attrs.get("dim0", -1)) == 1 and int(node.attrs.get("dim1", -1)) == 2 + return False + + +def _lower_softplus(g: Graph, x: Tensor) -> Tensor: + # Stable fp16 softplus: relu(x) + log(1 + exp(-abs(x))). + abs_x = g.abs(x) + neg_abs_x = g.scalar_multiply(abs_x, -1.0) + exp_term = g.scalar_exp(neg_abs_x) + log_term = g.scalar_log(g.scalar_add(exp_term, 1.0)) + return g.add(g.relu(x), log_term) + + +def _normalize_dim(dim: int, rank: int) -> int: + if dim < 0: + dim += rank + return dim + + +def _normalize_reduction_axes(axis: Any, rank: int) -> tuple[int, ...]: + if axis is None: + return tuple(range(rank)) + if isinstance(axis, int): + return (_normalize_dim(axis, rank),) + if isinstance(axis, (list, tuple)): + normalized: list[int] = [] + seen: set[int] = set() + for raw_axis in axis: + if not isinstance(raw_axis, int): + raise NotImplementedError("reduction axes must be integers") + reduced_axis = _normalize_dim(raw_axis, rank) + if reduced_axis in seen: + continue + seen.add(reduced_axis) + normalized.append(reduced_axis) + if normalized: + return tuple(sorted(normalized)) + raise NotImplementedError("reduction axes must be an int, a tuple/list of ints, or None") + + +def _lower_reduction(g: Graph, op: str, x: Tensor, *, axes: tuple[int, ...]) -> Tensor: + if not axes: + return x + + fn = getattr(g, op) + if len(axes) == 1: + return fn(x, axes[0]) + + if op == "variance": + flattened = g.flatten(x, start_dim=axes[0], end_dim=axes[-1]) + collapsed_axes = set(axes) + expected_axes = set(range(axes[0], axes[-1] + 1)) + if collapsed_axes != expected_axes: + raise NotImplementedError("variance currently requires contiguous multi-axis reductions") + return fn(flattened, axes[0]) + + reduced = x + for axis in sorted(axes, reverse=True): + reduced = fn(reduced, axis) + return reduced + + +def _normalize_index(index: int, dim_size: int) -> int: + if index < 0: + index += dim_size + return index + + +def _static_shape(shape: Any) -> tuple[int, ...] | None: + if not isinstance(shape, tuple): + return None + dims: list[int] = [] + for dim in shape: + try: + dims.append(int(dim)) + except Exception: + return None + return tuple(dims) + + +def _constant_tensor_from_ir(ir: IRGraph, value_id: str) -> torch.Tensor | None: + const = ir.constants.get(value_id) + if isinstance(const, torch.nn.Parameter): + const = const.detach() + if isinstance(const, torch.Tensor): + return const.detach().cpu() + return None + + +def _try_lower_identity_broadcast_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + if len(node.inputs) != 3: + return None + + source = _tensor(env, node.inputs[0]) + source_shape = tuple(int(dim) for dim in source.shape) + if len(source_shape) != 2 or int(source_shape[0]) != 1: + return None + + output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + if output_shape is None: + return None + + batch_index = _constant_tensor_from_ir(ir, node.inputs[1]) + token_index = _constant_tensor_from_ir(ir, node.inputs[2]) + if batch_index is None or token_index is None: + return None + + try: + batch_index = torch.broadcast_to(batch_index.to(torch.int64), output_shape) + token_index = torch.broadcast_to(token_index.to(torch.int64), output_shape) + except RuntimeError: + return None + + if torch.count_nonzero(batch_index).item() != 0: + return None + + flat_index = token_index.reshape(-1) + expected = torch.arange(int(source_shape[1]), dtype=torch.int64) + if flat_index.numel() != expected.numel() or not torch.equal(flat_index, expected): + return None + + return g.reshape(source, output_shape) + + +def _try_lower_prefix_mask_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + if len(node.inputs) != 2: + return None + + source = _tensor(env, node.inputs[0]) + mask = _tensor(env, node.inputs[1]) + source_shape = tuple(int(dim) for dim in source.shape) + mask_shape = tuple(int(dim) for dim in mask.shape) + if not mask_shape or len(mask_shape) > len(source_shape): + return None + if source_shape[: len(mask_shape)] != mask_shape: + return None + selected = g.masked_select_prefix(source, mask) + + output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + if output_shape is None or not output_shape: + return selected + if len(output_shape) != len(selected.shape): + return selected + expected_prefix = int(output_shape[0]) + if expected_prefix < int(selected.shape[0]): + selected = g.slice(selected, axis=0, start=0, length=expected_prefix) + return selected + + +def _try_lower_advanced_index( + g: Graph, + node: IRNode, + env: dict[str, Any], + ir: IRGraph, +) -> Tensor | None: + lowered = _try_lower_identity_broadcast_advanced_index(g, node, env, ir) + if lowered is not None: + return lowered + return _try_lower_prefix_mask_advanced_index(g, node, env, ir) + + +def _normalize_slice_end(end: int, dim_size: int) -> int: + if end < 0: + end += dim_size + return max(0, min(end, dim_size)) + + +def _legalize_elementwise_binary_inputs(g: Graph, lhs: Tensor, rhs: Tensor) -> tuple[Tensor, Tensor]: + if lhs.shape == rhs.shape: + return lhs, rhs + + lhs_rank = len(lhs.shape) + rhs_rank = len(rhs.shape) + + if lhs_rank > rhs_rank: + rhs = _reshape_for_trailing_broadcast(g, rhs, lhs.shape) + elif rhs_rank > lhs_rank: + lhs = _reshape_for_trailing_broadcast(g, lhs, rhs.shape) + + if lhs.shape == rhs.shape: + return lhs, rhs + + return lhs, rhs + + +def _is_scalar_like(value: Any) -> bool: + return isinstance(value, (int, float, bool)) + + +def _lower_binary_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: + if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): + lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) + target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 + lhs = _ensure_tensor_dtype(g, lhs, target_dtype) + rhs = _ensure_tensor_dtype(g, rhs, target_dtype) + if op == "add": + return g.add(lhs, rhs) + if op == "subtract": + return g.subtract(lhs, rhs) + if op == "multiply": + return g.multiply(lhs, rhs) + if op == "divide": + return g.divide(lhs, rhs) + raise NotImplementedError(f"unsupported binary op: {op}") + + if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): + lhs_value = _ensure_tensor_dtype( + g, + lhs_value, + Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + scalar = float(rhs_value) + if op == "add": + return g.scalar_add(lhs_value, scalar) + if op == "subtract": + return g.scalar_subtract(lhs_value, scalar) + if op == "multiply": + return g.scalar_multiply(lhs_value, scalar) + if op == "divide": + return g.scalar_divide(lhs_value, scalar) + raise NotImplementedError(f"unsupported binary op: {op}") + + if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): + rhs_value = _ensure_tensor_dtype( + g, + rhs_value, + Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + scalar = float(lhs_value) + if op == "add": + return g.scalar_add(rhs_value, scalar) + if op == "subtract": + return g.scalar_add(g.scalar_multiply(rhs_value, -1.0), scalar) + if op == "multiply": + return g.scalar_multiply(rhs_value, scalar) + if op == "divide": + raise NotImplementedError("scalar/tensor divide is not directly supported by Cactus graph ops") + raise NotImplementedError(f"unsupported binary op: {op}") + + raise TypeError( + f"unsupported lowered operand types for {op}: " + f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" + ) + + +def _lower_compare_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tensor: + if op == "equal": + not_equal = _lower_compare_op(g, lhs_value, rhs_value, "not_equal") + return g.scalar_not_equal(not_equal, 1.0) + + if op == "greater": + delta = _lower_binary_op(g, lhs_value, rhs_value, "subtract") + delta = _ensure_fp16_tensor(g, delta) + return g.scalar_not_equal(g.relu(delta), 0.0) + + if op == "less_equal": + greater = _lower_compare_op(g, lhs_value, rhs_value, "greater") + return g.scalar_not_equal(greater, 1.0) + + if op != "not_equal": + raise NotImplementedError(f"unsupported compare op: {op}") + + if isinstance(lhs_value, Tensor) and isinstance(rhs_value, Tensor): + lhs, rhs = _legalize_elementwise_binary_inputs(g, lhs_value, rhs_value) + target_dtype = Graph.FP16 if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 else Graph.FP32 + lhs = _ensure_tensor_dtype(g, lhs, target_dtype) + rhs = _ensure_tensor_dtype(g, rhs, target_dtype) + return g.not_equal(lhs, rhs) + + if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): + lhs_value = _ensure_tensor_dtype( + g, + lhs_value, + Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + return g.scalar_not_equal(lhs_value, float(rhs_value)) + + if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): + rhs_value = _ensure_tensor_dtype( + g, + rhs_value, + Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, + ) + return g.scalar_not_equal(rhs_value, float(lhs_value)) + + raise TypeError( + f"unsupported lowered operand types for {op}: " + f"{type(lhs_value).__name__}, {type(rhs_value).__name__}" + ) + + +def _reshape_for_trailing_broadcast(g: Graph, tensor: Tensor, target_shape: tuple[int, ...]) -> Tensor: + tensor_shape = tuple(tensor.shape) + target_rank = len(target_shape) + tensor_rank = len(tensor_shape) + + if tensor_rank > target_rank: + return tensor + + padded_shape = (1,) * (target_rank - tensor_rank) + tensor_shape + + # Only legalize cases that are valid trailing broadcasts, e.g.: + # (H,) -> (1, H), (1, 1, H), etc. + for src_dim, tgt_dim in zip(padded_shape, target_shape): + if src_dim != 1 and src_dim != tgt_dim: + return tensor + + if padded_shape == tensor_shape: + return tensor + + return g.reshape(tensor, padded_shape) + + +def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_scalar_math_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype in (Graph.FP16, Graph.FP32): + return tensor + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_tensor_dtype(g: Graph, tensor: Tensor, dtype: int) -> Tensor: + if tensor.dtype == dtype: + return tensor + return g.precision_cast(tensor, dtype) + + +def _cat_with_legalized_dtype(g: Graph, tensors: list[Tensor], *, axis: int, dtype: int | None) -> Tensor: + if not tensors: + raise NotImplementedError("cat requires at least one tensor") + if dtype is None: + dtypes = {tensor.dtype for tensor in tensors} + dtype = tensors[0].dtype if len(dtypes) == 1 else Graph.FP16 + legalized = [_ensure_tensor_dtype(g, tensor, dtype) for tensor in tensors] + return g.cat(legalized, axis=axis) + + +def _invert_permutation(permutation: tuple[int, ...]) -> tuple[int, ...]: + inverse = [0] * len(permutation) + for index, source_axis in enumerate(permutation): + inverse[int(source_axis)] = index + return tuple(inverse) + + +def _lower_static_strided_slice_via_gather( + g: Graph, + x: Tensor, + *, + axis: int, + indices: list[int], +) -> Tensor: + if not indices: + return g.slice(x, axis=axis, start=0, length=0) + + indices_tensor = _materialize_constant_tensor(g, torch.tensor(indices, dtype=torch.float32)) + if axis == 0: + return g.gather(x, indices_tensor) + + permutation = (axis, *[dim for dim in range(len(x.shape)) if dim != axis]) + transposed = g.permute(x, permutation) + gathered = g.gather(transposed, indices_tensor) + inverse_permutation = _invert_permutation(permutation) + return g.permute(gathered, inverse_permutation) + + +def _lower_where_branch_term(g: Graph, branch_value: Any, mask: Tensor, *, dtype: int | None) -> Tensor: + if dtype is not None: + mask = _ensure_tensor_dtype(g, mask, dtype) + if isinstance(branch_value, Tensor): + branch_mask = mask if dtype is None else _ensure_tensor_dtype(g, mask, branch_value.dtype) + return _lower_binary_op(g, branch_value, branch_mask, "multiply") + return g.scalar_multiply(mask, _clamp_scalar_for_dtype(float(branch_value), mask.dtype)) + + +def _clamp_scalar_for_dtype(value: float, dtype: int | None) -> float: + if dtype != Graph.FP16: + return value + if math.isnan(value): + return value + if value == math.inf: + return 65504.0 + if value == -math.inf: + return -65504.0 + return max(-65504.0, min(65504.0, value)) + + +def _lower_where_op(g: Graph, node: IRNode, env: dict[str, Any]) -> Tensor: + condition = _lower_compare_op(g, env[node.inputs[0]], 0.0, "not_equal") + false_mask = g.scalar_add(g.scalar_multiply(condition, -1.0), 1.0) + + input_index = 1 + if bool(node.attrs.get("true_is_scalar", False)): + true_value: Any = float(node.attrs["true_value"]) + else: + true_value = env[node.inputs[input_index]] + input_index += 1 + + if bool(node.attrs.get("false_is_scalar", False)): + false_value: Any = float(node.attrs["false_value"]) + else: + false_value = env[node.inputs[input_index]] + + result_dtype: int | None = None + if isinstance(true_value, Tensor): + result_dtype = true_value.dtype + elif isinstance(false_value, Tensor): + result_dtype = false_value.dtype + + true_term = _lower_where_branch_term(g, true_value, condition, dtype=result_dtype) + false_term = _lower_where_branch_term(g, false_value, false_mask, dtype=result_dtype) + return _lower_binary_op(g, true_term, false_term, "add") + + +def _flatten_to_2d_for_linear(g: Graph, tensor: Tensor) -> Tensor: + shape = tuple(tensor.shape) + if len(shape) <= 2: + return tensor + leading = 1 + for dim in shape[:-1]: + leading *= int(dim) + return g.reshape(tensor, (leading, int(shape[-1]))) + + +def _broadcast_batch_shape(lhs_batch: tuple[int, ...], rhs_batch: tuple[int, ...]) -> tuple[int, ...] | None: + rank = max(len(lhs_batch), len(rhs_batch)) + lhs_padded = (1,) * (rank - len(lhs_batch)) + lhs_batch + rhs_padded = (1,) * (rank - len(rhs_batch)) + rhs_batch + result: list[int] = [] + for lhs_dim, rhs_dim in zip(lhs_padded, rhs_padded): + if lhs_dim == rhs_dim: + result.append(int(lhs_dim)) + elif lhs_dim == 1: + result.append(int(rhs_dim)) + elif rhs_dim == 1: + result.append(int(lhs_dim)) + else: + return None + return tuple(result) + + +def _project_broadcast_coord(coord: tuple[int, ...], operand_batch: tuple[int, ...]) -> tuple[int, ...]: + if not operand_batch: + return () + offset = len(coord) - len(operand_batch) + projected: list[int] = [] + for axis, dim in enumerate(operand_batch): + value = coord[offset + axis] + projected.append(0 if int(dim) == 1 else int(value)) + return tuple(projected) + + +def _index_batch_prefix(g: Graph, tensor: Tensor, coord: tuple[int, ...]) -> Tensor: + for index_value in coord: + tensor = g.index(tensor, index_value, axis=0) + return tensor + + +def _lower_static_batched_matmul( + g: Graph, + lhs: Tensor, + rhs: Tensor, + *, + output_dtype: int | None = None, +) -> Tensor | None: + lhs_shape = tuple(int(dim) for dim in lhs.shape) + rhs_shape = tuple(int(dim) for dim in rhs.shape) + if len(lhs_shape) < 3 or len(rhs_shape) < 3: + return None + if lhs_shape[-1] != rhs_shape[-2]: + return None + + lhs_batch = lhs_shape[:-2] + rhs_batch = rhs_shape[:-2] + batch_shape = _broadcast_batch_shape(lhs_batch, rhs_batch) + if batch_shape is None: + return None + + output_matrix_shape = (int(lhs_shape[-2]), int(rhs_shape[-1])) + rank = len(batch_shape) + + def _build(dim: int, coord_prefix: tuple[int, ...]) -> Tensor: + if dim == rank: + lhs_coord = _project_broadcast_coord(coord_prefix, lhs_batch) + rhs_coord = _project_broadcast_coord(coord_prefix, rhs_batch) + lhs_slice = _index_batch_prefix(g, lhs, lhs_coord) + rhs_slice = _index_batch_prefix(g, rhs, rhs_coord) + out = _matmul_with_quantized_rhs_legalization( + g, + lhs_slice, + rhs_slice, + output_dtype=output_dtype, + ) + return g.reshape(out, (1,) * rank + output_matrix_shape) + + pieces = [_build(dim + 1, coord_prefix + (index_value,)) for index_value in range(int(batch_shape[dim]))] + if len(pieces) == 1: + return pieces[0] + return g.cat(pieces, axis=dim) + + return _build(0, ()) + + +def _legalize_matmul_inputs( + g: Graph, + lhs: Tensor, + rhs: Tensor, + node: IRNode, +) -> tuple[Tensor, Tensor, tuple[int, ...]] | None: + lhs_shape = tuple(lhs.shape) + rhs_shape = tuple(rhs.shape) + if len(lhs_shape) <= 2 and len(rhs_shape) <= 2: + return None + + if len(lhs_shape) > 2 and len(rhs_shape) == 2 and lhs_shape[-1] == rhs_shape[0]: + leading = 1 + for dim in lhs_shape[:-1]: + leading *= int(dim) + lhs_2d = g.reshape(lhs, (leading, int(lhs_shape[-1]))) + output_shape = lhs_shape[:-1] + (int(rhs_shape[1]),) + return lhs_2d, rhs, output_shape + + # Cactus matmul is 2D-only. Legalize the narrow case where both operands have + # only singleton leading dims, e.g. rotary helper matmuls: + # (1, M, K) @ (1, K, N) -> reshape to (M, K) @ (K, N) -> reshape back. + if any(dim != 1 for dim in lhs_shape[:-2]): + return None + if any(dim != 1 for dim in rhs_shape[:-2]): + return None + + lhs_2d_shape = lhs_shape[-2:] + rhs_2d_shape = rhs_shape[-2:] + if lhs_2d_shape[-1] != rhs_2d_shape[0]: + return None + + output_shape = node.meta.get("shape") + if not isinstance(output_shape, tuple): + output_shape = lhs_shape[:-2] + (lhs_2d_shape[0], rhs_2d_shape[1]) + output_shape = tuple(int(v) for v in output_shape) + + lhs_2d = g.reshape(lhs, lhs_2d_shape) + rhs_2d = g.reshape(rhs, rhs_2d_shape) + return lhs_2d, rhs_2d, output_shape + + +def _resolve_expand_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: + if len(requested_shape) < len(input_shape): + raise NotImplementedError(f"expand cannot reduce rank: {input_shape} -> {requested_shape}") + + padded_input = (1,) * (len(requested_shape) - len(input_shape)) + input_shape + resolved: list[int] = [] + for in_dim, req_dim in zip(padded_input, requested_shape): + if req_dim == -1: + resolved.append(in_dim) + continue + if req_dim < -1: + raise NotImplementedError(f"invalid expand dimension: {req_dim}") + resolved.append(int(req_dim)) + return tuple(resolved) + + +def _resolve_reshape_shape(input_shape: tuple[int, ...], requested_shape: tuple[int, ...]) -> tuple[int, ...]: + resolved = [int(v) for v in requested_shape] + unknown_indices = [idx for idx, dim in enumerate(resolved) if dim == -1] + if len(unknown_indices) > 1: + raise NotImplementedError(f"reshape with multiple inferred dimensions is unsupported: {requested_shape}") + if not unknown_indices: + return tuple(resolved) + + input_elements = 1 + for dim in input_shape: + input_elements *= int(dim) + + known_elements = 1 + for dim in resolved: + if dim != -1: + known_elements *= int(dim) + + if known_elements == 0 or input_elements % known_elements != 0: + raise NotImplementedError(f"cannot infer reshape target {requested_shape} from input shape {input_shape}") + + resolved[unknown_indices[0]] = input_elements // known_elements + return tuple(resolved) + + +def _match_gqa_expand_alias(ir: IRGraph, expand_node: IRNode) -> str | None: + target_shape = tuple(int(v) for v in expand_node.attrs["shape"]) + if len(target_shape) != 5: + return None + + current_value_id = expand_node.inputs[0] + base_value_id: str | None = None + + while True: + producer_id = ir.values[current_value_id].producer + if producer_id is None: + return None + producer = ir.nodes[producer_id] + if producer.op == "slice": + axis = int(producer.attrs.get("axis", -1)) + input_shape = ir.values[producer.inputs[0]].shape + if input_shape is None: + return None + normalized_axis = _normalize_dim(axis, len(input_shape)) + start = int(producer.attrs.get("start", 0)) + end = int(producer.attrs.get("end", input_shape[normalized_axis])) + if start != 0 or end < input_shape[normalized_axis]: + return None + current_value_id = producer.inputs[0] + continue + if producer.op == "unsqueeze": + base_value_id = producer.inputs[0] + base_shape = ir.values[base_value_id].shape + if base_shape is None: + return None + unsqueezed_dim = _normalize_dim(int(producer.attrs.get("dim", 0)), len(base_shape) + 1) + if unsqueezed_dim != 2: + return None + break + if producer.op in {"reshape", "view"}: + base_value_id = producer.inputs[0] + base_shape = ir.values[base_value_id].shape + view_shape = ir.values[current_value_id].shape + if base_shape is None or view_shape is None: + return None + if len(base_shape) != 4 or len(view_shape) != 5: + return None + if tuple(int(v) for v in view_shape) != ( + int(base_shape[0]), + int(base_shape[1]), + 1, + int(base_shape[2]), + int(base_shape[3]), + ): + return None + break + return None + + if base_value_id is None: + return None + base_shape = ir.values[base_value_id].shape + if base_shape is None or len(base_shape) != 4: + return None + + expected_target = ( + int(base_shape[0]), + int(base_shape[1]), + int(target_shape[2]), + int(base_shape[2]), + int(base_shape[3]), + ) + if target_shape != expected_target: + return None + if int(target_shape[2]) <= 1: + return None + + return base_value_id + + +def _map_ir_dtype(dtype: str) -> int: + if dtype == "bf16": + return Graph.FP32 + if dtype == "fp16": + return Graph.FP16 + if dtype in ("fp32", "fp64"): + return Graph.FP32 + if dtype == "int8": + return Graph.INT8 + if dtype in ("int32", "int64", "bool"): + return Graph.FP32 + + raise NotImplementedError(f"unsupported IR dtype: {dtype}") + + +def _map_ir_or_torch_dtype(dtype: Any) -> int: + if dtype is None: + raise NotImplementedError("missing dtype for precision_cast") + if isinstance(dtype, str): + if dtype.startswith("torch."): + return _map_torch_dtype(getattr(torch, dtype.split(".", 1)[1])) + return _map_ir_dtype(dtype) + return _map_torch_dtype(dtype) + + +def _map_torch_dtype(dtype: Any) -> int: + if dtype == torch.bfloat16: + return Graph.FP32 + if dtype == torch.float16: + return Graph.FP16 + if dtype == torch.float32: + return Graph.FP32 + if dtype == torch.float64: + return Graph.FP32 + if dtype == torch.int8: + return Graph.INT8 + if dtype in (torch.int16, torch.int32, torch.int64, torch.bool): + return Graph.FP32 + raise NotImplementedError(f"unsupported torch dtype: {dtype}") + + +def _materialize_constant_tensor(g: Graph, tensor_value: torch.Tensor) -> Tensor: + graph_dtype = _map_torch_dtype(tensor_value.dtype) + materialized = tensor_value.detach().cpu() + if graph_dtype == Graph.FP32 and materialized.dtype not in (torch.float32, torch.float64): + materialized = materialized.to(torch.float32) + elif graph_dtype == Graph.FP16 and materialized.dtype != torch.float16: + materialized = materialized.to(torch.float16) + elif graph_dtype == Graph.INT8 and materialized.dtype != torch.int8: + materialized = materialized.to(torch.int8) + + tensor = g.input(shape=tuple(materialized.shape), dtype=graph_dtype) + g.set_input(tensor, materialized, dtype=graph_dtype) + materialized_constants = getattr(g, "_transpile_materialized_constants", None) + if isinstance(materialized_constants, list): + materialized_constants.append(tensor) + return tensor + + +def _materialize_constant_torch_dtype(dtype: Any) -> torch.dtype: + if dtype is None: + return torch.float32 + if isinstance(dtype, str): + if dtype.startswith("torch."): + return getattr(torch, dtype.split(".", 1)[1]) + if dtype in ("fp16", "bf16"): + return torch.float16 + if dtype in ("fp32", "fp64", "int16", "int32", "int64", "bool"): + return torch.float32 + if dtype == "int8": + return torch.int8 + if isinstance(dtype, torch.dtype): + return dtype + raise NotImplementedError(f"unsupported dtype for constant materialization: {dtype}") + + +def _torch_dtype_for_graph_dtype(dtype: int) -> torch.dtype: + if dtype == Graph.FP16: + return torch.float16 + if dtype == Graph.FP32: + return torch.float32 + if dtype == Graph.INT8: + return torch.int8 + raise NotImplementedError(f"unsupported graph dtype for constant materialization: {dtype}") diff --git a/python/src/transpile/model_adapters.py b/python/src/transpile/model_adapters.py new file mode 100644 index 000000000..38476d8ec --- /dev/null +++ b/python/src/transpile/model_adapters.py @@ -0,0 +1,2507 @@ +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import inspect +import struct +import os +from pathlib import Path +from typing import Callable + +import numpy as np +import torch +import torch.nn.functional as F + +from src.transpile.component_pipeline import ComponentModuleSpec +from src.tensor_io import CACTUS_MAGIC +from src.tensor_io import align_offset + + +_GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE = 1.0 / 64.0 + + +@dataclass +class CanonicalizedModel: + module: torch.nn.Module + task: str + family: str + input_names: tuple[str, ...] = () + + +@dataclass(frozen=True) +class _Gemma4NativeMergeSegment: + kind: str + input_start: int + length: int + feature_start: int = 0 + + +@dataclass(frozen=True) +class _Gemma4NativeMergePlan: + segments: tuple[_Gemma4NativeMergeSegment, ...] + pli_token_ids: tuple[int, ...] + + +def _model_name_or_path(model: torch.nn.Module) -> str: + value = getattr(model, "name_or_path", None) + if isinstance(value, str) and value: + return value + config = getattr(model, "config", None) + value = getattr(config, "_name_or_path", None) + if isinstance(value, str) and value: + return value + return "" + + +def _transpile_graph_meta(model: torch.nn.Module, *, adapter_family: str, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: + return { + "adapter_family": adapter_family, + "adapter_type": adapter_type, + "model_name_or_path": _model_name_or_path(model), + "input_names": input_names, + } + + +def _extract_tensor_output(output: object, *, preferred_field: str | None = None) -> torch.Tensor: + if preferred_field is not None: + value = getattr(output, preferred_field, None) + if isinstance(value, torch.Tensor): + return value + + if isinstance(output, torch.Tensor): + return output + if isinstance(output, tuple) and output and isinstance(output[0], torch.Tensor): + return output[0] + + for field_name in ("last_hidden_state", "logits"): + value = getattr(output, field_name, None) + if isinstance(value, torch.Tensor): + return value + + raise TypeError(f"could not extract tensor output from {type(output).__name__}") + + +def _select_last_active_token(hidden_or_logits: torch.Tensor, attention_mask: torch.Tensor | None) -> torch.Tensor: + if attention_mask is None or hidden_or_logits.ndim < 3: + return hidden_or_logits[:, -1:, ...] + + if attention_mask.ndim != 2: + raise ValueError(f"expected 2D attention mask, got shape {tuple(attention_mask.shape)}") + if attention_mask.shape[1] != hidden_or_logits.shape[1]: + raise ValueError( + "attention mask / hidden sequence length mismatch: " + f"{tuple(attention_mask.shape)} vs {tuple(hidden_or_logits.shape)}" + ) + + trailing_zero = attention_mask[:, :1] - attention_mask[:, :1] + shifted_mask = torch.cat((attention_mask[:, 1:], trailing_zero), dim=1) + last_active_mask = torch.logical_and(attention_mask != 0, shifted_mask == 0) + expanded_mask = last_active_mask.to(dtype=hidden_or_logits.dtype) + for _ in range(hidden_or_logits.ndim - 2): + expanded_mask = expanded_mask.unsqueeze(-1) + return (hidden_or_logits * expanded_mask).sum(dim=1, keepdim=True) + + +def _module_floating_dtype(module: torch.nn.Module) -> torch.dtype | None: + for parameter in module.parameters(): + if parameter.is_floating_point(): + return parameter.dtype + for buffer in module.buffers(): + if buffer.is_floating_point(): + return buffer.dtype + return None + + +def _module_device(module: torch.nn.Module) -> torch.device | None: + for parameter in module.parameters(): + return parameter.device + for buffer in module.buffers(): + return buffer.device + return None + + +def _unique_modules(*candidates: object) -> tuple[torch.nn.Module, ...]: + modules: list[torch.nn.Module] = [] + seen: set[int] = set() + for candidate in candidates: + if not isinstance(candidate, torch.nn.Module): + continue + candidate_id = id(candidate) + if candidate_id in seen: + continue + seen.add(candidate_id) + modules.append(candidate) + return tuple(modules) + + +def _gemma4_vision_modules(multimodal_backbone: torch.nn.Module) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + getattr(multimodal_backbone, "vision_tower", None), + getattr(multimodal_backbone, "embed_vision", None), + ) + + +def _gemma4_audio_modules(multimodal_backbone: torch.nn.Module) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + getattr(multimodal_backbone, "audio_tower", None), + getattr(multimodal_backbone, "embed_audio", None), + ) + + +def _gemma4_text_config(multimodal_backbone: torch.nn.Module) -> object: + config = getattr(multimodal_backbone, "config", None) + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + return get_text_config() + except Exception: + pass + return config + + +def _gemma4_special_token_ids(multimodal_backbone: torch.nn.Module) -> tuple[int, int, int]: + config = getattr(multimodal_backbone, "config", None) + text_config = _gemma4_text_config(multimodal_backbone) + image_token_id = int(getattr(config, "image_token_id", 0) or 0) + audio_token_id = int(getattr(config, "audio_token_id", 0) or 0) + pad_token_id = getattr(text_config, "pad_token_id", None) + if pad_token_id is None: + pad_token_id = getattr(config, "pad_token_id", 0) + return image_token_id, audio_token_id, int(pad_token_id or 0) + + +@contextmanager +def _temporary_cpu_float32_modules(modules: tuple[torch.nn.Module, ...]): + promoted: list[tuple[torch.nn.Module, torch.dtype]] = [] + for module in modules: + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + promoted.append((module, dtype)) + module.to(dtype=torch.float32) + try: + yield + finally: + for module, dtype in reversed(promoted): + module.to(dtype=dtype) + + +def _torch_is_compiling() -> bool: + dynamo = getattr(torch, "_dynamo", None) + is_compiling = getattr(dynamo, "is_compiling", None) + if callable(is_compiling): + try: + return bool(is_compiling()) + except Exception: + return False + return False + + +def _gemma4_cpu_safe_text_mlp_enabled(layer: torch.nn.Module, hidden_states: torch.Tensor) -> bool: + if hidden_states.device.type != "cpu" or hidden_states.dtype != torch.float16: + return False + if bool(getattr(layer, "enable_moe_block", False)): + return False + mlp = getattr(layer, "mlp", None) + if mlp is None: + return False + for attr_name in ("gate_proj", "up_proj", "down_proj", "act_fn"): + if not hasattr(mlp, attr_name): + return False + return True + + +def _gemma4_cpu_safe_text_mlp_forward(mlp: torch.nn.Module, hidden_states: torch.Tensor) -> torch.Tensor: + gate = F.linear( + hidden_states, + mlp.gate_proj.weight, + mlp.gate_proj.bias, + ) + up = F.linear( + hidden_states, + mlp.up_proj.weight, + mlp.up_proj.bias, + ) + # Gemma4 text MLPs can overflow in FP16 at the gated product. + # Scaling one branch here keeps the product and down-projection finite. + # The following post-feedforward RMSNorm is scale-invariant, so this + # constant factor cancels back out in the normalized output. + activated = mlp.act_fn(gate) * _GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE + return F.linear( + activated * up, + mlp.down_proj.weight, + mlp.down_proj.bias, + ) + + +def _gemma4_text_attention_forward( + attn: torch.nn.Module, + hidden_states: torch.Tensor, + *, + position_embeddings: torch.Tensor | tuple[torch.Tensor, torch.Tensor] | None, + attention_mask: torch.Tensor | None, + position_ids: torch.LongTensor | None, + past_key_values: object | None, + use_cache: bool, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None, +) -> torch.Tensor: + if shared_kv_states is None: + attn_out, _ = attn( + hidden_states=hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + ) + return attn_out + + from transformers.models.gemma4.modeling_gemma4 import ALL_ATTENTION_FUNCTIONS # type: ignore + from transformers.models.gemma4.modeling_gemma4 import apply_rotary_pos_emb # type: ignore + from transformers.models.gemma4.modeling_gemma4 import eager_attention_forward # type: ignore + + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, attn.head_dim) + if position_embeddings is None: + raise ValueError("Gemma4 text attention requires position embeddings") + cos, sin = position_embeddings + + query_states = attn.q_proj(hidden_states).view(hidden_shape) + query_states = attn.q_norm(query_states) + query_states = apply_rotary_pos_emb(query_states, cos, sin, unsqueeze_dim=2) + query_states = query_states.transpose(1, 2) + + kv_shared_layer_index = getattr(attn, "kv_shared_layer_index", None) + if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_layer_index in shared_kv_states: + key_states, value_states = shared_kv_states[int(kv_shared_layer_index)] + key_states = key_states.to(query_states.device) + value_states = value_states.to(query_states.device) + else: + key_states = attn.k_proj(hidden_states).view(hidden_shape) + value_states = attn.v_proj(hidden_states).view(hidden_shape) if attn.v_proj is not None else key_states + + key_states = attn.k_norm(key_states) + key_states = apply_rotary_pos_emb(key_states, cos, sin, unsqueeze_dim=2) + key_states = key_states.transpose(1, 2) + + value_states = attn.v_norm(value_states) + value_states = value_states.transpose(1, 2) + + if bool(getattr(attn, "store_full_length_kv", False)): + shared_kv_states[int(attn.layer_idx)] = (key_states, value_states) + + attention_interface = eager_attention_forward + if getattr(attn.config, "_attn_implementation", "eager") != "eager": + attention_interface = ALL_ATTENTION_FUNCTIONS[attn.config._attn_implementation] + + attn_output, _ = attention_interface( + attn, + query_states, + key_states, + value_states, + attention_mask, + dropout=attn.attention_dropout if attn.training else 0.0, + scaling=attn.scaling, + sliding_window=attn.sliding_window, + ) + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + return attn.o_proj(attn_output) + + +def _gemma4_text_decoder_layer_forward( + layer: torch.nn.Module, + hidden_states: torch.Tensor, + *, + position_embeddings: torch.Tensor | tuple[torch.Tensor, torch.Tensor] | None = None, + per_layer_input: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: object | None = None, + use_cache: bool = False, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, +) -> torch.Tensor: + if not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): + return layer( + hidden_states, + position_embeddings=position_embeddings, + per_layer_input=per_layer_input, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + ) + + residual = hidden_states + + hidden_states = layer.input_layernorm(hidden_states) + hidden_states = _gemma4_text_attention_forward( + layer.self_attn, + hidden_states, + position_embeddings=position_embeddings, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + shared_kv_states=shared_kv_states, + ) + hidden_states = layer.post_attention_layernorm(hidden_states) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = layer.pre_feedforward_layernorm(hidden_states) + mlp_out = _gemma4_cpu_safe_text_mlp_forward(layer.mlp, hidden_states) + hidden_states = layer.post_feedforward_layernorm(mlp_out) + hidden_states = residual.float() + hidden_states + hidden_states = hidden_states.to(dtype=residual.dtype) + + if getattr(layer, "hidden_size_per_layer_input", 0): + residual = hidden_states + hidden_states = layer.per_layer_input_gate(hidden_states) + hidden_states = layer.act_fn(hidden_states) + if per_layer_input is None: + raise ValueError("Gemma4 layer expected per_layer_input but none was provided") + hidden_states = torch.multiply(hidden_states, per_layer_input) + hidden_states = layer.per_layer_projection(hidden_states) + hidden_states = layer.post_per_layer_input_norm(hidden_states) + hidden_states = residual + hidden_states + + hidden_states = hidden_states * layer.layer_scalar + return hidden_states + + +def _gemma4_text_backbone_forward( + backbone: torch.nn.Module, + *, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor | None, + causal_mask_mapping: dict[str, torch.Tensor], + position_ids: torch.LongTensor, +) -> torch.Tensor: + hidden_states = inputs_embeds + layer_types = tuple(dict.fromkeys(getattr(backbone.config, "layer_types", ()))) + position_embeddings = { + layer_type: backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in layer_types + } + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + + for decoder_layer in backbone.layers[: backbone.config.num_hidden_layers]: + layer_per_input = None + if per_layer_inputs is not None: + layer_per_input = per_layer_inputs[:, :, decoder_layer.layer_idx, :] + hidden_states = _gemma4_text_decoder_layer_forward( + decoder_layer, + hidden_states, + per_layer_input=layer_per_input, + attention_mask=causal_mask_mapping[decoder_layer.attention_type], + position_embeddings=position_embeddings[decoder_layer.attention_type], + position_ids=position_ids, + past_key_values=None, + use_cache=False, + shared_kv_states=shared_kv_states, + ) + + return backbone.norm(hidden_states) + + +def _gemma4_strip_audio_padding(audio_output: object) -> torch.Tensor: + audio_features = getattr(audio_output, "pooler_output", None) + audio_mask_from_encoder = getattr(audio_output, "audio_mel_mask", None) + if not isinstance(audio_features, torch.Tensor): + raise TypeError("Gemma4 audio output did not expose tensor pooler_output") + if not isinstance(audio_mask_from_encoder, torch.Tensor): + return audio_features + all_real_tokens: list[torch.Tensor] = [] + for encodings, padding_mask in zip(audio_features, audio_mask_from_encoder, strict=True): + all_real_tokens.append(encodings[~padding_mask]) + return torch.cat(all_real_tokens, dim=0).unsqueeze(0) + + +def _gemma4_rms_norm_no_scale(hidden_states: torch.Tensor, *, eps: float) -> torch.Tensor: + hidden_states_f32 = hidden_states.float() + mean_squared = hidden_states_f32.pow(2).mean(-1, keepdim=True) + eps + return hidden_states_f32 * torch.pow(mean_squared, -0.5) + + +def _gemma4_rms_norm(hidden_states: torch.Tensor, weight: torch.Tensor, *, eps: float) -> torch.Tensor: + normed = _gemma4_rms_norm_no_scale(hidden_states, eps=eps) + return normed * weight.to(device=normed.device, dtype=normed.dtype) + + +def _gemma4_layer_norm_eps(multimodal_backbone: torch.nn.Module) -> float: + config = getattr(multimodal_backbone, "config", None) + text_config = None + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + text_config = get_text_config() + except Exception: + text_config = None + for candidate in (text_config, config): + value = getattr(candidate, "rms_norm_eps", None) + if value is not None: + return float(value) + value = getattr(candidate, "layer_norm_eps", None) + if value is not None: + return float(value) + return 1e-6 + + +def _gemma4_load_cactus_fp_tensor(path: Path) -> torch.Tensor: + with path.open("rb") as handle: + header = handle.read(84) + if len(header) < 84 or header[:4] != CACTUS_MAGIC: + raise RuntimeError(f"Gemma4 Cactus tensor is missing a valid header: {path}") + + alignment = max(1, int(struct.unpack_from(" 0) + precision = int(struct.unpack_from(" 0 else 0 + data_offset = ( + align_offset(scales_offset + scales_bytes, alignment) + if scales_bytes > 0 + else aligned_header + ) + data_count = byte_size // np.dtype(dtype).itemsize + array = np.memmap(path, mode="r", dtype=dtype, offset=data_offset, shape=(data_count,)) + tensor = torch.from_numpy(np.array(array, copy=True)) + if shape: + tensor = tensor.reshape(shape) + return tensor + + +def _gemma4_load_vision_post_proj_norm(weights_dir: str | Path | None) -> torch.Tensor | None: + if weights_dir is None: + return None + path = Path(weights_dir).expanduser() / "embed_vision_post_proj_norm.weights" + if not path.exists(): + return None + tensor = _gemma4_load_cactus_fp_tensor(path) + if tensor.ndim != 1: + raise RuntimeError(f"Gemma4 vision post-projection norm must be 1D, got shape={tuple(tensor.shape)}") + return tensor.float() + + +def _gemma4_can_use_native_like_vision_features(multimodal_backbone: torch.nn.Module) -> bool: + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + return ( + isinstance(vision_tower, torch.nn.Module) + and isinstance(embed_vision, torch.nn.Module) + and hasattr(vision_tower, "patch_embedder") + and hasattr(vision_tower, "encoder") + and hasattr(embed_vision, "embedding_projection") + ) + + +def _gemma4_can_use_native_like_audio_features(multimodal_backbone: torch.nn.Module) -> bool: + audio_tower = getattr(multimodal_backbone, "audio_tower", None) + embed_audio = getattr(multimodal_backbone, "embed_audio", None) + return ( + isinstance(audio_tower, torch.nn.Module) + and isinstance(embed_audio, torch.nn.Module) + and hasattr(audio_tower, "subsample_conv_projection") + and hasattr(audio_tower, "conformer") + and hasattr(embed_audio, "embedding_projection") + ) + + +def _gemma4_pool_vision_hidden_native_like( + vision_tower: torch.nn.Module, + hidden_states: torch.Tensor, + pixel_position_ids: torch.Tensor, +) -> torch.Tensor: + output_length = int(getattr(vision_tower, "default_output_length")) + pooling_kernel_size = int(getattr(vision_tower, "pooling_kernel_size")) + padding_positions = (pixel_position_ids == -1).all(dim=-1) + pooled_batches: list[torch.Tensor] = [] + for hidden_row, position_row, padding_row in zip(hidden_states, pixel_position_ids, padding_positions, strict=True): + valid_hidden = hidden_row[~padding_row].float() + valid_positions = position_row[~padding_row].clamp(min=0) + max_x = valid_positions[:, 0].max() + 1 + kernel_positions = torch.div(valid_positions, pooling_kernel_size, rounding_mode="floor") + kernel_indices = kernel_positions[:, 0] + torch.div( + max_x, + pooling_kernel_size, + rounding_mode="floor", + ) * kernel_positions[:, 1] + weights = F.one_hot(kernel_indices.long(), output_length).float() / float(pooling_kernel_size**2) + pooled_full = weights.transpose(0, 1) @ valid_hidden + valid_bins = torch.logical_not((weights == 0).all(dim=0)) + pooled_batches.append(pooled_full[valid_bins]) + return torch.cat(pooled_batches, dim=0) + + +def _gemma4_compute_native_like_image_features( + multimodal_backbone: torch.nn.Module, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + *, + post_proj_norm_weight: torch.Tensor | None = None, +) -> torch.Tensor: + if pixel_position_ids is None: + raise TypeError("Gemma4 native-like vision feature path requires pixel_position_ids") + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + if not isinstance(vision_tower, torch.nn.Module) or not isinstance(embed_vision, torch.nn.Module): + raise TypeError("Gemma4 multimodal backbone is missing native-like vision modules") + + padding_positions = (pixel_position_ids == -1).all(dim=-1) + vision_inputs = vision_tower.patch_embedder( + pixel_values, + pixel_position_ids, + padding_positions, + ) + vision_outputs = vision_tower.encoder( + inputs_embeds=vision_inputs, + attention_mask=~padding_positions, + pixel_position_ids=pixel_position_ids, + ) + vision_hidden = _extract_tensor_output(vision_outputs, preferred_field="last_hidden_state") + pooled_hidden = _gemma4_pool_vision_hidden_native_like( + vision_tower, + vision_hidden, + pixel_position_ids, + ) + projection = getattr(embed_vision, "embedding_projection", None) + if not isinstance(projection, torch.nn.Linear): + raise TypeError("Gemma4 vision embedder is missing embedding_projection") + projected = F.linear( + pooled_hidden, + projection.weight.float(), + None if projection.bias is None else projection.bias.float(), + ) + if post_proj_norm_weight is not None and post_proj_norm_weight.numel() > 0: + return _gemma4_rms_norm( + projected, + post_proj_norm_weight, + eps=_gemma4_layer_norm_eps(multimodal_backbone), + ) + return _gemma4_rms_norm_no_scale(projected, eps=float(getattr(embed_vision, "eps", 1e-6))) + + +def _gemma4_compute_native_like_audio_features( + multimodal_backbone: torch.nn.Module, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, +) -> torch.Tensor: + audio_tower = getattr(multimodal_backbone, "audio_tower", None) + embed_audio = getattr(multimodal_backbone, "embed_audio", None) + if not isinstance(audio_tower, torch.nn.Module) or not isinstance(embed_audio, torch.nn.Module): + raise TypeError("Gemma4 multimodal backbone is missing native-like audio modules") + + audio_output = audio_tower(input_features, ~input_features_mask) + if not isinstance(audio_output, tuple) or len(audio_output) != 2: + raise TypeError("Gemma4 audio tower did not return (audio_encodings, audio_mel_mask)") + audio_encodings, audio_mel_mask = audio_output + if not isinstance(audio_encodings, torch.Tensor): + raise TypeError("Gemma4 audio tower did not return tensor audio encodings") + if not isinstance(audio_mel_mask, torch.Tensor): + raise TypeError("Gemma4 audio tower did not return tensor audio mask") + + projection = getattr(embed_audio, "embedding_projection", None) + if not isinstance(projection, torch.nn.Linear): + raise TypeError("Gemma4 audio embedder is missing embedding_projection") + normed = _gemma4_rms_norm_no_scale(audio_encodings, eps=float(getattr(embed_audio, "eps", 1e-6))) + projected = F.linear( + normed, + projection.weight.float(), + None if projection.bias is None else projection.bias.float(), + ) + projected = projected * (1.0 / 16.0) + all_real_tokens: list[torch.Tensor] = [] + for encodings, padding_mask in zip(projected, audio_mel_mask, strict=True): + all_real_tokens.append(encodings[~padding_mask]) + return torch.cat(all_real_tokens, dim=0).unsqueeze(0) + + +def _gemma4_feature_token_count(features: torch.Tensor | None) -> int: + if features is None or features.numel() == 0: + return 0 + if features.ndim == 3: + return int(features.shape[1]) + if features.ndim == 2: + return int(features.shape[0]) + if features.ndim >= 1: + return int(features.reshape(-1, features.shape[-1]).shape[0]) + return 0 + + +def _gemma4_build_native_merge_plan( + multimodal_backbone: torch.nn.Module, + input_ids: torch.Tensor, + *, + image_feature_count: int, + audio_feature_count: int, +) -> _Gemma4NativeMergePlan: + if input_ids.ndim != 2 or int(input_ids.shape[0]) != 1: + raise ValueError( + "Gemma4 native multimodal merge currently expects a static batch-1 input_ids tensor, " + f"got shape {tuple(input_ids.shape)}" + ) + + image_token_id, audio_token_id, pad_token_id = _gemma4_special_token_ids(multimodal_backbone) + token_ids = [int(token) for token in input_ids.detach().cpu().reshape(-1).tolist()] + segments: list[_Gemma4NativeMergeSegment] = [] + pli_token_ids: list[int] = [] + image_offset = 0 + audio_offset = 0 + index = 0 + + while index < len(token_ids): + token_id = token_ids[index] + is_image = image_token_id != 0 and token_id == image_token_id + is_audio = audio_token_id != 0 and token_id == audio_token_id + + if is_image or is_audio: + region_start = index + while index < len(token_ids) and token_ids[index] == token_id: + index += 1 + placeholder_count = index - region_start + if is_image: + insert_count = min(placeholder_count, max(0, image_feature_count - image_offset)) + if insert_count > 0: + segments.append( + _Gemma4NativeMergeSegment( + kind="image", + input_start=region_start, + length=insert_count, + feature_start=image_offset, + ) + ) + pli_token_ids.extend([pad_token_id] * insert_count) + image_offset += insert_count + else: + insert_count = min(placeholder_count, max(0, audio_feature_count - audio_offset)) + if insert_count > 0: + segments.append( + _Gemma4NativeMergeSegment( + kind="audio", + input_start=region_start, + length=insert_count, + feature_start=audio_offset, + ) + ) + pli_token_ids.extend([pad_token_id] * insert_count) + audio_offset += insert_count + continue + + text_start = index + while index < len(token_ids): + next_token = token_ids[index] + if image_token_id != 0 and next_token == image_token_id: + break + if audio_token_id != 0 and next_token == audio_token_id: + break + index += 1 + text_tokens = token_ids[text_start:index] + if text_tokens: + segments.append( + _Gemma4NativeMergeSegment( + kind="text", + input_start=text_start, + length=len(text_tokens), + ) + ) + pli_token_ids.extend(text_tokens) + + if not segments: + raise RuntimeError("Gemma4 native multimodal merge built no input segments") + return _Gemma4NativeMergePlan( + segments=tuple(segments), + pli_token_ids=tuple(pli_token_ids), + ) + + +def _gemma4_feature_sequence( + features: torch.Tensor, + *, + batch_size: int, + feature_dim: int, +) -> torch.Tensor: + if features.ndim == 3: + if int(features.shape[0]) == batch_size: + return features + if int(features.shape[0]) == 1: + return features.expand(batch_size, -1, -1) + if features.ndim == 2: + if int(features.shape[-1]) != feature_dim: + raise ValueError(f"Gemma4 feature dim mismatch: expected {feature_dim}, got {tuple(features.shape)}") + return features.unsqueeze(0).expand(batch_size, -1, -1) + if features.ndim >= 1 and int(features.shape[-1]) == feature_dim: + flattened = features.reshape(-1, feature_dim) + return flattened.unsqueeze(0).expand(batch_size, -1, -1) + raise ValueError(f"unsupported Gemma4 feature tensor shape: {tuple(features.shape)}") + + +def _gemma4_apply_native_merge_plan( + model: torch.nn.Module, + *, + input_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + merge_plan: _Gemma4NativeMergePlan, + pli_token_ids: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + embedding = model.get_input_embeddings() + if not isinstance(embedding, torch.nn.Module): + raise TypeError("Gemma4 model is missing input embeddings") + batch_size = int(input_ids.shape[0]) + feature_dim = int(getattr(embedding, "embedding_dim", 0) or image_features.shape[-1] or audio_features.shape[-1]) + target_dtype = _module_floating_dtype(embedding) or image_features.dtype + image_sequence = _gemma4_feature_sequence( + image_features, + batch_size=batch_size, + feature_dim=feature_dim, + ).to(device=input_ids.device, dtype=target_dtype) + audio_sequence = _gemma4_feature_sequence( + audio_features, + batch_size=batch_size, + feature_dim=feature_dim, + ).to(device=input_ids.device, dtype=target_dtype) + + embedded_segments: list[torch.Tensor] = [] + model_config = getattr(model, "config", None) + text_config = getattr(model_config, "text_config", None) + hidden_scale = float(getattr(model_config, "hidden_size", 0) or 0) + if hidden_scale <= 0.0: + hidden_scale = float(getattr(text_config, "hidden_size", 0) or 0) + if hidden_scale <= 0.0: + hidden_scale = float(feature_dim) + hidden_scale = float(hidden_scale) ** 0.5 + for segment in merge_plan.segments: + if segment.kind == "text": + text_tokens = input_ids[:, segment.input_start : segment.input_start + segment.length] + embedded_segments.append(embedding(text_tokens) * hidden_scale) + elif segment.kind == "image": + embedded_segments.append( + image_sequence[:, segment.feature_start : segment.feature_start + segment.length, :] + ) + elif segment.kind == "audio": + embedded_segments.append( + audio_sequence[:, segment.feature_start : segment.feature_start + segment.length, :] + ) + else: + raise RuntimeError(f"unknown Gemma4 merge segment kind: {segment.kind!r}") + + inputs_embeds = torch.cat(embedded_segments, dim=1) + if pli_token_ids.numel() != inputs_embeds.shape[1]: + raise RuntimeError( + "Gemma4 native merge PLI token count mismatch: " + f"{int(pli_token_ids.numel())} vs {int(inputs_embeds.shape[1])}" + ) + pli_tokens = pli_token_ids.to(device=input_ids.device, dtype=input_ids.dtype).unsqueeze(0) + if batch_size != 1: + pli_tokens = pli_tokens.expand(batch_size, -1) + return inputs_embeds, pli_tokens + + +def _gemma4_remap_sequence_tensor( + tensor: torch.Tensor | None, + *, + merge_plan: _Gemma4NativeMergePlan, +) -> torch.Tensor | None: + if tensor is None: + return None + if tensor.ndim != 2: + raise ValueError(f"Gemma4 merge remap expects a rank-2 tensor, got shape {tuple(tensor.shape)}") + remapped_segments: list[torch.Tensor] = [] + for segment in merge_plan.segments: + remapped_segments.append( + tensor[:, segment.input_start : segment.input_start + segment.length] + ) + if not remapped_segments: + return tensor[:, :0] + return torch.cat(remapped_segments, dim=1) + + +def _gemma4_build_standard_causal_mask_mapping( + *, + create_causal_mask: Callable[..., torch.Tensor] | None, + create_sliding_window_causal_mask: Callable[..., torch.Tensor] | None, + config: object, + inputs_embeds: torch.Tensor, + attention_mask: torch.Tensor | None, + position_ids: torch.Tensor, +) -> dict[str, torch.Tensor]: + if not callable(create_causal_mask) or not callable(create_sliding_window_causal_mask): + raise RuntimeError("Gemma4 standard causal mask helpers are unavailable") + mask_kwargs = { + "config": config, + "inputs_embeds": inputs_embeds, + "attention_mask": attention_mask, + "past_key_values": None, + "position_ids": position_ids, + } + return { + "full_attention": create_causal_mask(**mask_kwargs), + "sliding_attention": create_sliding_window_causal_mask(**mask_kwargs), + } + + +def _infer_input_names(module: torch.nn.Module, *, preferred: tuple[str, ...]) -> tuple[str, ...]: + try: + signature = inspect.signature(module.forward) + except (TypeError, ValueError): + return preferred[:1] + + control_names = { + "self", + "return_dict", + "use_cache", + "past_key_values", + "cache_position", + "position_ids", + "labels", + "decoder_input_ids", + "decoder_attention_mask", + "output_attentions", + "output_hidden_states", + } + available = [ + name + for name, parameter in signature.parameters.items() + if name not in control_names + and parameter.kind in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + ] + matched = [name for name in preferred if name in available] + if matched: + return tuple(matched) + if available: + return tuple(available[: min(2, len(available))]) + return preferred[:1] + + +class BoundInputAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str, metadata_task: str): + super().__init__() + self.model = model + self.input_names = tuple(input_names) + self.family = family + self.metadata_task = metadata_task + + def _kwargs_from_bound_inputs(self, *bound_inputs: torch.Tensor | None) -> dict[str, torch.Tensor]: + provided = tuple(bound_inputs) + if len(self.input_names) > len(provided): + raise ValueError( + f"adapter expected at most {len(provided)} bound inputs, got {len(self.input_names)} names" + ) + kwargs: dict[str, torch.Tensor] = {} + for index, name in enumerate(self.input_names): + value = provided[index] + if value is None: + raise ValueError(f"missing required bound input {index} for {name}") + kwargs[name] = value + return kwargs + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family=self.family, + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + "task": self.metadata_task, + } + } + + +def _gemma4_import_hints(backbone: torch.nn.Module, *, module_path_suffix_prefix: str) -> list[dict[str, object]]: + sliding_window = getattr(backbone.config, "sliding_window", None) + layer_types = list(getattr(backbone.config, "layer_types", [])) + import_hints: list[dict[str, object]] = [] + for layer_index, layer_type in enumerate(layer_types): + attrs: dict[str, object] = {} + if layer_type == "sliding_attention" and sliding_window is not None: + attrs["window_size"] = int(sliding_window) + import_hints.append( + { + "module_path_suffix": f"{module_path_suffix_prefix}.layers.{layer_index}.self_attn", + "op": "scaled_dot_product_attention", + "attrs": attrs, + "meta": { + "attention_layer_type": layer_type, + "attention_layer_index": layer_index, + }, + } + ) + return import_hints + + +class CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + + def forward(self, input_ids: torch.Tensor): + return self.model( + input_ids=input_ids, + use_cache=False, + return_dict=False, + )[0] + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="generic", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + } + } + + +class GemmaCausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.gemma.modeling_gemma import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + causal_mask = self._create_causal_mask( + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=None, + past_key_values=None, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.backbone.rotary_emb(hidden_states, position_ids=position_ids) + checkpoints: list[torch.Tensor] = [] + + for decoder_layer in self.backbone.layers[: self.backbone.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=None, + use_cache=False, + position_embeddings=position_embeddings, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return _gemma4_apply_final_logit_softcapping(self.model, self.model.lm_head(hidden_states)), checkpoints + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + } + } + + +class Gemma3CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.gemma3.modeling_gemma3 import create_causal_mask # type: ignore + from transformers.models.gemma3.modeling_gemma3 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.config.layer_types + } + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], + position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], + position_ids=position_ids, + past_key_values=None, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma3", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + }, + "import_hints": _gemma4_import_hints(self.backbone, module_path_suffix_prefix="backbone"), + } + + +class Gemma4CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + model_backbone = model.model + self.backbone = getattr(model_backbone, "language_model", model_backbone) + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + per_layer_inputs = None + if self.backbone.hidden_size_per_layer_input: + per_layer_inputs = self.backbone.get_per_layer_inputs(input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.unique_layer_types + } + shared_kv_states: dict[str, torch.Tensor] = {} + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + per_layer_input = per_layer_inputs[:, :, i, :] if per_layer_inputs is not None else None + hidden_states = decoder_layer( + hidden_states, + per_layer_input, + shared_kv_states=shared_kv_states, + position_embeddings=position_embeddings[self.backbone.config.layer_types[i]], + attention_mask=causal_mask_mapping[self.backbone.config.layer_types[i]], + position_ids=position_ids, + past_key_values=None, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def debug_first_block(self, input_ids: torch.Tensor) -> dict[str, torch.Tensor]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + per_layer_inputs = None + if self.backbone.hidden_size_per_layer_input: + per_layer_inputs = self.backbone.get_per_layer_inputs(input_ids, inputs_embeds) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + mask_kwargs = { + "config": self.backbone.config, + "inputs_embeds": inputs_embeds, + "attention_mask": None, + "past_key_values": None, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": self._create_causal_mask(**mask_kwargs), + "sliding_attention": self._create_sliding_window_causal_mask(**mask_kwargs), + } + + hidden_states = inputs_embeds + position_embeddings = { + layer_type: self.backbone.rotary_emb(hidden_states, position_ids, layer_type) + for layer_type in self.backbone.unique_layer_types + } + shared_kv_states: dict[str, torch.Tensor] = {} + layer = self.backbone.layers[0] + layer_type = self.backbone.config.layer_types[0] + per_layer_input = per_layer_inputs[:, :, 0, :] if per_layer_inputs is not None else None + + checkpoints: dict[str, torch.Tensor] = {} + + residual = hidden_states + normed = layer.input_layernorm(hidden_states) + checkpoints["pre_attn_norm"] = normed + + attn_out = layer.self_attn( + normed, + position_embeddings=position_embeddings[layer_type], + attention_mask=causal_mask_mapping[layer_type], + position_ids=position_ids, + past_key_values=None, + shared_kv_states=shared_kv_states, + ) + if isinstance(attn_out, tuple): + attn_out = attn_out[0] + checkpoints["attn_o_proj"] = attn_out + + post_attn_norm = layer.post_attention_layernorm(attn_out) + checkpoints["post_attn_norm"] = post_attn_norm + + after_attention = residual + post_attn_norm + checkpoints["after_attention_residual"] = after_attention + + pre_ffn_norm = layer.pre_feedforward_layernorm(after_attention) + checkpoints["pre_ffn_norm"] = pre_ffn_norm + + mlp_out = layer.mlp(pre_ffn_norm) + checkpoints["mlp_down"] = mlp_out + + post_ffn_norm = layer.post_feedforward_layernorm(mlp_out) + checkpoints["post_ffn_norm"] = post_ffn_norm + + after_ffn = after_attention + post_ffn_norm + checkpoints["after_ffn_residual"] = after_ffn + + if per_layer_input is not None: + gated = layer.per_layer_input_gate(after_ffn) + gated = layer.act_fn(gated) + projected = gated * per_layer_input + per_layer_proj = layer.per_layer_projection(projected) + checkpoints["per_layer_input_proj"] = per_layer_proj + post_per_layer_input_norm = layer.post_per_layer_input_norm(per_layer_proj) + checkpoints["post_per_layer_input_norm"] = post_per_layer_input_norm + after_ffn = after_ffn + post_per_layer_input_norm + + layer_scalar = getattr(layer, "layer_scalar", None) + if layer_scalar is not None: + after_ffn = after_ffn * layer_scalar + checkpoints["layer_scalar_out"] = after_ffn + return checkpoints + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + }, + "import_hints": _gemma4_import_hints(self.backbone, module_path_suffix_prefix="backbone"), + } + + +def _gemma4_apply_final_logit_softcapping(model: torch.nn.Module, logits: torch.Tensor) -> torch.Tensor: + config = getattr(model, "config", None) + text_config = None + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + try: + text_config = get_text_config() + except Exception: + text_config = None + cap = getattr(text_config, "final_logit_softcapping", None) + if cap is None: + cap = getattr(config, "final_logit_softcapping", None) + if cap is None: + return logits + cap_value = float(cap) + if cap_value <= 0.0: + return logits + return torch.tanh(logits / cap_value) * cap_value + + +class Gemma4MultimodalCausalLMLogitsAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], weights_dir: str | None = None): + super().__init__( + model, + input_names=input_names, + family="gemma4", + metadata_task="multimodal_causal_lm_logits", + ) + model_backbone = model.model + self.multimodal_backbone = model_backbone + self.backbone = getattr(model_backbone, "language_model", model_backbone) + self.last_token_logits_only = False + self._use_cached_multimodal_features = False + self._native_merge_plan: _Gemma4NativeMergePlan | None = None + self.register_buffer("_cached_image_features", torch.empty(0), persistent=False) + self.register_buffer("_cached_audio_features", torch.empty(0), persistent=False) + self.register_buffer("_native_merge_pli_token_ids", torch.empty(0, dtype=torch.long), persistent=False) + vision_post_proj_norm = _gemma4_load_vision_post_proj_norm(weights_dir) + if vision_post_proj_norm is None: + vision_post_proj_norm = torch.empty(0) + self.register_buffer("_cactus_vision_post_proj_norm", vision_post_proj_norm, persistent=False) + self._capture_cpu_float32_text_modules: list[tuple[torch.nn.Module, torch.dtype]] = [] + self._create_causal_mask_mapping = None + self._create_masks_for_generate = None + self._create_causal_mask = None + self._create_sliding_window_causal_mask = None + try: + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_masks_for_generate # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_causal_mask_mapping = create_causal_mask_mapping + self._create_masks_for_generate = create_masks_for_generate + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + except Exception: + pass + + def _apply_final_logit_softcapping(self, logits: torch.Tensor) -> torch.Tensor: + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def _capture_text_modules(self) -> tuple[torch.nn.Module, ...]: + return _unique_modules( + self.backbone, + getattr(self.model, "get_input_embeddings", lambda: None)(), + getattr(self.model, "lm_head", None), + ) + + def _compute_image_features( + self, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + get_image_features: Callable[..., object], + ) -> torch.Tensor: + multimodal_backbone = self.multimodal_backbone + if _gemma4_can_use_native_like_vision_features(multimodal_backbone): + return _gemma4_compute_native_like_image_features( + multimodal_backbone, + pixel_values, + pixel_position_ids, + post_proj_norm_weight=self._cactus_vision_post_proj_norm, + ) + vision_tower = getattr(multimodal_backbone, "vision_tower", None) + embed_vision = getattr(multimodal_backbone, "embed_vision", None) + vision_modules = _gemma4_vision_modules(multimodal_backbone) + if ( + pixel_values.device.type != "cpu" + or pixel_values.dtype != torch.float16 + or len(vision_modules) != 2 + or _torch_is_compiling() + ): + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + vision_dtype = _module_floating_dtype(vision_tower) + embed_dtype = _module_floating_dtype(embed_vision) + if vision_dtype != torch.float16 and embed_dtype != torch.float16: + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + # Gemma4's CPU float16 vision path can emit non-finite soft tokens; upcast only + # the static image feature extraction path and restore the original module dtypes. + with _temporary_cpu_float32_modules(vision_modules): + return get_image_features( + pixel_values.float(), + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + def _compute_audio_features( + self, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, + get_audio_features: Callable[..., object], + ) -> torch.Tensor: + multimodal_backbone = self.multimodal_backbone + if _gemma4_can_use_native_like_audio_features(multimodal_backbone): + return _gemma4_compute_native_like_audio_features( + multimodal_backbone, + input_features, + input_features_mask, + ) + audio_modules = _gemma4_audio_modules(multimodal_backbone) + if ( + input_features.device.type != "cpu" + or input_features.dtype != torch.float16 + or len(audio_modules) != 2 + or _torch_is_compiling() + ): + audio_output = get_audio_features(input_features, ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + if all(_module_floating_dtype(module) != torch.float16 for module in audio_modules): + audio_output = get_audio_features(input_features, ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + with _temporary_cpu_float32_modules(audio_modules): + audio_output = get_audio_features(input_features.float(), ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + def prepare_cpu_float32_capture(self) -> None: + self._capture_cpu_float32_text_modules.clear() + if os.environ.get("CACTUS_GEMMA4_CAPTURE_FP32") != "1": + return + for module in self._capture_text_modules(): + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + self._capture_cpu_float32_text_modules.append((module, dtype)) + module.to(dtype=torch.float32) + + def restore_cpu_float32_capture(self) -> None: + for module, dtype in reversed(self._capture_cpu_float32_text_modules): + module.to(dtype=dtype) + self._capture_cpu_float32_text_modules.clear() + + def _resolve_image_features( + self, + *, + inputs_embeds: torch.Tensor, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + get_image_features: Callable[..., object], + ) -> torch.Tensor | None: + if pixel_values is None: + return None + if self._use_cached_multimodal_features and self._cached_image_features.numel() > 0: + image_features = self._cached_image_features + else: + image_features = self._compute_image_features( + pixel_values, + pixel_position_ids, + get_image_features, + ) + return image_features.to(inputs_embeds.device, inputs_embeds.dtype) + + def _resolve_audio_features( + self, + *, + inputs_embeds: torch.Tensor, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_audio_features: Callable[..., object], + ) -> torch.Tensor | None: + if input_features is None or input_features_mask is None: + return None + if self._use_cached_multimodal_features and self._cached_audio_features.numel() > 0: + audio_features = self._cached_audio_features + else: + audio_features = self._compute_audio_features( + input_features, + input_features_mask, + get_audio_features, + ) + return audio_features.to(inputs_embeds.device, inputs_embeds.dtype) + + def _prepare_text_backbone_inputs( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor | None, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_placeholder_mask: Callable[..., object], + get_image_features: Callable[..., object], + get_audio_features: Callable[..., object], + ) -> tuple[torch.Tensor, torch.Tensor | None, dict[str, torch.Tensor], torch.LongTensor]: + inputs_embeds = self.model.get_input_embeddings()(input_ids) + + image_features = self._resolve_image_features( + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + get_image_features=get_image_features, + ) + audio_features = self._resolve_audio_features( + inputs_embeds=inputs_embeds, + input_features=input_features, + input_features_mask=input_features_mask, + get_audio_features=get_audio_features, + ) + + if image_features is not None and audio_features is not None and self._native_merge_plan is not None: + inputs_embeds, per_layer_inputs_tokens = _gemma4_apply_native_merge_plan( + self.model, + input_ids=input_ids, + image_features=image_features, + audio_features=audio_features, + merge_plan=self._native_merge_plan, + pli_token_ids=self._native_merge_pli_token_ids, + ) + attention_mask = _gemma4_remap_sequence_tensor( + attention_mask, + merge_plan=self._native_merge_plan, + ) + token_type_ids = _gemma4_remap_sequence_tensor( + token_type_ids, + merge_plan=self._native_merge_plan, + ) + if token_type_ids is None: + raise RuntimeError("Gemma4 native merge requires token_type_ids for multimodal attention masking") + else: + text_mask, image_mask, audio_mask = get_placeholder_mask( + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + if image_features is not None: + inputs_embeds = inputs_embeds.masked_scatter( + image_mask.unsqueeze(-1).expand_as(inputs_embeds), + image_features, + ) + if audio_features is not None: + inputs_embeds = inputs_embeds.masked_scatter( + audio_mask.unsqueeze(-1).expand_as(inputs_embeds), + audio_features, + ) + per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype) + + per_layer_inputs = None + text_config = _gemma4_text_config(self.multimodal_backbone) + if getattr(text_config, "hidden_size_per_layer_input", None): + per_layer_inputs = self.backbone.get_per_layer_inputs(per_layer_inputs_tokens) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + if self._native_merge_plan is not None: + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + elif getattr(text_config, "use_bidirectional_attention", None) == "vision": + causal_mask_mapping = self._create_causal_mask_mapping( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + token_type_ids, + pixel_values, + is_training=self.training, + ) + else: + causal_mask_mapping = self._create_masks_for_generate( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + ) + return inputs_embeds, per_layer_inputs, causal_mask_mapping, position_ids + + def _forward_hidden_states( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor | None, + pixel_values: torch.Tensor | None, + pixel_position_ids: torch.Tensor | None, + input_features: torch.Tensor | None, + input_features_mask: torch.Tensor | None, + get_placeholder_mask: Callable[..., object], + get_image_features: Callable[..., object], + get_audio_features: Callable[..., object], + ) -> torch.Tensor: + inputs_embeds, per_layer_inputs, causal_mask_mapping, position_ids = self._prepare_text_backbone_inputs( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + input_features=input_features, + input_features_mask=input_features_mask, + get_placeholder_mask=get_placeholder_mask, + get_image_features=get_image_features, + get_audio_features=get_audio_features, + ) + return _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + + def prime_static_multimodal_features(self, *bound_inputs: torch.Tensor | None) -> None: + kwargs = self._kwargs_from_bound_inputs(*bound_inputs) + input_ids = kwargs["input_ids"] + token_type_ids = kwargs.get("token_type_ids") + pixel_values = kwargs.get("pixel_values") + pixel_position_ids = kwargs.get("pixel_position_ids") + input_features = kwargs.get("input_features") + input_features_mask = kwargs.get("input_features_mask") + + multimodal_backbone = self.multimodal_backbone + get_placeholder_mask = getattr(multimodal_backbone, "get_placeholder_mask", None) + get_image_features = getattr(multimodal_backbone, "get_image_features", None) + get_audio_features = getattr(multimodal_backbone, "get_audio_features", None) + if not callable(get_placeholder_mask): + return + + with torch.no_grad(): + inputs_embeds = self.model.get_input_embeddings()(input_ids) + _, image_mask, audio_mask = get_placeholder_mask( + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + + if pixel_values is not None and callable(get_image_features) and image_mask.any(): + image_features = self._compute_image_features( + pixel_values, + pixel_position_ids, + get_image_features, + ) + self._cached_image_features = image_features.detach() + else: + self._cached_image_features = self._cached_image_features.new_empty(0) + + if ( + input_features is not None + and input_features_mask is not None + and callable(get_audio_features) + and audio_mask.any() + ): + self._cached_audio_features = self._compute_audio_features( + input_features, + input_features_mask, + get_audio_features, + ).detach() + else: + self._cached_audio_features = self._cached_audio_features.new_empty(0) + + if self._cached_image_features.numel() > 0 and self._cached_audio_features.numel() > 0: + plan = _gemma4_build_native_merge_plan( + self.multimodal_backbone, + input_ids, + image_feature_count=_gemma4_feature_token_count(self._cached_image_features), + audio_feature_count=_gemma4_feature_token_count(self._cached_audio_features), + ) + self._native_merge_plan = plan + self._native_merge_pli_token_ids = torch.tensor( + plan.pli_token_ids, + dtype=torch.long, + ) + else: + self._native_merge_plan = None + self._native_merge_pli_token_ids = self._native_merge_pli_token_ids.new_empty(0) + + self._use_cached_multimodal_features = True + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + kwargs = self._kwargs_from_bound_inputs(*bound_inputs) + input_ids = kwargs["input_ids"] + attention_mask = kwargs.get("attention_mask") + token_type_ids = kwargs.get("token_type_ids") + pixel_values = kwargs.get("pixel_values") + pixel_position_ids = kwargs.get("pixel_position_ids") + input_features = kwargs.get("input_features") + input_features_mask = kwargs.get("input_features_mask") + + multimodal_backbone = self.multimodal_backbone + get_placeholder_mask = getattr(multimodal_backbone, "get_placeholder_mask", None) + get_image_features = getattr(multimodal_backbone, "get_image_features", None) + get_audio_features = getattr(multimodal_backbone, "get_audio_features", None) + lm_head = getattr(self.model, "lm_head", None) + if ( + not callable(get_placeholder_mask) + or not callable(get_image_features) + or not callable(get_audio_features) + or not callable(self._create_causal_mask_mapping) + or not callable(self._create_masks_for_generate) + or not isinstance(lm_head, torch.nn.Module) + ): + outputs = self.model( + return_dict=True, + use_cache=False, + **kwargs, + ) + logits = _extract_tensor_output(outputs, preferred_field="logits") + if self.last_token_logits_only and logits.ndim >= 3: + return _select_last_active_token(logits, attention_mask) + return logits + + hidden_states = self._forward_hidden_states( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + pixel_values=pixel_values, + pixel_position_ids=pixel_position_ids, + input_features=input_features, + input_features_mask=input_features_mask, + get_placeholder_mask=get_placeholder_mask, + get_image_features=get_image_features, + get_audio_features=get_audio_features, + ) + if self.last_token_logits_only: + hidden_states = _select_last_active_token(hidden_states, attention_mask) + return self._apply_final_logit_softcapping(lm_head(hidden_states)) + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=type(self).__name__, + input_names=self.input_names, + ), + "task": self.metadata_task, + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + "last_token_logits_only": bool(self.last_token_logits_only), + }, + "import_hints": _gemma4_import_hints( + self.backbone, + module_path_suffix_prefix="backbone", + ), + } + + +_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS = ( + "inputs_embeds", + "per_layer_inputs", + "full_attention_mask", + "sliding_attention_mask", + "position_ids", +) + + +class _Gemma4MultimodalComponentBase(torch.nn.Module): + def __init__( + self, + model: torch.nn.Module, + *, + input_names: tuple[str, ...], + weights_dir: str | None = None, + native_merge_plan: _Gemma4NativeMergePlan | None = None, + ): + super().__init__() + self.model = model + self.input_names = input_names + self.multimodal_backbone = model.model + model_backbone = model.model + self.backbone = getattr(model_backbone, "language_model", model_backbone) + self._native_merge_plan = native_merge_plan + self.register_buffer( + "_native_merge_pli_token_ids", + torch.tensor(native_merge_plan.pli_token_ids, dtype=torch.long) + if native_merge_plan is not None + else torch.empty(0, dtype=torch.long), + persistent=False, + ) + vision_post_proj_norm = _gemma4_load_vision_post_proj_norm(weights_dir) + if vision_post_proj_norm is None: + vision_post_proj_norm = torch.empty(0) + self.register_buffer("_cactus_vision_post_proj_norm", vision_post_proj_norm, persistent=False) + self._create_causal_mask_mapping = None + self._create_masks_for_generate = None + self._create_causal_mask = None + self._create_sliding_window_causal_mask = None + try: + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_causal_mask_mapping # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_masks_for_generate # type: ignore + from transformers.models.gemma4.modeling_gemma4 import create_sliding_window_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + self._create_causal_mask_mapping = create_causal_mask_mapping + self._create_masks_for_generate = create_masks_for_generate + self._create_sliding_window_causal_mask = create_sliding_window_causal_mask + except Exception: + pass + self._capture_modules: list[tuple[torch.nn.Module, torch.dtype]] = [] + + def _base_graph_meta(self, *, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + **_transpile_graph_meta( + self.model, + adapter_family="gemma4", + adapter_type=adapter_type, + input_names=input_names, + ), + "task": "multimodal_causal_lm_logits", + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + } + + def _compute_image_features( + self, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor | None, + ) -> torch.Tensor: + if _gemma4_can_use_native_like_vision_features(self.multimodal_backbone): + return _gemma4_compute_native_like_image_features( + self.multimodal_backbone, + pixel_values, + pixel_position_ids, + post_proj_norm_weight=self._cactus_vision_post_proj_norm, + ) + get_image_features = getattr(self.multimodal_backbone, "get_image_features", None) + vision_tower = getattr(self.multimodal_backbone, "vision_tower", None) + embed_vision = getattr(self.multimodal_backbone, "embed_vision", None) + vision_modules = _gemma4_vision_modules(self.multimodal_backbone) + if not callable(get_image_features): + raise TypeError("Gemma4 multimodal backbone is missing get_image_features") + if ( + pixel_values.device.type != "cpu" + or pixel_values.dtype != torch.float16 + or len(vision_modules) != 2 + or _torch_is_compiling() + ): + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + vision_dtype = _module_floating_dtype(vision_tower) + embed_dtype = _module_floating_dtype(embed_vision) + if vision_dtype != torch.float16 and embed_dtype != torch.float16: + return get_image_features( + pixel_values, + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + with _temporary_cpu_float32_modules(vision_modules): + return get_image_features( + pixel_values.float(), + pixel_position_ids, + None, + return_dict=True, + ).pooler_output + + def _compute_audio_features( + self, + input_features: torch.Tensor, + input_features_mask: torch.Tensor, + ) -> torch.Tensor: + if _gemma4_can_use_native_like_audio_features(self.multimodal_backbone): + return _gemma4_compute_native_like_audio_features( + self.multimodal_backbone, + input_features, + input_features_mask, + ) + get_audio_features = getattr(self.multimodal_backbone, "get_audio_features", None) + if not callable(get_audio_features): + raise TypeError("Gemma4 multimodal backbone is missing get_audio_features") + audio_modules = _gemma4_audio_modules(self.multimodal_backbone) + if ( + input_features.device.type == "cpu" + and input_features.dtype == torch.float16 + and len(audio_modules) == 2 + and not _torch_is_compiling() + and any(_module_floating_dtype(module) == torch.float16 for module in audio_modules) + ): + with _temporary_cpu_float32_modules(audio_modules): + audio_output = get_audio_features(input_features.float(), ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + audio_output = get_audio_features(input_features, ~input_features_mask, return_dict=True) + return _gemma4_strip_audio_padding(audio_output) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def prepare_for_capture(self, **_: object) -> None: + self.restore_after_capture() + modules = self._modules_to_prepare_for_capture() + if not modules: + return + promoted: list[tuple[torch.nn.Module, torch.dtype]] = [] + for module in modules: + device = _module_device(module) + dtype = _module_floating_dtype(module) + if device is None or device.type != "cpu" or dtype is None or dtype == torch.float32: + continue + promoted.append((module, dtype)) + module.to(dtype=torch.float32) + self._capture_modules = promoted + + def restore_after_capture(self, **_: object) -> None: + for module, dtype in reversed(self._capture_modules): + module.to(dtype=dtype) + self._capture_modules.clear() + + def _prepare_decoder_inputs( + self, + *, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.LongTensor]: + if self._native_merge_plan is not None: + inputs_embeds, per_layer_inputs_tokens = _gemma4_apply_native_merge_plan( + self.model, + input_ids=input_ids, + image_features=image_features, + audio_features=audio_features, + merge_plan=self._native_merge_plan, + pli_token_ids=self._native_merge_pli_token_ids, + ) + attention_mask = _gemma4_remap_sequence_tensor( + attention_mask, + merge_plan=self._native_merge_plan, + ) + token_type_ids = _gemma4_remap_sequence_tensor( + token_type_ids, + merge_plan=self._native_merge_plan, + ) + if token_type_ids is None: + raise RuntimeError("Gemma4 native merge requires token_type_ids for multimodal attention masking") + else: + get_placeholder_mask = getattr(self.multimodal_backbone, "get_placeholder_mask", None) + if not callable(get_placeholder_mask): + raise TypeError("Gemma4 multimodal backbone is missing get_placeholder_mask") + + inputs_embeds = self.model.get_input_embeddings()(input_ids) + text_mask, image_mask, audio_mask = get_placeholder_mask( + token_type_ids=token_type_ids, + input_ids=input_ids, + inputs_embeds=inputs_embeds, + ) + inputs_embeds = inputs_embeds.masked_scatter( + image_mask.unsqueeze(-1).expand_as(inputs_embeds), + image_features.to(inputs_embeds.device, inputs_embeds.dtype), + ) + inputs_embeds = inputs_embeds.masked_scatter( + audio_mask.unsqueeze(-1).expand_as(inputs_embeds), + audio_features.to(inputs_embeds.device, inputs_embeds.dtype), + ) + per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype) + + per_layer_inputs: torch.Tensor + text_config = _gemma4_text_config(self.multimodal_backbone) + if getattr(text_config, "hidden_size_per_layer_input", None): + per_layer_inputs = self.backbone.get_per_layer_inputs(per_layer_inputs_tokens) + per_layer_inputs = self.backbone.project_per_layer_inputs(inputs_embeds, per_layer_inputs) + else: + per_layer_inputs = inputs_embeds.new_empty((inputs_embeds.shape[0], inputs_embeds.shape[1], 0, 0)) + + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) + if self._native_merge_plan is not None: + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + elif getattr(text_config, "use_bidirectional_attention", None) == "vision": + if not callable(self._create_causal_mask_mapping): + raise RuntimeError("Gemma4 component LM encoder requires create_causal_mask_mapping") + causal_mask_mapping = self._create_causal_mask_mapping( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + token_type_ids, + None, + is_training=self.training, + ) + else: + if not callable(self._create_masks_for_generate): + raise RuntimeError("Gemma4 component LM encoder requires create_masks_for_generate") + causal_mask_mapping = self._create_masks_for_generate( + self.multimodal_backbone.config, + inputs_embeds, + attention_mask, + None, + position_ids, + ) + full_attention_mask = causal_mask_mapping["full_attention"] + sliding_attention_mask = causal_mask_mapping.get("sliding_attention", full_attention_mask) + return ( + inputs_embeds, + per_layer_inputs, + full_attention_mask, + sliding_attention_mask, + position_ids, + ) + + +class Gemma4VisionEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=("pixel_values", "pixel_position_ids"), weights_dir=weights_dir) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def forward(self, pixel_values: torch.Tensor, pixel_position_ids: torch.Tensor | None) -> torch.Tensor: + return self._compute_image_features(pixel_values, pixel_position_ids) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("pixel_values", "pixel_position_ids"), + ), + } + + +class Gemma4AudioEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=("input_features", "input_features_mask"), weights_dir=weights_dir) + + def _modules_to_prepare_for_capture(self) -> tuple[torch.nn.Module, ...]: + return () + + def forward(self, input_features: torch.Tensor, input_features_mask: torch.Tensor) -> torch.Tensor: + return self._compute_audio_features(input_features, input_features_mask) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("input_features", "input_features_mask"), + ), + } + + +class Gemma4LMEncoderAdapter(_Gemma4MultimodalComponentBase): + def __init__( + self, + model: torch.nn.Module, + *, + weights_dir: str | None = None, + native_merge_plan: _Gemma4NativeMergePlan | None = None, + ): + super().__init__( + model, + input_names=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + weights_dir=weights_dir, + native_merge_plan=native_merge_plan, + ) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None, + token_type_ids: torch.Tensor, + image_features: torch.Tensor, + audio_features: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.LongTensor]: + return self._prepare_decoder_inputs( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + image_features=image_features, + audio_features=audio_features, + ) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + ), + } + + +class Gemma4DecoderAdapter(_Gemma4MultimodalComponentBase): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__(model, input_names=_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS, weights_dir=weights_dir) + + def forward( + self, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor, + full_attention_mask: torch.Tensor, + sliding_attention_mask: torch.Tensor, + position_ids: torch.LongTensor, + ) -> torch.Tensor: + normalized_per_layer_inputs = per_layer_inputs + if normalized_per_layer_inputs.numel() == 0: + normalized_per_layer_inputs = None + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping={ + "full_attention": full_attention_mask, + "sliding_attention": sliding_attention_mask, + }, + position_ids=position_ids, + ) + logits = self.model.lm_head(hidden_states[:, -1:, :]) + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def get_transpile_metadata(self): + return { + "graph": self._base_graph_meta( + adapter_type=type(self).__name__, + input_names=_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS, + ), + "import_hints": _gemma4_import_hints( + self.backbone, + module_path_suffix_prefix="backbone", + ), + } + + +def _build_gemma4_multimodal_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None, + components: tuple[str, ...] | None = None, +) -> list[ComponentModuleSpec]: + pixel_values = named_tensors["pixel_values"] + pixel_position_ids = named_tensors.get("pixel_position_ids") + input_features = named_tensors["input_features"] + input_features_mask = named_tensors["input_features_mask"] + input_ids = named_tensors["input_ids"] + attention_mask = named_tensors.get("attention_mask") + token_type_ids = named_tensors["token_type_ids"] + + requested_components = tuple(components or ("vision_encoder", "audio_encoder", "lm_encoder", "decoder")) + requested_set = set(requested_components) + if not requested_set: + return [] + + expanded_components: list[str] = [] + + def _require(component: str) -> None: + if component not in expanded_components: + expanded_components.append(component) + + if "decoder" in requested_set: + _require("vision_encoder") + _require("audio_encoder") + _require("lm_encoder") + _require("decoder") + elif "lm_encoder" in requested_set: + _require("vision_encoder") + _require("audio_encoder") + _require("lm_encoder") + else: + if "vision_encoder" in requested_set: + _require("vision_encoder") + if "audio_encoder" in requested_set: + _require("audio_encoder") + + vision_encoder = Gemma4VisionEncoderAdapter(model, weights_dir=weights_dir).eval() + audio_encoder = Gemma4AudioEncoderAdapter(model, weights_dir=weights_dir).eval() + + image_features: torch.Tensor | None = None + audio_features: torch.Tensor | None = None + decoder_inputs: tuple[torch.Tensor, ...] | None = None + native_merge_plan: _Gemma4NativeMergePlan | None = None + + with torch.no_grad(): + if "vision_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): + image_features = vision_encoder(pixel_values, pixel_position_ids) + if "audio_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): + audio_features = audio_encoder(input_features, input_features_mask) + if "lm_encoder" in expanded_components or "decoder" in expanded_components: + if image_features is None: + image_features = vision_encoder(pixel_values, pixel_position_ids) + if audio_features is None: + audio_features = audio_encoder(input_features, input_features_mask) + native_merge_plan = _gemma4_build_native_merge_plan( + getattr(model, "model", model), + input_ids, + image_feature_count=_gemma4_feature_token_count(image_features), + audio_feature_count=_gemma4_feature_token_count(audio_features), + ) + lm_encoder = Gemma4LMEncoderAdapter( + model, + weights_dir=weights_dir, + native_merge_plan=native_merge_plan, + ).eval() + decoder = Gemma4DecoderAdapter(model, weights_dir=weights_dir).eval() + + with torch.no_grad(): + if "decoder" in expanded_components: + decoder_inputs = lm_encoder( + input_ids, + attention_mask, + token_type_ids, + image_features, + audio_features, + ) + + common_graph_meta = { + "weights_dir": weights_dir, + "task": "multimodal_causal_lm_logits", + "adapter_family": "gemma4", + } + specs: list[ComponentModuleSpec] = [] + if "vision_encoder" in expanded_components: + specs.append(ComponentModuleSpec( + component="vision_encoder", + module=vision_encoder, + example_inputs=(pixel_values, pixel_position_ids), + input_keys=("pixel_values", "pixel_position_ids"), + output_keys=("image_features",), + graph_meta={**common_graph_meta, "component": "vision_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "audio_encoder" in expanded_components: + specs.append(ComponentModuleSpec( + component="audio_encoder", + module=audio_encoder, + example_inputs=(input_features, input_features_mask), + input_keys=("input_features", "input_features_mask"), + output_keys=("audio_features",), + graph_meta={**common_graph_meta, "component": "audio_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "lm_encoder" in expanded_components: + if image_features is None or audio_features is None: + raise RuntimeError("Gemma4 lm_encoder spec requires precomputed image/audio features") + specs.append(ComponentModuleSpec( + component="lm_encoder", + module=lm_encoder, + example_inputs=(input_ids, attention_mask, token_type_ids, image_features, audio_features), + input_keys=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), + output_keys=_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS, + graph_meta={**common_graph_meta, "component": "lm_encoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder" in expanded_components: + if decoder_inputs is None: + raise RuntimeError("Gemma4 decoder spec requires precomputed decoder inputs") + specs.append(ComponentModuleSpec( + component="decoder", + module=decoder, + example_inputs=tuple(decoder_inputs), + input_keys=_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS, + output_keys=("logits",), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, + )) + return specs + + +class Qwen35CausalLMLogitsAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + from transformers.models.qwen3_5.modeling_qwen3_5 import create_causal_mask # type: ignore + + self._create_causal_mask = create_causal_mask + + def forward(self, input_ids: torch.Tensor): + return self.debug_forward(input_ids)[0] + + def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[torch.Tensor]]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1) + text_position_ids = position_ids[0] + multimodal_position_ids = position_ids[1:] + + causal_mask = self._create_causal_mask( + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=None, + past_key_values=None, + position_ids=text_position_ids, + ) + linear_attn_mask = self.backbone._update_linear_attn_mask(None, None) + + hidden_states = inputs_embeds + checkpoints: list[torch.Tensor] = [] + position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) + + for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): + layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask + hidden_states = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + attention_mask=layer_mask, + position_ids=text_position_ids, + past_key_values=None, + use_cache=False, + ) + checkpoints.append(hidden_states) + + hidden_states = self.backbone.norm(hidden_states) + checkpoints.append(hidden_states) + return self.model.lm_head(hidden_states), checkpoints + + def get_transpile_metadata(self): + layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + import_hints: list[dict[str, object]] = [] + for layer_index, layer_type in enumerate(layer_types): + import_hints.append( + { + "module_path_suffix": f"backbone.layers.{layer_index}.self_attn", + "meta": { + "attention_layer_type": layer_type, + "attention_layer_index": layer_index, + }, + } + ) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="qwen3_5", + adapter_type=type(self).__name__, + input_names=("input_ids",), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": layer_types, + }, + "import_hints": import_hints, + } + + +class CTCLogitsAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): + super().__init__(model, input_names=input_names, family=family, metadata_task="ctc_logits") + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + outputs = self.model(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) + return _extract_tensor_output(outputs, preferred_field="logits") + + +class EncoderHiddenStatesAdapter(BoundInputAdapter): + def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], family: str): + encoder = None + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder = get_encoder() + if encoder is None: + encoder = getattr(model, "encoder", None) + if encoder is None: + model_attr = getattr(model, "model", None) + if model_attr is not None: + encoder = getattr(model_attr, "encoder", None) + if not isinstance(encoder, torch.nn.Module): + raise NotImplementedError(f"{type(model).__name__} does not expose an encoder module") + super().__init__(model, input_names=input_names, family=family, metadata_task="encoder_hidden_states") + self.encoder = encoder + + def forward(self, *bound_inputs: torch.Tensor | None) -> torch.Tensor: + outputs = self.encoder(return_dict=True, **self._kwargs_from_bound_inputs(*bound_inputs)) + return _extract_tensor_output(outputs, preferred_field="last_hidden_state") + + +def _family_key(model: torch.nn.Module) -> str: + explicit_family = getattr(model, "family", None) + if isinstance(explicit_family, str) and explicit_family: + return explicit_family.lower() + module_name = type(model).__module__ + if module_name.startswith("transformers.models.whisper."): + return "whisper" + if module_name.startswith("transformers.models.gemma4."): + return "gemma4" + if module_name.startswith("transformers.models.gemma3."): + return "gemma3" + if module_name.startswith("transformers.models.gemma."): + return "gemma" + if module_name.startswith("transformers.models.qwen3_5."): + return "qwen3_5" + if module_name.startswith("transformers.models.lfm2_vl."): + return "lfm2_vl" + if module_name.startswith("transformers.models.lfm2_moe."): + return "lfm2_moe" + if module_name.startswith("transformers.models.lfm2."): + return "lfm2" + return "generic" + + +def build_component_module_specs( + model: torch.nn.Module, + *, + task: str, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, + components: tuple[str, ...] | None = None, +) -> list[ComponentModuleSpec] | None: + family = _family_key(model) + if family == "gemma4" and task == "multimodal_causal_lm_logits": + return _build_gemma4_multimodal_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + components=components, + ) + if family == "parakeet_tdt" and task == "tdt_transcription": + from src.transpile.parakeet_tdt_local import build_parakeet_tdt_component_specs + + return build_parakeet_tdt_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + ) + return None + + +def canonicalize_model_interface( + model: torch.nn.Module, + task: str = "causal_lm_logits", + *, + input_names: tuple[str, ...] | None = None, + weights_dir: str | None = None, +) -> CanonicalizedModel: + family = _family_key(model) + adapter_factory: Callable[[torch.nn.Module], torch.nn.Module] + resolved_input_names = tuple(input_names or ()) + + if task == "causal_lm_logits": + if family == "gemma": + adapter_factory = GemmaCausalLMLogitsAdapter + elif family == "gemma4": + adapter_factory = Gemma4CausalLMLogitsAdapter + elif family == "gemma3": + adapter_factory = Gemma3CausalLMLogitsAdapter + elif family == "qwen3_5": + adapter_factory = Qwen35CausalLMLogitsAdapter + else: + adapter_factory = CausalLMLogitsAdapter + resolved_input_names = ("input_ids",) + elif task == "multimodal_causal_lm_logits": + if family != "gemma4": + raise NotImplementedError(f"{type(model).__name__} does not support task={task}") + if not resolved_input_names: + resolved_input_names = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", + ) + adapter_factory = lambda inner_model: Gemma4MultimodalCausalLMLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + weights_dir=weights_dir, + ) + elif task == "ctc_logits": + if not resolved_input_names: + resolved_input_names = _infer_input_names( + model, + preferred=("input_values", "input_features", "attention_mask"), + ) + adapter_factory = lambda inner_model: CTCLogitsAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + elif task == "encoder_hidden_states": + encoder_module = None + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder_module = get_encoder() + if encoder_module is None: + encoder_module = getattr(model, "encoder", None) + if encoder_module is None and getattr(model, "model", None) is not None: + encoder_module = getattr(model.model, "encoder", None) + if not isinstance(encoder_module, torch.nn.Module): + raise NotImplementedError(f"{type(model).__name__} does not support task={task}") + if not resolved_input_names: + resolved_input_names = _infer_input_names( + encoder_module, + preferred=("input_features", "input_values", "attention_mask"), + ) + adapter_factory = lambda inner_model: EncoderHiddenStatesAdapter( # type: ignore[assignment] + inner_model, + input_names=resolved_input_names, + family=family, + ) + else: + raise NotImplementedError(f"unsupported canonicalization task: {task}") + + return CanonicalizedModel( + module=adapter_factory(model).eval(), + task=task, + family=family, + input_names=resolved_input_names, + ) diff --git a/python/src/transpile/model_patterns.py b/python/src/transpile/model_patterns.py new file mode 100644 index 000000000..22be81bda --- /dev/null +++ b/python/src/transpile/model_patterns.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GoldPattern: + name: str + source_models: tuple[str, ...] + semantic_ops: tuple[str, ...] + description: str + + +GOLD_PATTERNS: tuple[GoldPattern, ...] = ( + GoldPattern( + name="decoder_attention_gqa", + source_models=("model_gemma.cpp", "model_qwen.cpp", "model_lfm2.cpp", "model_gemma4.cpp"), + semantic_ops=("rms_norm", "rope", "attention", "attention_int8_hybrid"), + description=( + "Q/K/V projections from the same normalized hidden state; Q and K optionally get per-head RMSNorm, " + "Q/K are rotary-embedded, then Cactus attention consumes native Q-head and KV-head counts." + ), + ), + GoldPattern( + name="gated_mlp_gelu", + source_models=("model_gemma.cpp", "model_gemma4.cpp"), + semantic_ops=("matmul", "gelu", "multiply"), + description="Two projection branches from the same normalized input: GELU(gate) * up, then a down projection.", + ), + GoldPattern( + name="gated_mlp_silu", + source_models=("model_qwen.cpp", "model_lfm2.cpp"), + semantic_ops=("matmul", "silu", "multiply"), + description="Two projection branches from the same normalized input: SiLU(gate) * up, then a down projection.", + ), + GoldPattern( + name="decoder_block_post_attn_norm", + source_models=("model_gemma.cpp", "model_gemma4.cpp", "model_gemma3n.cpp"), + semantic_ops=("rms_norm", "attention", "add_clipped"), + description=( + "Pre-norm attention block with post-attention RMSNorm before the residual add, followed by a second " + "pre-FFN RMSNorm and a post-FFN RMSNorm before the final residual add." + ), + ), + GoldPattern( + name="decoder_block_simple_residual", + source_models=("model_qwen.cpp", "model_lfm2.cpp"), + semantic_ops=("rms_norm", "attention", "add"), + description=( + "Pre-norm attention block with a direct residual add after attention, followed by a second RMSNorm and " + "a gated MLP branch with another direct residual add." + ), + ), + GoldPattern( + name="gemma4_partial_rope_attention", + source_models=("model_gemma4.cpp",), + semantic_ops=("rms_norm", "rope", "concat", "attention"), + description=( + "Gemma4 attention where only a prefix of the head dimension is rotary-embedded, optionally with shared " + "K/V heads and sliding-window attention." + ), + ), + GoldPattern( + name="sliding_window_attention_mask", + source_models=("model_gemma.cpp", "model_gemma4.cpp"), + semantic_ops=("attention",), + description=( + "Exported PyTorch may build a boolean mask with diff/cumsum/comparisons, but the handwritten Cactus " + "models encode the same behavior directly as an attention `window_size`." + ), + ), +) + + +def gold_patterns_by_name() -> dict[str, GoldPattern]: + return {pattern.name: pattern for pattern in GOLD_PATTERNS} diff --git a/python/src/transpile/normalize.py b/python/src/transpile/normalize.py new file mode 100644 index 000000000..b2d3f2de3 --- /dev/null +++ b/python/src/transpile/normalize.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from typing import Any + +import torch + + +def normalize_target(target: str) -> str: + if target.startswith("aten.mul_"): + return "multiply_inplace" + if target.startswith("aten._assert_tensor_metadata"): + return "identity" + if target.startswith("aten.lift_fresh_copy"): + return "identity" + if target.startswith("aten.clone"): + return "identity" + if target.startswith("aten.alias"): + return "identity" + if target.startswith("aten.detach"): + return "identity" + if target.startswith("aten.dropout"): + return "identity" + if target.startswith("aten.native_dropout"): + return "identity" + if target.startswith("aten.arange.start"): + return "arange" + if target.startswith("aten.arange"): + return "arange" + if target.startswith("aten.add"): + return "add" + if target.startswith("aten.sub"): + return "subtract" + if target.startswith("aten.mul"): + return "multiply" + if target.startswith("aten.floor_divide"): + return "floor_divide" + if target.startswith("aten.div"): + return "divide" + if target.startswith("aten.to.dtype"): + return "precision_cast" + if target.startswith("aten.to.device"): + return "identity" + if target.startswith("aten.type_as"): + return "type_as" + if target.startswith("aten.neg"): + return "negate" + if target.startswith("aten.ne.") or target == "aten.ne": + return "not_equal" + if target.startswith("aten.eq.") or target == "aten.eq": + return "equal" + if target.startswith("aten.logical_or") or target.startswith("aten.__or__"): + return "logical_or" + if target.startswith("aten.logical_and") or target.startswith("aten.__and__"): + return "logical_and" + if target.startswith("aten.logical_not") or target.startswith("aten.bitwise_not"): + return "logical_not" + if target.startswith("aten.abs"): + return "abs" + if target.startswith("aten.clamp"): + return "clamp" + if target.startswith("aten.where"): + return "where" + if target.startswith("aten.masked_scatter"): + return "masked_scatter" + if target.startswith("aten.masked_fill"): + return "masked_fill" + if target.startswith("aten.expand"): + return "expand" + if target.startswith("aten.exp"): + return "scalar_exp" + if target.startswith("aten.sqrt"): + return "scalar_sqrt" + if target.startswith("aten.reciprocal"): + return "reciprocal" + if target.startswith("aten.log"): + return "scalar_log" + if target.startswith("aten.rsqrt"): + return "rsqrt" + if target.startswith("aten.cos"): + return "cos" + if target.startswith("aten.sin"): + return "sin" + if target in ("aten.view.default", "aten.reshape.default", "aten._unsafe_view.default"): + return "reshape" + if target == "aten.flatten.using_ints": + return "flatten" + if target.startswith("aten.squeeze"): + return "squeeze" + if target.startswith("aten.unsqueeze"): + return "unsqueeze" + if target == "aten.t.default" or target.startswith("aten.transpose"): + return "transpose" + if target.startswith("aten.permute"): + return "permute" + if target.startswith("aten.numpy_T"): + return "numpy_T" + if target.startswith("aten.movedim"): + return "movedim" + if target.startswith("aten.contiguous"): + return "contiguous" + if target.startswith("aten.mm") or target.startswith("aten.matmul") or target.startswith("aten.bmm"): + return "matmul" + if target.startswith("aten.linear"): + return "linear" + if target.startswith("aten.addmm"): + return "addmm" + if target.startswith("aten.relu"): + return "relu" + if target.startswith("aten.silu"): + return "silu" + if target == "aten.gelu.erf": + return "gelu_erf" + if target.startswith("aten.gelu"): + return "gelu" + if target.startswith("aten.sigmoid"): + return "sigmoid" + if target.startswith("aten.glu"): + return "glu" + if target.startswith("aten.softplus"): + return "softplus" + if target.startswith("aten.tanh"): + return "tanh" + if target.startswith("aten.softmax"): + return "softmax" + if target.startswith("aten.scaled_dot_product_attention"): + return "scaled_dot_product_attention" + if target.startswith("aten.sum"): + return "sum" + if target.startswith("aten.mean"): + return "mean" + if target.startswith("aten.all"): + return "min" + if target.startswith("aten.var"): + return "variance" + if target.startswith("aten.min") or target.startswith("aten.amin"): + return "min" + if target.startswith("aten.max") or target.startswith("aten.amax"): + return "max" + if target.startswith("aten.cat"): + return "cat" + if target.startswith("aten.split_with_sizes"): + return "split_with_sizes" + if target.startswith("aten.chunk"): + return "chunk" + if target.startswith("aten.ones"): + return "ones" + if target.startswith("aten.slice"): + return "slice" + if target.startswith("aten.select"): + return "index" + if target.startswith("aten.gather"): + return "gather" + if target.startswith("aten.pad"): + return "pad" + if target.startswith("aten.one_hot"): + return "one_hot" + if target.startswith("aten.tril"): + return "tril" + if target.startswith("aten.unfold"): + return "unfold" + if target.startswith("aten.embedding"): + return "embedding" + if target.startswith("aten.conv1d"): + return "conv1d" + if target.startswith("aten.conv2d"): + return "conv2d" + if target.startswith("aten.pow"): + return "pow" + if target.startswith("aten.layer_norm"): + return "layer_norm" + if target.startswith("aten.native_layer_norm"): + return "layer_norm" + if target.startswith("aten.group_norm"): + return "group_norm" + if target.startswith("aten.batch_norm") or target.startswith("aten._native_batch_norm_legit_no_training"): + return "batch_norm" + if target.startswith("aten.rms_norm"): + return "rms_norm" + if target == "": + return "getitem" + return target + + +def dtype_to_ir(dtype: Any | None) -> str | None: + if dtype is None: + return None + + dtype_map = { + torch.float16: "fp16", + torch.float32: "fp32", + torch.float64: "fp64", + torch.bfloat16: "bf16", + torch.int8: "int8", + torch.int16: "int16", + torch.int32: "int32", + torch.int64: "int64", + torch.bool: "bool", + } + return dtype_map.get(dtype, str(dtype)) diff --git a/python/src/transpile/ops.py b/python/src/transpile/ops.py new file mode 100644 index 000000000..924336fef --- /dev/null +++ b/python/src/transpile/ops.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class OpSchema: + name: str + num_inputs: int | None = None + attrs: tuple[str, ...] = () + backend_op: str | None = None + is_layout: bool = False + is_reduction: bool = False + is_fusible: bool = False + + +# Small canonical op registry for the transpiler. +# This is metadata for canonical IR ops, not another IR layer. + +OPS: tuple[OpSchema, ...] = ( + OpSchema("add", num_inputs=2, backend_op="add", is_fusible=True), + OpSchema("add_clipped", num_inputs=2, backend_op="add_clipped", is_fusible=True), + OpSchema("subtract", num_inputs=2, backend_op="subtract", is_fusible=True), + OpSchema("multiply", num_inputs=2, backend_op="multiply", is_fusible=True), + OpSchema("divide", num_inputs=2, backend_op="divide", is_fusible=True), + OpSchema("not_equal", num_inputs=2, backend_op="not_equal"), + OpSchema("equal", num_inputs=2), + OpSchema("greater", num_inputs=2), + OpSchema("less_equal", num_inputs=2), + OpSchema("logical_and", num_inputs=2), + OpSchema("logical_or", num_inputs=2), + OpSchema("logical_not", num_inputs=1), + OpSchema("abs", num_inputs=1, backend_op="abs", is_fusible=True), + OpSchema("clamp", num_inputs=1, attrs=("min", "max")), + OpSchema("pow", num_inputs=1, attrs=("exponent",), backend_op="pow"), + OpSchema("negate", num_inputs=1, is_fusible=True), + OpSchema("scalar_add", num_inputs=1, attrs=("value",), backend_op="scalar_add", is_fusible=True), + OpSchema("scalar_subtract", num_inputs=1, attrs=("value",), backend_op="scalar_subtract", is_fusible=True), + OpSchema("scalar_subtract_reverse", num_inputs=1, attrs=("value",)), + OpSchema("scalar_multiply", num_inputs=1, attrs=("value",), backend_op="scalar_multiply", is_fusible=True), + OpSchema("scalar_divide", num_inputs=1, attrs=("value",), backend_op="scalar_divide", is_fusible=True), + OpSchema("scalar_floor_divide", num_inputs=1, attrs=("value",), backend_op="scalar_floor_divide"), + OpSchema("scalar_divide_reverse", num_inputs=1, attrs=("value",)), + OpSchema("scalar_not_equal", num_inputs=1, attrs=("value",), backend_op="scalar_not_equal"), + OpSchema("scalar_equal", num_inputs=1, attrs=("value",)), + OpSchema("scalar_exp", num_inputs=1, backend_op="scalar_exp", is_fusible=True), + OpSchema("scalar_sqrt", num_inputs=1, backend_op="scalar_sqrt", is_fusible=True), + OpSchema("scalar_log", num_inputs=1, backend_op="scalar_log", is_fusible=True), + OpSchema("scalar_cos", num_inputs=1, backend_op="scalar_cos"), + OpSchema("scalar_sin", num_inputs=1, backend_op="scalar_sin"), + OpSchema("precision_cast", num_inputs=1, attrs=("dtype",), backend_op="precision_cast"), + OpSchema("view", num_inputs=1, attrs=("shape",), backend_op="reshape", is_layout=True), + OpSchema("permute", num_inputs=1, attrs=("permutation",), backend_op="permute", is_layout=True), + OpSchema("squeeze", num_inputs=1, attrs=("dim",), is_layout=True), + OpSchema("movedim", num_inputs=1, attrs=("source", "destination"), is_layout=True), + OpSchema("slice", num_inputs=1, attrs=("axis", "start", "end", "step"), backend_op="slice"), + OpSchema("split_with_sizes", num_inputs=1, attrs=("sizes", "axis")), + OpSchema("chunk", num_inputs=1, attrs=("chunks", "axis")), + OpSchema("ones", num_inputs=0, attrs=("shape", "dtype")), + OpSchema("one_hot", num_inputs=1, attrs=("num_classes",)), + OpSchema("tril", num_inputs=1, attrs=("diagonal",)), + OpSchema("unfold", num_inputs=1, attrs=("dimension", "size", "step")), + OpSchema("index", num_inputs=1, attrs=("axis", "index_value"), backend_op="index"), + OpSchema("advanced_index", num_inputs=None), + OpSchema( + "where", + num_inputs=None, + attrs=("true_is_scalar", "true_value", "false_is_scalar", "false_value"), + ), + OpSchema("masked_fill", num_inputs=2, attrs=("value",)), + OpSchema("masked_scatter", num_inputs=3), + OpSchema("getitem", num_inputs=1, attrs=("index",)), + OpSchema("cat", num_inputs=None, attrs=("axis",), backend_op="cat"), + OpSchema("matmul", num_inputs=2, backend_op="matmul"), + OpSchema("linear", num_inputs=None, attrs=("has_bias",), backend_op="matmul"), + OpSchema("addmm", num_inputs=3), + OpSchema("gather", num_inputs=2, attrs=("axis",), backend_op="gather"), + OpSchema("embedding", num_inputs=2, backend_op="embedding_from_tensor"), + OpSchema("sum", num_inputs=1, attrs=("axis", "keepdim"), backend_op="sum", is_reduction=True), + OpSchema("mean", num_inputs=1, attrs=("axis", "keepdim"), backend_op="mean", is_reduction=True), + OpSchema("variance", num_inputs=1, attrs=("axis", "keepdim"), backend_op="variance", is_reduction=True), + OpSchema("min", num_inputs=1, attrs=("axis", "keepdim"), backend_op="min", is_reduction=True), + OpSchema("max", num_inputs=1, attrs=("axis", "keepdim"), backend_op="max", is_reduction=True), + OpSchema("cumsum", num_inputs=1, attrs=("axis",), backend_op="cumsum"), + OpSchema("relu", num_inputs=1, backend_op="relu", is_fusible=True), + OpSchema("silu", num_inputs=1, backend_op="silu", is_fusible=True), + OpSchema("gelu", num_inputs=1, backend_op="gelu", is_fusible=True), + OpSchema("gelu_erf", num_inputs=1, backend_op="gelu_erf", is_fusible=True), + OpSchema("sigmoid", num_inputs=1, backend_op="sigmoid", is_fusible=True), + OpSchema("glu", num_inputs=1, attrs=("axis",), backend_op="glu"), + OpSchema("tanh", num_inputs=1, backend_op="tanh", is_fusible=True), + OpSchema("softplus", num_inputs=1), + OpSchema("softmax", num_inputs=1, attrs=("axis",), backend_op="softmax"), + OpSchema("layer_norm", num_inputs=None, attrs=("eps",), backend_op="layer_norm"), + OpSchema("group_norm", num_inputs=3, attrs=("num_groups", "eps"), backend_op="group_norm"), + OpSchema("batch_norm", num_inputs=5, attrs=("axis", "eps"), backend_op="batch_norm"), + OpSchema("conv1d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv1d"), + OpSchema("conv2d", num_inputs=2, attrs=("stride", "padding", "dilation", "groups"), backend_op="conv2d"), + OpSchema("pad", num_inputs=1, attrs=("pads", "value", "mode")), + OpSchema("lstm_cell", num_inputs=7, backend_op="lstm_cell"), + + # CUSTOM / FUSED OPS + OpSchema("rms_norm", num_inputs=2, attrs=("eps",), backend_op="rms_norm"), + OpSchema("rope", num_inputs=1, attrs=("theta", "position_offset"), backend_op="rope"), + OpSchema("rel_pos_bias", num_inputs=2, attrs=("scale",), backend_op="rel_pos_bias"), + OpSchema( + "attention", + num_inputs=3, + attrs=("scale", "is_causal", "window_size", "mask", "additive_mask"), + backend_op="attention", + ), + OpSchema( + "attention_block", + num_inputs=None, + attrs=( + "scale", + "is_causal", + "window_size", + "has_mask", + "additive_mask", + "has_gate", + "has_bias", + "attention_output_shape", + ), + is_fusible=True, + ), + OpSchema( + "self_attention_block", + num_inputs=None, + attrs=( + "scale", + "is_causal", + "window_size", + "has_mask", + "additive_mask", + "has_gate", + "has_bias", + "has_query_projection_bias", + "has_query_add", + "has_rel_query_add", + "has_key_projection_bias", + "has_value_projection_bias", + "has_rel_pos_bias", + "has_relative_key_projection_bias", + "query_shape", + "key_shape", + "value_shape", + "relative_key_shape", + "rel_pos_scale", + "attention_output_shape", + ), + is_fusible=True, + ), + OpSchema( + "conv_module", + num_inputs=None, + attrs=( + "eps", + "has_pointwise1_bias", + "has_depthwise_bias", + "has_pointwise2_bias", + "depthwise_kernel_size", + "depthwise_padding", + ), + is_fusible=True, + ), + OpSchema( + "gated_deltanet_prefill", + num_inputs=None, + attrs=( + "num_k_heads", + "num_v_heads", + "key_dim", + "value_dim", + "eps", + "chunk_size", + "has_z", + "has_dt_bias", + "has_a_log", + "has_conv", + ), + is_fusible=True, + ), + OpSchema( + "gated_deltanet_decode", + num_inputs=None, + attrs=( + "num_k_heads", + "num_v_heads", + "key_dim", + "value_dim", + "eps", + "has_z", + "has_dt_bias", + "has_a_log", + "has_conv", + ), + is_fusible=True, + ), +) + + +ALL_OPS: dict[str, OpSchema] = {op.name: op for op in OPS} + + +# Import/capture spellings reduced into the canonical registry above. +ALIASES: dict[str, str] = { + "reshape": "view", + "flatten": "view", + "unsqueeze": "view", + "expand": "view", + "contiguous": "view", + "identity": "view", + "transpose": "permute", + "concat": "cat", + "type_as": "precision_cast", + "scaled_dot_product_attention": "attention", + "cos": "scalar_cos", + "sin": "scalar_sin", +} + + +PRECISION_OP = "precision_cast" +SUPPORTED_PRECISIONS: tuple[str, ...] = ("int4", "int8", "fp16", "fp32") + + +def canonicalize_op(name: str) -> str: + return ALIASES.get(name, name) + + +def get_op(name: str) -> OpSchema: + return ALL_OPS[canonicalize_op(name)] + + +def has_op(name: str) -> bool: + return canonicalize_op(name) in ALL_OPS diff --git a/python/src/transpile/optimize_graph.py b/python/src/transpile/optimize_graph.py new file mode 100644 index 000000000..7c622dbf8 --- /dev/null +++ b/python/src/transpile/optimize_graph.py @@ -0,0 +1,1135 @@ +from __future__ import annotations + +from dataclasses import dataclass +import re + +import torch + +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.canonicalize.utils import normalize_dtype_name +from src.transpile.canonicalize.utils import remove_node +from src.transpile.canonicalize.utils import rebuild_graph +from src.transpile.fusion import match_attention +from src.transpile.fusion import match_attention_block +from src.transpile.fusion import match_conv_module +from src.transpile.fusion import match_gated_deltanet +from src.transpile.fusion import match_gated_mlp +from src.transpile.fusion import match_lstm_cell +from src.transpile.fusion import match_rel_pos_bias +from src.transpile.fusion import match_rms_norm +from src.transpile.fusion import match_rope +from src.transpile.fusion import match_self_attention_block +from src.transpile.fusion.common import producer +from src.transpile.fusion.common import strip_passthrough +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.graph_ir import verify_ir +from src.transpile.model_patterns import GOLD_PATTERNS +from src.transpile.normalize import dtype_to_ir + + +@dataclass(frozen=True) +class DetectedPattern: + name: str + anchor_node_id: str + node_ids: tuple[str, ...] + value_ids: tuple[str, ...] + details: dict[str, object] + + +@dataclass(frozen=True) +class FusionConfig: + enable_gated_deltanet: bool = True + enable_lstm_cell: bool = True + enable_rms_norm: bool = True + enable_rope: bool = True + enable_rel_pos_bias: bool = True + enable_attention: bool = True + enable_self_attention_block: bool = True + enable_attention_block: bool = True + enable_conv_module: bool = True + enable_add_clipped: bool = True + + +_DECODER_SELF_ATTN_OUTPUT_WEIGHT_RE = re.compile(r"(?:^|.*\.)layers\.(\d+)\.self_attn\.(?:o_proj|out_proj)\.weight$") +_CAPTURED_LAYER_SLICE_WEIGHT_RE = re.compile( + r"(?:^|.*)backbone_layers_slice_none__\d+__none____modules__(\d+)___self_attn_(?:o_proj|out_proj)_weight$" +) + + +def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: + config = config or FusionConfig() + verify_ir(graph) + canonicalize_exported_graph(graph) + + for _ in range(max_passes): + changed = False + if config.enable_gated_deltanet and fuse_gated_deltanet(graph): + changed = True + if config.enable_lstm_cell and fuse_lstm_cells(graph): + changed = True + if config.enable_rms_norm: + if fuse_rms_norm(graph): + changed = True + if fuse_rms_norm_scale_multiply(graph): + changed = True + if config.enable_rope and fuse_rope(graph): + changed = True + if config.enable_rel_pos_bias and fuse_rel_pos_bias(graph): + changed = True + if config.enable_attention and fuse_attention(graph): + changed = True + if config.enable_self_attention_block and fuse_self_attention_blocks(graph): + changed = True + if config.enable_attention_block and fuse_attention_blocks(graph): + changed = True + if normalize_gemma4_decoder_attention_semantics(graph): + changed = True + if config.enable_conv_module and fuse_conv_modules(graph): + changed = True + if config.enable_add_clipped and fuse_add_clipped(graph): + changed = True + if not changed: + break + canonicalize_exported_graph(graph) + + annotate_gold_patterns(graph) + _prune_unused_inputs(graph) + verify_ir(graph) + return graph + + +def fuse_gated_deltanet(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_gated_deltanet(graph, node) + if match is None: + continue + + inputs = [ + match.normalized_input_value_id, + match.qkv_weight_value_id, + match.a_weight_value_id, + match.b_weight_value_id, + match.norm_weight_value_id, + ] + if match.z_weight_value_id is not None: + inputs.append(match.z_weight_value_id) + if match.dt_bias_value_id is not None: + inputs.append(match.dt_bias_value_id) + if match.a_log_value_id is not None: + inputs.append(match.a_log_value_id) + if match.conv_weight_value_id is not None: + inputs.append(match.conv_weight_value_id) + + node.op = f"gated_deltanet_{match.mode}" + node.inputs = inputs + node.attrs = { + "num_k_heads": int(match.num_k_heads), + "num_v_heads": int(match.num_v_heads), + "key_dim": int(match.key_dim), + "value_dim": int(match.value_dim), + "eps": float(match.eps), + "chunk_size": int(match.chunk_size), + "has_z": bool(match.z_weight_value_id is not None), + "has_dt_bias": bool(match.dt_bias_value_id is not None), + "has_a_log": bool(match.a_log_value_id is not None), + "has_conv": bool(match.conv_weight_value_id is not None), + } + node.kind = "semantic" + node.meta["gated_deltanet_mode"] = match.mode + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_lstm_cells(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "multiply": + continue + + match = match_lstm_cell(graph, node) + if match is None: + continue + + bias_hh_value_id = match.bias_hh_value_id + if bias_hh_value_id is None: + bias_hh_value_id = _materialize_zero_like_constant( + graph, + match.bias_ih_value_id, + suffix="lstm_bias_hh_zero", + ) + + node.op = "lstm_cell" + node.inputs = [ + match.x_value_id, + match.h_prev_value_id, + match.c_prev_value_id, + match.weight_ih_value_id, + match.weight_hh_value_id, + match.bias_ih_value_id, + bias_hh_value_id, + ] + node.outputs = [match.h_output_value_id, match.c_output_value_id] + node.attrs = {} + node.kind = "semantic" + node.meta["lstm_cell_nodes"] = match.node_ids + node.meta["lstm_cell_bias_hh_zero"] = bool(match.bias_hh_value_id is None) + + for other_node_id in match.node_ids: + if other_node_id == node.id: + continue + graph.nodes.pop(other_node_id, None) + if other_node_id in graph.order: + graph.order.remove(other_node_id) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def annotate_gold_patterns(graph: IRGraph) -> list[DetectedPattern]: + _clear_gold_pattern_annotations(graph) + patterns = [ + *_detect_gated_mlps(graph), + *_detect_decoder_attentions(graph), + *_detect_transformer_blocks(graph), + ] + + for pattern in patterns: + for node_id in pattern.node_ids: + node = graph.nodes.get(node_id) + if node is None: + continue + node.meta.setdefault("gold_patterns", []) + node.meta["gold_patterns"].append(pattern.name) + + anchor = graph.nodes.get(pattern.anchor_node_id) + if anchor is not None: + anchor.meta["gold_pattern_anchor"] = True + anchor.meta["gold_pattern_details"] = pattern.details + + graph.meta["gold_patterns_catalog"] = tuple(pattern.name for pattern in GOLD_PATTERNS) + graph.meta["detected_gold_patterns"] = patterns + return patterns + + +def summarize_detected_gold_patterns(graph: IRGraph) -> dict[str, int]: + summary: dict[str, int] = {} + for pattern in annotate_gold_patterns(graph): + summary[pattern.name] = summary.get(pattern.name, 0) + 1 + return summary + + +def fuse_rms_norm(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"multiply", "type_as", "precision_cast"}: + continue + + match = match_rms_norm(graph, node) + if match is None: + continue + input_value = graph.values.get(match.input_value_id) + if input_value is not None and normalize_dtype_name(input_value.dtype) == "fp32": + # Keep FP32 normalization paths unfused so they can execute with FP32 + # reductions and elementwise math instead of the FP16-only RMS kernel. + continue + + weight_value_id = match.weight_value_id + if weight_value_id is None: + if input_value is None or input_value.shape is None or not input_value.shape: + continue + hidden_dim = input_value.shape[-1] + if not isinstance(hidden_dim, int) or hidden_dim <= 0: + continue + weight_value_id = _materialize_ones_constant( + graph, + hidden_dim, + dtype=input_value.dtype, + suffix="rms_norm_ones", + ) + if float(match.weight_offset) != 0.0: + weight_value_id = _materialize_shifted_constant( + graph, + weight_value_id, + float(match.weight_offset), + suffix="rms_norm_scale", + ) + + node.op = "rms_norm" + node.inputs = [match.input_value_id, weight_value_id] + node.attrs = {"eps": float(match.eps)} + node.kind = "semantic" + node.meta["rms_weight_offset"] = float(match.weight_offset) + node.meta["rms_input_value_id"] = match.input_value_id + node.meta["rms_weight_value_id"] = weight_value_id + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rms_norm_scale_multiply(graph: IRGraph) -> bool: + if not _is_gemma4_graph(graph): + return False + + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "multiply" or len(node.inputs) != 2 or len(node.outputs) != 1: + continue + + rms_node: IRNode | None = None + rms_output_id: str | None = None + scale_value_id: str | None = None + for input_id, other_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + candidate = producer(graph, input_id) + if candidate is None or candidate.op != "rms_norm" or len(candidate.inputs) != 2: + continue + if not _is_materialized_ones_constant(graph, candidate.inputs[1]): + continue + rms_node = candidate + rms_output_id = input_id + scale_value_id = other_id + break + + if rms_node is None or rms_output_id is None or scale_value_id is None: + continue + if rms_node.outputs != [rms_output_id]: + continue + if _has_other_users(graph, rms_output_id, excluding=node.id) or rms_output_id in graph.outputs: + continue + + node.op = "rms_norm" + node.inputs = [rms_node.inputs[0], scale_value_id] + node.attrs = dict(rms_node.attrs) + node.kind = "semantic" + node.meta["rms_folded_scale_multiply"] = True + node.meta["rms_input_value_id"] = rms_node.inputs[0] + node.meta["rms_weight_value_id"] = scale_value_id + remove_node(graph, rms_node.id) + graph.values.pop(rms_output_id, None) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rope(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or not node.outputs: + continue + + match = match_rope(graph, node.outputs[0]) + if match is None or match.partial: + continue + + node.op = "rope" + node.inputs = [match.input_value_id] + node.attrs = { + "theta": float(match.theta), + "position_offset": int(match.position_offset), + } + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_attention(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: + continue + + match = match_attention(graph, node) + if match is None: + continue + + window_size = int(node.attrs.get("window_size", 0)) + if len(node.inputs) > 3: + mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) + if mask_info is not None: + node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) + if window_size == 0: + window_size = int(mask_info["window_size"]) + node.meta["window_size_source"] = "mask_pattern" + + if window_size != 0 and "window_size_source" not in node.meta: + node.meta["window_size_source"] = "import_attr" + + semantic_attrs = { + key: value + for key, value in node.attrs.items() + if key not in {"mask", "dropout_p", "scale", "is_causal"} + } + semantic_attrs.update( + { + "scale": float(node.attrs.get("scale", 0.0)), + "is_causal": bool(node.attrs.get("is_causal", True)), + "window_size": window_size, + } + ) + + if ( + node.op == "attention" + and node.inputs[:3] == [match.query_value_id, match.key_value_id, match.value_value_id] + and node.attrs == semantic_attrs + ): + continue + + node.op = "attention" + node.inputs = [match.query_value_id, match.key_value_id, match.value_value_id] + node.attrs = semantic_attrs + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_rel_pos_bias(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_rel_pos_bias(graph, node) + if match is None: + continue + + node.op = "rel_pos_bias" + node.inputs = [match.query_value_id, match.relative_key_value_id] + node.attrs = {"scale": float(match.scale)} + node.kind = "semantic" + node.meta["rel_pos_bias_nodes"] = match.node_ids + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_attention_blocks(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_attention_block(graph, node) + if match is None: + continue + + inputs = [match.query_value_id, match.key_value_id, match.value_value_id] + if match.mask_value_id is not None: + inputs.append(match.mask_value_id) + if match.gate_value_id is not None: + inputs.append(match.gate_value_id) + inputs.append(match.output_projection_weight_value_id) + if match.output_projection_bias_value_id is not None: + inputs.append(match.output_projection_bias_value_id) + + node.op = "attention_block" + node.inputs = inputs + node.attrs = { + "scale": float(match.scale), + "is_causal": bool(match.is_causal), + "window_size": int(match.window_size), + "has_mask": bool(match.mask_value_id is not None), + "additive_mask": bool(match.additive_mask), + "has_gate": bool(match.gate_value_id is not None), + "has_bias": bool(match.output_projection_bias_value_id is not None), + "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), + } + node.kind = "semantic" + node.meta["attention_block_source"] = match.attention_node_id + _apply_decoder_attention_layer_hints( + graph, + node, + output_projection_weight_value_id=match.output_projection_weight_value_id, + ) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: + if not _is_gemma4_graph(graph): + return False + + changed = False + sliding_window = _graph_sliding_window(graph) + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"attention", "scaled_dot_product_attention", "attention_block"}: + continue + if str(node.meta.get("attention_layer_type") or "").strip().lower() != "sliding_attention": + continue + if bool(node.attrs.get("additive_mask", False)): + continue + + mask_input_index: int | None = None + if node.op == "attention_block": + if bool(node.attrs.get("has_mask", False)) and len(node.inputs) > 3: + mask_input_index = 3 + elif len(node.inputs) > 3: + mask_input_index = 3 + + if mask_input_index is not None: + mask_value_id = node.inputs[mask_input_index] + mask_value = graph.values.get(mask_value_id) + if mask_value is None or mask_value.dtype not in (None, "bool"): + continue + mask_info = _extract_sliding_window_mask(graph, mask_value_id) + mask_input_name = _graph_input_name(graph, mask_value_id) + if mask_info is None and mask_input_name != "sliding_attention_mask": + continue + del node.inputs[mask_input_index] + node.meta["gemma4_sliding_mask_elided"] = True + if mask_info is not None: + node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) + if int(node.attrs.get("window_size", 0)) == 0 and int(mask_info["window_size"]) > 0: + node.attrs["window_size"] = int(mask_info["window_size"]) + node.meta.setdefault("window_size_source", "mask_pattern") + elif mask_input_name is not None: + node.meta["gemma4_elided_mask_input_name"] = mask_input_name + changed = True + + if node.op == "attention_block" and bool(node.attrs.get("has_mask", False)): + node.attrs["has_mask"] = False + changed = True + if not bool(node.attrs.get("is_causal", False)): + node.attrs["is_causal"] = True + changed = True + if ( + int(node.attrs.get("window_size", 0)) == 0 + and sliding_window is not None + and sliding_window > 0 + ): + node.attrs["window_size"] = int(sliding_window) + node.meta.setdefault("window_size_source", "layer_type_config") + changed = True + if "additive_mask" in node.attrs and not bool(node.attrs.get("additive_mask", False)): + node.attrs.pop("additive_mask", None) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_self_attention_blocks(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_self_attention_block(graph, node) + if match is None: + continue + + inputs = [match.hidden_value_id, match.query_weight_value_id] + if match.query_projection_bias_value_id is not None: + inputs.append(match.query_projection_bias_value_id) + if match.query_add_value_id is not None: + inputs.append(match.query_add_value_id) + if match.rel_query_add_value_id is not None: + inputs.append(match.rel_query_add_value_id) + inputs.append(match.key_weight_value_id) + if match.key_projection_bias_value_id is not None: + inputs.append(match.key_projection_bias_value_id) + inputs.append(match.value_weight_value_id) + if match.value_projection_bias_value_id is not None: + inputs.append(match.value_projection_bias_value_id) + if match.mask_value_id is not None: + inputs.append(match.mask_value_id) + if match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None: + inputs.append(match.relative_key_input_value_id) + inputs.append(match.relative_key_weight_value_id) + if match.relative_key_projection_bias_value_id is not None: + inputs.append(match.relative_key_projection_bias_value_id) + if match.gate_value_id is not None: + inputs.append(match.gate_value_id) + inputs.append(match.output_projection_weight_value_id) + if match.output_projection_bias_value_id is not None: + inputs.append(match.output_projection_bias_value_id) + + node.op = "self_attention_block" + node.inputs = inputs + node.attrs = { + "scale": float(match.scale), + "is_causal": bool(match.is_causal), + "window_size": int(match.window_size), + "has_mask": bool(match.mask_value_id is not None), + "additive_mask": bool(match.additive_mask), + "has_gate": bool(match.gate_value_id is not None), + "has_bias": bool(match.output_projection_bias_value_id is not None), + "has_query_projection_bias": bool(match.query_projection_bias_value_id is not None), + "has_query_add": bool(match.query_add_value_id is not None), + "has_rel_query_add": bool(match.rel_query_add_value_id is not None), + "has_key_projection_bias": bool(match.key_projection_bias_value_id is not None), + "has_value_projection_bias": bool(match.value_projection_bias_value_id is not None), + "has_rel_pos_bias": bool(match.relative_key_input_value_id is not None and match.relative_key_weight_value_id is not None), + "has_relative_key_projection_bias": bool(match.relative_key_projection_bias_value_id is not None), + "query_shape": tuple(int(dim) for dim in match.query_shape), + "key_shape": tuple(int(dim) for dim in match.key_shape), + "value_shape": tuple(int(dim) for dim in match.value_shape), + "relative_key_shape": tuple(int(dim) for dim in match.relative_key_shape) if match.relative_key_shape is not None else (), + "rel_pos_scale": float(match.rel_pos_scale) if match.rel_pos_scale is not None else 1.0, + "attention_output_shape": tuple(int(dim) for dim in match.attention_output_shape), + } + node.kind = "semantic" + node.meta["self_attention_block_source"] = match.attention_node_id + _apply_decoder_attention_layer_hints( + graph, + node, + output_projection_weight_value_id=match.output_projection_weight_value_id, + ) + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_add_clipped(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op != "add" or len(node.inputs) != 2: + continue + + lhs = strip_passthrough(graph, node.inputs[0]) + rhs = strip_passthrough(graph, node.inputs[1]) + if not (_looks_like_gemma_residual_add(graph, lhs, rhs) or _looks_like_gemma_residual_add(graph, rhs, lhs)): + continue + + node.op = "add_clipped" + node.kind = "semantic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def fuse_conv_modules(graph: IRGraph) -> bool: + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_conv_module(graph, node) + if match is None: + continue + + inputs = [ + match.input_value_id, + match.pointwise1_weight_value_id, + ] + if match.pointwise1_bias_value_id is not None: + inputs.append(match.pointwise1_bias_value_id) + inputs.append(match.depthwise_weight_value_id) + if match.depthwise_bias_value_id is not None: + inputs.append(match.depthwise_bias_value_id) + inputs.extend( + [ + match.batch_norm_weight_value_id, + match.batch_norm_bias_value_id, + match.batch_norm_running_mean_value_id, + match.batch_norm_running_var_value_id, + match.pointwise2_weight_value_id, + ] + ) + if match.pointwise2_bias_value_id is not None: + inputs.append(match.pointwise2_bias_value_id) + + node.op = "conv_module" + node.inputs = inputs + node.attrs = { + "eps": float(match.eps), + "has_pointwise1_bias": bool(match.pointwise1_bias_value_id is not None), + "has_depthwise_bias": bool(match.depthwise_bias_value_id is not None), + "has_pointwise2_bias": bool(match.pointwise2_bias_value_id is not None), + "depthwise_kernel_size": int(match.depthwise_kernel_size), + "depthwise_padding": int(match.depthwise_padding), + } + node.kind = "semantic" + node.meta["conv_module_nodes"] = match.node_ids + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _materialize_shifted_constant(graph: IRGraph, value_id: str, offset: float, *, suffix: str) -> str: + base = graph.constants[value_id] + if not isinstance(base, torch.Tensor): + raise NotImplementedError(f"expected tensor constant for {value_id}, got {type(base).__name__}") + + new_value_id = f"{value_id}_{suffix}" + if new_value_id in graph.constants: + return new_value_id + + shifted = base.detach().cpu() + offset + graph.constants[new_value_id] = shifted + meta: dict[str, object] = {} + source_value = graph.values.get(value_id) + if _is_gemma4_graph(graph) and float(offset) == 1.0 and source_value is not None: + path = source_value.meta.get("path") + kind = source_value.meta.get("kind") + source_name = source_value.meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + meta = dict(source_value.meta) + meta["materialized_from_value_id"] = value_id + meta["materialized_by_op"] = "scalar_add" + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=tuple(shifted.shape), + dtype=dtype_to_ir(shifted.dtype), + producer=None, + users=[], + meta=meta, + ) + return new_value_id + + +def _materialize_zero_like_constant(graph: IRGraph, value_id: str, *, suffix: str) -> str: + base = graph.constants[value_id] + if not isinstance(base, torch.Tensor): + raise NotImplementedError(f"expected tensor constant for {value_id}, got {type(base).__name__}") + + new_value_id = f"{value_id}_{suffix}" + if new_value_id in graph.constants: + return new_value_id + + zero = torch.zeros_like(base.detach().cpu()) + graph.constants[new_value_id] = zero + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=tuple(zero.shape), + dtype=dtype_to_ir(zero.dtype), + producer=None, + users=[], + ) + return new_value_id + + +def _materialize_ones_constant(graph: IRGraph, size: int, *, dtype: str | None, suffix: str) -> str: + dtype_map = { + "fp16": torch.float16, + "bf16": torch.bfloat16, + "fp32": torch.float32, + "fp64": torch.float64, + } + torch_dtype = dtype_map.get(dtype, torch.float32) + new_value_id = f"c_{suffix}_{size}_{dtype or 'fp32'}" + if new_value_id in graph.constants: + return new_value_id + + value = torch.ones((size,), dtype=torch_dtype) + graph.constants[new_value_id] = value + graph.values[new_value_id] = IRValue( + id=new_value_id, + shape=(size,), + dtype=dtype_to_ir(value.dtype), + producer=None, + users=[], + ) + return new_value_id + + +def _is_gemma4_graph(graph: IRGraph) -> bool: + return str(graph.meta.get("adapter_family") or graph.meta.get("family") or "").lower() == "gemma4" + + +def _prune_unused_inputs(graph: IRGraph) -> bool: + kept_inputs: list[str] = [] + changed = False + for value_id in graph.inputs: + value = graph.values.get(value_id) + if value_id in graph.outputs: + kept_inputs.append(value_id) + continue + if value is not None and value.users: + kept_inputs.append(value_id) + continue + changed = True + + if not changed: + return False + + graph.inputs = kept_inputs + rebuild_graph(graph) + return True + + +def _graph_input_name(graph: IRGraph, value_id: str) -> str | None: + input_names = _coerce_string_tuple(graph.meta.get("input_names")) + match = re.match(r"^v_args_(\d+)$", value_id) + if match is not None: + input_index = int(match.group(1)) + if input_index < len(input_names): + return input_names[input_index] + try: + input_index = list(graph.inputs).index(value_id) + except ValueError: + return None + if input_index < len(input_names): + return input_names[input_index] + return None + + +def _is_materialized_ones_constant(graph: IRGraph, value_id: str) -> bool: + value = graph.constants.get(value_id) + if not isinstance(value, torch.Tensor): + return False + if value.numel() == 0: + return False + return bool(torch.all(value.detach().cpu() == 1).item()) + + +def _has_other_users(graph: IRGraph, value_id: str, *, excluding: str) -> bool: + for other_id, other in graph.nodes.items(): + if other_id == excluding: + continue + if value_id in other.inputs: + return True + return False + + +def _clear_gold_pattern_annotations(graph: IRGraph) -> None: + for node in graph.nodes.values(): + node.meta.pop("gold_patterns", None) + node.meta.pop("gold_pattern_anchor", None) + node.meta.pop("gold_pattern_details", None) + + +def _detect_gated_mlps(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None: + continue + + match = match_gated_mlp(graph, node) + if match is None: + continue + + pattern_name = "gated_mlp_gelu" if match.activation == "gelu" else "gated_mlp_silu" + patterns.append( + DetectedPattern( + name=pattern_name, + anchor_node_id=node.id, + node_ids=match.node_ids, + value_ids=(match.input_value_id,), + details={ + "activation": match.activation, + "input_value_id": match.input_value_id, + "gate_weight_value_id": match.gate_weight_value_id, + "up_weight_value_id": match.up_weight_value_id, + "down_weight_value_id": match.down_weight_value_id, + }, + ) + ) + return patterns + + +def _detect_decoder_attentions(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scaled_dot_product_attention", "attention"}: + continue + + match = match_attention(graph, node) + if match is None: + continue + + q_input, k_input, v_input = match.source_input_value_ids + pattern_name = "decoder_attention_gqa" if match.has_gqa_repeat else "decoder_attention" + + node_ids = set(match.node_ids) + details = { + "input_value_ids": (q_input, k_input, v_input), + "q_linear_weight_value_id": match.weight_value_ids[0], + "k_linear_weight_value_id": match.weight_value_ids[1], + "v_linear_weight_value_id": match.weight_value_ids[2], + "has_rope": bool(match.has_rope), + "has_qk_rms_norm": bool(match.has_qk_norm), + "has_gqa_repeat": bool(match.has_gqa_repeat), + "is_causal": bool(match.is_causal), + "scale": float(match.scale), + "window_size_hint": int(match.window_size), + } + + if len(node.inputs) > 3: + details["mask_value_id"] = node.inputs[3] + mask_info = _extract_sliding_window_mask(graph, node.inputs[3]) + if mask_info is not None: + details["mask_pattern"] = "sliding_window_attention_mask" + details["window_size_hint"] = int(mask_info["window_size"]) + node_ids.update(mask_info["node_ids"]) + + patterns.append( + DetectedPattern( + name=pattern_name, + anchor_node_id=node.id, + node_ids=tuple(sorted(node_ids)), + value_ids=(q_input, k_input, v_input), + details=details, + ) + ) + return patterns + + +def _detect_transformer_blocks(graph: IRGraph) -> list[DetectedPattern]: + patterns: list[DetectedPattern] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"add", "add_clipped"} or len(node.inputs) != 2: + continue + + rhs_pattern = _find_anchor_pattern(graph, node.inputs[1], {"gated_mlp_gelu", "gated_mlp_silu"}) + lhs_pattern = _find_anchor_pattern(graph, node.inputs[0], {"gated_mlp_gelu", "gated_mlp_silu"}) + mlp_pattern = rhs_pattern or lhs_pattern + if mlp_pattern is None: + continue + + residual_value_id = node.inputs[0] if rhs_pattern is not None else node.inputs[1] + attn_pattern = _find_anchor_pattern( + graph, + residual_value_id, + {"decoder_attention_gqa", "decoder_attention", "gemma4_partial_rope_attention"}, + ) + if attn_pattern is None: + continue + + node_ids = {node.id} + for candidate in graph.nodes.values(): + gold_patterns = candidate.meta.get("gold_patterns", []) + if attn_pattern in gold_patterns or mlp_pattern in gold_patterns: + node_ids.add(candidate.id) + + patterns.append( + DetectedPattern( + name="decoder_block_post_attn_norm" if node.op == "add_clipped" else "decoder_block_simple_residual", + anchor_node_id=node.id, + node_ids=tuple(sorted(node_ids)), + value_ids=(residual_value_id,), + details={ + "residual_value_id": residual_value_id, + "attention_pattern": attn_pattern, + "mlp_pattern": mlp_pattern, + "residual_op": node.op, + }, + ) + ) + return patterns + + +def _looks_like_gemma_residual_add(graph: IRGraph, residual_value_id: str, branch_value_id: str) -> bool: + branch_node = producer(graph, strip_passthrough(graph, branch_value_id)) + if branch_node is None or branch_node.op != "rms_norm": + return False + return strip_passthrough(graph, residual_value_id) != strip_passthrough(graph, branch_node.inputs[0]) + + +def _find_anchor_pattern(graph: IRGraph, value_id: str, names: set[str]) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + current = strip_passthrough(graph, current) + node = producer(graph, current) + if node is None: + return None + + for name in node.meta.get("gold_patterns", []): + if name in names and node.meta.get("gold_pattern_anchor"): + return name + + if len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _extract_sliding_window_mask(graph: IRGraph, value_id: str) -> dict[str, object] | None: + node_ids: set[str] = set() + top_and = producer(graph, strip_passthrough(graph, value_id)) + if top_and is None or top_and.op not in {"aten.__and__.Tensor", "logical_and"}: + return None + node_ids.add(top_and.id) + + stack = list(top_and.inputs) + saw_diff = False + saw_cumsum = False + window_candidates: list[int] = [] + + while stack: + current = strip_passthrough(graph, stack.pop()) + node = producer(graph, current) + if node is None or node.id in node_ids: + continue + node_ids.add(node.id) + + if node.op in {"aten.diff.default", "diff"}: + saw_diff = True + elif node.op in {"aten.cumsum.default", "cumsum"}: + saw_cumsum = True + elif node.op == "scalar_subtract": + maybe_window = int(node.attrs.get("value", 0)) + if maybe_window > 0: + window_candidates.append(maybe_window) + + stack.extend(node.inputs) + + if not saw_diff or not saw_cumsum: + return None + + return { + "window_size": max((candidate for candidate in window_candidates if candidate > 1), default=0), + "node_ids": tuple(sorted(node_ids)), + } + + +def _apply_decoder_attention_layer_hints( + graph: IRGraph, + node: IRNode, + *, + output_projection_weight_value_id: str, +) -> None: + layer_config = _decoder_attention_layer_config( + graph, + output_projection_weight_value_id=output_projection_weight_value_id, + ) + if layer_config is None: + return + + layer_index, layer_type, sliding_window = layer_config + node.meta.setdefault("attention_layer_index", layer_index) + node.meta.setdefault("attention_layer_type", layer_type) + node.meta.setdefault("attention_layer_source", "weight_source_name") + + if ( + layer_type == "sliding_attention" + and int(node.attrs.get("window_size", 0)) == 0 + and sliding_window is not None + and sliding_window > 0 + ): + node.attrs["window_size"] = int(sliding_window) + node.meta.setdefault("window_size_source", "layer_type_config") + + +def _decoder_attention_layer_config( + graph: IRGraph, + *, + output_projection_weight_value_id: str, +) -> tuple[int, str, int | None] | None: + layer_types = _graph_layer_types(graph) + if not layer_types: + return None + + layer_index = _decoder_attention_layer_index(graph, output_projection_weight_value_id) + if layer_index is None or layer_index < 0 or layer_index >= len(layer_types): + return None + + sliding_window = _graph_sliding_window(graph) + return (layer_index, layer_types[layer_index], sliding_window) + + +def _decoder_attention_layer_index(graph: IRGraph, output_projection_weight_value_id: str) -> int | None: + weight_value = graph.values.get(output_projection_weight_value_id) + candidates: list[str] = [output_projection_weight_value_id] + if weight_value is not None and isinstance(weight_value.meta, dict): + source_name = weight_value.meta.get("source_name") + if isinstance(source_name, str) and source_name: + candidates.insert(0, source_name) + + for candidate in candidates: + match = _DECODER_SELF_ATTN_OUTPUT_WEIGHT_RE.match(candidate) + if match is not None: + return int(match.group(1)) + match = _CAPTURED_LAYER_SLICE_WEIGHT_RE.match(candidate) + if match is not None: + return int(match.group(1)) + return None + + +def _graph_layer_types(graph: IRGraph) -> tuple[str, ...]: + layer_types = _coerce_string_tuple(graph.meta.get("layer_types")) + if layer_types: + return layer_types + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + layer_types = _coerce_string_tuple(provider_meta.get("layer_types")) + if layer_types: + return layer_types + return () + + +def _graph_sliding_window(graph: IRGraph) -> int | None: + sliding_window = _coerce_optional_int(graph.meta.get("sliding_window")) + if sliding_window is not None: + return sliding_window + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + sliding_window = _coerce_optional_int(provider_meta.get("sliding_window")) + if sliding_window is not None: + return sliding_window + return None + + +def _coerce_string_tuple(value: object) -> tuple[str, ...]: + if not isinstance(value, (tuple, list)): + return () + result: list[str] = [] + for item in value: + if not isinstance(item, str): + return () + result.append(item) + return tuple(result) + + +def _coerce_optional_int(value: object) -> int | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return int(value) + return None diff --git a/python/src/transpile/parakeet_tdt_local.py b/python/src/transpile/parakeet_tdt_local.py new file mode 100644 index 000000000..ed2e0a275 --- /dev/null +++ b/python/src/transpile/parakeet_tdt_local.py @@ -0,0 +1,756 @@ +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import replace +import json +from pathlib import Path +import re +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from src.transpile.component_pipeline import ComponentModuleSpec +from src.transpile.audio_preprocess import prepare_native_parakeet_audio_features + + +def _cfg_get(config: dict[str, Any], key: str, default: Any = None) -> Any: + value = config.get(key, default) + return default if value is None else value + + +def _load_tensor_state_dict(model_source: str) -> dict[str, torch.Tensor]: + root = Path(model_source) + safetensors_path = root / "model.safetensors" + if safetensors_path.exists(): + from safetensors.torch import load_file + + return dict(load_file(str(safetensors_path))) + + bin_path = root / "pytorch_model.bin" + if bin_path.exists(): + loaded = torch.load(bin_path, map_location="cpu") + if isinstance(loaded, dict): + return { + str(key): value + for key, value in loaded.items() + if isinstance(value, torch.Tensor) + } + raise RuntimeError(f"unsupported Parakeet TDT checkpoint format in {model_source}") + + +@dataclass +class ParakeetTDTConfig: + model_source: str + sample_rate: int + num_mel_bins: int + hidden_dim: int + num_layers: int + attention_heads: int + attention_head_dim: int + attention_scale: float + ff_intermediate_dim: int + conv_kernel_size: int + subsampling_factor: int + subsampling_conv_channels: int + predictor_hidden_dim: int + predictor_num_layers: int + joint_dim: int + num_tdt_durations: int + tdt_durations: tuple[int, ...] + blank_id: int + vocabulary: tuple[str, ...] + encoder_hidden_act: str + + +def prepare_parakeet_tdt_audio_features( + audio_file: str | Path, + *, + expected_frames: int | None, + expected_mels: int, + torch_dtype: torch.dtype, +) -> tuple[torch.Tensor, int]: + return prepare_native_parakeet_audio_features( + audio_file, + expected_frames=expected_frames, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + + +def greedy_decode_parakeet_tdt_token_ids( + *, + config: ParakeetTDTConfig, + encoder_hidden_states: np.ndarray, + initial_states: tuple[np.ndarray, ...], + step, +) -> list[int]: + hidden = np.ascontiguousarray(np.asarray(encoder_hidden_states)) + if hidden.ndim != 3 or hidden.shape[0] != 1: + raise ValueError( + f"expected encoder_hidden_states with shape [1, T, D], got {tuple(hidden.shape)}" + ) + predictor_layers = int(config.predictor_num_layers) + if len(initial_states) != predictor_layers * 2: + raise ValueError( + f"expected {predictor_layers * 2} initial state tensors, got {len(initial_states)}" + ) + + states = tuple(np.ascontiguousarray(np.asarray(state)) for state in initial_states) + durations = [int(value) for value in config.tdt_durations] + if not durations and int(config.num_tdt_durations) > 0: + durations = list(range(int(config.num_tdt_durations))) + duration_class_count = max(len(durations), int(config.num_tdt_durations)) + last_token = _initial_parakeet_tdt_blank_id(config) + emitted: list[int] = [] + time_index = 0 + + while time_index < int(hidden.shape[1]): + frame = np.ascontiguousarray(hidden[:, time_index, :]) + advanced = False + symbols_added = 0 + + while symbols_added < 10: + logits, next_states = step(frame, last_token, states) + logits_array = np.asarray(logits, dtype=np.float32) + total_classes = int(logits_array.shape[-1]) + active_duration_count = duration_class_count + if active_duration_count <= 0 or active_duration_count >= total_classes: + active_duration_count = max(1, min(total_classes - 1, int(config.num_tdt_durations) or len(durations) or 1)) + if len(durations) != active_duration_count: + durations = list(range(active_duration_count)) + token_class_count = total_classes - active_duration_count + blank_id = _effective_parakeet_tdt_blank_id( + config, + token_class_count=token_class_count, + ) + if last_token < 0 or last_token >= token_class_count: + last_token = blank_id + token_scores = logits_array[:, :token_class_count] + duration_scores = logits_array[:, token_class_count:] + next_token = int(np.argmax(token_scores[0])) + duration_index = int(np.argmax(duration_scores[0])) + skip = int(durations[min(duration_index, len(durations) - 1)]) if durations else 1 + + if next_token != blank_id: + emitted.append(next_token) + last_token = next_token + states = tuple(np.ascontiguousarray(np.asarray(state).copy()) for state in next_states) + + symbols_added += 1 + + if skip > 0: + time_index += skip + advanced = True + break + if next_token == blank_id: + time_index += 1 + advanced = True + break + + if not advanced: + time_index += 1 + + return emitted + + +def _initial_parakeet_tdt_blank_id(config: ParakeetTDTConfig) -> int: + blank_id = int(config.blank_id) + vocab_size = len(config.vocabulary) + if vocab_size > 0 and blank_id == vocab_size - 1: + return vocab_size + return blank_id + + +def _effective_parakeet_tdt_blank_id( + config: ParakeetTDTConfig, + *, + token_class_count: int, +) -> int: + blank_id = int(config.blank_id) + vocab_size = len(config.vocabulary) + if vocab_size > 0 and blank_id == vocab_size - 1 and token_class_count == vocab_size + 1: + return token_class_count - 1 + if 0 <= blank_id < token_class_count: + return blank_id + if vocab_size > 0 and token_class_count == vocab_size + 1: + return token_class_count - 1 + return max(0, token_class_count - 1) + + +def load_parakeet_tdt_config(model_source: str) -> ParakeetTDTConfig: + config_path = Path(model_source) / "config.json" + if not config_path.exists(): + raise FileNotFoundError(f"missing config.json for Parakeet TDT: {config_path}") + root = json.loads(config_path.read_text()) + encoder = root.get("encoder") or {} + decoder = root.get("decoder") or {} + prediction = decoder.get("prediction") or decoder.get("prednet") or {} + joint = root.get("joint") or {} + jointnet = joint.get("jointnet") or {} + model_defaults = root.get("model_defaults") or {} + preprocessor = root.get("preprocessor") or {} + + hidden_dim = int(_cfg_get(encoder, "d_model", _cfg_get(encoder, "hidden_size", 0))) + attention_heads = int(_cfg_get(encoder, "n_heads", _cfg_get(encoder, "num_attention_heads", 0))) + attention_head_dim = hidden_dim // max(attention_heads, 1) + tdt_durations = tuple(int(value) for value in _cfg_get(model_defaults, "tdt_durations", (0, 1, 2, 3, 4))) + vocabulary = tuple(str(value) for value in _cfg_get(joint, "vocabulary", ())) + decoder_vocab_size = int(_cfg_get(decoder, "vocab_size", len(vocabulary))) + decoding = root.get("decoding") or {} + blank_id_raw = _cfg_get(decoding, "blank_id", _cfg_get(decoder, "blank_id", None)) + if blank_id_raw is None: + blank_id = decoder_vocab_size + else: + try: + blank_id = int(blank_id_raw) + except (TypeError, ValueError): + blank_id = decoder_vocab_size + if blank_id < 0: + blank_id = decoder_vocab_size + + return ParakeetTDTConfig( + model_source=model_source, + sample_rate=int(_cfg_get(preprocessor, "sample_rate", 16000)), + num_mel_bins=int(_cfg_get(preprocessor, "features", _cfg_get(encoder, "feat_in", 128))), + hidden_dim=hidden_dim, + num_layers=int(_cfg_get(encoder, "n_layers", _cfg_get(encoder, "num_hidden_layers", 0))), + attention_heads=attention_heads, + attention_head_dim=attention_head_dim, + attention_scale=float(attention_head_dim ** -0.5 if attention_head_dim > 0 else 1.0), + ff_intermediate_dim=int( + _cfg_get( + encoder, + "ffn_hidden_size", + round(hidden_dim * float(_cfg_get(encoder, "ff_expansion_factor", 4.0))), + ) + ), + conv_kernel_size=int(_cfg_get(encoder, "conv_kernel_size", 9)), + subsampling_factor=int(_cfg_get(encoder, "subsampling_factor", 8)), + subsampling_conv_channels=int(_cfg_get(encoder, "subsampling_conv_channels", 256)), + predictor_hidden_dim=int(_cfg_get(prediction, "pred_hidden", _cfg_get(model_defaults, "pred_hidden", 640))), + predictor_num_layers=int(_cfg_get(prediction, "pred_rnn_layers", 1)), + joint_dim=int(_cfg_get(jointnet, "joint_hidden", _cfg_get(model_defaults, "joint_hidden", 640))), + num_tdt_durations=int(_cfg_get(model_defaults, "num_tdt_durations", len(tdt_durations))), + tdt_durations=tdt_durations, + blank_id=blank_id, + vocabulary=vocabulary, + encoder_hidden_act=str(_cfg_get(encoder, "activation", "silu")).lower(), + ) + + +def _copy_linear_weight(linear: nn.Linear, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + linear.weight.data.copy_(weight.to(dtype=linear.weight.dtype, device=linear.weight.device)) + if linear.bias is not None: + if bias is None: + linear.bias.data.zero_() + else: + linear.bias.data.copy_(bias.to(dtype=linear.bias.dtype, device=linear.bias.device)) + + +def _copy_conv2d_weight(conv: nn.Conv2d, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + conv.weight.data.copy_(weight.permute(0, 3, 1, 2).contiguous().to(dtype=conv.weight.dtype, device=conv.weight.device)) + if conv.bias is not None: + if bias is None: + conv.bias.data.zero_() + else: + conv.bias.data.copy_(bias.to(dtype=conv.bias.dtype, device=conv.bias.device)) + + +def _copy_conv1d_weight(conv: nn.Conv1d, weight: torch.Tensor, *, bias: torch.Tensor | None = None) -> None: + tensor = weight + if tensor.ndim == 3 and tensor.shape[1] != conv.in_channels // conv.groups: + tensor = tensor.permute(0, 2, 1).contiguous() + conv.weight.data.copy_(tensor.to(dtype=conv.weight.dtype, device=conv.weight.device)) + if conv.bias is not None: + if bias is None: + conv.bias.data.zero_() + else: + conv.bias.data.copy_(bias.to(dtype=conv.bias.dtype, device=conv.bias.device)) + + +def _apply_activation(x: torch.Tensor, activation: str) -> torch.Tensor: + if "gelu" in activation: + return F.gelu(x) + if activation == "relu": + return F.relu(x) + return F.silu(x) + + +def _relative_position_embeddings(*, seq_len: int, hidden_dim: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: + rel_positions = torch.arange(seq_len - 1, -seq_len, -1, device=device, dtype=torch.float32) + half_dim = hidden_dim // 2 + inv_freq = 1.0 / ( + 10000.0 + ** (torch.arange(0, half_dim, device=device, dtype=torch.float32) * 2.0 / float(max(hidden_dim, 1))) + ) + angles = rel_positions.unsqueeze(1) * inv_freq.unsqueeze(0) + sin = torch.sin(angles) + cos = torch.cos(angles) + embeddings = torch.stack((sin, cos), dim=-1).reshape(2 * seq_len - 1, half_dim * 2) + if embeddings.shape[1] < hidden_dim: + pad = torch.zeros( + (embeddings.shape[0], hidden_dim - embeddings.shape[1]), + device=device, + dtype=embeddings.dtype, + ) + embeddings = torch.cat((embeddings, pad), dim=1) + elif embeddings.shape[1] > hidden_dim: + embeddings = embeddings[:, :hidden_dim] + return embeddings.to(dtype=dtype) + + +def _relative_position_bias(query: torch.Tensor, relative_key: torch.Tensor, *, scale: float) -> torch.Tensor: + batch, heads, seq_len, _ = query.shape + scores = torch.matmul(query, relative_key.transpose(-1, -2)) + rel_index = ( + torch.arange(seq_len, device=query.device).view(seq_len, 1) + - torch.arange(seq_len, device=query.device).view(1, seq_len) + + (seq_len - 1) + ) + rel_index = rel_index.view(1, 1, seq_len, seq_len).expand(batch, heads, seq_len, seq_len) + gathered = scores.gather(-1, rel_index) + return gathered * float(scale) + + +class ParakeetTDTFeedForward(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.linear1 = nn.Linear(config.hidden_dim, config.ff_intermediate_dim, bias=False) + self.linear2 = nn.Linear(config.ff_intermediate_dim, config.hidden_dim, bias=False) + _copy_linear_weight(self.linear1, state_dict[f"{prefix}.linear1.weight"]) + _copy_linear_weight(self.linear2, state_dict[f"{prefix}.linear2.weight"]) + + def forward(self, x: torch.Tensor, *, activation: str) -> torch.Tensor: + return self.linear2(_apply_activation(self.linear1(x), activation)) + + +class ParakeetTDTSelfAttention(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.linear_q = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_k = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_v = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_out = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + self.linear_pos = nn.Linear(config.hidden_dim, config.hidden_dim, bias=False) + _copy_linear_weight(self.linear_q, state_dict[f"{prefix}.linear_q.weight"]) + _copy_linear_weight(self.linear_k, state_dict[f"{prefix}.linear_k.weight"]) + _copy_linear_weight(self.linear_v, state_dict[f"{prefix}.linear_v.weight"]) + _copy_linear_weight(self.linear_out, state_dict[f"{prefix}.linear_out.weight"]) + _copy_linear_weight(self.linear_pos, state_dict[f"{prefix}.linear_pos.weight"]) + self.pos_bias_u = nn.Parameter(state_dict[f"{prefix}.pos_bias_u"].clone()) + self.pos_bias_v = nn.Parameter(state_dict[f"{prefix}.pos_bias_v"].clone()) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + batch, seq_len, _ = x.shape + num_heads = self.config.attention_heads + head_dim = self.config.attention_head_dim + + q = self.linear_q(x).view(batch, seq_len, num_heads, head_dim) + k = self.linear_k(x).view(batch, seq_len, num_heads, head_dim) + v = self.linear_v(x).view(batch, seq_len, num_heads, head_dim) + + rel_pos = _relative_position_embeddings( + seq_len=seq_len, + hidden_dim=self.config.hidden_dim, + device=x.device, + dtype=x.dtype, + ) + rel_k = self.linear_pos(rel_pos).view(1, 2 * seq_len - 1, num_heads, head_dim) + + q_u = q + self.pos_bias_u.view(1, 1, num_heads, head_dim).to(dtype=x.dtype, device=x.device) + q_v = q + self.pos_bias_v.view(1, 1, num_heads, head_dim).to(dtype=x.dtype, device=x.device) + + q_u_heads = q_u.permute(0, 2, 1, 3) + q_v_heads = q_v.permute(0, 2, 1, 3) + k_heads = k.permute(0, 2, 1, 3) + v_heads = v.permute(0, 2, 1, 3) + rel_k_heads = rel_k.permute(0, 2, 1, 3) + + rel_bias = _relative_position_bias( + q_v_heads, + rel_k_heads, + scale=self.config.attention_scale, + ) + attn = F.scaled_dot_product_attention( + q_u_heads, + k_heads, + v_heads, + attn_mask=rel_bias, + dropout_p=0.0, + is_causal=False, + ) + attn = attn.permute(0, 2, 1, 3).reshape(batch, seq_len, self.config.hidden_dim) + return self.linear_out(attn) + + +class ParakeetTDTConformerConv(nn.Module): + def __init__(self, config: ParakeetTDTConfig, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + hidden_dim = config.hidden_dim + kernel = config.conv_kernel_size + self.pointwise_conv1 = nn.Conv1d(hidden_dim, hidden_dim * 2, kernel_size=1, bias=False) + self.depthwise_conv = nn.Conv1d( + hidden_dim, + hidden_dim, + kernel_size=kernel, + padding=kernel // 2, + groups=hidden_dim, + bias=False, + ) + self.batch_norm = nn.BatchNorm1d(hidden_dim, eps=1e-5, affine=True, track_running_stats=True) + self.pointwise_conv2 = nn.Conv1d(hidden_dim, hidden_dim, kernel_size=1, bias=False) + _copy_conv1d_weight(self.pointwise_conv1, state_dict[f"{prefix}.pointwise_conv1.weight"]) + _copy_conv1d_weight(self.depthwise_conv, state_dict[f"{prefix}.depthwise_conv.weight"]) + _copy_conv1d_weight(self.pointwise_conv2, state_dict[f"{prefix}.pointwise_conv2.weight"]) + self.batch_norm.weight.data.copy_(state_dict[f"{prefix}.batch_norm.weight"].to(dtype=self.batch_norm.weight.dtype)) + self.batch_norm.bias.data.copy_(state_dict[f"{prefix}.batch_norm.bias"].to(dtype=self.batch_norm.bias.dtype)) + self.batch_norm.running_mean.data.copy_( + state_dict[f"{prefix}.batch_norm.running_mean"].to(dtype=self.batch_norm.running_mean.dtype) + ) + self.batch_norm.running_var.data.copy_( + state_dict[f"{prefix}.batch_norm.running_var"].to(dtype=self.batch_norm.running_var.dtype) + ) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.transpose(1, 2) + x = self.pointwise_conv1(x) + x = F.glu(x, dim=1) + x = self.depthwise_conv(x) + x = self.batch_norm(x) + x = _apply_activation(x, self.config.encoder_hidden_act) + x = self.pointwise_conv2(x) + return x.transpose(1, 2) + + +class ParakeetTDTEncoderLayer(nn.Module): + def __init__(self, config: ParakeetTDTConfig, layer_index: int, state_dict: dict[str, torch.Tensor]): + super().__init__() + prefix = f"encoder.layers.{layer_index}" + self.feed_forward1 = ParakeetTDTFeedForward(config, f"{prefix}.feed_forward1", state_dict) + self.feed_forward2 = ParakeetTDTFeedForward(config, f"{prefix}.feed_forward2", state_dict) + self.self_attn = ParakeetTDTSelfAttention(config, f"{prefix}.self_attn", state_dict) + self.conv = ParakeetTDTConformerConv(config, f"{prefix}.conv", state_dict) + self.norm_feed_forward1 = nn.LayerNorm(config.hidden_dim) + self.norm_self_att = nn.LayerNorm(config.hidden_dim) + self.norm_conv = nn.LayerNorm(config.hidden_dim) + self.norm_feed_forward2 = nn.LayerNorm(config.hidden_dim) + self.norm_out = nn.LayerNorm(config.hidden_dim) + for name in ("norm_feed_forward1", "norm_self_att", "norm_conv", "norm_feed_forward2", "norm_out"): + module = getattr(self, name) + module.weight.data.copy_(state_dict[f"{prefix}.{name}.weight"].to(dtype=module.weight.dtype)) + module.bias.data.copy_(state_dict[f"{prefix}.{name}.bias"].to(dtype=module.bias.dtype)) + self.config = config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + 0.5 * self.feed_forward1(self.norm_feed_forward1(x), activation=self.config.encoder_hidden_act) + x = x + self.self_attn(self.norm_self_att(x)) + x = x + self.conv(self.norm_conv(x)) + x = x + 0.5 * self.feed_forward2(self.norm_feed_forward2(x), activation=self.config.encoder_hidden_act) + return self.norm_out(x) + + +class ParakeetTDTPreEncode(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + channels = config.subsampling_conv_channels + self.conv = nn.ModuleList( + [ + nn.Conv2d(1, channels, kernel_size=3, stride=2, padding=1), + nn.Identity(), + nn.Conv2d(channels, channels, kernel_size=3, stride=2, padding=1, groups=channels), + nn.Conv2d(channels, channels, kernel_size=1, stride=1, padding=0), + nn.Identity(), + nn.Conv2d(channels, channels, kernel_size=3, stride=2, padding=1, groups=channels), + nn.Conv2d(channels, channels, kernel_size=1, stride=1, padding=0), + ] + ) + _copy_conv2d_weight(self.conv[0], state_dict["encoder.pre_encode.conv.0.weight"], bias=state_dict["encoder.pre_encode.conv.0.bias"]) + _copy_conv2d_weight(self.conv[2], state_dict["encoder.pre_encode.conv.2.weight"], bias=state_dict["encoder.pre_encode.conv.2.bias"]) + _copy_conv2d_weight(self.conv[3], state_dict["encoder.pre_encode.conv.3.weight"], bias=state_dict["encoder.pre_encode.conv.3.bias"]) + _copy_conv2d_weight(self.conv[5], state_dict["encoder.pre_encode.conv.5.weight"], bias=state_dict["encoder.pre_encode.conv.5.bias"]) + _copy_conv2d_weight(self.conv[6], state_dict["encoder.pre_encode.conv.6.weight"], bias=state_dict["encoder.pre_encode.conv.6.bias"]) + projected_width = config.num_mel_bins + for _ in range(3): + projected_width = (projected_width + 2 - 3) // 2 + 1 + self.out = nn.Linear(channels * projected_width, config.hidden_dim) + _copy_linear_weight( + self.out, + state_dict["encoder.pre_encode.out.weight"], + bias=state_dict["encoder.pre_encode.out.bias"], + ) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + if input_features.ndim != 3: + raise ValueError(f"expected input_features [batch, frames, mels], got {tuple(input_features.shape)}") + x = input_features.unsqueeze(1) + x = F.relu(self.conv[0](x)) + x = F.relu(self.conv[3](self.conv[2](x))) + x = F.relu(self.conv[6](self.conv[5](x))) + x = x.permute(0, 2, 1, 3).reshape(x.shape[0], x.shape[2], -1) + return self.out(x) + + +class ParakeetTDTEncoder(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.pre_encode = ParakeetTDTPreEncode(config, state_dict) + self.layers = nn.ModuleList( + [ParakeetTDTEncoderLayer(config, index, state_dict) for index in range(config.num_layers)] + ) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + x = self.pre_encode(input_features) + for layer in self.layers: + x = layer(x) + return x + + +class ParakeetTDTDecoderCell(nn.Module): + def __init__(self, hidden_dim: int, prefix: str, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.Wx = nn.Parameter(state_dict[f"{prefix}.Wx"].clone()) + self.Wh = nn.Parameter(state_dict[f"{prefix}.Wh"].clone()) + self.bias = nn.Parameter(state_dict[f"{prefix}.bias"].clone()) + self.hidden_dim = hidden_dim + + def forward(self, x: torch.Tensor, h: torch.Tensor, c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + gates = F.linear(x, self.Wx) + F.linear(h, self.Wh) + self.bias + i, f, g, o = torch.chunk(gates, 4, dim=-1) + i = torch.sigmoid(i) + f = torch.sigmoid(f) + g = torch.tanh(g) + o = torch.sigmoid(o) + c_next = f * c + i * g + h_next = o * torch.tanh(c_next) + return h_next, c_next + + +class ParakeetTDTDecoderPrediction(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + vocab_size = max(config.blank_id + 1, state_dict["decoder.prediction.embed.weight"].shape[0]) + self.embed = nn.Embedding(vocab_size, config.predictor_hidden_dim) + self.embed.weight.data.copy_(state_dict["decoder.prediction.embed.weight"].to(dtype=self.embed.weight.dtype)) + self.dec_rnn = nn.Module() + self.dec_rnn.lstm = nn.ModuleList( + [ + ParakeetTDTDecoderCell( + config.predictor_hidden_dim, + f"decoder.prediction.dec_rnn.lstm.{index}", + state_dict, + ) + for index in range(config.predictor_num_layers) + ] + ) + + def forward( + self, + token_ids: torch.Tensor, + state_h: tuple[torch.Tensor, ...], + state_c: tuple[torch.Tensor, ...], + ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...], tuple[torch.Tensor, ...]]: + x = self.embed(token_ids) + if x.ndim == 3 and x.shape[1] == 1: + x = x[:, 0, :] + next_h: list[torch.Tensor] = [] + next_c: list[torch.Tensor] = [] + for index, cell in enumerate(self.dec_rnn.lstm): + h_new, c_new = cell(x, state_h[index], state_c[index]) + next_h.append(h_new) + next_c.append(c_new) + x = h_new + return x, tuple(next_h), tuple(next_c) + + +class ParakeetTDTJoint(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.enc = nn.Linear(config.hidden_dim, config.joint_dim) + self.pred = nn.Linear(config.predictor_hidden_dim, config.joint_dim) + output_classes = int(state_dict["joint.joint_net.2.weight"].shape[0]) + self.joint_net = nn.Sequential( + nn.Identity(), + nn.ReLU(), + nn.Linear( + config.joint_dim, + output_classes, + ), + ) + _copy_linear_weight(self.enc, state_dict["joint.enc.weight"], bias=state_dict["joint.enc.bias"]) + _copy_linear_weight(self.pred, state_dict["joint.pred.weight"], bias=state_dict["joint.pred.bias"]) + _copy_linear_weight( + self.joint_net[2], + state_dict["joint.joint_net.2.weight"], + bias=state_dict["joint.joint_net.2.bias"], + ) + + def forward(self, encoder_frame: torch.Tensor, predictor_hidden: torch.Tensor) -> torch.Tensor: + return self.joint_net(F.relu(self.enc(encoder_frame) + self.pred(predictor_hidden))) + + +class ParakeetTDTDecoderStep(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.prediction = ParakeetTDTDecoderPrediction(config, state_dict) + self.joint = ParakeetTDTJoint(config, state_dict) + self.config = config + + def forward(self, encoder_frame: torch.Tensor, token_ids: torch.Tensor, *state_tensors: torch.Tensor) -> tuple[torch.Tensor, ...]: + if len(state_tensors) != self.config.predictor_num_layers * 2: + raise ValueError( + f"expected {self.config.predictor_num_layers * 2} predictor state tensors, got {len(state_tensors)}" + ) + state_h = tuple(state_tensors[0::2]) + state_c = tuple(state_tensors[1::2]) + predictor_hidden, next_h, next_c = self.prediction(token_ids, state_h, state_c) + logits = self.joint(encoder_frame, predictor_hidden) + outputs: list[torch.Tensor] = [logits] + for h, c in zip(next_h, next_c, strict=True): + outputs.append(h) + outputs.append(c) + return tuple(outputs) + + +class ParakeetTDTLocalModel(nn.Module): + def __init__(self, config: ParakeetTDTConfig, state_dict: dict[str, torch.Tensor]): + super().__init__() + self.name_or_path = config.model_source + self.family = "parakeet_tdt" + self.config = config + self.encoder = ParakeetTDTEncoder(config, state_dict) + self.decoder = nn.Module() + self.decoder.prediction = ParakeetTDTDecoderPrediction(config, state_dict) + self.joint = ParakeetTDTJoint(config, state_dict) + self.decoder_step = ParakeetTDTDecoderStep(config, state_dict) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + return self.encoder(input_features) + + def initial_decoder_state(self, *, batch_size: int, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, ...]: + state: list[torch.Tensor] = [] + for _ in range(self.config.predictor_num_layers): + state.append(torch.zeros((batch_size, self.config.predictor_hidden_dim), device=device, dtype=dtype)) + state.append(torch.zeros((batch_size, self.config.predictor_hidden_dim), device=device, dtype=dtype)) + return tuple(state) + + def greedy_decode_token_ids(self, input_features: torch.Tensor) -> list[int]: + with torch.no_grad(): + encoder_hidden = self.encoder(input_features) + batch = int(encoder_hidden.shape[0]) + if batch != 1: + raise ValueError("Parakeet TDT local greedy decode currently expects batch size 1") + states = self.initial_decoder_state( + batch_size=batch, + device=encoder_hidden.device, + dtype=encoder_hidden.dtype, + ) + state_arrays = tuple(state.detach().cpu().numpy() for state in states) + encoder_hidden_np = encoder_hidden.detach().cpu().numpy() + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + frame_tensor = torch.from_numpy(frame).to(device=encoder_hidden.device, dtype=encoder_hidden.dtype) + token_tensor = torch.tensor([[token_id]], device=encoder_hidden.device, dtype=torch.long) + state_tensors = tuple( + torch.from_numpy(value).to(device=encoder_hidden.device, dtype=encoder_hidden.dtype) + for value in state_values + ) + outputs = self.decoder_step(frame_tensor, token_tensor, *state_tensors) + logits = outputs[0].detach().cpu().numpy() + next_states = tuple(output.detach().cpu().numpy() for output in outputs[1:]) + return logits, next_states + + return greedy_decode_parakeet_tdt_token_ids( + config=self.config, + encoder_hidden_states=encoder_hidden_np, + initial_states=state_arrays, + step=_step, + ) + + def decode_token_ids(self, token_ids: list[int]) -> str: + pieces: list[str] = [] + for token_id in token_ids: + if token_id < 0 or token_id >= len(self.config.vocabulary): + continue + piece = self.config.vocabulary[token_id] + if piece.startswith("<|") and piece.endswith("|>"): + continue + pieces.append(piece) + text = "".join(pieces).replace("▁", " ") + return re.sub(r"\s+", " ", text).strip() + + +def load_parakeet_tdt_local_model(model_source: str, *, torch_dtype: torch.dtype) -> ParakeetTDTLocalModel: + config = load_parakeet_tdt_config(model_source) + state_dict = _load_tensor_state_dict(model_source) + predictor_vocab_size = int(state_dict["decoder.prediction.embed.weight"].shape[0]) + if config.blank_id < 0 or config.blank_id >= predictor_vocab_size: + config = replace(config, blank_id=predictor_vocab_size - 1) + elif config.blank_id == len(config.vocabulary) - 1 and predictor_vocab_size == len(config.vocabulary) + 1: + config = replace(config, blank_id=predictor_vocab_size - 1) + model = ParakeetTDTLocalModel(config, state_dict).eval() + model.to(dtype=torch_dtype) + return model + + +def build_parakeet_tdt_component_specs( + model: ParakeetTDTLocalModel, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, +) -> list[ComponentModuleSpec]: + input_features = named_tensors["input_features"] + example_hidden = model.encoder(input_features) + batch_size = int(example_hidden.shape[0]) + initial_states = model.initial_decoder_state( + batch_size=batch_size, + device=example_hidden.device, + dtype=example_hidden.dtype, + ) + example_frame = example_hidden[:, :1, :].reshape(batch_size, example_hidden.shape[-1]) + example_token_id = torch.full( + (batch_size,), + int(model.config.blank_id), + device=example_hidden.device, + dtype=torch.long, + ) + + state_input_keys: list[str] = [] + state_output_keys: list[str] = [] + for index in range(model.config.predictor_num_layers): + state_input_keys.extend((f"state_h_{index}", f"state_c_{index}")) + state_output_keys.extend((f"state_h_{index}", f"state_c_{index}")) + + common_graph_meta = { + "weights_dir": weights_dir, + "task": "tdt_transcription", + "adapter_family": "parakeet_tdt", + } + return [ + ComponentModuleSpec( + component="audio_encoder", + module=model.encoder, + example_inputs=(input_features,), + input_keys=("input_features",), + output_keys=("encoder_hidden_states",), + graph_meta={**common_graph_meta, "component": "audio_encoder"}, + metadata={"family": "parakeet_tdt", "task": "tdt_transcription"}, + ), + ComponentModuleSpec( + component="decoder", + module=model.decoder_step, + example_inputs=(example_frame, example_token_id, *initial_states), + input_keys=("encoder_frame", "token_ids", *state_input_keys), + output_keys=("step_logits", *state_output_keys), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata={"family": "parakeet_tdt", "task": "tdt_transcription"}, + ), + ] diff --git a/python/src/transpile/weight_binding.py b/python/src/transpile/weight_binding.py new file mode 100644 index 000000000..da05532c0 --- /dev/null +++ b/python/src/transpile/weight_binding.py @@ -0,0 +1,461 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re + + +@dataclass(frozen=True) +class WeightBinding: + path: str + kind: str # "weight" | "embedding" + source_name: str + + +_PROJECT_ROOT = Path(__file__).resolve().parents[3] + +_PARAKEET_GLOBAL_FILENAMES: dict[str, str] = { + "ctc_head.weight": "ctc_head_weight.weights", + "ctc_head.bias": "ctc_head_bias.bias", + "encoder.subsampling.layers.0.weight": "subsampling_conv0_weight.weights", + "encoder.subsampling.layers.0.bias": "subsampling_conv0_bias.bias", + "encoder.subsampling.layers.2.weight": "subsampling_depthwise1_weight.weights", + "encoder.subsampling.layers.2.bias": "subsampling_depthwise1_bias.bias", + "encoder.subsampling.layers.3.weight": "subsampling_pointwise1_weight.weights", + "encoder.subsampling.layers.3.bias": "subsampling_pointwise1_bias.bias", + "encoder.subsampling.layers.5.weight": "subsampling_depthwise2_weight.weights", + "encoder.subsampling.layers.5.bias": "subsampling_depthwise2_bias.bias", + "encoder.subsampling.layers.6.weight": "subsampling_pointwise2_weight.weights", + "encoder.subsampling.layers.6.bias": "subsampling_pointwise2_bias.bias", + "encoder.subsampling.linear.weight": "subsampling_linear_weight.weights", + "encoder.subsampling.linear.bias": "subsampling_linear_bias.bias", + "encoder.pre_encode.conv.0.weight": "subsampling_conv0_weight.weights", + "encoder.pre_encode.conv.0.bias": "subsampling_conv0_bias.bias", + "encoder.pre_encode.conv.2.weight": "subsampling_depthwise1_weight.weights", + "encoder.pre_encode.conv.2.bias": "subsampling_depthwise1_bias.bias", + "encoder.pre_encode.conv.3.weight": "subsampling_pointwise1_weight.weights", + "encoder.pre_encode.conv.3.bias": "subsampling_pointwise1_bias.bias", + "encoder.pre_encode.conv.5.weight": "subsampling_depthwise2_weight.weights", + "encoder.pre_encode.conv.5.bias": "subsampling_depthwise2_bias.bias", + "encoder.pre_encode.conv.6.weight": "subsampling_pointwise2_weight.weights", + "encoder.pre_encode.conv.6.bias": "subsampling_pointwise2_bias.bias", + "encoder.pre_encode.out.weight": "subsampling_linear_weight.weights", + "encoder.pre_encode.out.bias": "subsampling_linear_bias.bias", + "decoder.prediction.embed.weight": "tdt_predictor_embed.weights", + "joint.enc.weight": "tdt_joint_enc.weights", + "joint.enc.bias": "tdt_joint_enc.bias", + "joint.pred.weight": "tdt_joint_pred.weights", + "joint.pred.bias": "tdt_joint_pred.bias", + "joint.joint_net.2.weight": "tdt_joint_out.weights", + "joint.joint_net.2.bias": "tdt_joint_out.bias", +} + +_PARAKEET_LAYER_FILENAMES: dict[str, str] = { + "feed_forward1.linear1.weight": "ff1_linear1.weights", + "feed_forward1.linear1.bias": "ff1_linear1.bias", + "feed_forward1.linear2.weight": "ff1_linear2.weights", + "feed_forward1.linear2.bias": "ff1_linear2.bias", + "feed_forward2.linear1.weight": "ff2_linear1.weights", + "feed_forward2.linear1.bias": "ff2_linear1.bias", + "feed_forward2.linear2.weight": "ff2_linear2.weights", + "feed_forward2.linear2.bias": "ff2_linear2.bias", + "self_attn.q_proj.weight": "self_attn_q.weights", + "self_attn.q_proj.bias": "self_attn_q.bias", + "self_attn.k_proj.weight": "self_attn_k.weights", + "self_attn.k_proj.bias": "self_attn_k.bias", + "self_attn.v_proj.weight": "self_attn_v.weights", + "self_attn.v_proj.bias": "self_attn_v.bias", + "self_attn.o_proj.weight": "self_attn_output.weights", + "self_attn.o_proj.bias": "self_attn_output.bias", + "self_attn.relative_k_proj.weight": "self_attn_relative_k.weights", + "self_attn.linear_q.weight": "self_attn_q.weights", + "self_attn.linear_q.bias": "self_attn_q.bias", + "self_attn.linear_k.weight": "self_attn_k.weights", + "self_attn.linear_k.bias": "self_attn_k.bias", + "self_attn.linear_v.weight": "self_attn_v.weights", + "self_attn.linear_v.bias": "self_attn_v.bias", + "self_attn.linear_out.weight": "self_attn_output.weights", + "self_attn.linear_out.bias": "self_attn_output.bias", + "self_attn.linear_pos.weight": "self_attn_relative_k.weights", + "self_attn.bias_u": "self_attn_bias_u.weights", + "self_attn.bias_v": "self_attn_bias_v.weights", + "self_attn.pos_bias_u": "self_attn_bias_u.weights", + "self_attn.pos_bias_v": "self_attn_bias_v.weights", + "conv.pointwise_conv1.weight": "conv_pointwise1.weights", + "conv.pointwise_conv1.bias": "conv_pointwise1.bias", + "conv.depthwise_conv.weight": "conv_depthwise.weights", + "conv.depthwise_conv.bias": "conv_depthwise.bias", + "conv.pointwise_conv2.weight": "conv_pointwise2.weights", + "conv.pointwise_conv2.bias": "conv_pointwise2.bias", + "conv.norm.weight": "conv_batchnorm_weight.weights", + "conv.norm.bias": "conv_batchnorm_bias.bias", + "conv.norm.running_mean": "conv_batchnorm_running_mean.weights", + "conv.norm.running_var": "conv_batchnorm_running_var.weights", + "conv.batch_norm.weight": "conv_batchnorm_weight.weights", + "conv.batch_norm.bias": "conv_batchnorm_bias.bias", + "conv.batch_norm.running_mean": "conv_batchnorm_running_mean.weights", + "conv.batch_norm.running_var": "conv_batchnorm_running_var.weights", + "norm_feed_forward1.weight": "norm_ff1.weights", + "norm_feed_forward1.bias": "norm_ff1.bias", + "norm_self_att.weight": "norm_self_attn.weights", + "norm_self_att.bias": "norm_self_attn.bias", + "norm_conv.weight": "norm_conv.weights", + "norm_conv.bias": "norm_conv.bias", + "norm_feed_forward2.weight": "norm_ff2.weights", + "norm_feed_forward2.bias": "norm_ff2.bias", + "norm_out.weight": "norm_out.weights", + "norm_out.bias": "norm_out.bias", +} + +_GEMMA3N_VISION_TOWER_PREFIX = "model.vision_tower.timm_model." +_GEMMA3N_AUDIO_TOWER_PREFIX = "model.audio_tower." +_GEMMA4_VISION_TOWER_PREFIX = "model.vision_tower." +_GEMMA4_AUDIO_TOWER_PREFIX = "model.audio_tower." + +_GEMMA_GLOBAL_FILENAMES: dict[str, tuple[str, str]] = { + "model.language_model.embed_tokens_per_layer.weight": ("embed_tokens_per_layer.weights", "embedding"), + "model.language_model.per_layer_model_projection.weight": ("per_layer_model_proj.weights", "weight"), + "model.language_model.per_layer_projection_norm.weight": ("per_layer_proj_norm.weights", "weight"), + "model.embed_vision.embedding.weight": ("embed_vision_embedding.weights", "embedding"), + "model.embed_vision.embedding_projection.weight": ("embed_vision_proj.weights", "weight"), + "model.embed_vision.soft_embedding_norm.weight": ("embed_vision_soft_norm.weights", "weight"), + "model.embed_vision.hard_embedding_norm.weight": ("embed_vision_hard_norm.weights", "weight"), + "model.embed_audio.embedding.weight": ("embed_audio_embedding.weights", "embedding"), + "model.embed_audio.embedding_projection.weight": ("embed_audio_proj.weights", "weight"), + "model.embed_audio.soft_embedding_norm.weight": ("embed_audio_soft_norm.weights", "weight"), + "model.embed_audio.hard_embedding_norm.weight": ("embed_audio_hard_norm.weights", "weight"), +} + + + +def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: + candidates: list[str] = [] + raw = model_name_or_path.strip() + if not raw: + return candidates + + def _add(name: str) -> None: + name = name.strip().lower() + if name and name not in candidates: + candidates.append(name) + + _add(raw.split("/")[-1]) + + path = Path(raw) + parts = path.parts + for part in parts: + if part.startswith("models--"): + _add(part[len("models--"):].split("--")[-1]) + break + + return candidates + + +def _default_weights_dir_for_model_name(model_name_or_path: str) -> str | None: + if not model_name_or_path: + return None + for model_dir_name in _candidate_model_dir_names(model_name_or_path): + candidate = _PROJECT_ROOT / "weights" / model_dir_name + if candidate.exists(): + return str(candidate) + return None + + +def _normalized_source_candidates(source_name: str) -> list[str]: + candidates: list[str] = [] + + def _add(name: str) -> None: + if name and name not in candidates: + candidates.append(name) + + def _add_backbone_aliases(name: str) -> None: + if name.startswith("backbone."): + tail = name[len("backbone.") :] + _add(f"model.{tail}") + _add(f"model.language_model.{tail}") + if name.startswith("language_model."): + tail = name[len("language_model.") :] + _add(f"model.language_model.{tail}") + if name.startswith("multimodal_backbone."): + tail = name[len("multimodal_backbone.") :] + _add(tail) + _add(f"model.{tail}") + _add(f"model.multimodal_backbone.{tail}") + if name.startswith("model.encoder."): + _add(name[len("model.") :]) + if name.startswith("model.decoder."): + _add(name[len("model.") :]) + if name.startswith("layers."): + _add(f"encoder.{name}") + if name.startswith("pre_encode."): + _add(f"encoder.{name}") + if name.startswith("prediction."): + _add(f"decoder.{name}") + + raw = source_name.strip() + _add(raw) + _add_backbone_aliases(raw) + + for prefix in ("p_", "b_", "c_"): + if raw.startswith(prefix): + stripped = raw[len(prefix):] + _add(stripped) + _add_backbone_aliases(stripped) + + stripped = raw + while True: + for prefix in ("adapter.model.", "adapter.module.", "adapter.encoder.", "adapter.decoder.", "module."): + if stripped.startswith(prefix): + tail = stripped[len(prefix) :] + if prefix == "adapter.encoder.": + stripped = f"encoder.{tail}" + elif prefix == "adapter.decoder.": + stripped = f"decoder.{tail}" + else: + stripped = tail + _add(stripped) + _add_backbone_aliases(stripped) + break + else: + break + + for candidate in tuple(candidates): + for prefix in ( + "module.backbone.", + "adapter.backbone.", + "adapter.model.model.", + "adapter.model.language_model.", + ): + if not candidate.startswith(prefix): + continue + tail = candidate[len(prefix):] + _add(tail) + _add_backbone_aliases(tail) + _add(f"model.{tail}") + _add(f"model.language_model.{tail}") + if tail.startswith("layers."): + _add(f"model.{tail}") + _add(f"model.language_model.{tail}") + break + + if stripped == "adapter.model.lm_head.weight": + _add("lm_head.weight") + _add("model.embed_tokens.weight") + _add("model.language_model.embed_tokens.weight") + if stripped == "lm_head.weight": + _add("model.embed_tokens.weight") + _add("model.language_model.embed_tokens.weight") + + layer_match = re.match( + r"^(?:(?:module|adapter)_)?backbone_layers_slice_none__\d+__none____modules__(\d+)___(.+)$", + stripped, + ) + if layer_match: + layer_index = int(layer_match.group(1)) + tail = layer_match.group(2) + tail_map = { + "self_attn_q_proj_weight": "self_attn.q_proj.weight", + "self_attn_k_proj_weight": "self_attn.k_proj.weight", + "self_attn_v_proj_weight": "self_attn.v_proj.weight", + "self_attn_o_proj_weight": "self_attn.o_proj.weight", + "self_attn_q_norm_weight": "self_attn.q_norm.weight", + "self_attn_k_norm_weight": "self_attn.k_norm.weight", + "mlp_gate_proj_weight": "mlp.gate_proj.weight", + "mlp_up_proj_weight": "mlp.up_proj.weight", + "mlp_down_proj_weight": "mlp.down_proj.weight", + "input_layernorm_weight": "input_layernorm.weight", + "post_attention_layernorm_weight": "post_attention_layernorm.weight", + "linear_attn_in_proj_qkv_weight": "linear_attn.in_proj_qkv.weight", + "linear_attn_conv1d_weight": "linear_attn.conv1d.weight", + "linear_attn_norm_weight": "linear_attn.norm.weight", + "linear_attn_dt_bias": "linear_attn.dt_bias", + "linear_attn_A_log": "linear_attn.A_log", + } + dotted_tail = tail_map.get(tail) + if dotted_tail is not None: + _add(f"model.layers.{layer_index}.{dotted_tail}") + _add(f"model.language_model.layers.{layer_index}.{dotted_tail}") + + embed_map = { + "module_backbone_embed_tokens_weight": [ + "model.embed_tokens.weight", + "model.language_model.embed_tokens.weight", + ], + "adapter_backbone_embed_tokens_weight": [ + "model.embed_tokens.weight", + "model.language_model.embed_tokens.weight", + ], + "module_backbone_embed_tokens_per_layer_weight": [ + "model.embed_tokens_per_layer.weight", + "model.language_model.embed_tokens_per_layer.weight", + ], + "adapter_backbone_embed_tokens_per_layer_weight": [ + "model.embed_tokens_per_layer.weight", + "model.language_model.embed_tokens_per_layer.weight", + ], + "module_backbone_per_layer_model_projection_weight": [ + "model.per_layer_model_projection.weight", + "model.language_model.per_layer_model_projection.weight", + ], + "adapter_backbone_per_layer_model_projection_weight": [ + "model.per_layer_model_projection.weight", + "model.language_model.per_layer_model_projection.weight", + ], + "module_backbone_norm_weight": [ + "model.norm.weight", + "model.language_model.norm.weight", + ], + "adapter_backbone_norm_weight": [ + "model.norm.weight", + "model.language_model.norm.weight", + ], + "module_model_lm_head_weight": [ + "lm_head.weight", + ], + "adapter_model_lm_head_weight": [ + "lm_head.weight", + ], + } + for key, mapped in embed_map.items(): + if stripped == key: + for item in mapped: + _add(item) + + return candidates + + +def _gemma_tower_output_name(hf_key: str, strip_prefix: str, add_prefix: str) -> str: + name = hf_key[len(strip_prefix) :] + if name.endswith(".weight"): + name = name[: -len(".weight")] + ext = ".weights" + elif name.endswith(".bias"): + name = name[: -len(".bias")] + ext = ".bias" + else: + ext = ".weights" + if name.endswith(".linear"): + name = name[: -len(".linear")] + elif name.endswith("_linear"): + name = name[: -len("_linear")] + name = name.replace(".", "_") + return add_prefix + name + ext + + +def _fallback_filename_candidates(source_name: str) -> list[tuple[str, str]]: + filenames: list[tuple[str, str]] = [] + + def _add(filename: str | None, *, kind: str = "weight") -> None: + if not filename: + return + candidate = (filename, kind) + if candidate not in filenames: + filenames.append(candidate) + + for candidate in _normalized_source_candidates(source_name): + _add(_PARAKEET_GLOBAL_FILENAMES.get(candidate)) + gemma_global = _GEMMA_GLOBAL_FILENAMES.get(candidate) + if gemma_global is not None: + _add(gemma_global[0], kind=gemma_global[1]) + if candidate in {"model.embed_tokens_per_layer.weight", "model.language_model.embed_tokens_per_layer.weight"}: + _add("embed_tokens_per_layer.weights", kind="embedding") + elif candidate in {"model.per_layer_model_projection.weight", "model.language_model.per_layer_model_projection.weight"}: + _add("per_layer_model_proj.weights") + elif candidate in {"model.per_layer_projection_norm.weight", "model.language_model.per_layer_projection_norm.weight"}: + _add("per_layer_proj_norm.weights") + elif candidate in {"lm_head.weight", "model.lm_head.weight"}: + _add("token_embeddings.weights", kind="embedding") + elif candidate in { + "_cactus_vision_post_proj_norm", + "model._cactus_vision_post_proj_norm", + }: + _add("embed_vision_post_proj_norm.weights") + + if candidate.startswith(_GEMMA3N_VISION_TOWER_PREFIX): + _add(_gemma_tower_output_name(candidate, _GEMMA3N_VISION_TOWER_PREFIX, "vision_")) + elif candidate.startswith(_GEMMA3N_AUDIO_TOWER_PREFIX): + _add(_gemma_tower_output_name(candidate, _GEMMA3N_AUDIO_TOWER_PREFIX, "audio_")) + elif candidate.startswith(_GEMMA4_VISION_TOWER_PREFIX): + _add(_gemma_tower_output_name(candidate, _GEMMA4_VISION_TOWER_PREFIX, "vision_")) + elif candidate.startswith(_GEMMA4_AUDIO_TOWER_PREFIX): + _add(_gemma_tower_output_name(candidate, _GEMMA4_AUDIO_TOWER_PREFIX, "audio_")) + + predictor_match = re.match(r"^decoder\.prediction\.dec_rnn\.lstm\.(\d+)\.(Wx|Wh|bias)$", candidate) + if predictor_match: + layer_index = int(predictor_match.group(1)) + suffix = predictor_match.group(2) + if suffix == "Wx": + _add(f"tdt_predictor_lstm_{layer_index}_weight_ih.weights") + elif suffix == "Wh": + _add(f"tdt_predictor_lstm_{layer_index}_weight_hh.weights") + elif suffix == "bias": + _add(f"tdt_predictor_lstm_{layer_index}_bias.weights") + + layer_match = re.match(r"^encoder\.layers\.(\d+)\.(.+)$", candidate) + if layer_match: + layer_index = int(layer_match.group(1)) + suffix = layer_match.group(2) + mapped = _PARAKEET_LAYER_FILENAMES.get(suffix) + if mapped is not None: + _add(f"layer_{layer_index}_{mapped}") + + return filenames + + +def resolve_transpile_weights_dir(graph_meta: dict[str, object]) -> str | None: + explicit = graph_meta.get("weights_dir") + if isinstance(explicit, str) and explicit: + return explicit + + family = str(graph_meta.get("adapter_family", "")).upper() + family_env = f"CACTUS_TRANSPILER_WEIGHTS_DIR_{family}" + if family and family_env in os.environ and os.environ[family_env]: + return os.environ[family_env] + + generic = os.environ.get("CACTUS_TRANSPILER_WEIGHTS_DIR") + if generic: + return generic + + model_name_or_path = graph_meta.get("model_name_or_path") + if isinstance(model_name_or_path, str) and model_name_or_path: + default_dir = _default_weights_dir_for_model_name(model_name_or_path) + if default_dir: + return default_dir + return None + + +def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> WeightBinding | None: + if not weights_dir: + return None + root = Path(weights_dir) + if not root.exists(): + return None + manifest_path = root / "weights_manifest.json" + if not manifest_path.exists(): + return None + try: + manifest = json.loads(manifest_path.read_text()) + except Exception: + return None + if not isinstance(manifest, dict): + return None + for candidate_name in _normalized_source_candidates(source_name): + entry = manifest.get(candidate_name) + if not isinstance(entry, dict): + continue + filename = entry.get("filename") + kind = entry.get("kind", "weight") + if not isinstance(filename, str) or not isinstance(kind, str): + continue + candidate = root / filename + if not candidate.exists(): + continue + return WeightBinding(path=str(candidate), kind=kind, source_name=candidate_name) + + for filename, kind in _fallback_filename_candidates(source_name): + candidate = root / filename + if candidate.exists(): + return WeightBinding(path=str(candidate), kind=kind, source_name=source_name) + return None From b9e2fe12814be707f1ea75762b1c0a6586a6dae4 Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Tue, 12 May 2026 09:04:22 -0700 Subject: [PATCH 05/11] Added cli support to transpile and run the transpiled models through cactus cli --- cactus-transpile/CMakeLists.txt | 211 ++ cactus-transpile/cactus.h | 12 + cactus-transpile/engine/engine.h | 1079 +++++++ cactus-transpile/engine/engine_bpe.cpp | 538 ++++ cactus-transpile/engine/engine_cache.cpp | 691 ++++ .../engine/engine_constraints.cpp | 1137 +++++++ cactus-transpile/engine/engine_image.cpp | 600 ++++ cactus-transpile/engine/engine_index.cpp | 780 +++++ cactus-transpile/engine/engine_model.cpp | 1098 +++++++ cactus-transpile/engine/engine_signals.cpp | 775 +++++ cactus-transpile/engine/engine_sp.cpp | 485 +++ cactus-transpile/engine/engine_tokenizer.cpp | 866 +++++ cactus-transpile/ffi/cactus_cloud.cpp | 452 +++ cactus-transpile/ffi/cactus_cloud.h | 48 + cactus-transpile/ffi/cactus_complete.cpp | 1177 +++++++ cactus-transpile/ffi/cactus_diarize.cpp | 200 ++ cactus-transpile/ffi/cactus_embed.cpp | 273 ++ cactus-transpile/ffi/cactus_embed_speaker.cpp | 125 + cactus-transpile/ffi/cactus_ffi.h | 489 +++ cactus-transpile/ffi/cactus_graph.cpp | 1432 +++++++++ cactus-transpile/ffi/cactus_index.cpp | 458 +++ cactus-transpile/ffi/cactus_init.cpp | 489 +++ cactus-transpile/ffi/cactus_log.cpp | 23 + cactus-transpile/ffi/cactus_rag.cpp | 459 +++ cactus-transpile/ffi/cactus_stream.cpp | 2714 ++++++++++++++++ cactus-transpile/ffi/cactus_telemetry.cpp | 22 + cactus-transpile/ffi/cactus_transcribe.cpp | 829 +++++ cactus-transpile/ffi/cactus_utils.h | 1850 +++++++++++ cactus-transpile/ffi/cactus_vad.cpp | 176 + cactus-transpile/ffi/gemma_tools.h | 576 ++++ cactus-transpile/graph/graph.h | 805 +++++ cactus-transpile/graph/graph_builder.cpp | 1879 +++++++++++ cactus-transpile/graph/graph_core.cpp | 265 ++ cactus-transpile/graph/graph_execute.cpp | 948 ++++++ cactus-transpile/graph/graph_io.cpp | 809 +++++ cactus-transpile/graph/graph_ops_math.cpp | 737 +++++ cactus-transpile/graph/graph_ops_nn.cpp | 2754 ++++++++++++++++ cactus-transpile/graph/graph_ops_sample.cpp | 134 + cactus-transpile/graph/graph_ops_tensor.cpp | 813 +++++ cactus-transpile/graph/graph_param_io.cpp | 394 +++ cactus-transpile/graph/graph_param_io.h | 11 + cactus-transpile/kernel/kernel.h | 488 +++ cactus-transpile/kernel/kernel_altup.cpp | 180 ++ cactus-transpile/kernel/kernel_attention.cpp | 1365 ++++++++ cactus-transpile/kernel/kernel_bilstm.cpp | 370 +++ cactus-transpile/kernel/kernel_blas.cpp | 639 ++++ cactus-transpile/kernel/kernel_conv.cpp | 1663 ++++++++++ cactus-transpile/kernel/kernel_deltanet.cpp | 677 ++++ cactus-transpile/kernel/kernel_i8mm.cpp | 189 ++ cactus-transpile/kernel/kernel_lstm.cpp | 117 + cactus-transpile/kernel/kernel_matmul.cpp | 787 +++++ cactus-transpile/kernel/kernel_nn.cpp | 874 +++++ cactus-transpile/kernel/kernel_quants.cpp | 312 ++ cactus-transpile/kernel/kernel_reduce.cpp | 430 +++ cactus-transpile/kernel/kernel_scalar.cpp | 119 + cactus-transpile/kernel/kernel_sme2.cpp | 616 ++++ cactus-transpile/kernel/kernel_utils.h | 724 +++++ .../models/gemma4/model_gemma4.cpp | 594 ++++ cactus-transpile/models/gemma4/model_gemma4.h | 404 +++ .../models/gemma4/model_gemma4_audio.cpp | 777 +++++ .../models/gemma4/model_gemma4_mm.cpp | 459 +++ .../models/gemma4/model_gemma4_vision.cpp | 540 ++++ cactus-transpile/models/model.h | 1773 +++++++++++ cactus-transpile/models/model_gemma.cpp | 192 ++ cactus-transpile/models/model_gemma3n.cpp | 475 +++ cactus-transpile/models/model_lfm2.cpp | 389 +++ cactus-transpile/models/model_lfm2moe.cpp | 456 +++ cactus-transpile/models/model_lfm2vl.cpp | 607 ++++ cactus-transpile/models/model_moonshine.cpp | 596 ++++ cactus-transpile/models/model_needle.cpp | 337 ++ cactus-transpile/models/model_nomic.cpp | 201 ++ cactus-transpile/models/model_parakeet.cpp | 860 +++++ .../models/model_parakeet_tdt.cpp | 1107 +++++++ cactus-transpile/models/model_pyannote.cpp | 201 ++ cactus-transpile/models/model_qwen.cpp | 171 + cactus-transpile/models/model_qwen3p5.cpp | 771 +++++ cactus-transpile/models/model_siglip2.cpp | 387 +++ cactus-transpile/models/model_silero_vad.cpp | 338 ++ cactus-transpile/models/model_wespeaker.cpp | 204 ++ cactus-transpile/models/model_whisper.cpp | 778 +++++ cactus-transpile/models/model_youtu.cpp | 246 ++ cactus-transpile/npu/npu.cpp | 22 + cactus-transpile/npu/npu.h | 89 + cactus-transpile/npu/npu_ane.h | 154 + cactus-transpile/npu/npu_ane.mm | 1305 ++++++++ cactus-transpile/telemetry/telemetry.cpp | 1546 +++++++++ cactus-transpile/telemetry/telemetry.h | 44 + python/examples/compare_gemma4_multimodal.py | 933 ++++++ .../examples/compare_parakeet_performance.py | 793 +++++ python/examples/inspect_gemma4_cleanup.py | 241 ++ python/examples/inspect_transpile_cleanup.py | 138 + python/examples/test_torch_capture.py | 70 + python/examples/transpile_hf_causal_lm.py | 17 + python/examples/transpile_hf_model.py | 2832 +++++++++++++++++ python/src/cactus.py | 127 +- python/src/cli.py | 156 +- python/src/converter.py | 211 +- python/src/graph.py | 188 +- python/src/tensor_io.py | 327 +- 99 files changed, 61115 insertions(+), 104 deletions(-) create mode 100644 cactus-transpile/CMakeLists.txt create mode 100644 cactus-transpile/cactus.h create mode 100644 cactus-transpile/engine/engine.h create mode 100644 cactus-transpile/engine/engine_bpe.cpp create mode 100644 cactus-transpile/engine/engine_cache.cpp create mode 100644 cactus-transpile/engine/engine_constraints.cpp create mode 100644 cactus-transpile/engine/engine_image.cpp create mode 100644 cactus-transpile/engine/engine_index.cpp create mode 100644 cactus-transpile/engine/engine_model.cpp create mode 100644 cactus-transpile/engine/engine_signals.cpp create mode 100644 cactus-transpile/engine/engine_sp.cpp create mode 100644 cactus-transpile/engine/engine_tokenizer.cpp create mode 100644 cactus-transpile/ffi/cactus_cloud.cpp create mode 100644 cactus-transpile/ffi/cactus_cloud.h create mode 100644 cactus-transpile/ffi/cactus_complete.cpp create mode 100644 cactus-transpile/ffi/cactus_diarize.cpp create mode 100644 cactus-transpile/ffi/cactus_embed.cpp create mode 100644 cactus-transpile/ffi/cactus_embed_speaker.cpp create mode 100644 cactus-transpile/ffi/cactus_ffi.h create mode 100644 cactus-transpile/ffi/cactus_graph.cpp create mode 100644 cactus-transpile/ffi/cactus_index.cpp create mode 100644 cactus-transpile/ffi/cactus_init.cpp create mode 100644 cactus-transpile/ffi/cactus_log.cpp create mode 100644 cactus-transpile/ffi/cactus_rag.cpp create mode 100644 cactus-transpile/ffi/cactus_stream.cpp create mode 100644 cactus-transpile/ffi/cactus_telemetry.cpp create mode 100644 cactus-transpile/ffi/cactus_transcribe.cpp create mode 100644 cactus-transpile/ffi/cactus_utils.h create mode 100644 cactus-transpile/ffi/cactus_vad.cpp create mode 100644 cactus-transpile/ffi/gemma_tools.h create mode 100644 cactus-transpile/graph/graph.h create mode 100644 cactus-transpile/graph/graph_builder.cpp create mode 100644 cactus-transpile/graph/graph_core.cpp create mode 100644 cactus-transpile/graph/graph_execute.cpp create mode 100644 cactus-transpile/graph/graph_io.cpp create mode 100644 cactus-transpile/graph/graph_ops_math.cpp create mode 100644 cactus-transpile/graph/graph_ops_nn.cpp create mode 100644 cactus-transpile/graph/graph_ops_sample.cpp create mode 100644 cactus-transpile/graph/graph_ops_tensor.cpp create mode 100644 cactus-transpile/graph/graph_param_io.cpp create mode 100644 cactus-transpile/graph/graph_param_io.h create mode 100644 cactus-transpile/kernel/kernel.h create mode 100644 cactus-transpile/kernel/kernel_altup.cpp create mode 100644 cactus-transpile/kernel/kernel_attention.cpp create mode 100644 cactus-transpile/kernel/kernel_bilstm.cpp create mode 100644 cactus-transpile/kernel/kernel_blas.cpp create mode 100644 cactus-transpile/kernel/kernel_conv.cpp create mode 100644 cactus-transpile/kernel/kernel_deltanet.cpp create mode 100644 cactus-transpile/kernel/kernel_i8mm.cpp create mode 100644 cactus-transpile/kernel/kernel_lstm.cpp create mode 100644 cactus-transpile/kernel/kernel_matmul.cpp create mode 100644 cactus-transpile/kernel/kernel_nn.cpp create mode 100644 cactus-transpile/kernel/kernel_quants.cpp create mode 100644 cactus-transpile/kernel/kernel_reduce.cpp create mode 100644 cactus-transpile/kernel/kernel_scalar.cpp create mode 100644 cactus-transpile/kernel/kernel_sme2.cpp create mode 100644 cactus-transpile/kernel/kernel_utils.h create mode 100644 cactus-transpile/models/gemma4/model_gemma4.cpp create mode 100644 cactus-transpile/models/gemma4/model_gemma4.h create mode 100644 cactus-transpile/models/gemma4/model_gemma4_audio.cpp create mode 100644 cactus-transpile/models/gemma4/model_gemma4_mm.cpp create mode 100644 cactus-transpile/models/gemma4/model_gemma4_vision.cpp create mode 100644 cactus-transpile/models/model.h create mode 100644 cactus-transpile/models/model_gemma.cpp create mode 100644 cactus-transpile/models/model_gemma3n.cpp create mode 100644 cactus-transpile/models/model_lfm2.cpp create mode 100644 cactus-transpile/models/model_lfm2moe.cpp create mode 100644 cactus-transpile/models/model_lfm2vl.cpp create mode 100644 cactus-transpile/models/model_moonshine.cpp create mode 100644 cactus-transpile/models/model_needle.cpp create mode 100644 cactus-transpile/models/model_nomic.cpp create mode 100644 cactus-transpile/models/model_parakeet.cpp create mode 100644 cactus-transpile/models/model_parakeet_tdt.cpp create mode 100644 cactus-transpile/models/model_pyannote.cpp create mode 100644 cactus-transpile/models/model_qwen.cpp create mode 100644 cactus-transpile/models/model_qwen3p5.cpp create mode 100644 cactus-transpile/models/model_siglip2.cpp create mode 100644 cactus-transpile/models/model_silero_vad.cpp create mode 100644 cactus-transpile/models/model_wespeaker.cpp create mode 100644 cactus-transpile/models/model_whisper.cpp create mode 100644 cactus-transpile/models/model_youtu.cpp create mode 100644 cactus-transpile/npu/npu.cpp create mode 100644 cactus-transpile/npu/npu.h create mode 100644 cactus-transpile/npu/npu_ane.h create mode 100644 cactus-transpile/npu/npu_ane.mm create mode 100644 cactus-transpile/telemetry/telemetry.cpp create mode 100644 cactus-transpile/telemetry/telemetry.h create mode 100644 python/examples/compare_gemma4_multimodal.py create mode 100644 python/examples/compare_parakeet_performance.py create mode 100644 python/examples/inspect_gemma4_cleanup.py create mode 100644 python/examples/inspect_transpile_cleanup.py create mode 100644 python/examples/test_torch_capture.py create mode 100644 python/examples/transpile_hf_causal_lm.py create mode 100644 python/examples/transpile_hf_model.py diff --git a/cactus-transpile/CMakeLists.txt b/cactus-transpile/CMakeLists.txt new file mode 100644 index 000000000..3901084e2 --- /dev/null +++ b/cactus-transpile/CMakeLists.txt @@ -0,0 +1,211 @@ +cmake_minimum_required(VERSION 3.10) +project(Cactus LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +set(_CACTUS_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../CACTUS_VERSION") +if(EXISTS "${_CACTUS_VERSION_FILE}") + file(READ "${_CACTUS_VERSION_FILE}" _CACTUS_VERSION_CONTENT) + string(STRIP "${_CACTUS_VERSION_CONTENT}" CACTUS_VERSION_STR) + message(STATUS "Cactus version: ${CACTUS_VERSION_STR}") +else() + set(CACTUS_VERSION_STR "") +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) + message(STATUS "CMAKE_BUILD_TYPE was not set, defaulting to Release.") +endif() + +find_package(CURL QUIET) + +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +set(CACTUS_CURL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../libs/curl" CACHE PATH "Path to vendored libcurl artifacts") + +function(cactus_require_vendored_curl include_dir library_path platform_label) + if(NOT EXISTS "${include_dir}/curl/curl.h" OR NOT EXISTS "${library_path}") + message(WARNING "==============================================================================") + message(WARNING " Vendored libcurl is required for ${platform_label} builds but is missing.") + message(WARNING " Expected header: ${include_dir}/curl/curl.h") + message(WARNING " Expected library: ${library_path}") + message(WARNING " Populate libs/curl from your local curl build before running cactus build.") + message(WARNING "==============================================================================") + message(FATAL_ERROR "Missing vendored libcurl artifacts for ${platform_label}") + endif() +endfunction() + +if(APPLE) + set(CMAKE_OSX_ARCHITECTURES "arm64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64 -march=armv8.2-a+fp16+simd+dotprod+i8mm -pthread -Wall -Wextra -pedantic -O3 -Wno-missing-field-initializers") + add_compile_definitions( + __ARM_NEON=1 + __ARM_FEATURE_FP16_VECTOR_ARITHMETIC=1 + __ARM_FEATURE_DOTPROD=1 + __ARM_FEATURE_MATMUL_INT8=1 + ACCELERATE_NEW_LAPACK + ) + + enable_language(OBJCXX) + set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") + set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") + cactus_require_vendored_curl("${CACTUS_CURL_INCLUDE_DIR}" "${CACTUS_CURL_LIBRARY_MACOS}" "macOS") + find_library(COREML_FRAMEWORK CoreML REQUIRED) + find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) + find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) + find_library(SECURITY_FRAMEWORK Security REQUIRED) + find_library(SYSTEMCONFIGURATION_FRAMEWORK SystemConfiguration REQUIRED) + find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8.2-a+fp16+simd+dotprod+i8mm -pthread -Wall -Wextra -pedantic -O3 -Wno-missing-field-initializers") + add_compile_definitions( + __ARM_NEON=1 + __ARM_FEATURE_FP16_VECTOR_ARITHMETIC=1 + __ARM_FEATURE_DOTPROD=1 + __ARM_FEATURE_MATMUL_INT8=1 + ) +endif() + +file(GLOB ENGINE_SOURCES CONFIGURE_DEPENDS "engine/*.cpp") +file(GLOB GRAPH_SOURCES CONFIGURE_DEPENDS "graph/*.cpp") +file(GLOB KERNEL_SOURCES CONFIGURE_DEPENDS "kernel/*.cpp") +file(GLOB FFI_SOURCES CONFIGURE_DEPENDS "ffi/*.cpp") +file(GLOB MODEL_SOURCES CONFIGURE_DEPENDS "models/*.cpp" "models/gemma4/*.cpp") +file(GLOB TELEMETRY_SOURCES CONFIGURE_DEPENDS "telemetry/*.cpp") + +set(SME2_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/kernel/kernel_sme2.cpp") +set(I8MM_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/kernel/kernel_i8mm.cpp") +list(REMOVE_ITEM KERNEL_SOURCES "${SME2_SOURCE}") +list(REMOVE_ITEM KERNEL_SOURCES "${I8MM_SOURCE}") + +if(APPLE) + set(NPU_SOURCES "npu/npu_ane.mm") + set_source_files_properties(npu/npu_ane.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc") + message(STATUS "Apple platform detected - NPU/ANE acceleration enabled") +else() + set(NPU_SOURCES "npu/npu.cpp") +endif() + +set(COMMON_SOURCES + ${ENGINE_SOURCES} + ${GRAPH_SOURCES} + ${KERNEL_SOURCES} + ${FFI_SOURCES} + ${MODEL_SOURCES} + ${NPU_SOURCES} + ${TELEMETRY_SOURCES} +) + +set(COMMON_INCLUDES + ${CMAKE_CURRENT_SOURCE_DIR} + engine + graph + kernel + ffi + models + npu + telemetry +) + +function(configure_cactus_target target_name) + target_include_directories(${target_name} PUBLIC ${COMMON_INCLUDES}) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + target_compile_options(${target_name} PRIVATE -Wno-c99-extensions) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options(${target_name} PRIVATE -Wno-pedantic) + endif() + + if(NOT "${CACTUS_VERSION_STR}" STREQUAL "") + target_compile_definitions(${target_name} PRIVATE CACTUS_COMPILE_TIME_VERSION="${CACTUS_VERSION_STR}") + endif() + + if(APPLE) + target_link_libraries(${target_name} PUBLIC + ${COREML_FRAMEWORK} + ${FOUNDATION_FRAMEWORK} + ${ACCELERATE_FRAMEWORK} + ${SECURITY_FRAMEWORK} + ${SYSTEMCONFIGURATION_FRAMEWORK} + ${CFNETWORK_FRAMEWORK} + ${CACTUS_CURL_LIBRARY_MACOS} + ) + target_include_directories(${target_name} PUBLIC ${CACTUS_CURL_INCLUDE_DIR}) + target_compile_definitions(${target_name} PRIVATE CACTUS_USE_CURL=1) + elseif(CURL_FOUND) + target_include_directories(${target_name} PRIVATE ${CURL_INCLUDE_DIRS}) + target_link_libraries(${target_name} PUBLIC ${CURL_LIBRARIES}) + target_compile_definitions(${target_name} PRIVATE CACTUS_USE_CURL) + endif() +endfunction() + +set(ENABLE_SME2 "AUTO" CACHE STRING "SME2 build mode: AUTO or OFF") +set_property(CACHE ENABLE_SME2 PROPERTY STRINGS AUTO OFF) + +set(BUILD_SME2 0) +set(SME2_MARCH_FLAG "-march=armv9.2-a+nosve+nosve2+sme2") +if(ENABLE_SME2 STREQUAL "AUTO") + if(APPLE AND CMAKE_OSX_ARCHITECTURES STREQUAL "arm64") + include(CheckCXXSourceCompiles) + set(CMAKE_REQUIRED_FLAGS "-arch arm64 ${SME2_MARCH_FLAG}") + check_cxx_source_compiles(" + #include + int main() { return 0; } + " COMPILER_SUPPORTS_SME2) + unset(CMAKE_REQUIRED_FLAGS) + + if(COMPILER_SUPPORTS_SME2) + set(BUILD_SME2 1) + message(STATUS "Toolchain supports SME2 - building SME2 kernels") + else() + message(STATUS "Compiler does not support SME2 - building without SME2") + endif() + + else() + message(STATUS "Non-arm64 target - building without SME2") + endif() + +endif() + +if(BUILD_SME2) + add_library(cactus_sme2_obj OBJECT ${SME2_SOURCE}) + target_include_directories(cactus_sme2_obj PRIVATE ${COMMON_INCLUDES}) + target_compile_options(cactus_sme2_obj PRIVATE ${SME2_MARCH_FLAG}) + set_target_properties(cactus_sme2_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(cactus_sme2_obj PRIVATE CACTUS_COMPILE_SME2) +endif() + +# I8MM support: compile kernel_i8mm.cpp with +i8mm +set(I8MM_MARCH_FLAG "-march=armv8.2-a+fp16+simd+dotprod+i8mm") +add_library(cactus_i8mm_obj OBJECT ${I8MM_SOURCE}) +target_include_directories(cactus_i8mm_obj PRIVATE ${COMMON_INCLUDES}) +target_compile_options(cactus_i8mm_obj PRIVATE ${I8MM_MARCH_FLAG}) +set_target_properties(cactus_i8mm_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_compile_definitions(cactus_i8mm_obj PRIVATE CACTUS_COMPILE_I8MM __ARM_FEATURE_MATMUL_INT8=1) + +set(EXTRA_OBJECTS $) +if(BUILD_SME2) + set(EXTRA_OBJECTS ${EXTRA_OBJECTS} $) +endif() + +add_library(cactus STATIC ${COMMON_SOURCES} ${EXTRA_OBJECTS}) +configure_cactus_target(cactus) +target_compile_definitions(cactus PRIVATE CACTUS_COMPILE_I8MM) + +if(BUILD_SME2) + target_compile_definitions(cactus PRIVATE CACTUS_COMPILE_SME2) +endif() + +add_library(cactus_ffi SHARED ${COMMON_SOURCES} ${EXTRA_OBJECTS}) +configure_cactus_target(cactus_ffi) +set_target_properties(cactus_ffi PROPERTIES + POSITION_INDEPENDENT_CODE ON + OUTPUT_NAME "cactus" +) +target_compile_definitions(cactus_ffi PRIVATE CACTUS_COMPILE_I8MM) + +if(BUILD_SME2) + target_compile_definitions(cactus_ffi PRIVATE CACTUS_COMPILE_SME2) +endif() diff --git a/cactus-transpile/cactus.h b/cactus-transpile/cactus.h new file mode 100644 index 000000000..b5b6d803f --- /dev/null +++ b/cactus-transpile/cactus.h @@ -0,0 +1,12 @@ +#ifndef CACTUS_H +#define CACTUS_H + +#include "graph/graph.h" +#include "kernel/kernel.h" +#include "kernel/kernel_utils.h" +#include "engine/engine.h" +#include "models/model.h" +#include "ffi/cactus_ffi.h" +#include "npu/npu.h" + +#endif // CACTUS_H \ No newline at end of file diff --git a/cactus-transpile/engine/engine.h b/cactus-transpile/engine/engine.h new file mode 100644 index 000000000..c289466f6 --- /dev/null +++ b/cactus-transpile/engine/engine.h @@ -0,0 +1,1079 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../graph/graph.h" + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc99-extensions" +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +extern "C" { + #include "../../libs/stb/stb_image.h" + #include "../../libs/stb/stb_image_resize2.h" +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +class CactusGraph; + +namespace cactus { +namespace npu { + class NPUPrefill; +} +namespace engine { + +class Siglip2Preprocessor; + +struct Config { + uint32_t vocab_size = 151936; + uint32_t bos_token_id = 151643; + uint32_t eos_token_id = 151645; + uint32_t num_layers = 28; + uint32_t hidden_dim = 1024; + uint32_t ffn_intermediate_dim = 3072; + uint32_t attention_heads = 16; + uint32_t attention_kv_heads = 8; + uint32_t attention_head_dim = 128; + float layer_norm_eps = 1e-6f; + float rope_theta = 1000000.0f; + uint32_t num_experts = 0; + uint32_t num_shared_experts = 0; + uint32_t num_top_experts = 0; + uint32_t moe_every_n_layers = 0; + uint32_t moe_intermediate_dim = 0; + uint32_t num_dense_layers = 0; + uint32_t num_experts_per_tok = 0; + bool norm_topk_prob = false; + bool use_expert_bias = false; + float routed_scaling_factor = 1.0f; + bool tie_word_embeddings = true; + + uint32_t vision_hidden_dim = 0; + uint32_t vision_num_layers = 0; + uint32_t vision_attention_heads = 0; + uint32_t vision_image_size = 0; + uint32_t vision_patch_size = 0; + uint32_t vision_num_channels = 3; + uint32_t vision_embed_dim = 0; + uint32_t visual_tokens_per_img = 0; + bool use_pixel_shuffle = false; + uint32_t pixel_shuffle_factor = 1; + bool use_image_tokens = false; + uint32_t image_token_id = 0; + bool use_layout_tags = false; + uint32_t image_seq_len = 64; + + uint32_t global_image_size = 2048; + uint32_t max_tile_size = 512; + float rescale_factor = 0.00392156862745098f; + float image_mean = 0.5f; + float image_std = 0.5f; + + uint32_t downsample_factor = 2; + uint32_t min_tiles = 2; + uint32_t max_tiles = 10; + bool use_thumbnail = true; + uint32_t min_image_tokens = 64; + uint32_t max_image_tokens = 256; + uint32_t max_num_patches = 1024; + uint32_t tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_image_splitting = true; + bool encoder_act_gelu = false; + bool decoder_act_gelu = false; + uint32_t num_encoder_layers = 0; + uint32_t num_decoder_layers = 0; + float partial_rotary_factor = 0.0f; + uint32_t pad_token_id = 0; + uint32_t conv_kernel_size = 0; + uint32_t subsampling_conv_kernel_size = 0; + uint32_t subsampling_conv_stride = 0; + uint32_t subsampling_conv_channels = 0; + uint32_t subsampling_factor = 0; + uint32_t num_mel_bins = 80; + std::string encoder_hidden_act = "silu"; + uint32_t linear_num_key_heads = 0; + uint32_t linear_key_head_dim = 0; + uint32_t linear_num_value_heads = 0; + uint32_t linear_value_head_dim = 0; + uint32_t linear_q_proj_dim = 0; + uint32_t linear_k_proj_dim = 0; + uint32_t linear_v_proj_dim = 0; + + uint32_t kv_lora_rank = 0; + uint32_t q_lora_rank = 0; + uint32_t qk_head_dim = 0; + uint32_t qk_nope_head_dim = 0; + uint32_t qk_rope_head_dim = 0; + uint32_t v_head_dim = 0; + uint32_t rope_interleave = 0; + bool attention_bias = false; + float rope_scaling_factor = 1.0f; + float rope_mscale_all_dim = 0.0f; + + enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, PYANNOTE = 16, WESPEAKER = 17, NEEDLE = 18}; + uint32_t predictor_hidden_dim = 0; + uint32_t predictor_num_layers = 0; + uint32_t tdt_joint_dim = 0; + uint32_t tdt_num_durations = 0; + uint32_t tdt_blank_id = 0; + std::vector tdt_durations; + + ModelType model_type = ModelType::QWEN; + + enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3}; + ModelVariant model_variant = ModelVariant::DEFAULT; + + enum class Activation {GELU = 0, SILU = 1}; + Activation activation = Activation::SILU; + + enum class Backend {CPU = 0, NPU = 1}; + Backend default_backend = Backend::CPU; + + enum class Precision {INT8 = 0, FP16 = 1, FP32 = 2}; + Precision precision = Precision::FP32; + + float default_temperature = 0.6f; + float default_top_p = 0.95f; + size_t default_top_k = 20; + float default_max_tps = -1.0f; + float default_cloud_handoff_threshold = 0.0f; + + std::vector layer_types; + size_t conv_L_cache = 0; + + uint32_t altup_num_inputs = 4; + uint32_t laurel_rank = 64; + uint32_t hidden_size_per_layer_input = 256; + uint32_t num_kv_shared_layers = 0; + uint32_t sliding_window = 512; + float rope_local_base_freq = 10000.0f; + float final_logit_softcapping = 0.0f; + float global_partial_rotary_factor = 1.0f; + uint32_t expert_intermediate_size = 0; + uint32_t global_head_dim = 0; + uint32_t num_global_kv_heads = 0; + bool attention_k_eq_v = false; + bool enable_moe_block = false; + std::vector activation_sparsity_ppf; + + uint32_t vision_head_dim = 64; + uint32_t vision_kv_heads = 12; + uint32_t vision_intermediate_size = 3072; + uint32_t vision_position_embedding_size = 10240; + uint32_t vision_pooling_kernel_size = 3; + uint32_t vision_default_output_length = 280; + float vision_rope_theta = 100.0f; + + uint32_t audio_hidden_dim = 0; + uint32_t audio_num_layers = 0; + uint32_t audio_num_heads = 0; + uint32_t audio_head_dim = 0; + uint32_t audio_input_feat_size = 128; + uint32_t audio_conf_conv_kernel_size = 5; + uint32_t audio_chunk_size = 12; + uint32_t audio_context_left = 13; + uint32_t audio_context_right = 0; + float audio_logit_cap = 50.0f; + float audio_residual_weight = 0.5f; + uint32_t audio_output_proj_dims = 0; + uint32_t audio_vocab_size = 128; + uint32_t audio_vocab_offset = 0; + uint32_t audio_soft_tokens = 188; + uint32_t audio_sscp_conv0_channels = 128; + uint32_t audio_sscp_conv1_channels = 32; + float audio_sscp_conv_eps = 1e-3f; + float audio_rms_norm_eps = 1e-6f; + uint32_t audio_fft_length = 1024; + uint32_t audio_token_id = 0; + bool audio_fft_overdrive = false; + uint32_t channel_open_token_id = 100; + uint32_t channel_close_token_id = 101; + + static bool is_gemma_family(ModelType t) { + return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; + } + + bool from_json(const std::string& json_path); + std::string to_json() const; +}; + + + +struct MergeRule { + std::string first; + std::string second; + std::string merged; + uint32_t priority; + + MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) + : first(f), second(s), merged(m), priority(p) {} +}; + + +struct ToolCallInfo { + std::string name; + std::string arguments; +}; + +struct ChatMessage { + std::string role; + std::string content; + std::string name; + std::vector images; + std::vector audio; + size_t audio_soft_token_count = 0; + std::vector tool_calls; +}; + +inline std::string format_needle_query_text(const std::vector& messages) { + std::string system_text; + std::string user_query; + + for (const auto& msg : messages) { + if (msg.role == "system") { + if (!system_text.empty()) { + system_text += "\n"; + } + system_text += msg.content; + } else if (msg.role == "user") { + user_query = msg.content; + } + } + + if (user_query.empty() && !messages.empty()) { + user_query = messages.back().content; + } + if (system_text.empty()) { + return user_query; + } + if (user_query.empty()) { + return system_text; + } + return system_text + "\n\n" + user_query; +} + +struct ToolConstraintSpec { + std::string name; + std::vector parameter_names; + std::vector required_parameter_names; +}; + +struct TokenizerRuntimeConfig { + enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE }; + enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN }; + enum class Normalizer { NONE, METASPACE, BYTE_LEVEL }; + enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL }; + + TokenizerType tokenizer_type = TokenizerType::UNKNOWN; + VocabFormat vocab_format = VocabFormat::UNKNOWN; + Normalizer normalizer = Normalizer::NONE; + Decoder decoder = Decoder::NONE; + bool byte_fallback = false; + bool has_chat_template = false; +}; + +TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); +void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); +std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); + +class Tokenizer { +public: + virtual ~Tokenizer() = default; + + virtual std::vector encode(const std::string& text) const = 0; + virtual std::string decode(const std::vector& tokens) const = 0; + + virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; + virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = true) const; + + virtual uint32_t get_vocab_size() const = 0; + virtual uint32_t get_unk_token() const = 0; + virtual uint32_t get_bos_token() const = 0; + virtual uint32_t get_eos_token() const = 0; + virtual bool has_chat_template() const { return has_chat_template_; } + std::string get_default_stop_sequence() const; + + virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; + + uint32_t get_image_token_id() const { return image_token_id_; } + uint32_t get_fake_token_id() const { return fake_token_id_; } + uint32_t get_global_img_token_id() const { return global_img_token_id_; } + +protected: + enum class ModelType { UNKNOWN, QWEN, QWEN3P5, GEMMA, GEMMA4, LFM2, BERT, WHISPER, PARAKEET, YOUTU, NEEDLE}; + ModelType model_type_ = ModelType::UNKNOWN; + enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG}; + ModelVariant model_variant_ = ModelVariant::DEFAULT; + bool has_chat_template_ = false; + std::string chat_template_; + + uint32_t image_token_id_ = 396; + uint32_t fake_token_id_ = 49189; + uint32_t global_img_token_id_ = 49152; + + + uint32_t vision_patch_size_ = 16; + uint32_t vision_pooling_kernel_size_ = 3; + uint32_t vision_default_output_length_ = 280; + uint32_t vision_image_size_ = 768; + TokenizerRuntimeConfig runtime_config_; + + void detect_model_type(const std::string& config_path); + void load_chat_template(const std::string& template_file); + std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = true) const; + std::string format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = true) const; + std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_lfm2_vl_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_needle_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; +}; + +class BPETokenizer : public Tokenizer { +public: + BPETokenizer(); + ~BPETokenizer(); + + bool load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file); + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector merge_rules_; + std::unordered_map merge_map_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void* merges_mmap_ptr_; + size_t merges_mmap_size_; + + std::vector apply_bpe(const std::vector& tokens) const; + std::pair find_best_merge_fast(const std::vector& tokens) const; + + std::string bytes_to_unicode(const std::string& text) const; + std::string unicode_to_bytes(const std::string& text) const; + std::vector byte_level_split(const std::string& text) const; + std::vector utf8_split(const std::string& text) const; + + void cleanup_mmap(); + +private: + mutable std::unordered_map byte_to_unicode_; + mutable std::unordered_map unicode_to_byte_; + void init_byte_mappings() const; + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class SPTokenizer : public Tokenizer { +public: + SPTokenizer(); + ~SPTokenizer(); + + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + struct TrieNode { + std::unordered_map> children; + int32_t token_id = -1; + float score = 0.0f; + }; + + std::unique_ptr trie_root_; + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector token_scores_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + uint32_t pad_token_id_; + + bool sp_bpe_mode_ = false; + bool sp_add_dummy_prefix_ = false; + bool sp_byte_fallback_ = false; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void build_trie(); + std::vector> tokenize_with_trie(const std::string& text) const; + std::vector tokenize_with_bpe(const std::string& text) const; + std::string preprocess_text(const std::string& text) const; + std::string postprocess_text(const std::string& text) const; + std::vector split_by_unicode_spaces(const std::string& text) const; + + void cleanup_mmap(); + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class ConvCache { +public: + struct CircularView { + const void* ptr1; + size_t len1; + const void* ptr2; + size_t len2; + size_t total_len; + }; + + void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); + CircularView get_window(size_t layer) const; + void update(CactusGraph* gb, size_t layer, const size_t latest_token); + void reset(); + + bool is_empty() const { return num_layers == 0; } + + size_t num_layers = 0; + size_t hidden_size = 0; + size_t window_size = 0; + Precision precision = Precision::FP32; + size_t element_size = 4; + +private: + struct LayerState { + std::vector data; + size_t head = 0; + size_t count = 0; + }; + + std::vector layer_states; +}; + +struct KVCache { + static constexpr size_t DEFAULT_WINDOW_SIZE = 1024; + static constexpr size_t DEFAULT_SINK_SIZE = 4; + + struct LayerCache { + std::vector keys; + std::vector values; + std::vector key_scales; + std::vector value_scales; + size_t head_dim = 0; + size_t kv_heads = 0; + }; + + std::vector layer_caches; + + size_t window_size = DEFAULT_WINDOW_SIZE; + size_t sink_size = DEFAULT_SINK_SIZE; + size_t current_seq_len = 0; + size_t total_seq_len = 0; + size_t max_seq_len = 2048; + size_t num_layers = 0; + Precision precision; + size_t element_size = 4; + + void set_window_size(size_t window, size_t sink = DEFAULT_SINK_SIZE); + size_t get_effective_seq_len() const { return current_seq_len; } + size_t get_total_seq_len() const { return total_seq_len; } + size_t get_layer_head_dim(size_t layer_idx) const { return layer_caches[layer_idx].head_dim; } + size_t get_layer_kv_heads(size_t layer_idx) const { return layer_caches[layer_idx].kv_heads; } + + void init(size_t num_layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision); + void reset(); + void update_from_graph(CactusGraph* gb, const std::vector& k_nodes, + const std::vector& v_nodes, size_t seq_len, + size_t num_layers); + + void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, + size_t num_tokens, size_t kv_heads, size_t head_dim); + + bool is_empty() const { return current_seq_len == 0; } + void* get_key_ptr(size_t layer); + void* get_value_ptr(size_t layer); + + struct CircularView { + const void* ptr1; + const void* ptr2; + size_t len1; + size_t len2; + size_t total_len; + }; + + CircularView get_key_view(size_t layer); + CircularView get_value_view(size_t layer); + + const int8_t* get_keys_int8(size_t layer) const; + const int8_t* get_values_int8(size_t layer) const; + const float* get_key_scales(size_t layer) const; + const float* get_value_scales(size_t layer) const; + + void remove_token_range(size_t start, size_t count); + void compact_to_windows(const std::vector& target_windows); +}; + +class ToolCallConstrainer { +public: + enum class State { + DONE, + + QWEN_START, + QWEN_EXPECT_OPEN_BRACE, + QWEN_EXPECT_NAME_KEY, + QWEN_EXPECT_NAME_COLON, + QWEN_EXPECT_NAME_VALUE, + QWEN_EXPECT_COMMA, + QWEN_EXPECT_ARGS_KEY, + QWEN_EXPECT_ARGS_COLON, + QWEN_IN_ARGUMENTS, + QWEN_EXPECT_CLOSE_BRACE, + QWEN_EXPECT_END, + + NEEDLE_START, + + LFM_START, + LFM_EXPECT_BRACKET, + LFM_IN_FUNC_NAME, + LFM_EXPECT_PAREN, + LFM_IN_ARGUMENTS, + LFM_EXPECT_BRACKET_CLOSE, + LFM_EXPECT_END, + + GEMMA_START, + GEMMA_EXPECT_CALL, + GEMMA_IN_FUNC_NAME, + GEMMA_EXPECT_BRACE, + GEMMA_IN_ARGUMENTS, + GEMMA_EXPECT_END + }; + + void init(Config::ModelType model_type, + const std::vector& tools, + Tokenizer* tokenizer); + + const std::unordered_map& get_bias() const { return current_bias_; } + + void update(uint32_t token_id, const std::string& decoded_text); + + void reset(); + + bool is_active() const { return active_; } + +private: + bool active_ = false; + State state_ = State::QWEN_START; + Config::ModelType model_type_ = Config::ModelType::QWEN; + Tokenizer* tokenizer_ = nullptr; + + bool is_gemma_family() const { return Config::is_gemma_family(model_type_); } + bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } + + enum class NeedleJsonState { + FREE, + IN_NAME, + IN_ARG_KEY, + }; + + struct NeedleTrieNode { + std::unordered_map> children; + bool is_terminal = false; + }; + + std::vector tool_specs_; + std::vector function_names_; + std::string generated_text_; + int brace_depth_ = 0; + + std::string call_start_tag_; + std::string call_end_tag_; + + std::unordered_set qwen_tool_call_start_tokens_; + std::unordered_set qwen_tool_call_end_tokens_; + std::unordered_set open_brace_tokens_; + std::unordered_set close_brace_tokens_; + std::unordered_set colon_tokens_; + std::unordered_set comma_tokens_; + std::unordered_set name_key_tokens_; + std::unordered_set args_key_tokens_; + std::unordered_set quote_tokens_; + std::unordered_set backtick_tokens_; + std::unordered_set all_func_name_tokens_; + std::unordered_map> func_name_sequences_; + NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; + std::string needle_buffer_; + std::string needle_constrained_buf_; + std::string needle_current_function_; + bool needle_in_arguments_ = false; + int needle_arguments_depth_ = 0; + int needle_nesting_depth_ = 0; + bool needle_in_string_value_ = false; + bool needle_between_pairs_ = false; + bool needle_prev_char_escape_ = false; + std::unique_ptr needle_name_trie_; + std::unordered_map> needle_param_tries_; + std::unordered_map> needle_required_params_; + std::unordered_set needle_seen_arg_keys_; + std::vector needle_token_strings_; + std::unordered_map> needle_token_index_; + std::unordered_set needle_arg_close_tokens_; + + std::unordered_set tool_start_tokens_; + std::unordered_set tool_end_tokens_; + std::unordered_set bracket_open_tokens_; + std::unordered_set bracket_close_tokens_; + std::unordered_set paren_open_tokens_; + std::unordered_set paren_close_tokens_; + std::unordered_set equals_tokens_; + + std::unordered_set gemma_call_start_tokens_; + std::unordered_set gemma_call_end_tokens_; + std::unordered_set gemma_response_start_tokens_; + std::unordered_set gemma_call_prefix_tokens_; + std::unordered_set escape_tokens_; + + std::unordered_map current_bias_; + + void compute_bias(); + void tokenize_grammar_elements(); + void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); + void tokenize_function_names(bool quote_names); + void init_common_tokens(); + void init_needle_constraints(); + void reset_needle_constraints(); + void feed_needle_text(const std::string& text); + void feed_needle_char(char ch); + bool needle_at_arg_key_start() const; + bool needle_is_value_string_start() const; + void needle_insert_word(NeedleTrieNode* root, const std::string& word); + const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; + bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; + bool needle_has_unseen_completion(const NeedleTrieNode* node, std::string& partial) const; +}; + +class Model { +public: + struct DebugNode { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + Model(); + explicit Model(const Config& config); + virtual ~Model(); + + const Config& get_config() const { return config_; } + Tokenizer* get_tokenizer() const { return tokenizer_.get(); } + const std::vector& get_debug_nodes() const; + + virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); + + virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true); + + virtual uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + + virtual void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file = ""); + + virtual uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, + float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, float temperature = 0.0f, float top_p = 0.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); + + std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); + + virtual std::vector get_image_embeddings(const std::string& image_path); + + virtual std::vector get_audio_embeddings(const std::vector& audio_features); + + virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } + void record_sampled_token(uint32_t token) { + if (token_history_.size() >= MAX_TOKEN_HISTORY) { + token_history_.erase(token_history_.begin(), token_history_.begin() + (MAX_TOKEN_HISTORY / 2)); + } + token_history_.push_back(token); + } + + double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored); + + + + void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); } + + bool load_npu_prefill(const std::string& model_path); + bool has_npu_prefill() const; + size_t get_prefill_chunk_size() const; + + virtual void remove_thinking_tokens(const std::vector>& ranges); + virtual void compact_kv_cache() {} + + void set_tool_constraints(const std::vector& tools); + void clear_tool_constraints(); + void update_tool_constraints(uint32_t token_id); + + void* graph_handle_; + + void set_vocab_bias(const std::unordered_map& bias) { + vocab_bias_ = bias; + } + + void clear_vocab_bias() { + vocab_bias_.clear(); + } + + bool has_vocab_bias() const { + return !vocab_bias_.empty(); + } + + const std::unordered_map& get_vocab_bias() const { + return vocab_bias_; + } + +protected: + size_t sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, + float min_p, float repetition_penalty, + const std::unordered_map* extra_bias = nullptr) const; + + static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); + + virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; + + virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); + + virtual void load_weights_to_graph(CactusGraph* gb) = 0; + + virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + + virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const = 0; + virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + void update_kv_cache(CactusGraph* gb, size_t seq_len); + virtual std::vector get_kv_layer_dims() const { + return std::vector(config_.num_layers, config_.attention_head_dim); + } + virtual std::vector get_kv_layer_heads() const { + return std::vector(config_.num_layers, config_.attention_kv_heads); + } + virtual void post_init() {} + virtual void post_execute_updates(CactusGraph*, size_t) {} + Config config_; + std::unique_ptr tokenizer_; + + bool initialized_; + float attention_scale_; + +protected: + KVCache kv_cache_; + std::vector cache_k_output_nodes_; + std::vector cache_v_output_nodes_; + + std::string embedding_file_path_; + size_t embedding_node_id_; + std::string model_folder_path_; + size_t output_weight_node_id_; + + mutable std::vector debug_nodes_; + + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const; + void clear_debug_nodes(); + + bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup); + bool owns_graph_; + + std::unique_ptr npu_prefill_; + void prefill_npu(const std::vector& tokens); + virtual std::vector<__fp16> get_token_embeddings(const std::vector& tokens); + + static constexpr size_t MAX_TOKEN_HISTORY = 128; + ToolCallConstrainer tool_constrainer_; + std::vector token_history_; + +private: + std::unordered_map vocab_bias_; +}; + +std::unique_ptr create_model(const std::string& model_folder); + +class Siglip2Preprocessor { +public: + struct Config { + int patch_size = 16; + int downsample_factor = 2; + int min_tiles = 2; + int max_tiles = 10; + bool use_thumbnail = true; + int min_image_tokens = 64; + int max_image_tokens = 256; + int max_num_patches = 1024; + int tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_resize = true; + bool do_rescale = true; + bool do_normalize = true; + bool do_convert_rgb = true; + bool do_image_splitting = true; + float rescale_factor = 1.0f / 255.0f; + float image_mean[3] = {0.5f, 0.5f, 0.5f}; + float image_std[3] = {0.5f, 0.5f, 0.5f}; + }; + + struct PreprocessedImage { + std::vector pixel_values; + std::vector pixel_attention_mask; + std::vector> spatial_shapes; + std::vector pixel_values_shape; + std::vector pixel_attention_mask_shape; + std::vector spatial_shapes_shape; + int num_patches_height; + int num_patches_width; + int actual_num_patches; + int num_tiles; + int patch_dim; + int max_patches_per_tile; + + int image_rows; + int image_cols; + int image_height; + int image_width; + int tokens_per_tile; + int thumbnail_tokens; + + ~PreprocessedImage(); + }; + + struct SpatialShapeResult { + std::vector> shapes; + int grid_rows; + int grid_cols; + }; + + explicit Siglip2Preprocessor(const Config& config); + Siglip2Preprocessor(); + ~Siglip2Preprocessor(); + + PreprocessedImage preprocess_from_file(const std::string& image_path); + PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels); + SpatialShapeResult compute_spatial_shapes(int height, int width); + +private: + Config config_; + + std::pair compute_pixel_limits() const; + std::vector convert_to_rgb(const unsigned char* img_data, int width, int height, int channels); + std::pair smart_resize(int height, int width); + bool is_image_too_large(int height, int width); + std::pair get_grid_layout(int height, int width); + std::pair find_closest_aspect_ratio(float aspect_ratio, int width, int height); + std::vector resize_image(const unsigned char* img_data, int src_width, int src_height, + int dst_width, int dst_height, int channels); + std::vector normalize_image(const float* img_data, int width, int height, int channels); + std::vector> convert_image_to_patches( + const std::vector& image, int width, int height, int channels, int patch_size); + PreprocessedImage pad_patches(const std::vector>& tile_patches, + const std::vector>& spatial_shapes, + int patch_dim, + int max_patches_per_tile); + int round_by_factor(int number, int factor); +}; + +class AudioProcessor { +public: + struct SpectrogramConfig { + size_t n_fft = 400; + size_t hop_length = 160; + size_t frame_length = 400; + float power = 2.0f; + bool center = true; + const char* pad_mode = "reflect"; + bool onesided = true; + float dither = 0.0f; + float mel_floor = 1e-10f; + const char* log_mel = nullptr; + float reference = 1.0f; + float min_value = 1e-10f; + bool remove_dc_offset = false; + float preemphasis = 0.0f; + bool hann_periodic = true; + float window_a0 = 0.5f; + size_t fft_override = 0; + bool mel_floor_additive = false; + }; + + AudioProcessor(); + ~AudioProcessor(); + + void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters, + float min_freq, float max_freq, size_t sampling_rate, + const char* norm = "slaney", const char* mel_scale = "slaney"); + + std::vector compute_spectrogram( + const std::vector& waveform, + const SpectrogramConfig& config); + + static std::vector compute_irfft( + const std::vector& complex_input, + size_t n, + const char* norm = "backward"); + + const std::vector& get_mel_filters() const { return mel_filters_; } + + size_t get_num_mel_filters() const { return num_mel_filters_; } + size_t get_num_frequency_bins() const { return num_frequency_bins_; } + +private: + std::vector mel_filters_; + size_t num_frequency_bins_; + size_t num_mel_filters_; +}; + +namespace index { + constexpr uint32_t MAGIC = 0x43414354; + constexpr uint32_t VERSION = 1; + + struct Document { + int id; + std::vector embedding; + std::string content; + std::string metadata; + }; + + struct QueryResult { + int doc_id; + float score; + + QueryResult(int doc_id, float score) : doc_id(doc_id), score(score) {} + }; + + struct QueryOptions { + size_t top_k = 10; + float score_threshold = -1.0f; + }; + + class Index { + public: + Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim); + ~Index(); + + Index(const Index&) = delete; + Index& operator=(const Index&) = delete; + Index(Index&&) = delete; + Index& operator=(Index&&) = delete; + + void add_documents(const std::vector& documents); + void delete_documents(const std::vector& doc_ids); + std::vector get_documents(const std::vector& doc_ids); + std::vector> query(const std::vector>& embeddings, const QueryOptions& options); + void compact(); + + private: + struct IndexHeader { + uint32_t magic; + uint32_t version; + uint32_t embedding_dim; + uint32_t num_documents; + }; + + struct IndexEntry { + int32_t doc_id; + uint64_t data_offset; + uint8_t flags; // bit 0: tombstone + + const __fp16* embedding() const { + return reinterpret_cast(this + 1); + } + + static size_t size(size_t embedding_dim) { + return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16); + } + }; + + struct DataHeader { + uint32_t magic; + uint32_t version; + }; + + struct DataEntry { + uint16_t content_len; + uint16_t metadata_len; + + const char* content() const { + return reinterpret_cast(this + 1); + } + + const char* metadata() const { + return content() + content_len; + } + }; + + void parse_index_header(); + void parse_data_header(); + void build_doc_id_map(); + void validate_documents(const std::vector& documents); + void validate_doc_ids(const std::vector& doc_ids); + ssize_t write_full(int fd, const void* buf, size_t count); + + std::unordered_map doc_id_map_; + + std::string index_path_, data_path_; + size_t embedding_dim_; + size_t index_entry_size_; + uint32_t num_documents_; + + int index_fd_, data_fd_; + void *mapped_index_, *mapped_data_; + size_t index_file_size_, data_file_size_; + }; +} // namespace index + +} +} diff --git a/cactus-transpile/engine/engine_bpe.cpp b/cactus-transpile/engine/engine_bpe.cpp new file mode 100644 index 000000000..6a882c6f1 --- /dev/null +++ b/cactus-transpile/engine/engine_bpe.cpp @@ -0,0 +1,538 @@ +#include "engine.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +namespace { +constexpr const char* kMetaspace = "\xE2\x96\x81"; +} // namespace + +BPETokenizer::BPETokenizer() + : vocab_size_(0), unk_token_id_(0), bos_token_id_(1), eos_token_id_(2), + vocab_mmap_ptr_(nullptr), vocab_mmap_size_(0), + merges_mmap_ptr_(nullptr), merges_mmap_size_(0) { + has_chat_template_ = false; +} + +BPETokenizer::~BPETokenizer() { + cleanup_mmap(); +} + +void BPETokenizer::cleanup_mmap() { + if (vocab_mmap_ptr_ && vocab_mmap_ptr_ != MAP_FAILED) { + munmap(vocab_mmap_ptr_, vocab_mmap_size_); + vocab_mmap_ptr_ = nullptr; + } + if (merges_mmap_ptr_ && merges_mmap_ptr_ != MAP_FAILED) { + munmap(merges_mmap_ptr_, merges_mmap_size_); + merges_mmap_ptr_ = nullptr; + } +} + +bool BPETokenizer::load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file) { + int vocab_fd = open(vocab_file.c_str(), O_RDONLY); + if (vocab_fd == -1) return false; + + struct stat vocab_stat; + if (fstat(vocab_fd, &vocab_stat) == -1) { + close(vocab_fd); + return false; + } + + vocab_mmap_size_ = vocab_stat.st_size; + vocab_mmap_ptr_ = mmap(nullptr, vocab_mmap_size_, PROT_READ, MAP_PRIVATE, vocab_fd, 0); + close(vocab_fd); + + if (vocab_mmap_ptr_ == MAP_FAILED) return false; + + std::string vocab_content(static_cast(vocab_mmap_ptr_), vocab_mmap_size_); + std::istringstream vocab_stream(vocab_content); + + auto rtrim_cr = [](std::string& s) { + if (!s.empty() && s.back() == '\r') s.pop_back(); + }; + + std::string line; + token_to_id_.clear(); + id_to_token_.clear(); + special_tokens_.clear(); + const bool use_id_tab_vocab = + runtime_config_.vocab_format == TokenizerRuntimeConfig::VocabFormat::ID_TAB_TOKEN; + + if (use_id_tab_vocab) { + while (std::getline(vocab_stream, line)) { + rtrim_cr(line); + + std::string token; + uint32_t id = UINT32_MAX; + + std::istringstream iss(line); + if (iss >> id) { + if (std::getline(iss, token) && !token.empty() && token[0] == '\t') { + token = token.substr(1); + } + + if (token.empty()) { + auto last_pos = vocab_stream.tellg(); + while (std::getline(vocab_stream, line)) { + rtrim_cr(line); + if (!line.empty()) { + break; + } + token += '\n'; + last_pos = vocab_stream.tellg(); + } + vocab_stream.seekg(last_pos); + } + } + + if (!token.empty() && id != UINT32_MAX) { + token_to_id_[token] = id; + if (id >= id_to_token_.size()) { + id_to_token_.resize(id + 1); + } + id_to_token_[id] = token; + } + } + vocab_size_ = static_cast(id_to_token_.size()); + } else { + uint32_t id = 0; + while (std::getline(vocab_stream, line)) { + rtrim_cr(line); + token_to_id_[line] = id; + id_to_token_.push_back(line); + ++id; + } + vocab_size_ = id; + } + + int merges_fd = open(merges_file.c_str(), O_RDONLY); + if (merges_fd == -1) return false; + + struct stat merges_stat; + if (fstat(merges_fd, &merges_stat) == -1) { + close(merges_fd); + return false; + } + + merges_mmap_size_ = merges_stat.st_size; + merges_mmap_ptr_ = mmap(nullptr, merges_mmap_size_, PROT_READ, MAP_PRIVATE, merges_fd, 0); + close(merges_fd); + + if (merges_mmap_ptr_ == MAP_FAILED) return false; + + std::string merges_content(static_cast(merges_mmap_ptr_), merges_mmap_size_); + std::istringstream merges_stream(merges_content); + + merge_rules_.clear(); + uint32_t priority = 0; + + while (std::getline(merges_stream, line)) { + rtrim_cr(line); + if (line.empty() || line[0] == '#') continue; + + std::istringstream iss(line); + std::string first, second; + if (iss >> first >> second) { + rtrim_cr(first); + rtrim_cr(second); + + std::string merged = first + second; + merge_rules_.emplace_back(first, second, merged, priority); + + std::string key = first + "\x00" + second; + auto it = merge_map_.find(key); + if (it == merge_map_.end() || priority < it->second) { + merge_map_[key] = priority; + } + priority++; + } + } + + std::sort(merge_rules_.begin(), merge_rules_.end(), + [](const MergeRule& a, const MergeRule& b) { + return a.priority < b.priority; + }); + + return true; +} + +bool BPETokenizer::load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) { + runtime_config_ = load_tokenizer_runtime_config(config_file); + if (!load_vocabulary_mmap(vocab_file, merges_file)) { + return false; + } + + std::ifstream config_stream(config_file); + if (!config_stream.is_open()) { + return true; + } + + std::string line; + while (std::getline(config_stream, line)) { + if (line.empty() || line[0] == '#') continue; + + size_t eq_pos = line.find('='); + if (eq_pos == std::string::npos) continue; + + std::string key = line.substr(0, eq_pos); + std::string value = line.substr(eq_pos + 1); + + key.erase(0, key.find_first_not_of(" \t")); + key.erase(key.find_last_not_of(" \t") + 1); + value.erase(0, value.find_first_not_of(" \t")); + value.erase(value.find_last_not_of(" \t") + 1); + + if (key == "eos_token_id") { + eos_token_id_ = std::stoul(value); + } else if (key == "pad_token_id") { + if (unk_token_id_ == 0) { + unk_token_id_ = std::stoul(value); + } + } else if (key == "unk_token_id" && value != "null") { + unk_token_id_ = std::stoul(value); + } else if (key == "bos_token_id" && value != "null") { + bos_token_id_ = std::stoul(value); + } else if (key == "vocab_size") { + if (std::stoul(value) != vocab_size_) { + } + } + } + + std::string dir = config_file.substr(0, config_file.find_last_of("/\\")); + std::string special_tokens_path = dir + "/special_tokens.json"; + load_special_tokens(special_tokens_path); + + std::string template_path = dir + "/chat_template.jinja2"; + load_chat_template(template_path); + + std::string config_path = dir + "/config.txt"; + detect_model_type(config_path); + + return true; +} + +void BPETokenizer::load_special_tokens(const std::string& config_file) { + load_special_tokens_map(config_file, special_tokens_); +} + +std::vector BPETokenizer::split_with_special_tokens(const std::string& text) const { + return cactus::engine::split_with_special_tokens(text, special_tokens_); +} + +void BPETokenizer::init_byte_mappings() const { + if (!byte_to_unicode_.empty()) return; + + std::vector bytes; + + for (int i = 33; i <= 126; ++i) { + bytes.push_back(i); + } + + + for (int i = 161; i <= 255; ++i) { + bytes.push_back(i); + } + + std::vector remaining_bytes; + for (int i = 0; i <= 32; ++i) remaining_bytes.push_back(i); + remaining_bytes.push_back(127); + for (int i = 128; i <= 160; ++i) remaining_bytes.push_back(i); + + int unicode_start = 256; + for (int byte : remaining_bytes) { + bytes.push_back(byte); + } + + for (size_t i = 0; i < bytes.size(); ++i) { + uint8_t byte = static_cast(bytes[i]); + + if (byte >= 33 && byte <= 126) { + std::string unicode_char(1, static_cast(byte)); + byte_to_unicode_[byte] = unicode_char; + unicode_to_byte_[unicode_char] = byte; + } else if (byte >= 161 && byte <= 255) { + std::string unicode_char; + unicode_char += static_cast(0xC0 | (byte >> 6)); + unicode_char += static_cast(0x80 | (byte & 0x3F)); + byte_to_unicode_[byte] = unicode_char; + unicode_to_byte_[unicode_char] = byte; + } else { + int unicode_point = unicode_start++; + std::string unicode_char; + if (unicode_point < 0x800) { + unicode_char += static_cast(0xC0 | (unicode_point >> 6)); + unicode_char += static_cast(0x80 | (unicode_point & 0x3F)); + } else { + unicode_char += static_cast(0xE0 | (unicode_point >> 12)); + unicode_char += static_cast(0x80 | ((unicode_point >> 6) & 0x3F)); + unicode_char += static_cast(0x80 | (unicode_point & 0x3F)); + } + byte_to_unicode_[byte] = unicode_char; + unicode_to_byte_[unicode_char] = byte; + } + } +} + +std::string BPETokenizer::bytes_to_unicode(const std::string& text) const { + init_byte_mappings(); + + std::string result; + for (uint8_t byte : text) { + result += byte_to_unicode_.at(byte); + } + return result; +} + +std::string BPETokenizer::unicode_to_bytes(const std::string& text) const { + init_byte_mappings(); + + std::string result; + size_t i = 0; + while (i < text.length()) { + std::string unicode_char; + + if ((text[i] & 0x80) == 0) { + unicode_char = text.substr(i, 1); + i += 1; + } else if ((text[i] & 0xE0) == 0xC0) { + unicode_char = text.substr(i, 2); + i += 2; + } else if ((text[i] & 0xF0) == 0xE0) { + unicode_char = text.substr(i, 3); + i += 3; + } else { + unicode_char = text.substr(i, 1); + i += 1; + } + + auto it = unicode_to_byte_.find(unicode_char); + if (it != unicode_to_byte_.end()) { + result += static_cast(it->second); + } else if (unicode_char.size() == 1 && (unsigned char)unicode_char[0] < 128) { + result += unicode_char[0]; + } else { + result += '?'; + } + } + return result; +} + +std::vector BPETokenizer::byte_level_split(const std::string& text) const { + std::string unicode_text = bytes_to_unicode(text); + + std::vector chars; + size_t i = 0; + while (i < unicode_text.length()) { + size_t char_len = 1; + + if ((unicode_text[i] & 0x80) == 0) { + char_len = 1; + } else if ((unicode_text[i] & 0xE0) == 0xC0) { + char_len = 2; + } else if ((unicode_text[i] & 0xF0) == 0xE0) { + char_len = 3; + } else if ((unicode_text[i] & 0xF8) == 0xF0) { + char_len = 4; + } + + if (i + char_len <= unicode_text.length()) { + chars.push_back(unicode_text.substr(i, char_len)); + } + i += char_len; + } + + return chars; +} + +std::vector BPETokenizer::utf8_split(const std::string& text) const { + std::vector chars; + size_t i = 0; + while (i < text.length()) { + size_t char_len = 1; + const unsigned char byte = static_cast(text[i]); + + if ((byte & 0x80) == 0) { + char_len = 1; + } else if ((byte & 0xE0) == 0xC0) { + char_len = 2; + } else if ((byte & 0xF0) == 0xE0) { + char_len = 3; + } else if ((byte & 0xF8) == 0xF0) { + char_len = 4; + } + + if (i + char_len <= text.length()) { + chars.push_back(text.substr(i, char_len)); + } + i += char_len; + } + + return chars; +} + + +std::pair BPETokenizer::find_best_merge_fast(const std::vector& tokens) const { + int best_pos = -1; + uint32_t best_priority = UINT32_MAX; + + for (size_t i = 0; i < tokens.size() - 1; ++i) { + std::string key = tokens[i] + "\x00" + tokens[i + 1]; + auto it = merge_map_.find(key); + if (it != merge_map_.end()) { + if (it->second < best_priority) { + best_priority = it->second; + best_pos = static_cast(i); + } + } + } + + return {best_pos, best_priority}; +} + +std::vector BPETokenizer::apply_bpe(const std::vector& tokens) const { + if (tokens.size() <= 1) return tokens; + + std::vector current_tokens = tokens; + + + while (true) { + auto [merge_pos, priority] = find_best_merge_fast(current_tokens); + if (merge_pos == -1) break; + + + std::vector new_tokens; + new_tokens.reserve(current_tokens.size() - 1); + + for (int i = 0; i < static_cast(current_tokens.size()); ++i) { + if (i == merge_pos) { + std::string merged = current_tokens[i] + current_tokens[i + 1]; + new_tokens.push_back(merged); + i++; + } else { + new_tokens.push_back(current_tokens[i]); + } + } + current_tokens = std::move(new_tokens); + } + + return current_tokens; +} + +std::vector BPETokenizer::encode(const std::string& text) const { + if (text.empty()) return {}; + + + auto text_segments = split_with_special_tokens(text); + + + std::vector token_ids; + + for (const auto& segment : text_segments) { + auto special_it = special_tokens_.find(segment); + if (special_it != special_tokens_.end()) { + token_ids.push_back(special_it->second); + } else { + std::string normalized_segment = segment; + std::vector chars; + if (runtime_config_.normalizer == TokenizerRuntimeConfig::Normalizer::METASPACE) { + size_t pos = 0; + while ((pos = normalized_segment.find(' ', pos)) != std::string::npos) { + normalized_segment.replace(pos, 1, kMetaspace); + pos += 3; + } + chars = utf8_split(normalized_segment); + } else { + chars = byte_level_split(segment); + } + auto bpe_tokens = apply_bpe(chars); + + for (const auto& token : bpe_tokens) { + auto it = token_to_id_.find(token); + if (it != token_to_id_.end()) { + token_ids.push_back(it->second); + } else { + if (!runtime_config_.byte_fallback) { + token_ids.push_back(unk_token_id_); + continue; + } + + std::vector fallback_ids; + fallback_ids.reserve(token.size()); + for (unsigned char byte : token) { + char fallback_token[7]; + std::snprintf(fallback_token, sizeof(fallback_token), "<0x%02X>", byte); + auto fallback_it = token_to_id_.find(fallback_token); + if (fallback_it == token_to_id_.end()) { + fallback_ids.clear(); + break; + } + fallback_ids.push_back(fallback_it->second); + } + + if (fallback_ids.empty()) { + token_ids.push_back(unk_token_id_); + } else { + token_ids.insert(token_ids.end(), fallback_ids.begin(), fallback_ids.end()); + } + } + } + } + } + + return token_ids; +} + +std::string BPETokenizer::decode(const std::vector& tokens) const { + if (runtime_config_.decoder == TokenizerRuntimeConfig::Decoder::REPLACE_METASPACE) { + std::string result; + result.reserve(tokens.size() * 4); + + for (uint32_t token_id : tokens) { + if (token_id >= id_to_token_.size()) continue; + result += id_to_token_[token_id]; + } + + size_t pos = 0; + while ((pos = result.find(kMetaspace, pos)) != std::string::npos) { + result.replace(pos, 3, " "); + pos += 1; + } + return result; + } + + std::string unicode_result; + unicode_result.reserve(tokens.size() * 4); + + for (uint32_t token_id : tokens) { + if (token_id >= id_to_token_.size()) continue; + const std::string& tok = id_to_token_[token_id]; + + size_t pos = 0; + while (pos < tok.size()) { + if (pos + 3 <= tok.size() && + (unsigned char)tok[pos] == 0xE2 && + (unsigned char)tok[pos+1] == 0x96 && + (unsigned char)tok[pos+2] == 0x81) { + unicode_result.push_back(' '); + pos += 3; + } else { + unicode_result.push_back(tok[pos++]); + } + } + } + + return unicode_to_bytes(unicode_result); +} + + +} +} diff --git a/cactus-transpile/engine/engine_cache.cpp b/cactus-transpile/engine/engine_cache.cpp new file mode 100644 index 000000000..77a7381b4 --- /dev/null +++ b/cactus-transpile/engine/engine_cache.cpp @@ -0,0 +1,691 @@ +#include "engine.h" +#include "../graph/graph.h" +#include "../kernel/kernel.h" +#include +#include +#include + +namespace cactus { +namespace engine { + +void KVCache::init(size_t layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision) { + num_layers = layers; + max_seq_len = max_seq; + precision = model_precision; + element_size = PrecisionTraits::size_of(precision); + + layer_caches.resize(num_layers); + for (size_t i = 0; i < num_layers; i++) { + layer_caches[i].head_dim = layer_dims[i]; + layer_caches[i].kv_heads = layer_kv_heads[i]; + } + + current_seq_len = 0; + total_seq_len = 0; +} + +void KVCache::set_window_size(size_t window, size_t sink) { + window_size = window; + sink_size = sink; + + if (window_size > 0) { + for (size_t layer_idx = 0; layer_idx < num_layers; layer_idx++) { + size_t head_dim = get_layer_head_dim(layer_idx); + if (head_dim == 0) continue; + size_t layer_kv = get_layer_kv_heads(layer_idx); + size_t cache_bytes = window_size * layer_kv * head_dim * element_size; + size_t num_groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + size_t num_scales = window_size * layer_kv * num_groups; + auto& cache = layer_caches[layer_idx]; + + cache.keys.resize(cache_bytes); + cache.values.resize(cache_bytes); + std::memset(cache.keys.data(), 0, cache_bytes); + std::memset(cache.values.data(), 0, cache_bytes); + + cache.key_scales.resize(num_scales); + cache.value_scales.resize(num_scales); + std::fill(cache.key_scales.begin(), cache.key_scales.end(), 1.0f); + std::fill(cache.value_scales.begin(), cache.value_scales.end(), 1.0f); + } + } +} + +void KVCache::reset() { + current_seq_len = 0; + total_seq_len = 0; +} + +void* KVCache::get_key_ptr(size_t layer) { + if (current_seq_len == 0 || layer >= num_layers) return nullptr; + return layer_caches[layer].keys.data(); +} + +void* KVCache::get_value_ptr(size_t layer) { + if (current_seq_len == 0 || layer >= num_layers) return nullptr; + return layer_caches[layer].values.data(); +} + +KVCache::CircularView KVCache::get_key_view(size_t layer) { + CircularView view; + if (layer >= num_layers || current_seq_len == 0) { + view.ptr1 = nullptr; + view.ptr2 = nullptr; + view.len1 = 0; + view.len2 = 0; + view.total_len = 0; + return view; + } + + view.ptr1 = layer_caches[layer].keys.data(); + view.ptr2 = nullptr; + view.len1 = current_seq_len; + view.len2 = 0; + view.total_len = current_seq_len; + return view; +} + +KVCache::CircularView KVCache::get_value_view(size_t layer) { + CircularView view; + if (layer >= num_layers || current_seq_len == 0) { + view.ptr1 = nullptr; + view.ptr2 = nullptr; + view.len1 = 0; + view.len2 = 0; + view.total_len = 0; + return view; + } + + view.ptr1 = layer_caches[layer].values.data(); + view.ptr2 = nullptr; + view.len1 = current_seq_len; + view.len2 = 0; + view.total_len = current_seq_len; + return view; +} + +void KVCache::update_from_graph(CactusGraph* gb, const std::vector& k_nodes, + const std::vector& v_nodes, size_t seq_len, + size_t layers) { + size_t old_seq_len = current_seq_len; + size_t new_total_len = old_seq_len + seq_len; + + total_seq_len += seq_len; + + size_t effective_seq_len; + bool use_sliding_window = (window_size > 0 && new_total_len > window_size); + + if (use_sliding_window) { + effective_seq_len = window_size; + } else { + effective_seq_len = new_total_len; + } + + bool any_layer_updated = false; + + for (size_t layer_idx = 0; layer_idx < layers; layer_idx++) { + if (k_nodes[layer_idx] == 0 || v_nodes[layer_idx] == 0) + continue; + + auto& cache = layer_caches[layer_idx]; + + void* k_output = gb->get_output(k_nodes[layer_idx]); + void* v_output = gb->get_output(v_nodes[layer_idx]); + + if (k_output && v_output) { + const auto& k_buffer = gb->get_output_buffer(k_nodes[layer_idx]); + const auto& v_buffer = gb->get_output_buffer(v_nodes[layer_idx]); + + size_t dim = get_layer_head_dim(layer_idx); + size_t kv_heads = get_layer_kv_heads(layer_idx); + size_t elements_per_token = kv_heads * dim; + size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + size_t scales_per_token = kv_heads * num_groups; + size_t bytes_per_token = elements_per_token * element_size; + + size_t expected_elements = new_total_len * elements_per_token; + + if (k_buffer.total_size == expected_elements && v_buffer.total_size == expected_elements) { + any_layer_updated = true; + + if (!use_sliding_window) { + size_t total_bytes = new_total_len * bytes_per_token; + cache.keys.resize(total_bytes); + cache.values.resize(total_bytes); + + if (precision == Precision::INT8) { + size_t num_scales = new_total_len * scales_per_token; + cache.key_scales.resize(num_scales); + cache.value_scales.resize(num_scales); + + cactus_quantize_kv_fp16_to_int8( + static_cast(k_output), + reinterpret_cast(cache.keys.data()), + cache.key_scales.data(), + new_total_len, kv_heads, dim); + + cactus_quantize_kv_fp16_to_int8( + static_cast(v_output), + reinterpret_cast(cache.values.data()), + cache.value_scales.data(), + new_total_len, kv_heads, dim); + } else { + std::memcpy(cache.keys.data(), k_output, total_bytes); + std::memcpy(cache.values.data(), v_output, total_bytes); + } + } else { + size_t cache_bytes = window_size * bytes_per_token; + size_t remaining_window = window_size - sink_size; + size_t skip_tokens = new_total_len - window_size; + + bool first_slide = (cache.keys.size() != cache_bytes); + + if (first_slide) { + cache.keys.resize(cache_bytes); + cache.values.resize(cache_bytes); + } + + if (precision == Precision::INT8) { + size_t num_scales = window_size * scales_per_token; + if (first_slide) { + cache.key_scales.resize(num_scales); + cache.value_scales.resize(num_scales); + } + + const __fp16* k_fp16 = static_cast(k_output); + const __fp16* v_fp16 = static_cast(v_output); + + if (first_slide) { + cactus_quantize_kv_fp16_to_int8( + k_fp16, + reinterpret_cast(cache.keys.data()), + cache.key_scales.data(), + sink_size, kv_heads, dim); + + cactus_quantize_kv_fp16_to_int8( + v_fp16, + reinterpret_cast(cache.values.data()), + cache.value_scales.data(), + sink_size, kv_heads, dim); + } + + size_t src_offset = (sink_size + skip_tokens) * elements_per_token; + size_t dst_offset = sink_size * elements_per_token; + size_t scale_dst_offset = sink_size * scales_per_token; + + cactus_quantize_kv_fp16_to_int8( + k_fp16 + src_offset, + reinterpret_cast(cache.keys.data()) + dst_offset, + cache.key_scales.data() + scale_dst_offset, + remaining_window, kv_heads, dim); + + cactus_quantize_kv_fp16_to_int8( + v_fp16 + src_offset, + reinterpret_cast(cache.values.data()) + dst_offset, + cache.value_scales.data() + scale_dst_offset, + remaining_window, kv_heads, dim); + } else { + if (first_slide) { + size_t sink_bytes = sink_size * bytes_per_token; + std::memcpy(cache.keys.data(), k_output, sink_bytes); + std::memcpy(cache.values.data(), v_output, sink_bytes); + } + + const uint8_t* k_src = static_cast(k_output) + + (sink_size + skip_tokens) * bytes_per_token; + const uint8_t* v_src = static_cast(v_output) + + (sink_size + skip_tokens) * bytes_per_token; + size_t sink_bytes = sink_size * bytes_per_token; + size_t recent_bytes = remaining_window * bytes_per_token; + + std::memcpy(cache.keys.data() + sink_bytes, k_src, recent_bytes); + std::memcpy(cache.values.data() + sink_bytes, v_src, recent_bytes); + } + } + } + else if (seq_len * elements_per_token == k_buffer.total_size && + seq_len * elements_per_token == v_buffer.total_size) { + any_layer_updated = true; + + if (!use_sliding_window) { + size_t old_bytes = old_seq_len * bytes_per_token; + size_t new_bytes = seq_len * bytes_per_token; + size_t total_bytes = old_bytes + new_bytes; + cache.keys.resize(total_bytes); + cache.values.resize(total_bytes); + + if (precision == Precision::INT8) { + size_t num_scales = (old_seq_len + seq_len) * scales_per_token; + cache.key_scales.resize(num_scales); + cache.value_scales.resize(num_scales); + + size_t dst_offset = old_seq_len * elements_per_token; + size_t scale_offset = old_seq_len * scales_per_token; + + cactus_quantize_kv_fp16_to_int8( + static_cast(k_output), + reinterpret_cast(cache.keys.data()) + dst_offset, + cache.key_scales.data() + scale_offset, + seq_len, kv_heads, dim); + + cactus_quantize_kv_fp16_to_int8( + static_cast(v_output), + reinterpret_cast(cache.values.data()) + dst_offset, + cache.value_scales.data() + scale_offset, + seq_len, kv_heads, dim); + } else { + std::memcpy(cache.keys.data() + old_bytes, k_output, new_bytes); + std::memcpy(cache.values.data() + old_bytes, v_output, new_bytes); + } + } else { + size_t cache_bytes = window_size * bytes_per_token; + + if (cache.keys.size() != cache_bytes) { + cache.keys.resize(cache_bytes); + cache.values.resize(cache_bytes); + } + + size_t tokens_to_shift = window_size - sink_size - seq_len; + size_t sink_bytes = sink_size * bytes_per_token; + + if (tokens_to_shift > 0 && old_seq_len > sink_size) { + size_t shift_src = old_seq_len - tokens_to_shift; + if (shift_src > sink_size) { + size_t shift_bytes = tokens_to_shift * bytes_per_token; + + std::memmove(cache.keys.data() + sink_bytes, + cache.keys.data() + shift_src * bytes_per_token, + shift_bytes); + std::memmove(cache.values.data() + sink_bytes, + cache.values.data() + shift_src * bytes_per_token, + shift_bytes); + + if (precision == Precision::INT8) { + size_t sink_scale_offset = sink_size * scales_per_token; + size_t shift_src_scale_offset = shift_src * scales_per_token; + size_t shift_scale_count = tokens_to_shift * scales_per_token; + + std::memmove(cache.key_scales.data() + sink_scale_offset, + cache.key_scales.data() + shift_src_scale_offset, + shift_scale_count * sizeof(float)); + std::memmove(cache.value_scales.data() + sink_scale_offset, + cache.value_scales.data() + shift_src_scale_offset, + shift_scale_count * sizeof(float)); + } + } + } + + size_t append_token_offset = window_size - seq_len; + size_t append_bytes_offset = append_token_offset * bytes_per_token; + size_t new_bytes = seq_len * bytes_per_token; + + if (precision == Precision::INT8) { + size_t append_offset = append_token_offset * elements_per_token; + size_t scale_offset = append_token_offset * scales_per_token; + + cactus_quantize_kv_fp16_to_int8( + static_cast(k_output), + reinterpret_cast(cache.keys.data()) + append_offset, + cache.key_scales.data() + scale_offset, + seq_len, kv_heads, dim); + + cactus_quantize_kv_fp16_to_int8( + static_cast(v_output), + reinterpret_cast(cache.values.data()) + append_offset, + cache.value_scales.data() + scale_offset, + seq_len, kv_heads, dim); + } else { + std::memcpy(cache.keys.data() + append_bytes_offset, k_output, new_bytes); + std::memcpy(cache.values.data() + append_bytes_offset, v_output, new_bytes); + } + } + } + } + } + + if (any_layer_updated) { + current_seq_len = effective_seq_len; + } +} + +void KVCache::update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, + size_t num_tokens, size_t kv_heads, size_t dim) { + if (layer_idx >= num_layers || !k_data || !v_data || num_tokens == 0) { + return; + } + + auto& cache = layer_caches[layer_idx]; + size_t old_seq_len = current_seq_len; + size_t new_total_len = old_seq_len + num_tokens; + size_t elements_per_token = kv_heads * dim; + size_t bytes_per_token = elements_per_token * element_size; + size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + size_t scales_per_token = kv_heads * num_groups; + + if (layer_idx == 0) { + total_seq_len += num_tokens; + } + + bool use_sliding_window = (window_size > 0 && new_total_len > window_size); + size_t effective_seq_len = use_sliding_window ? window_size : new_total_len; + + if (!use_sliding_window) { + size_t total_bytes = new_total_len * bytes_per_token; + cache.keys.resize(total_bytes); + cache.values.resize(total_bytes); + + size_t num_scales = new_total_len * scales_per_token; + cache.key_scales.resize(num_scales); + cache.value_scales.resize(num_scales); + + size_t dst_offset = old_seq_len * elements_per_token; + size_t scale_offset = old_seq_len * scales_per_token; + + cactus_quantize_kv_fp16_to_int8( + k_data, + reinterpret_cast(cache.keys.data()) + dst_offset, + cache.key_scales.data() + scale_offset, + num_tokens, kv_heads, dim); + + cactus_quantize_kv_fp16_to_int8( + v_data, + reinterpret_cast(cache.values.data()) + dst_offset, + cache.value_scales.data() + scale_offset, + num_tokens, kv_heads, dim); + } else { + size_t cache_bytes = window_size * bytes_per_token; + + if (cache.keys.size() != cache_bytes) { + cache.keys.resize(cache_bytes); + cache.values.resize(cache_bytes); + + size_t num_scales = window_size * scales_per_token; + cache.key_scales.resize(num_scales); + cache.value_scales.resize(num_scales); + } + + size_t remaining_window = window_size - sink_size; + + if (num_tokens >= remaining_window) { + size_t skip_tokens = num_tokens - remaining_window; + size_t dst_offset = sink_size * elements_per_token; + size_t scale_offset = sink_size * scales_per_token; + + cactus_quantize_kv_fp16_to_int8( + k_data + skip_tokens * elements_per_token, + reinterpret_cast(cache.keys.data()) + dst_offset, + cache.key_scales.data() + scale_offset, + remaining_window, kv_heads, dim); + + cactus_quantize_kv_fp16_to_int8( + v_data + skip_tokens * elements_per_token, + reinterpret_cast(cache.values.data()) + dst_offset, + cache.value_scales.data() + scale_offset, + remaining_window, kv_heads, dim); + } else { + size_t tokens_to_shift = remaining_window - num_tokens; + + if (tokens_to_shift > 0 && old_seq_len > sink_size) { + size_t shift_src = old_seq_len - tokens_to_shift; + if (shift_src > sink_size) { + size_t sink_offset = sink_size * elements_per_token; + size_t shift_src_offset = shift_src * elements_per_token; + size_t shift_bytes = tokens_to_shift * bytes_per_token; + + std::memmove(cache.keys.data() + sink_offset, + cache.keys.data() + shift_src_offset, + shift_bytes); + std::memmove(cache.values.data() + sink_offset, + cache.values.data() + shift_src_offset, + shift_bytes); + + size_t sink_scale_offset = sink_size * scales_per_token; + size_t shift_src_scale_offset = shift_src * scales_per_token; + size_t shift_scale_count = tokens_to_shift * scales_per_token; + + std::memmove(cache.key_scales.data() + sink_scale_offset, + cache.key_scales.data() + shift_src_scale_offset, + shift_scale_count * sizeof(float)); + std::memmove(cache.value_scales.data() + sink_scale_offset, + cache.value_scales.data() + shift_src_scale_offset, + shift_scale_count * sizeof(float)); + } + } + + size_t append_token_offset = window_size - num_tokens; + size_t append_offset = append_token_offset * elements_per_token; + size_t scale_offset = append_token_offset * scales_per_token; + + cactus_quantize_kv_fp16_to_int8( + k_data, + reinterpret_cast(cache.keys.data()) + append_offset, + cache.key_scales.data() + scale_offset, + num_tokens, kv_heads, dim); + + cactus_quantize_kv_fp16_to_int8( + v_data, + reinterpret_cast(cache.values.data()) + append_offset, + cache.value_scales.data() + scale_offset, + num_tokens, kv_heads, dim); + } + } + + if (layer_idx == num_layers - 1) { + current_seq_len = effective_seq_len; + } +} + +void KVCache::remove_token_range(size_t start, size_t count) { + if (count == 0 || start >= current_seq_len || start + count > current_seq_len) return; + + size_t tail_tokens = current_seq_len - start - count; + + auto erase_bytes = [](std::vector& buf, size_t offset, size_t remove, size_t tail) { + std::memmove(buf.data() + offset, buf.data() + offset + remove, tail); + buf.resize(buf.size() - remove); + }; + + auto erase_floats = [](std::vector& buf, size_t offset, size_t remove, size_t tail) { + std::memmove(buf.data() + offset, buf.data() + offset + remove, tail * sizeof(float)); + buf.resize(buf.size() - remove); + }; + + for (size_t i = 0; i < layer_caches.size(); i++) { + size_t dim = get_layer_head_dim(i); + if (dim == 0) continue; + auto& layer = layer_caches[i]; + size_t num_kv_heads = get_layer_kv_heads(i); + size_t bytes_per_tok = num_kv_heads * dim * element_size; + + erase_bytes(layer.keys, start * bytes_per_tok, count * bytes_per_tok, tail_tokens * bytes_per_tok); + erase_bytes(layer.values, start * bytes_per_tok, count * bytes_per_tok, tail_tokens * bytes_per_tok); + + if (precision == Precision::INT8 && !layer.key_scales.empty()) { + size_t scaler_per_tok = num_kv_heads * dim / KV_QUANT_GROUP_SIZE; + erase_floats(layer.key_scales, start * scaler_per_tok, count * scaler_per_tok, tail_tokens * scaler_per_tok); + erase_floats(layer.value_scales, start * scaler_per_tok, count * scaler_per_tok, tail_tokens * scaler_per_tok); + } + } + + current_seq_len -= count; + total_seq_len -= count; +} + +void KVCache::compact_to_windows(const std::vector& target_windows) { + for (size_t i = 0; i < layer_caches.size(); i++) { + size_t dim = get_layer_head_dim(i); + if (dim == 0) continue; + size_t target = (i < target_windows.size()) ? target_windows[i] : 0; + if (target == 0) continue; + + auto& layer = layer_caches[i]; + size_t num_kv_heads = get_layer_kv_heads(i); + size_t layer_tokens = layer.keys.size() / (num_kv_heads * dim * element_size); + if (layer_tokens <= target) continue; + + size_t bytes_per_token = num_kv_heads * dim * element_size; + size_t keep_bytes = target * bytes_per_token; + size_t discard_bytes = (layer_tokens - target) * bytes_per_token; + + std::memmove(layer.keys.data(), layer.keys.data() + discard_bytes, keep_bytes); + layer.keys.resize(keep_bytes); + std::memmove(layer.values.data(), layer.values.data() + discard_bytes, keep_bytes); + layer.values.resize(keep_bytes); + + if (precision == Precision::INT8 && !layer.key_scales.empty()) { + size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + size_t scales_per_token = num_kv_heads * num_groups; + size_t keep_scales = target * scales_per_token; + size_t discard_scales = (layer_tokens - target) * scales_per_token; + + std::memmove(layer.key_scales.data(), layer.key_scales.data() + discard_scales, keep_scales * sizeof(float)); + layer.key_scales.resize(keep_scales); + std::memmove(layer.value_scales.data(), layer.value_scales.data() + discard_scales, keep_scales * sizeof(float)); + layer.value_scales.resize(keep_scales); + } + } +} + +const int8_t* KVCache::get_keys_int8(size_t layer) const { + if (layer >= num_layers || current_seq_len == 0) { + return nullptr; + } + return reinterpret_cast(layer_caches[layer].keys.data()); +} + +const int8_t* KVCache::get_values_int8(size_t layer) const { + if (layer >= num_layers || current_seq_len == 0) { + return nullptr; + } + return reinterpret_cast(layer_caches[layer].values.data()); +} + +const float* KVCache::get_key_scales(size_t layer) const { + if (layer >= num_layers || current_seq_len == 0) { + return nullptr; + } + return layer_caches[layer].key_scales.data(); +} + +const float* KVCache::get_value_scales(size_t layer) const { + if (layer >= num_layers || current_seq_len == 0) { + return nullptr; + } + return layer_caches[layer].value_scales.data(); +} + +void ConvCache::init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision) { + num_layers = layers; + hidden_size = hidden_dim; + window_size = window_len; + precision = model_precision; + element_size = PrecisionTraits::size_of(precision); + + size_t state_bytes = window_size * hidden_size * element_size; + layer_states.resize(num_layers); + for (auto& state : layer_states) { + state.data.resize(state_bytes); + std::memset(state.data.data(), 0, state_bytes); + state.head = 0; + state.count = 0; + } +} + +ConvCache::CircularView ConvCache::get_window(size_t layer) const { + CircularView view; + if (layer >= num_layers) { + view.ptr1 = nullptr; + view.len1 = 0; + view.ptr2 = nullptr; + view.len2 = 0; + view.total_len = 0; + return view; + } + + const auto& state = layer_states[layer]; + if (state.count == 0) { + view.ptr1 = nullptr; + view.len1 = 0; + view.ptr2 = nullptr; + view.len2 = 0; + view.total_len = 0; + return view; + } + + size_t stride = hidden_size * element_size; + + if (state.count < window_size) { + view.ptr1 = state.data.data(); + view.len1 = state.count; + view.ptr2 = nullptr; + view.len2 = 0; + view.total_len = state.count; + return view; + } + + view.ptr1 = state.data.data(); + view.len1 = state.head; + view.ptr2 = state.data.data() + state.head * stride; + view.len2 = window_size - state.head; + view.total_len = window_size; + return view; +} + +void ConvCache::update(CactusGraph* gb, size_t layer, const size_t bx_node) { + if (layer >= num_layers || !bx_node || window_size == 0 || hidden_size == 0) { + return; + } + + auto& state = layer_states[layer]; + const void* output_ptr = gb->get_output(bx_node); + if (!output_ptr) { + return; + } + + const auto& buffer = gb->get_output_buffer(bx_node); + const size_t stride_bytes = hidden_size * element_size; + + size_t rows = 1; + if (!buffer.shape.empty()) { + if (buffer.shape.size() == 1) { + rows = 1; + } else { + rows = buffer.shape[0]; + } + } + + if (buffer.total_size > 0 && hidden_size > 0) { + size_t inferred = buffer.total_size / hidden_size; + if (inferred > 0) { + rows = inferred; + } + } + + if (rows == 0) { + return; + } + + size_t copy_rows = std::min(rows, window_size); + size_t start_row = rows > window_size ? rows - window_size : 0; + + const uint8_t* src = static_cast(output_ptr) + start_row * stride_bytes; + + for (size_t i = 0; i < copy_rows; ++i) { + std::memcpy(state.data.data() + state.head * stride_bytes, src + i * stride_bytes, stride_bytes); + state.head = (state.head + 1) % window_size; + if (state.count < window_size) { + ++state.count; + } + } +} + +void ConvCache::reset() { + for (auto& state : layer_states) { + std::fill(state.data.begin(), state.data.end(), 0); + state.head = 0; + state.count = 0; + } +} + +} +} \ No newline at end of file diff --git a/cactus-transpile/engine/engine_constraints.cpp b/cactus-transpile/engine/engine_constraints.cpp new file mode 100644 index 000000000..2e18c7c14 --- /dev/null +++ b/cactus-transpile/engine/engine_constraints.cpp @@ -0,0 +1,1137 @@ +#include "engine.h" + +namespace cactus { +namespace engine { + +constexpr float FORCE_BIAS = 500.0f; +constexpr float BLOCK_BIAS = -500.0f; + +void ToolCallConstrainer::add_tokens_for_string(const std::string& str, std::unordered_set& token_set) { + if (!tokenizer_) return; + auto tokens = tokenizer_->encode(str); + for (uint32_t t : tokens) { + token_set.insert(t); + } +} + +void ToolCallConstrainer::tokenize_function_names(bool quote_names) { + all_func_name_tokens_.clear(); + func_name_sequences_.clear(); + + for (const auto& name : function_names_) { + std::string name_to_encode = quote_names ? ("\"" + name + "\"") : name; + auto tokens = tokenizer_->encode(name_to_encode); + func_name_sequences_[name] = tokens; + for (uint32_t t : tokens) { + all_func_name_tokens_.insert(t); + } + if (quote_names) { + auto unquoted_tokens = tokenizer_->encode(name); + for (uint32_t t : unquoted_tokens) { + all_func_name_tokens_.insert(t); + } + } + } +} + +void ToolCallConstrainer::init_common_tokens() { + backtick_tokens_.clear(); + add_tokens_for_string("`", backtick_tokens_); + add_tokens_for_string("``", backtick_tokens_); + add_tokens_for_string("```", backtick_tokens_); + add_tokens_for_string("````", backtick_tokens_); + add_tokens_for_string("```json", backtick_tokens_); + add_tokens_for_string("```JSON", backtick_tokens_); + add_tokens_for_string("``` json", backtick_tokens_); + add_tokens_for_string("```\n", backtick_tokens_); + add_tokens_for_string("` ", backtick_tokens_); +} + +void ToolCallConstrainer::needle_insert_word(NeedleTrieNode* root, const std::string& word) { + if (!root) return; + + NeedleTrieNode* node = root; + for (char ch : word) { + auto& child = node->children[ch]; + if (!child) { + child = std::make_unique(); + } + node = child.get(); + } + node->is_terminal = true; +} + +const ToolCallConstrainer::NeedleTrieNode* ToolCallConstrainer::needle_get_trie_node( + const NeedleTrieNode* root, + const std::string& prefix +) const { + const NeedleTrieNode* node = root; + for (char ch : prefix) { + if (!node) return nullptr; + auto it = node->children.find(ch); + if (it == node->children.end()) { + return nullptr; + } + node = it->second.get(); + } + return node; +} + +bool ToolCallConstrainer::needle_has_unseen_completion(const NeedleTrieNode* node, std::string& partial) const { + if (!node) return false; + if (node->is_terminal && !needle_seen_arg_keys_.count(partial)) return true; + for (const auto& [ch, child] : node->children) { + partial.push_back(ch); + if (needle_has_unseen_completion(child.get(), partial)) { + partial.pop_back(); + return true; + } + partial.pop_back(); + } + return false; +} + +bool ToolCallConstrainer::needle_check_token_valid(const std::string& token_text, + const NeedleTrieNode* trie_node) const { + const NeedleTrieNode* node = trie_node; + if (!node) return false; + + for (char ch : token_text) { + if (ch == '"') { + return node->is_terminal; + } + auto it = node->children.find(ch); + if (it == node->children.end()) { + return false; + } + node = it->second.get(); + } + return true; +} + +void ToolCallConstrainer::reset_needle_constraints() { + needle_json_state_ = NeedleJsonState::FREE; + needle_buffer_.clear(); + needle_constrained_buf_.clear(); + needle_current_function_.clear(); + needle_seen_arg_keys_.clear(); + needle_in_arguments_ = false; + needle_arguments_depth_ = 0; + needle_nesting_depth_ = 0; + needle_in_string_value_ = false; + needle_between_pairs_ = false; + needle_prev_char_escape_ = false; +} + +bool ToolCallConstrainer::needle_at_arg_key_start() const { + if (needle_buffer_.size() < 2) return false; + return needle_buffer_.compare(needle_buffer_.size() - 2, 2, "{\"") == 0 || + needle_buffer_.compare(needle_buffer_.size() - 2, 2, ",\"") == 0; +} + +bool ToolCallConstrainer::needle_is_value_string_start() const { + if (needle_buffer_.empty()) return false; + for (size_t i = needle_buffer_.size() - 1; i-- > 0;) { + char ch = needle_buffer_[i]; + if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r') { + continue; + } + return ch == ':'; + } + return false; +} + +void ToolCallConstrainer::feed_needle_char(char ch) { + if (needle_json_state_ == NeedleJsonState::IN_NAME || + needle_json_state_ == NeedleJsonState::IN_ARG_KEY) { + if (ch == '"') { + if (needle_json_state_ == NeedleJsonState::IN_NAME) { + needle_current_function_ = needle_constrained_buf_; + needle_seen_arg_keys_.clear(); + } else { + needle_seen_arg_keys_.insert(needle_constrained_buf_); + } + needle_constrained_buf_.clear(); + needle_json_state_ = NeedleJsonState::FREE; + } else { + needle_constrained_buf_.push_back(ch); + } + needle_buffer_.push_back(ch); + return; + } + + needle_buffer_.push_back(ch); + + if (needle_in_string_value_) { + if (needle_prev_char_escape_) { + needle_prev_char_escape_ = false; + return; + } + if (ch == '\\') { + needle_prev_char_escape_ = true; + return; + } + if (ch == '"') { + needle_in_string_value_ = false; + if (needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { + needle_between_pairs_ = true; + } + } + return; + } + + if (ch == '{' || ch == '[') { + needle_nesting_depth_++; + } else if (ch == '}' || ch == ']') { + needle_nesting_depth_ = std::max(0, needle_nesting_depth_ - 1); + if (ch == '}' && needle_in_arguments_ && needle_nesting_depth_ < needle_arguments_depth_) { + needle_in_arguments_ = false; + } + } else if (ch == ',' && needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { + needle_between_pairs_ = true; + } + + if (needle_buffer_.size() >= 8 && + needle_buffer_.compare(needle_buffer_.size() - 8, 8, "\"name\":\"") == 0) { + needle_json_state_ = NeedleJsonState::IN_NAME; + needle_constrained_buf_.clear(); + return; + } + + if (needle_buffer_.size() >= 13 && + needle_buffer_.compare(needle_buffer_.size() - 13, 13, "\"arguments\":{") == 0) { + needle_in_arguments_ = true; + needle_arguments_depth_ = needle_nesting_depth_; + return; + } + + if (needle_in_arguments_ && needle_at_arg_key_start()) { + needle_json_state_ = NeedleJsonState::IN_ARG_KEY; + needle_constrained_buf_.clear(); + needle_between_pairs_ = false; + return; + } + + if (ch == '"' && needle_is_value_string_start()) { + needle_in_string_value_ = true; + } +} + +void ToolCallConstrainer::feed_needle_text(const std::string& text) { + for (char ch : text) { + feed_needle_char(ch); + } +} + +void ToolCallConstrainer::init_needle_constraints() { + reset_needle_constraints(); + + needle_name_trie_ = std::make_unique(); + needle_param_tries_.clear(); + needle_required_params_.clear(); + for (const auto& tool : tool_specs_) { + if (!tool.name.empty()) { + needle_insert_word(needle_name_trie_.get(), tool.name); + } + + auto param_root = std::make_unique(); + for (const auto& param_name : tool.parameter_names) { + if (!param_name.empty()) { + needle_insert_word(param_root.get(), param_name); + } + } + needle_param_tries_[tool.name] = std::move(param_root); + needle_required_params_[tool.name] = tool.required_parameter_names; + } + + const uint32_t vocab_size = tokenizer_ ? tokenizer_->get_vocab_size() : 0; + if (needle_token_strings_.size() != vocab_size) { + needle_token_strings_.assign(vocab_size, ""); + needle_token_index_.clear(); + needle_arg_close_tokens_.clear(); + + auto would_close_args = [](const std::string& tok, bool start_in_str) -> bool { + bool in_str = start_in_str; + bool esc = false; + int depth = 0; + for (char ch : tok) { + if (in_str) { + if (esc) { esc = false; } + else if (ch == '\\') { esc = true; } + else if (ch == '"') { in_str = false; } + continue; + } + if (ch == '"') { in_str = true; continue; } + if (ch == '{' || ch == '[') { depth++; continue; } + if (ch == '}') { if (depth == 0) return true; depth--; } + else if (ch == ']' && depth > 0) depth--; + } + return false; + }; + + for (uint32_t token_id = 0; token_id < vocab_size; ++token_id) { + std::string decoded = tokenizer_->decode({token_id}); + needle_token_strings_[token_id] = decoded; + if (decoded.empty()) continue; + needle_token_index_[decoded.front()].push_back(token_id); + if (would_close_args(decoded, false) || would_close_args(decoded, true)) { + needle_arg_close_tokens_.insert(token_id); + } + } + } +} + +void ToolCallConstrainer::tokenize_grammar_elements() { + if (!tokenizer_) return; + + // Clear all token sets + open_brace_tokens_.clear(); + close_brace_tokens_.clear(); + colon_tokens_.clear(); + comma_tokens_.clear(); + name_key_tokens_.clear(); + args_key_tokens_.clear(); + quote_tokens_.clear(); + tool_start_tokens_.clear(); + tool_end_tokens_.clear(); + bracket_open_tokens_.clear(); + bracket_close_tokens_.clear(); + paren_open_tokens_.clear(); + paren_close_tokens_.clear(); + equals_tokens_.clear(); + + init_common_tokens(); + + if (model_type_ == Config::ModelType::LFM2) { + add_tokens_for_string("<|tool_call_start|>", tool_start_tokens_); + add_tokens_for_string("<|tool_call_end|>", tool_end_tokens_); + add_tokens_for_string("[", bracket_open_tokens_); + add_tokens_for_string("]", bracket_close_tokens_); + add_tokens_for_string("(", paren_open_tokens_); + add_tokens_for_string(")", paren_close_tokens_); + add_tokens_for_string("=", equals_tokens_); + add_tokens_for_string(",", comma_tokens_); + add_tokens_for_string("\"", quote_tokens_); + + tokenize_function_names(false); + } else if (is_gemma_family()) { + gemma_call_start_tokens_.clear(); + gemma_call_end_tokens_.clear(); + gemma_call_prefix_tokens_.clear(); + escape_tokens_.clear(); + gemma_response_start_tokens_.clear(); + + add_tokens_for_string(call_start_tag_, gemma_call_start_tokens_); + add_tokens_for_string(call_end_tag_, gemma_call_end_tokens_); + if (model_type_ == Config::ModelType::GEMMA4) { + add_tokens_for_string("<|tool_response>", gemma_response_start_tokens_); + } else { + add_tokens_for_string("", gemma_response_start_tokens_); + add_tokens_for_string("", escape_tokens_); + } + add_tokens_for_string("call:", gemma_call_prefix_tokens_); + + add_tokens_for_string("{", open_brace_tokens_); + add_tokens_for_string("}", close_brace_tokens_); + add_tokens_for_string(":", colon_tokens_); + add_tokens_for_string(",", comma_tokens_); + + tokenize_function_names(false); + } else if (is_needle()) { + qwen_tool_call_start_tokens_.clear(); + add_tokens_for_string("", qwen_tool_call_start_tokens_); + } else { + qwen_tool_call_start_tokens_.clear(); + qwen_tool_call_end_tokens_.clear(); + + add_tokens_for_string("", qwen_tool_call_start_tokens_); + add_tokens_for_string("", qwen_tool_call_end_tokens_); + + add_tokens_for_string("{", open_brace_tokens_); + add_tokens_for_string("}", close_brace_tokens_); + add_tokens_for_string(":", colon_tokens_); + add_tokens_for_string(",", comma_tokens_); + add_tokens_for_string("\"", quote_tokens_); + + add_tokens_for_string("name", name_key_tokens_); + + add_tokens_for_string("arguments", args_key_tokens_); + + tokenize_function_names(true); + } +} + +void ToolCallConstrainer::init(Config::ModelType model_type, + const std::vector& tools, + Tokenizer* tokenizer) { + model_type_ = model_type; + tool_specs_ = tools; + function_names_.clear(); + function_names_.reserve(tool_specs_.size()); + for (const auto& tool : tool_specs_) { + function_names_.push_back(tool.name); + } + tokenizer_ = tokenizer; + generated_text_.clear(); + brace_depth_ = 0; + active_ = !function_names_.empty() && tokenizer != nullptr; + + if (model_type_ == Config::ModelType::LFM2) { + state_ = State::LFM_START; + } else if (is_gemma_family()) { + state_ = State::GEMMA_START; + if (model_type_ == Config::ModelType::GEMMA4) { + call_start_tag_ = "<|tool_call>"; + call_end_tag_ = ""; + } else { + call_start_tag_ = ""; + call_end_tag_ = ""; + } + } else if (is_needle()) { + state_ = State::NEEDLE_START; + } else { + state_ = State::QWEN_START; + } + + if (!active_) { + return; + } + + tokenize_grammar_elements(); + if (is_needle()) { + init_needle_constraints(); + } + compute_bias(); +} + +void ToolCallConstrainer::update(uint32_t /*token_id*/, const std::string& decoded_text) { + if (!active_) return; + + generated_text_ += decoded_text; + + if (model_type_ == Config::ModelType::LFM2) { + switch (state_) { + case State::LFM_START: + if (generated_text_.find("<|tool_call_start|>") != std::string::npos) { + state_ = State::LFM_EXPECT_BRACKET; + generated_text_.clear(); + } + break; + + case State::LFM_EXPECT_BRACKET: + if (generated_text_.find("[") != std::string::npos) { + state_ = State::LFM_IN_FUNC_NAME; + generated_text_.clear(); + } + break; + + case State::LFM_IN_FUNC_NAME: + for (const auto& name : function_names_) { + if (generated_text_.find(name) != std::string::npos) { + state_ = State::LFM_EXPECT_PAREN; + generated_text_.clear(); + break; + } + } + break; + + case State::LFM_EXPECT_PAREN: + if (generated_text_.find("(") != std::string::npos) { + state_ = State::LFM_IN_ARGUMENTS; + generated_text_.clear(); + } + break; + + case State::LFM_IN_ARGUMENTS: + if (decoded_text.find(")") != std::string::npos) { + state_ = State::LFM_EXPECT_BRACKET_CLOSE; + generated_text_.clear(); + } else { + generated_text_.clear(); + } + break; + + case State::LFM_EXPECT_BRACKET_CLOSE: + if (generated_text_.find("]") != std::string::npos) { + state_ = State::LFM_EXPECT_END; + generated_text_.clear(); + } + break; + + case State::LFM_EXPECT_END: + if (generated_text_.find("<|tool_call_end|>") != std::string::npos) { + state_ = State::DONE; + generated_text_.clear(); + } + break; + + default: + break; + } + } else if (is_gemma_family()) { + switch (state_) { + case State::GEMMA_START: + if (generated_text_.find(call_start_tag_) != std::string::npos) { + state_ = State::GEMMA_EXPECT_CALL; + generated_text_.clear(); + } + break; + + case State::GEMMA_EXPECT_CALL: + if (generated_text_.find("call:") != std::string::npos) { + state_ = State::GEMMA_IN_FUNC_NAME; + generated_text_.clear(); + } + break; + + case State::GEMMA_IN_FUNC_NAME: + for (const auto& name : function_names_) { + if (generated_text_.find(name) != std::string::npos) { + state_ = State::GEMMA_EXPECT_BRACE; + generated_text_.clear(); + break; + } + } + break; + + case State::GEMMA_EXPECT_BRACE: + if (generated_text_.find("{") != std::string::npos) { + state_ = State::GEMMA_IN_ARGUMENTS; + brace_depth_ = 1; + generated_text_.clear(); + } + break; + + case State::GEMMA_IN_ARGUMENTS: + generated_text_.clear(); + for (char c : decoded_text) { + if (c == '{') brace_depth_++; + else if (c == '}') { + brace_depth_--; + if (brace_depth_ == 0) { + state_ = State::GEMMA_EXPECT_END; + break; + } + } + } + break; + + case State::GEMMA_EXPECT_END: + if (generated_text_.find(call_end_tag_) != std::string::npos) { + state_ = State::DONE; + generated_text_.clear(); + } + break; + + default: + break; + } + } else if (is_needle()) { + switch (state_) { + case State::NEEDLE_START: + if (generated_text_.find("") != std::string::npos) { + state_ = State::DONE; + generated_text_.clear(); + } + break; + + case State::DONE: + feed_needle_text(decoded_text); + generated_text_.clear(); + break; + + default: + break; + } + } else { + switch (state_) { + case State::QWEN_START: + if (generated_text_.find("") != std::string::npos) { + state_ = State::QWEN_EXPECT_OPEN_BRACE; + generated_text_.clear(); + } + break; + + case State::QWEN_EXPECT_OPEN_BRACE: { + size_t pos = generated_text_.find("{"); + if (pos != std::string::npos) { + state_ = State::QWEN_EXPECT_NAME_KEY; + generated_text_ = generated_text_.substr(pos + 1); + } + break; + } + + case State::QWEN_EXPECT_NAME_KEY: + if (generated_text_.find("name\"") != std::string::npos) { + state_ = State::QWEN_EXPECT_NAME_COLON; + generated_text_.clear(); + } + break; + + case State::QWEN_EXPECT_NAME_COLON: + if (generated_text_.find(":") != std::string::npos) { + state_ = State::QWEN_EXPECT_NAME_VALUE; + generated_text_.clear(); + } + break; + + case State::QWEN_EXPECT_NAME_VALUE: + for (const auto& name : function_names_) { + if (generated_text_.find(name + "\"") != std::string::npos) { + state_ = State::QWEN_EXPECT_COMMA; + generated_text_.clear(); + break; + } + } + break; + + case State::QWEN_EXPECT_COMMA: + if (generated_text_.find(",") != std::string::npos) { + state_ = State::QWEN_EXPECT_ARGS_KEY; + generated_text_.clear(); + } else if (generated_text_.find("}") != std::string::npos) { + state_ = State::QWEN_EXPECT_END; + generated_text_.clear(); + } + break; + + case State::QWEN_EXPECT_ARGS_KEY: + if (generated_text_.find("arguments\"") != std::string::npos) { + state_ = State::QWEN_EXPECT_ARGS_COLON; + generated_text_.clear(); + } + break; + + case State::QWEN_EXPECT_ARGS_COLON: + if (generated_text_.find(":") != std::string::npos) { + state_ = State::QWEN_IN_ARGUMENTS; + brace_depth_ = 0; + generated_text_.clear(); + } + break; + + case State::QWEN_IN_ARGUMENTS: + generated_text_.clear(); + for (char c : decoded_text) { + if (c == '{') brace_depth_++; + else if (c == '}') { + if (brace_depth_ > 0) { + brace_depth_--; + } else { + state_ = State::QWEN_EXPECT_CLOSE_BRACE; + generated_text_.clear(); + break; + } + } + } + break; + + case State::QWEN_EXPECT_CLOSE_BRACE: + if (generated_text_.find("}") != std::string::npos) { + state_ = State::QWEN_EXPECT_END; + generated_text_.clear(); + } + break; + + case State::QWEN_EXPECT_END: + if (generated_text_.find("") != std::string::npos) { + state_ = State::DONE; + generated_text_.clear(); + } + break; + + case State::DONE: + if (generated_text_.find("") != std::string::npos) { + state_ = State::QWEN_EXPECT_OPEN_BRACE; + generated_text_.clear(); + } + break; + + default: + break; + } + } + + compute_bias(); +} + +void ToolCallConstrainer::compute_bias() { + current_bias_.clear(); + + if (!active_) return; + + for (uint32_t t : backtick_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + + if (model_type_ == Config::ModelType::LFM2) { + switch (state_) { + case State::LFM_START: + for (uint32_t t : tool_start_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : bracket_open_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : paren_open_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::LFM_EXPECT_BRACKET: + for (uint32_t t : bracket_open_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : paren_open_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : bracket_close_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::LFM_IN_FUNC_NAME: + for (uint32_t t : all_func_name_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : bracket_close_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : paren_close_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : equals_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::LFM_EXPECT_PAREN: + for (uint32_t t : paren_open_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : bracket_close_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : equals_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::LFM_IN_ARGUMENTS: + for (uint32_t t : paren_close_tokens_) { + current_bias_[t] = 15.0f; + } + for (uint32_t t : equals_tokens_) { + current_bias_[t] = 10.0f; + } + for (uint32_t t : comma_tokens_) { + current_bias_[t] = 8.0f; + } + for (uint32_t t : quote_tokens_) { + current_bias_[t] = 5.0f; + } + for (uint32_t t : bracket_close_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : tool_end_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::LFM_EXPECT_BRACKET_CLOSE: + for (uint32_t t : bracket_close_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : paren_open_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : equals_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::LFM_EXPECT_END: + for (uint32_t t : tool_end_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : bracket_open_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : paren_open_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + default: + break; + } + } else if (is_gemma_family()) { + for (uint32_t t : gemma_response_start_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + + switch (state_) { + case State::GEMMA_START: + for (uint32_t t : gemma_call_start_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : close_brace_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::GEMMA_EXPECT_CALL: + for (uint32_t t : gemma_call_prefix_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : gemma_call_end_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::GEMMA_IN_FUNC_NAME: + for (uint32_t t : all_func_name_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : close_brace_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : gemma_call_end_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::GEMMA_EXPECT_BRACE: + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : gemma_call_end_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::GEMMA_IN_ARGUMENTS: + for (uint32_t t : colon_tokens_) { + current_bias_[t] = 10.0f; + } + for (uint32_t t : comma_tokens_) { + current_bias_[t] = 8.0f; + } + for (uint32_t t : escape_tokens_) { + current_bias_[t] = 5.0f; + } + for (uint32_t t : close_brace_tokens_) { + current_bias_[t] = 3.0f; + } + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = 3.0f; + } + for (uint32_t t : gemma_call_end_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::GEMMA_EXPECT_END: + for (uint32_t t : gemma_call_end_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : gemma_call_start_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + default: + break; + } + } else if (is_needle()) { + switch (state_) { + case State::NEEDLE_START: + for (uint32_t t : qwen_tool_call_start_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + if (tokenizer_) { + current_bias_[tokenizer_->get_eos_token()] = FORCE_BIAS; + } + break; + + default: + if (needle_json_state_ == NeedleJsonState::FREE) { + if (needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { + auto param_it = needle_param_tries_.find(needle_current_function_); + if (param_it != needle_param_tries_.end()) { + std::string partial; + bool all_seen = !needle_has_unseen_completion(param_it->second.get(), partial); + if (all_seen && needle_between_pairs_) { + for (uint32_t t : needle_arg_close_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + } else if (!all_seen && (needle_between_pairs_ || needle_in_string_value_)) { + auto req_it = needle_required_params_.find(needle_current_function_); + if (req_it != needle_required_params_.end()) { + for (const auto& req : req_it->second) { + if (!needle_seen_arg_keys_.count(req)) { + for (uint32_t t : needle_arg_close_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + } + } + } + } + } + } + break; + } + + const NeedleTrieNode* trie_node = nullptr; + if (needle_json_state_ == NeedleJsonState::IN_NAME) { + trie_node = needle_get_trie_node(needle_name_trie_.get(), needle_constrained_buf_); + } else if (needle_json_state_ == NeedleJsonState::IN_ARG_KEY) { + auto it = needle_param_tries_.find(needle_current_function_); + if (it != needle_param_tries_.end()) { + trie_node = needle_get_trie_node(it->second.get(), needle_constrained_buf_); + } + } + + if (!trie_node) { + break; + } + + std::vector valid_tokens(needle_token_strings_.size(), false); + bool has_valid = false; + auto mark_valid_tokens = [&](char first_char) { + auto idx_it = needle_token_index_.find(first_char); + if (idx_it == needle_token_index_.end()) { + return; + } + for (uint32_t token_id : idx_it->second) { + if (!valid_tokens[token_id] && + needle_check_token_valid(needle_token_strings_[token_id], trie_node)) { + valid_tokens[token_id] = true; + has_valid = true; + } + } + }; + + for (const auto& [first_char, _] : trie_node->children) { + mark_valid_tokens(first_char); + } + if (trie_node->is_terminal) { + mark_valid_tokens('"'); + } + + if (!has_valid) { + break; + } + + if (needle_json_state_ == NeedleJsonState::IN_ARG_KEY && !needle_seen_arg_keys_.empty()) { + for (size_t token_id = 0; token_id < valid_tokens.size(); ++token_id) { + if (!valid_tokens[token_id]) continue; + const std::string& tok = needle_token_strings_[token_id]; + size_t quote_pos = tok.find('"'); + if (quote_pos != std::string::npos) { + if (needle_seen_arg_keys_.count(needle_constrained_buf_ + tok.substr(0, quote_pos))) { + valid_tokens[token_id] = false; + } + } else { + std::string partial = needle_constrained_buf_ + tok; + const NeedleTrieNode* new_node = needle_get_trie_node(trie_node, tok); + if (!needle_has_unseen_completion(new_node, partial)) { + valid_tokens[token_id] = false; + } + } + } + } + + for (size_t token_id = 0; token_id < valid_tokens.size(); ++token_id) { + if (!valid_tokens[token_id]) { + current_bias_[static_cast(token_id)] = BLOCK_BIAS; + } + } + break; + } + } else { + switch (state_) { + case State::QWEN_START: + for (uint32_t t : qwen_tool_call_start_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : close_brace_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::QWEN_EXPECT_OPEN_BRACE: + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : qwen_tool_call_end_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::QWEN_EXPECT_NAME_KEY: { + bool has_name = generated_text_.find("name") != std::string::npos; + bool has_quote = generated_text_.find("\"") != std::string::npos; + if (has_name) { + for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } + } else if (has_quote) { + for (uint32_t t : name_key_tokens_) { current_bias_[t] = FORCE_BIAS; } + for (uint32_t t : quote_tokens_) { current_bias_[t] = BLOCK_BIAS; } + } else { + for (uint32_t t : all_func_name_tokens_) { current_bias_[t] = BLOCK_BIAS; } + for (uint32_t t : args_key_tokens_) { current_bias_[t] = BLOCK_BIAS; } + for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } + } + break; + } + + case State::QWEN_EXPECT_NAME_COLON: + for (uint32_t t : colon_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + break; + + case State::QWEN_EXPECT_NAME_VALUE: { + bool name_complete = false; + for (const auto& name : function_names_) { + if (generated_text_.find(name) != std::string::npos) { name_complete = true; break; } + } + bool has_open_quote = generated_text_.find("\"") != std::string::npos; + if (name_complete) { + for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } + } else if (has_open_quote) { + for (uint32_t t : all_func_name_tokens_) { current_bias_[t] = FORCE_BIAS; } + for (uint32_t t : quote_tokens_) { current_bias_[t] = 5.0f; } + } else { + for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } + } + break; + } + + case State::QWEN_EXPECT_COMMA: + for (uint32_t t : comma_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : close_brace_tokens_) { + current_bias_[t] = 5.0f; + } + break; + + case State::QWEN_EXPECT_ARGS_KEY: { + bool has_args = generated_text_.find("arguments") != std::string::npos; + bool has_quote = generated_text_.find("\"") != std::string::npos; + if (has_args) { + for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } + } else if (has_quote) { + for (uint32_t t : args_key_tokens_) { current_bias_[t] = FORCE_BIAS; } + for (uint32_t t : quote_tokens_) { current_bias_[t] = BLOCK_BIAS; } + } else { + for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } + } + break; + } + + case State::QWEN_EXPECT_ARGS_COLON: + for (uint32_t t : colon_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + break; + + case State::QWEN_IN_ARGUMENTS: + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = 3.0f; + } + for (uint32_t t : close_brace_tokens_) { + current_bias_[t] = 3.0f; + } + for (uint32_t t : colon_tokens_) { + current_bias_[t] = 2.0f; + } + for (uint32_t t : comma_tokens_) { + current_bias_[t] = 2.0f; + } + for (uint32_t t : quote_tokens_) { + current_bias_[t] = 2.0f; + } + for (uint32_t t : qwen_tool_call_end_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + case State::QWEN_EXPECT_CLOSE_BRACE: + for (uint32_t t : close_brace_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + break; + + case State::QWEN_EXPECT_END: + for (uint32_t t : qwen_tool_call_end_tokens_) { + current_bias_[t] = FORCE_BIAS; + } + for (uint32_t t : open_brace_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + for (uint32_t t : qwen_tool_call_start_tokens_) { + current_bias_[t] = BLOCK_BIAS; + } + break; + + default: + break; + } + } +} + +void ToolCallConstrainer::reset() { + generated_text_.clear(); + current_bias_.clear(); + brace_depth_ = 0; + reset_needle_constraints(); + + if (model_type_ == Config::ModelType::LFM2) { + state_ = State::LFM_START; + } else if (is_gemma_family()) { + state_ = State::GEMMA_START; + } else if (is_needle()) { + state_ = State::NEEDLE_START; + } else { + state_ = State::QWEN_START; + } + + if (active_) { + compute_bias(); + } +} + + +void Model::set_tool_constraints(const std::vector& tools) { + tool_constrainer_.init(config_.model_type, tools, tokenizer_.get()); +} + +void Model::clear_tool_constraints() { + tool_constrainer_.reset(); + tool_constrainer_.init(config_.model_type, {}, tokenizer_.get()); +} + +void Model::update_tool_constraints(uint32_t token_id) { + if (tool_constrainer_.is_active() && tokenizer_) { + std::string decoded = tokenizer_->decode({token_id}); + tool_constrainer_.update(token_id, decoded); + } +} + +} // namespace engine +} // namespace cactus diff --git a/cactus-transpile/engine/engine_image.cpp b/cactus-transpile/engine/engine_image.cpp new file mode 100644 index 000000000..cf066e99e --- /dev/null +++ b/cactus-transpile/engine/engine_image.cpp @@ -0,0 +1,600 @@ +#define STB_IMAGE_IMPLEMENTATION +#define STB_IMAGE_RESIZE_IMPLEMENTATION + +#define STBI_NO_BMP +#define STBI_NO_PSD +#define STBI_NO_HDR +#define STBI_NO_PIC +#define STBI_NO_PNM +#define STBI_NO_TGA + +#include "engine.h" +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +Siglip2Preprocessor::PreprocessedImage::~PreprocessedImage() { + pixel_values.clear(); + pixel_attention_mask.clear(); +} + +Siglip2Preprocessor::Siglip2Preprocessor(const Config& config) + : config_(config) {} + +Siglip2Preprocessor::Siglip2Preprocessor() : config_() {} + +Siglip2Preprocessor::~Siglip2Preprocessor() = default; + +std::pair Siglip2Preprocessor::compute_pixel_limits() const { + int64_t min_pixels = static_cast(config_.min_image_tokens) * + config_.patch_size * config_.patch_size * + config_.downsample_factor * config_.downsample_factor; + int64_t max_pixels = static_cast(config_.max_image_tokens) * + config_.patch_size * config_.patch_size * + config_.downsample_factor * config_.downsample_factor; + return {min_pixels, max_pixels}; +} + +int Siglip2Preprocessor::round_by_factor(int number, int factor) { + if (factor == 0) { + return number; + } + double scaled = static_cast(number) / static_cast(factor); + long long rounded = std::llround(scaled); + + return static_cast(rounded * factor); +} + +std::pair Siglip2Preprocessor::smart_resize(int height, int width) { + const int total_factor = config_.patch_size * config_.downsample_factor; + auto [min_pixels, max_pixels] = compute_pixel_limits(); + + int h_bar = std::max(total_factor, round_by_factor(height, total_factor)); + int w_bar = std::max(total_factor, round_by_factor(width, total_factor)); + + + if (static_cast(h_bar) * static_cast(w_bar) > max_pixels) { + double beta = std::sqrt((static_cast(height) * static_cast(width)) / + static_cast(max_pixels)); + h_bar = std::max(total_factor, + static_cast(std::floor(height / beta / total_factor)) * total_factor); + w_bar = std::max(total_factor, + static_cast(std::floor(width / beta / total_factor)) * total_factor); + } else if (static_cast(h_bar) * static_cast(w_bar) < min_pixels) { + double beta = std::sqrt(static_cast(min_pixels) / + (static_cast(height) * static_cast(width))); + h_bar = std::max(total_factor, + static_cast(std::ceil(height * beta / total_factor)) * total_factor); + w_bar = std::max(total_factor, + static_cast(std::ceil(width * beta / total_factor)) * total_factor); + } + + return {w_bar, h_bar}; +} + + +bool Siglip2Preprocessor::is_image_too_large(int height, int width) { + const int total_factor = config_.patch_size * config_.downsample_factor; + const int h_bar = std::max(config_.patch_size, round_by_factor(height, total_factor)); + const int w_bar = std::max(config_.patch_size, round_by_factor(width, total_factor)); + + const int64_t pixels = static_cast(h_bar) * static_cast(w_bar); + auto [min_pixels, max_pixels] = compute_pixel_limits(); + const double max_pixels_with_tolerance = static_cast(max_pixels) * config_.max_pixels_tolerance; + + bool result = static_cast(pixels) > max_pixels_with_tolerance; + + return result; +} + + +std::pair Siglip2Preprocessor::find_closest_aspect_ratio(float aspect_ratio, int width, int height) { + float best_ratio_diff = std::numeric_limits::infinity(); + std::pair best_ratio = {1, 1}; + int area = width * height; + + std::vector> target_ratios; + for (int n = config_.min_tiles; n <= config_.max_tiles; ++n) { + for (int w = 1; w <= n; ++w) { + for (int h = 1; h <= n; ++h) { + int total_tiles = w * h; + if (total_tiles >= config_.min_tiles && total_tiles <= config_.max_tiles) { + target_ratios.push_back({w, h}); + } + } + } + } + + std::sort(target_ratios.begin(), target_ratios.end()); + target_ratios.erase(std::unique(target_ratios.begin(), target_ratios.end()), target_ratios.end()); + std::sort(target_ratios.begin(), target_ratios.end(), [](const auto& a, const auto& b) { + return (a.first * a.second) < (b.first * b.second); + }); + + for (const auto& ratio : target_ratios) { + float target_aspect_ratio = static_cast(ratio.first) / ratio.second; + float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio); + + if (ratio_diff < best_ratio_diff) { + best_ratio_diff = ratio_diff; + best_ratio = ratio; + } else if (ratio_diff == best_ratio_diff) { + int target_area = config_.tile_size * config_.tile_size * ratio.first * ratio.second; + if (area > 0.5f * target_area) { + best_ratio = ratio; + } + } + } + + + return best_ratio; +} + +std::pair Siglip2Preprocessor::get_grid_layout(int height, int width) { + float aspect_ratio = (float)width / (float)height; + auto [grid_width, grid_height] = find_closest_aspect_ratio(aspect_ratio, width, height); + + int target_width = config_.tile_size * grid_width; + int target_height = config_.tile_size * grid_height; + + return {target_width, target_height}; +} + +Siglip2Preprocessor::SpatialShapeResult Siglip2Preprocessor::compute_spatial_shapes(int height, int width) { + if (height <= 0 || width <= 0) { + throw std::runtime_error("Image dimensions must be positive"); + } + + if (config_.patch_size <= 0) { + throw std::runtime_error("Patch size must be positive"); + } + + const int patch = config_.patch_size; + auto [resized_width, resized_height] = smart_resize(height, width); + const bool should_split = config_.do_image_splitting && is_image_too_large(height, width); + + + SpatialShapeResult result; + result.grid_rows = 1; + result.grid_cols = 1; + + if (should_split) { + if (config_.tile_size % patch != 0) { + throw std::runtime_error("Tile size must be divisible by patch size"); + } + + auto [grid_target_width, grid_target_height] = get_grid_layout(height, width); + result.grid_cols = grid_target_width / config_.tile_size; + result.grid_rows = grid_target_height / config_.tile_size; + + + const int patches_per_tile_side = config_.tile_size / patch; + const auto tile_shape = std::make_pair(patches_per_tile_side, patches_per_tile_side); + + result.shapes.reserve(static_cast(result.grid_rows) * result.grid_cols + 1); + for (int row = 0; row < result.grid_rows; ++row) { + for (int col = 0; col < result.grid_cols; ++col) { + result.shapes.push_back(tile_shape); + + } + } + + if (config_.use_thumbnail && result.grid_rows * result.grid_cols != 1) { + if (resized_height % patch != 0 || resized_width % patch != 0) { + throw std::runtime_error("Resized thumbnail dimensions must be divisible by patch size"); + } + result.shapes.emplace_back(resized_height / patch, resized_width / patch); + + } + } else { + int target_width = resized_width; + int target_height = resized_height; + if (!config_.do_resize) { + target_width = width; + target_height = height; + } + + if (target_height % patch != 0 || target_width % patch != 0) { + throw std::runtime_error("Target dimensions must be divisible by patch size"); + } + + result.shapes.emplace_back(target_height / patch, target_width / patch); + + } + + return result; +} + + +Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::preprocess_from_file(const std::string& image_path) { + int width, height, channels; + unsigned char* img_data = stbi_load(image_path.c_str(), &width, &height, &channels, 0); + + if (!img_data) { + throw std::runtime_error("Failed to load image: " + image_path + " - " + std::string(stbi_failure_reason())); + } + + PreprocessedImage result; + result = preprocess_from_memory(img_data, width, height, channels); + + stbi_image_free(img_data); + return result; +} + +Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::preprocess_from_memory( + const unsigned char* img_data, int width, int height, int channels) { + if (!img_data) { + throw std::runtime_error("Invalid image data pointer"); + } + + const int expected_channels = 3; + std::vector rgb_data; + const unsigned char* source_data = img_data; + int source_channels = channels; + + if (config_.do_convert_rgb && channels != expected_channels) { + rgb_data = convert_to_rgb(img_data, width, height, channels); + source_data = rgb_data.data(); + source_channels = expected_channels; + + } + + if (source_channels != expected_channels) { + throw std::runtime_error("Image must have 3 channels (RGB)"); + } + + const int patch = config_.patch_size; + const int downsample = config_.downsample_factor; + const int patch_dim = patch * patch * expected_channels; + + if (patch <= 0) { + throw std::runtime_error("Patch size must be positive"); + } + if (config_.tile_size % patch != 0) { + throw std::runtime_error("Tile size must be divisible by patch size"); + } + + auto [resized_width, resized_height] = smart_resize(height, width); + + + const int patches_per_tile_side = config_.tile_size / patch; + const int tile_patch_count = patches_per_tile_side * patches_per_tile_side; + const int thumbnail_patch_cap = config_.max_image_tokens * downsample * downsample; + int max_patches_per_tile = config_.max_num_patches; + max_patches_per_tile = std::max(max_patches_per_tile, tile_patch_count); + max_patches_per_tile = std::max(max_patches_per_tile, thumbnail_patch_cap); + + const bool allow_splitting = config_.do_image_splitting; + const bool should_split = allow_splitting && is_image_too_large(height, width); + + + size_t expected_tiles = should_split ? static_cast(config_.max_tiles) : 1; + if (should_split && config_.use_thumbnail) { + expected_tiles += 1; + } + + std::vector> tile_patches; + tile_patches.reserve(expected_tiles); + std::vector> spatial_shapes; + spatial_shapes.reserve(expected_tiles); + + auto normalize_and_patchify = [&](const float* data_ptr, int img_width, int img_height) { + if (img_height % patch != 0 || img_width % patch != 0) { + throw std::runtime_error("Image dimensions must be divisible by patch size"); + } + std::vector normalized = normalize_image(data_ptr, img_width, img_height, expected_channels); + auto patches = convert_image_to_patches(normalized, img_width, img_height, expected_channels, patch); + std::vector flattened(patches.size() * patch_dim); + for (size_t idx = 0; idx < patches.size(); ++idx) { + std::copy(patches[idx].begin(), patches[idx].end(), flattened.begin() + idx * patch_dim); + } + tile_patches.push_back(std::move(flattened)); + spatial_shapes.emplace_back(img_height / patch, img_width / patch); + }; + + int grid_rows = 1; + int grid_cols = 1; + bool thumbnail_added = false; + + if (should_split) { + auto [grid_target_width, grid_target_height] = get_grid_layout(height, width); + grid_cols = grid_target_width / config_.tile_size; + grid_rows = grid_target_height / config_.tile_size; + + std::vector resized_grid = resize_image( + source_data, width, height, grid_target_width, grid_target_height, expected_channels); + + std::vector tile_buffer( + static_cast(config_.tile_size) * config_.tile_size * expected_channels); + + for (int row = 0; row < grid_rows; ++row) { + for (int col = 0; col < grid_cols; ++col) { + for (int y = 0; y < config_.tile_size; ++y) { + const float* src_row = resized_grid.data() + + ((row * config_.tile_size + y) * grid_target_width + col * config_.tile_size) * + expected_channels; + float* dst_row = tile_buffer.data() + (y * config_.tile_size) * expected_channels; + std::copy_n(src_row, + static_cast(config_.tile_size) * expected_channels, + dst_row); + } + normalize_and_patchify(tile_buffer.data(), config_.tile_size, config_.tile_size); + + } + } + + if (config_.use_thumbnail && grid_rows * grid_cols != 1) { + std::vector thumbnail_bytes = resize_image( + source_data, width, height, resized_width, resized_height, expected_channels); + normalize_and_patchify(thumbnail_bytes.data(), resized_width, resized_height); + thumbnail_added = true; + + } + } else { + int target_width = resized_width; + int target_height = resized_height; + + const bool needs_resize = config_.do_resize && (width != target_width || height != target_height); + + std::vector resized_image; + if (needs_resize) { + resized_image = resize_image(source_data, width, height, target_width, target_height, expected_channels); + normalize_and_patchify(resized_image.data(), target_width, target_height); + + } else { + resized_image.resize(static_cast(width) * height * expected_channels); + for (size_t idx = 0; idx < resized_image.size(); ++idx) { + resized_image[idx] = static_cast(source_data[idx]); + } + normalize_and_patchify(resized_image.data(), width, height); + target_width = width; + target_height = height; + resized_width = target_width; + resized_height = target_height; + + } + + grid_rows = 1; + grid_cols = 1; + } + + PreprocessedImage result = pad_patches(tile_patches, spatial_shapes, patch_dim, max_patches_per_tile); + + + result.image_rows = grid_rows; + result.image_cols = grid_cols; + result.image_width = resized_width; + result.image_height = resized_height; + + auto compute_tokens = [&](int patches_h, int patches_w) -> int { + int tokens_h = (patches_h + downsample - 1) / downsample; + int tokens_w = (patches_w + downsample - 1) / downsample; + return tokens_h * tokens_w; + }; + + result.tokens_per_tile = spatial_shapes.empty() + ? 0 + : compute_tokens(spatial_shapes.front().first, spatial_shapes.front().second); + + + if (thumbnail_added && !spatial_shapes.empty()) { + const auto& thumb_shape = spatial_shapes.back(); + result.thumbnail_tokens = compute_tokens(thumb_shape.first, thumb_shape.second); + + } else { + result.thumbnail_tokens = 0; + } + + return result; +} + +std::vector Siglip2Preprocessor::convert_to_rgb( + const unsigned char* img_data, int width, int height, int channels) { + + std::vector rgb_data(width * height * 3); + + if (channels == 1) { + for (int i = 0; i < width * height; ++i) { + rgb_data[i * 3 + 0] = img_data[i]; + rgb_data[i * 3 + 1] = img_data[i]; + rgb_data[i * 3 + 2] = img_data[i]; + } + } else if (channels == 4) { + for (int i = 0; i < width * height; ++i) { + rgb_data[i * 3 + 0] = img_data[i * 4 + 0]; + rgb_data[i * 3 + 1] = img_data[i * 4 + 1]; + rgb_data[i * 3 + 2] = img_data[i * 4 + 2]; + } + } else if (channels == 2) { + for (int i = 0; i < width * height; ++i) { + rgb_data[i * 3 + 0] = img_data[i * 2 + 0]; + rgb_data[i * 3 + 1] = img_data[i * 2 + 0]; + rgb_data[i * 3 + 2] = img_data[i * 2 + 0]; + } + } else { + throw std::runtime_error("Unsupported number of channels: " + std::to_string(channels)); + } + + return rgb_data; +} + +std::vector Siglip2Preprocessor::resize_image( + const unsigned char* img_data, int src_width, int src_height, + int dst_width, int dst_height, int channels) { + + const size_t src_elements = static_cast(src_width) * src_height * channels; + std::vector src_float(src_elements); + for (size_t idx = 0; idx < src_elements; ++idx) { + src_float[idx] = static_cast(img_data[idx]); + } + + + std::vector resized_data(static_cast(dst_width) * dst_height * channels); + + stbir_pixel_layout layout = (channels == 1) ? STBIR_1CHANNEL : + (channels == 3) ? STBIR_RGB : STBIR_RGBA; + + float* result = stbir_resize_float_linear( + src_float.data(), src_width, src_height, 0, + resized_data.data(), dst_width, dst_height, 0, + layout + ); + + if (!result) { + throw std::runtime_error("Failed to resize image"); + } + + + return resized_data; +} + +std::vector Siglip2Preprocessor::normalize_image( + const float* img_data, int width, int height, int channels) { + + size_t total_pixels = width * height * channels; + std::vector normalized(total_pixels); + + for (size_t i = 0; i < static_cast(width * height); ++i) { + for (int c = 0; c < channels; ++c) { + size_t idx = i * channels + c; + float pixel = img_data[idx]; + + if (config_.do_rescale) { + pixel *= config_.rescale_factor; + } + + if (config_.do_normalize) { + pixel = (pixel - config_.image_mean[c]) / config_.image_std[c]; + } + + normalized[idx] = pixel; + } + } + + + return normalized; +} + +std::vector> Siglip2Preprocessor::convert_image_to_patches( + const std::vector& image, int width, int height, int channels, int patch_size) { + + int num_patches_height = height / patch_size; + int num_patches_width = width / patch_size; + int num_patches = num_patches_height * num_patches_width; + int patch_elements = patch_size * patch_size * channels; + + std::vector> patches(num_patches, std::vector(patch_elements)); + + for (int ph = 0; ph < num_patches_height; ++ph) { + for (int pw = 0; pw < num_patches_width; ++pw) { + int patch_idx = ph * num_patches_width + pw; + + for (int y = 0; y < patch_size; ++y) { + for (int x = 0; x < patch_size; ++x) { + int img_y = ph * patch_size + y; + int img_x = pw * patch_size + x; + int img_idx = (img_y * width + img_x) * channels; + int patch_offset = (y * patch_size + x) * channels; + + for (int c = 0; c < channels; ++c) { + patches[patch_idx][patch_offset + c] = image[img_idx + c]; + } + } + } + } + } + + return patches; +} + +Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::pad_patches( + const std::vector>& tile_patches, + const std::vector>& spatial_shapes, + int patch_dim, + int max_patches_per_tile) { + + if (tile_patches.size() != spatial_shapes.size()) { + throw std::runtime_error("Mismatch between tile data and spatial shapes"); + } + + PreprocessedImage result; + + const int num_tiles = static_cast(tile_patches.size()); + result.num_tiles = num_tiles; + result.patch_dim = patch_dim; + result.max_patches_per_tile = max_patches_per_tile; + result.spatial_shapes = spatial_shapes; + result.actual_num_patches = 0; + + const size_t total_values = static_cast(num_tiles) * max_patches_per_tile * patch_dim; + result.pixel_values.assign(total_values, 0.0f); + result.pixel_attention_mask.assign(static_cast(num_tiles) * max_patches_per_tile, 0); + + + for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { + const auto& [patches_h, patches_w] = spatial_shapes[tile_idx]; + const int actual_patches = patches_h * patches_w; + + if (actual_patches > max_patches_per_tile) { + throw std::runtime_error("Actual patches exceed max_patches_per_tile"); + } + + const auto& flattened = tile_patches[tile_idx]; + const size_t expected_size = static_cast(actual_patches) * patch_dim; + if (flattened.size() != expected_size) { + throw std::runtime_error("Tile patch data has unexpected size"); + } + + float* destination = result.pixel_values.data() + + static_cast(tile_idx) * max_patches_per_tile * patch_dim; + std::memcpy(destination, flattened.data(), expected_size * sizeof(float)); + + + int mask_offset = tile_idx * max_patches_per_tile; + for (int p = 0; p < actual_patches; ++p) { + result.pixel_attention_mask[mask_offset + p] = 1; + } + + result.actual_num_patches += actual_patches; + } + + if (!spatial_shapes.empty()) { + result.num_patches_height = spatial_shapes.front().first; + result.num_patches_width = spatial_shapes.front().second; + } else { + result.num_patches_height = 0; + result.num_patches_width = 0; + } + + result.pixel_values_shape = { + static_cast(num_tiles), + static_cast(max_patches_per_tile), + static_cast(patch_dim) + }; + + + result.pixel_attention_mask_shape = { + static_cast(num_tiles), + static_cast(max_patches_per_tile) + }; + + result.spatial_shapes_shape = { + static_cast(num_tiles), + static_cast(2) + }; + + + return result; +} + +} // namespace engine +} // namespace cactus + + diff --git a/cactus-transpile/engine/engine_index.cpp b/cactus-transpile/engine/engine_index.cpp new file mode 100644 index 000000000..13d08920a --- /dev/null +++ b/cactus-transpile/engine/engine_index.cpp @@ -0,0 +1,780 @@ +#include "engine.h" +#include "kernel/kernel.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { +namespace index { + + __fp16 dot_product(const __fp16 *a, const __fp16 *b, size_t dim) { + __fp16 result; + cactus_matmul_f16(a, b, &result, 1, dim, 1); + return result; + } + + void normalize(__fp16 *v, size_t dim) { + __fp16 x = dot_product(v, v, dim); + if (x < 1e-6f) { + throw std::runtime_error("Cannot normalize zero vector"); + } + cactus_scalar_op_f16(&x, &x, 1, 0, ScalarOpType::SQRT); + cactus_scalar_op_f16(v, v, dim, x, ScalarOpType::DIVIDE); + } + + Index::Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim): + index_path_(index_path), data_path_(data_path), embedding_dim_(embedding_dim), + index_entry_size_(IndexEntry::size(embedding_dim)), num_documents_(0), + index_fd_(-1), data_fd_(-1), + mapped_index_(nullptr), mapped_data_(nullptr) { + + bool index_exists = (access(index_path.c_str(), F_OK) == 0); + bool data_exists = (access(data_path.c_str(), F_OK) == 0); + + if (index_exists != data_exists) { + throw std::runtime_error("Index and data files must both exist or both not exist"); + } + + index_fd_ = open(index_path.c_str(), O_RDWR | O_CREAT, 0644); + if (index_fd_ < 0) { + throw std::runtime_error("Cannot open index file: " + index_path); + } + + data_fd_ = open(data_path.c_str(), O_RDWR | O_CREAT, 0644); + if (data_fd_ < 0) { + close(index_fd_); + throw std::runtime_error("Cannot open data file: " + data_path); + } + + auto cleanup_and_throw = [this](const std::string& msg) { + close(index_fd_); + close(data_fd_); + throw std::runtime_error(msg); + }; + + struct stat index_st, data_st; + if (fstat(index_fd_, &index_st)) { + cleanup_and_throw("Cannot get index file size: " + index_path); + } + index_file_size_ = index_st.st_size; + + if (fstat(data_fd_, &data_st)) { + cleanup_and_throw("Cannot get data file size: " + data_path); + } + data_file_size_ = data_st.st_size; + + if (!index_exists) { + index_file_size_ = sizeof(IndexHeader); + data_file_size_ = sizeof(DataHeader); + + if (ftruncate(index_fd_, index_file_size_) != 0) { + cleanup_and_throw("Failed to resize index file"); + } + + if (ftruncate(data_fd_, data_file_size_) != 0) { + cleanup_and_throw("Failed to resize data file"); + } + } + + mapped_index_ = mmap(nullptr, index_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, index_fd_, 0); + if (mapped_index_ == MAP_FAILED) { + cleanup_and_throw("Cannot map file: " + index_path); + } + + mapped_data_ = mmap(nullptr, data_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, data_fd_, 0); + if (mapped_data_ == MAP_FAILED) { + munmap(mapped_index_, index_file_size_); + close(index_fd_); + close(data_fd_); + throw std::runtime_error("Cannot map file: " + data_path); + } + + if (!index_exists) { + IndexHeader index_header = { + MAGIC, + VERSION, + static_cast(embedding_dim), + 0 + }; + memcpy(mapped_index_, &index_header, sizeof(IndexHeader)); + + DataHeader data_header = { MAGIC, VERSION }; + memcpy(mapped_data_, &data_header, sizeof(DataHeader)); + + if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { + cleanup_and_throw("Failed to sync index file"); + } + + if (msync(mapped_data_, data_file_size_, MS_SYNC) != 0) { + cleanup_and_throw("Failed to sync data file"); + } + } + + parse_index_header(); + parse_data_header(); + + build_doc_id_map(); + } + + Index::~Index() { + if (mapped_index_ != nullptr && mapped_index_ != MAP_FAILED) { + madvise(mapped_index_, index_file_size_, MADV_DONTNEED); + munmap(mapped_index_, index_file_size_); + } + if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { + madvise(mapped_data_, data_file_size_, MADV_DONTNEED); + munmap(mapped_data_, data_file_size_); + } + if (index_fd_ != -1) { + close(index_fd_); + } + if (data_fd_ != -1) { + close(data_fd_); + } + } + + void Index::add_documents(const std::vector& documents) { + validate_documents(documents); + + size_t added_data_size = 0; + for (const auto& doc : documents) { + added_data_size += sizeof(DataEntry) + doc.content.size() + doc.metadata.size(); + } + + size_t new_index_size = index_file_size_ + documents.size() * index_entry_size_; + size_t new_data_size = data_file_size_ + added_data_size; + + if (ftruncate(index_fd_, new_index_size) != 0) { + throw std::runtime_error("Failed to resize index file"); + } + + if (ftruncate(data_fd_, new_data_size) != 0) { + throw std::runtime_error("Failed to resize data file"); + } + + munmap(mapped_index_, index_file_size_); + munmap(mapped_data_, data_file_size_); + + mapped_index_ = mmap(nullptr, new_index_size, PROT_READ | PROT_WRITE, MAP_SHARED, index_fd_, 0); + if (mapped_index_ == MAP_FAILED) { + throw std::runtime_error("Failed to remap index file"); + } + + mapped_data_ = mmap(nullptr, new_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, data_fd_, 0); + if (mapped_data_ == MAP_FAILED) { + munmap(mapped_index_, new_index_size); + throw std::runtime_error("Failed to remap data file"); + } + + size_t old_index_size = index_file_size_; + size_t old_data_size = data_file_size_; + + index_file_size_ = new_index_size; + data_file_size_ = new_data_size; + + char* index_write_pos = static_cast(mapped_index_) + old_index_size; + char* data_write_pos = static_cast(mapped_data_) + old_data_size; + + uint64_t data_offset = old_data_size; + doc_id_map_.reserve(doc_id_map_.size() + documents.size()); + + for (size_t i = 0; i < documents.size(); ++i) { + const auto& doc = documents[i]; + + DataEntry data_entry{ + static_cast(doc.content.size()), + static_cast(doc.metadata.size()) + }; + + memcpy(data_write_pos, &data_entry, sizeof(DataEntry)); + data_write_pos += sizeof(DataEntry); + memcpy(data_write_pos, doc.content.data(), doc.content.size()); + data_write_pos += doc.content.size(); + memcpy(data_write_pos, doc.metadata.data(), doc.metadata.size()); + data_write_pos += doc.metadata.size(); + + IndexEntry index_entry{doc.id, data_offset, 0}; + memcpy(index_write_pos, &index_entry, sizeof(IndexEntry)); + index_write_pos += sizeof(IndexEntry); + + std::vector<__fp16> normalized_embedding(embedding_dim_); + cactus_fp32_to_fp16(doc.embedding.data(), normalized_embedding.data(), embedding_dim_); + normalize(normalized_embedding.data(), embedding_dim_); + + size_t embedding_bytes = embedding_dim_ * sizeof(__fp16); + memcpy(index_write_pos, normalized_embedding.data(), embedding_bytes); + index_write_pos += embedding_bytes; + + doc_id_map_[doc.id] = num_documents_ + static_cast(i); + data_offset += sizeof(DataEntry) + doc.content.size() + doc.metadata.size(); + } + + num_documents_ += static_cast(documents.size()); + IndexHeader* header = reinterpret_cast(mapped_index_); + header->num_documents = num_documents_; + + if (msync(mapped_data_, data_file_size_, MS_SYNC) != 0) { + throw std::runtime_error("Failed to sync data file to disk"); + } + + if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { + throw std::runtime_error("Failed to sync index file to disk"); + } + } + + void Index::delete_documents(const std::vector& doc_ids) { + validate_doc_ids(doc_ids); + + char* index_ptr = static_cast(mapped_index_); + char* entries = index_ptr + sizeof(IndexHeader); + + if (num_documents_ > 0) { + size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; + if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { + throw std::runtime_error("File corrupted: index entry extends beyond file size"); + } + } + + for (int doc_id : doc_ids) { + uint32_t i = doc_id_map_.at(doc_id); + IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); + + entry.flags |= 0x1; + doc_id_map_.erase(doc_id); + } + + if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { + throw std::runtime_error("Failed to sync index file to disk"); + } + } + + std::vector Index::get_documents(const std::vector& doc_ids) { + validate_doc_ids(doc_ids); + + const char* index_ptr = static_cast(mapped_index_); + const char* entries = index_ptr + sizeof(IndexHeader); + const char* data_ptr = static_cast(mapped_data_); + + if (num_documents_ > 0) { + size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; + if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { + throw std::runtime_error("File corrupted: index entry extends beyond file size"); + } + } + + std::vector results; + results.reserve(doc_ids.size()); + + for (int doc_id : doc_ids) { + uint32_t i = doc_id_map_.at(doc_id); + const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); + + if (static_cast(entry.data_offset) + sizeof(DataEntry) > data_file_size_) { + throw std::runtime_error("File corrupted: data entry extends beyond file size"); + } + + const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); + size_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; + + if (static_cast(entry.data_offset) + data_entry_size > data_file_size_) { + throw std::runtime_error("File corrupted: data entry extends beyond file size"); + } + + std::vector embedding_f32(embedding_dim_); + cactus_fp16_to_fp32(entry.embedding(), embedding_f32.data(), embedding_dim_); + + results.emplace_back( + doc_id, + std::move(embedding_f32), + std::string(data_entry->content(), data_entry->content_len), + std::string(data_entry->metadata(), data_entry->metadata_len) + ); + } + + return results; + } + + std::vector> Index::query(const std::vector>& embeddings, const QueryOptions& options) { + if (embeddings.empty()) { + return {}; + } + + if (options.top_k == 0) { + return std::vector>(embeddings.size()); + } + + if (num_documents_ > 0) { + size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; + if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { + throw std::runtime_error("File corrupted: index entry extends beyond file size"); + } + } + + std::vector> normalized_embeddings; + normalized_embeddings.reserve(embeddings.size()); + + for (const auto& embedding : embeddings) { + std::vector<__fp16> normalized_embedding(embedding_dim_); + cactus_fp32_to_fp16(embedding.data(), normalized_embedding.data(), embedding_dim_); + normalize(normalized_embedding.data(), embedding_dim_); + normalized_embeddings.emplace_back(std::move(normalized_embedding)); + } + + std::vector> all_results; + all_results.reserve(embeddings.size()); + + const char* index_ptr = static_cast(mapped_index_); + const char* entries = index_ptr + sizeof(IndexHeader); + + auto cmp = [](const QueryResult& a, const QueryResult& b) { + return a.score > b.score; + }; + + for (const auto& normalized_embedding : normalized_embeddings) { + std::priority_queue, decltype(cmp)> top_results(cmp); + + for (uint32_t i = 0; i < num_documents_; ++i) { + const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); + + if (entry.flags & 0x1) { + continue; + } + + float score = static_cast( + dot_product(normalized_embedding.data(), entry.embedding(), embedding_dim_) + ); + + if (score < options.score_threshold) { + continue; + } + + if (top_results.size() < options.top_k) { + top_results.emplace(entry.doc_id, score); + } else if (score > top_results.top().score) { + top_results.pop(); + top_results.emplace(entry.doc_id, score); + } + } + + std::vector results; + results.reserve(top_results.size()); + + while(!top_results.empty()) { + results.emplace_back(top_results.top()); + top_results.pop(); + } + std::reverse(results.begin(), results.end()); + + all_results.emplace_back(std::move(results)); + } + + return all_results; + } + + void Index::compact() { + std::string temp_index_path = index_path_ + ".tmp"; + std::string temp_data_path = data_path_ + ".tmp"; + + const char* index_ptr = static_cast(mapped_index_); + const char* entries = index_ptr + sizeof(IndexHeader); + const char* data_ptr = static_cast(mapped_data_); + + uint32_t compacted_count = static_cast(doc_id_map_.size()); + + if (num_documents_ > 0) { + size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; + if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { + throw std::runtime_error("Compaction failed: File corrupted: index entry extends beyond file size"); + } + } + + off_t new_data_offset = sizeof(DataHeader); + size_t new_data_size = sizeof(DataHeader); + + for (const auto& [doc_id, index] : doc_id_map_) { + const IndexEntry& entry = *reinterpret_cast(entries + index * index_entry_size_); + + if (static_cast(entry.data_offset) + sizeof(DataEntry) > data_file_size_) { + throw std::runtime_error("Compaction failed: File corrupted: data entry extends beyond file size"); + } + + const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); + uint32_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; + + if (static_cast(entry.data_offset) + data_entry_size > data_file_size_) { + throw std::runtime_error("Compaction failed: File corrupted: data entry extends beyond file size"); + } + + new_data_size += data_entry_size; + } + + size_t new_index_size = sizeof(IndexHeader) + compacted_count * index_entry_size_; + + int temp_index_fd = open(temp_index_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644); + if (temp_index_fd < 0) { + throw std::runtime_error("Cannot create temporary index file: " + temp_index_path); + } + + int temp_data_fd = open(temp_data_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644); + if (temp_data_fd < 0) { + close(temp_index_fd); + unlink(temp_index_path.c_str()); + throw std::runtime_error("Cannot create temporary data file: " + temp_data_path); + } + + auto cleanup_and_throw = [&temp_index_fd, &temp_data_fd, &temp_index_path, &temp_data_path](const std::string& msg) { + close(temp_index_fd); + close(temp_data_fd); + unlink(temp_index_path.c_str()); + unlink(temp_data_path.c_str()); + throw std::runtime_error(msg); + }; + + if (ftruncate(temp_index_fd, new_index_size) != 0) { + cleanup_and_throw("Failed to resize temporary index file"); + } + + if (ftruncate(temp_data_fd, new_data_size) != 0) { + cleanup_and_throw("Failed to resize temporary data file"); + } + + void* temp_index_map = mmap(nullptr, new_index_size, PROT_READ | PROT_WRITE, MAP_SHARED, temp_index_fd, 0); + if (temp_index_map == MAP_FAILED) { + cleanup_and_throw("Cannot map temporary index file"); + } + + void* temp_data_map = mmap(nullptr, new_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, temp_data_fd, 0); + if (temp_data_map == MAP_FAILED) { + munmap(temp_index_map, new_index_size); + cleanup_and_throw("Cannot map temporary data file"); + } + + IndexHeader new_header = { + MAGIC, + VERSION, + static_cast(embedding_dim_), + compacted_count + }; + + DataHeader new_data_header = { + MAGIC, + VERSION + }; + + memcpy(temp_index_map, &new_header, sizeof(IndexHeader)); + memcpy(temp_data_map, &new_data_header, sizeof(DataHeader)); + + char* temp_index_ptr = static_cast(temp_index_map); + char* temp_data_ptr = static_cast(temp_data_map); + char* index_write_pos = temp_index_ptr + sizeof(IndexHeader); + char* data_write_pos = temp_data_ptr + sizeof(DataHeader); + + std::unordered_map new_doc_id_map; + new_doc_id_map.reserve(compacted_count); + + uint32_t new_index = 0; + new_data_offset = sizeof(DataHeader); + + for (const auto& [doc_id, index] : doc_id_map_) { + const IndexEntry& entry = *reinterpret_cast(entries + index * index_entry_size_); + const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); + uint32_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; + + memcpy(data_write_pos, data_entry, data_entry_size); + data_write_pos += data_entry_size; + + IndexEntry new_entry = {entry.doc_id, static_cast(new_data_offset), 0}; + + memcpy(index_write_pos, &new_entry, sizeof(IndexEntry)); + index_write_pos += sizeof(IndexEntry); + + size_t embedding_bytes = embedding_dim_ * sizeof(__fp16); + memcpy(index_write_pos, entry.embedding(), embedding_bytes); + index_write_pos += embedding_bytes; + + new_doc_id_map[doc_id] = new_index; + ++new_index; + + if (static_cast(data_entry_size) > std::numeric_limits::max() - new_data_offset) { + munmap(temp_index_map, new_index_size); + munmap(temp_data_map, new_data_size); + cleanup_and_throw("Data offset overflow during compaction"); + } + new_data_offset += data_entry_size; + } + + if (msync(temp_data_map, new_data_size, MS_SYNC) != 0) { + munmap(temp_index_map, new_index_size); + munmap(temp_data_map, new_data_size); + cleanup_and_throw("Failed to sync temporary data file"); + } + + if (msync(temp_index_map, new_index_size, MS_SYNC) != 0) { + munmap(temp_index_map, new_index_size); + munmap(temp_data_map, new_data_size); + cleanup_and_throw("Failed to sync temporary index file"); + } + + munmap(temp_index_map, new_index_size); + munmap(temp_data_map, new_data_size); + + std::string backup_index = index_path_ + ".backup"; + std::string backup_data = data_path_ + ".backup"; + + if (access(backup_index.c_str(), F_OK) == 0 || access(backup_data.c_str(), F_OK) == 0) { + cleanup_and_throw("Backup files already exist, previous compaction may have failed"); + } + + close(temp_index_fd); + close(temp_data_fd); + + munmap(mapped_index_, index_file_size_); + munmap(mapped_data_, data_file_size_); + close(index_fd_); + close(data_fd_); + + if (rename(index_path_.c_str(), backup_index.c_str()) != 0) { + cleanup_and_throw("Failed to backup index file"); + } + if (rename(data_path_.c_str(), backup_data.c_str()) != 0) { + rename(backup_index.c_str(), index_path_.c_str()); + cleanup_and_throw("Failed to backup data file"); + } + + if (rename(temp_data_path.c_str(), data_path_.c_str()) != 0) { + rename(backup_data.c_str(), data_path_.c_str()); + rename(backup_index.c_str(), index_path_.c_str()); + cleanup_and_throw("Failed to rename data file"); + } + + if (rename(temp_index_path.c_str(), index_path_.c_str()) != 0) { + unlink(data_path_.c_str()); + rename(backup_data.c_str(), data_path_.c_str()); + rename(backup_index.c_str(), index_path_.c_str()); + cleanup_and_throw("Failed to rename index file"); + } + + auto sync_dir = [](const std::string& file_path) { + size_t last_slash = file_path.rfind('/'); + std::string dir_path = (last_slash != std::string::npos) ? file_path.substr(0, last_slash) : "."; + int dir_fd = open(dir_path.c_str(), O_RDONLY); + if (dir_fd < 0) { + throw std::runtime_error("Cannot open directory for fsync: " + dir_path); + } + if (fsync(dir_fd) != 0) { + close(dir_fd); + throw std::runtime_error("Failed to fsync directory: " + dir_path); + } + close(dir_fd); + }; + + sync_dir(index_path_); + sync_dir(data_path_); + + unlink(backup_index.c_str()); + unlink(backup_data.c_str()); + + int new_index_fd = open(index_path_.c_str(), O_RDWR); + if (new_index_fd < 0) { + throw std::runtime_error("Cannot open new index file: " + index_path_); + } + + int new_data_fd = open(data_path_.c_str(), O_RDWR); + if (new_data_fd < 0) { + close(new_index_fd); + throw std::runtime_error("Cannot open new data file: " + data_path_); + } + + auto cleanup_fds = [&new_index_fd, &new_data_fd](const std::string& msg) { + close(new_index_fd); + close(new_data_fd); + throw std::runtime_error(msg); + }; + + struct stat index_st, data_st; + if (fstat(new_index_fd, &index_st)) { + cleanup_fds("Cannot get index file size: " + index_path_); + } + index_file_size_ = index_st.st_size; + + if (fstat(new_data_fd, &data_st)) { + cleanup_fds("Cannot get data file size: " + data_path_); + } + data_file_size_ = data_st.st_size; + + mapped_index_ = mmap(nullptr, index_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, new_index_fd, 0); + if (mapped_index_ == MAP_FAILED) { + cleanup_fds("Cannot map file: " + index_path_); + } + + mapped_data_ = mmap(nullptr, data_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, new_data_fd, 0); + if (mapped_data_ == MAP_FAILED) { + munmap(mapped_index_, index_file_size_); + close(new_index_fd); + close(new_data_fd); + throw std::runtime_error("Cannot map file: " + data_path_); + } + + index_fd_ = new_index_fd; + data_fd_ = new_data_fd; + + num_documents_ = compacted_count; + doc_id_map_ = std::move(new_doc_id_map); + } + + void Index::parse_index_header() { + const char* index_ptr = static_cast(mapped_index_); + size_t offset = 0; + + if (index_file_size_ < sizeof(IndexHeader)) { + throw std::runtime_error("Index file too small: insufficient data for header"); + } + + IndexHeader header; + header.magic = *reinterpret_cast(index_ptr + offset); + offset += sizeof(header.magic); + + if (header.magic != MAGIC) { + throw std::runtime_error("Invalid index file magic number"); + } + + header.version = *reinterpret_cast(index_ptr + offset); + offset += sizeof(header.version); + + if (header.version != VERSION) { + throw std::runtime_error("Index file version mismatch"); + } + + header.embedding_dim = *reinterpret_cast(index_ptr + offset); + offset += sizeof(header.embedding_dim); + + if (static_cast(header.embedding_dim) != embedding_dim_) { + throw std::runtime_error("Embedding dimension mismatch"); + } + + num_documents_ = *reinterpret_cast(index_ptr + offset); + } + + void Index::parse_data_header() { + const char* data_ptr = static_cast(mapped_data_); + size_t offset = 0; + + if (data_file_size_ < sizeof(DataHeader)) { + throw std::runtime_error("Data file too small: insufficient data for header"); + } + + DataHeader header; + header.magic = *reinterpret_cast(data_ptr + offset); + offset += sizeof(header.magic); + + if (header.magic != MAGIC) { + throw std::runtime_error("Invalid data file magic number"); + } + + header.version = *reinterpret_cast(data_ptr + offset); + offset += sizeof(header.version); + + if (header.version != VERSION) { + throw std::runtime_error("Data file version mismatch"); + } + } + + void Index::build_doc_id_map() { + const char* index_ptr = static_cast(mapped_index_); + const char* entries = index_ptr + sizeof(IndexHeader); + + if (num_documents_ > 0) { + size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; + if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { + throw std::runtime_error("File corrupted: index entry extends beyond file size"); + } + } + + doc_id_map_.reserve(static_cast(num_documents_)); + + for (uint32_t i = 0; i < num_documents_; ++i) { + const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); + + if (entry.flags & 0x1) { + continue; + } + + doc_id_map_[entry.doc_id] = i; + } + } + + void Index::validate_documents(const std::vector& documents) { + if (documents.empty()) { + throw std::runtime_error("Documents vector is empty"); + } + + std::unordered_set seen_ids; + for (const auto& doc : documents) { + if (doc.content.size() > 65535) { + throw std::runtime_error("Document content too long (max 65535 bytes) for ID: " + std::to_string(doc.id)); + } + + if (doc.metadata.size() > 65535) { + throw std::runtime_error("Document metadata too long (max 65535 bytes) for ID: " + std::to_string(doc.id)); + } + + if (doc.embedding.size() != embedding_dim_) { + throw std::runtime_error("Document embedding dimension mismatch for ID: " + std::to_string(doc.id)); + } + + if (seen_ids.find(doc.id) != seen_ids.end()) { + throw std::runtime_error("Duplicate document ID in input: " + std::to_string(doc.id)); + } + seen_ids.insert(doc.id); + + if (doc_id_map_.find(doc.id) != doc_id_map_.end()) { + throw std::runtime_error("Document ID already exists: " + std::to_string(doc.id)); + } + } + } + + void Index::validate_doc_ids(const std::vector& doc_ids) { + if (doc_ids.empty()) { + throw std::runtime_error("Document IDs vector is empty"); + } + + std::unordered_set seen_ids; + for (int doc_id : doc_ids) { + if (seen_ids.find(doc_id) != seen_ids.end()) { + throw std::runtime_error("Duplicate document ID in input: " + std::to_string(doc_id)); + } + seen_ids.insert(doc_id); + + if (doc_id_map_.find(doc_id) == doc_id_map_.end()) { + throw std::runtime_error("Document ID not found: " + std::to_string(doc_id)); + } + } + } + + ssize_t Index::write_full(int fd, const void* buf, size_t count) { + size_t written = 0; + const char* ptr = static_cast(buf); + while (written < count) { + ssize_t n = write(fd, ptr + written, count - written); + if (n < 0) { + if (errno == EINTR) continue; + return -1; + } + if (n == 0) return written; + written += n; + } + return written; + } + +} // namespace index +} // namespace engine +} // namespace cactus diff --git a/cactus-transpile/engine/engine_model.cpp b/cactus-transpile/engine/engine_model.cpp new file mode 100644 index 000000000..42575da0b --- /dev/null +++ b/cactus-transpile/engine/engine_model.cpp @@ -0,0 +1,1098 @@ +#include "engine.h" +#include "../models/model.h" +#include "../graph/graph.h" +#include "../npu/npu.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + + +Model::Model() + : graph_handle_(nullptr), + config_(), + tokenizer_(nullptr), + initialized_(false), + attention_scale_(0.0f), + output_weight_node_id_(0), + owns_graph_(false) { +} + +Model::Model(const Config& config) + : graph_handle_(nullptr), + config_(config), + tokenizer_(nullptr), + initialized_(false), + attention_scale_(0.0f), + output_weight_node_id_(0), + owns_graph_(false) { +} + +Model::~Model() { + if (graph_handle_ && owns_graph_) { + delete static_cast(graph_handle_); + } +} + +bool Model::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { + if (initialized_) { + return true; + } + auto* gb = new CactusGraph(); + graph_handle_ = gb; + owns_graph_ = true; + embedding_file_path_ = model_folder + "/token_embeddings.weights"; + return init_internal(gb, model_folder, context_size, system_prompt, do_warmup); +} + +bool Model::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup) { + if (!external_graph) { + throw std::invalid_argument("External graph pointer must not be null"); + } + if (initialized_) { + graph_handle_ = external_graph; + owns_graph_ = false; + return true; + } + + owns_graph_ = false; + graph_handle_ = external_graph; + return init_internal(external_graph, model_folder, context_size, system_prompt, do_warmup); +} + +bool Model::init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup) { + + CACTUS_LOG_DEBUG("model", "Initializing model from: " << model_folder); + model_folder_path_ = model_folder; + std::string config_path = model_folder + "/config.txt"; + + if (!config_.from_json(config_path)) { + CACTUS_LOG_ERROR("model", "Model initialization failed - config not loaded from: " << model_folder); + return false; + } + + std::string vocab_file = model_folder + "/vocab.txt"; + std::string merges_file = model_folder + "/merges.txt"; + std::string tokenizer_config_file = model_folder + "/tokenizer_config.txt"; + TokenizerRuntimeConfig tokenizer_runtime_config = load_tokenizer_runtime_config(tokenizer_config_file); + + std::ifstream merges_check(merges_file); + bool has_merges = false; + if (merges_check.is_open()) { + std::string line; + int line_count = 0; + while (std::getline(merges_check, line) && line_count < 10) { + if (!line.empty() && line[0] != '#') { + has_merges = true; + break; + } + line_count++; + } + merges_check.close(); + } + + if (tokenizer_runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::BPE || + (tokenizer_runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::UNKNOWN && has_merges)) { + tokenizer_ = std::make_unique(); + } else { + tokenizer_ = std::make_unique(); + } + + if (!tokenizer_->load_vocabulary_with_config(vocab_file, merges_file, tokenizer_config_file)) { + return false; + } + + graph_handle_ = gb; + + if(config_.model_type == Config::ModelType::WHISPER){ + embedding_file_path_ = model_folder+"/decoder_token_embeddings.weights"; + } + else{ + embedding_file_path_ = model_folder + "/token_embeddings.weights"; + } + + load_weights_to_graph(gb); + + if (config_.model_type == Config::ModelType::GEMMA3N || config_.model_type == Config::ModelType::GEMMA4) { + attention_scale_ = 1.0f; + } else if (config_.model_type == Config::ModelType::GEMMA) { + attention_scale_ = 1.0f / std::sqrt(256.0f); + } else { + attention_scale_ = 1.0f / std::sqrt(static_cast(config_.attention_head_dim)); + } + + Precision cache_precision = (config_.model_type == Config::ModelType::WHISPER || + config_.model_type == Config::ModelType::MOONSHINE || + config_.model_type == Config::ModelType::PARAKEET || + config_.model_type == Config::ModelType::PARAKEET_TDT || + config_.model_type == Config::ModelType::NEEDLE) + ? Precision::FP16 + : Precision::INT8; + kv_cache_.init(config_.num_layers, context_size, get_kv_layer_dims(), get_kv_layer_heads(), cache_precision); + + size_t window_size = std::min(context_size, size_t(512)); + size_t sink_size = 4; + const char* env_window = std::getenv("CACTUS_KV_WINDOW_SIZE"); + const char* env_sink = std::getenv("CACTUS_KV_SINK_SIZE"); + if (env_window) { + window_size = std::stoul(env_window); + } + if (env_sink) { + sink_size = std::stoul(env_sink); + } + kv_cache_.set_window_size(window_size, sink_size); + cache_k_output_nodes_.resize(config_.num_layers); + cache_v_output_nodes_.resize(config_.num_layers); + + post_init(); + + initialized_ = true; + + if (do_warmup && + config_.model_type != Config::ModelType::WHISPER && + config_.model_type != Config::ModelType::MOONSHINE && + config_.model_type != Config::ModelType::PARAKEET && + config_.model_type != Config::ModelType::PARAKEET_TDT && + config_.model_type != Config::ModelType::NEEDLE) { + std::string warmup_text = system_prompt.empty() ? "Hello" : system_prompt; + auto warmup_tokens = tokenizer_->encode(warmup_text); + if (config_.model_type == Config::ModelType::GEMMA4) { + warmup_tokens = {2}; + } + forward(warmup_tokens); + auto* gb = static_cast(graph_handle_); + gb->execute(); + } + + reset_cache(); + return true; +} + +size_t Model::forward(const std::vector& /*mel_bins*/, const std::vector& tokens, bool use_cache){ + return forward(tokens, use_cache); +} + +void Model::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { + if (tokens.empty()) { + return; + } + + if (has_npu_prefill()) { + size_t npu_chunk_size = static_cast(npu_prefill_->get_chunk_size()); + if (tokens.size() > npu_chunk_size) { + prefill_npu(tokens); + return; + } + } + + auto* gb = static_cast(graph_handle_); + + auto process_chunk = [&](const std::vector& chunk) { + forward(chunk, true); + gb->execute(profile_file); + post_execute_updates(gb, chunk.size()); + update_kv_cache(gb, chunk.size()); + }; + + if (tokens.size() <= chunk_size) { + process_chunk(tokens); + return; + } + + size_t num_full_chunks = (tokens.size() - 1) / chunk_size; + + for (size_t chunk_idx = 0; chunk_idx < num_full_chunks; ++chunk_idx) { + size_t start = chunk_idx * chunk_size; + size_t end = start + chunk_size; + std::vector chunk(tokens.begin() + start, tokens.begin() + end); + if (chunk_idx == 1) { + gb->set_prefill_mode(true); + } + process_chunk(chunk); + } + + gb->set_prefill_mode(false); + size_t final_start = num_full_chunks * chunk_size; + std::vector final_chunk(tokens.begin() + final_start, tokens.end()); + process_chunk(final_chunk); +} + +void Model::prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file) { + (void)image_paths; + prefill(tokens, get_prefill_chunk_size(), profile_file); +} + +uint32_t Model::decode(const std::vector& tokens, float temperature, float top_p, + size_t top_k, const std::string& profile_file, float* out_entropy, + float min_p, float repetition_penalty) { + + if (temperature < 0) { + temperature = config_.default_temperature; + } + if (top_p < 0) { + top_p = config_.default_top_p; + } + if (top_k == 0) { + top_k = config_.default_top_k; + } + auto final_hidden = forward(tokens, true); + + auto* gb = static_cast(graph_handle_); + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + auto last_hidden = gb->index(final_hidden, tokens.size() - 1, 0); + const auto& last_hidden_buf = gb->get_output_buffer(last_hidden); + size_t hidden_dim = last_hidden_buf.shape[0]; + last_hidden = gb->reshape(last_hidden, {1, hidden_dim}); + + auto logits_node_id = gb->matmul(last_hidden, output_weight_node_id_, true, backend); + + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits_node_id = gb->scalar_multiply(logits_node_id, inv_cap); + logits_node_id = gb->tanh(logits_node_id); + logits_node_id = gb->scalar_multiply(logits_node_id, config_.final_logit_softcapping); + } + auto sampled_token_id = sample_token(gb, logits_node_id, temperature, top_p, top_k, min_p, repetition_penalty); + + gb->execute(profile_file); + + compute_entropy(gb, logits_node_id, out_entropy); + + post_execute_updates(gb, tokens.size()); + update_kv_cache(gb, tokens.size()); + + auto* output_ptr = gb->get_output(sampled_token_id); + uint32_t result_token = *static_cast(output_ptr); + record_sampled_token(result_token); + return result_token; +} + +size_t Model::sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, + float min_p, float repetition_penalty, + const std::unordered_map* extra_bias) const { + auto combined_bias = tool_constrainer_.get_bias(); + for (const auto& [token_id, boost] : vocab_bias_) { + combined_bias[token_id] += boost; + } + if (extra_bias) { + for (const auto& [token_id, boost] : *extra_bias) { + combined_bias[token_id] += boost; + } + } + if (!token_history_.empty() && repetition_penalty > 1.0f && std::isfinite(repetition_penalty)) { + float log_penalty = std::log(repetition_penalty); + for (uint32_t tok : token_history_) { + combined_bias[tok] -= log_penalty; + } + } + return gb->sample_with_options(logits_node_id, temperature, top_p, min_p, 1.0f, top_k, combined_bias); +} + +void Model::compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy) { + if (!out_entropy) return; + + const auto& logits_buf = gb->get_output_buffer(logits_node_id); + void* logits_ptr = gb->get_output(logits_node_id); + size_t vocab_size = logits_buf.shape.back(); + size_t seq_len = 1; + if (logits_buf.shape.size() >= 2) + seq_len = logits_buf.shape[logits_buf.shape.size() - 2]; + size_t row_offset = (seq_len > 0 ? (seq_len - 1) * vocab_size : 0); + + std::vector logits(vocab_size); + if (logits_buf.precision == Precision::FP32) { + float* src = static_cast(logits_ptr) + row_offset; + std::copy(src, src + vocab_size, logits.begin()); + } else if (logits_buf.precision == Precision::FP16) { + __fp16* src = static_cast<__fp16*>(logits_ptr) + row_offset; + Quantization::fp16_to_fp32(src, logits.data(), vocab_size); + } else { + int8_t* src = static_cast(logits_ptr) + row_offset; + Quantization::int8_to_fp32(src, logits.data(), vocab_size, 1.0f); + } + + float max_logit = *std::max_element(logits.begin(), logits.end()); + double sum_exp = 0.0; + for (size_t i = 0; i < vocab_size; ++i) + sum_exp += std::exp(static_cast(logits[i] - max_logit)); + double log_sum_exp = static_cast(max_logit) + std::log(sum_exp); + + double entropy = 0.0; + for (size_t i = 0; i < vocab_size; ++i) { + double log_prob = static_cast(logits[i]) - log_sum_exp; + double prob = std::exp(log_prob); + if (prob > 1e-10) + entropy -= prob * log_prob; + } + + double max_entropy = std::log(static_cast(vocab_size)); + *out_entropy = static_cast(entropy / max_entropy); +} + +uint32_t Model::decode_with_audio(const std::vector& tokens, const std::vector& /*mel_bins*/, float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, + float min_p, float repetition_penalty, + float* /*out_token_time_start*/, float* /*out_token_time_end*/){ + return decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); +} + +uint32_t Model::decode_with_images(const std::vector& tokens, const std::vector& image_paths, + float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, + float min_p, float repetition_penalty) { + (void)image_paths; + return decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); +} + +std::vector Model::get_image_embeddings(const std::string& /*image_path*/) { + throw std::runtime_error("Image embeddings not supported for this model type"); +} + +std::vector Model::get_audio_embeddings(const std::vector& /*mel_bins*/) { + throw std::runtime_error("Audio embeddings not supported for this model type"); +} + +void Model::update_kv_cache(CactusGraph* gb, size_t seq_len) { + kv_cache_.update_from_graph(gb, cache_k_output_nodes_, cache_v_output_nodes_, + seq_len, config_.num_layers); +} + +void Model::remove_thinking_tokens(const std::vector>& ranges) { + for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) + kv_cache_.remove_token_range(it->first, it->second); +} + +std::vector Model::get_embeddings(const std::vector& tokens, bool pooled, bool normalize, const std::string& profile_file) { + std::vector embeddings; + auto final_hidden = forward(tokens); + + auto* gb = static_cast(graph_handle_); + auto* output_ptr = gb->get_output(final_hidden); + const auto& output_buffer = gb->get_output_buffer(final_hidden); + + if (pooled) { + auto pooled_hidden = gb->mean(final_hidden, 0); + + if (!profile_file.empty()) { + gb->execute(profile_file); + } else { + gb->execute(); + } + post_execute_updates(gb, tokens.size()); + auto* pooled_ptr = gb->get_output(pooled_hidden); + const auto& pooled_buffer = gb->get_output_buffer(pooled_hidden); + + size_t hidden_dim = pooled_buffer.total_size; + embeddings.resize(hidden_dim); + + if (pooled_buffer.precision == Precision::FP32) { + float* pooled_data = static_cast(pooled_ptr); + std::copy(pooled_data, pooled_data + hidden_dim, embeddings.begin()); + } else if (pooled_buffer.precision == Precision::FP16) { + __fp16* pooled_data = static_cast<__fp16*>(pooled_ptr); + Quantization::fp16_to_fp32(pooled_data, embeddings.data(), hidden_dim); + } else if (pooled_buffer.precision == Precision::INT8) { + int8_t* pooled_data = static_cast(pooled_ptr); + Quantization::int8_to_fp32(pooled_data, embeddings.data(), hidden_dim, 1.0f); + } + } else { + if (!profile_file.empty()) { + gb->execute(profile_file); + } else { + gb->execute(); + } + post_execute_updates(gb, tokens.size()); + + size_t total_size = output_buffer.total_size; + embeddings.resize(total_size); + + if (output_buffer.precision == Precision::FP32) { + float* hidden_states = static_cast(output_ptr); + std::copy(hidden_states, hidden_states + total_size, embeddings.begin()); + } else if (output_buffer.precision == Precision::FP16) { + __fp16* hidden_states = static_cast<__fp16*>(output_ptr); + for (size_t i = 0; i < total_size; i++) { + embeddings[i] = static_cast(hidden_states[i]); + } + } else if (output_buffer.precision == Precision::INT8) { + int8_t* hidden_states = static_cast(output_ptr); + for (size_t i = 0; i < total_size; i++) { + embeddings[i] = static_cast(hidden_states[i]); + } + } + } + + if (normalize && !embeddings.empty()) { + float norm_sq = 0.0f; + for (float v : embeddings) { + norm_sq += v * v; + } + float norm = std::sqrt(norm_sq); + if (norm > 1e-12f) { + float inv_norm = 1.0f / norm; + for (float& v : embeddings) { + v *= inv_norm; + } + } + } + + kv_cache_.reset(); + + return embeddings; +} + +bool Config::from_json(const std::string& config_path) { + std::ifstream file(config_path); + if (!file) { + CACTUS_LOG_ERROR("config", "Failed to open config file: " << config_path); + return false; + } + + std::string line; + while (std::getline(file, line)) { + if (line.empty() || line[0] == '#') continue; + + size_t eq_pos = line.find('='); + if (eq_pos == std::string::npos) continue; + + std::string key = line.substr(0, eq_pos); + std::string value = line.substr(eq_pos + 1); + + key.erase(0, key.find_first_not_of(" \t")); + key.erase(key.find_last_not_of(" \t") + 1); + value.erase(0, value.find_first_not_of(" \t")); + value.erase(value.find_last_not_of(" \t") + 1); + + if (key == "vocab_size") vocab_size = static_cast(std::stoul(value)); + else if (key == "bos_token_id") bos_token_id = static_cast(std::stoul(value)); + else if (key == "eos_token_id") eos_token_id = static_cast(std::stoul(value)); + else if (key == "num_layers") num_layers = static_cast(std::stoul(value)); + else if (key == "hidden_dim") hidden_dim = static_cast(std::stoul(value)); + else if (key == "ffn_intermediate_dim") ffn_intermediate_dim = static_cast(std::stoul(value)); + else if (key == "attention_heads") attention_heads = static_cast(std::stoul(value)); + else if (key == "attention_kv_heads") attention_kv_heads = static_cast(std::stoul(value)); + else if (key == "attention_head_dim") attention_head_dim = static_cast(std::stoul(value)); + else if (key == "layer_norm_eps") layer_norm_eps = std::stof(value); + else if (key == "rope_theta") rope_theta = std::stof(value); + else if (key == "num_experts") num_experts = static_cast(std::stoul(value)); + else if (key == "num_shared_experts") num_shared_experts = static_cast(std::stoul(value)); + else if (key == "num_top_experts") num_top_experts = static_cast(std::stoul(value)); + else if (key == "moe_every_n_layers") moe_every_n_layers = static_cast(std::stoul(value)); + else if (key == "moe_intermediate_dim" || key == "moe_intermediate_size") moe_intermediate_dim = static_cast(std::stoul(value)); + else if (key == "num_dense_layers") num_dense_layers = static_cast(std::stoul(value)); + else if (key == "num_experts_per_tok") num_experts_per_tok = static_cast(std::stoul(value)); + else if (key == "norm_topk_prob") norm_topk_prob = (value == "true" || value == "1"); + else if (key == "use_expert_bias") use_expert_bias = (value == "true" || value == "1"); + else if (key == "routed_scaling_factor") routed_scaling_factor = std::stof(value); + else if (key == "tie_word_embeddings") tie_word_embeddings = (value == "true" || value == "1"); + else if (key == "vision_hidden_dim" || key == "vision_hidden_size") vision_hidden_dim = static_cast(std::stoul(value)); + else if (key == "vision_num_layers") vision_num_layers = static_cast(std::stoul(value)); + else if (key == "vision_attention_heads") vision_attention_heads = static_cast(std::stoul(value)); + else if (key == "vision_image_size") vision_image_size = static_cast(std::stoul(value)); + else if (key == "vision_patch_size") vision_patch_size = static_cast(std::stoul(value)); + else if (key == "vision_num_channels") vision_num_channels = static_cast(std::stoul(value)); + else if (key == "vision_embed_dim") vision_embed_dim = static_cast(std::stoul(value)); + else if (key == "visual_tokens_per_img") visual_tokens_per_img = static_cast(std::stoul(value)); + else if (key == "use_pixel_shuffle") use_pixel_shuffle = (value == "true" || value == "1"); + else if (key == "pixel_shuffle_factor") pixel_shuffle_factor = static_cast(std::stoul(value)); + else if (key == "use_image_tokens") use_image_tokens = (value == "true" || value == "1"); + else if (key == "image_token_id") image_token_id = static_cast(std::stoul(value)); + else if (key == "use_layout_tags") use_layout_tags = (value == "true" || value == "1"); + else if (key == "image_seq_len") image_seq_len = static_cast(std::stoul(value)); + else if (key == "global_image_size") global_image_size = static_cast(std::stoul(value)); + else if (key == "max_tile_size") max_tile_size = static_cast(std::stoul(value)); + else if (key == "rescale_factor") rescale_factor = std::stof(value); + else if (key == "image_mean") image_mean = std::stof(value); + else if (key == "image_std") image_std = std::stof(value); + else if (key == "downsample_factor") downsample_factor = static_cast(std::stoul(value)); + else if (key == "min_tiles") min_tiles = static_cast(std::stoul(value)); + else if (key == "max_tiles") max_tiles = static_cast(std::stoul(value)); + else if (key == "use_thumbnail") use_thumbnail = (value == "true" || value == "1"); + else if (key == "min_image_tokens") min_image_tokens = static_cast(std::stoul(value)); + else if (key == "max_image_tokens") max_image_tokens = static_cast(std::stoul(value)); + else if (key == "tile_size") tile_size = static_cast(std::stoul(value)); + else if (key == "max_pixels_tolerance") max_pixels_tolerance = std::stof(value); + else if (key == "do_image_splitting") do_image_splitting = (value == "true" || value == "1"); + else if (key == "precision") { + if (value == "INT8") precision = Precision::INT8; + else if (value == "FP16") precision = Precision::FP16; + else precision = Precision::FP32; + } + else if (key == "model_type") { + std::string model_type_value = value; + std::transform(model_type_value.begin(), model_type_value.end(), model_type_value.begin(), ::tolower); + if (value == "gemma" || value == "GEMMA") model_type = ModelType::GEMMA; + else if (value == "lfm2" || value == "LFM2" || value == "lfm2_moe" || value == "LFM2_MOE") model_type = ModelType::LFM2; + else if (value == "bert" || value == "BERT") model_type = ModelType::NOMIC; + else if (value == "whisper" || value == "WHISPER") model_type = ModelType::WHISPER; + else if (value == "moonshine" || value == "MOONSHINE") model_type = ModelType::MOONSHINE; + else if (value == "silero_vad" || value == "SILERO_VAD") model_type = ModelType::SILERO_VAD; + else if (value == "parakeet" || value == "PARAKEET") model_type = ModelType::PARAKEET; + else if (model_type_value.rfind("qwen3_5", 0) == 0) model_type = ModelType::QWEN3P5; + else if (value == "parakeet_tdt" || value == "PARAKEET_TDT") model_type = ModelType::PARAKEET_TDT; + else if (value == "gemma3n" || value == "GEMMA3N") model_type = ModelType::GEMMA3N; + else if (value == "needle" || value == "NEEDLE") model_type = ModelType::NEEDLE; + else if (value == "gemma4" || value == "GEMMA4" || value == "tinyllama" || value == "TINYLLAMA") model_type = ModelType::GEMMA4; + else if (value == "youtu" || value == "YOUTU") model_type = ModelType::YOUTU; + else if (value == "pyannote" || value == "PYANNOTE") model_type = ModelType::PYANNOTE; + else if (value == "wespeaker" || value == "WESPEAKER") model_type = ModelType::WESPEAKER; + else model_type = ModelType::QWEN; + } + else if (key == "model_variant") { + std::string v = value; + std::transform(v.begin(), v.end(), v.begin(), ::tolower); + if (v == "vlm") model_variant = ModelVariant::VLM; + else if (v == "extract") model_variant = ModelVariant::EXTRACT; + else if (v == "rag") model_variant = ModelVariant::RAG; + else model_variant = ModelVariant::DEFAULT; + } + else if (key == "conv_L_cache") conv_L_cache = static_cast(std::stoul(value)); + else if (key == "layer_types") { + layer_types.clear(); + std::string sanitized; + sanitized.reserve(value.size()); + for (char c : value) { + if (c == '[' || c == ']' || c == '\'' || c == '"') { + continue; + } + sanitized.push_back(c); + } + std::stringstream ss(sanitized); + std::string item; + while (std::getline(ss, item, ',')) { + if (!item.empty()) { + item.erase(0, item.find_first_not_of(" \t")); + item.erase(item.find_last_not_of(" \t") + 1); + if (!item.empty()) layer_types.push_back(item); + } + } + } + else if (key == "enc_hidden_act") encoder_act_gelu = (value == "gelu"); + else if (key == "dec_hidden_act") decoder_act_gelu = (value == "gelu"); + else if (key == "num_encoder_layers") num_encoder_layers = static_cast(std::stoul(value)); + else if (key == "num_decoder_layers") num_decoder_layers = static_cast(std::stoul(value)); + else if (key == "partial_rotary_factor") partial_rotary_factor = std::stof(value); + else if (key == "pad_token_id") pad_token_id = static_cast(std::stoul(value)); + else if (key == "conv_kernel_size") conv_kernel_size = static_cast(std::stoul(value)); + else if (key == "subsampling_conv_kernel_size") subsampling_conv_kernel_size = static_cast(std::stoul(value)); + else if (key == "subsampling_conv_stride") subsampling_conv_stride = static_cast(std::stoul(value)); + else if (key == "subsampling_conv_channels") subsampling_conv_channels = static_cast(std::stoul(value)); + else if (key == "subsampling_factor") subsampling_factor = static_cast(std::stoul(value)); + else if (key == "num_mel_bins") num_mel_bins = static_cast(std::stoul(value)); + else if (key == "encoder_hidden_act") encoder_hidden_act = value; + else if (key == "linear_num_key_heads") linear_num_key_heads = static_cast(std::stoul(value)); + else if (key == "linear_key_head_dim") linear_key_head_dim = static_cast(std::stoul(value)); + else if (key == "linear_num_value_heads") linear_num_value_heads = static_cast(std::stoul(value)); + else if (key == "linear_value_head_dim") linear_value_head_dim = static_cast(std::stoul(value)); + else if (key == "linear_q_proj_dim") linear_q_proj_dim = static_cast(std::stoul(value)); + else if (key == "kv_lora_rank") kv_lora_rank = static_cast(std::stoul(value)); + else if (key == "q_lora_rank") q_lora_rank = static_cast(std::stoul(value)); + else if (key == "qk_head_dim") qk_head_dim = static_cast(std::stoul(value)); + else if (key == "qk_nope_head_dim") qk_nope_head_dim = static_cast(std::stoul(value)); + else if (key == "qk_rope_head_dim") qk_rope_head_dim = static_cast(std::stoul(value)); + else if (key == "v_head_dim") v_head_dim = static_cast(std::stoul(value)); + else if (key == "rope_interleave") rope_interleave = (value == "true" || value == "1"); + else if (key == "attention_bias") attention_bias = (value == "true" || value == "1"); + else if (key == "rope_scaling_factor") rope_scaling_factor = std::stof(value); + else if (key == "rope_mscale_all_dim") rope_mscale_all_dim = std::stof(value); + else if (key == "linear_k_proj_dim") linear_k_proj_dim = static_cast(std::stoul(value)); + else if (key == "linear_v_proj_dim") linear_v_proj_dim = static_cast(std::stoul(value)); + else if (key == "predictor_hidden_dim") predictor_hidden_dim = static_cast(std::stoul(value)); + else if (key == "predictor_num_layers") predictor_num_layers = static_cast(std::stoul(value)); + else if (key == "tdt_joint_dim") tdt_joint_dim = static_cast(std::stoul(value)); + else if (key == "tdt_num_durations") tdt_num_durations = static_cast(std::stoul(value)); + else if (key == "tdt_blank_id") tdt_blank_id = static_cast(std::stoul(value)); + else if (key == "tdt_durations") { + tdt_durations.clear(); + std::stringstream ss(value); + std::string item; + while (std::getline(ss, item, ',')) { + size_t first = item.find_first_not_of(" \t"); + if (first == std::string::npos) continue; + size_t last = item.find_last_not_of(" \t"); + item = item.substr(first, last - first + 1); + tdt_durations.push_back(static_cast(std::stoul(item))); + } + } + else if (key == "altup_num_inputs") altup_num_inputs = static_cast(std::stoul(value)); + else if (key == "laurel_rank") laurel_rank = static_cast(std::stoul(value)); + else if (key == "hidden_size_per_layer_input") hidden_size_per_layer_input = static_cast(std::stoul(value)); + else if (key == "num_kv_shared_layers") num_kv_shared_layers = static_cast(std::stoul(value)); + else if (key == "sliding_window") sliding_window = static_cast(std::stoul(value)); + else if (key == "rope_local_base_freq") rope_local_base_freq = std::stof(value); + else if (key == "final_logit_softcapping") final_logit_softcapping = std::stof(value); + else if (key == "global_partial_rotary_factor") global_partial_rotary_factor = std::stof(value); + else if (key == "expert_intermediate_size") expert_intermediate_size = static_cast(std::stoul(value)); + else if (key == "global_head_dim") global_head_dim = static_cast(std::stoul(value)); + else if (key == "num_global_kv_heads" || key == "num_global_key_value_heads") num_global_kv_heads = static_cast(std::stoul(value)); + else if (key == "attention_k_eq_v") attention_k_eq_v = (value == "true" || value == "1"); + else if (key == "enable_moe_block") enable_moe_block = (value == "true" || value == "1"); + else if (key == "vision_head_dim") vision_head_dim = static_cast(std::stoul(value)); + else if (key == "vision_kv_heads") vision_kv_heads = static_cast(std::stoul(value)); + else if (key == "vision_intermediate_size") vision_intermediate_size = static_cast(std::stoul(value)); + else if (key == "vision_position_embedding_size") vision_position_embedding_size = static_cast(std::stoul(value)); + else if (key == "vision_pooling_kernel_size") vision_pooling_kernel_size = static_cast(std::stoul(value)); + else if (key == "vision_default_output_length") vision_default_output_length = static_cast(std::stoul(value)); + else if (key == "vision_rope_theta") vision_rope_theta = std::stof(value); + else if (key == "audio_hidden_dim") audio_hidden_dim = static_cast(std::stoul(value)); + else if (key == "audio_num_layers") audio_num_layers = static_cast(std::stoul(value)); + else if (key == "audio_num_heads") audio_num_heads = static_cast(std::stoul(value)); + else if (key == "audio_head_dim") audio_head_dim = static_cast(std::stoul(value)); + else if (key == "audio_input_feat_size") audio_input_feat_size = static_cast(std::stoul(value)); + else if (key == "audio_conf_conv_kernel_size") audio_conf_conv_kernel_size = static_cast(std::stoul(value)); + else if (key == "audio_chunk_size") audio_chunk_size = static_cast(std::stoul(value)); + else if (key == "audio_context_left") audio_context_left = static_cast(std::stoul(value)); + else if (key == "audio_context_right") audio_context_right = static_cast(std::stoul(value)); + else if (key == "audio_logit_cap") audio_logit_cap = std::stof(value); + else if (key == "audio_residual_weight") audio_residual_weight = std::stof(value); + else if (key == "audio_output_proj_dims") audio_output_proj_dims = static_cast(std::stoul(value)); + else if (key == "audio_vocab_size") audio_vocab_size = static_cast(std::stoul(value)); + else if (key == "audio_vocab_offset") audio_vocab_offset = static_cast(std::stoul(value)); + else if (key == "audio_soft_tokens") audio_soft_tokens = static_cast(std::stoul(value)); + else if (key == "audio_sscp_conv0_channels") audio_sscp_conv0_channels = static_cast(std::stoul(value)); + else if (key == "audio_sscp_conv1_channels") audio_sscp_conv1_channels = static_cast(std::stoul(value)); + else if (key == "audio_sscp_conv_eps") audio_sscp_conv_eps = std::stof(value); + else if (key == "audio_rms_norm_eps") audio_rms_norm_eps = std::stof(value); + else if (key == "audio_fft_length") audio_fft_length = static_cast(std::stoul(value)); + else if (key == "audio_fft_overdrive") { + audio_fft_overdrive = (value == "true" || value == "1"); + audio_fft_length = audio_fft_overdrive ? 1024u : 512u; + } + else if (key == "audio_token_id") audio_token_id = static_cast(std::stoul(value)); + else if (key == "channel_open_token_id") channel_open_token_id = static_cast(std::stoul(value)); + else if (key == "channel_close_token_id") channel_close_token_id = static_cast(std::stoul(value)); + else if (key == "activation_sparsity_ppf") { + activation_sparsity_ppf.clear(); + std::stringstream ss(value); + std::string item; + while (std::getline(ss, item, ',')) { + size_t first = item.find_first_not_of(" \t"); + if (first == std::string::npos) continue; + size_t last = item.find_last_not_of(" \t"); + item = item.substr(first, last - first + 1); + activation_sparsity_ppf.push_back(std::stof(item)); + } + } + } + + if (is_gemma_family(model_type)) { + default_temperature = 1.0f; + default_top_p = 0.95f; + default_top_k = 64; + } else if (model_type == ModelType::LFM2) { + default_temperature = 0.3f; + default_top_p = 0.95f; + default_top_k = 20; + } else if (model_type == ModelType::QWEN) { + default_temperature = 0.6f; + default_top_p = 0.95f; + default_top_k = 20; + } else if (model_type == ModelType::QWEN3P5) { + default_temperature = 0.7f; + default_top_p = 0.8f; + default_top_k = 20; + } else if (model_type == ModelType::WHISPER) { + default_temperature = 0.0f; + default_top_p = 0.0f; + default_top_k = 0; + default_cloud_handoff_threshold = 0.4f; + } else if (model_type == ModelType::MOONSHINE) { + default_temperature = 0.0f; + default_top_p = 0.0f; + default_top_k = 0; + default_max_tps = 6.5f; + default_cloud_handoff_threshold = 0.35f; + } else if (model_type == ModelType::PARAKEET) { + default_temperature = 0.0f; + default_top_p = 0.0f; + default_top_k = 0; + default_max_tps = 8.0f; + default_cloud_handoff_threshold = 0.35f; + } else if (model_type == ModelType::PARAKEET_TDT) { + default_temperature = 0.0f; + default_top_p = 0.0f; + default_top_k = 0; + default_max_tps = 8.0f; + default_cloud_handoff_threshold = 0.35f; + } else if (model_type == ModelType::NEEDLE) { + default_temperature = 0.0f; + default_top_p = 0.0f; + default_top_k = 0; + } else if (model_type == ModelType::YOUTU) { + default_temperature = 1.0f; + default_top_p = 0.95f; + default_top_k = 20; + } + + return true; +} + +std::string Config::to_json() const { + return "{}"; +} + +std::unique_ptr create_model(const std::string& model_folder) { + CACTUS_LOG_DEBUG("model", "Creating model from: " << model_folder); + Config config; + std::string config_path = model_folder + "/config.txt"; + + if (!config.from_json(config_path)) { + CACTUS_LOG_ERROR("model", "Failed to create model - cannot load config from: " << model_folder); + return nullptr; + } + + const bool has_vision_support = + config.use_image_tokens || + config.vision_num_layers > 0 || + config.vision_embed_dim > 0 || + config.vision_hidden_dim > 0 || + config.visual_tokens_per_img > 0; + + if (config.model_type == Config::ModelType::LFM2 && has_vision_support) { + return std::make_unique(config); + } + + const bool has_audio_support = + config.audio_num_layers > 0 || + config.audio_hidden_dim > 0; + + if (config.model_type == Config::ModelType::GEMMA4 && (has_vision_support || has_audio_support)) { + return std::make_unique(config); + } + + switch (config.model_type) { + case Config::ModelType::QWEN: + return std::make_unique(config); + case Config::ModelType::QWEN3P5: + return std::make_unique(config); + case Config::ModelType::GEMMA: + return std::make_unique(config); + case Config::ModelType::GEMMA3N: + return std::make_unique(config); + case Config::ModelType::LFM2: + if (config.num_experts > 0 && config.moe_intermediate_dim > 0 && config.num_experts_per_tok > 0) { + return std::make_unique(config); + } + return std::make_unique(config); + case Config::ModelType::NOMIC: + return std::make_unique(config); + case Config::ModelType::WHISPER: + return std::make_unique(config); + case Config::ModelType::MOONSHINE: + return std::make_unique(config); + case Config::ModelType::SILERO_VAD: + return std::make_unique(config); + case Config::ModelType::PARAKEET: + return std::make_unique(config); + case Config::ModelType::PARAKEET_TDT: + return std::make_unique(config); + case Config::ModelType::NEEDLE: + return std::make_unique(config); + case Config::ModelType::GEMMA4: + return std::make_unique(config); + case Config::ModelType::YOUTU: + return std::make_unique(config); + case Config::ModelType::PYANNOTE: + return std::make_unique(config); + case Config::ModelType::WESPEAKER: + return std::make_unique(config); + default: + return std::make_unique(config); + } +} + +void Model::capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const { + auto* graph = static_cast(graph_handle_); + if (!graph) { + return; + } + graph->capture_debug_node(layer_idx, name, node_id); +} + +void Model::clear_debug_nodes() { + auto* graph = static_cast(graph_handle_); + if (!graph) { + return; + } + graph->clear_debug_nodes(); +} + +const std::vector& Model::get_debug_nodes() const { + auto* graph = static_cast(graph_handle_); + debug_nodes_.clear(); + if (!graph) { + return debug_nodes_; + } + + const auto& entries = graph->get_debug_nodes(); + debug_nodes_.reserve(entries.size()); + for (const auto& entry : entries) { + debug_nodes_.push_back({entry.layer_idx, entry.name, entry.node_id}); + } + return debug_nodes_; +} + +bool Model::load_npu_prefill(const std::string& model_path) { + CACTUS_LOG_DEBUG("npu", "Attempting to load NPU prefill from: " << model_path); + + npu_prefill_ = npu::create_prefill(); + if (!npu_prefill_) { + CACTUS_LOG_DEBUG("npu", "NPU prefill creation failed (not supported on this device)"); + return false; + } + + bool loaded = npu_prefill_->load(model_path); + if (loaded) { + CACTUS_LOG_INFO("npu", "NPU prefill loaded successfully from: " << model_path); + } else { + CACTUS_LOG_DEBUG("npu", "NPU prefill model not found at: " << model_path); + } + return loaded; +} + +bool Model::has_npu_prefill() const { + return npu_prefill_ && npu_prefill_->is_available(); +} + +size_t Model::get_prefill_chunk_size() const { + if (has_npu_prefill()) { + return static_cast(npu_prefill_->get_chunk_size()); + } + return 256; // default chunk size +} + +std::vector<__fp16> Model::get_token_embeddings(const std::vector& tokens) { + auto* gb = static_cast(graph_handle_); + if (!gb || tokens.empty()) { + return {}; + } + + gb->soft_reset(); + + size_t tok_input = gb->input({tokens.size()}, Precision::FP32); + std::vector tok_f(tokens.size()); + for (size_t i = 0; i < tokens.size(); i++) { + tok_f[i] = static_cast(tokens[i]); + } + gb->set_input(tok_input, tok_f.data(), Precision::FP32); + + size_t embedding_node = gb->embedding(embedding_node_id_, tok_input); + + gb->execute(); + + const auto& emb_buf = gb->get_output_buffer(embedding_node); + void* emb_ptr = gb->get_output(embedding_node); + + size_t num_tokens = tokens.size(); + size_t hidden_dim = config_.hidden_dim; + std::vector<__fp16> embeddings(num_tokens * hidden_dim); + + if (emb_buf.precision == Precision::FP16) { + __fp16* src = static_cast<__fp16*>(emb_ptr); + std::copy(src, src + num_tokens * hidden_dim, embeddings.begin()); + } else if (emb_buf.precision == Precision::FP32) { + float* src = static_cast(emb_ptr); + for (size_t i = 0; i < num_tokens * hidden_dim; i++) { + embeddings[i] = static_cast<__fp16>(src[i]); + } + } else if (emb_buf.precision == Precision::INT8) { + int8_t* src = static_cast(emb_ptr); + for (size_t i = 0; i < num_tokens * hidden_dim; i++) { + embeddings[i] = static_cast<__fp16>(src[i]); + } + } + + return embeddings; +} + +void Model::prefill_npu(const std::vector& tokens) { + if (!npu_prefill_ || !npu_prefill_->is_available()) { + throw std::runtime_error("NPU prefill not available"); + } + + const int chunk_size = npu_prefill_->get_chunk_size(); + const int hidden_dim = npu_prefill_->get_hidden_dim(); + const int num_layers = npu_prefill_->get_num_layers(); + const int fallback_num_kv_heads = npu_prefill_->get_num_kv_heads(); + const int fallback_head_dim = npu_prefill_->get_head_dim(); + + const std::vector layer_dims = get_kv_layer_dims(); + const std::vector layer_heads = get_kv_layer_heads(); + const int layers_to_update = std::min(num_layers, static_cast(config_.num_layers)); + + std::vector<__fp16> all_embeddings = get_token_embeddings(tokens); + if (all_embeddings.empty()) { + throw std::runtime_error("Failed to get token embeddings for NPU prefill"); + } + + if (Config::is_gemma_family(config_.model_type)) { + float scale = std::sqrt(static_cast(hidden_dim)); + for (size_t i = 0; i < all_embeddings.size(); i++) { + all_embeddings[i] = __fp16(static_cast(all_embeddings[i]) * scale); + } + } + + size_t num_tokens = tokens.size(); + size_t num_chunks = (num_tokens + chunk_size - 1) / chunk_size; + + for (size_t c = 0; c < num_chunks; c++) { + size_t start = c * chunk_size; + size_t actual_tokens = std::min(static_cast(chunk_size), num_tokens - start); + + std::vector<__fp16> chunk_embeddings(chunk_size * hidden_dim, __fp16(0)); + std::copy(all_embeddings.begin() + start * hidden_dim, + all_embeddings.begin() + (start + actual_tokens) * hidden_dim, + chunk_embeddings.begin()); + + int position_offset = static_cast(start); + + npu::NPUPrefillDirectResult direct_result = npu_prefill_->prefill_chunk_direct(chunk_embeddings, position_offset); + + if (direct_result.valid) { + for (int layer_idx = 0; layer_idx < layers_to_update; layer_idx++) { + const auto& k_ref = direct_result.k_caches[layer_idx]; + const auto& v_ref = direct_result.v_caches[layer_idx]; + + if (k_ref.data && v_ref.data) { + size_t layer_kv_heads = layer_idx < static_cast(layer_heads.size()) + ? layer_heads[layer_idx] + : static_cast(fallback_num_kv_heads); + size_t layer_head_dim = layer_idx < static_cast(layer_dims.size()) + ? layer_dims[layer_idx] + : static_cast(fallback_head_dim); + + size_t expected = static_cast(chunk_size) * layer_kv_heads * layer_head_dim; + if (expected > 0 && (k_ref.count < expected || v_ref.count < expected)) { + CACTUS_LOG_WARN( + "npu", + "NPU prefill cache output too small for layer " << layer_idx + << " (expected>=" << expected + << ", got k=" << k_ref.count << ", v=" << v_ref.count << "); skipping layer"); + continue; + } + + kv_cache_.update_from_npu(layer_idx, k_ref.data, v_ref.data, + actual_tokens, layer_kv_heads, layer_head_dim); + } + } + } + } +} + +double Model::score_tokens_window_logprob( + const std::vector& tokens, + size_t start, + size_t end, + size_t context, + size_t* tokens_scored +) { + if (tokens_scored) + *tokens_scored = 0; + + if (tokens.empty()) + return 0.0; + + if (end > tokens.size()) + end = tokens.size(); + + if (start >= end) + return 0.0; + + if (start == 0) + start = 1; + + if (start >= end) + return 0.0; + + const size_t target_len = end - start; + const size_t ctx_begin = (start > context) ? (start - context) : 0; + + if (end < 2) return 0.0; + const size_t input_end = end - 1; + + if (input_end <= ctx_begin) + return 0.0; + + std::vector input_tokens(tokens.begin() + ctx_begin,tokens.begin() + input_end); + + if (tokens_scored) + *tokens_scored = target_len; + + reset_cache(); + + auto* gb = static_cast(graph_handle_); + const auto backend = (config_.default_backend == Config::Backend::CPU) ? ComputeBackend::CPU : ComputeBackend::NPU; + + const size_t hidden_node = forward(input_tokens, /*use_cache=*/false); + const auto& hidden_buf = gb->get_output_buffer(hidden_node); + + if (hidden_buf.shape.size() != 2) { + throw std::runtime_error("Expected hidden to be rank-2 [L, hidden_dim]"); + } + + + const size_t first_pos = start - ctx_begin - 1; + const size_t hidden_slice = gb->slice(hidden_node, /*axis=*/0, first_pos, target_len); + bool transpose_w = true; + const size_t logits_node = gb->matmul(hidden_slice, output_weight_node_id_, transpose_w, backend); + gb->execute(); + + const auto& logits_buf = gb->get_output_buffer(logits_node); + if (logits_buf.shape.size() != 2) + throw std::runtime_error("Expected logits to be rank-2 [T, vocab]"); + + const size_t T = logits_buf.shape[0]; + const size_t vocab_size = logits_buf.shape[1]; + + if (T != target_len) + throw std::runtime_error("Logits T dimension does not match target_len"); + + void* logits_ptr = gb->get_output(logits_node); + std::vector row(vocab_size); + double total_logprob = 0.0; + + for (size_t i = 0; i < target_len; ++i) { + const uint32_t y = tokens[start + i]; + if (y >= vocab_size) + throw std::runtime_error("Target token out of vocab range"); + + if (logits_buf.precision == Precision::FP32) { + const float* src = static_cast(logits_ptr) + i * vocab_size; + std::memcpy(row.data(), src, vocab_size * sizeof(float)); + } + else if (logits_buf.precision == Precision::FP16) { + const __fp16* src = static_cast(logits_ptr) + i * vocab_size; + Quantization::fp16_to_fp32(const_cast<__fp16*>(src), row.data(), vocab_size); + } + else { + const int8_t* src = static_cast(logits_ptr) + i * vocab_size; + Quantization::int8_to_fp32(const_cast(src), row.data(), vocab_size, 1.0f); + } + + float max_logit = *std::max_element(row.begin(), row.end()); + double sum = 0.0; + + for (size_t j = 0; j < vocab_size; ++j) + sum += std::exp(double(row[j] - max_logit)); + + const double lse = double(max_logit) + std::log(sum); + total_logprob += double(row[y]) - lse; + } + + return total_logprob; +} +} +} diff --git a/cactus-transpile/engine/engine_signals.cpp b/cactus-transpile/engine/engine_signals.cpp new file mode 100644 index 000000000..38468b4c0 --- /dev/null +++ b/cactus-transpile/engine/engine_signals.cpp @@ -0,0 +1,775 @@ +#include "engine.h" +#include "../kernel/kernel_utils.h" +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +namespace cactus { +namespace engine { + +static void to_db( + float* spectrogram, + size_t size, + float reference, + float min_value, + const float* db_range, + float multiplier) +{ + if (reference <= 0.0f) { + throw std::invalid_argument("reference must be greater than zero"); + } + if (min_value <= 0.0f) { + throw std::invalid_argument("min_value must be greater than zero"); + } + + reference = std::max(min_value, reference); + const float log_ref = std::log10(reference); + + CactusThreading::parallel_for(size, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { + for (size_t i = start; i < end; i++) { + float value = std::max(min_value, spectrogram[i]); + spectrogram[i] = multiplier * (std::log10(value) - log_ref); + } + }); + + if (db_range != nullptr) { + if (*db_range <= 0.0f) { + throw std::invalid_argument("db_range must be greater than zero"); + } + + float max_db = CactusThreading::parallel_reduce, float, std::function>( + size, CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) { + float local_max = -std::numeric_limits::infinity(); + for (size_t i = start; i < end; i++) { + local_max = std::max(local_max, spectrogram[i]); + } + return local_max; + }, + -std::numeric_limits::infinity(), + [](float a, float b) { return std::max(a, b); } + ); + + float min_db = max_db - *db_range; + CactusThreading::parallel_for(size, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { + for (size_t i = start; i < end; i++) { + spectrogram[i] = std::max(min_db, spectrogram[i]); + } + }); + } +} + +static size_t bit_reverse(size_t x, size_t log2n) { + size_t result = 0; + for (size_t i = 0; i < log2n; i++) { + result = (result << 1) | (x & 1); + x >>= 1; + } + return result; +} + +static bool is_power_of_two(size_t n) { + return n != 0 && (n & (n - 1)) == 0; +} + +static size_t log2_power_of_two(size_t n) { + size_t log2n = 0; + for (size_t temp = n; temp > 1; temp >>= 1) { + log2n++; + } + return log2n; +} + +static void fft_radix2(float* re, float* im, size_t n, bool inverse) { + if (!is_power_of_two(n)) return; + + const size_t log2n = log2_power_of_two(n); + + for (size_t i = 0; i < n; i++) { + size_t j = bit_reverse(i, log2n); + if (i < j) { + std::swap(re[i], re[j]); + std::swap(im[i], im[j]); + } + } + + for (size_t s = 1; s <= log2n; s++) { + size_t m = size_t{1} << s; + size_t m2 = m >> 1; + float w_re = 1.0f; + float w_im = 0.0f; + const float angle = static_cast(M_PI) / static_cast(m2); + const float wm_re = std::cos(angle); + const float wm_im = inverse ? std::sin(angle) : -std::sin(angle); + + for (size_t j = 0; j < m2; j++) { + for (size_t k = j; k < n; k += m) { + size_t k_m2 = k + m2; + float t_re = w_re * re[k_m2] - w_im * im[k_m2]; + float t_im = w_re * im[k_m2] + w_im * re[k_m2]; + float u_re = re[k]; + float u_im = im[k]; + re[k] = u_re + t_re; + im[k] = u_im + t_im; + re[k_m2] = u_re - t_re; + im[k_m2] = u_im - t_im; + } + float new_w_re = w_re * wm_re - w_im * wm_im; + float new_w_im = w_re * wm_im + w_im * wm_re; + w_re = new_w_re; + w_im = new_w_im; + } + } +} + +enum class FFTNorm { + BACKWARD, + FORWARD, + ORTHO +}; + +static FFTNorm parse_fft_norm(const char* norm) { + if (!norm || std::strcmp(norm, "backward") == 0) { + return FFTNorm::BACKWARD; + } + if (std::strcmp(norm, "forward") == 0) { + return FFTNorm::FORWARD; + } + if (std::strcmp(norm, "ortho") == 0) { + return FFTNorm::ORTHO; + } + throw std::invalid_argument("norm must be one of {\"backward\",\"forward\",\"ortho\"}"); +} + +static size_t next_power_of_two(size_t n) { + size_t padded_n = 1; + while (padded_n < n) { + padded_n <<= 1; + } + return padded_n; +} + +static float fft_norm_factor(size_t n, FFTNorm norm_mode, bool inverse) { + if (norm_mode == FFTNorm::ORTHO) { + return 1.0f / std::sqrt(static_cast(n)); + } + if (norm_mode == FFTNorm::FORWARD) { + return inverse ? 1.0f : (1.0f / static_cast(n)); + } + return inverse ? (1.0f / static_cast(n)) : 1.0f; +} + +static void ensure_irfft_scratch(std::vector& re, std::vector& im, size_t n) { + if (re.size() < n) { + re.resize(n); + } + if (im.size() < n) { + im.resize(n); + } +} + +static void idft_r_f32_1d_dft( + const float* input, + float* output, + const size_t n, + const float norm_factor) { + const size_t in_len = n / 2 + 1; + const float two_pi_over_n = (2.0f * static_cast(M_PI)) / static_cast(n); + for (size_t t = 0; t < n; ++t) { + float sum = input[0]; + for (size_t k = 1; k < in_len; ++k) { + const float re = input[k * 2]; + const float im = input[k * 2 + 1]; + const float angle = two_pi_over_n * static_cast(k * t); + const float c = std::cos(angle); + const float s = std::sin(angle); + const bool self_conjugate = (k * 2 == n); + if (self_conjugate) { + sum += re * c; + } else { + sum += 2.0f * (re * c - im * s); + } + } + output[t] = sum * norm_factor; + } +} + +static void rfft_f32_1d(const float* input, float* output, const size_t n, const char* norm) { + const size_t out_len = n / 2 + 1; + const FFTNorm norm_mode = parse_fft_norm(norm); + const float norm_factor = fft_norm_factor(n, norm_mode, false); + + if (n == 1) { + output[0] = input[0] * norm_factor; + output[1] = 0.0f; + return; + } + +#ifdef __APPLE__ + { + size_t log2n = 0; + size_t padded_n = 1; + while (padded_n < n) { + padded_n <<= 1; + log2n++; + } + + FFTSetup fft_setup = vDSP_create_fftsetup(log2n, FFT_RADIX2); + if (fft_setup) { + std::vector buffer(padded_n, 0.0f); + std::copy(input, input + n, buffer.begin()); + + DSPSplitComplex split; + std::vector real_part(padded_n / 2); + std::vector imag_part(padded_n / 2); + split.realp = real_part.data(); + split.imagp = imag_part.data(); + + vDSP_ctoz(reinterpret_cast(buffer.data()), 2, &split, 1, padded_n / 2); + + vDSP_fft_zrip(fft_setup, &split, 1, log2n, FFT_FORWARD); + + float scale = 0.5f * norm_factor; + + output[0] = split.realp[0] * scale * 2.0f; + output[1] = 0.0f; + + if (out_len > 1) { + size_t nyquist_idx = padded_n / 2; + if (nyquist_idx < out_len) { + output[nyquist_idx * 2] = split.imagp[0] * scale * 2.0f; + output[nyquist_idx * 2 + 1] = 0.0f; + } + } + + for (size_t i = 1; i < out_len && i < padded_n / 2; i++) { + output[i * 2] = split.realp[i] * scale * 2.0f; + output[i * 2 + 1] = split.imagp[i] * scale * 2.0f; + } + + vDSP_destroy_fftsetup(fft_setup); + return; + } + } +#endif + + const size_t padded_n = next_power_of_two(n); + + std::vector re(padded_n, 0.0f), im(padded_n, 0.0f); + std::copy(input, input + n, re.begin()); + + fft_radix2(re.data(), im.data(), padded_n, false); + + for (size_t i = 0; i < out_len; i++) { + output[i * 2] = re[i] * norm_factor; + output[i * 2 + 1] = im[i] * norm_factor; + } +} + +static void idft_r_f32_1d(const float* input, float* output, const size_t n, const char* norm) { + const size_t in_len = n / 2 + 1; + const FFTNorm norm_mode = parse_fft_norm(norm); + const float norm_factor = fft_norm_factor(n, norm_mode, true); + + if (n == 1) { + output[0] = input[0] * norm_factor; + return; + } + + if (!is_power_of_two(n)) { + idft_r_f32_1d_dft(input, output, n, norm_factor); + return; + } + + const size_t padded_n = next_power_of_two(n); + const size_t padded_half_plus_one = padded_n / 2 + 1; + thread_local std::vector re_scratch; + thread_local std::vector im_scratch; + ensure_irfft_scratch(re_scratch, im_scratch, padded_n); + std::fill(re_scratch.begin(), re_scratch.begin() + padded_n, 0.0f); + std::fill(im_scratch.begin(), im_scratch.begin() + padded_n, 0.0f); + float* re = re_scratch.data(); + float* im = im_scratch.data(); + for (size_t i = 0; i < in_len; ++i) { + re[i] = input[i * 2]; + im[i] = input[i * 2 + 1]; + } + + im[0] = 0.0f; + const bool has_padded_nyquist_bin = in_len == padded_half_plus_one; + if (has_padded_nyquist_bin) { + im[padded_n / 2] = 0.0f; + } + + const size_t mirror_limit = has_padded_nyquist_bin ? (in_len - 1) : in_len; + for (size_t i = 1; i < mirror_limit; ++i) { + const size_t mirrored = padded_n - i; + re[mirrored] = re[i]; + im[mirrored] = -im[i]; + } + + fft_radix2(re, im, padded_n, true); + + for (size_t i = 0; i < n; ++i) { + output[i] = re[i] * norm_factor; + } +} + +static float hertz_to_mel(float freq, const char* mel_scale) { + if (std::strcmp(mel_scale, "htk") == 0) { + return 2595.0f * std::log10(1.0f + (freq / 700.0f)); + } else if (std::strcmp(mel_scale, "kaldi") == 0) { + return 1127.0f * std::log(1.0f + (freq / 700.0f)); + } + + const float min_log_hertz = 1000.0f; + const float min_log_mel = 15.0f; + const float logstep = 27.0f / std::log(6.4f); + float mels = 3.0f * freq / 200.0f; + + if (freq >= min_log_hertz) { + mels = min_log_mel + std::log(freq / min_log_hertz) * logstep; + } + + return mels; +} + +static float mel_to_hertz(float mels, const char* mel_scale) { + if (std::strcmp(mel_scale, "htk") == 0) { + return 700.0f * (std::pow(10.0f, mels / 2595.0f) - 1.0f); + } else if (std::strcmp(mel_scale, "kaldi") == 0) { + return 700.0f * (std::exp(mels / 1127.0f) - 1.0f); + } + + const float min_log_hertz = 1000.0f; + const float min_log_mel = 15.0f; + const float logstep = std::log(6.4f) / 27.0f; + float freq = 200.0f * mels / 3.0f; + + if (mels >= min_log_mel) { + freq = min_log_hertz * std::exp(logstep * (mels - min_log_mel)); + } + + return freq; +} + +static void generate_mel_filter_bank( + float* mel_filters, + const int num_frequency_bins, + const int num_mel_filters, + const float min_frequency, + const float max_frequency, + const int sampling_rate, + const char* norm, + const char* mel_scale, + const bool triangularize_in_mel_space) +{ + if (norm != nullptr && std::strcmp(norm, "slaney") != 0) { + throw std::invalid_argument("norm must be one of None or \"slaney\""); + } + + if (std::strcmp(mel_scale, "htk") != 0 && std::strcmp(mel_scale, "kaldi") != 0 && std::strcmp(mel_scale, "slaney") != 0) { + throw std::invalid_argument("mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\"."); + } + + if (num_frequency_bins < 2) { + throw std::invalid_argument( + "Require num_frequency_bins: " + std::to_string(num_frequency_bins) + " >= 2"); + } + + if (min_frequency > max_frequency) { + throw std::invalid_argument( + "Require min_frequency: " + std::to_string(min_frequency) + + " <= max_frequency: " + std::to_string(max_frequency)); + } + + const float mel_min = hertz_to_mel(min_frequency, mel_scale); + const float mel_max = hertz_to_mel(max_frequency, mel_scale); + + std::vector mel_freqs(num_mel_filters + 2); + for (int i = 0; i < num_mel_filters + 2; i++) { + mel_freqs[i] = mel_min + (mel_max - mel_min) * i / (num_mel_filters + 1); + } + + std::vector filter_freqs(num_mel_filters + 2); + for (int i = 0; i < num_mel_filters + 2; i++) { + filter_freqs[i] = mel_to_hertz(mel_freqs[i], mel_scale); + } + + std::vector fft_freqs(num_frequency_bins); + if (triangularize_in_mel_space) { + float fft_bin_width = static_cast(sampling_rate) / ((num_frequency_bins - 1) * 2); + for (int i = 0; i < num_frequency_bins; i++) { + fft_freqs[i] = hertz_to_mel(fft_bin_width * i, mel_scale); + } + filter_freqs = mel_freqs; + } else { + for (int i = 0; i < num_frequency_bins; i++) { + fft_freqs[i] = (static_cast(sampling_rate) / 2.0f) * i / (num_frequency_bins - 1); + } + } + + for (int i = 0; i < num_mel_filters; i++) { + float left_edge = filter_freqs[i]; + float center = filter_freqs[i + 1]; + float right_edge = filter_freqs[i + 2]; + + for (int j = 0; j < num_frequency_bins; j++) { + float freq = fft_freqs[j]; + float down_slope = (freq - left_edge) / (center - left_edge); + float up_slope = (right_edge - freq) / (right_edge - center); + + mel_filters[i * num_frequency_bins + j] = std::max(0.0f, std::min(down_slope, up_slope)); + } + } + + if (norm != nullptr && std::strcmp(norm, "slaney") == 0) { + for (int i = 0; i < num_mel_filters; i++) { + float enorm = 2.0f / (filter_freqs[i + 2] - filter_freqs[i]); + for (int j = 0; j < num_frequency_bins; j++) { + mel_filters[i * num_frequency_bins + j] *= enorm; + } + } + } +} + +static void compute_spectrogram_f32( + const float* waveform, + size_t waveform_length, + const float* window, + size_t window_length, + size_t frame_length, + size_t hop_length, + const size_t* fft_length, + float* spectrogram, + float power, + bool center, + const char* pad_mode, + bool onesided [[maybe_unused]], + float dither, + const float* preemphasis, + const float* mel_filters, + size_t mel_filters_size, + float mel_floor, + const char* log_mel, + float reference, + float min_value, + const float* db_range, + bool remove_dc_offset) +{ + size_t actual_fft_length; + if (fft_length == nullptr) { + actual_fft_length = frame_length; + } else { + actual_fft_length = *fft_length; + } + + if (frame_length > actual_fft_length) { + throw std::invalid_argument( + "frame_length (" + std::to_string(frame_length) + + ") may not be larger than fft_length (" + + std::to_string(actual_fft_length) + ")"); + } + + std::vector hann_window; + const float* actual_window = window; + + if (window == nullptr) { + size_t length = frame_length + 1; + hann_window.resize(frame_length); + for (size_t i = 0; i < frame_length; i++) { + hann_window[i] = 0.5f * (1.0f - std::cos(2.0f * static_cast(M_PI) * i / (length - 1))); + } + actual_window = hann_window.data(); + } else if (window_length != frame_length) { + throw std::invalid_argument( + "Length of the window (" + std::to_string(window_length) + + ") must equal frame_length (" + std::to_string(frame_length) + ")"); + } + + if (hop_length <= 0) { + throw std::invalid_argument("hop_length must be greater than zero"); + } + + if (power == 0.0f && mel_filters != nullptr) { + throw std::invalid_argument( + "You have provided `mel_filters` but `power` is `None`. " + "Mel spectrogram computation is not yet supported for complex-valued spectrogram. " + "Specify `power` to fix this issue."); + } + + std::vector padded_waveform; + const float* input_waveform = waveform; + size_t input_length = waveform_length; + + if (center) { + size_t pad_length = frame_length / 2; + size_t padded_length = waveform_length + 2 * pad_length; + padded_waveform.assign(padded_length, 0.0f); + + if (std::strcmp(pad_mode, "reflect") == 0) { + for (size_t i = 0; i < pad_length; i++) { + padded_waveform[i] = waveform[pad_length - i]; + } + + std::copy(waveform, waveform + waveform_length, padded_waveform.data() + pad_length); + + for (size_t i = 0; i < pad_length; i++) { + padded_waveform[pad_length + waveform_length + i] = waveform[waveform_length - 2 - i]; + } + } else if (std::strcmp(pad_mode, "constant") == 0) { + std::copy(waveform, waveform + waveform_length, padded_waveform.data() + pad_length); + } else { + throw std::invalid_argument("Unsupported pad_mode: " + std::string(pad_mode)); + } + + input_waveform = padded_waveform.data(); + input_length = padded_length; + } + + const size_t num_frames = 1 + (input_length - frame_length) / hop_length; + const size_t num_frequency_bins = (actual_fft_length / 2) + 1; + + std::vector buffer(actual_fft_length); + std::vector raw_complex_frequencies(num_frequency_bins * 2); + + const size_t num_mel_bins = mel_filters != nullptr ? mel_filters_size / num_frequency_bins : 0; + const size_t spectrogram_bins = mel_filters != nullptr ? num_mel_bins : num_frequency_bins; + + std::vector temp_spectrogram(num_frames * num_frequency_bins); + + CactusThreading::parallel_for(num_frames, CactusThreading::Thresholds::SCALAR_EXPENSIVE, [&](size_t start_frame, size_t end_frame) { + std::vector local_buffer(actual_fft_length); + std::vector local_complex_frequencies(num_frequency_bins * 2); + + for (size_t frame_idx = start_frame; frame_idx < end_frame; frame_idx++) { + size_t timestep = frame_idx * hop_length; + std::fill(local_buffer.begin(), local_buffer.end(), 0.0f); + + size_t available_length = std::min(frame_length, input_length - timestep); + std::copy(input_waveform + timestep, input_waveform + timestep + available_length, local_buffer.data()); + + if (dither != 0.0f) { + thread_local std::mt19937 rng(std::random_device{}()); + std::normal_distribution dist(0.0f, 1.0f); + for (size_t i = 0; i < frame_length; i++) { + local_buffer[i] += dither * dist(rng); + } + } + + if (remove_dc_offset) { + float mean = 0.0f; + for (size_t i = 0; i < frame_length; i++) { + mean += local_buffer[i]; + } + mean /= static_cast(frame_length); + + for (size_t i = 0; i < frame_length; i++) { + local_buffer[i] -= mean; + } + } + + if (preemphasis != nullptr) { + float preemph_coef = *preemphasis; + for (size_t i = frame_length - 1; i > 0; i--) { + local_buffer[i] -= preemph_coef * local_buffer[i - 1]; + } + local_buffer[0] *= (1.0f - preemph_coef); + } + + for (size_t i = 0; i < frame_length; i++) { + local_buffer[i] *= actual_window[i]; + } + + rfft_f32_1d(local_buffer.data(), local_complex_frequencies.data(), actual_fft_length, "backward"); + + for (size_t i = 0; i < num_frequency_bins; i++) { + float real = local_complex_frequencies[i * 2]; + float imag = local_complex_frequencies[i * 2 + 1]; + float magnitude = std::hypot(real, imag); + temp_spectrogram[frame_idx * num_frequency_bins + i] = std::pow(magnitude, power); + } + } + }); + + if (mel_filters != nullptr) { + CactusThreading::parallel_for_2d(num_mel_bins, num_frames, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t m, size_t t) { + float sum = 0.0f; + for (size_t f = 0; f < num_frequency_bins; f++) { + sum += mel_filters[m * num_frequency_bins + f] * temp_spectrogram[t * num_frequency_bins + f]; + } + spectrogram[m * num_frames + t] = std::max(mel_floor, sum); + }); + } else { + CactusThreading::parallel_for_2d(num_frames, num_frequency_bins, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t t, size_t f) { + spectrogram[f * num_frames + t] = temp_spectrogram[t * num_frequency_bins + f]; + }); + } + + if (power != 0.0f && log_mel != nullptr) { + const size_t total_elements = spectrogram_bins * num_frames; + + if (std::strcmp(log_mel, "log") == 0) { + CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { + for (size_t i = start; i < end; i++) { + spectrogram[i] = std::log(spectrogram[i]); + } + }); + } else if (std::strcmp(log_mel, "log10") == 0) { + CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { + for (size_t i = start; i < end; i++) { + spectrogram[i] = std::log10(spectrogram[i]); + } + }); + } else if (std::strcmp(log_mel, "dB") == 0) { + if (power == 1.0f) { + to_db(spectrogram, total_elements, reference, min_value, db_range, 20.0f); + } else if (power == 2.0f) { + to_db(spectrogram, total_elements, reference, min_value, db_range, 10.0f); + } else { + throw std::invalid_argument( + "Cannot use log_mel option 'dB' with power " + std::to_string(power)); + } + } else { + throw std::invalid_argument("Unknown log_mel option: " + std::string(log_mel)); + } + } +} + +AudioProcessor::AudioProcessor() + : num_frequency_bins_(0), num_mel_filters_(0) {} + +AudioProcessor::~AudioProcessor() = default; + +void AudioProcessor::init_mel_filters(size_t num_frequency_bins, + size_t num_mel_filters, + float min_freq, + float max_freq, + size_t sampling_rate, + const char* norm, + const char* mel_scale) { + num_frequency_bins_ = num_frequency_bins; + num_mel_filters_ = num_mel_filters; + mel_filters_.resize(num_mel_filters * num_frequency_bins); + + generate_mel_filter_bank( + mel_filters_.data(), + num_frequency_bins, + num_mel_filters, + min_freq, + max_freq, + sampling_rate, + norm, + mel_scale, + false + ); +} + +std::vector AudioProcessor::compute_spectrogram( + const std::vector& waveform, + const SpectrogramConfig& config) { + + if (mel_filters_.empty()) { + throw std::runtime_error("Mel filters not initialized. Call init_mel_filters() first."); + } + + const size_t n_samples = waveform.size(); + const size_t fft_size = config.fft_override > 0 ? config.fft_override : config.n_fft; + const size_t analysis_frame_length = config.n_fft; + const size_t window_length = std::min(config.frame_length, analysis_frame_length); + const size_t pad_length = config.center ? analysis_frame_length / 2 : 0; + const size_t padded_length = n_samples + 2 * pad_length; + if (analysis_frame_length == 0 || padded_length < analysis_frame_length || config.hop_length == 0) { + return {}; + } + const size_t num_frames = 1 + (padded_length - analysis_frame_length) / config.hop_length; + + std::vector output(num_mel_filters_ * num_frames); + std::vector window(analysis_frame_length, 0.0f); + if (window_length == 1) { + window[analysis_frame_length / 2] = 1.0f; + } else if (window_length > 1) { + const float denom = config.hann_periodic + ? static_cast(window_length) + : static_cast(window_length - 1); + const float a0 = config.window_a0; + const size_t left_pad = (analysis_frame_length - window_length) / 2; + for (size_t i = 0; i < window_length; ++i) { + const float w = a0 - (1.0f - a0) * std::cos(2.0f * static_cast(M_PI) * static_cast(i) / denom); + window[left_pad + i] = w; + } + } + + float effective_floor = config.mel_floor_additive ? 0.0f : config.mel_floor; + const char* effective_log = config.mel_floor_additive ? nullptr : config.log_mel; + + compute_spectrogram_f32( + waveform.data(), + waveform.size(), + window.data(), + window.size(), + analysis_frame_length, + config.hop_length, + &fft_size, + output.data(), + config.power, + config.center, + config.pad_mode, + config.onesided, + config.dither, + config.preemphasis != 0.0f ? &config.preemphasis : nullptr, + mel_filters_.data(), + mel_filters_.size(), + effective_floor, + effective_log, + config.reference, + config.min_value, + nullptr, + config.remove_dc_offset + ); + + if (config.mel_floor_additive) { + for (size_t i = 0; i < output.size(); i++) { +#ifdef __APPLE__ + // vDSP FFT returns 2x standard DFT magnitudes, so raw mel + // values are 2x too large. Halve before adding floor and log. + output[i] = std::log(output[i] * 0.5f + config.mel_floor); +#else + output[i] = std::log(output[i] + config.mel_floor); +#endif + } + } + + return output; +} + +std::vector AudioProcessor::compute_irfft( + const std::vector& complex_input, + size_t n, + const char* norm) { + if (n == 0) { + throw std::invalid_argument("compute_irfft: n must be greater than 0"); + } + const size_t expected_in_len = (n / 2 + 1) * 2; + if (complex_input.size() != expected_in_len) { + throw std::invalid_argument("compute_irfft: complex_input must contain exactly (n/2+1)*2 values"); + } + std::vector output(n); + idft_r_f32_1d(complex_input.data(), output.data(), n, norm); + return output; +} + +} +} diff --git a/cactus-transpile/engine/engine_sp.cpp b/cactus-transpile/engine/engine_sp.cpp new file mode 100644 index 000000000..41cd506c6 --- /dev/null +++ b/cactus-transpile/engine/engine_sp.cpp @@ -0,0 +1,485 @@ +#include "engine.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +SPTokenizer::SPTokenizer() + : trie_root_(std::make_unique()), + vocab_size_(0), + unk_token_id_(3), + bos_token_id_(2), + eos_token_id_(1), + pad_token_id_(0), + vocab_mmap_ptr_(nullptr), + vocab_mmap_size_(0) { + has_chat_template_ = false; +} + +SPTokenizer::~SPTokenizer() { + cleanup_mmap(); +} + +void SPTokenizer::cleanup_mmap() { + if (vocab_mmap_ptr_ && vocab_mmap_ptr_ != MAP_FAILED) { + munmap(vocab_mmap_ptr_, vocab_mmap_size_); + vocab_mmap_ptr_ = nullptr; + } +} + +bool SPTokenizer::load_vocabulary_with_config(const std::string& vocab_file, const std::string& /*merges_file*/, const std::string& config_file) { + runtime_config_ = load_tokenizer_runtime_config(config_file); + std::string config_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/config.txt"; + detect_model_type(config_path); + + std::ifstream vocab_stream(vocab_file); + if (!vocab_stream.is_open()) return false; + + token_to_id_.clear(); + id_to_token_.clear(); + token_scores_.clear(); + + std::string first_line; + std::getline(vocab_stream, first_line); + vocab_stream.seekg(0); + + bool is_id_token_format = false; + if (!first_line.empty()) { + is_id_token_format = (std::isdigit(first_line[0]) && + first_line.find('\t') != std::string::npos); + } + + if (is_id_token_format) { + std::string line = ""; + while (std::getline(vocab_stream, line)) { + std::string token = ""; + uint32_t id = UINT32_MAX; + + std::istringstream iss(line); + if (iss >> id) { + if (std::getline(iss, token)) { + if (!token.empty() && token[0] == '\t') { + token = token.substr(1); + } + } + + if (token.empty()) { + auto last_pos = vocab_stream.tellg(); + while (std::getline(vocab_stream, line)) { + if (!line.empty()) { + break; + } + token += '\n'; + last_pos = vocab_stream.tellg(); + } + vocab_stream.seekg(last_pos); + } + } + + if (!token.empty() && id != UINT32_MAX) { + float score = -static_cast(id); + size_t tab_in_token = token.find('\t'); + if (tab_in_token != std::string::npos) { + std::string score_str = token.substr(tab_in_token + 1); + if (!score_str.empty()) score = std::stof(score_str); + token = token.substr(0, tab_in_token); + } + token_to_id_[token] = id; + if (id >= id_to_token_.size()) { + id_to_token_.resize(id + 1); + token_scores_.resize(id + 1, 0.0f); + } + id_to_token_[id] = token; + token_scores_[id] = score; + } + } + vocab_size_ = id_to_token_.size(); + } else { + std::string line; + uint32_t id = 0; + + vocab_stream.seekg(0); + while (std::getline(vocab_stream, line)) { + token_to_id_[line] = id; + id_to_token_.push_back(line); + token_scores_.push_back(0.0f); + id++; + } + vocab_size_ = id; + } + + vocab_stream.close(); + + build_trie(); + + std::ifstream config_stream(config_file); + if (config_stream.is_open()) { + std::string config_line; + while (std::getline(config_stream, config_line)) { + if (config_line.empty() || config_line[0] == '#') continue; + + size_t eq_pos = config_line.find('='); + if (eq_pos == std::string::npos) continue; + + std::string key = config_line.substr(0, eq_pos); + std::string value = config_line.substr(eq_pos + 1); + + key.erase(0, key.find_first_not_of(" \t")); + key.erase(key.find_last_not_of(" \t") + 1); + value.erase(0, value.find_first_not_of(" \t")); + value.erase(value.find_last_not_of(" \t") + 1); + + if (key == "eos_token_id") { + eos_token_id_ = std::stoul(value); + } else if (key == "pad_token_id") { + pad_token_id_ = std::stoul(value); + } else if (key == "unk_token_id") { + unk_token_id_ = std::stoul(value); + } else if (key == "bos_token_id") { + bos_token_id_ = std::stoul(value); + } else if (key == "sp_model_type") { + sp_bpe_mode_ = (value == "bpe" || value == "BPE"); + } else if (key == "sp_add_dummy_prefix") { + sp_add_dummy_prefix_ = (value == "true" || value == "1"); + } else if (key == "sp_byte_fallback") { + sp_byte_fallback_ = (value == "true" || value == "1"); + } + } + } + + std::string special_tokens_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/special_tokens.json"; + load_special_tokens(special_tokens_path); + + std::string template_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/chat_template.jinja2"; + load_chat_template(template_path); + + return true; +} + +void SPTokenizer::build_trie() { + for (uint32_t id = 0; id < id_to_token_.size(); ++id) { + const std::string& token = id_to_token_[id]; + if (token.empty()) continue; + + std::u32string u32_token; + size_t pos = 0; + while (pos < token.length()) { + char32_t codepoint = 0; + unsigned char byte = token[pos]; + + if (byte < 0x80) { + codepoint = byte; + pos++; + } else if ((byte & 0xE0) == 0xC0) { + if (pos + 1 < token.length()) { + codepoint = ((byte & 0x1F) << 6) | (token[pos + 1] & 0x3F); + pos += 2; + } else break; + } else if ((byte & 0xF0) == 0xE0) { + if (pos + 2 < token.length()) { + codepoint = ((byte & 0x0F) << 12) | + ((token[pos + 1] & 0x3F) << 6) | + (token[pos + 2] & 0x3F); + pos += 3; + } else break; + } else if ((byte & 0xF8) == 0xF0) { + if (pos + 3 < token.length()) { + codepoint = ((byte & 0x07) << 18) | + ((token[pos + 1] & 0x3F) << 12) | + ((token[pos + 2] & 0x3F) << 6) | + (token[pos + 3] & 0x3F); + pos += 4; + } else break; + } else { + pos++; + continue; + } + + u32_token.push_back(codepoint); + } + + if (u32_token.empty()) continue; + + TrieNode* current = trie_root_.get(); + for (char32_t ch : u32_token) { + if (current->children.find(ch) == current->children.end()) { + current->children[ch] = std::make_unique(); + } + current = current->children[ch].get(); + } + current->token_id = static_cast(id); + current->score = token_scores_[id]; + } +} + +std::string SPTokenizer::preprocess_text(const std::string& text) const { + if (text.empty()) return text; + + if (sp_bpe_mode_) { + std::string processed; + if (sp_add_dummy_prefix_) processed += "\xE2\x96\x81"; + for (char c : text) { + if (c == ' ') processed += "\xE2\x96\x81"; + else processed += c; + } + return processed; + } + + std::string processed = ""; + if (model_type_ == ModelType::BERT) { + processed = "▁"; + } + + for (size_t i = text.find_first_not_of(" "); i < text.length(); i++) { + char c = text[i]; + if (c == ' ') { + processed += "▁"; + } else { + processed += c; + } + } + + return processed; +} + +std::string SPTokenizer::postprocess_text(const std::string& text) const { + std::string result; + size_t i = 0; + while (i < text.length()) { + if (i + 2 < text.length() && + static_cast(text[i]) == 0xE2 && + static_cast(text[i+1]) == 0x96 && + static_cast(text[i+2]) == 0x81) { + result += ' '; + i += 3; + } else { + result += text[i]; + i++; + } + } + if (!result.empty() && result[0] == ' ') { + result = result.substr(1); + } + return result; +} + +std::vector> SPTokenizer::tokenize_with_trie(const std::string& text) const { + std::vector> result; + + std::u32string u32_text; + size_t pos = 0; + while (pos < text.length()) { + char32_t codepoint = 0; + unsigned char byte = text[pos]; + + if (byte < 0x80) { + codepoint = byte; + pos++; + } else if ((byte & 0xE0) == 0xC0) { + if (pos + 1 < text.length()) { + codepoint = ((byte & 0x1F) << 6) | (text[pos + 1] & 0x3F); + pos += 2; + } else break; + } else if ((byte & 0xF0) == 0xE0) { + if (pos + 2 < text.length()) { + codepoint = ((byte & 0x0F) << 12) | + ((text[pos + 1] & 0x3F) << 6) | + (text[pos + 2] & 0x3F); + pos += 3; + } else break; + } else if ((byte & 0xF8) == 0xF0) { + if (pos + 3 < text.length()) { + codepoint = ((byte & 0x07) << 18) | + ((text[pos + 1] & 0x3F) << 12) | + ((text[pos + 2] & 0x3F) << 6) | + (text[pos + 3] & 0x3F); + pos += 4; + } else break; + } else { + pos++; + continue; + } + + u32_text.push_back(codepoint); + } + + if (u32_text.empty()) { + result.push_back({text, unk_token_id_}); + return result; + } + + pos = 0; + while (pos < u32_text.length()) { + TrieNode* current = trie_root_.get(); + size_t best_match_len = 0; + int32_t best_token_id = -1; + + for (size_t len = 0; pos + len < u32_text.length(); ++len) { + char32_t ch = u32_text[pos + len]; + if (current->children.find(ch) == current->children.end()) { + break; + } + current = current->children[ch].get(); + if (current->token_id >= 0) { + best_match_len = len + 1; + best_token_id = current->token_id; + } + } + + if (best_match_len > 0) { + std::u32string u32_token = u32_text.substr(pos, best_match_len); + + std::string token; + for (char32_t cp : u32_token) { + if (cp < 0x80) { + token.push_back(static_cast(cp)); + } else if (cp < 0x800) { + token.push_back(static_cast(0xC0 | (cp >> 6))); + token.push_back(static_cast(0x80 | (cp & 0x3F))); + } else if (cp < 0x10000) { + token.push_back(static_cast(0xE0 | (cp >> 12))); + token.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + token.push_back(static_cast(0x80 | (cp & 0x3F))); + } else { + token.push_back(static_cast(0xF0 | (cp >> 18))); + token.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); + token.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + token.push_back(static_cast(0x80 | (cp & 0x3F))); + } + } + result.push_back({token, static_cast(best_token_id)}); + pos += best_match_len; + } else { + char32_t cp = u32_text[pos]; + std::string char_str; + if (cp < 0x80) { + char_str.push_back(static_cast(cp)); + } else if (cp < 0x800) { + char_str.push_back(static_cast(0xC0 | (cp >> 6))); + char_str.push_back(static_cast(0x80 | (cp & 0x3F))); + } else if (cp < 0x10000) { + char_str.push_back(static_cast(0xE0 | (cp >> 12))); + char_str.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + char_str.push_back(static_cast(0x80 | (cp & 0x3F))); + } else { + char_str.push_back(static_cast(0xF0 | (cp >> 18))); + char_str.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); + char_str.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + char_str.push_back(static_cast(0x80 | (cp & 0x3F))); + } + result.push_back({char_str, unk_token_id_}); + pos++; + } + } + + return result; +} + +std::vector SPTokenizer::tokenize_with_bpe(const std::string& text) const { + std::vector symbols; + for (size_t i = 0; i < text.size(); ) { + size_t char_len = 1; + unsigned char byte = static_cast(text[i]); + if ((byte & 0xE0) == 0xC0) char_len = 2; + else if ((byte & 0xF0) == 0xE0) char_len = 3; + else if ((byte & 0xF8) == 0xF0) char_len = 4; + if (i + char_len <= text.size()) symbols.push_back(text.substr(i, char_len)); + i += char_len; + } + + while (symbols.size() > 1) { + int best_index = -1; + float best_score = -std::numeric_limits::infinity(); + for (size_t i = 0; i + 1 < symbols.size(); ++i) { + auto it = token_to_id_.find(symbols[i] + symbols[i + 1]); + if (it == token_to_id_.end()) continue; + float score = it->second < token_scores_.size() + ? token_scores_[it->second] + : -static_cast(it->second); + if (score > best_score) { best_score = score; best_index = static_cast(i); } + } + if (best_index < 0) break; + symbols[best_index] += symbols[best_index + 1]; + symbols.erase(symbols.begin() + best_index + 1); + } + + std::vector result; + for (const auto& symbol : symbols) { + auto it = token_to_id_.find(symbol); + if (it != token_to_id_.end()) { result.push_back(it->second); continue; } + if (sp_byte_fallback_) { + for (unsigned char b : symbol) { + char buf[7]; std::snprintf(buf, sizeof(buf), "<0x%02X>", b); + auto byte_it = token_to_id_.find(buf); + result.push_back(byte_it != token_to_id_.end() ? byte_it->second : unk_token_id_); + } + continue; + } + result.push_back(unk_token_id_); + } + return result; +} + +std::vector SPTokenizer::split_with_special_tokens(const std::string& text) const { + return cactus::engine::split_with_special_tokens(text, special_tokens_); +} + +std::vector SPTokenizer::encode(const std::string& text) const { + if (text.empty()) return {}; + + auto text_segments = split_with_special_tokens(text); + std::vector token_ids; + + for (const auto& segment : text_segments) { + auto special_it = special_tokens_.find(segment); + if (special_it != special_tokens_.end()) { + token_ids.push_back(special_it->second); + } else { + std::string processed = preprocess_text(segment); + if (processed.empty()) continue; + + if (sp_bpe_mode_) { + auto ids = tokenize_with_bpe(processed); + token_ids.insert(token_ids.end(), ids.begin(), ids.end()); + } else { + auto token_pairs = tokenize_with_trie(processed); + for (const auto& [token, id] : token_pairs) { + token_ids.push_back(id); + } + } + } + } + + return token_ids; +} + +std::string SPTokenizer::decode(const std::vector& tokens) const { + std::string raw; + for (uint32_t token_id : tokens) { + if (token_id >= id_to_token_.size()) continue; + const std::string& piece = id_to_token_[token_id]; + unsigned int byte_val; + if (piece.size() == 6 && std::sscanf(piece.c_str(), "<0x%02X>", &byte_val) == 1) { + raw.push_back(static_cast(byte_val)); + } else { + raw += piece; + } + } + return postprocess_text(raw); +} + +void SPTokenizer::load_special_tokens(const std::string& config_file) { + load_special_tokens_map(config_file, special_tokens_); +} + +} // namespace engine +} // namespace cactus diff --git a/cactus-transpile/engine/engine_tokenizer.cpp b/cactus-transpile/engine/engine_tokenizer.cpp new file mode 100644 index 000000000..566fea2f8 --- /dev/null +++ b/cactus-transpile/engine/engine_tokenizer.cpp @@ -0,0 +1,866 @@ +#include "engine.h" +#include +#include +#include +#include +#include + +extern "C" { + #include "../../libs/stb/stb_image.h" +} + +namespace cactus { +namespace engine { + +namespace { + +std::string trim_copy(const std::string& value) { + size_t start = value.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) return ""; + size_t end = value.find_last_not_of(" \t\r\n"); + return value.substr(start, end - start + 1); +} + +TokenizerRuntimeConfig::TokenizerType parse_tokenizer_type(const std::string& value) { + if (value == "bpe") return TokenizerRuntimeConfig::TokenizerType::BPE; + if (value == "sentencepiece") return TokenizerRuntimeConfig::TokenizerType::SENTENCEPIECE; + return TokenizerRuntimeConfig::TokenizerType::UNKNOWN; +} + +TokenizerRuntimeConfig::VocabFormat parse_vocab_format(const std::string& value) { + if (value == "id_tab_token") return TokenizerRuntimeConfig::VocabFormat::ID_TAB_TOKEN; + if (value == "line_token") return TokenizerRuntimeConfig::VocabFormat::LINE_TOKEN; + return TokenizerRuntimeConfig::VocabFormat::UNKNOWN; +} + +TokenizerRuntimeConfig::Normalizer parse_normalizer(const std::string& value) { + if (value == "metaspace") return TokenizerRuntimeConfig::Normalizer::METASPACE; + if (value == "byte_level") return TokenizerRuntimeConfig::Normalizer::BYTE_LEVEL; + return TokenizerRuntimeConfig::Normalizer::NONE; +} + +TokenizerRuntimeConfig::Decoder parse_decoder(const std::string& value) { + if (value == "replace_metaspace") return TokenizerRuntimeConfig::Decoder::REPLACE_METASPACE; + if (value == "byte_level") return TokenizerRuntimeConfig::Decoder::BYTE_LEVEL; + return TokenizerRuntimeConfig::Decoder::NONE; +} + +} // namespace + +TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file) { + TokenizerRuntimeConfig config; + + std::ifstream file(config_file); + if (!file.is_open()) { + return config; + } + + std::string line; + while (std::getline(file, line)) { + if (line.empty() || line[0] == '#') continue; + + size_t eq_pos = line.find('='); + if (eq_pos == std::string::npos) continue; + + const std::string key = trim_copy(line.substr(0, eq_pos)); + const std::string value = trim_copy(line.substr(eq_pos + 1)); + + if (key == "tokenizer_type") { + config.tokenizer_type = parse_tokenizer_type(value); + } else if (key == "vocab_format") { + config.vocab_format = parse_vocab_format(value); + } else if (key == "normalizer") { + config.normalizer = parse_normalizer(value); + } else if (key == "decoder") { + config.decoder = parse_decoder(value); + } else if (key == "byte_fallback") { + config.byte_fallback = (value == "true" || value == "1"); + } else if (key == "has_chat_template") { + config.has_chat_template = (value == "true" || value == "1"); + } + } + + return config; +} + +void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens) { + special_tokens.clear(); + + std::ifstream file(config_file); + if (!file.is_open()) { + return; + } + + std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + + size_t pos = content.find("\"special_tokens\""); + if (pos == std::string::npos) return; + + pos = content.find("{", pos); + if (pos == std::string::npos) return; + + size_t end_pos = content.find("}", pos); + if (end_pos == std::string::npos) return; + + std::string special_tokens_section = content.substr(pos + 1, end_pos - pos - 1); + std::istringstream iss(special_tokens_section); + std::string line; + + while (std::getline(iss, line)) { + size_t colon_pos = line.find(":"); + if (colon_pos == std::string::npos) continue; + + std::string id_part = line.substr(0, colon_pos); + std::string token_part = line.substr(colon_pos + 1); + + size_t id_start = id_part.find("\""); + size_t id_end = id_part.find("\"", id_start + 1); + if (id_start == std::string::npos || id_end == std::string::npos) continue; + + uint32_t token_id = static_cast(std::stoul(id_part.substr(id_start + 1, id_end - id_start - 1))); + + size_t token_start = token_part.find("\""); + size_t token_end = token_part.rfind("\""); + if (token_start == std::string::npos || token_end == std::string::npos || token_start >= token_end) continue; + + std::string token_content = token_part.substr(token_start + 1, token_end - token_start - 1); + special_tokens[token_content] = token_id; + } +} + +std::vector split_with_special_tokens(const std::string& text, + const std::unordered_map& special_tokens) { + std::vector result; + size_t start = 0; + while (start < text.size()) { + size_t best_match_pos = text.size(); + size_t best_match_len = 0; + std::string best_special_token; + + for (const auto& [special_token, token_id] : special_tokens) { + size_t pos = text.find(special_token, start); + if (pos != std::string::npos && + (pos < best_match_pos || (pos == best_match_pos && special_token.length() > best_match_len))) { + best_match_pos = pos; + best_match_len = special_token.length(); + best_special_token = special_token; + } + } + + if (best_match_pos < text.size()) { + if (best_match_pos > start) + result.push_back(text.substr(start, best_match_pos - start)); + result.push_back(best_special_token); + start = best_match_pos + best_match_len; + } else { + if (start < text.size()) + result.push_back(text.substr(start)); + break; + } + } + return result; +} + +void Tokenizer::load_chat_template(const std::string& template_file) { + std::ifstream file(template_file); + if (!file.is_open()) { + has_chat_template_ = false; + return; + } + chat_template_ = std::string((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + has_chat_template_ = !chat_template_.empty(); +} + +void Tokenizer::detect_model_type(const std::string& config_path) { + std::ifstream file(config_path); + if (!file.is_open()) { + model_type_ = ModelType::UNKNOWN; + return; + } + + std::string line; + while (std::getline(file, line)) { + size_t pos = line.find("model_type"); + if (pos != std::string::npos) { + std::transform(line.begin(), line.end(), line.begin(), ::tolower); + + if (line.find("qwen3_5") != std::string::npos) { + model_type_ = ModelType::QWEN3P5; + break; + } else if (line.find("qwen") != std::string::npos) { + model_type_ = ModelType::QWEN; + break; + } else if (line.find("gemma4") != std::string::npos || line.find("tinyllama") != std::string::npos) { + model_type_ = ModelType::GEMMA4; + break; + } else if (line.find("gemma") != std::string::npos) { + model_type_ = ModelType::GEMMA; + break; + } else if(line.find("lfm2") != std::string::npos) { + model_type_ = ModelType::LFM2; + } else if (line.find("bert") != std::string::npos) { + model_type_ = ModelType::BERT; + break; + } else if (line.find("whisper") != std::string::npos) { + model_type_ = ModelType::WHISPER; + break; + } else if (line.find("parakeet") != std::string::npos) { + model_type_ = ModelType::PARAKEET; + break; + } else if (line.find("needle") != std::string::npos) { + model_type_ = ModelType::NEEDLE; + break; + } else if (line.find("youtu") != std::string::npos) { + model_type_ = ModelType::YOUTU; + break; + } else { + model_type_ = ModelType::UNKNOWN; + } + } + } + file.clear(); + file.seekg(0); + + while (std::getline(file, line)) { + size_t pos2 = line.find("model_variant"); + if (pos2 != std::string::npos) { + std::transform(line.begin(), line.end(), line.begin(), ::tolower); + + if (line.find("vlm") != std::string::npos) { + model_variant_ = ModelVariant::VLM; + break; + } else if (line.find("extract") != std::string::npos) { + model_variant_ = ModelVariant::EXTRACT; + break; + } else if (line.find("rag") != std::string::npos) { + model_variant_ = ModelVariant::RAG; + break; + } else { + model_variant_ = ModelVariant::DEFAULT; + } + } + } + + file.clear(); + file.seekg(0); + while (std::getline(file, line)) { + auto parse_uint = [&](const std::string& key, uint32_t& out) { + size_t p = line.find(key + "="); + if (p != std::string::npos) { + out = static_cast(std::stoul(line.substr(p + key.size() + 1))); + } + }; + parse_uint("vision_patch_size", vision_patch_size_); + parse_uint("vision_pooling_kernel_size", vision_pooling_kernel_size_); + parse_uint("vision_default_output_length", vision_default_output_length_); + parse_uint("vision_image_size", vision_image_size_); + } + + file.close(); +} + +std::string Tokenizer::get_default_stop_sequence() const { + switch (model_type_) { + case ModelType::GEMMA: + return ""; + case ModelType::GEMMA4: + return ""; + case ModelType::QWEN: + case ModelType::QWEN3P5: + case ModelType::LFM2: + return "<|im_end|>"; + case ModelType::NEEDLE: + return ""; + default: + return "<|im_end|>"; + } +} + +std::vector Tokenizer::apply_chat_template(const std::vector& messages, bool add_generation_prompt) const { + std::string formatted_prompt = format_chat_prompt(messages, add_generation_prompt); + return encode(formatted_prompt); +} + +std::string Tokenizer::format_chat_prompt(const std::vector& messages, bool add_generation_prompt, + const std::string& tools_json, bool enable_thinking_if_supported) const { + bool has_images = false; + for (const auto& msg : messages) { + if (!msg.images.empty()) { + has_images = true; + break; + } + } + if (model_type_ == ModelType::LFM2 && has_images) { + return format_lfm2_vl_style(messages, add_generation_prompt, tools_json); + } + + switch (model_type_) { + case ModelType::QWEN: + case ModelType::QWEN3P5: + return format_qwen_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); + case ModelType::GEMMA: + return format_gemma_style(messages, add_generation_prompt, tools_json); + case ModelType::GEMMA4: + return format_gemma4_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); + case ModelType::LFM2: + return format_lfm2_style(messages, add_generation_prompt, tools_json); + case ModelType::NEEDLE: + return format_needle_style(messages, add_generation_prompt, tools_json); + case ModelType::YOUTU: + return format_youtu_style(messages, add_generation_prompt, tools_json); + default: + return format_qwen_style(messages, add_generation_prompt, tools_json); + } +} + +std::string Tokenizer::format_needle_style(const std::vector& messages, + bool /*add_generation_prompt*/, + const std::string& tools_json) const { + std::string serialized_tools = tools_json.empty() ? "[]" : tools_json; + return format_needle_query_text(messages) + "" + serialized_tools + ""; +} + +std::string Tokenizer::format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported) const { + std::string result; + size_t start_idx = 0; + + if (!tools_json.empty()) { + result += "<|im_start|>system\n"; + if (!messages.empty() && messages[0].role == "system") { + result += messages[0].content + "\n\n"; + start_idx = 1; + } + result += "# Tools\n\n"; + result += "You may call one or more functions to assist with the user query.\n\n"; + result += "You are provided with function signatures within XML tags:\n"; + result += ""; + result += tools_json; + result += "\n\n\n"; + result += "For each function call, return a json object with function name and arguments within XML tags:\n"; + result += "\n"; + result += "{\"name\": , \"arguments\": }\n"; + result += ""; + result += "<|im_end|>\n"; + } else { + if (!messages.empty() && messages[0].role == "system") { + result += "<|im_start|>system\n" + messages[0].content + "<|im_end|>\n"; + start_idx = 1; + } + } + + size_t last_query_index = messages.empty() ? 0 : messages.size() - 1; + for (size_t idx = messages.size(); idx > 0; idx--) { + const auto& m = messages[idx - 1]; + if (m.role == "user") { + bool is_tool_response = m.content.size() >= 31 && + m.content.substr(0, 15) == "" && + m.content.substr(m.content.size() - 16) == ""; + if (!is_tool_response) { + last_query_index = idx - 1; + break; + } + } + } + + for (size_t i = start_idx; i < messages.size(); i++) { + const auto& msg = messages[i]; + if (msg.role == "user" || (msg.role == "system" && i > 0)) { + result += "<|im_start|>" + msg.role + "\n" + msg.content + "<|im_end|>\n"; + } else if (msg.role == "assistant") { + if (i > last_query_index && i == messages.size() - 1) { + result += "<|im_start|>assistant\n\n\n\n\n"; + std::string content = msg.content; + size_t lstrip = content.find_first_not_of('\n'); + if (lstrip != std::string::npos) { + result += content.substr(lstrip); + } + } else { + result += "<|im_start|>assistant\n" + msg.content; + } + if (!msg.tool_calls.empty()) { + for (size_t j = 0; j < msg.tool_calls.size(); j++) { + if ((j == 0 && !msg.content.empty()) || j > 0) { + result += "\n"; + } + result += "\n{\"name\": \"" + msg.tool_calls[j].name + + "\", \"arguments\": " + msg.tool_calls[j].arguments + + "}\n"; + } + } + result += "<|im_end|>\n"; + } else if (msg.role == "tool") { + bool first_tool = (i == 0 || messages[i - 1].role != "tool"); + bool last_tool = (i == messages.size() - 1 || messages[i + 1].role != "tool"); + if (first_tool) { + result += "<|im_start|>user"; + } + result += "\n\n" + msg.content + "\n"; + if (last_tool) { + result += "<|im_end|>\n"; + } + } + } + + if (add_generation_prompt) { + result += "<|im_start|>assistant\n"; + if (!enable_thinking_if_supported) { + result += "\n\n\n\n"; + } + } + + return result; +} + +std::string Tokenizer::format_lfm2_style(const std::vector& messages, + bool add_generation_prompt, + const std::string& tools_json) const +{ + std::string result = "<|startoftext|>"; + + std::string sys_content; + bool has_system_msg = false; + for (const auto& msg : messages) { + if (msg.role == "system") { + sys_content = msg.content; + has_system_msg = true; + break; + } + } + + if (!tools_json.empty()) { + if (!sys_content.empty()) { + sys_content += "\n"; + } + sys_content += "List of tools: <|tool_list_start|>["; + if (!tools_json.empty()) { + sys_content += "\n"; + sys_content += tools_json; + sys_content += "\n"; + } + sys_content += "]<|tool_list_end|>"; + sys_content += "\n\nWhen you need to call a tool, use this exact format:\n"; + sys_content += "<|tool_call_start|>[function_name(arg1=\"value1\", arg2=\"value2\")]<|tool_call_end|>\n"; + sys_content += "You can call multiple tools by using multiple tool call blocks."; + } + + if (!sys_content.empty()) { + result += "<|im_start|>system\n"; + result += sys_content; + result += "<|im_end|>\n"; + } + + for (const auto& msg : messages) { + if (msg.role == "system" && has_system_msg) { + has_system_msg = false; + continue; + } + result += "<|im_start|>" + msg.role + "\n"; + if (msg.role == "tool") { + result += "<|tool_response_start|>"; + result += msg.content; + result += "<|tool_response_end|>"; + } else { + result += msg.content; + if (msg.role == "assistant" && !msg.tool_calls.empty()) { + for (const auto& tc : msg.tool_calls) { + result += "<|tool_call_start|>[{\"name\": \"" + tc.name + + "\", \"arguments\": " + tc.arguments + "}]<|tool_call_end|>"; + } + } + } + result += "<|im_end|>\n"; + } + + if (add_generation_prompt) { + result += "<|im_start|>assistant\n"; + } + + return result; +} + +static std::string format_tool_call_for_prompt(const std::string& name, const std::string& args_json, bool use_pipe_tags) { + const char* tag_start = use_pipe_tags ? "<|tool_call>" : ""; + const char* tag_end = use_pipe_tags ? "" : ""; + + std::string formatted_args = "{"; + if (!args_json.empty()) { + std::map args; + size_t pos = 0; + while (pos < args_json.length() && args_json[pos] != '{') pos++; + if (pos < args_json.length()) pos++; + + while (pos < args_json.length()) { + while (pos < args_json.length() && std::isspace(static_cast(args_json[pos]))) pos++; + if (pos >= args_json.length() || args_json[pos] == '}') break; + if (args_json[pos] == ',') { pos++; continue; } + if (args_json[pos] != '"') break; + pos++; + + std::string key; + while (pos < args_json.length() && args_json[pos] != '"') { + if (args_json[pos] == '\\' && pos + 1 < args_json.length()) { pos++; } + key += args_json[pos++]; + } + if (pos < args_json.length()) pos++; + + while (pos < args_json.length() && (std::isspace(static_cast(args_json[pos])) || args_json[pos] == ':')) pos++; + + std::string value; + if (pos < args_json.length() && args_json[pos] == '"') { + pos++; + std::string str_val; + while (pos < args_json.length() && args_json[pos] != '"') { + if (args_json[pos] == '\\' && pos + 1 < args_json.length()) { + pos++; + if (args_json[pos] == 'n') str_val += '\n'; + else if (args_json[pos] == 't') str_val += '\t'; + else str_val += args_json[pos]; + } else { + str_val += args_json[pos]; + } + pos++; + } + if (pos < args_json.length()) pos++; + value = "" + str_val + ""; + } else { + size_t val_start = pos; + while (pos < args_json.length() && args_json[pos] != ',' && args_json[pos] != '}') pos++; + value = args_json.substr(val_start, pos - val_start); + while (!value.empty() && std::isspace(static_cast(value.back()))) value.pop_back(); + } + + if (!key.empty()) args[key] = value; + } + + bool first = true; + for (const auto& [k, v] : args) { + if (!first) formatted_args += ","; + first = false; + formatted_args += k + ":" + v; + } + } + formatted_args += "}"; + + return std::string(tag_start) + "call:" + name + formatted_args + tag_end; +} + +std::string Tokenizer::format_gemma4_style(const std::vector& messages, bool add_generation_prompt, + const std::string& tools_json, bool enable_thinking_if_supported) const { + std::string result = ""; + + std::string sys_content; + size_t first_msg = 0; + if (!messages.empty() && (messages[0].role == "system" || messages[0].role == "developer")) { + sys_content = messages[0].content; + first_msg = 1; + } + + if (enable_thinking_if_supported || !sys_content.empty() || !tools_json.empty()) { + result += "<|turn>system\n"; + if (enable_thinking_if_supported) { + result += "<|think|>"; + } + result += sys_content; + result += tools_json; + result += "\n"; + } + + auto strip_channel = [](const std::string& text) -> std::string { + const std::string open_tag = "<|channel>"; + const std::string close_tag = ""; + std::string out; + size_t pos = 0; + while (pos < text.size()) { + size_t open_pos = text.find(open_tag, pos); + if (open_pos == std::string::npos) { + out += text.substr(pos); + break; + } + out += text.substr(pos, open_pos - pos); + size_t close_pos = text.find(close_tag, open_pos + open_tag.size()); + if (close_pos == std::string::npos) { + break; + } + pos = close_pos + close_tag.size(); + } + return out; + }; + + auto compute_soft_tokens = [&](const std::string& image_path) -> size_t { + int w = 0, h = 0, c = 0; + unsigned char* data = stbi_load(image_path.c_str(), &w, &h, &c, 3); + if (!data) return 0; + stbi_image_free(data); + + uint32_t p = vision_patch_size_; + uint32_t k = vision_pooling_kernel_size_; + uint32_t side = k * p; + uint32_t max_patches = vision_default_output_length_ * k * k; + float factor = std::sqrt(static_cast(max_patches) * p * p / + (static_cast(h) * w)); + int th = static_cast(std::floor(factor * h / side)) * side; + int tw = static_cast(std::floor(factor * w / side)) * side; + if (th == 0) th = side; + if (tw == 0) tw = side; + return static_cast((th / p / k) * (tw / p / k)); + }; + + for (size_t i = first_msg; i < messages.size(); i++) { + const auto& msg = messages[i]; + std::string role = (msg.role == "assistant") ? "model" : msg.role; + result += "<|turn>" + role + "\n"; + if (role == "model") { + result += strip_channel(msg.content); + if (!msg.tool_calls.empty()) { + for (const auto& tc : msg.tool_calls) { + result += format_tool_call_for_prompt(tc.name, tc.arguments, true); + } + } + } else { + for (const auto& image_path : msg.images) { + size_t n = compute_soft_tokens(image_path); + if (n > 0) { + result += "\n\n<|image>"; + for (size_t j = 0; j < n; j++) + result += "<|image|>"; + result += "\n\n"; + } + } + result += msg.content; + if (msg.audio_soft_token_count > 0) { + result += "<|audio>"; + for (size_t j = 0; j < msg.audio_soft_token_count; j++) + result += "<|audio|>"; + result += ""; + } + } + result += "\n"; + } + + if (add_generation_prompt) { + result += "<|turn>model\n"; + } + + return result; +} + +std::string Tokenizer::format_lfm2_vl_style( + const std::vector& messages, + bool add_generation_prompt, + const std::string& tools_json) const +{ + if (!tools_json.empty()) { + return "ERROR: Tool calls are not supported for LFM2-VL models"; + } + + std::string result = "<|startoftext|>"; + + for (const auto& msg : messages) { + result += "<|im_start|>" + msg.role + "\n"; + for (const auto& image_path : msg.images) { + int width = 0, height = 0, channels = 0; + unsigned char* img_data = stbi_load(image_path.c_str(), &width, &height, &channels, 0); + + if (img_data) { + Siglip2Preprocessor preprocessor; + auto shape_result = preprocessor.compute_spatial_shapes(height, width); + int downsample_factor = 2; + bool use_thumbnail = true; + int grid_rows = shape_result.grid_rows; + int grid_cols = shape_result.grid_cols; + int num_tiles = grid_rows * grid_cols; + result += "<|image_start|>"; + + if (num_tiles > 1) { + for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { + int row = tile_idx / grid_cols; + int col = tile_idx % grid_cols; + + result += "<|img_row_" + std::to_string(row + 1) + "_col_" + std::to_string(col + 1) + "|>"; + auto [tile_height, tile_width] = shape_result.shapes[tile_idx]; + int tile_tokens = (tile_height * tile_width) / (downsample_factor * downsample_factor); + + for (int t = 0; t < tile_tokens; ++t) { + result += ""; + } + } + if (use_thumbnail && static_cast(num_tiles) < shape_result.shapes.size()) { + result += "<|img_thumbnail|>"; + + auto [thumb_height, thumb_width] = shape_result.shapes[num_tiles]; + int thumbnail_tokens = (thumb_height * thumb_width) / (downsample_factor * downsample_factor); + + for (int t = 0; t < thumbnail_tokens; ++t) { + result += ""; + } + } + } else if (num_tiles == 1) { + auto [thumb_height, thumb_width] = shape_result.shapes[0]; + int thumbnail_tokens = (thumb_height * thumb_width) / (downsample_factor * downsample_factor); + + for (int t = 0; t < thumbnail_tokens; ++t) { + result += ""; + } + } + + result += "<|image_end|>"; + + stbi_image_free(img_data); + } else { + result += ""; + } + } + result += msg.content; + + result += "<|im_end|>\n"; + } + + if (add_generation_prompt) { + result += "<|im_start|>assistant\n"; + } + + return result; +} + + +std::string Tokenizer::format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const { + auto trim = [](const std::string& s) -> std::string { + size_t start = 0; + while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; + size_t end = s.size(); + while (end > start && std::isspace(static_cast(s[end - 1]))) --end; + return s.substr(start, end - start); + }; + + std::string result = ""; + + std::string system_content; + size_t start_idx = 0; + + if (!messages.empty() && (messages[0].role == "system" || messages[0].role == "developer")) { + system_content = messages[0].content; + start_idx = 1; + } + + if (!tools_json.empty() || !system_content.empty()) { + result += "developer\n"; + if (!system_content.empty()) { + result += trim(system_content); + } + if (!tools_json.empty()) { + result += tools_json; + } + result += "\n"; + } + + std::string prev_message_type; + + for (size_t i = start_idx; i < messages.size(); i++) { + const auto& msg = messages[i]; + + if (msg.role == "tool") { + std::string func_name = msg.name.empty() ? "tool" : msg.name; + result += "response:" + func_name + "{" + msg.content + "}"; + prev_message_type = "tool_response"; + + } else if (msg.role == "user") { + if (prev_message_type != "tool_response") { + result += "user\n"; + } + result += trim(msg.content); + result += "\n"; + prev_message_type = "content"; + + } else if (msg.role == "assistant" || msg.role == "model") { + if (prev_message_type != "tool_response") { + result += "model\n"; + } + if (!msg.content.empty()) { + result += trim(msg.content); + } + if (!msg.tool_calls.empty()) { + for (const auto& tc : msg.tool_calls) { + result += format_tool_call_for_prompt(tc.name, tc.arguments, false); + } + if (i == messages.size() - 1) { + result += ""; + } + prev_message_type = "tool_call"; + } else { + result += "\n"; + prev_message_type = "content"; + } + } + } + + if (add_generation_prompt) { + if (prev_message_type != "tool_response" && prev_message_type != "tool_call") { + result += "model\n"; + } + } + + return result; +} + +std::string Tokenizer::format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const { + std::string result = "<|begin_of_text|>"; + + std::string system_block; + for (const auto& msg : messages) { + if (msg.role == "system") { system_block = msg.content; break; } + } + + if (!tools_json.empty()) { + std::string tool_desc = + "<|begin_of_tool_description|>Tool calling capabilities.\n" + "You may call one or more functions to assist with the user query. " + "You have the following functions available:\n" + "```json\n" + tools_json + "\n```\n" + "For tool call returns, you MUST use the following format:\n" + "{\"name\": \"function-name\", \"arguments\": {\"param1\": \"value1\", \"param2\": \"value2\"}}\n" + "<|end_of_tool_description|>"; + system_block = system_block.empty() ? tool_desc : system_block + "\n\n" + tool_desc; + } + + result += system_block; + + bool is_last_user = false; + bool is_tool = false; + bool is_output_first = true; + + for (const auto& msg : messages) { + if (msg.role == "system") { + continue; + } else if (msg.role == "user") { + is_last_user = true; is_tool = false; + result += "<|User|>" + msg.content; + } else if (msg.role == "tool") { + is_tool = true; is_last_user = false; + if (is_output_first) { + result += "<|User|>" + msg.content + ""; + is_output_first = false; + } else { + result += "\n" + msg.content + ""; + } + } else if (msg.role == "assistant" || msg.role == "model") { + is_last_user = false; is_tool = false; is_output_first = true; + result += "<|Assistant|>" + msg.content; + if (!msg.tool_calls.empty()) { + for (const auto& tc : msg.tool_calls) { + result += "{\"name\": \"" + tc.name + + "\", \"arguments\": " + tc.arguments + "}"; + } + } + result += "<|end_of_text|>"; + } + } + + if (add_generation_prompt && (is_last_user || is_tool)) { + result += "<|Assistant|>"; + } + + return result; +} + +} // namespace engine +} // namespace cactus diff --git a/cactus-transpile/ffi/cactus_cloud.cpp b/cactus-transpile/ffi/cactus_cloud.cpp new file mode 100644 index 000000000..2116f48f3 --- /dev/null +++ b/cactus-transpile/ffi/cactus_cloud.cpp @@ -0,0 +1,452 @@ +#include "cactus_cloud.h" +#include "telemetry/telemetry.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef CACTUS_USE_CURL +#include +#endif + +namespace cactus { +namespace ffi { + +namespace { + +std::string compact_error_detail(std::string detail) { + detail = trim_string(detail); + if (detail.empty()) return {}; + + for (char& c : detail) { + if (c == '\n' || c == '\r' || c == '\t') c = ' '; + } + while (detail.find(" ") != std::string::npos) { + detail.erase(detail.find(" "), 1); + } + + constexpr size_t kMaxLen = 160; + if (detail.size() > kMaxLen) { + detail = detail.substr(0, kMaxLen) + "..."; + } + return detail; +} + +} // namespace + +std::string resolve_cloud_api_key(const char* cloud_key_param) { + const char* env_cloud = std::getenv("CACTUS_CLOUD_KEY"); + const char* env_cloud_legacy = std::getenv("CACTUS_CLOUD_API_KEY"); // keeping this cuz Matthew Hayes had some code reliant on Cactus_cloud_api_key + std::string resolved_cloud_key; + bool key_from_param_or_env = false; + + if (cloud_key_param && *cloud_key_param) { + resolved_cloud_key = cloud_key_param; + key_from_param_or_env = true; + } else if (env_cloud && *env_cloud) { + resolved_cloud_key = env_cloud; + key_from_param_or_env = true; + } else if (env_cloud_legacy && *env_cloud_legacy) { + resolved_cloud_key = env_cloud_legacy; + key_from_param_or_env = true; + } else { + resolved_cloud_key = cactus::telemetry::loadCachedCloudApiKey(); + } + + resolved_cloud_key = trim_string(resolved_cloud_key); + + const bool should_cache = + !resolved_cloud_key.empty() && + key_from_param_or_env; + if (should_cache) { + cactus::telemetry::cacheCloudApiKey(resolved_cloud_key.c_str()); + } + + return resolved_cloud_key; +} + +namespace { + +#ifdef CACTUS_USE_CURL +static std::atomic g_warned_missing_cloud_api_key{false}; + +static void apply_curl_tls_trust(CURL* curl) { + if (!curl) return; + const char* ca_bundle = std::getenv("CACTUS_CA_BUNDLE"); + if (ca_bundle && ca_bundle[0] != '\0') { + curl_easy_setopt(curl, CURLOPT_CAINFO, ca_bundle); + } +#if defined(__ANDROID__) + const char* ca_path = std::getenv("CACTUS_CA_PATH"); + if (ca_path && ca_path[0] != '\0') { + curl_easy_setopt(curl, CURLOPT_CAPATH, ca_path); + } else { + curl_easy_setopt(curl, CURLOPT_CAPATH, "/system/etc/security/cacerts"); + } +#endif +} + +static size_t curl_write_cb(void* ptr, size_t size, size_t nmemb, void* userdata) { + auto* s = static_cast(userdata); + s->append(static_cast(ptr), size * nmemb); + return size * nmemb; +} + +static bool read_file_bytes(const std::string& path, std::vector& out) { + std::ifstream in(path, std::ios::binary); + if (!in.is_open()) return false; + in.seekg(0, std::ios::end); + std::streamsize size = in.tellg(); + if (size <= 0) return false; + in.seekg(0, std::ios::beg); + + out.resize(static_cast(size)); + if (!in.read(reinterpret_cast(out.data()), size)) return false; + return true; +} + +static std::string infer_mime_type(const std::string& path) { + auto dot = path.find_last_of('.'); + if (dot == std::string::npos) return "image/jpeg"; + std::string ext = path.substr(dot + 1); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + + if (ext == "png") return "image/png"; + if (ext == "webp") return "image/webp"; + if (ext == "gif") return "image/gif"; + return "image/jpeg"; +} + +static std::string build_cloud_text_prompt(const CloudCompletionRequest& request) { + std::ostringstream oss; + oss << "You are continuing an existing assistant conversation.\\n"; + oss << "Output contract:\\n"; + oss << "1) Never include role prefixes like 'assistant:'.\\n"; + oss << "2) Never include markdown/code fences/backticks.\\n"; + oss << "3) Return only the final assistant answer text unless a tool call is required.\\n"; + oss << "4) If a tool call is required, return ONLY JSON with this exact shape:\\n"; + oss << "[{\"name\":\"tool_name\",\"arguments\":{\"arg\":\"value\"}}]\\n"; + oss << "5) Do not include any prose before or after that JSON tool-call output.\\n"; + oss << "\\nConversation:\\n"; + for (const auto& m : request.messages) { + oss << "[" << m.role << "] " << m.content << "\\n"; + } + + if (!request.tools.empty()) { + oss << "\\nAvailable tools JSON (use only these tool names and arguments):\\n"; + oss << serialize_tools_json(request.tools) << "\\n"; + oss << "If tools are relevant, prefer the strict JSON tool-call output contract above.\\n"; + } + + if (!request.local_output.empty()) { + oss << "\\nLocal model draft (useful fallback reference, may be low confidence):\\n"; + oss << request.local_output << "\\n"; + } + + return oss.str(); +} + +static std::string call_cloud_endpoint(const std::string& url, + const std::string& payload, + long timeout_ms, + const char* cloud_key_param, + std::string& err_out) { + std::string api_key = resolve_cloud_api_key(cloud_key_param); + if (api_key.empty()) { + if (!g_warned_missing_cloud_api_key.exchange(true)) { + CACTUS_LOG_WARN("cloud_handoff", "No cloud key found (cloud_key param, CACTUS_CLOUD_KEY, or CACTUS_CLOUD_API_KEY env); cloud handoff will fall back to local output"); + } + err_out = "missing_api_key"; + return {}; + } + + CURL* curl = curl_easy_init(); + if (!curl) { + err_out = "curl_init_failed"; + return {}; + } + + std::string response_body; + struct curl_slist* headers = nullptr; + headers = curl_slist_append(headers, ("X-API-Key: " + api_key).c_str()); + headers = curl_slist_append(headers, "Content-Type: application/json"); + + const char* extra_hdrs = std::getenv("CACTUS_CLOUD_HEADERS"); + if (extra_hdrs && extra_hdrs[0] != '\0') { + std::istringstream stream(extra_hdrs); + std::string pair; + while (std::getline(stream, pair, ',')) { + auto eq = pair.find('='); + if (eq != std::string::npos && eq > 0) { + std::string header = pair.substr(0, eq) + ": " + pair.substr(eq + 1); + headers = curl_slist_append(headers, header.c_str()); + } + } + } + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.size())); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_body); + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout_ms); + + if (!env_flag_enabled("CACTUS_CLOUD_STRICT_SSL")) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); + } else { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); + apply_curl_tls_trust(curl); + } + + CURLcode res = curl_easy_perform(curl); + long http_status = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_status); + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (res != CURLE_OK) { + err_out = curl_easy_strerror(res); + return {}; + } + + if (http_status >= 400) { + std::string detail = json_string_field(response_body, "error"); + if (detail.empty()) detail = json_string_field(response_body, "message"); + if (detail.empty()) detail = json_string_field(response_body, "detail"); + detail = compact_error_detail(detail); + + err_out = "http_" + std::to_string(http_status); + if (!detail.empty()) { + err_out += ":" + detail; + } + return {}; + } + + return response_body; +} +#endif + +} // namespace + +std::string cloud_base64_encode(const uint8_t* data, size_t len) { + static const char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + out.reserve(((len + 2) / 3) * 4); + for (size_t i = 0; i < len; i += 3) { + uint32_t n = static_cast(data[i]) << 16; + if (i + 1 < len) n |= static_cast(data[i + 1]) << 8; + if (i + 2 < len) n |= static_cast(data[i + 2]); + out += table[(n >> 18) & 0x3F]; + out += table[(n >> 12) & 0x3F]; + out += (i + 1 < len) ? table[(n >> 6) & 0x3F] : '='; + out += (i + 2 < len) ? table[n & 0x3F] : '='; + } + return out; +} + +std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes) { + constexpr uint32_t sample_rate = 16000; + constexpr uint16_t channels = 1; + constexpr uint16_t bits = 16; + const uint32_t byte_rate = sample_rate * channels * bits / 8; + const uint16_t block_align = channels * bits / 8; + const uint32_t data_size = static_cast(pcm_bytes); + const uint32_t file_size = 36 + data_size; + + std::vector wav(44 + pcm_bytes); + auto w16 = [&](size_t off, uint16_t v) { + wav[off] = v & 0xFF; + wav[off + 1] = v >> 8; + }; + auto w32 = [&](size_t off, uint32_t v) { + wav[off] = v & 0xFF; + wav[off + 1] = (v >> 8) & 0xFF; + wav[off + 2] = (v >> 16) & 0xFF; + wav[off + 3] = (v >> 24) & 0xFF; + }; + + std::memcpy(wav.data(), "RIFF", 4); + w32(4, file_size); + std::memcpy(wav.data() + 8, "WAVE", 4); + std::memcpy(wav.data() + 12, "fmt ", 4); + w32(16, 16); + w16(20, 1); + w16(22, channels); + w32(24, sample_rate); + w32(28, byte_rate); + w16(32, block_align); + w16(34, bits); + std::memcpy(wav.data() + 36, "data", 4); + w32(40, data_size); + std::memcpy(wav.data() + 44, pcm, pcm_bytes); + return wav; +} + +CloudResponse cloud_transcribe_request(const std::string& audio_b64, + const std::string& fallback_text, + long timeout_seconds, + const char* cloud_key) { +#ifdef CACTUS_USE_CURL + std::string base = env_or_default("CACTUS_CLOUD_API_BASE", "https://104.198.76.3/api/v1"); + std::string endpoint = base + "/transcribe"; + + std::string payload = "{\"audio\":\"" + audio_b64 + "\",\"mime_type\":\"audio/wav\",\"language\":\"en-US\"}"; + + std::string err; + std::string body = call_cloud_endpoint(endpoint, payload, timeout_seconds * 1000L, cloud_key, err); + if (body.empty()) { + return {fallback_text, "", false, err.empty() ? "request_failed" : err}; + } + + std::string transcript = json_string_field(body, "transcript"); + if (transcript.empty()) { + transcript = json_string_field(body, "text"); + } + if (transcript.empty()) { + transcript = json_string_field(body, "response"); + } + if (transcript.empty()) { + transcript = json_string_field(body, "analysis"); + } + if (transcript.empty()) { + std::string detail = json_string_field(body, "error"); + if (detail.empty()) detail = json_string_field(body, "message"); + if (detail.empty()) detail = json_string_field(body, "detail"); + detail = compact_error_detail(detail); + if (!detail.empty()) { + return {fallback_text, "", false, "missing_transcript:" + detail}; + } + return {fallback_text, "", false, "missing_transcript"}; + } + + std::string api_key_hash = json_string_field(body, "api_key_hash"); + return {transcript, api_key_hash, true, ""}; +#else + (void)audio_b64; + (void)timeout_seconds; + return {fallback_text, "", false, "curl_not_enabled"}; +#endif +} + +CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, + long timeout_ms) { +#ifdef CACTUS_USE_CURL + std::string base = env_or_default("CACTUS_CLOUD_API_BASE", "https://104.198.76.3/api/v1"); + std::string text_model = env_or_default("CACTUS_CLOUD_TEXT_MODEL", "gemini-2.5-flash"); + std::string vlm_model = env_or_default("CACTUS_CLOUD_VLM_MODEL", "gemini-2.5-flash"); + + std::string endpoint; + std::string payload; + + if (request.has_images) { + std::vector image_paths; + for (const auto& message : request.messages) { + for (const auto& image : message.images) { + if (!image.empty()) { + image_paths.push_back(image); + } + } + } + + if (image_paths.size() > 1) { + return {false, false, "", {}, "multiple_images_not_supported"}; + } + + std::string image_path; + if (!image_paths.empty()) { + image_path = image_paths.back(); + } + if (image_path.empty()) { + return {false, false, "", {}, "missing_image_path"}; + } + + std::vector img_bytes; + if (!read_file_bytes(image_path, img_bytes)) { + return {false, false, "", {}, "image_read_failed"}; + } + + std::string img_b64 = cloud_base64_encode(img_bytes.data(), img_bytes.size()); + std::string mime = infer_mime_type(image_path); + std::string prompt = build_cloud_text_prompt(request); + + endpoint = base + "/vlm"; + payload = "{" + "\"image\":\"" + img_b64 + "\"," + "\"mime_type\":\"" + mime + "\"," + "\"prompt\":\"" + escape_json_string(prompt) + "\"," + "\"language\":\"en-US\"," + "\"model\":\"" + escape_json_string(vlm_model) + "\"" + "}"; + } else { + endpoint = base + "/text"; + std::string text = build_cloud_text_prompt(request); + payload = "{" + "\"text\":\"" + escape_json_string(text) + "\"," + "\"language\":\"en-US\"," + "\"model\":\"" + escape_json_string(text_model) + "\"" + "}"; + } + + std::string err; + const char* cloud_key = request.cloud_key.empty() ? nullptr : request.cloud_key.c_str(); + std::string body = call_cloud_endpoint(endpoint, payload, timeout_ms, cloud_key, err); + if (body.empty()) { + return {false, false, "", {}, err.empty() ? "request_failed" : err}; + } + + std::vector function_calls; + std::string calls_json = json_array_field(body, "function_calls"); + if (calls_json != "[]") { + function_calls = split_json_array(calls_json); + } + + std::string response = json_string_field(body, "text"); + if (response.empty()) { + response = json_string_field(body, "analysis"); + } + + if (!response.empty() && function_calls.empty()) { + std::string regular_response; + parse_function_calls_from_response(response, regular_response, function_calls); + response = regular_response; + } + + if (!response.empty() && function_calls.empty()) { + auto first = response.find_first_not_of(" \t\n\r"); + auto last = response.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) { + std::string trimmed = response.substr(first, last - first + 1); + if (!trimmed.empty() && trimmed.front() == '[' && trimmed.back() == ']' && + trimmed.find("\"name\"") != std::string::npos) { + function_calls = split_json_array(trimmed); + response.clear(); + } + } + } + + if (response.empty() && function_calls.empty()) { + return {false, false, "", {}, "missing_text"}; + } + + CloudCompletionResult out; + out.ok = true; + out.used_cloud = true; + out.response = response; + out.function_calls = std::move(function_calls); + return out; +#else + (void)request; + (void)timeout_ms; + return {false, false, "", {}, "curl_not_enabled"}; +#endif +} + +} // namespace ffi +} // namespace cactus diff --git a/cactus-transpile/ffi/cactus_cloud.h b/cactus-transpile/ffi/cactus_cloud.h new file mode 100644 index 000000000..e61841d7c --- /dev/null +++ b/cactus-transpile/ffi/cactus_cloud.h @@ -0,0 +1,48 @@ +#ifndef CACTUS_CLOUD_H +#define CACTUS_CLOUD_H + +#include "cactus_utils.h" +#include +#include + +namespace cactus { +namespace ffi { + +struct CloudResponse { + std::string transcript; + std::string api_key_hash; + bool used_cloud = false; + std::string error; +}; + +struct CloudCompletionRequest { + std::vector messages; + std::vector tools; + std::string local_output; + std::vector local_function_calls; + bool has_images = false; + std::string cloud_key; +}; + +struct CloudCompletionResult { + bool ok = false; + bool used_cloud = false; + std::string response; + std::vector function_calls; + std::string error; +}; + +std::string cloud_base64_encode(const uint8_t* data, size_t len); +std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes); +std::string resolve_cloud_api_key(const char* cloud_key_param); +CloudResponse cloud_transcribe_request(const std::string& audio_b64, + const std::string& fallback_text, + long timeout_seconds = 15L, + const char* cloud_key = nullptr); +CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, + long timeout_ms); + +} // namespace ffi +} // namespace cactus + +#endif // CACTUS_CLOUD_H diff --git a/cactus-transpile/ffi/cactus_complete.cpp b/cactus-transpile/ffi/cactus_complete.cpp new file mode 100644 index 000000000..cd25d88a1 --- /dev/null +++ b/cactus-transpile/ffi/cactus_complete.cpp @@ -0,0 +1,1177 @@ +#include "cactus_ffi.h" +#include "cactus_cloud.h" +#include "cactus_utils.h" +#include "telemetry/telemetry.h" +#include "../../libs/audio/wav.h" +#include +#include +#include +#include +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; + +static constexpr size_t ROLLING_ENTROPY_WINDOW = 10; + +namespace { + +std::vector> extract_schema_property_types(const std::string& schema); +std::vector extract_schema_required(const std::string& schema); + +std::string extract_last_user_query(const std::vector& messages) { + for (auto it = messages.rbegin(); it != messages.rend(); ++it) { + if (it->role == "user") { + return it->content; + } + } + return {}; +} + +std::vector encode_needle_tools_suffix(Tokenizer* tokenizer, const std::string& formatted_tools) { + auto prefix = tokenizer->encode(""); + auto body = tokenizer->encode(formatted_tools); + std::vector tokens; + tokens.reserve(prefix.size() + body.size() + 1); + tokens.insert(tokens.end(), prefix.begin(), prefix.end()); + tokens.insert(tokens.end(), body.begin(), body.end()); + tokens.push_back(tokenizer->get_eos_token()); + return tokens; +} + +void inject_rag_context(CactusModelHandle* handle, std::vector& messages) { + if (!handle->corpus_index) return; + + std::string query = extract_last_user_query(messages); + if (query.empty()) return; + + std::string rag_context = retrieve_rag_context(handle, query); + if (rag_context.empty()) return; + + if (!messages.empty() && messages[0].role == "system") { + messages[0].content = rag_context + messages[0].content; + } else { + ChatMessage system_msg; + system_msg.role = "system"; + system_msg.content = rag_context + "Answer the user's question using ONLY the context above. Do not use any prior knowledge. If the answer cannot be found in the context, respond with \"I don't have enough information to answer that.\""; + messages.insert(messages.begin(), system_msg); + } +} + +std::vector build_tool_constraint_specs(const std::vector& tools) { + std::vector specs; + specs.reserve(tools.size()); + + for (const auto& tool : tools) { + ToolConstraintSpec spec; + spec.name = tool.name; + + auto schema_it = tool.parameters.find("schema"); + if (schema_it != tool.parameters.end()) { + auto properties = extract_schema_property_types(schema_it->second); + spec.parameter_names.reserve(properties.size()); + for (const auto& [name, _] : properties) { + spec.parameter_names.push_back(name); + } + spec.required_parameter_names = extract_schema_required(schema_it->second); + } + + specs.push_back(std::move(spec)); + } + + return specs; +} + +void strip_thinking_from_cache(CactusModelHandle* handle, + const std::vector& generated_tokens, + size_t prompt_len) { + const auto& cfg = handle->model->get_config(); + uint32_t open_id = cfg.channel_open_token_id; + uint32_t close_id = cfg.channel_close_token_id; + auto ranges = find_channel_token_ranges(generated_tokens, prompt_len, + open_id, close_id); + if (ranges.empty()) return; + + handle->model->remove_thinking_tokens(ranges); + for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) { + auto start = handle->processed_tokens.begin() + it->first; + handle->processed_tokens.erase(start, start + it->second); + } +} + +void setup_tool_constraints(CactusModelHandle* handle, const std::vector& tools, + bool force_tools, float& temperature) { + if (!force_tools || tools.empty()) return; + + handle->model->set_tool_constraints(build_tool_constraint_specs(tools)); + + if (temperature == 0.0f) { + temperature = 0.01f; + } +} + +size_t find_json_block_end(const std::string& json, size_t start) { + if (start >= json.size() || json[start] != '{') { + return std::string::npos; + } + + int depth = 1; + bool in_string = false; + bool escaped = false; + size_t pos = start + 1; + while (pos < json.size() && depth > 0) { + char c = json[pos]; + if (in_string) { + if (escaped) { + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '"') { + in_string = false; + } + } else { + if (c == '"') { + in_string = true; + } else if (c == '{') { + depth++; + } else if (c == '}') { + depth--; + } + } + ++pos; + } + + return depth == 0 ? pos : std::string::npos; +} + +std::string extract_json_object_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t key_pos = json.find(pattern); + if (key_pos == std::string::npos) { + return {}; + } + + size_t object_start = json.find('{', key_pos + pattern.size()); + if (object_start == std::string::npos) { + return {}; + } + + size_t object_end = find_json_block_end(json, object_start); + if (object_end == std::string::npos) { + return {}; + } + + return json.substr(object_start, object_end - object_start); +} + +std::vector> extract_schema_property_types(const std::string& schema) { + std::vector> properties; + std::string properties_object = extract_json_object_field(schema, "properties"); + if (properties_object.empty() || properties_object.size() < 2) { + return properties; + } + + size_t pos = 1; + while (pos + 1 < properties_object.size()) { + size_t key_start = properties_object.find('"', pos); + if (key_start == std::string::npos || key_start + 1 >= properties_object.size()) { + break; + } + size_t key_end = properties_object.find('"', key_start + 1); + if (key_end == std::string::npos) { + break; + } + + std::string name = properties_object.substr(key_start + 1, key_end - key_start - 1); + size_t value_start = properties_object.find('{', key_end); + if (value_start == std::string::npos) { + break; + } + size_t value_end = find_json_block_end(properties_object, value_start); + if (value_end == std::string::npos) { + break; + } + + std::string value = properties_object.substr(value_start, value_end - value_start); + std::string type = "string"; + std::string type_pattern = "\"type\":\""; + size_t type_pos = value.find(type_pattern); + if (type_pos != std::string::npos) { + size_t type_start = type_pos + type_pattern.size(); + size_t type_end = value.find('"', type_start); + if (type_end != std::string::npos) { + type = value.substr(type_start, type_end - type_start); + } + } else if (value.find("\"enum\"") != std::string::npos) { + type = "string"; + } else if (value.find("\"properties\"") != std::string::npos) { + type = "object"; + } + + properties.emplace_back(std::move(name), std::move(type)); + pos = value_end; + } + + return properties; +} + +std::vector extract_schema_required(const std::string& schema) { + std::vector required; + std::string key = "\"required\""; + size_t key_pos = schema.find(key); + if (key_pos == std::string::npos) return required; + size_t arr_start = schema.find('[', key_pos + key.size()); + if (arr_start == std::string::npos) return required; + size_t arr_end = schema.find(']', arr_start); + if (arr_end == std::string::npos) return required; + size_t pos = arr_start + 1; + while (pos < arr_end) { + size_t qs = schema.find('"', pos); + if (qs == std::string::npos || qs >= arr_end) break; + size_t qe = schema.find('"', qs + 1); + if (qe == std::string::npos || qe > arr_end) break; + required.push_back(schema.substr(qs + 1, qe - qs - 1)); + pos = qe + 1; + } + return required; +} + +std::string serialize_needle_tools_json(const std::vector& tools, const char* raw_tools_json) { + if (tools.empty()) { + return (raw_tools_json && std::strlen(raw_tools_json) > 0) ? raw_tools_json : "[]"; + } + + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < tools.size(); ++i) { + if (i > 0) { + oss << ","; + } + + oss << "{\"name\":\"" << escape_json_string(tools[i].name) << "\""; + if (!tools[i].description.empty()) { + oss << ",\"description\":\"" << escape_json_string(tools[i].description) << "\""; + } + + oss << ",\"parameters\":"; + auto schema_it = tools[i].parameters.find("schema"); + if (schema_it == tools[i].parameters.end()) { + oss << "{}"; + } else { + auto properties = extract_schema_property_types(schema_it->second); + if (properties.empty()) { + oss << schema_it->second; + } else { + oss << "{"; + for (size_t p = 0; p < properties.size(); ++p) { + if (p > 0) { + oss << ","; + } + oss << "\"" << escape_json_string(properties[p].first) << "\":" + << "\"" << escape_json_string(properties[p].second) << "\""; + } + oss << "}"; + } + } + + oss << "}"; + } + oss << "]"; + return oss.str(); +} + +std::vector> build_stop_sequences( + Tokenizer* tokenizer, + const std::vector& stop_sequences, + Config::ModelType model_type, + bool has_tools +) { + std::vector> stop_token_sequences; + stop_token_sequences.push_back({tokenizer->get_eos_token()}); + + std::vector sequences = stop_sequences; + if (sequences.empty()) { + std::string default_stop = tokenizer->get_default_stop_sequence(); + if (!default_stop.empty()) { + sequences.push_back(default_stop); + } + } + for (const auto& stop_seq : sequences) { + stop_token_sequences.push_back(tokenizer->encode(stop_seq)); + } + + if ((model_type == Config::ModelType::GEMMA || model_type == Config::ModelType::GEMMA3N) && has_tools) { + stop_token_sequences.push_back(tokenizer->encode("")); + stop_token_sequences.push_back(tokenizer->encode("")); + } + + if (model_type == Config::ModelType::GEMMA4) { + stop_token_sequences.push_back(tokenizer->encode("")); + if (has_tools) { + stop_token_sequences.push_back(tokenizer->encode("")); + stop_token_sequences.push_back(tokenizer->encode("<|tool_response>")); + } + } + + return stop_token_sequences; +} + +void trim_stop_suffix(std::vector& generated_tokens, + const std::vector>& stop_token_sequences, + bool include_stop_sequences) { + if (include_stop_sequences) return; + for (const auto& stop_seq : stop_token_sequences) { + if (stop_seq.empty()) continue; + if (generated_tokens.size() >= stop_seq.size() && + std::equal(stop_seq.rbegin(), stop_seq.rend(), generated_tokens.rbegin())) { + generated_tokens.resize(generated_tokens.size() - stop_seq.size()); + break; + } + } +} + +void reset_cache(CactusModelHandle* handle) { + handle->model->reset_cache(); + handle->processed_tokens.clear(); + handle->processed_images.clear(); +} + +struct PrefillResult { + std::vector remaining_tokens; + size_t prefilled_count = 0; + bool was_prefix = false; + bool was_exact_match = false; +}; + +struct EntropyState { + std::vector window; + float window_sum = 0.0f; + float total_sum = 0.0f; + size_t total_count = 0; + bool spike_handoff = false; + + void add(float entropy) { + window.push_back(entropy); + window_sum += entropy; + total_sum += entropy; + total_count++; + + if (window.size() > ROLLING_ENTROPY_WINDOW) { + window_sum -= window.front(); + window.erase(window.begin()); + } + } + + float rolling_confidence() const { + return 1.0f - (window_sum / window.size()); + } + + float mean_confidence() const { + return 1.0f - (total_sum / static_cast(total_count)); + } +}; + +struct PreparedPrompt { + InferenceOptions options; + Config::ModelType model_type = Config::ModelType::QWEN; + std::vector image_paths; + std::vector audio_paths; + std::vector messages; + std::vector tools; + std::vector tokens; + size_t context_token_count = 0; + std::vector> images; + + std::vector audio_features; + size_t audio_num_frames = 0; + + bool has_images() const { + return std::any_of(images.begin(), images.end(), + [](const auto& msg_imgs) { return !msg_imgs.empty(); }); + } + + bool has_audio() const { + return !audio_features.empty(); + } +}; + +CactusModelHandle::ProcessedImage image_signature(const std::string& image_path) { + std::filesystem::path normalized_path(image_path); + std::error_code ec; + + auto absolute_path = std::filesystem::absolute(normalized_path, ec); + if (!ec) { + normalized_path = absolute_path; + } + + CactusModelHandle::ProcessedImage image; + image.path = normalized_path.string(); + + ec.clear(); + auto status = std::filesystem::status(normalized_path, ec); + if (!ec && std::filesystem::is_regular_file(status)) { + std::error_code time_ec; + auto mtime = std::filesystem::last_write_time(normalized_path, time_ec); + if (!time_ec) { + image.last_modified_timestamp = static_cast(mtime.time_since_epoch().count()); + } + } + + return image; +} + +std::vector> images_from_message(const std::vector& messages) { + std::vector> message_signatures; + message_signatures.reserve(messages.size()); + + for (const auto& message : messages) { + std::vector image_signatures; + image_signatures.reserve(message.images.size()); + for (const auto& image_path : message.images) { + image_signatures.push_back(image_signature(image_path)); + } + message_signatures.push_back(std::move(image_signatures)); + } + + return message_signatures; +} + +bool image_context_prefix_matches( + const std::vector>& prefix, + const std::vector>& full +) { + return prefix.size() <= full.size() && + std::equal(prefix.begin(), prefix.end(), full.begin()); +} + +bool prompt_context_matches( + const CactusModelHandle* handle, + const PreparedPrompt& prompt +) { + if (handle->processed_tokens.empty()) { + return false; + } + if (handle->model && + handle->model->get_config().model_type == Config::ModelType::NEEDLE && + prompt.tokens.size() != handle->processed_tokens.size() + 1) { + return false; + } + if (prompt.context_token_count < handle->processed_tokens.size()) { + return false; + } + if (!std::equal(handle->processed_tokens.begin(), handle->processed_tokens.end(), prompt.tokens.begin())) { + return false; + } + if (prompt.has_images()) { + return image_context_prefix_matches(handle->processed_images, prompt.images); + } + return !prompt.has_images(); +} + +PreparedPrompt prepare_prompt( + CactusModelHandle* handle, + const char* messages_json, + const char* options_json, + const char* tools_json, + bool apply_tool_constraints, + bool add_generation_prompt, + const uint8_t* pcm_buffer = nullptr, + size_t pcm_buffer_size = 0 +) { + if (!handle || !handle->model) { + throw std::runtime_error("Invalid model handle"); + } + + PreparedPrompt prompt; + prompt.options = parse_inference_options_json(options_json ? options_json : ""); + prompt.messages = parse_messages_json(messages_json, prompt.image_paths, &prompt.audio_paths); + if (prompt.messages.empty()) { + throw std::runtime_error("No messages provided"); + } + + inject_rag_context(handle, prompt.messages); + + if (tools_json && std::strlen(tools_json) > 0) { + prompt.tools = parse_tools_json(tools_json); + } + + if (prompt.options.tool_rag_top_k > 0 && prompt.tools.size() > prompt.options.tool_rag_top_k) { + std::string query = extract_last_user_query(prompt.messages); + if (!query.empty()) { + prompt.tools = select_relevant_tools(handle, query, prompt.tools, prompt.options.tool_rag_top_k); + } + } + + if (prompt.model_type == Config::ModelType::NEEDLE && !prompt.tools.empty()) { + prompt.options.force_tools = true; + } + + if (apply_tool_constraints) { + setup_tool_constraints(handle, prompt.tools, prompt.options.force_tools, prompt.options.temperature); + } + + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) { + throw std::runtime_error("Tokenizer unavailable"); + } + + prompt.model_type = handle->model->get_config().model_type; + + if (prompt.model_type == Config::ModelType::GEMMA4) { + std::vector audio_samples; + if (pcm_buffer != nullptr && pcm_buffer_size > 1) { + auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); + audio_samples = resample_to_16k_fp32(waveform_fp32, 16000); + } else if (!prompt.audio_paths.empty()) { + for (auto it = prompt.messages.rbegin(); it != prompt.messages.rend(); ++it) { + if (!it->audio.empty()) { + const std::string& audio_path = it->audio.back(); + AudioFP32 wav = load_wav(audio_path); + audio_samples = resample_to_16k_fp32(wav.samples, wav.sample_rate); + break; + } + } + } + if (!audio_samples.empty()) { + auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, handle->model->get_config()); + prompt.audio_features = std::move(audio_prep.features); + prompt.audio_num_frames = audio_prep.num_frames; + for (auto it = prompt.messages.rbegin(); it != prompt.messages.rend(); ++it) { + if (it->role == "user") { + it->audio_soft_token_count = audio_prep.num_soft_tokens; + break; + } + } + } + } + + std::string formatted_tools; + if (Config::is_gemma_family(prompt.model_type)) { + formatted_tools = gemma::format_tools(prompt.tools, prompt.model_type == Config::ModelType::GEMMA4); + } else if (prompt.model_type == Config::ModelType::NEEDLE) { + formatted_tools = serialize_needle_tools_json(prompt.tools, tools_json); + } else if (prompt.model_type == Config::ModelType::QWEN || prompt.model_type == Config::ModelType::QWEN3P5) { + formatted_tools = serialize_tools_for_template(prompt.tools); + } else { + formatted_tools = serialize_tools_json(prompt.tools); + } + + if (prompt.model_type == Config::ModelType::NEEDLE) { + std::string query_text = cactus::engine::format_needle_query_text(prompt.messages); + prompt.tokens = tokenizer->encode(query_text); + auto suffix_tokens = encode_needle_tools_suffix(tokenizer, formatted_tools); + prompt.tokens.insert(prompt.tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); + } else { + std::string full_prompt = tokenizer->format_chat_prompt( + prompt.messages, + add_generation_prompt, + formatted_tools, + prompt.options.enable_thinking_if_supported + ); + if (full_prompt.find("ERROR:") == 0) { + throw std::runtime_error(full_prompt.substr(6)); + } + prompt.tokens = tokenizer->encode(full_prompt); + } + prompt.context_token_count = prompt.tokens.size(); + prompt.images = images_from_message(prompt.messages); + return prompt; +} + +PrefillResult do_prefill( + CactusModelHandle* handle, + const PreparedPrompt& prompt, + const std::vector& target_tokens +) { + PrefillResult result = {}; + bool has_images = prompt.has_images(); + + result.was_prefix = prompt_context_matches(handle, prompt); + result.was_exact_match = result.was_prefix && + target_tokens.size() == handle->processed_tokens.size(); + + if (result.was_exact_match) { + return result; + } + + std::vector tokens_to_process; + if (!result.was_prefix) { + reset_cache(handle); + tokens_to_process = target_tokens; + } else { + tokens_to_process.assign( + target_tokens.begin() + handle->processed_tokens.size(), + target_tokens.end() + ); + } + + if (tokens_to_process.size() > 1) { + std::vector prefill_tokens(tokens_to_process.begin(), tokens_to_process.end() - 1); + result.prefilled_count = prefill_tokens.size(); + if (has_images) { + std::vector delta_image_paths; + if (result.was_prefix) { + size_t cached_image_count = 0; + for (const auto& msg_imgs : handle->processed_images) { + cached_image_count += msg_imgs.size(); + } + delta_image_paths.assign( + prompt.image_paths.begin() + cached_image_count, + prompt.image_paths.end() + ); + } else { + delta_image_paths = prompt.image_paths; + } + handle->model->prefill_with_images(prefill_tokens, delta_image_paths); + } else { + handle->model->prefill(prefill_tokens, handle->model->get_prefill_chunk_size()); + } + result.remaining_tokens = {tokens_to_process.back()}; + } else { + result.remaining_tokens = tokens_to_process; + } + + return result; +} + +uint32_t decode( + std::unique_ptr& model, + const std::vector& tokens, + const InferenceOptions& options, + float* out_entropy +) { + return model->decode(tokens, options.temperature, options.top_p, options.top_k, + "", out_entropy, options.min_p, options.repetition_penalty); +} + +uint32_t generate_first_token( + CactusModelHandle* handle, + const PrefillResult& prefill_result, + const PreparedPrompt& prompt, + float* first_token_entropy +) { + if (prefill_result.was_exact_match || prefill_result.remaining_tokens.empty()) { + if (handle->processed_tokens.empty()) { + throw std::runtime_error("Cannot generate from empty prompt"); + } + return decode(handle->model, {handle->processed_tokens.back()}, prompt.options, first_token_entropy); + } + return decode(handle->model, prefill_result.remaining_tokens, prompt.options, first_token_entropy); +} + +std::string construct_prefill_response_json( + bool success, + const std::string* error, + size_t prefill_tokens, + double prefill_tps, + double total_time_ms +) { + std::ostringstream json; + json << "{"; + json << "\"success\":" << (success ? "true" : "false") << ","; + if (error) { + json << "\"error\":\"" << escape_json_string(*error) << "\","; + } else { + json << "\"error\":null,"; + } + json << "\"prefill_tokens\":" << prefill_tokens << ","; + json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); + json << "}"; + return json.str(); +} + +} // anonymous namespace + +extern "C" { + +int cactus_complete( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const char* tools_json, + cactus_token_callback callback, + void* user_data, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +) { + if (!model) { + std::string error_msg = last_error_message.empty() ? + "Model not initialized. Check model path and files." : last_error_message; + CACTUS_LOG_ERROR("complete", error_msg); + handle_error_response(error_msg, response_buffer, buffer_size); + return -1; + } + + if (!messages_json || !response_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR("complete", "Invalid parameters: messages_json, response_buffer, or buffer_size"); + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + + try { + auto start_time = std::chrono::high_resolution_clock::now(); + + auto* handle = static_cast(model); + handle->should_stop = false; + auto* tokenizer = handle->model->get_tokenizer(); + auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, true, true, pcm_buffer, pcm_buffer_size); + + CACTUS_LOG_DEBUG("complete", "Prompt tokens: " << prompt.tokens.size() + << ", max_tokens: " << prompt.options.max_tokens); + + bool has_images = prompt.has_images(); + bool has_audio = prompt.has_audio(); + + auto stop_token_sequences = build_stop_sequences(tokenizer, prompt.options.stop_sequences, prompt.model_type, !prompt.tools.empty()); + + std::vector generated_tokens; + double time_to_first_token = 0.0; + float first_token_entropy = 0.0f; + uint32_t next_token; + size_t prompt_tokens; + + if (has_audio) { + prompt_tokens = prompt.tokens.size(); + next_token = handle->model->decode_with_audio( + prompt.tokens, prompt.audio_features, + prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, + "", &first_token_entropy, + prompt.options.min_p, prompt.options.repetition_penalty); + } else { + auto prefill_result = do_prefill(handle, prompt, prompt.tokens); + prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); + next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); + } + + handle->processed_tokens = prompt.tokens; + handle->processed_images = prompt.images; + + auto token_end = std::chrono::high_resolution_clock::now(); + time_to_first_token = std::chrono::duration_cast(token_end - start_time).count() / 1000.0; + + float confidence = 1.0f - first_token_entropy; + bool cloud_used = false; + std::string cloud_error; + std::future cloud_future; + bool cloud_future_started = false; + const bool cloud_eligible = prompt.options.auto_handoff && (!has_images || prompt.options.handoff_with_images); + + auto maybe_start_cloud_handoff = [&](const std::string& local_output_hint, + const std::vector& local_calls_hint) { + if (!cloud_eligible || cloud_future_started) { + return; + } + CloudCompletionRequest request; + request.messages = prompt.messages; + request.tools = prompt.tools; + request.local_output = local_output_hint; + request.local_function_calls = local_calls_hint; + request.has_images = has_images; + request.cloud_key = resolve_cloud_api_key(nullptr); + + cloud_future_started = true; + cloud_future = std::async(std::launch::async, [request, &prompt]() { + return cloud_complete_request(request, static_cast(prompt.options.cloud_timeout_ms)); + }); + }; + + if (confidence < prompt.options.confidence_threshold) { + maybe_start_cloud_handoff("", {}); + } + + generated_tokens.push_back(next_token); + handle->processed_tokens.push_back(next_token); + + if (prompt.options.force_tools && !prompt.tools.empty()) { + handle->model->update_tool_constraints(next_token); + } + + EntropyState entropy; + entropy.add(first_token_entropy); + + if (!matches_stop_sequence(generated_tokens, stop_token_sequences)) { + if (callback) { + std::string new_text = tokenizer->decode({next_token}); + callback(new_text.c_str(), next_token, user_data); + } + + for (size_t i = 1; i < prompt.options.max_tokens; i++) { + if (handle->should_stop) break; + + float token_entropy = 0.0f; + if (has_audio) { + next_token = handle->model->decode_with_audio( + handle->processed_tokens, prompt.audio_features, + prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, + "", &token_entropy, + prompt.options.min_p, prompt.options.repetition_penalty); + } else { + next_token = decode(handle->model, {next_token}, prompt.options, &token_entropy); + } + handle->processed_tokens.push_back(next_token); + generated_tokens.push_back(next_token); + + entropy.add(token_entropy); + + if (entropy.rolling_confidence() < prompt.options.confidence_threshold) { + entropy.spike_handoff = true; + maybe_start_cloud_handoff("", {}); + } + + if (prompt.options.force_tools && !prompt.tools.empty()) { + handle->model->update_tool_constraints(next_token); + } + + if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { + trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); + break; + } + + if (callback) { + std::string new_text = tokenizer->decode({next_token}); + callback(new_text.c_str(), next_token, user_data); + } + } + } else { + trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); + } + + confidence = entropy.mean_confidence(); + + if (prompt.options.force_tools && !prompt.tools.empty()) { + handle->model->clear_tool_constraints(); + } + + if (prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported && !generated_tokens.empty()) { + strip_thinking_from_cache(handle, generated_tokens, prompt.tokens.size()); + } + + if (prompt.model_type == Config::ModelType::GEMMA4) { + handle->model->compact_kv_cache(); + } + + auto end_time = std::chrono::high_resolution_clock::now(); + double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + + size_t completion_tokens = generated_tokens.size(); + double decode_time_ms = total_time_ms - time_to_first_token; + double prefill_tps = time_to_first_token > 0 ? (prompt_tokens * 1000.0) / time_to_first_token : 0.0; + double decode_tps = (completion_tokens > 1 && decode_time_ms > 0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; + + std::string response_text = tokenizer->decode(generated_tokens); + + std::string regular_response; + std::vector function_calls; + parse_function_calls_from_response(response_text, regular_response, function_calls); + + std::string thinking_text; + if (prompt.model_type == Config::ModelType::GEMMA4 || prompt.options.enable_thinking_if_supported) { + std::string stripped_content; + strip_thinking_block(regular_response, thinking_text, stripped_content); + regular_response = stripped_content; + if (!prompt.options.enable_thinking_if_supported) { + thinking_text.clear(); + } + } + + if (confidence < prompt.options.confidence_threshold) { + maybe_start_cloud_handoff(regular_response, function_calls); + } + + std::string local_completion = regular_response; + if (local_completion.empty() && function_calls.empty()) { + local_completion = response_text; + } + std::string primary_response = local_completion; + std::vector primary_function_calls = function_calls; + + if (cloud_future_started) { + auto status = cloud_future.wait_for(std::chrono::milliseconds(prompt.options.cloud_timeout_ms)); + if (status == std::future_status::ready) { + CloudCompletionResult cloud_result = cloud_future.get(); + if (cloud_result.ok && (!cloud_result.response.empty() || !cloud_result.function_calls.empty())) { + cloud_used = true; + if (!cloud_result.response.empty()) { + primary_response = cloud_result.response; + } + if (!cloud_result.function_calls.empty()) { + primary_function_calls = cloud_result.function_calls; + } + } else { + cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; + CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed, falling back to local output: " << cloud_error); + } + } else { + cloud_error = "timeout"; + CACTUS_LOG_WARN("cloud_handoff", "Cloud completion timed out, falling back to local output: " << cloud_error); + } + } + + const bool handoff_succeeded = cloud_used; + std::string result = construct_response_json(primary_response, primary_function_calls, time_to_first_token, + total_time_ms, prefill_tps, decode_tps, prompt_tokens, + completion_tokens, confidence, handoff_succeeded, + thinking_text); + + if (result.length() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + + std::strcpy(response_buffer, result.c_str()); + + std::string function_calls_json = serialize_function_calls(primary_function_calls); + cactus::telemetry::CompletionMetrics metrics{}; + metrics.success = true; + metrics.cloud_handoff = handoff_succeeded; + metrics.ttft_ms = time_to_first_token; + metrics.prefill_tps = prefill_tps; + metrics.decode_tps = decode_tps; + metrics.response_time_ms = total_time_ms; + metrics.confidence = confidence; + metrics.ram_usage_mb = get_ram_usage_mb(); + metrics.prefill_tokens = prompt_tokens; + metrics.decode_tokens = completion_tokens; + metrics.error_message = nullptr; + metrics.function_calls_json = nullptr; + cactus::telemetry::recordCompletion(handle->model_name.c_str(), metrics); + + return static_cast(result.length()); + + } catch (const std::exception& e) { + CACTUS_LOG_ERROR("complete", "Exception: " << e.what()); + handle_error_response(e.what(), response_buffer, buffer_size); + + cactus::telemetry::CompletionMetrics metrics{}; + metrics.success = false; + metrics.cloud_handoff = false; + metrics.ttft_ms = 0.0; + metrics.prefill_tps = 0.0; + metrics.decode_tps = 0.0; + metrics.response_time_ms = 0.0; + metrics.confidence = 0.0; + metrics.ram_usage_mb = get_ram_usage_mb(); + metrics.prefill_tokens = 0; + metrics.decode_tokens = 0; + metrics.error_message = e.what(); + metrics.function_calls_json = nullptr; + auto* h = static_cast(model); + cactus::telemetry::recordCompletion(h ? h->model_name.c_str() : "unknown", metrics); + + return -1; + } catch (...) { + CACTUS_LOG_ERROR("complete", "Unknown exception during completion"); + handle_error_response("Unknown error during completion", response_buffer, buffer_size); + + cactus::telemetry::CompletionMetrics metrics{}; + metrics.success = false; + metrics.cloud_handoff = false; + metrics.ttft_ms = 0.0; + metrics.prefill_tps = 0.0; + metrics.decode_tps = 0.0; + metrics.response_time_ms = 0.0; + metrics.confidence = 0.0; + metrics.ram_usage_mb = get_ram_usage_mb(); + metrics.prefill_tokens = 0; + metrics.decode_tokens = 0; + metrics.error_message = "Unknown error during completion"; + metrics.function_calls_json = nullptr; + auto* h = static_cast(model); + cactus::telemetry::recordCompletion(h ? h->model_name.c_str() : "unknown", metrics); + + return -1; + } +} + +int cactus_prefill( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const char* tools_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +) { + if (!model) { + std::string error_msg = last_error_message.empty() + ? "Model not initialized. Check model path and files." + : last_error_message; + if (response_buffer && buffer_size > 0) { + std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); + if (result.size() < buffer_size) { + std::strcpy(response_buffer, result.c_str()); + } + } + return -1; + } + + if (!messages_json || !response_buffer || buffer_size == 0) { + std::string error_msg = "Invalid parameters"; + if (response_buffer && buffer_size > 0) { + std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); + if (result.size() < buffer_size) { + std::strcpy(response_buffer, result.c_str()); + } + } + return -1; + } + + try { + auto start_time = std::chrono::high_resolution_clock::now(); + + auto* handle = static_cast(model); + auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, false, false, pcm_buffer, pcm_buffer_size); + + std::vector context_tokens(prompt.tokens.begin(), prompt.tokens.begin() + prompt.context_token_count); + auto prefill_result = do_prefill(handle, prompt, context_tokens); + + if (!prefill_result.was_exact_match) { + handle->processed_tokens = context_tokens; + if (!handle->processed_tokens.empty()) { + handle->processed_tokens.pop_back(); + } + } + handle->processed_images = prompt.images; + + auto end_time = std::chrono::high_resolution_clock::now(); + double elapsed_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + double prefill_tps = (prefill_result.prefilled_count > 0 && elapsed_ms > 0.0) + ? (static_cast(prefill_result.prefilled_count) * 1000.0) / elapsed_ms + : 0.0; + + std::string result = construct_prefill_response_json(true, nullptr, prefill_result.prefilled_count, prefill_tps, elapsed_ms); + if (result.size() >= buffer_size) { + std::string error_msg = "Response buffer too small"; + std::string error_json = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); + if (error_json.size() < buffer_size) { + std::strcpy(response_buffer, error_json.c_str()); + } + return -1; + } + + std::strcpy(response_buffer, result.c_str()); + return static_cast(result.size()); + } catch (const std::exception& e) { + std::string error_msg = e.what(); + std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); + if (result.size() < buffer_size) { + std::strcpy(response_buffer, result.c_str()); + } + return -1; + } catch (...) { + std::string error_msg = "Unknown error during prefill"; + std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); + if (result.size() < buffer_size) { + std::strcpy(response_buffer, result.c_str()); + } + return -1; + } +} + +int cactus_tokenize( + cactus_model_t model, + const char* text, + uint32_t* token_buffer, + size_t token_buffer_len, + size_t* out_token_len +) { + if (!model || !text || !out_token_len) return -1; + + try { + auto* handle = static_cast(model); + auto* tokenizer = handle->model->get_tokenizer(); + + std::vector toks = tokenizer->encode(std::string(text)); + *out_token_len = toks.size(); + + if (!token_buffer || token_buffer_len == 0) return 0; + if (token_buffer_len < toks.size()) return -2; + + std::memcpy(token_buffer, toks.data(), toks.size() * sizeof(uint32_t)); + return 0; + } catch (...) { + return -1; + } +} + +int cactus_decode_tokens( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + float temperature, + float top_p, + size_t top_k, + uint32_t* out_token_id +) { + if (!model || !tokens || token_len == 0 || !out_token_id) { + last_error_message = "Invalid parameters"; + return -1; + } + + try { + auto* handle = static_cast(model); + std::vector vec(tokens, tokens + token_len); + handle->model->reset_cache(); + *out_token_id = handle->model->decode(vec, temperature, top_p, top_k); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } catch (...) { + last_error_message = "Unknown error during raw token decode"; + return -1; + } +} + +int cactus_score_window( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + size_t start, + size_t end, + size_t context, + char* response_buffer, + size_t buffer_size +) { + if (!model || !tokens || token_len == 0 || !response_buffer || buffer_size == 0) { + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + + try { + auto* handle = static_cast(model); + + std::vector vec(tokens, tokens + token_len); + + size_t scored = 0; + double logprob = handle->model->score_tokens_window_logprob(vec, start, end, context, &scored); + + std::ostringstream oss; + oss << "{" + << "\"success\":true," + << "\"logprob\":" << std::setprecision(10) << logprob << "," + << "\"tokens\":" << scored + << "}"; + + std::string result = oss.str(); + if (result.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + + std::strcpy(response_buffer, result.c_str()); + return (int)result.size(); + + } catch (const std::exception& e) { + handle_error_response(e.what(), response_buffer, buffer_size); + return -1; + } +} + +} diff --git a/cactus-transpile/ffi/cactus_diarize.cpp b/cactus-transpile/ffi/cactus_diarize.cpp new file mode 100644 index 000000000..358c7aa37 --- /dev/null +++ b/cactus-transpile/ffi/cactus_diarize.cpp @@ -0,0 +1,200 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include "../models/model.h" +#include "../../libs/audio/wav.h" +#include +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; + +static constexpr int DIARIZE_MAX_SPEAKERS = 3; + +struct DiarizeOptions { + size_t step_samples = 16000; + float threshold = -1.0f; + int num_speakers = -1; + int min_speakers = -1; + int max_speakers = -1; + bool raw_powerset = false; +}; + +static DiarizeOptions parse_diarize_options(const std::string& json) { + DiarizeOptions opts; + if (json.empty()) return opts; + + size_t pos; + + pos = json.find("\"step_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + int v = std::stoi(json.substr(pos)); + if (v > 0) opts.step_samples = static_cast(v) * 16; + } + + pos = json.find("\"threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + opts.threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"num_speakers\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + opts.num_speakers = std::stoi(json.substr(pos)); + } + + pos = json.find("\"min_speakers\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + opts.min_speakers = std::stoi(json.substr(pos)); + } + + pos = json.find("\"max_speakers\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + opts.max_speakers = std::stoi(json.substr(pos)); + } + + pos = json.find("\"raw_powerset\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + opts.raw_powerset = (json.substr(pos, 4) == "true"); + } + + return opts; +} + +static void apply_speaker_filter(std::vector& scores, size_t total_frames, const DiarizeOptions& opts) { + int keep = DIARIZE_MAX_SPEAKERS; + if (opts.num_speakers > 0) keep = std::min(opts.num_speakers, DIARIZE_MAX_SPEAKERS); + if (opts.max_speakers > 0) keep = std::min(keep, opts.max_speakers); + if (opts.min_speakers > 0) keep = std::max(keep, std::min(opts.min_speakers, DIARIZE_MAX_SPEAKERS)); + if (keep >= DIARIZE_MAX_SPEAKERS) return; + + float activity[DIARIZE_MAX_SPEAKERS] = {}; + for (size_t f = 0; f < total_frames; ++f) + for (int s = 0; s < DIARIZE_MAX_SPEAKERS; ++s) + activity[s] += scores[f * DIARIZE_MAX_SPEAKERS + s]; + + int order[DIARIZE_MAX_SPEAKERS] = {0, 1, 2}; + std::sort(order, order + DIARIZE_MAX_SPEAKERS, [&](int a, int b) { return activity[a] > activity[b]; }); + + bool active[DIARIZE_MAX_SPEAKERS] = {}; + for (int i = 0; i < keep; ++i) active[order[i]] = true; + + for (size_t f = 0; f < total_frames; ++f) + for (int s = 0; s < DIARIZE_MAX_SPEAKERS; ++s) + if (!active[s]) scores[f * DIARIZE_MAX_SPEAKERS + s] = 0.0f; +} + +static void apply_threshold(std::vector& scores, float threshold) { + for (float& v : scores) + v = (v >= threshold) ? v : 0.0f; +} + +extern "C" { + +int cactus_diarize( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +) { + if (validate_audio_params("diarize", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) + return -1; + + try { + auto start_time = std::chrono::high_resolution_clock::now(); + auto* handle = static_cast(model); + auto* pyannote = dynamic_cast(handle->model.get()); + if (!pyannote) { + CACTUS_LOG_ERROR("diarize", "Model is not a PyAnnote diarization model"); + handle_error_response("Model is not a PyAnnote diarization model", response_buffer, buffer_size); + return -1; + } + + std::vector audio; + if (audio_file_path == nullptr) { + audio = pcm_to_float(pcm_buffer, pcm_buffer_size); + } else { + AudioFP32 wav_audio = load_wav(audio_file_path); + audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); + } + + if (audio.empty()) { + last_error_message = "Failed to load audio or audio is empty"; + CACTUS_LOG_ERROR("diarize", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } + + const DiarizeOptions opts = parse_diarize_options(options_json ? options_json : ""); + auto scores = pyannote->diarize(audio.data(), audio.size(), opts.step_samples, opts.raw_powerset); + audio = {}; + + const int num_channels = opts.raw_powerset ? 7 : DIARIZE_MAX_SPEAKERS; + const size_t total_frames = scores.size() / num_channels; + + if (!opts.raw_powerset) { + if (opts.num_speakers > 0 || opts.min_speakers > 0 || opts.max_speakers > 0) + apply_speaker_filter(scores, total_frames, opts); + + if (opts.threshold >= 0.0f) + apply_threshold(scores, opts.threshold); + } + + auto end_time = std::chrono::high_resolution_clock::now(); + double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"num_speakers\":" << num_channels << ","; + json << "\"scores\":["; + json << std::fixed << std::setprecision(6); + for (size_t i = 0; i < scores.size(); ++i) { + if (i > 0) json << ","; + json << scores[i]; + } + json << "],"; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); + json << "}"; + + std::string response = json.str(); + if (response.length() >= buffer_size) { + last_error_message = "Response buffer too small"; + CACTUS_LOG_ERROR("diarize", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } + + std::strcpy(response_buffer, response.c_str()); + return static_cast(response.length()); + + } catch (const std::exception& e) { + last_error_message = "Exception during diarization: " + std::string(e.what()); + CACTUS_LOG_ERROR("diarize", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } catch (...) { + last_error_message = "Unknown exception during diarization"; + CACTUS_LOG_ERROR("diarize", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } +} + +} diff --git a/cactus-transpile/ffi/cactus_embed.cpp b/cactus-transpile/ffi/cactus_embed.cpp new file mode 100644 index 000000000..ba6da2420 --- /dev/null +++ b/cactus-transpile/ffi/cactus_embed.cpp @@ -0,0 +1,273 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include "../../libs/audio/wav.h" +#include +#include +#include +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; +using cactus::audio::WHISPER_SAMPLE_RATE; +using cactus::audio::apply_preemphasis; +using cactus::audio::get_parakeet_spectrogram_config; +using cactus::audio::get_whisper_spectrogram_config; +using cactus::audio::init_whisper_mel_filters; +using cactus::audio::normalize_parakeet_log_mel; +using cactus::audio::normalize_whisper_mel; +using cactus::audio::preprocess_audio_for_gemma4; +using cactus::audio::trim_mel_frames; + +static std::vector compute_mel_from_wav(const std::string& wav_path, bool is_parakeet, size_t mel_bins) { + AudioFP32 audio = load_wav(wav_path); + std::vector waveform_16k = resample_to_16k_fp32(audio.samples, audio.sample_rate); + + auto cfg = is_parakeet ? get_parakeet_spectrogram_config() : get_whisper_spectrogram_config(); + const size_t num_mel_filters = std::max(1, mel_bins); + + AudioProcessor ap; + if (is_parakeet) { + ap.init_mel_filters(cfg.n_fft / 2 + 1, num_mel_filters, 0.0f, 8000.0f, WHISPER_SAMPLE_RATE); + } else { + init_whisper_mel_filters(ap, cfg, num_mel_filters); + } + const size_t waveform_samples = waveform_16k.size(); + if (is_parakeet) { + apply_preemphasis(waveform_16k, 0.97f); + } + std::vector mel = ap.compute_spectrogram(waveform_16k, cfg); + + if (mel.empty()) return mel; + if (is_parakeet) { + normalize_parakeet_log_mel(mel, num_mel_filters); + size_t valid_frames = waveform_samples / cfg.hop_length; + if (valid_frames == 0) valid_frames = 1; + trim_mel_frames(mel, num_mel_filters, valid_frames); + return mel; + } + + const bool is_v3 = num_mel_filters > 80; + return normalize_whisper_mel(mel, num_mel_filters, is_v3); +} + +extern "C" { + +int cactus_embed( + cactus_model_t model, + const char* text, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim, + bool normalize +) { + if (!model || !text || !embeddings_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR("embed", "Invalid parameters for text embedding"); + return -1; + } + + try { + auto* handle = static_cast(model); + auto* tokenizer = handle->model->get_tokenizer(); + + std::vector tokens = tokenizer->encode(text); + if (tokens.empty()) { + CACTUS_LOG_ERROR("embed", "Tokenization produced empty result"); + return -1; + } + + std::vector embeddings = handle->model->get_embeddings(tokens, true, normalize); + if (embeddings.size() * sizeof(float) > buffer_size) { + CACTUS_LOG_ERROR("embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes, got " << buffer_size); + return -2; + } + + std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); + if (embedding_dim) *embedding_dim = embeddings.size(); + + return static_cast(embeddings.size()); + + } catch (const std::exception& e) { + last_error_message = e.what(); + CACTUS_LOG_ERROR("embed", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during embedding"; + CACTUS_LOG_ERROR("embed", last_error_message); + return -1; + } +} + +int cactus_image_embed( + cactus_model_t model, + const char* image_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +) { + if (!model || !image_path || !embeddings_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR("image_embed", "Invalid parameters for image embedding"); + return -1; + } + + try { + auto* handle = static_cast(model); + + CACTUS_LOG_DEBUG("image_embed", "Processing image: " << image_path); + std::vector embeddings = handle->model->get_image_embeddings(image_path); + if (embeddings.empty()) { + CACTUS_LOG_ERROR("image_embed", "Image embedding returned empty result"); + return -1; + } + if (embeddings.size() * sizeof(float) > buffer_size) { + CACTUS_LOG_ERROR("image_embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes"); + return -2; + } + + std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); + if (embedding_dim) *embedding_dim = embeddings.size(); + + return static_cast(embeddings.size()); + + } catch (const std::exception& e) { + last_error_message = e.what(); + CACTUS_LOG_ERROR("image_embed", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during image embedding"; + CACTUS_LOG_ERROR("image_embed", last_error_message); + return -1; + } +} + +int cactus_audio_embed( + cactus_model_t model, + const char* audio_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +) { + if (!model || !audio_path || !embeddings_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR("audio_embed", "Invalid parameters for audio embedding"); + return -1; + } + + try { + auto* handle = static_cast(model); + + CACTUS_LOG_DEBUG("audio_embed", "Processing audio: " << audio_path); + const bool is_parakeet = + handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET || + handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; + const bool is_gemma4 = + handle->model->get_config().model_type == cactus::engine::Config::ModelType::GEMMA4; + std::vector mel_bins; + if (is_gemma4) { + AudioFP32 audio = load_wav(audio_path); + std::vector waveform_16k = resample_to_16k_fp32(audio.samples, audio.sample_rate); + auto prepared = preprocess_audio_for_gemma4(std::move(waveform_16k), handle->model->get_config()); + mel_bins = std::move(prepared.features); + } else { + mel_bins = compute_mel_from_wav(audio_path, is_parakeet, handle->model->get_config().num_mel_bins); + } + if (mel_bins.empty()) { + last_error_message = "Failed to compute mel spectrogram"; + CACTUS_LOG_ERROR("audio_embed", last_error_message << " for: " << audio_path); + return -1; + } + + std::vector embeddings = handle->model->get_audio_embeddings(mel_bins); + if (embeddings.empty()) { + CACTUS_LOG_ERROR("audio_embed", "Audio embedding returned empty result"); + return -1; + } + if (embeddings.size() * sizeof(float) > buffer_size) { + CACTUS_LOG_ERROR("audio_embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes"); + return -2; + } + + std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); + if (embedding_dim) *embedding_dim = embeddings.size(); + + return static_cast(embeddings.size()); + + } catch (const std::exception& e) { + last_error_message = e.what(); + CACTUS_LOG_ERROR("audio_embed", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during audio embedding"; + CACTUS_LOG_ERROR("audio_embed", last_error_message); + return -1; + } +} + +int cactus_preprocess_audio_features( + const char* audio_path, + const char* model_type, + size_t mel_bins, + float* features_buffer, + size_t buffer_size, + size_t* feature_count, + size_t* out_mel_bins, + size_t* out_frames +) { + if (!audio_path || !model_type || !features_buffer || buffer_size == 0 || mel_bins == 0) { + CACTUS_LOG_ERROR("audio_features", "Invalid parameters for audio feature preprocessing"); + return -1; + } + + try { + std::string type(model_type); + std::transform(type.begin(), type.end(), type.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + const bool is_parakeet = type.find("parakeet") != std::string::npos; + const bool is_gemma4 = type.find("gemma4") != std::string::npos; + + std::vector features; + if (is_gemma4) { + AudioFP32 audio = load_wav(audio_path); + std::vector waveform_16k = resample_to_16k_fp32(audio.samples, audio.sample_rate); + cactus::engine::Config config; + config.audio_input_feat_size = static_cast(mel_bins); + config.audio_fft_length = 512; + auto prepared = preprocess_audio_for_gemma4(std::move(waveform_16k), config); + features = std::move(prepared.features); + } else { + features = compute_mel_from_wav(audio_path, is_parakeet, mel_bins); + } + if (features.empty()) { + last_error_message = "Failed to compute audio features"; + CACTUS_LOG_ERROR("audio_features", last_error_message << " for: " << audio_path); + return -1; + } + if ((features.size() % mel_bins) != 0) { + last_error_message = "Computed audio features have invalid mel/frame shape"; + CACTUS_LOG_ERROR("audio_features", last_error_message); + return -1; + } + + if (feature_count) *feature_count = features.size(); + if (out_mel_bins) *out_mel_bins = mel_bins; + if (out_frames) *out_frames = features.size() / mel_bins; + + const size_t bytes = features.size() * sizeof(float); + if (bytes > buffer_size) { + CACTUS_LOG_ERROR("audio_features", "Buffer too small: need " << bytes << " bytes"); + return -2; + } + + std::memcpy(features_buffer, features.data(), bytes); + return static_cast(features.size()); + } catch (const std::exception& e) { + last_error_message = e.what(); + CACTUS_LOG_ERROR("audio_features", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during audio feature preprocessing"; + CACTUS_LOG_ERROR("audio_features", last_error_message); + return -1; + } +} + +} diff --git a/cactus-transpile/ffi/cactus_embed_speaker.cpp b/cactus-transpile/ffi/cactus_embed_speaker.cpp new file mode 100644 index 000000000..ddb42707e --- /dev/null +++ b/cactus-transpile/ffi/cactus_embed_speaker.cpp @@ -0,0 +1,125 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include "../models/model.h" +#include "../../libs/audio/wav.h" +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; +using cactus::audio::get_wespeaker_spectrogram_config; + +extern "C" { + +int cactus_embed_speaker( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + const float* mask_weights, + size_t mask_num_frames +) { + (void)options_json; + if (validate_audio_params("embed_speaker", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) + return -1; + + try { + auto start_time = std::chrono::high_resolution_clock::now(); + auto* handle = static_cast(model); + auto* wespeaker = dynamic_cast(handle->model.get()); + if (!wespeaker) { + CACTUS_LOG_ERROR("embed_speaker", "Model is not a WeSpeaker embedding model"); + handle_error_response("Model is not a WeSpeaker embedding model", response_buffer, buffer_size); + return -1; + } + + std::vector audio; + if (audio_file_path == nullptr) { + audio = pcm_to_float(pcm_buffer, pcm_buffer_size); + } else { + AudioFP32 wav_audio = load_wav(audio_file_path); + audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); + } + + if (audio.empty()) { + last_error_message = "Failed to load audio or audio is empty"; + CACTUS_LOG_ERROR("embed_speaker", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } + + static const auto s_cfg = get_wespeaker_spectrogram_config(); + static AudioProcessor s_ap = []() { + AudioProcessor p; + p.init_mel_filters(get_wespeaker_spectrogram_config().n_fft / 2 + 1, + 80, 20.0f, 8000.0f, 16000, nullptr, "htk"); + return p; + }(); + + std::vector mel = s_ap.compute_spectrogram(audio, s_cfg); + audio = {}; + + static constexpr size_t NUM_MEL_BINS = 80; + const size_t num_frames = mel.size() / NUM_MEL_BINS; + + for (size_t m = 0; m < NUM_MEL_BINS; ++m) { + float* row = mel.data() + m * num_frames; + float sum = 0.0f; + for (size_t t = 0; t < num_frames; ++t) sum += row[t]; + const float mean = sum / static_cast(num_frames); + for (size_t t = 0; t < num_frames; ++t) row[t] -= mean; + } + + std::vector embedding; + if (mask_weights && mask_num_frames > 0) { + embedding = wespeaker->embed(mel.data(), mel.size(), mask_weights, mask_num_frames); + } else { + embedding = wespeaker->embed(mel.data(), mel.size()); + } + + auto end_time = std::chrono::high_resolution_clock::now(); + double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"embedding\":["; + json << std::fixed << std::setprecision(6); + for (size_t i = 0; i < embedding.size(); ++i) { + if (i > 0) json << ","; + json << embedding[i]; + } + json << "],"; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); + json << "}"; + + std::string response = json.str(); + if (response.length() >= buffer_size) { + last_error_message = "Response buffer too small"; + CACTUS_LOG_ERROR("embed_speaker", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } + + std::strcpy(response_buffer, response.c_str()); + return static_cast(response.length()); + + } catch (const std::exception& e) { + last_error_message = "Exception during speaker embedding: " + std::string(e.what()); + CACTUS_LOG_ERROR("embed_speaker", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } catch (...) { + last_error_message = "Unknown exception during speaker embedding"; + CACTUS_LOG_ERROR("embed_speaker", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } +} + +} diff --git a/cactus-transpile/ffi/cactus_ffi.h b/cactus-transpile/ffi/cactus_ffi.h new file mode 100644 index 000000000..9a12e3afa --- /dev/null +++ b/cactus-transpile/ffi/cactus_ffi.h @@ -0,0 +1,489 @@ +#ifndef CACTUS_FFI_H +#define CACTUS_FFI_H + +#include +#include +#include + +#if __GNUC__ >= 4 + #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) + #define CACTUS_FFI_LOCAL __attribute__((visibility("hidden"))) +#else + #define CACTUS_FFI_EXPORT + #define CACTUS_FFI_LOCAL +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* cactus_model_t; +typedef void* cactus_index_t; +typedef void* cactus_stream_transcribe_t; + +typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); + +CACTUS_FFI_EXPORT cactus_model_t cactus_init( + const char* model_path, + const char* corpus_dir, // optional: NULL if no RAG corpus + bool cache_index // false = always rebuild index, true = load cached if available +); + +CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); + +CACTUS_FFI_EXPORT int cactus_complete( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_prefill( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_tokenize( + cactus_model_t model, + const char* text, + uint32_t* token_buffer, + size_t token_buffer_len, + size_t* out_token_len +); + +CACTUS_FFI_EXPORT int cactus_decode_tokens( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + float temperature, + float top_p, + size_t top_k, + uint32_t* out_token_id +); + +CACTUS_FFI_EXPORT int cactus_score_window( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + size_t start, + size_t end, + size_t context, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_transcribe( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + const char* prompt, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_detect_language( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_start( + cactus_model_t model, + const char* options_json // optional +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_process( + cactus_stream_transcribe_t stream, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop( + cactus_stream_transcribe_t stream, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed( + cactus_model_t model, + const char* text, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim, + bool normalize +); + +CACTUS_FFI_EXPORT int cactus_image_embed( + cactus_model_t model, + const char* image_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_audio_embed( + cactus_model_t model, + const char* audio_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_preprocess_audio_features( + const char* audio_path, + const char* model_type, + size_t mel_bins, + float* features_buffer, + size_t buffer_size, + size_t* feature_count, + size_t* out_mel_bins, + size_t* out_frames +); + +CACTUS_FFI_EXPORT int cactus_vad( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_diarize( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed_speaker( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + const float* mask_weights, + size_t mask_num_frames +); + +CACTUS_FFI_EXPORT int cactus_rag_query( + cactus_model_t model, + const char* query, + char* response_buffer, + size_t buffer_size, + size_t top_k +); + +CACTUS_FFI_EXPORT cactus_index_t cactus_index_init( + const char* index_dir, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_add( + cactus_index_t index, + const int* ids, + const char** documents, + const char** metadatas, // optional: can be NULL + const float** embeddings, + size_t count, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_delete( + cactus_index_t index, + const int* ids, + size_t ids_count +); + +CACTUS_FFI_EXPORT int cactus_index_get( + cactus_index_t index, + const int* ids, + size_t ids_count, + char** document_buffers, + size_t* document_buffer_sizes, + char** metadata_buffers, + size_t* metadata_buffer_sizes, + float** embedding_buffers, + size_t* embedding_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_query( + cactus_index_t index, + const float** embeddings, + size_t embeddings_count, + size_t embedding_dim, + const char* options_json, // optional + int** id_buffers, + size_t* id_buffer_sizes, + float** score_buffers, + size_t* score_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_compact(cactus_index_t index); +CACTUS_FFI_EXPORT void cactus_index_destroy(cactus_index_t index); + +CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); + +// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE +CACTUS_FFI_EXPORT void cactus_log_set_level(int level); + +typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); +CACTUS_FFI_EXPORT void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); + +CACTUS_FFI_EXPORT void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); +CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); +CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); +CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); + +// cactus graph export +typedef void* cactus_graph_t; +typedef uint64_t cactus_node_t; + +typedef struct { + int32_t precision; + size_t rank; + size_t shape[8]; + size_t num_elements; + size_t byte_size; +} cactus_tensor_info_t; + +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); +CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); + +CACTUS_FFI_EXPORT int cactus_graph_input( + cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, +cactus_node_t* out_node); + +CACTUS_FFI_EXPORT int cactus_graph_set_input( + cactus_graph_t graph, cactus_node_t node, const void* data, int32_t +precision); +CACTUS_FFI_EXPORT int cactus_graph_set_external_input( + cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); + +CACTUS_FFI_EXPORT int cactus_graph_precision_cast( + cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_quantize_activations( + cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_not_equal(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_floor_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_not_equal(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_masked_select_prefix(cactus_graph_t graph, cactus_node_t input, cactus_node_t mask, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_masked_scatter(cactus_graph_t graph, cactus_node_t input, cactus_node_t mask, cactus_node_t source, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, +float exponent, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_view( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_flatten( + cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_reshape( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_expand( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose( + cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose_n( + cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_slice( + cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_index( + cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_concat( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, +cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_cat( + cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t +axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_matmul( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, int32_t output_precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gather( + cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( + cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( + cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( + cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_set_grouped_scales( + cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr); +CACTUS_FFI_EXPORT int cactus_graph_set_interleaved( + cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n); +CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_layernorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_groupnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_batchnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rms_norm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( + cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s1p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, + size_t stride, size_t padding, size_t dilation, size_t groups, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( + cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_stft( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_altup_predict( + cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_altup_correct( + cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( + cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_sample( + cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( + cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_persistent( + cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_is_populated( + cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); +CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( + cactus_graph_t graph, cactus_node_t persistent_node); + +CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, +cactus_node_t node, void** out_ptr); +CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, +cactus_node_t node, cactus_tensor_info_t* out_info); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_FFI_H diff --git a/cactus-transpile/ffi/cactus_graph.cpp b/cactus-transpile/ffi/cactus_graph.cpp new file mode 100644 index 000000000..5590e38aa --- /dev/null +++ b/cactus-transpile/ffi/cactus_graph.cpp @@ -0,0 +1,1432 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include "../graph/graph.h" +#include +#include + +using namespace cactus::ffi; + +struct GraphHandle { + CactusGraph graph; +}; + +static GraphHandle* as_graph(cactus_graph_t g) { + return reinterpret_cast(g); +} + +static int fail_invalid(const char* msg) { + last_error_message = msg; + return -1; +} + +#define GRAPH_WRAP_UNARY(fn_name, method_name) \ +int fn_name(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { \ + if (!graph || !out) return fail_invalid("Invalid args to " #fn_name); \ + try { \ + *out = static_cast(as_graph(graph)->graph.method_name(static_cast(x))); \ + return 0; \ + } catch (const std::exception& e) { \ + last_error_message = e.what(); \ + return -1; \ + } \ +} + +#define GRAPH_WRAP_BINARY(fn_name, method_name) \ +int fn_name(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { \ + if (!graph || !out) return fail_invalid("Invalid args to " #fn_name); \ + try { \ + *out = static_cast(as_graph(graph)->graph.method_name(static_cast(a), static_cast(b))); \ + return 0; \ + } catch (const std::exception& e) { \ + last_error_message = e.what(); \ + return -1; \ + } \ +} + +extern "C" { + +cactus_graph_t cactus_graph_create(void) { + try { + return reinterpret_cast(new GraphHandle()); + } catch (const std::exception& e) { + last_error_message = e.what(); + return nullptr; + } catch (...) { + last_error_message = "Unknown error creating graph"; + return nullptr; + } +} + +void cactus_graph_destroy(cactus_graph_t graph) { + delete as_graph(graph); +} + +int cactus_graph_hard_reset(cactus_graph_t graph) { + if (!graph) { + last_error_message = "Invalid args to cactus_graph_hard_reset"; + return -1; + } + try { + as_graph(graph)->graph.hard_reset(); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_save(cactus_graph_t graph, const char* filename) { + if (!graph || !filename) { + last_error_message = "Invalid args to cactus_graph_save"; + return -1; + } + try { + as_graph(graph)->graph.save(filename); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + + +cactus_graph_t cactus_graph_load(const char* filename) { + if (!filename) { + last_error_message = "Invalid args to cactus_graph_load"; + return nullptr; + } + try { + auto handle = std::make_unique(); + handle->graph = CactusGraph::load(std::string(filename)); + return reinterpret_cast(handle.release()); + } catch (const std::exception& e) { + last_error_message = e.what(); + return nullptr; + } catch (...) { + last_error_message = "Unknown error loading graph"; + return nullptr; + } +} + +int cactus_graph_input(cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, cactus_node_t* out_node) { + if (!graph || !shape || rank == 0 || !out_node) { + last_error_message = "Invalid args to cactus_graph_input"; + return -1; + } + try { + std::vector s(shape, shape + rank); + auto id = as_graph(graph)->graph.input(s, static_cast(precision)); + *out_node = static_cast(id); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_set_input(cactus_graph_t graph, cactus_node_t node, const void* data, int32_t precision) { + if (!graph || !data) { + last_error_message = "Invalid args to cactus_graph_set_input"; + return -1; + } + try { + as_graph(graph)->graph.set_input(static_cast(node), data, static_cast(precision)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_set_external_input(cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision) { + if (!graph || !data) { + return fail_invalid("Invalid args to cactus_graph_set_external_input"); + } + try { + as_graph(graph)->graph.set_external_input(static_cast(node), data, static_cast(precision)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_precision_cast(cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out) { + if (!graph || !out) { + return fail_invalid("Invalid args to cactus_graph_precision_cast"); + } + try { + *out = static_cast(as_graph(graph)->graph.precision_cast(static_cast(input), static_cast(target_precision))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_quantize_activations(cactus_graph_t graph, cactus_node_t input, cactus_node_t* out) { + if (!graph || !out) { + return fail_invalid("Invalid args to cactus_graph_quantize_activations"); + } + try { + *out = static_cast(as_graph(graph)->graph.quantize_activations(static_cast(input))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_add"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.add(static_cast(a), static_cast(b))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_add_clipped"); + try { + *out = static_cast(as_graph(graph)->graph.add_clipped(static_cast(a), static_cast(b))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_subtract"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.subtract(static_cast(a), static_cast(b))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_multiply"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.multiply(static_cast(a), static_cast(b))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_divide(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_divide"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.divide(static_cast(a), static_cast(b))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_not_equal(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_not_equal"); + try { + *out = static_cast(as_graph(graph)->graph.not_equal(static_cast(a), static_cast(b))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_scalar_add"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.scalar_add(static_cast(x), value)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_subtract"); + try { + *out = static_cast(as_graph(graph)->graph.scalar_subtract(static_cast(x), value)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_scalar_multiply"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.scalar_multiply(static_cast(x), value)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_scalar_divide"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.scalar_divide(static_cast(x), value)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_floor_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_scalar_floor_divide"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.scalar_floor_divide(static_cast(x), value)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_not_equal(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_not_equal"); + try { + *out = static_cast(as_graph(graph)->graph.scalar_not_equal(static_cast(x), value)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_masked_select_prefix(cactus_graph_t graph, cactus_node_t input, cactus_node_t mask, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_masked_select_prefix"); + try { + *out = static_cast( + as_graph(graph)->graph.masked_select_prefix(static_cast(input), static_cast(mask)) + ); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_masked_scatter( + cactus_graph_t graph, + cactus_node_t input, + cactus_node_t mask, + cactus_node_t source, + cactus_node_t* out +) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_masked_scatter"); + try { + *out = static_cast( + as_graph(graph)->graph.masked_scatter( + static_cast(input), + static_cast(mask), + static_cast(source) + ) + ); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t *out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_scalar_exp"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.scalar_exp(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t *out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_scalar_sqrt"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.scalar_sqrt(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_cos"); + try { + *out = static_cast(as_graph(graph)->graph.scalar_cos(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_sin"); + try { + *out = static_cast(as_graph(graph)->graph.scalar_sin(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_log"); + try { + *out = static_cast(as_graph(graph)->graph.scalar_log(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_abs"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.abs(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, float exponent, cactus_node_t* out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_pow"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.pow(static_cast(x), exponent)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_view(cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out) { + if (!graph || !shape || rank == 0 || !out) { + last_error_message = "Invalid args to cactus_graph_view"; + return -1; + } + try { + std::vector s(shape, shape + rank); + *out = static_cast(as_graph(graph)->graph.view(static_cast(x), s)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_flatten(cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, cactus_node_t* out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_flatten"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.flatten(static_cast(x), start_dim, end_dim)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_reshape(cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out) { + if (!graph || !shape || rank == 0 || !out) return fail_invalid("Invalid args to cactus_graph_reshape"); + try { + std::vector s(shape, shape + rank); + *out = static_cast(as_graph(graph)->graph.reshape(static_cast(x), s)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_expand(cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out) { + if (!graph || !shape || rank == 0 || !out) return fail_invalid("Invalid args to cactus_graph_expand"); + try { + std::vector s(shape, shape + rank); + *out = static_cast(as_graph(graph)->graph.expand(static_cast(x), s)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_transpose(cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_transpose"); + try { + *out = static_cast(as_graph(graph)->graph.transpose(static_cast(x), static_cast(backend))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_transpose_n(cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out) { + if (!graph || !permutation || rank == 0 || !out) return fail_invalid("Invalid args to cactus_graph_transpose_n"); + try { + std::vector p(permutation, permutation + rank); + *out = static_cast(as_graph(graph)->graph.transposeN(static_cast(x), p, static_cast(backend))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_slice(cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_slice"); + try { + *out = static_cast(as_graph(graph)->graph.slice(static_cast(x), axis, start, length)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_index(cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_index"); + try { + *out = static_cast(as_graph(graph)->graph.index(static_cast(x), index_value, dim)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sum"); + try { + *out = static_cast(as_graph(graph)->graph.sum(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_mean"); + try { + *out = static_cast(as_graph(graph)->graph.mean(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_variance"); + try { + *out = static_cast(as_graph(graph)->graph.variance(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_min"); + try { + *out = static_cast(as_graph(graph)->graph.min(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_max"); + try { + *out = static_cast(as_graph(graph)->graph.max(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_cumsum"); + try { + *out = static_cast(as_graph(graph)->graph.cumsum(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_concat(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, cactus_node_t* out) { + if (!graph || !out) { + last_error_message = "Invalid args to cactus_graph_concat"; + return -1; + } + try { + *out = static_cast(as_graph(graph)->graph.concat(static_cast(a), static_cast(b), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_cat(cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t axis, cactus_node_t* out) { + if (!graph || !nodes || !out || count == 0) { + last_error_message = "Invalid args to cactus_graph_cat"; + return -1; + } + try { + std::vector ns; + for (size_t i = 0; i < count; ++i) { + ns.push_back(static_cast(nodes[i])); + } + *out = static_cast(as_graph(graph)->graph.cat(ns, axis)); + return 0; + + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_matmul( + cactus_graph_t graph, + cactus_node_t a, + cactus_node_t b, + bool pretransposed_rhs, + int32_t backend, + int32_t output_precision, + cactus_node_t* out +) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_matmul"); + try { + *out = static_cast( + as_graph(graph)->graph.matmul( + static_cast(a), + static_cast(b), + pretransposed_rhs, + static_cast(backend), + static_cast(output_precision) + ) + ); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_gather(cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gather"); + try { + *out = static_cast(as_graph(graph)->graph.gather( + static_cast(tensor), + static_cast(indices), + axis + )); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_embedding_from_tensor(cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_embedding_from_tensor"); + try { + *out = static_cast(as_graph(graph)->graph.embedding(static_cast(embedding_tensor), static_cast(indices))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_embedding_from_file(cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out) { + if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_embedding_from_file"); + try { + *out = static_cast(as_graph(graph)->graph.embedding(std::string(filename), static_cast(indices))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_mmap_embeddings(cactus_graph_t graph, const char* filename, cactus_node_t* out) { + if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_mmap_embeddings"); + try { + *out = static_cast(as_graph(graph)->graph.mmap_embeddings(std::string(filename))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_mmap_weights(cactus_graph_t graph, const char* filename, cactus_node_t* out) { + if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_mmap_weights"); + try { + *out = static_cast(as_graph(graph)->graph.mmap_weights(std::string(filename))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_bilinear_interpolation(cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_bilinear_interpolation"); + try { + *out = static_cast(as_graph(graph)->graph.bilinear_interpolation(static_cast(pos_embeds), dst_height, dst_width)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_set_grouped_scales(cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr) { + if (!graph || !scales_ptr) return fail_invalid("Invalid args to cactus_graph_set_grouped_scales"); + try { + as_graph(graph)->graph.set_grouped_scales(static_cast(node), group_size, num_groups, scales_ptr); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_set_interleaved(cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n) { + if (!graph) return fail_invalid("Invalid args to cactus_graph_set_interleaved"); + try { + as_graph(graph)->graph.set_interleaved(static_cast(node), interleaved, original_n); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node) { + if (!graph) return fail_invalid("Invalid args to cactus_graph_release_weight_pages"); + try { + as_graph(graph)->graph.release_weight_pages(static_cast(node)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node) { + if (!graph) return fail_invalid("Invalid args to cactus_graph_prefetch_weight_pages"); + try { + as_graph(graph)->graph.prefetch_weight_pages(static_cast(node)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_release_all_weight_pages(cactus_graph_t graph) { + if (!graph) return fail_invalid("Invalid args to cactus_graph_release_all_weight_pages"); + try { + as_graph(graph)->graph.release_all_weight_pages(); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_relu"); + try { + *out = static_cast(as_graph(graph)->graph.relu(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_silu"); + try { + *out = static_cast(as_graph(graph)->graph.silu(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gelu"); + try { + *out = static_cast(as_graph(graph)->graph.gelu(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gelu_erf"); + try { + *out = static_cast(as_graph(graph)->graph.gelu_erf(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sigmoid"); + try { + *out = static_cast(as_graph(graph)->graph.sigmoid(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_tanh"); + try { + *out = static_cast(as_graph(graph)->graph.tanh(static_cast(x))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_glu"); + try { + *out = static_cast(as_graph(graph)->graph.glu(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_layernorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_layernorm"); + try { + if (has_bias) { + *out = static_cast(as_graph(graph)->graph.layernorm(static_cast(input), static_cast(weight), static_cast(bias), epsilon)); + } else { + *out = static_cast(as_graph(graph)->graph.layernorm(static_cast(input), static_cast(weight), epsilon)); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_groupnorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_groupnorm"); + try { + *out = static_cast(as_graph(graph)->graph.groupnorm(static_cast(input), static_cast(weight), static_cast(bias), num_groups, epsilon)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_batchnorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_batchnorm"); + try { + *out = static_cast(as_graph(graph)->graph.batchnorm(static_cast(input), static_cast(weight), static_cast(bias), static_cast(running_mean), static_cast(running_var), axis, epsilon)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_topk"); + try { + *out = static_cast(as_graph(graph)->graph.topk(static_cast(input), k)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_rms_norm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rms_norm"); + try { + *out = static_cast(as_graph(graph)->graph.rms_norm(static_cast(input), static_cast(weight), epsilon)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_rope(cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rope"); + try { + *out = static_cast(as_graph(graph)->graph.rope(static_cast(input), theta, position_offset, static_cast(backend))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_rope_gptj(cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rope_gptj"); + try { + *out = static_cast(as_graph(graph)->graph.rope_gptj(static_cast(input), theta, position_offset, rot_dim, static_cast(backend))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_softmax"); + try { + *out = static_cast(as_graph(graph)->graph.softmax(static_cast(input), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_attention(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_attention"); + try { + if (use_mask) { + *out = static_cast(as_graph(graph)->graph.attention_masked(static_cast(query), static_cast(key), static_cast(value), static_cast(mask), scale, is_causal, static_cast(backend), additive_mask, position_offset, window_size)); + } else if (window_size > 0 || position_offset > 0) { + *out = static_cast(as_graph(graph)->graph.attention(static_cast(query), static_cast(key), static_cast(value), scale, position_offset, window_size, static_cast(backend))); + } else { + *out = static_cast(as_graph(graph)->graph.attention(static_cast(query), static_cast(key), static_cast(value), scale, is_causal, static_cast(backend))); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_rel_pos_bias(cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rel_pos_bias"); + try { + *out = static_cast(as_graph(graph)->graph.rel_pos_bias(static_cast(query), static_cast(relative_key), scale)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_attention_int8_hybrid(cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_attention_int8_hybrid"); + try { + *out = static_cast(as_graph(graph)->graph.attention_int8_hybrid(static_cast(query), static_cast(key_new), static_cast(value_new), scale, position_offset, cached_keys, cached_values, k_scales, v_scales, cache_len, num_kv_heads, head_dim, window_size)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv1d_causal(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_causal"); + try { + *out = static_cast(as_graph(graph)->graph.conv1d_causal(static_cast(input), static_cast(weight), kernel_size, dilation)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv1d_k3(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_k3"); + try { + *out = static_cast(as_graph(graph)->graph.conv1d_k3(static_cast(input), static_cast(weight), stride)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv1d_k7s3(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_k7s3"); + try { + *out = static_cast(as_graph(graph)->graph.conv1d_k7s3(static_cast(input), static_cast(weight), static_cast(bias))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv1d(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d"); + try { + if (has_bias) { + *out = static_cast(as_graph(graph)->graph.conv1d(static_cast(input), static_cast(weight), static_cast(bias), stride)); + } else { + *out = static_cast(as_graph(graph)->graph.conv1d(static_cast(input), static_cast(weight), stride)); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv1d_same_depthwise_k9(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_same_depthwise_k9"); + try { + if (has_bias) { + *out = static_cast(as_graph(graph)->graph.conv1d_same_depthwise_k9(static_cast(input), static_cast(weight), static_cast(bias))); + } else { + *out = static_cast(as_graph(graph)->graph.conv1d_same_depthwise_k9(static_cast(input), static_cast(weight))); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv1d_pointwise(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_pointwise"); + try { + if (has_bias) { + *out = static_cast(as_graph(graph)->graph.conv1d_pointwise(static_cast(input), static_cast(weight), static_cast(bias))); + } else { + *out = static_cast(as_graph(graph)->graph.conv1d_pointwise(static_cast(input), static_cast(weight))); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv2d_k3s2p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_k3s2p1"); + try { + if (has_bias) { + *out = static_cast(as_graph(graph)->graph.conv2d_k3s2p1(static_cast(input), static_cast(weight), static_cast(bias))); + } else { + *out = static_cast(as_graph(graph)->graph.conv2d_k3s2p1(static_cast(input), static_cast(weight))); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv2d_depthwise_k3s2p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_depthwise_k3s2p1"); + try { + if (has_bias) { + *out = static_cast(as_graph(graph)->graph.conv2d_depthwise_k3s2p1(static_cast(input), static_cast(weight), static_cast(bias))); + } else { + *out = static_cast(as_graph(graph)->graph.conv2d_depthwise_k3s2p1(static_cast(input), static_cast(weight))); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv2d_pointwise_1x1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_pointwise_1x1"); + try { + if (has_bias) { + *out = static_cast(as_graph(graph)->graph.conv2d_pointwise_1x1(static_cast(input), static_cast(weight), static_cast(bias))); + } else { + *out = static_cast(as_graph(graph)->graph.conv2d_pointwise_1x1(static_cast(input), static_cast(weight))); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv2d_k3s1p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_k3s1p1"); + try { + if (has_bias) { + *out = static_cast(as_graph(graph)->graph.conv2d_k3s1p1(static_cast(input), static_cast(weight), static_cast(bias))); + } else { + *out = static_cast(as_graph(graph)->graph.conv2d_k3s1p1(static_cast(input), static_cast(weight))); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_conv2d( + cactus_graph_t graph, + cactus_node_t input, + cactus_node_t weight, + bool has_bias, + cactus_node_t bias, + size_t stride, + size_t padding, + size_t dilation, + size_t groups, + cactus_node_t* out +) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d"); + try { + if (has_bias) { + *out = static_cast( + as_graph(graph)->graph.conv2d( + static_cast(input), + static_cast(weight), + static_cast(bias), + stride, + padding, + dilation, + groups + ) + ); + } else { + *out = static_cast( + as_graph(graph)->graph.conv2d( + static_cast(input), + static_cast(weight), + stride, + padding, + dilation, + groups + ) + ); + } + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_lstm_cell(cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_lstm_cell"); + try { + *out = static_cast(as_graph(graph)->graph.lstm_cell(static_cast(input), static_cast(h_prev), static_cast(c_prev), static_cast(weight_ih), static_cast(weight_hh), static_cast(bias_ih), static_cast(bias_hh))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_gated_deltanet_decode(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gated_deltanet_decode"); + try { + *out = static_cast(as_graph(graph)->graph.gated_deltanet_decode(static_cast(query), static_cast(key), static_cast(value), static_cast(gate_log), static_cast(beta), static_cast(initial_state), scale)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_gated_deltanet_prefill(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gated_deltanet_prefill"); + try { + *out = static_cast(as_graph(graph)->graph.gated_deltanet_prefill(static_cast(query), static_cast(key), static_cast(value), static_cast(gate_log), static_cast(beta), static_cast(initial_state), chunk_size, scale)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_stft(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_stft"); + try { + *out = static_cast(as_graph(graph)->graph.stft(static_cast(input), static_cast(weight), stride, num_fft_bins)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_altup_predict(cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out) { + if (!graph || !streams || num_streams == 0 || !out) return fail_invalid("Invalid args to cactus_graph_altup_predict"); + try { + std::vector s(num_streams); + for (size_t i = 0; i < num_streams; ++i) s[i] = static_cast(streams[i]); + *out = static_cast(as_graph(graph)->graph.altup_predict(static_cast(coefs), s.data(), num_streams)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_altup_correct(cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out) { + if (!graph || !predictions || num_predictions == 0 || !out) return fail_invalid("Invalid args to cactus_graph_altup_correct"); + try { + std::vector p(num_predictions); + for (size_t i = 0; i < num_predictions; ++i) p[i] = static_cast(predictions[i]); + *out = static_cast(as_graph(graph)->graph.altup_correct(static_cast(coefs), static_cast(innovation), p.data(), num_predictions)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_gaussian_topk(cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gaussian_topk"); + try { + *out = static_cast(as_graph(graph)->graph.gaussian_topk(static_cast(input), ppf)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_moe_layer_gated(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out) { + if (!graph || !w1_weights || !w3_weights || !w2_weights || !out) return fail_invalid("Invalid args to cactus_graph_moe_layer_gated"); + try { + std::vector w1(num_experts), w3(num_experts), w2(num_experts); + for (size_t i = 0; i < num_experts; ++i) { + w1[i] = static_cast(w1_weights[i]); + w3[i] = static_cast(w3_weights[i]); + w2[i] = static_cast(w2_weights[i]); + } + *out = static_cast(as_graph(graph)->graph.moe_layer(static_cast(hidden), static_cast(routing_probs), static_cast(topk_indices), w1, w3, w2, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_moe_layer_ungated(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out) { + if (!graph || !w1_weights || !w2_weights || !out) return fail_invalid("Invalid args to cactus_graph_moe_layer_ungated"); + try { + std::vector w1(num_experts), w2(num_experts); + for (size_t i = 0; i < num_experts; ++i) { + w1[i] = static_cast(w1_weights[i]); + w2[i] = static_cast(w2_weights[i]); + } + *out = static_cast(as_graph(graph)->graph.moe_layer(static_cast(hidden), static_cast(routing_probs), static_cast(topk_indices), w1, w2, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor, static_cast(activation))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_sample(cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sample"); + try { + std::unordered_map empty_bias; + *out = static_cast(as_graph(graph)->graph.sample(static_cast(logits), temperature, top_p, top_k, empty_bias)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_scatter_topk(cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scatter_topk"); + try { + *out = static_cast(as_graph(graph)->graph.scatter_topk(static_cast(indices), static_cast(values), num_classes)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_persistent(cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_persistent"); + try { + *out = static_cast(as_graph(graph)->graph.persistent(static_cast(source_node))); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_is_populated(cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated) { + if (!graph || !out_is_populated) return fail_invalid("Invalid args to cactus_graph_is_populated"); + try { + *out_is_populated = as_graph(graph)->graph.is_populated(static_cast(persistent_node)) ? 1 : 0; + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_invalidate_persistent(cactus_graph_t graph, cactus_node_t persistent_node) { + if (!graph) return fail_invalid("Invalid args to cactus_graph_invalidate_persistent"); + try { + as_graph(graph)->graph.invalidate_persistent(static_cast(persistent_node)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_execute(cactus_graph_t graph) { + if (!graph) { + last_error_message = "Graph is null"; + return -1; + } + try { + as_graph(graph)->graph.execute(); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_get_output_ptr(cactus_graph_t graph, cactus_node_t node, void** out_ptr) { + if (!graph || !out_ptr) { + last_error_message = "Invalid args to cactus_graph_get_output_ptr"; + return -1; + } + try { + *out_ptr = as_graph(graph)->graph.get_output(static_cast(node)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + +int cactus_graph_get_output_info(cactus_graph_t graph, cactus_node_t node, cactus_tensor_info_t* out_info) { + if (!graph || !out_info) { + last_error_message = "Invalid args to cactus_graph_get_output_info"; + return -1; + } + try { + const auto& buf = as_graph(graph)->graph.get_output_buffer(static_cast(node)); + out_info->precision = static_cast(buf.precision); + out_info->rank = buf.shape.size(); + if (out_info->rank > 8) { + last_error_message = "Rank exceeds cactus_tensor_info_t shape capacity"; + return -1; + } + for (size_t i = 0; i < out_info->rank; ++i) { + out_info->shape[i] = buf.shape[i]; + } + for (size_t i = out_info->rank; i < 8; ++i) { + out_info->shape[i] = 0; + } + out_info->num_elements = buf.total_size; + out_info->byte_size = buf.byte_size; + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} +} diff --git a/cactus-transpile/ffi/cactus_index.cpp b/cactus-transpile/ffi/cactus_index.cpp new file mode 100644 index 000000000..a81f62377 --- /dev/null +++ b/cactus-transpile/ffi/cactus_index.cpp @@ -0,0 +1,458 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include + +using namespace cactus::ffi; + +struct CactusIndexHandle { + std::unique_ptr index; +}; + +static cactus::engine::index::QueryOptions parse_query_options_json(const std::string& json) { + cactus::engine::index::QueryOptions options; + + if (json.empty()) return options; + + size_t pos = json.find("\"top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"score_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.score_threshold = std::stof(json.substr(pos)); + } + + return options; +} + +extern "C" { + +cactus_index_t cactus_index_init(const char* index_dir, size_t embedding_dim) { + if (!index_dir) { + last_error_message = "Index directory path cannot be null"; + CACTUS_LOG_ERROR("index_init", last_error_message); + return nullptr; + } + + if (embedding_dim == 0) { + last_error_message = "Embedding dimension must be greater than 0"; + CACTUS_LOG_ERROR("index_init", last_error_message); + return nullptr; + } + + std::string dir_path(index_dir); + if (dir_path.empty()) { + last_error_message = "Index directory path cannot be empty"; + CACTUS_LOG_ERROR("index_init", last_error_message); + return nullptr; + } + + std::string index_path_str = dir_path + "/index.bin"; + std::string data_path_str = dir_path + "/data.bin"; + + CACTUS_LOG_INFO("index_init", "Initializing index in directory: " << dir_path << ", dim: " << embedding_dim); + + try { + auto* handle = new CactusIndexHandle(); + handle->index = std::make_unique( + index_path_str, + data_path_str, + embedding_dim + ); + + if (!handle->index) { + last_error_message = "Failed to create index instance"; + CACTUS_LOG_ERROR("index_init", last_error_message); + + delete handle; + return nullptr; + } + + CACTUS_LOG_INFO("index_init", "Index initialized successfully"); + return handle; + + } catch (const std::exception& e) { + last_error_message = "Exception during index init: " + std::string(e.what()); + CACTUS_LOG_ERROR("index_init", last_error_message); + return nullptr; + } catch (...) { + last_error_message = "Unknown exception during index initialization"; + CACTUS_LOG_ERROR("index_init", last_error_message); + return nullptr; + } +} + +int cactus_index_add( + cactus_index_t index, + const int* ids, + const char** documents, + const char** metadatas, + const float** embeddings, + size_t count, + size_t embedding_dim +) { + if (!index) { + last_error_message = "Index not initialized"; + CACTUS_LOG_ERROR("index_add", last_error_message); + return -1; + } + + if (!ids || !documents || !embeddings) { + last_error_message = "Invalid parameters: ids, documents, or embeddings is null"; + CACTUS_LOG_ERROR("index_add", last_error_message); + return -1; + } + + if (count == 0) { + last_error_message = "Invalid parameter: count must be greater than 0"; + CACTUS_LOG_ERROR("index_add", last_error_message); + return -1; + } + + if (embedding_dim == 0) { + last_error_message = "Invalid parameter: embedding_dim must be greater than 0"; + CACTUS_LOG_ERROR("index_add", last_error_message); + return -1; + } + + try { + auto* handle = static_cast(index); + + std::vector docs; + docs.reserve(count); + + for (size_t i = 0; i < count; ++i) { + if (!embeddings[i]) { + last_error_message = "Embedding at index " + std::to_string(i) + " is null"; + CACTUS_LOG_ERROR("index_add", last_error_message); + return -1; + } + + docs.emplace_back( + ids[i], + std::vector(embeddings[i], embeddings[i] + embedding_dim), + documents[i] ? std::string(documents[i]) : "", + metadatas && metadatas[i] ? std::string(metadatas[i]) : "" + ); + } + + CACTUS_LOG_INFO("index_add", "Adding " << docs.size() << " documents"); + handle->index->add_documents(docs); + + return 0; + + } catch (const std::exception& e) { + last_error_message = std::string(e.what()); + CACTUS_LOG_ERROR("index_add", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during add"; + CACTUS_LOG_ERROR("index_add", last_error_message); + return -1; + } +} + +int cactus_index_delete( + cactus_index_t index, + const int* ids, + size_t ids_count +) { + if (!index) { + last_error_message = "Index not initialized"; + CACTUS_LOG_ERROR("index_delete", last_error_message); + return -1; + } + + if (!ids || ids_count == 0) { + last_error_message = "Invalid parameters"; + CACTUS_LOG_ERROR("index_delete", last_error_message); + return -1; + } + + try { + auto* handle = static_cast(index); + + std::vector doc_ids(ids, ids + ids_count); + + CACTUS_LOG_INFO("index_delete", "Deleting " << doc_ids.size() << " documents"); + handle->index->delete_documents(doc_ids); + + return 0; + + } catch (const std::exception& e) { + last_error_message = std::string(e.what()); + CACTUS_LOG_ERROR("index_delete", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during delete"; + CACTUS_LOG_ERROR("index_delete", last_error_message); + return -1; + } +} + +int cactus_index_get( + cactus_index_t index, + const int* ids, + size_t ids_count, + char** document_buffers, + size_t* document_buffer_sizes, + char** metadata_buffers, + size_t* metadata_buffer_sizes, + float** embedding_buffers, + size_t* embedding_buffer_sizes +) { + if (!index) { + last_error_message = "Index not initialized"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + + if (!ids) { + last_error_message = "Invalid parameter: ids is null"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + + if (ids_count == 0) { + last_error_message = "Invalid parameter: ids_count must be greater than 0"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + + if (document_buffers && !document_buffer_sizes) { + last_error_message = "Invalid parameters: document_buffers provided but document_buffer_sizes is null"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + + if (metadata_buffers && !metadata_buffer_sizes) { + last_error_message = "Invalid parameters: metadata_buffers provided but metadata_buffer_sizes is null"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + + if (embedding_buffers && !embedding_buffer_sizes) { + last_error_message = "Invalid parameters: embedding_buffers provided but embedding_buffer_sizes is null"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + + try { + auto* handle = static_cast(index); + + std::vector doc_ids(ids, ids + ids_count); + + CACTUS_LOG_INFO("index_get", "Getting " << doc_ids.size() << " documents"); + std::vector docs = handle->index->get_documents(doc_ids); + + for (size_t i = 0; i < docs.size(); ++i) { + if (document_buffers) { + size_t document_size = docs[i].content.size() + 1; + if (document_buffer_sizes[i] < document_size) { + last_error_message = "Document buffer too small at index " + std::to_string(i) + + " (required: " + std::to_string(document_size) + + ", provided: " + std::to_string(document_buffer_sizes[i]) + ")"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + } + + if (metadata_buffers) { + size_t metadata_size = docs[i].metadata.size() + 1; + if (metadata_buffer_sizes[i] < metadata_size) { + last_error_message = "Metadata buffer too small at index " + std::to_string(i) + + " (required: " + std::to_string(metadata_size) + + ", provided: " + std::to_string(metadata_buffer_sizes[i]) + ")"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + } + + if (embedding_buffers) { + size_t embedding_size = docs[i].embedding.size(); + if (embedding_buffer_sizes[i] < embedding_size) { + last_error_message = "Embedding buffer too small at index " + std::to_string(i) + + " (required: " + std::to_string(embedding_size) + + ", provided: " + std::to_string(embedding_buffer_sizes[i]) + ")"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } + } + } + + for (size_t i = 0; i < docs.size(); ++i) { + if (document_buffers) { + size_t document_size = docs[i].content.size() + 1; + std::memcpy(document_buffers[i], docs[i].content.c_str(), document_size); + document_buffer_sizes[i] = document_size; + } + + if (metadata_buffers) { + size_t metadata_size = docs[i].metadata.size() + 1; + std::memcpy(metadata_buffers[i], docs[i].metadata.c_str(), metadata_size); + metadata_buffer_sizes[i] = metadata_size; + } + + if (embedding_buffers) { + size_t embedding_size = docs[i].embedding.size(); + std::memcpy(embedding_buffers[i], docs[i].embedding.data(), embedding_size * sizeof(float)); + embedding_buffer_sizes[i] = embedding_size; + } + } + + return 0; + + } catch (const std::exception& e) { + last_error_message = std::string(e.what()); + CACTUS_LOG_ERROR("index_get", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during get"; + CACTUS_LOG_ERROR("index_get", last_error_message); + return -1; + } +} + +int cactus_index_query( + cactus_index_t index, + const float** embeddings, + size_t embeddings_count, + size_t embedding_dim, + const char* options_json, + int** id_buffers, + size_t* id_buffer_sizes, + float** score_buffers, + size_t* score_buffer_sizes +) { + if (!index) { + last_error_message = "Index not initialized"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } + + if (!embeddings) { + last_error_message = "Invalid parameter: embeddings is null"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } + + if (!id_buffers || !score_buffers) { + last_error_message = "Invalid parameters: id_buffers or score_buffers is null"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } + + if (!id_buffer_sizes || !score_buffer_sizes) { + last_error_message = "Invalid parameters: buffer size arrays cannot be null"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } + + if (embeddings_count == 0) { + last_error_message = "Invalid parameter: embeddings_count must be greater than 0"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } + + if (embedding_dim == 0) { + last_error_message = "Invalid parameter: embedding_dim must be greater than 0"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } + + try { + auto* handle = static_cast(index); + + std::vector> embeddings_vec; + embeddings_vec.reserve(embeddings_count); + + for (size_t i = 0; i < embeddings_count; ++i) { + if (!embeddings[i]) { + last_error_message = "Embedding at index " + std::to_string(i) + " is null"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } + embeddings_vec.emplace_back(embeddings[i], embeddings[i] + embedding_dim); + } + + cactus::engine::index::QueryOptions options; + if (options_json && std::strlen(options_json) > 0) { + options = parse_query_options_json(options_json); + } + + CACTUS_LOG_DEBUG("index_query", "Querying with " << embeddings_count << " queries, dim: " << embedding_dim << ", top_k: " << options.top_k); + std::vector> results = handle->index->query(embeddings_vec, options); + + for (size_t i = 0; i < results.size(); ++i) { + size_t result_count = results[i].size(); + if (id_buffer_sizes[i] < result_count || score_buffer_sizes[i] < result_count) { + last_error_message = "Result buffer too small at query index " + std::to_string(i) + + " (required: " + std::to_string(result_count) + + ", id_buffer: " + std::to_string(id_buffer_sizes[i]) + + ", score_buffer: " + std::to_string(score_buffer_sizes[i]) + ")"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } + } + + for (size_t i = 0; i < results.size(); ++i) { + size_t result_count = results[i].size(); + + for (size_t j = 0; j < result_count; ++j) { + id_buffers[i][j] = results[i][j].doc_id; + score_buffers[i][j] = results[i][j].score; + } + + id_buffer_sizes[i] = result_count; + score_buffer_sizes[i] = result_count; + } + + return 0; + + } catch (const std::exception& e) { + last_error_message = std::string(e.what()); + CACTUS_LOG_ERROR("index_query", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during query"; + CACTUS_LOG_ERROR("index_query", last_error_message); + return -1; + } +} + +int cactus_index_compact(cactus_index_t index) { + if (!index) { + last_error_message = "Index not initialized"; + CACTUS_LOG_ERROR("index_compact", last_error_message); + return -1; + } + + try { + auto* handle = static_cast(index); + + CACTUS_LOG_INFO("index_compact", "Compacting index"); + handle->index->compact(); + + return 0; + + } catch (const std::exception& e) { + last_error_message = std::string(e.what()); + CACTUS_LOG_ERROR("index_compact", "Exception: " << e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown error during compact"; + CACTUS_LOG_ERROR("index_compact", last_error_message); + return -1; + } +} + +void cactus_index_destroy(cactus_index_t index) { + if (index) delete static_cast(index); +} + +} diff --git a/cactus-transpile/ffi/cactus_init.cpp b/cactus-transpile/ffi/cactus_init.cpp new file mode 100644 index 000000000..13cc35851 --- /dev/null +++ b/cactus-transpile/ffi/cactus_init.cpp @@ -0,0 +1,489 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include "telemetry/telemetry.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; + +static constexpr size_t RAG_MAX_CHUNK_TOKENS = 128; +static constexpr size_t RAG_MIN_CHUNK_TOKENS = 24; +static constexpr size_t RAG_CHUNK_OVERLAP = 32; + +static void apply_no_cloud_telemetry_env() { + if (cactus::ffi::env_flag_enabled("CACTUS_NO_CLOUD_TELE")) { + cactus::telemetry::setCloudDisabled(true); + } +} + +static time_t get_file_mtime(const std::string& path) { + struct stat st; + if (stat(path.c_str(), &st) == 0) { + return st.st_mtime; + } + return 0; +} + +static bool corpus_is_stale(const std::string& corpus_dir) { + std::string index_path = corpus_dir + "/index.bin"; + time_t index_mtime = get_file_mtime(index_path); + if (index_mtime == 0) return true; + + DIR* dir = opendir(corpus_dir.c_str()); + if (!dir) return true; + + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + std::string name = entry->d_name; + if (name == "." || name == "..") continue; + if (name == "index.bin" || name == "data.bin") continue; + + bool is_corpus_file = false; + if (name.size() > 4 && name.substr(name.size() - 4) == ".txt") is_corpus_file = true; + if (name.size() > 3 && name.substr(name.size() - 3) == ".md") is_corpus_file = true; + + if (is_corpus_file) { + std::string full_path = corpus_dir + "/" + name; + time_t file_mtime = get_file_mtime(full_path); + if (file_mtime > index_mtime) { + closedir(dir); + CACTUS_LOG_INFO("init", "Corpus file " << name << " is newer than index, rebuilding"); + return true; + } + } + } + closedir(dir); + return false; +} + +static std::string read_file_contents(const std::string& path) { + std::ifstream file(path); + if (!file.is_open()) return ""; + std::stringstream buffer; + buffer << file.rdbuf(); + return buffer.str(); +} + +static std::vector scan_corpus_files(const std::string& corpus_dir) { + std::vector files; + DIR* dir = opendir(corpus_dir.c_str()); + if (!dir) return files; + + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + std::string name = entry->d_name; + if (name == "." || name == "..") continue; + if (name == "index.bin" || name == "data.bin") continue; + + std::string full_path = corpus_dir + "/" + name; + struct stat st; + if (stat(full_path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) { + if (name.size() > 4 && (name.substr(name.size() - 4) == ".txt" || name.substr(name.size() - 3) == ".md")) { + files.push_back(full_path); + } + } + } + closedir(dir); + return files; +} + +static std::vector split_into_paragraphs(const std::string& content) { + std::vector paragraphs; + std::string current; + + size_t i = 0; + while (i < content.size()) { + // Check for markdown header + if (content[i] == '#' && (i == 0 || content[i-1] == '\n')) { + if (!current.empty()) { + paragraphs.push_back(current); + current.clear(); + } + // Include the header line + while (i < content.size() && content[i] != '\n') { + current += content[i++]; + } + if (i < content.size()) current += content[i++]; + continue; + } + + // Check for double newline (paragraph break) + if (content[i] == '\n' && i + 1 < content.size() && content[i+1] == '\n') { + current += content[i]; + if (!current.empty() && current != "\n") { + paragraphs.push_back(current); + current.clear(); + } + i++; + // Skip multiple blank lines + while (i < content.size() && content[i] == '\n') i++; + continue; + } + + current += content[i++]; + } + + if (!current.empty()) { + paragraphs.push_back(current); + } + + return paragraphs; +} + +static std::vector> chunk_corpus( + const std::vector& file_paths, + Tokenizer* tokenizer +) { + std::vector> chunks; + + for (const auto& path : file_paths) { + std::string content = read_file_contents(path); + if (content.empty()) continue; + + std::string filename = path; + size_t last_slash = path.find_last_of("/\\"); + if (last_slash != std::string::npos) { + filename = path.substr(last_slash + 1); + } + + auto paragraphs = split_into_paragraphs(content); + + std::string current_chunk; + size_t current_tokens = 0; + + for (const auto& para : paragraphs) { + std::vector para_tokens = tokenizer->encode(para); + size_t para_token_count = para_tokens.size(); + + // If paragraph alone is too large, split it by tokens + if (para_token_count > RAG_MAX_CHUNK_TOKENS) { + // Flush current chunk first + if (!current_chunk.empty()) { + chunks.emplace_back(current_chunk, filename); + current_chunk.clear(); + current_tokens = 0; + } + + // Split large paragraph by sentences or fixed tokens + size_t stride = RAG_MAX_CHUNK_TOKENS - RAG_CHUNK_OVERLAP; + for (size_t i = 0; i < para_tokens.size(); i += stride) { + size_t end = std::min(i + RAG_MAX_CHUNK_TOKENS, para_tokens.size()); + std::vector chunk_tokens(para_tokens.begin() + i, para_tokens.begin() + end); + std::string chunk_text = tokenizer->decode(chunk_tokens); + chunks.emplace_back(chunk_text, filename); + if (end >= para_tokens.size()) break; + } + continue; + } + + // Would adding this paragraph exceed max? + if (current_tokens + para_token_count > RAG_MAX_CHUNK_TOKENS && !current_chunk.empty()) { + chunks.emplace_back(current_chunk, filename); + current_chunk.clear(); + current_tokens = 0; + } + + // Add paragraph to current chunk + if (!current_chunk.empty()) current_chunk += "\n"; + current_chunk += para; + current_tokens += para_token_count; + } + + // Don't forget the last chunk + if (!current_chunk.empty() && current_tokens >= RAG_MIN_CHUNK_TOKENS) { + chunks.emplace_back(current_chunk, filename); + } else if (!current_chunk.empty() && !chunks.empty()) { + // Append small remaining chunk to previous + chunks.back().first += "\n" + current_chunk; + } else if (!current_chunk.empty()) { + chunks.emplace_back(current_chunk, filename); + } + } + + return chunks; +} + +static bool build_corpus_index(CactusModelHandle* handle, const std::string& corpus_dir) { + CACTUS_LOG_INFO("init", "Building corpus index from: " << corpus_dir); + + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) { + CACTUS_LOG_ERROR("init", "No tokenizer available for corpus indexing"); + return false; + } + + auto file_paths = scan_corpus_files(corpus_dir); + if (file_paths.empty()) { + CACTUS_LOG_WARN("init", "No .txt or .md files found in corpus directory"); + return false; + } + + CACTUS_LOG_INFO("init", "Found " << file_paths.size() << " corpus files"); + + auto chunks = chunk_corpus(file_paths, tokenizer); + if (chunks.empty()) { + CACTUS_LOG_WARN("init", "No chunks generated from corpus"); + return false; + } + + CACTUS_LOG_INFO("init", "Generated " << chunks.size() << " chunks from corpus"); + + std::vector test_tokens = tokenizer->encode("test"); + std::vector test_embedding = handle->model->get_embeddings(test_tokens, true, true); + if (test_embedding.empty()) { + CACTUS_LOG_ERROR("init", "Failed to get embedding dimension"); + return false; + } + size_t embedding_dim = test_embedding.size(); + handle->corpus_embedding_dim = embedding_dim; + + CACTUS_LOG_INFO("init", "Embedding dimension: " << embedding_dim); + + std::string index_path = corpus_dir + "/index.bin"; + std::string data_path = corpus_dir + "/data.bin"; + + std::remove(index_path.c_str()); + std::remove(data_path.c_str()); + + try { + handle->corpus_index = std::make_unique(index_path, data_path, embedding_dim); + } catch (const std::exception& e) { + CACTUS_LOG_ERROR("init", "Failed to create index: " << e.what()); + return false; + } + + std::vector docs; + docs.reserve(chunks.size()); + + for (size_t i = 0; i < chunks.size(); ++i) { + const auto& [chunk_text, source_file] = chunks[i]; + + std::vector tokens = tokenizer->encode(chunk_text); + std::vector embedding = handle->model->get_embeddings(tokens, true, true); + + if (embedding.size() != embedding_dim) { + CACTUS_LOG_WARN("init", "Skipping chunk " << i << " - embedding dimension mismatch"); + continue; + } + + docs.push_back(index::Document{ + static_cast(i), + std::move(embedding), + chunk_text, + source_file + }); + + if ((i + 1) % 50 == 0) { + CACTUS_LOG_INFO("init", "Embedded " << (i + 1) << "/" << chunks.size() << " chunks"); + } + } + + if (docs.empty()) { + CACTUS_LOG_ERROR("init", "No documents to add to index"); + return false; + } + + try { + handle->corpus_index->add_documents(docs); + } catch (const std::exception& e) { + CACTUS_LOG_ERROR("init", "Failed to add documents to index: " << e.what()); + return false; + } + + CACTUS_LOG_INFO("init", "Corpus index built successfully with " << docs.size() << " chunks"); + return true; +} + +static bool load_corpus_index(CactusModelHandle* handle, const std::string& corpus_dir) { + std::string index_path = corpus_dir + "/index.bin"; + std::string data_path = corpus_dir + "/data.bin"; + + struct stat st; + if (stat(index_path.c_str(), &st) != 0 || stat(data_path.c_str(), &st) != 0) { + return false; + } + + if (corpus_is_stale(corpus_dir)) { + return false; + } + + auto* tokenizer = handle->model->get_tokenizer(); + std::vector test_tokens = tokenizer->encode("test"); + std::vector test_embedding = handle->model->get_embeddings(test_tokens, true, true); + if (test_embedding.empty()) { + CACTUS_LOG_ERROR("init", "Failed to get embedding dimension for index loading"); + return false; + } + size_t embedding_dim = test_embedding.size(); + handle->corpus_embedding_dim = embedding_dim; + + try { + handle->corpus_index = std::make_unique(index_path, data_path, embedding_dim); + CACTUS_LOG_INFO("init", "Loaded existing corpus index from: " << corpus_dir); + return true; + } catch (const std::exception& e) { + CACTUS_LOG_WARN("init", "Failed to load existing index: " << e.what()); + return false; + } +} + +std::string last_error_message; + +bool matches_stop_sequence(const std::vector& generated_tokens, + const std::vector>& stop_sequences) { + for (const auto& stop_seq : stop_sequences) { + if (stop_seq.empty()) continue; + if (generated_tokens.size() >= stop_seq.size()) { + if (std::equal(stop_seq.rbegin(), stop_seq.rend(), generated_tokens.rbegin())) + return true; + } + } + return false; +} + +extern "C" { + +const char* cactus_get_last_error() { + return last_error_message.c_str(); +} + +cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool cache_index) { + constexpr size_t DEFAULT_CONTEXT_SIZE = 512; // matches default sliding window size + + std::string model_path_str = model_path ? std::string(model_path) : "unknown"; + + std::string model_name = model_path_str; + size_t last_slash = model_path_str.find_last_of("/\\"); + if (last_slash != std::string::npos) { + model_name = model_path_str.substr(last_slash + 1); + } + + CACTUS_LOG_INFO("init", "Loading model: " << model_name << " from " << model_path_str); + + apply_no_cloud_telemetry_env(); + cactus::telemetry::init(nullptr, model_path_str.c_str(), nullptr); + + auto __cactus_init_start = std::chrono::steady_clock::now(); + + try { + auto* handle = new CactusModelHandle(); + handle->model = create_model(model_path); + handle->model_name = model_name; + + if (!handle->model) { + last_error_message = "Failed to create model - check config.txt exists at: " + model_path_str; + CACTUS_LOG_ERROR("init", last_error_message); + { + auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); + cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); + } + delete handle; + return nullptr; + } + + if (!handle->model->init(model_path, DEFAULT_CONTEXT_SIZE)) { + last_error_message = "Failed to initialize model - check weight files at: " + model_path_str; + CACTUS_LOG_ERROR("init", last_error_message); + { + auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); + cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); + } + delete handle; + return nullptr; + } + + auto model_type = handle->model->get_config().model_type; + if (model_type == Config::ModelType::WHISPER || + model_type == Config::ModelType::MOONSHINE || + model_type == Config::ModelType::PARAKEET || + model_type == Config::ModelType::PARAKEET_TDT) { + std::string vad_path = model_path_str + "/vad"; + const std::filesystem::path vad_config_path = std::filesystem::path(vad_path) / "config.txt"; + if (std::filesystem::exists(vad_config_path)) { + handle->vad_model = create_model(vad_path); + if (!handle->vad_model) { + last_error_message = "Failed to create VAD model - check VAD weights at: " + vad_path; + CACTUS_LOG_ERROR("init", last_error_message); + delete handle; + return nullptr; + } + if (!handle->vad_model->init(vad_path, 0, "", false)) { + last_error_message = "Failed to initialize VAD model - check VAD weight files at: " + vad_path; + CACTUS_LOG_ERROR("init", last_error_message); + delete handle; + return nullptr; + } + } else { + CACTUS_LOG_WARN("init", "VAD weights not found at " << vad_path << "; continuing without VAD"); + } + } + + if (corpus_dir != nullptr && strlen(corpus_dir) > 0) { + handle->corpus_dir = std::string(corpus_dir); + + bool loaded = false; + if (cache_index) { + loaded = load_corpus_index(handle, handle->corpus_dir); + } + + if (!loaded) { + CACTUS_LOG_INFO("init", (cache_index ? "No existing index found, building new corpus index" : "Building fresh corpus index (caching disabled)")); + if (!build_corpus_index(handle, handle->corpus_dir)) { + CACTUS_LOG_WARN("init", "Failed to build corpus index - RAG disabled"); + } + } + } + + CACTUS_LOG_INFO("init", "Model loaded successfully: " << model_name); + { + auto __cactus_init_ok_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); + cactus::telemetry::recordInit(model_name.c_str(), true, static_cast(__cactus_init_ok_dur), ""); + } + + return handle; + } catch (const std::exception& e) { + last_error_message = "Exception during init: " + std::string(e.what()); + CACTUS_LOG_ERROR("init", last_error_message); + { + auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); + cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); + } + return nullptr; + } catch (...) { + last_error_message = "Unknown exception during model initialization"; + CACTUS_LOG_ERROR("init", last_error_message); + { + auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); + cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); + } + return nullptr; + } +} + +void cactus_destroy(cactus_model_t model) { + if (model) delete static_cast(model); +} + +void cactus_reset(cactus_model_t model) { + if (!model) return; + auto* handle = static_cast(model); + handle->model->reset_cache(); + handle->processed_tokens.clear(); + handle->processed_images.clear(); +} + +void cactus_stop(cactus_model_t model) { + if (!model) return; + auto* handle = static_cast(model); + handle->should_stop = true; +} + +} diff --git a/cactus-transpile/ffi/cactus_log.cpp b/cactus-transpile/ffi/cactus_log.cpp new file mode 100644 index 000000000..d067f9adf --- /dev/null +++ b/cactus-transpile/ffi/cactus_log.cpp @@ -0,0 +1,23 @@ +#include "cactus_ffi.h" +#include "graph/graph.h" + +extern "C" { + +void cactus_log_set_level(int level) { + cactus::Logger::instance().set_level(static_cast(level)); +} + +void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data) { + if (callback) { + cactus::Logger::instance().set_callback( + [callback, user_data](cactus::LogLevel level, + const std::string& component, + const std::string& message) { + callback(static_cast(level), component.c_str(), message.c_str(), user_data); + }); + } else { + cactus::Logger::instance().set_callback(nullptr); + } +} + +} diff --git a/cactus-transpile/ffi/cactus_rag.cpp b/cactus-transpile/ffi/cactus_rag.cpp new file mode 100644 index 000000000..a0765f787 --- /dev/null +++ b/cactus-transpile/ffi/cactus_rag.cpp @@ -0,0 +1,459 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; + +static constexpr size_t RAG_TOP_K = 5; +static constexpr size_t RAG_CANDIDATE_K = 20; +static constexpr float RRF_K = 60.0f; +static constexpr float RRF_EMB_WEIGHT = 0.8f; +static constexpr float RRF_BM25_WEIGHT = 0.2f; +static constexpr float BM25_K1 = 1.5f; +static constexpr float BM25_B = 0.75f; + +static std::vector> compute_rrf_scores( + const std::vector>& emb_ranked, + const std::vector>& bm25_ranked, + size_t num_items +) { + std::vector> emb_sorted = emb_ranked; + std::vector> bm25_sorted = bm25_ranked; + + std::sort(emb_sorted.begin(), emb_sorted.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + std::sort(bm25_sorted.begin(), bm25_sorted.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + + std::unordered_map emb_rank_map, bm25_rank_map; + for (size_t r = 0; r < emb_sorted.size(); ++r) { + emb_rank_map[emb_sorted[r].second] = r + 1; + } + for (size_t r = 0; r < bm25_sorted.size(); ++r) { + bm25_rank_map[bm25_sorted[r].second] = r + 1; + } + + std::vector> rrf_scored; + rrf_scored.reserve(num_items); + for (size_t i = 0; i < num_items; ++i) { + float rrf = RRF_EMB_WEIGHT / (RRF_K + emb_rank_map[i]) + + RRF_BM25_WEIGHT / (RRF_K + bm25_rank_map[i]); + rrf_scored.emplace_back(rrf, i); + } + + std::sort(rrf_scored.begin(), rrf_scored.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + + return rrf_scored; +} + +static std::vector tokenize_words(const std::string& text) { + std::vector words; + std::string current; + for (char c : text) { + if (std::isalnum(static_cast(c))) { + current += std::tolower(static_cast(c)); + } else if (!current.empty()) { + if (current.length() > 2) { + words.push_back(current); + } + current.clear(); + } + } + if (!current.empty() && current.length() > 2) { + words.push_back(current); + } + return words; +} + +static float compute_bm25_score( + const std::vector& query_words, + const std::string& doc_content, + float avg_doc_len, + const std::unordered_map& doc_freqs, + int total_docs +) { + auto doc_words = tokenize_words(doc_content); + float doc_len = static_cast(doc_words.size()); + + std::unordered_map tf; + for (const auto& w : doc_words) { + tf[w]++; + } + + float score = 0.0f; + std::unordered_set query_set(query_words.begin(), query_words.end()); + + for (const auto& term : query_set) { + auto tf_it = tf.find(term); + if (tf_it == tf.end()) continue; + + float term_freq = static_cast(tf_it->second); + float df = 1.0f; + auto df_it = doc_freqs.find(term); + if (df_it != doc_freqs.end()) { + df = static_cast(df_it->second); + } + + float idf = std::log((total_docs - df + 0.5f) / (df + 0.5f) + 1.0f); + + float tf_component = (term_freq * (BM25_K1 + 1.0f)) / + (term_freq + BM25_K1 * (1.0f - BM25_B + BM25_B * (doc_len / avg_doc_len))); + + score += idf * tf_component; + } + + return score; +} + +std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query) { + if (!handle->corpus_index || handle->corpus_embedding_dim == 0) { + return ""; + } + + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) return ""; + + std::vector query_tokens = tokenizer->encode(query); + if (query_tokens.empty()) return ""; + + std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); + if (query_embedding.size() != handle->corpus_embedding_dim) { + CACTUS_LOG_WARN("rag", "Query embedding dimension mismatch"); + return ""; + } + + index::QueryOptions options; + options.top_k = RAG_CANDIDATE_K; + options.score_threshold = 0.0f; + + std::vector> query_embeddings = {query_embedding}; + + try { + auto results = handle->corpus_index->query(query_embeddings, options); + if (results.empty() || results[0].empty()) { + return ""; + } + + std::vector doc_ids; + std::vector embedding_scores; + for (const auto& result : results[0]) { + doc_ids.push_back(result.doc_id); + embedding_scores.push_back(result.score); + } + + auto docs = handle->corpus_index->get_documents(doc_ids); + if (docs.empty()) { + return ""; + } + + auto query_words = tokenize_words(query); + + float total_len = 0.0f; + std::unordered_map doc_freqs; + for (const auto& doc : docs) { + auto words = tokenize_words(doc.content); + total_len += words.size(); + std::unordered_set unique_words(words.begin(), words.end()); + for (const auto& w : unique_words) { + doc_freqs[w]++; + } + } + float avg_doc_len = total_len / docs.size(); + + std::vector> emb_ranked; + std::vector> bm25_ranked; + for (size_t i = 0; i < docs.size(); ++i) { + emb_ranked.emplace_back(embedding_scores[i], i); + float bm25 = compute_bm25_score( + query_words, docs[i].content, avg_doc_len, doc_freqs, docs.size() + ); + bm25_ranked.emplace_back(bm25, i); + } + + auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, docs.size()); + + std::string context = "[Retrieved Context - Use ONLY this information to answer. If the answer is not in the context, say \"I don't have enough information to answer that.\"]\n"; + size_t count = std::min(RAG_TOP_K, rrf_scored.size()); + for (size_t i = 0; i < count; ++i) { + size_t idx = rrf_scored[i].second; + context += "---\n"; + context += docs[idx].content; + if (!docs[idx].metadata.empty()) { + context += "\n(Source: " + docs[idx].metadata + ")"; + } + context += "\n"; + } + context += "---\n\n"; + + CACTUS_LOG_DEBUG("rag", "Retrieved " << count << " RAG chunks (hybrid BM25+embedding)"); + return context; + + } catch (const std::exception& e) { + CACTUS_LOG_WARN("rag", "RAG retrieval failed: " << e.what()); + return ""; + } +} + +static std::string tool_to_text(const ToolFunction& tool) { + return tool.name + " " + tool.description; +} + +static float cosine_similarity(const std::vector& a, const std::vector& b) { + if (a.size() != b.size() || a.empty()) return 0.0f; + + float dot = 0.0f, norm_a = 0.0f, norm_b = 0.0f; + for (size_t i = 0; i < a.size(); ++i) { + dot += a[i] * b[i]; + norm_a += a[i] * a[i]; + norm_b += b[i] * b[i]; + } + + float denom = std::sqrt(norm_a) * std::sqrt(norm_b); + return denom > 0.0f ? dot / denom : 0.0f; +} + +std::vector select_relevant_tools( + CactusModelHandle* handle, + const std::string& query, + const std::vector& all_tools, + size_t top_k +) { + using cactus::ffi::ToolFunction; + if (!handle || all_tools.empty() || top_k == 0) return all_tools; + + // If we have fewer tools than top_k, just return all + if (all_tools.size() <= top_k) return all_tools; + + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) { + CACTUS_LOG_WARN("tool_rag", "No tokenizer available, returning all tools"); + return all_tools; + } + + // Build tool texts and embeddings if not cached or tool set changed + bool need_recompute = (handle->tool_texts.size() != all_tools.size()); + if (!need_recompute) { + for (size_t i = 0; i < all_tools.size(); ++i) { + if (tool_to_text(all_tools[i]) != handle->tool_texts[i]) { + need_recompute = true; + break; + } + } + } + + if (need_recompute) { + CACTUS_LOG_DEBUG("tool_rag", "Computing embeddings for " << all_tools.size() << " tools"); + handle->tool_texts.clear(); + handle->tool_embeddings.clear(); + handle->tool_texts.reserve(all_tools.size()); + handle->tool_embeddings.reserve(all_tools.size()); + + for (const auto& tool : all_tools) { + std::string text = tool_to_text(tool); + handle->tool_texts.push_back(text); + + std::vector tokens = tokenizer->encode(text); + if (!tokens.empty()) { + std::vector emb = handle->model->get_embeddings(tokens, true, true); + handle->tool_embeddings.push_back(std::move(emb)); + } else { + handle->tool_embeddings.push_back({}); + } + } + } + + // Embed the query + std::vector query_tokens = tokenizer->encode(query); + if (query_tokens.empty()) { + CACTUS_LOG_WARN("tool_rag", "Empty query, returning all tools"); + return all_tools; + } + + std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); + if (query_embedding.empty()) { + CACTUS_LOG_WARN("tool_rag", "Failed to get query embedding, returning all tools"); + return all_tools; + } + + auto query_words = tokenize_words(query); + + float total_len = 0.0f; + std::unordered_map doc_freqs; + for (const auto& text : handle->tool_texts) { + auto words = tokenize_words(text); + total_len += words.size(); + std::unordered_set unique_words(words.begin(), words.end()); + for (const auto& w : unique_words) { + doc_freqs[w]++; + } + } + float avg_doc_len = total_len / handle->tool_texts.size(); + + std::vector> emb_ranked; + for (size_t i = 0; i < handle->tool_embeddings.size(); ++i) { + float sim = cosine_similarity(query_embedding, handle->tool_embeddings[i]); + emb_ranked.emplace_back(sim, i); + } + + std::vector> bm25_ranked; + for (size_t i = 0; i < handle->tool_texts.size(); ++i) { + float bm25 = compute_bm25_score( + query_words, handle->tool_texts[i], avg_doc_len, doc_freqs, handle->tool_texts.size() + ); + bm25_ranked.emplace_back(bm25, i); + } + + auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, all_tools.size()); + + // Select top-k tools + std::vector selected; + selected.reserve(top_k); + for (size_t i = 0; i < top_k && i < rrf_scored.size(); ++i) { + size_t idx = rrf_scored[i].second; + selected.push_back(all_tools[idx]); + CACTUS_LOG_DEBUG("tool_rag", "Selected tool: " << all_tools[idx].name << " (RRF: " << rrf_scored[i].first << ")"); + } + + CACTUS_LOG_INFO("tool_rag", "Selected " << selected.size() << " of " << all_tools.size() << " tools using hybrid ranking"); + return selected; +} + +extern "C" { + +int cactus_rag_query( + cactus_model_t model, + const char* query, + char* response_buffer, + size_t buffer_size, + size_t top_k +) { + if (!model || !query || !response_buffer || buffer_size == 0) { + return -1; + } + + auto* handle = static_cast(model); + + if (!handle->corpus_index || handle->corpus_embedding_dim == 0) { + std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"No corpus index loaded\"}"); + return 0; + } + + try { + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) { + std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"No tokenizer\"}"); + return 0; + } + + std::vector query_tokens = tokenizer->encode(query); + if (query_tokens.empty()) { + std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Empty query\"}"); + return 0; + } + + std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); + if (query_embedding.size() != handle->corpus_embedding_dim) { + std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Embedding dimension mismatch\"}"); + return 0; + } + + index::QueryOptions options; + options.top_k = RAG_CANDIDATE_K; + options.score_threshold = 0.0f; + + std::vector> query_embeddings = {query_embedding}; + auto results = handle->corpus_index->query(query_embeddings, options); + + if (results.empty() || results[0].empty()) { + std::strcpy(response_buffer, "{\"chunks\":[]}"); + return 0; + } + + std::vector doc_ids; + std::vector embedding_scores; + for (const auto& result : results[0]) { + doc_ids.push_back(result.doc_id); + embedding_scores.push_back(result.score); + } + + auto docs = handle->corpus_index->get_documents(doc_ids); + + auto query_words = tokenize_words(query); + + float total_len = 0.0f; + std::unordered_map doc_freqs; + for (const auto& doc : docs) { + auto words = tokenize_words(doc.content); + total_len += words.size(); + std::unordered_set unique_words(words.begin(), words.end()); + for (const auto& w : unique_words) { + doc_freqs[w]++; + } + } + float avg_doc_len = docs.size() > 0 ? total_len / docs.size() : 1.0f; + + std::vector> emb_ranked; + std::vector> bm25_ranked; + for (size_t i = 0; i < docs.size(); ++i) { + emb_ranked.emplace_back(embedding_scores[i], i); + float bm25 = compute_bm25_score( + query_words, docs[i].content, avg_doc_len, doc_freqs, docs.size() + ); + bm25_ranked.emplace_back(bm25, i); + } + + auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, docs.size()); + + size_t result_count = std::min(top_k > 0 ? top_k : RAG_TOP_K, rrf_scored.size()); + + std::ostringstream oss; + oss << "{\"chunks\":["; + for (size_t i = 0; i < result_count; ++i) { + size_t idx = rrf_scored[i].second; + float final_score = rrf_scored[i].first; + + if (i > 0) oss << ","; + oss << "{\"score\":" << std::setprecision(4) << final_score + << ",\"source\":\"" << docs[idx].metadata << "\"" + << ",\"content\":\""; + for (char c : docs[idx].content) { + switch (c) { + case '"': oss << "\\\""; break; + case '\\': oss << "\\\\"; break; + case '\n': oss << "\\n"; break; + case '\r': oss << "\\r"; break; + case '\t': oss << "\\t"; break; + default: oss << c; + } + } + oss << "\"}"; + } + oss << "]}"; + + std::string result = oss.str(); + if (result.size() >= buffer_size) { + std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Buffer too small\"}"); + return -1; + } + + std::strcpy(response_buffer, result.c_str()); + return static_cast(result.size()); + + } catch (const std::exception& e) { + std::string error = "{\"chunks\":[],\"error\":\"" + std::string(e.what()) + "\"}"; + if (error.size() < buffer_size) { + std::strcpy(response_buffer, error.c_str()); + } + return -1; + } +} + +} diff --git a/cactus-transpile/ffi/cactus_stream.cpp b/cactus-transpile/ffi/cactus_stream.cpp new file mode 100644 index 000000000..b451b8dfd --- /dev/null +++ b/cactus-transpile/ffi/cactus_stream.cpp @@ -0,0 +1,2714 @@ +#include "cactus_ffi.h" +#include "cactus_cloud.h" +#include "cactus_utils.h" +#include "telemetry/telemetry.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace cactus::ffi; +using cactus::audio::WHISPER_SAMPLE_RATE; +using cactus::audio::apply_preemphasis; +using cactus::audio::get_parakeet_spectrogram_config; +using cactus::audio::normalize_parakeet_log_mel; +using cactus::audio::trim_mel_frames; + +double json_number(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return 0.0; + size_t start = pos + pattern.size(); + while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; + size_t end = start; + while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; + try { return std::stod(json.substr(start, end - start)); } + catch (...) { return 0.0; } +} + +std::string json_string(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return {}; + size_t start = pos + pattern.size(); + + while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; + if (start >= json.size() || json[start] != '"') return {}; + size_t q1 = start; + size_t q2 = json.find('"', q1 + 1); + if (q2 == std::string::npos) return {}; + return json.substr(q1 + 1, q2 - q1 - 1); +} + +inline std::string escape_json(const std::string& s) { + return escape_json_string(s); +} + +bool json_bool(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return false; + size_t start = pos + pattern.size(); + while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; + if (start + 4 <= json.size() && json.substr(start, 4) == "true") return true; + return false; +} + +static std::string ensure_json_bool_option(const std::string& json, + const std::string& key, + bool value) { + const std::string field = "\"" + key + "\""; + if (json.find(field) != std::string::npos) return json; + + const std::string bool_str = value ? "true" : "false"; + if (json.empty()) { + return "{\"" + key + "\":" + bool_str + "}"; + } + + std::string out = json; + const size_t close = out.rfind('}'); + const size_t open = out.find('{'); + if (open == std::string::npos || close == std::string::npos || close <= open) { + return json; + } + + size_t content_start = open + 1; + while (content_start < close && std::isspace(static_cast(out[content_start]))) { + ++content_start; + } + const bool has_entries = content_start < close; + const std::string insertion = (has_entries ? "," : "") + field + ":" + bool_str; + out.insert(close, insertion); + return out; +} + +std::string json_array(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return "[]"; + size_t start = pos + pattern.size(); + while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; + if (start >= json.size() || json[start] != '[') return "[]"; + int depth = 1; + size_t end = start + 1; + while (end < json.size() && depth > 0) { + if (json[end] == '[') depth++; + else if (json[end] == ']') depth--; + end++; + } + return json.substr(start, end - start); +} + +static std::string suppress_unwanted_text(const std::string& text) { + static const std::regex pattern(R"(\([^)]*\)|\[[^\]]*\]|\.\.\.)"); + std::string result = std::regex_replace(text, pattern, ""); + + size_t start = result.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) return ""; + size_t end = result.find_last_not_of(" \t\n\r"); + return result.substr(start, end - start + 1); +} + +static bool is_pcm_chunk_silent(const uint8_t* pcm_buffer, size_t pcm_buffer_size, float rms_threshold) { + if (!pcm_buffer || pcm_buffer_size < sizeof(int16_t)) return true; + + const size_t sample_count = pcm_buffer_size / sizeof(int16_t); + const auto* samples = reinterpret_cast(pcm_buffer); + double sum_sq = 0.0; + for (size_t i = 0; i < sample_count; ++i) { + const double normalized = static_cast(samples[i]) / 32768.0; + sum_sq += normalized * normalized; + } + + const double rms = std::sqrt(sum_sq / static_cast(sample_count)); + return rms < static_cast(rms_threshold); +} + +static void parse_stream_transcribe_init_options(const std::string& json, + double& confirmation_threshold, + size_t& min_chunk_size, + bool& telemetry_enabled, + std::string& language) { + confirmation_threshold = 0.99; + min_chunk_size = 32000; + telemetry_enabled = true; + language = "en"; + + if (json.empty()) return; + + size_t pos = json.find("\"confirmation_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + confirmation_threshold = std::stod(json.substr(pos)); + } + + pos = json.find("\"min_chunk_size\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + min_chunk_size = static_cast(std::stod(json.substr(pos))); + } + + pos = json.find("\"telemetry_enabled\""); + if (pos != std::string::npos) { + telemetry_enabled = json_bool(json, "telemetry_enabled"); + } + + language = json_string(json, "language"); + if (language.empty()) language = "en"; +} + +static bool build_parakeet_features_from_pcm( + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + const cactus::engine::Model* model, + std::vector& out_features) { + if (!pcm_buffer || pcm_buffer_size < sizeof(int16_t) || !model) { + return false; + } + + std::vector waveform = cactus::audio::pcm_buffer_to_float_samples( + pcm_buffer, pcm_buffer_size); + if (waveform.empty()) { + return false; + } + + const auto cfg = get_parakeet_spectrogram_config(); + const size_t mel_bins = std::max( + 1, static_cast(model->get_config().num_mel_bins)); + cactus::engine::AudioProcessor processor; + processor.init_mel_filters( + cfg.n_fft / 2 + 1, + mel_bins, + 0.0f, + 8000.0f, + WHISPER_SAMPLE_RATE); + + const size_t waveform_samples = waveform.size(); + apply_preemphasis(waveform, 0.97f); + out_features = processor.compute_spectrogram(waveform, cfg); + normalize_parakeet_log_mel(out_features, mel_bins); + size_t valid_frames = waveform_samples / cfg.hop_length; + if (valid_frames == 0) valid_frames = 1; + trim_mel_frames(out_features, mel_bins, valid_frames); + return !out_features.empty(); +} + +static size_t parakeet_tdt_pcm_bytes_to_encoder_frames( + const cactus::engine::Model* model, + size_t pcm_bytes) { + if (!model) return 0; + const auto cfg = get_parakeet_spectrogram_config(); + const double sample_count = static_cast(pcm_bytes) / 2.0; + const double feature_frames = sample_count / static_cast(cfg.hop_length); + const double subsampling = std::max(1, model->get_config().subsampling_factor); + return static_cast(std::llround(feature_frames / subsampling)); +} + +struct CactusStreamTranscribeHandle { + CactusModelHandle* model_handle; + + struct CactusStreamTranscribeOptions { + double confirmation_threshold; + size_t min_chunk_size; + std::string language; + } options; + std::string transcribe_options_json; + bool has_custom_vocabulary_bias = false; + std::vector custom_vocabulary; + + std::vector audio_buffer; + std::string previous_parakeet_pending; + size_t previous_parakeet_audio_buffer_size = 0; + size_t previous_parakeet_pending_ticks = 0; + size_t parakeet_silence_run_bytes = 0; + size_t parakeet_speech_run_bytes = 0; + bool parakeet_resume_guard_active = true; + std::string parakeet_committed_text; + size_t parakeet_chunk_cursor_bytes = 0; + float parakeet_committed_until_sec = 0.0f; + bool parakeet_onset_active = false; + size_t parakeet_onset_start_bytes = 0; + bool parakeet_tdt_chunked_stream = false; + struct ParakeetTdtDecodeContext { + std::vector audio_samples; + size_t samples_decoded_up_to = 0; + float audio_time_offset_sec = 0.0f; + cactus::engine::ParakeetTDTModel::ChunkStreamState decoder_state; + cactus::engine::AudioProcessor audio_processor; + size_t mel_bins = 0; + bool initialized = false; + } parakeet_tdt_decode_context; + bool parakeet_tdt_chunked_blocked_until_pause = false; + cactus::engine::ParakeetTDTModel::ChunkStreamState parakeet_tdt_decoder_state; + std::string parakeet_tdt_pending_raw; + struct ParakeetTdtResumeCheckpoint { + size_t end_bytes = 0; + cactus::engine::ParakeetTDTModel::ChunkStreamState state; + }; + std::deque parakeet_tdt_resume_checkpoints; + + std::vector previous_segments; + bool previous_cloud_handoff = false; + float stream_audio_offset_sec = 0.0f; + uint64_t next_cloud_job_id = 1; + + struct CloudJob { + uint64_t id; + std::future result; + }; + std::vector pending_cloud_jobs; + std::vector> completed_cloud_results; + + char transcribe_response_buffer[65536]; + + std::chrono::steady_clock::time_point stream_start; + bool stream_first_token_seen; + double stream_first_token_ms; + int stream_total_tokens; + + std::chrono::steady_clock::time_point stream_session_start; + bool stream_session_first_token_seen; + double stream_session_first_token_ms; + int stream_cumulative_tokens; + double last_stream_decode_tps = 0.0; + double last_stream_decode_tokens = 0.0; + double last_stream_total_time_ms = 0.0; + double last_stream_raw_decoder_tps = 0.0; +}; + +static std::vector parse_segments(const std::string& transcribe_json); + +struct StreamWindowDecodeResult { + std::string raw_json; + std::string response; + std::vector segments; + double decode_tokens = 0.0; + bool cloud_handoff = false; +}; + +static double pcm_bytes_to_seconds(size_t pcm_bytes) { + return static_cast(pcm_bytes) / (16000.0 * 2.0); +} + +static size_t seconds_to_pcm_bytes(double seconds) { + if (seconds <= 0.0) return 0; + const size_t raw = static_cast(std::llround(seconds * 16000.0 * 2.0)); + return raw - (raw % sizeof(int16_t)); +} + +static std::string join_segments_by_end_time( + const std::vector& segments, + float offset_sec, + float min_end_exclusive_sec, + float max_end_inclusive_sec, + float* out_max_emitted_end_sec = nullptr) { + std::string joined; + float max_end_sec = min_end_exclusive_sec; + for (const auto& seg : segments) { + const float abs_end_sec = offset_sec + seg.end; + if (abs_end_sec <= min_end_exclusive_sec || abs_end_sec > max_end_inclusive_sec) { + continue; + } + if (!joined.empty()) joined += ' '; + joined += seg.text; + if (abs_end_sec > max_end_sec) { + max_end_sec = abs_end_sec; + } + } + + if (out_max_emitted_end_sec) { + *out_max_emitted_end_sec = max_end_sec; + } + return suppress_unwanted_text(joined); +} + +static bool run_stream_window_transcribe( + CactusStreamTranscribeHandle* handle, + const char* transcribe_prompt, + const std::string& effective_transcribe_options_json, + const uint8_t* decode_pcm, + size_t decode_pcm_size, + StreamWindowDecodeResult& out) { + if (!handle || !handle->model_handle || !decode_pcm || decode_pcm_size == 0) { + return false; + } + + const auto model_type = handle->model_handle->model->get_config().model_type; + const bool use_model_stream_mode = + model_type != cactus::engine::Config::ModelType::PARAKEET_TDT; + cactus::telemetry::setStreamMode(use_model_stream_mode); + const int result = cactus_transcribe( + handle->model_handle, + nullptr, + transcribe_prompt, + handle->transcribe_response_buffer, + sizeof(handle->transcribe_response_buffer), + effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), + nullptr, + nullptr, + decode_pcm, + decode_pcm_size); + cactus::telemetry::setStreamMode(false); + + if (result < 0) { + return false; + } + + out.raw_json.assign(handle->transcribe_response_buffer); + out.response = suppress_unwanted_text(json_string(out.raw_json, "response")); + out.segments = parse_segments(out.raw_json); + out.decode_tokens = std::max(0.0, json_number(out.raw_json, "decode_tokens")); + out.cloud_handoff = json_bool(out.raw_json, "cloud_handoff"); + handle->last_stream_decode_tps = std::max(0.0, json_number(out.raw_json, "decode_tps")); + handle->last_stream_decode_tokens = out.decode_tokens; + handle->last_stream_total_time_ms = std::max(0.0, json_number(out.raw_json, "total_time_ms")); + handle->last_stream_raw_decoder_tps = std::max(0.0, json_number(out.raw_json, "raw_decoder_tps")); + return true; +} + +static bool run_parakeet_tdt_chunk_decode( + CactusStreamTranscribeHandle* handle, + size_t window_start_bytes, + size_t window_end_bytes, + size_t chunk_start_bytes, + size_t chunk_end_bytes, + cactus::engine::ParakeetTDTModel::ChunkStreamResult& out, + cactus::engine::ParakeetTDTModel::ChunkStreamState& out_state) { + if (!handle || !handle->model_handle || !handle->model_handle->model) { + return false; + } + if (chunk_end_bytes <= chunk_start_bytes || + window_end_bytes <= window_start_bytes || + window_end_bytes > handle->audio_buffer.size() || + chunk_end_bytes > window_end_bytes || + window_start_bytes > chunk_start_bytes) { + return false; + } + + auto* model = static_cast( + handle->model_handle->model.get()); + size_t replay_start_bytes = window_start_bytes; + cactus::engine::ParakeetTDTModel::ChunkStreamState base_state; + bool found_checkpoint = false; + for (auto it = handle->parakeet_tdt_resume_checkpoints.rbegin(); + it != handle->parakeet_tdt_resume_checkpoints.rend(); + ++it) { + if (it->end_bytes <= window_start_bytes) { + replay_start_bytes = std::max(window_start_bytes, it->end_bytes); + base_state = it->state; + found_checkpoint = true; + break; + } + } + if (!found_checkpoint) { + for (auto it = handle->parakeet_tdt_resume_checkpoints.rbegin(); + it != handle->parakeet_tdt_resume_checkpoints.rend(); + ++it) { + if (it->end_bytes <= chunk_start_bytes) { + replay_start_bytes = std::max(window_start_bytes, it->end_bytes); + base_state = it->state; + found_checkpoint = true; + break; + } + } + } + if (!found_checkpoint) { + replay_start_bytes = window_start_bytes; + base_state = {}; + } + + std::vector audio_features; + if (!build_parakeet_features_from_pcm( + handle->audio_buffer.data() + window_start_bytes, + window_end_bytes - window_start_bytes, + model, + audio_features)) { + return false; + } + + const size_t replay_start_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( + model, replay_start_bytes - window_start_bytes); + const size_t start_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( + model, chunk_start_bytes - window_start_bytes); + size_t end_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( + model, chunk_end_bytes - window_start_bytes); + if (end_frame <= start_frame) { + end_frame = start_frame + 1; + } + + const auto decode_start = std::chrono::steady_clock::now(); + std::lock_guard lock(handle->model_handle->model_mutex); + cactus::telemetry::setStreamMode(true); + try { + out_state = base_state; + out = model->decode_chunk_stream( + audio_features, + replay_start_frame, + start_frame, + end_frame, + out_state); + } catch (...) { + cactus::telemetry::setStreamMode(false); + throw; + } + cactus::telemetry::setStreamMode(false); + const auto decode_end = std::chrono::steady_clock::now(); + const double decode_ms = std::chrono::duration_cast( + decode_end - decode_start).count() / 1000.0; + handle->last_stream_total_time_ms = decode_ms; + handle->last_stream_decode_tokens = static_cast(out.token_count); + handle->last_stream_decode_tps = + (out.token_count > 1 && decode_ms > 0.0) + ? ((static_cast(out.token_count) - 1.0) * 1000.0) / decode_ms + : 0.0; + handle->last_stream_raw_decoder_tps = out.raw_decoder_tps; + return true; +} + +static void reset_parakeet_tdt_chunk_stream(CactusStreamTranscribeHandle* handle) { + if (!handle) return; + handle->parakeet_tdt_decoder_state = {}; + handle->parakeet_tdt_pending_raw.clear(); + handle->parakeet_tdt_resume_checkpoints.clear(); +} + +static std::vector> collect_parakeet_word_spans(const std::string& text); + +static std::string strip_unwanted_text_preserving_edges(const std::string& text) { + static const std::regex pattern(R"(\([^)]*\)|\[[^\]]*\]|\.\.\.)"); + return std::regex_replace(text, pattern, ""); +} + +static void parakeet_append_monotonic_text(std::string& dst, const std::string& piece) { + const std::string normalized = suppress_unwanted_text(piece); + if (normalized.empty()) return; + if (dst.empty()) { + dst = normalized; + return; + } + + const unsigned char first = static_cast(normalized.front()); + const bool attach_without_space = + std::ispunct(first) && normalized.front() != '"' && + normalized.front() != '\'' && normalized.front() != '('; + if (!std::isspace(static_cast(dst.back())) && !attach_without_space) { + dst += " "; + } + dst += normalized; +} + +static std::vector parse_segments(const std::string& transcribe_json) { + std::vector segs; + std::string arr = json_array(transcribe_json, "segments"); + size_t i = 1; + while (i < arr.size() - 1) { + while (i < arr.size() && (arr[i] == ',' || std::isspace((unsigned char)arr[i]))) ++i; + if (i >= arr.size() - 1 || arr[i] != '{') break; + int depth = 1; + size_t obj_start = i++; + while (i < arr.size() && depth > 0) { + if (arr[i] == '{') ++depth; + else if (arr[i] == '}') --depth; + ++i; + } + std::string obj = arr.substr(obj_start, i - obj_start); + segs.emplace_back( + static_cast(json_number(obj, "start")), + static_cast(json_number(obj, "end")), + json_string(obj, "text") + ); + } + return segs; +} + +static bool is_parakeet_word_char(unsigned char ch) { + return std::isalnum(ch) || ch == '\''; +} + +static std::vector> collect_parakeet_word_spans(const std::string& text) { + std::vector> spans; + bool in_word = false; + size_t word_start = 0; + + for (size_t i = 0; i < text.size(); ++i) { + const unsigned char ch = static_cast(text[i]); + if (is_parakeet_word_char(ch)) { + if (!in_word) { + in_word = true; + word_start = i; + } + } else if (in_word) { + spans.emplace_back(word_start, i); + in_word = false; + } + } + + if (in_word) { + spans.emplace_back(word_start, text.size()); + } + return spans; +} + +static std::string parakeet_word_from_span( + const std::string& text, + const std::pair& span) { + if (span.second <= span.first || span.second > text.size()) return ""; + return text.substr(span.first, span.second - span.first); +} + +static bool parakeet_words_fuzzy_equal(const std::string& a, const std::string& b) { + if (a.empty() || b.empty()) return false; + const size_t dist = levenshtein_ci(a, b); + if (dist == 0) return true; + const size_t max_len = std::max(a.size(), b.size()); + return max_len >= 5 && dist <= 1; +} + +static size_t parakeet_common_prefix_words(const std::string& left, const std::string& right) { + const auto left_spans = collect_parakeet_word_spans(left); + const auto right_spans = collect_parakeet_word_spans(right); + const size_t max_words = std::min(left_spans.size(), right_spans.size()); + size_t common = 0; + for (; common < max_words; ++common) { + const std::string left_word = parakeet_word_from_span(left, left_spans[common]); + const std::string right_word = parakeet_word_from_span(right, right_spans[common]); + if (!parakeet_words_fuzzy_equal(left_word, right_word)) { + break; + } + } + return common; +} + +static std::string parakeet_take_first_words(const std::string& text, size_t words) { + if (text.empty() || words == 0) return ""; + const auto spans = collect_parakeet_word_spans(text); + if (spans.empty()) return ""; + if (words >= spans.size()) return suppress_unwanted_text(text); + const size_t end_idx = spans[words].first; + return suppress_unwanted_text(text.substr(0, end_idx)); +} + +static std::string parakeet_drop_first_words(const std::string& text, size_t words) { + if (text.empty()) return ""; + const auto spans = collect_parakeet_word_spans(text); + if (spans.empty()) return suppress_unwanted_text(text); + if (words >= spans.size()) return ""; + const size_t start_idx = spans[words].first; + return suppress_unwanted_text(text.substr(start_idx)); +} + +static size_t parakeet_word_count(const std::string& text) { + return collect_parakeet_word_spans(text).size(); +} + +static size_t parakeet_overlap_suffix_prefix_words( + const std::string& left, + const std::string& right, + size_t max_words = 64) { + const auto left_spans = collect_parakeet_word_spans(left); + const auto right_spans = collect_parakeet_word_spans(right); + if (left_spans.empty() || right_spans.empty()) return 0; + + const size_t max_overlap = std::min({left_spans.size(), right_spans.size(), max_words}); + size_t best = 0; + for (size_t k = 1; k <= max_overlap; ++k) { + bool matched = true; + for (size_t i = 0; i < k; ++i) { + const auto& left_span = left_spans[left_spans.size() - k + i]; + const auto& right_span = right_spans[i]; + const std::string left_word = parakeet_word_from_span(left, left_span); + const std::string right_word = parakeet_word_from_span(right, right_span); + if (!parakeet_words_fuzzy_equal(left_word, right_word)) { + matched = false; + break; + } + } + if (matched) best = k; + } + return best; +} + +static std::string parakeet_strip_recent_aligned_overlap( + const std::string& committed_text, + const std::string& hypothesis, + size_t max_recent_words = 24) { + const std::string candidate = suppress_unwanted_text(hypothesis); + if (candidate.empty() || committed_text.empty()) return candidate; + + const auto committed_spans = collect_parakeet_word_spans(committed_text); + if (committed_spans.empty()) return candidate; + + const size_t recent_words = std::min(committed_spans.size(), max_recent_words); + const size_t recent_start = committed_spans[committed_spans.size() - recent_words].first; + const std::string recent_tail = suppress_unwanted_text(committed_text.substr(recent_start)); + const auto recent_spans = collect_parakeet_word_spans(recent_tail); + const auto candidate_spans = collect_parakeet_word_spans(candidate); + const size_t recent_count = recent_spans.size(); + const size_t candidate_count = candidate_spans.size(); + if (recent_count == 0 || candidate_count == 0) return candidate; + + size_t best_run = 0; + size_t best_candidate_start = candidate_count; + size_t best_candidate_end = 0; + size_t best_recent_start = recent_count; + + for (size_t i = 0; i < recent_count; ++i) { + for (size_t j = 0; j < candidate_count; ++j) { + size_t run = 0; + while (i + run < recent_count && j + run < candidate_count) { + const std::string recent_word = parakeet_word_from_span( + recent_tail, recent_spans[i + run]); + const std::string candidate_word = parakeet_word_from_span( + candidate, candidate_spans[j + run]); + if (!parakeet_words_fuzzy_equal(recent_word, candidate_word)) { + break; + } + ++run; + } + + const size_t candidate_end = j + run; + const bool better_run = run > best_run; + const bool better_end = run == best_run && candidate_end > best_candidate_end; + const bool better_start = run == best_run && + candidate_end == best_candidate_end && + j < best_candidate_start; + if (better_run || better_end || better_start) { + best_run = run; + best_candidate_start = j; + best_candidate_end = candidate_end; + best_recent_start = i; + } + } + } + + if (best_run == 0 || best_candidate_end == 0) return candidate; + + const size_t min_overlap_words = candidate_count <= 5 ? 2 : 3; + if (best_run < min_overlap_words) return candidate; + + if (best_candidate_start > 1) return candidate; + if (best_candidate_start == 1) { + const std::string lead_word = parakeet_word_from_span(candidate, candidate_spans[0]); + const std::string overlap_word = parakeet_word_from_span(candidate, candidate_spans[1]); + const std::string recent_word = parakeet_word_from_span( + recent_tail, recent_spans[best_recent_start]); + if (!parakeet_words_fuzzy_equal(lead_word, overlap_word) && + !parakeet_words_fuzzy_equal(lead_word, recent_word)) { + return candidate; + } + } + + return parakeet_drop_first_words(candidate, best_candidate_end); +} + +static std::string parakeet_strip_recent_committed_prefix( + const std::string& committed_text, + const std::string& hypothesis); + +static std::string parakeet_emit_delta( + const std::string& committed_text, + const std::string& newly_confirmed) { + std::string candidate = parakeet_strip_recent_committed_prefix( + committed_text, newly_confirmed); + if (candidate.empty()) return ""; + if (committed_text.empty()) return candidate; + + const size_t overlap_words = parakeet_overlap_suffix_prefix_words(committed_text, candidate); + if (overlap_words > 0) { + return parakeet_drop_first_words(candidate, overlap_words); + } + + return candidate; +} + +static std::string parakeet_strip_recent_committed_prefix( + const std::string& committed_text, + const std::string& hypothesis) { + const std::string candidate = suppress_unwanted_text(hypothesis); + if (candidate.empty() || committed_text.empty()) return candidate; + + const auto committed_spans = collect_parakeet_word_spans(committed_text); + if (committed_spans.empty()) return candidate; + + const size_t recent_words = std::min(committed_spans.size(), 24); + const size_t recent_start = committed_spans[committed_spans.size() - recent_words].first; + const std::string recent_tail = suppress_unwanted_text(committed_text.substr(recent_start)); + const size_t candidate_words = parakeet_word_count(candidate); + const size_t overlap_words = parakeet_overlap_suffix_prefix_words(recent_tail, candidate, recent_words); + + // Be conservative for tiny hypotheses, but strip even a single repeated + // boundary word when the new hypothesis is a longer continuation. + const size_t required_overlap = candidate_words >= 5 ? 1 : 2; + if (overlap_words >= required_overlap) { + return parakeet_drop_first_words(candidate, overlap_words); + } + + const std::string aligned_trimmed = parakeet_strip_recent_aligned_overlap( + committed_text, candidate, recent_words); + if (aligned_trimmed != candidate) { + return aligned_trimmed; + } + + const auto recent_spans = collect_parakeet_word_spans(recent_tail); + const auto candidate_spans = collect_parakeet_word_spans(candidate); + const size_t recent_count = recent_spans.size(); + const size_t candidate_count = candidate_spans.size(); + if (recent_count == 0 || candidate_count < 4) return candidate; + + std::vector> dp( + recent_count + 1, std::vector(candidate_count + 1, 0)); + for (size_t i = 1; i <= recent_count; ++i) { + for (size_t j = 1; j <= candidate_count; ++j) { + const std::string recent_word = parakeet_word_from_span(recent_tail, recent_spans[i - 1]); + const std::string candidate_word = parakeet_word_from_span(candidate, candidate_spans[j - 1]); + if (parakeet_words_fuzzy_equal(recent_word, candidate_word)) { + dp[i][j] = dp[i - 1][j - 1] + 1; + } else { + dp[i][j] = std::max(dp[i - 1][j], dp[i][j - 1]); + } + } + } + + const size_t lcs_words = dp[recent_count][candidate_count]; + if (lcs_words < 3 || lcs_words * 2 < candidate_count) return candidate; + + size_t i = recent_count; + size_t j = candidate_count; + size_t last_matched_candidate_word = 0; + bool found_match = false; + bool captured_last_match = false; + while (i > 0 && j > 0) { + const std::string recent_word = parakeet_word_from_span(recent_tail, recent_spans[i - 1]); + const std::string candidate_word = parakeet_word_from_span(candidate, candidate_spans[j - 1]); + if (parakeet_words_fuzzy_equal(recent_word, candidate_word)) { + if (!captured_last_match) { + last_matched_candidate_word = j; + captured_last_match = true; + } + found_match = true; + --i; + --j; + } else if (dp[i - 1][j] >= dp[i][j - 1]) { + --i; + } else { + --j; + } + } + + if (!found_match || last_matched_candidate_word >= candidate_count) return candidate; + return parakeet_drop_first_words(candidate, last_matched_candidate_word); + + return candidate; +} + +static std::string serialize_segments_with_offset( + const std::vector& segs, float offset_sec) { + std::ostringstream out; + out << "["; + for (size_t i = 0; i < segs.size(); ++i) { + if (i > 0) out << ","; + out << std::fixed << std::setprecision(3) + << "{\"start\":" << (segs[i].start + offset_sec) + << ",\"end\":" << (segs[i].end + offset_sec) + << ",\"text\":\"" << escape_json(segs[i].text) << "\"}"; + } + out << "]"; + return out.str(); +} + +static std::string build_stream_response( + const std::string& raw_json_str, + const std::string& error_msg, + const std::string& confirmed, + const std::string& pending, + const std::string& segments_json, + bool cloud_handoff, + double buffer_duration_ms, + uint64_t cloud_job_id, + uint64_t cloud_result_job_id, + const CloudResponse& cloud_result, + double override_time_to_first_token_ms = -1.0, + double override_total_time_ms = -1.0, + double override_decode_tps = -1.0, + double override_decode_tokens = -1.0, + double override_total_tokens = -1.0, + double override_raw_decoder_tps = -1.0 +) { + std::string function_calls = json_array(raw_json_str, "function_calls"); + double confidence = json_number(raw_json_str, "confidence"); + double time_to_first_token_ms = json_number(raw_json_str, "time_to_first_token_ms"); + double total_time_ms = json_number(raw_json_str, "total_time_ms"); + double prefill_tps = json_number(raw_json_str, "prefill_tps"); + double decode_tps = json_number(raw_json_str, "decode_tps"); + double ram_usage_mb = json_number(raw_json_str, "ram_usage_mb"); + double prefill_tokens = json_number(raw_json_str, "prefill_tokens"); + double decode_tokens = json_number(raw_json_str, "decode_tokens"); + double total_tokens = json_number(raw_json_str, "total_tokens"); + double raw_decoder_tps = json_number(raw_json_str, "raw_decoder_tps"); + if (override_time_to_first_token_ms >= 0.0) { + time_to_first_token_ms = override_time_to_first_token_ms; + } + if (override_total_time_ms >= 0.0) { + total_time_ms = override_total_time_ms; + } + if (override_decode_tps >= 0.0) { + decode_tps = override_decode_tps; + } + if (override_decode_tokens >= 0.0) { + decode_tokens = override_decode_tokens; + } + if (override_total_tokens >= 0.0) { + total_tokens = override_total_tokens; + } + if (override_raw_decoder_tps >= 0.0) { + raw_decoder_tps = override_raw_decoder_tps; + } + std::string effective_confirmed = confirmed; + if (cloud_result.used_cloud && !cloud_result.transcript.empty()) { + effective_confirmed = cloud_result.transcript; + } + + std::ostringstream json_builder; + json_builder << "{"; + json_builder << "\"success\":true,"; + json_builder << "\"buffer_duration_ms\":" << buffer_duration_ms << ","; + json_builder << "\"error\":" << (error_msg.empty() ? "null" : "\"" + escape_json(error_msg) + "\"") << ","; + json_builder << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; + json_builder << "\"cloud_job_id\":" << cloud_job_id << ","; + json_builder << "\"cloud_result_job_id\":" << cloud_result_job_id << ","; + json_builder << "\"cloud_result\":\"" << escape_json(cloud_result.transcript) << "\","; + json_builder << "\"cloud_result_used_cloud\":" << (cloud_result.used_cloud ? "true" : "false") << ","; + json_builder << "\"cloud_result_error\":"; + if (cloud_result.error.empty()) { + json_builder << "null,"; + } else { + json_builder << "\"" << escape_json(cloud_result.error) << "\","; + } + json_builder << "\"cloud_result_source\":\"" << (cloud_result.used_cloud ? "cloud" : "fallback") << "\","; + json_builder << "\"confirmed_local\":\"" << escape_json(confirmed) << "\","; + json_builder << "\"confirmed\":\"" << escape_json(effective_confirmed) << "\","; + json_builder << "\"pending\":\"" << escape_json(pending) << "\","; + json_builder << "\"segments\":" << segments_json << ","; + json_builder << "\"function_calls\":" << function_calls << ","; + json_builder << "\"confidence\":" << confidence << ","; + json_builder << "\"time_to_first_token_ms\":" << time_to_first_token_ms << ","; + json_builder << "\"total_time_ms\":" << total_time_ms << ","; + json_builder << "\"prefill_tps\":" << prefill_tps << ","; + json_builder << "\"decode_tps\":" << decode_tps << ","; + json_builder << "\"ram_usage_mb\":" << ram_usage_mb << ","; + json_builder << "\"prefill_tokens\":" << prefill_tokens << ","; + json_builder << "\"decode_tokens\":" << decode_tokens << ","; + json_builder << "\"total_tokens\":" << total_tokens << ","; + json_builder << "\"raw_decoder_tps\":" << raw_decoder_tps; + json_builder << "}"; + return json_builder.str(); +} + +extern "C" { + +cactus_stream_transcribe_t cactus_stream_transcribe_start(cactus_model_t model, const char* options_json) { + if (!model) { + last_error_message = "Model not initialized. Check model path and files."; + CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); + return nullptr; + } + + try { + auto* model_handle = static_cast(model); + if (!model_handle->model) { + last_error_message = "Invalid model handle."; + CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); + return nullptr; + } + + auto* stream_handle = new CactusStreamTranscribeHandle(); + stream_handle->model_handle = model_handle; + stream_handle->transcribe_response_buffer[0] = '\0'; + + auto session_start_time = std::chrono::steady_clock::now(); + stream_handle->stream_start = session_start_time; + stream_handle->stream_first_token_seen = false; + stream_handle->stream_first_token_ms = 0.0; + stream_handle->stream_total_tokens = 0; + + stream_handle->stream_session_start = session_start_time; + stream_handle->stream_session_first_token_seen = false; + stream_handle->stream_session_first_token_ms = 0.0; + stream_handle->stream_cumulative_tokens = 0; + + double confirmation_threshold; + size_t min_chunk_size; + bool telemetry_enabled; + std::string language; + parse_stream_transcribe_init_options( + options_json ? options_json : "", + confirmation_threshold, + min_chunk_size, + telemetry_enabled, + language + ); + + stream_handle->options = { confirmation_threshold, min_chunk_size, language }; + stream_handle->transcribe_options_json = options_json ? options_json : ""; + stream_handle->parakeet_tdt_chunked_stream = + model_handle->model->get_config().model_type == + cactus::engine::Config::ModelType::PARAKEET_TDT; + { + float vocabulary_boost = 5.0f; + parse_custom_vocabulary_options(stream_handle->transcribe_options_json, + stream_handle->custom_vocabulary, vocabulary_boost); + auto vocab_bias = build_custom_vocabulary_bias( + model_handle->model->get_tokenizer(), + stream_handle->custom_vocabulary, + vocabulary_boost + ); + stream_handle->has_custom_vocabulary_bias = !vocab_bias.empty(); + if (stream_handle->has_custom_vocabulary_bias) { + model_handle->model->set_vocab_bias(vocab_bias); + } + } + + if (stream_handle->parakeet_tdt_chunked_stream) { + auto& ctx = stream_handle->parakeet_tdt_decode_context; + ctx.mel_bins = std::max( + 1, static_cast(model_handle->model->get_config().num_mel_bins)); + auto cfg = get_parakeet_spectrogram_config(); + ctx.audio_processor.init_mel_filters( + cfg.n_fft / 2 + 1, + ctx.mel_bins, + 0.0f, + 8000.0f, + WHISPER_SAMPLE_RATE); + ctx.initialized = true; + } + + CACTUS_LOG_INFO("stream_transcribe_start", + "Stream transcription initialized for model: " << model_handle->model_name); + + return stream_handle; + } catch (const std::exception& e) { + last_error_message = "Exception during stream_transcribe_start: " + std::string(e.what()); + CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); + return nullptr; + } catch (...) { + last_error_message = "Unknown exception during stream transcription initialization"; + CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); + return nullptr; + } +} + +int cactus_stream_transcribe_process( + cactus_stream_transcribe_t stream, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + char* response_buffer, + size_t buffer_size +) { + if (!stream) { + last_error_message = "Stream not initialized."; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); + return -1; + } + + if (!pcm_buffer || pcm_buffer_size == 0) { + last_error_message = "Invalid parameters: pcm_buffer or pcm_buffer_size"; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); + return -1; + } + + if (!response_buffer || buffer_size == 0) { + last_error_message = "Invalid parameters: response_buffer or buffer_size"; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); + return -1; + } + + try { + auto* handle = static_cast(stream); + const auto model_type = handle->model_handle->model->get_config().model_type; + bool is_moonshine = model_type == cactus::engine::Config::ModelType::MOONSHINE; + bool is_parakeet_tdt = + model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; + bool is_parakeet = + model_type == cactus::engine::Config::ModelType::PARAKEET || + is_parakeet_tdt; + bool is_gemma4 = model_type == cactus::engine::Config::ModelType::GEMMA4; + + if (is_parakeet_tdt && + handle->parakeet_tdt_chunked_stream && + handle->parakeet_tdt_decode_context.initialized) { + auto& ctx = handle->parakeet_tdt_decode_context; + auto* tdt_model = static_cast( + handle->model_handle->model.get()); + auto new_samples = cactus::audio::pcm_buffer_to_float_samples( + pcm_buffer, pcm_buffer_size); + ctx.audio_samples.insert(ctx.audio_samples.end(), new_samples.begin(), new_samples.end()); + + constexpr size_t kTdtLeftContextSamples = 8 * 16000; + constexpr size_t kTdtRightContextSamples = 16000; + constexpr size_t kTdtChunkSamples = 16000; + constexpr size_t kTdtColdStartSamples = 6 * 16000; + + const size_t total_samples = ctx.audio_samples.size(); + const size_t decodable_up_to = total_samples > kTdtRightContextSamples + ? total_samples - kTdtRightContextSamples + : 0; + const size_t min_chunk_samples = ctx.samples_decoded_up_to == 0 + ? kTdtColdStartSamples + : kTdtChunkSamples; + + if (decodable_up_to <= ctx.samples_decoded_up_to || + decodable_up_to - ctx.samples_decoded_up_to < min_chunk_samples) { + const std::string json_response = build_stream_response( + "{}", + "", + "", + handle->previous_parakeet_pending, + "[]", + false, + 0.0, + 0, + 0, + CloudResponse{}, + handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, + handle->last_stream_total_time_ms, + handle->last_stream_decode_tps, + handle->last_stream_decode_tokens, + handle->last_stream_decode_tokens, + handle->last_stream_raw_decoder_tps); + if (json_response.length() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + std::strcpy(response_buffer, json_response.c_str()); + return static_cast(json_response.length()); + } + + const size_t window_start_sample = ctx.samples_decoded_up_to > kTdtLeftContextSamples + ? ctx.samples_decoded_up_to - kTdtLeftContextSamples + : 0; + const size_t window_end_sample = std::min( + total_samples, decodable_up_to + kTdtRightContextSamples); + std::vector window_audio( + ctx.audio_samples.begin() + window_start_sample, + ctx.audio_samples.begin() + window_end_sample); + + auto cfg = get_parakeet_spectrogram_config(); + const size_t waveform_samples = window_audio.size(); + apply_preemphasis(window_audio, 0.97f); + window_audio = ctx.audio_processor.compute_spectrogram(window_audio, cfg); + normalize_parakeet_log_mel(window_audio, ctx.mel_bins); + size_t valid_frames = waveform_samples / cfg.hop_length; + if (valid_frames == 0) valid_frames = 1; + trim_mel_frames(window_audio, ctx.mel_bins, valid_frames); + + const uint32_t subsampling = + std::max(1, handle->model_handle->model->get_config().subsampling_factor); + const size_t samples_per_enc_frame = cfg.hop_length * subsampling; + const size_t context_samples = ctx.samples_decoded_up_to - window_start_sample; + const size_t decode_start_frame = context_samples / samples_per_enc_frame; + const size_t new_samples_count = decodable_up_to - ctx.samples_decoded_up_to; + const size_t decode_end_frame = + decode_start_frame + (new_samples_count / samples_per_enc_frame); + + if (decode_end_frame <= decode_start_frame) { + const std::string json_response = build_stream_response( + "{}", + "", + "", + handle->previous_parakeet_pending, + "[]", + false, + 0.0, + 0, + 0, + CloudResponse{}, + handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, + handle->last_stream_total_time_ms, + handle->last_stream_decode_tps, + handle->last_stream_decode_tokens, + handle->last_stream_decode_tokens, + handle->last_stream_raw_decoder_tps); + if (json_response.length() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + std::strcpy(response_buffer, json_response.c_str()); + return static_cast(json_response.length()); + } + + cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; + { + const auto decode_start = std::chrono::steady_clock::now(); + std::lock_guard lock(handle->model_handle->model_mutex); + cactus::telemetry::setStreamMode(true); + try { + chunk_decode = tdt_model->decode_chunk_stream( + window_audio, + decode_start_frame, + decode_start_frame, + decode_end_frame, + ctx.decoder_state); + cactus_reset(handle->model_handle); + } catch (...) { + cactus::telemetry::setStreamMode(false); + throw; + } + cactus::telemetry::setStreamMode(false); + const auto decode_end = std::chrono::steady_clock::now(); + const double decode_ms = + std::chrono::duration_cast( + decode_end - decode_start).count() / 1000.0; + handle->last_stream_total_time_ms = decode_ms; + handle->last_stream_decode_tokens = static_cast(chunk_decode.token_count); + handle->last_stream_decode_tps = + (chunk_decode.token_count > 1 && decode_ms > 0.0) + ? ((static_cast(chunk_decode.token_count) - 1.0) * 1000.0) / decode_ms + : 0.0; + handle->last_stream_raw_decoder_tps = chunk_decode.raw_decoder_tps; + } + + const float window_offset_sec = ctx.audio_time_offset_sec + + static_cast(window_start_sample) / static_cast(WHISPER_SAMPLE_RATE); + std::vector confirmed_segments; + if (!chunk_decode.confirmed_text.empty()) { + confirmed_segments.emplace_back( + window_offset_sec + chunk_decode.start_sec, + window_offset_sec + chunk_decode.confirmed_end_sec, + suppress_unwanted_text(chunk_decode.confirmed_text)); + } + std::string confirmed_text = suppress_unwanted_text(chunk_decode.confirmed_text); + std::string pending_text = suppress_unwanted_text(chunk_decode.pending_text); + if (!confirmed_text.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(confirmed_text, handle->custom_vocabulary); + } + if (!pending_text.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(pending_text, handle->custom_vocabulary); + } + + if (!confirmed_text.empty()) { + if (chunk_decode.token_count > 0) { + handle->stream_total_tokens += static_cast(chunk_decode.token_count); + handle->stream_cumulative_tokens += static_cast(chunk_decode.token_count); + } + if (!handle->stream_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_first_token_ms = + std::chrono::duration_cast( + now - handle->stream_start).count(); + handle->stream_first_token_seen = true; + } + if (!handle->stream_session_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_session_first_token_ms = + std::chrono::duration_cast( + now - handle->stream_session_start).count(); + handle->stream_session_first_token_seen = true; + } + parakeet_append_monotonic_text(handle->parakeet_committed_text, confirmed_text); + } + + handle->previous_parakeet_pending = pending_text; + handle->previous_parakeet_pending_ticks = pending_text.empty() ? 0 : 1; + + if (chunk_decode.confirmed_token_count > 0) { + const size_t confirmed_samples_in_window = static_cast( + chunk_decode.confirmed_end_sec * WHISPER_SAMPLE_RATE); + ctx.samples_decoded_up_to = window_start_sample + confirmed_samples_in_window; + } else if (chunk_decode.token_count == 0) { + ctx.samples_decoded_up_to = decodable_up_to; + } + + if (ctx.samples_decoded_up_to > kTdtLeftContextSamples * 2) { + const size_t trim_samples = ctx.samples_decoded_up_to - kTdtLeftContextSamples; + ctx.audio_samples.erase( + ctx.audio_samples.begin(), + ctx.audio_samples.begin() + trim_samples); + ctx.samples_decoded_up_to -= trim_samples; + ctx.audio_time_offset_sec += static_cast(trim_samples) / 16000.0f; + } + + handle->parakeet_committed_until_sec = std::max( + handle->parakeet_committed_until_sec, + window_offset_sec + + (chunk_decode.confirmed_token_count > 0 ? chunk_decode.confirmed_end_sec : 0.0f)); + + const std::string json_response = build_stream_response( + "{}", + "", + confirmed_text, + pending_text, + serialize_segments_with_offset(confirmed_segments, 0.0f), + false, + 0.0, + 0, + 0, + CloudResponse{}, + handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, + handle->last_stream_total_time_ms, + handle->last_stream_decode_tps, + handle->last_stream_decode_tokens, + handle->last_stream_decode_tokens, + handle->last_stream_raw_decoder_tps); + if (json_response.length() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + std::strcpy(response_buffer, json_response.c_str()); + return static_cast(json_response.length()); + } + + handle->audio_buffer.insert( + handle->audio_buffer.end(), + pcm_buffer, + pcm_buffer + pcm_buffer_size + ); + CACTUS_LOG_DEBUG("stream_transcribe_process", + "Inserted " << pcm_buffer_size << " bytes, buffer size: " << handle->audio_buffer.size()); + + if (handle->audio_buffer.size() < handle->options.min_chunk_size * sizeof(int16_t)) { + std::string json_response = "{\"success\":true,\"confirmed\":\"\",\"pending\":\"\"}"; + + if (json_response.length() >= buffer_size) { + last_error_message = "Response buffer too small"; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); + return -1; + } + + std::strcpy(response_buffer, json_response.c_str()); + return static_cast(json_response.length()); + } + + std::string whisper_prompt = "<|startoftranscript|><|" + handle->options.language + "|><|transcribe|><|notimestamps|>"; + const char* transcribe_prompt = + is_gemma4 ? "Transcribe the audio." : + (is_moonshine || is_parakeet) ? "" : + whisper_prompt.c_str(); + std::string effective_transcribe_options_json = handle->transcribe_options_json; + if (is_parakeet && !is_parakeet_tdt) { + effective_transcribe_options_json = ensure_json_bool_option( + effective_transcribe_options_json, "use_vad", false); + } + + if (is_parakeet_tdt) { + constexpr double kParakeetTdtLeftContextSec = 2.0; + constexpr double kParakeetTdtChunkSec = 0.25; + constexpr double kParakeetTdtRightContextSec = 1.0; + constexpr size_t kParakeetTdtMaxChunksPerProcess = 3; + + const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); + const size_t chunk_bytes = std::max( + seconds_to_pcm_bytes(kParakeetTdtChunkSec), + handle->options.min_chunk_size * sizeof(int16_t)); + const size_t right_context_bytes = seconds_to_pcm_bytes(kParakeetTdtRightContextSec); + const std::string tdt_vad_options_json = ensure_json_bool_option( + handle->transcribe_options_json, "use_vad", true); + const std::string tdt_no_vad_options_json = ensure_json_bool_option( + handle->transcribe_options_json, "use_vad", false); + + if (handle->parakeet_tdt_chunked_stream && + !handle->parakeet_tdt_chunked_blocked_until_pause && + !handle->parakeet_committed_text.empty()) { + const size_t encoder_frame_bytes = + get_parakeet_spectrogram_config().hop_length * + std::max(1, handle->model_handle->model->get_config().subsampling_factor) * + sizeof(int16_t); + std::string confirmed; + size_t confirmed_audio_bytes = 0; + size_t processed_chunks = 0; + bool chunked_stalled_on_speech = false; + + while (processed_chunks < kParakeetTdtMaxChunksPerProcess) { + const bool chunked_warmup = + handle->parakeet_committed_text.empty() && + handle->parakeet_tdt_pending_raw.empty(); + const size_t available_chunk_bytes = + handle->audio_buffer.size() > handle->parakeet_chunk_cursor_bytes + right_context_bytes + ? handle->audio_buffer.size() - handle->parakeet_chunk_cursor_bytes - right_context_bytes + : 0; + const size_t chunk_decode_bytes = std::max( + encoder_frame_bytes, + ((chunked_warmup + ? available_chunk_bytes + : seconds_to_pcm_bytes(0.96)) / + encoder_frame_bytes) * encoder_frame_bytes); + if (handle->parakeet_chunk_cursor_bytes + chunk_decode_bytes + right_context_bytes > + handle->audio_buffer.size()) { + break; + } + + const size_t chunk_start_bytes = handle->parakeet_chunk_cursor_bytes; + const size_t chunk_end_bytes = chunk_start_bytes + chunk_decode_bytes; + const size_t window_start_bytes = chunk_start_bytes > left_context_bytes + ? chunk_start_bytes - left_context_bytes + : 0; + const size_t window_end_bytes = chunk_end_bytes + right_context_bytes; + + cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; + cactus::engine::ParakeetTDTModel::ChunkStreamState chunk_state; + if (!run_parakeet_tdt_chunk_decode( + handle, + window_start_bytes, + window_end_bytes, + chunk_start_bytes, + chunk_end_bytes, + chunk_decode, + chunk_state)) { + last_error_message = "Chunked TDT decode failed."; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription( + handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, + false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, + last_error_message.c_str()); + return -1; + } + + const bool chunk_silent = is_pcm_chunk_silent( + handle->audio_buffer.data() + chunk_start_bytes, + chunk_end_bytes - chunk_start_bytes, + 0.012f); + const std::string raw_chunk_text = + strip_unwanted_text_preserving_edges(chunk_decode.text); + const std::string raw_confirmed_text = + strip_unwanted_text_preserving_edges(chunk_decode.confirmed_text); + const std::string raw_pending_text = + strip_unwanted_text_preserving_edges(chunk_decode.pending_text); + + if (raw_chunk_text.empty() && !chunk_silent) { + chunked_stalled_on_speech = true; + break; + } + + if (!raw_chunk_text.empty()) { + std::string confirmed_delta = suppress_unwanted_text(raw_confirmed_text); + std::string pending_raw = raw_pending_text; + if (!confirmed_delta.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(confirmed_delta, handle->custom_vocabulary); + } + handle->parakeet_tdt_pending_raw = pending_raw; + if (!confirmed_delta.empty()) { + if (chunk_decode.token_count > 0) { + handle->stream_total_tokens += static_cast(chunk_decode.token_count); + handle->stream_cumulative_tokens += static_cast(chunk_decode.token_count); + } + + if (!handle->stream_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_first_token_ms = + std::chrono::duration_cast( + now - handle->stream_start).count(); + handle->stream_first_token_seen = true; + } + if (!handle->stream_session_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_session_first_token_ms = + std::chrono::duration_cast( + now - handle->stream_session_start).count(); + handle->stream_session_first_token_seen = true; + } + + parakeet_append_monotonic_text( + handle->parakeet_committed_text, + confirmed_delta); + parakeet_append_monotonic_text(confirmed, confirmed_delta); + const size_t confirmed_boundary_bytes = std::min( + chunk_end_bytes, + std::max( + chunk_start_bytes, + window_start_bytes + seconds_to_pcm_bytes(chunk_decode.confirmed_end_sec))); + if (confirmed_boundary_bytes > chunk_start_bytes) { + confirmed_audio_bytes += confirmed_boundary_bytes - chunk_start_bytes; + } + } + } + + handle->parakeet_chunk_cursor_bytes = chunk_end_bytes; + handle->parakeet_tdt_decoder_state = chunk_state; + const size_t checkpoint_end_bytes = std::min( + chunk_end_bytes, + std::max( + window_start_bytes, + window_start_bytes + seconds_to_pcm_bytes(chunk_decode.resume_end_sec))); + while (!handle->parakeet_tdt_resume_checkpoints.empty() && + handle->parakeet_tdt_resume_checkpoints.back().end_bytes >= checkpoint_end_bytes) { + handle->parakeet_tdt_resume_checkpoints.pop_back(); + } + if (handle->parakeet_tdt_resume_checkpoints.empty() || + handle->parakeet_tdt_resume_checkpoints.back().end_bytes < checkpoint_end_bytes) { + handle->parakeet_tdt_resume_checkpoints.push_back({checkpoint_end_bytes, chunk_state}); + } + while (handle->parakeet_tdt_resume_checkpoints.size() > 16) { + handle->parakeet_tdt_resume_checkpoints.pop_front(); + } + handle->parakeet_committed_until_sec = std::max( + handle->parakeet_committed_until_sec, + static_cast( + handle->stream_audio_offset_sec + + pcm_bytes_to_seconds(checkpoint_end_bytes))); + ++processed_chunks; + } + + const bool fallback_to_legacy_tdt = + chunked_stalled_on_speech && + confirmed.empty() && + handle->parakeet_tdt_pending_raw.empty(); + if (fallback_to_legacy_tdt) { + const bool bootstrap_reset = + handle->parakeet_committed_text.empty() && + handle->previous_parakeet_pending.empty(); + reset_parakeet_tdt_chunk_stream(handle); + handle->parakeet_tdt_chunked_blocked_until_pause = true; + if (bootstrap_reset) { + handle->parakeet_chunk_cursor_bytes = 0; + } + } + + if (!fallback_to_legacy_tdt && + handle->parakeet_chunk_cursor_bytes > left_context_bytes) { + const size_t trim_bytes = + handle->parakeet_chunk_cursor_bytes - left_context_bytes; + handle->stream_audio_offset_sec += static_cast( + pcm_bytes_to_seconds(trim_bytes)); + handle->audio_buffer.erase( + handle->audio_buffer.begin(), + handle->audio_buffer.begin() + trim_bytes); + handle->parakeet_chunk_cursor_bytes -= trim_bytes; + while (!handle->parakeet_tdt_resume_checkpoints.empty() && + handle->parakeet_tdt_resume_checkpoints.front().end_bytes <= trim_bytes) { + handle->parakeet_tdt_resume_checkpoints.pop_front(); + } + for (auto& checkpoint : handle->parakeet_tdt_resume_checkpoints) { + checkpoint.end_bytes -= trim_bytes; + } + } + + if (!fallback_to_legacy_tdt) { + std::string pending_output; + std::string raw_json = "{}"; + if (!handle->parakeet_tdt_pending_raw.empty()) { + pending_output = suppress_unwanted_text(handle->parakeet_tdt_pending_raw); + if (!pending_output.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction( + pending_output, handle->custom_vocabulary); + } + } + if (handle->audio_buffer.size() > handle->parakeet_chunk_cursor_bytes) { + const size_t preview_start_bytes = handle->parakeet_chunk_cursor_bytes > left_context_bytes + ? handle->parakeet_chunk_cursor_bytes - left_context_bytes + : 0; + StreamWindowDecodeResult preview_decode; + if (run_stream_window_transcribe( + handle, + transcribe_prompt, + tdt_no_vad_options_json, + handle->audio_buffer.data() + preview_start_bytes, + handle->audio_buffer.size() - preview_start_bytes, + preview_decode)) { + raw_json = preview_decode.raw_json; + if (pending_output.empty()) { + pending_output = parakeet_strip_recent_committed_prefix( + handle->parakeet_committed_text, + preview_decode.response); + if (!pending_output.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction( + pending_output, handle->custom_vocabulary); + } + } + } + } + + handle->previous_parakeet_pending = pending_output; + handle->previous_parakeet_pending_ticks = pending_output.empty() ? 0 : 1; + handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); + handle->previous_segments.clear(); + handle->previous_cloud_handoff = false; + handle->parakeet_onset_active = false; + handle->parakeet_resume_guard_active = false; + + CloudResponse empty_cloud_result; + const std::string json_response = build_stream_response( + raw_json, + "", + confirmed, + pending_output, + "[]", + false, + pcm_bytes_to_seconds(confirmed_audio_bytes) * 1000.0, + 0, + 0, + empty_cloud_result, + handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, + raw_json == "{}" ? handle->last_stream_total_time_ms : -1.0, + raw_json == "{}" ? handle->last_stream_decode_tps : -1.0, + raw_json == "{}" ? handle->last_stream_decode_tokens : -1.0, + raw_json == "{}" ? handle->last_stream_decode_tokens : -1.0, + raw_json == "{}" ? handle->last_stream_raw_decoder_tps : -1.0); + if (json_response.length() >= buffer_size) { + last_error_message = "Response buffer too small"; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); + return -1; + } + + std::strcpy(response_buffer, json_response.c_str()); + return static_cast(json_response.length()); + } + } + + auto decode_window = [&](size_t window_start_bytes, + size_t window_end_bytes, + StreamWindowDecodeResult& out, + float& out_offset_sec) -> bool { + if (window_end_bytes <= window_start_bytes || window_end_bytes > handle->audio_buffer.size()) { + return false; + } + out_offset_sec = static_cast( + handle->stream_audio_offset_sec + pcm_bytes_to_seconds(window_start_bytes)); + StreamWindowDecodeResult vad_decode; + const bool vad_ok = run_stream_window_transcribe( + handle, + transcribe_prompt, + tdt_vad_options_json, + handle->audio_buffer.data() + window_start_bytes, + window_end_bytes - window_start_bytes, + vad_decode); + if (vad_ok && (!vad_decode.response.empty() || !vad_decode.segments.empty())) { + out = std::move(vad_decode); + return true; + } + + return run_stream_window_transcribe( + handle, + transcribe_prompt, + tdt_no_vad_options_json, + handle->audio_buffer.data() + window_start_bytes, + window_end_bytes - window_start_bytes, + out); + }; + + constexpr size_t kParakeetHoldbackWords = 1; + constexpr size_t kParakeetSilenceFlushBytes = 16000; // ~0.5s silence + constexpr size_t kParakeetStableFlushWords = 10; + constexpr size_t kParakeetStableFlushTicks = 2; + constexpr size_t kParakeetResumeGuardBytes = 24000; // ~0.75s speech + constexpr size_t kParakeetOnsetLeadSilenceBytes = 16000; // ~0.5s lead-in + constexpr size_t kParakeetOnsetMinConfirmWords = 6; + + const bool chunk_silent = is_pcm_chunk_silent(pcm_buffer, pcm_buffer_size, 0.012f); + const size_t prior_silence_run_bytes = handle->parakeet_silence_run_bytes; + const bool speech_resumed = + !chunk_silent && prior_silence_run_bytes >= kParakeetSilenceFlushBytes; + const size_t speech_chunk_start_bytes = + handle->audio_buffer.size() >= pcm_buffer_size + ? handle->audio_buffer.size() - pcm_buffer_size + : 0; + + if (chunk_silent) { + handle->parakeet_silence_run_bytes += pcm_buffer_size; + handle->parakeet_speech_run_bytes = 0; + if (handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes) { + handle->parakeet_resume_guard_active = true; + handle->parakeet_onset_active = false; + reset_parakeet_tdt_chunk_stream(handle); + const size_t retained_silence_bytes = std::min( + handle->parakeet_silence_run_bytes, kParakeetOnsetLeadSilenceBytes); + handle->parakeet_onset_start_bytes = + handle->audio_buffer.size() > retained_silence_bytes + ? handle->audio_buffer.size() - retained_silence_bytes + : 0; + } + } else { + if (!handle->parakeet_onset_active && + handle->parakeet_committed_text.empty() && + handle->previous_parakeet_pending.empty()) { + handle->parakeet_onset_active = true; + handle->parakeet_onset_start_bytes = 0; + } + if (speech_resumed) { + handle->parakeet_resume_guard_active = true; + handle->parakeet_speech_run_bytes = 0; + handle->parakeet_onset_active = true; + const size_t retained_silence_bytes = std::min( + prior_silence_run_bytes, kParakeetOnsetLeadSilenceBytes); + handle->parakeet_onset_start_bytes = + speech_chunk_start_bytes > retained_silence_bytes + ? speech_chunk_start_bytes - retained_silence_bytes + : 0; + } + handle->parakeet_silence_run_bytes = 0; + if (handle->parakeet_resume_guard_active) { + handle->parakeet_speech_run_bytes += pcm_buffer_size; + if (handle->parakeet_speech_run_bytes >= kParakeetResumeGuardBytes) { + handle->parakeet_resume_guard_active = false; + handle->parakeet_speech_run_bytes = 0; + } + } + } + + const bool detected_pause = + handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes; + const bool should_hold_cursor = handle->parakeet_onset_active; + + size_t processed_chunks = 0; + if (!should_hold_cursor) { + while (handle->parakeet_chunk_cursor_bytes + chunk_bytes + right_context_bytes <= + handle->audio_buffer.size() && + processed_chunks < kParakeetTdtMaxChunksPerProcess) { + handle->parakeet_chunk_cursor_bytes += chunk_bytes; + ++processed_chunks; + } + } + + if (!handle->parakeet_onset_active && + handle->parakeet_chunk_cursor_bytes > left_context_bytes) { + const size_t trim_bytes = + handle->parakeet_chunk_cursor_bytes - left_context_bytes; + handle->stream_audio_offset_sec += static_cast( + pcm_bytes_to_seconds(trim_bytes)); + handle->audio_buffer.erase( + handle->audio_buffer.begin(), + handle->audio_buffer.begin() + trim_bytes); + handle->parakeet_chunk_cursor_bytes -= trim_bytes; + } + + const bool onset_mode_active = handle->parakeet_onset_active; + const size_t preview_start_bytes = onset_mode_active + ? std::min(handle->parakeet_onset_start_bytes, handle->audio_buffer.size()) + : (handle->parakeet_chunk_cursor_bytes > left_context_bytes + ? handle->parakeet_chunk_cursor_bytes - left_context_bytes + : 0); + const size_t preview_end_bytes = handle->audio_buffer.size(); + + StreamWindowDecodeResult response_decode; + float response_segments_offset_sec = 0.0f; + if (!decode_window(preview_start_bytes, preview_end_bytes, response_decode, response_segments_offset_sec)) { + last_error_message = "Transcription failed in stream process."; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription( + handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, + false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, + last_error_message.c_str()); + return -1; + } + + std::string confirmed; + const std::string previous_pending = handle->previous_parakeet_pending; + std::string pending_tail = previous_pending; + const std::string normalized_response = parakeet_strip_recent_committed_prefix( + handle->parakeet_committed_text, response_decode.response); + if (!normalized_response.empty()) { + if (previous_pending.empty() || (onset_mode_active && handle->parakeet_resume_guard_active)) { + pending_tail = normalized_response; + } else { + const size_t stable_words = parakeet_common_prefix_words( + previous_pending, normalized_response); + const size_t confirm_words = + stable_words > kParakeetHoldbackWords + ? stable_words - kParakeetHoldbackWords + : 0; + confirmed = parakeet_take_first_words(normalized_response, confirm_words); + pending_tail = confirm_words > 0 + ? parakeet_drop_first_words(normalized_response, confirm_words) + : normalized_response; + } + } + + const bool onset_commit_too_small = + onset_mode_active && + !confirmed.empty() && + !detected_pause && + parakeet_word_count(confirmed) < kParakeetOnsetMinConfirmWords; + if (onset_commit_too_small) { + confirmed.clear(); + pending_tail = normalized_response; + } + + const bool pending_equivalent = + pending_tail == previous_pending || + (!pending_tail.empty() && + !previous_pending.empty() && + levenshtein_ci(pending_tail, previous_pending) <= 1); + const bool pending_changed = !pending_equivalent; + if (pending_changed) { + handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); + } + + const bool stable_short_pending = + !pending_tail.empty() && + !handle->parakeet_resume_guard_active && + pending_equivalent && + handle->previous_parakeet_pending_ticks >= kParakeetStableFlushTicks && + parakeet_word_count(pending_tail) <= kParakeetStableFlushWords; + if ((detected_pause || stable_short_pending) && !pending_tail.empty()) { + if (!confirmed.empty()) confirmed += " "; + confirmed += pending_tail; + pending_tail.clear(); + } + + double buffer_duration_ms = 0.0; + bool cloud_handoff_triggered = false; + uint64_t cloud_job_id = 0; + uint64_t cloud_result_job_id = 0; + CloudResponse cloud_result; + + if (!confirmed.empty()) { + if (!handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); + } + + std::string confirmed_delta = parakeet_emit_delta( + handle->parakeet_committed_text, confirmed); + if (!confirmed_delta.empty()) { + if (response_decode.decode_tokens > 0.0) { + handle->stream_total_tokens += static_cast(std::round(response_decode.decode_tokens)); + handle->stream_cumulative_tokens += static_cast(std::round(response_decode.decode_tokens)); + } + + if (!handle->stream_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_first_token_ms = + std::chrono::duration_cast( + now - handle->stream_start).count(); + handle->stream_first_token_seen = true; + } + + if (!handle->stream_session_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_session_first_token_ms = + std::chrono::duration_cast( + now - handle->stream_session_start).count(); + handle->stream_session_first_token_seen = true; + } + + if (!handle->parakeet_committed_text.empty()) { + handle->parakeet_committed_text += " "; + } + handle->parakeet_committed_text += confirmed_delta; + confirmed = confirmed_delta; + if (onset_mode_active) { + handle->parakeet_onset_active = false; + handle->parakeet_onset_start_bytes = 0; + handle->parakeet_chunk_cursor_bytes = + handle->audio_buffer.size() > right_context_bytes + ? handle->audio_buffer.size() - right_context_bytes + : 0; + } + buffer_duration_ms = + static_cast(processed_chunks) * kParakeetTdtChunkSec * 1000.0; + } else { + confirmed.clear(); + } + } + + if (pending_tail.empty()) { + handle->previous_parakeet_pending_ticks = 0; + } else if (pending_equivalent) { + handle->previous_parakeet_pending_ticks = std::min( + handle->previous_parakeet_pending_ticks + 1, 1000000); + } else { + handle->previous_parakeet_pending_ticks = 1; + } + handle->previous_parakeet_pending = pending_tail; + if (pending_tail.empty()) { + handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); + } + + std::vector current_segments = response_decode.segments; + const float segments_offset_sec = response_segments_offset_sec; + const std::string json_str = response_decode.raw_json; + const std::string pending_output = pending_tail; + handle->previous_cloud_handoff = false; + + for (auto it = handle->pending_cloud_jobs.begin(); it != handle->pending_cloud_jobs.end(); ) { + if (it->result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { + CloudResponse job_result = it->result.get(); + CACTUS_LOG_INFO( + "cloud_handoff", + "Completed stream cloud job id=" << it->id + << " used_cloud=" << (job_result.used_cloud ? "true" : "false") + << " error=" << (job_result.error.empty() ? "none" : job_result.error) + << " transcript_chars=" << job_result.transcript.size()); + handle->completed_cloud_results.push_back({it->id, std::move(job_result)}); + it = handle->pending_cloud_jobs.erase(it); + } else { + ++it; + } + } + + if (!handle->completed_cloud_results.empty()) { + cloud_result_job_id = handle->completed_cloud_results.front().first; + cloud_result = handle->completed_cloud_results.front().second; + handle->completed_cloud_results.erase(handle->completed_cloud_results.begin()); + + if (!cloud_result.api_key_hash.empty()) { + cactus::telemetry::setCloudKey(cloud_result.api_key_hash.c_str()); + } + } + + constexpr int STREAM_TOKENS_CAP = 20000; + constexpr double STREAM_DURATION_CAP_MS = 600000.0; + auto now = std::chrono::steady_clock::now(); + double elapsed_ms = + std::chrono::duration_cast( + now - handle->stream_start).count(); + if (handle->stream_total_tokens >= STREAM_TOKENS_CAP || + elapsed_ms >= STREAM_DURATION_CAP_MS) { + double period_tps = (elapsed_ms > 0.0) + ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms + : 0.0; + + double cumulative_elapsed_ms = + std::chrono::duration_cast( + now - handle->stream_session_start).count(); + double cumulative_tps = (cumulative_elapsed_ms > 0.0) + ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / + cumulative_elapsed_ms + : 0.0; + + cactus::telemetry::recordStreamTranscription( + handle->model_handle->model_name.c_str(), + true, + handle->stream_first_token_ms, + period_tps, + elapsed_ms, + handle->stream_total_tokens, + handle->stream_session_first_token_ms, + cumulative_tps, + cumulative_elapsed_ms, + handle->stream_cumulative_tokens, + "" + ); + + handle->stream_start = std::chrono::steady_clock::now(); + handle->stream_first_token_seen = false; + handle->stream_first_token_ms = 0.0; + handle->stream_total_tokens = 0; + } + + std::string json_response = build_stream_response( + json_str, + json_string(json_str, "error"), + confirmed, + pending_output, + serialize_segments_with_offset(current_segments, segments_offset_sec), + cloud_handoff_triggered, + buffer_duration_ms, + cloud_job_id, + cloud_result_job_id, + cloud_result + ); + + if (json_response.length() >= buffer_size) { + last_error_message = "Response buffer too small"; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription( + handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, + false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, + last_error_message.c_str()); + return -1; + } + + std::strcpy(response_buffer, json_response.c_str()); + return static_cast(json_response.length()); + } + + const uint8_t* decode_pcm = handle->audio_buffer.data(); + size_t decode_pcm_size = handle->audio_buffer.size(); + if (is_parakeet) { + constexpr size_t kParakeetDecodeWindowBytes = 160000; // ~5.0s + if (decode_pcm_size > kParakeetDecodeWindowBytes) { + decode_pcm += (decode_pcm_size - kParakeetDecodeWindowBytes); + decode_pcm_size = kParakeetDecodeWindowBytes; + } + } + + cactus::telemetry::setStreamMode(true); + int result = cactus_transcribe( + handle->model_handle, + nullptr, + transcribe_prompt, + handle->transcribe_response_buffer, + sizeof(handle->transcribe_response_buffer), + effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), + nullptr, + nullptr, + decode_pcm, + decode_pcm_size); + cactus::telemetry::setStreamMode(false); + if (result < 0) { + last_error_message = "Transcription failed in stream process."; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); + return -1; + } + + std::string json_str(handle->transcribe_response_buffer); + std::string response = suppress_unwanted_text(json_string(json_str, "response")); + + cactus_reset(handle->model_handle); + + auto current_segments = parse_segments(json_str); + const float segments_offset_sec = static_cast(handle->stream_audio_offset_sec); + + std::string confirmed; + double buffer_duration_ms = 0.0; + bool cloud_handoff_triggered = false; + uint64_t cloud_job_id = 0; + uint64_t cloud_result_job_id = 0; + CloudResponse cloud_result; + double chunk_decode_tokens = json_number(json_str, "decode_tokens"); + if (chunk_decode_tokens < 0.0) { + chunk_decode_tokens = 0.0; + } + + if (is_parakeet) { + constexpr size_t kParakeetHoldbackWords = 1; + constexpr size_t kParakeetTailBytes = 64000; // ~2.0s left context + constexpr size_t kParakeetSilenceFlushBytes = 16000; // ~0.5s silence + constexpr size_t kParakeetStableFlushWords = 4; + constexpr size_t kParakeetStableFlushTicks = 2; + constexpr size_t kParakeetMediumTrimWords = 3; + constexpr size_t kParakeetAggressiveTrimWords = 6; + + const std::string previous_pending = handle->previous_parakeet_pending; + std::string pending_tail = previous_pending; + const bool chunk_silent = is_pcm_chunk_silent(pcm_buffer, pcm_buffer_size, 0.012f); + if (chunk_silent) { + handle->parakeet_silence_run_bytes += pcm_buffer_size; + } else { + handle->parakeet_silence_run_bytes = 0; + } + if (!response.empty()) { + const std::string normalized_response = parakeet_strip_recent_committed_prefix( + handle->parakeet_committed_text, response); + if (previous_pending.empty()) { + pending_tail = normalized_response; + } else { + const size_t stable_words = parakeet_common_prefix_words(previous_pending, normalized_response); + size_t confirm_words = 0; + if (stable_words > kParakeetHoldbackWords) { + confirm_words = stable_words - kParakeetHoldbackWords; + } + + confirmed = parakeet_take_first_words(normalized_response, confirm_words); + if (confirm_words > 0) { + pending_tail = parakeet_drop_first_words(normalized_response, confirm_words); + } else { + pending_tail = normalized_response; + } + } + } + + const bool pending_equivalent = + pending_tail == previous_pending || + (!pending_tail.empty() && + !previous_pending.empty() && + levenshtein_ci(pending_tail, previous_pending) <= 1); + const bool pending_changed = !pending_equivalent; + if (pending_changed) { + handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); + } + + const bool detected_pause = + handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes; + const bool stable_short_pending = + !pending_tail.empty() && + pending_equivalent && + handle->previous_parakeet_pending_ticks >= kParakeetStableFlushTicks && + parakeet_word_count(pending_tail) <= kParakeetStableFlushWords && + handle->audio_buffer.size() >= + (handle->previous_parakeet_audio_buffer_size + (kParakeetSilenceFlushBytes / 2)); + if ((detected_pause || stable_short_pending) && !pending_tail.empty()) { + if (!confirmed.empty()) confirmed += " "; + confirmed += pending_tail; + pending_tail.clear(); + } + + if (!confirmed.empty()) { + if (!handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); + } + + std::string confirmed_delta = parakeet_emit_delta( + handle->parakeet_committed_text, confirmed); + + const size_t max_trim = handle->audio_buffer.size() > kParakeetTailBytes + ? handle->audio_buffer.size() - kParakeetTailBytes + : 0; + const size_t confirmed_words = parakeet_word_count(confirmed); + const size_t hop_bytes = std::clamp( + pcm_buffer_size, static_cast(4000), static_cast(16000)); + size_t confirmed_bytes = std::min(hop_bytes, max_trim); + if (confirmed_words >= kParakeetAggressiveTrimWords || pending_tail.empty()) { + confirmed_bytes = max_trim; + } else if (confirmed_words >= kParakeetMediumTrimWords) { + confirmed_bytes = std::min(max_trim, hop_bytes * 2); + } + buffer_duration_ms = (confirmed_bytes / 2.0) / 16000.0 * 1000.0; + + if (!confirmed_delta.empty()) { + if (chunk_decode_tokens > 0.0) { + handle->stream_total_tokens += static_cast(std::round(chunk_decode_tokens)); + handle->stream_cumulative_tokens += static_cast(std::round(chunk_decode_tokens)); + } + + if (!handle->stream_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_first_token_ms = std::chrono::duration_cast(now - handle->stream_start).count(); + handle->stream_first_token_seen = true; + } + + if (!handle->stream_session_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_session_first_token_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); + handle->stream_session_first_token_seen = true; + } + + if (!handle->parakeet_committed_text.empty()) { + handle->parakeet_committed_text += " "; + } + handle->parakeet_committed_text += confirmed_delta; + } + + if (confirmed_bytes > 0) { + if (handle->previous_cloud_handoff && !confirmed.empty()) { + cloud_handoff_triggered = true; + std::vector confirmed_audio( + handle->audio_buffer.begin(), + handle->audio_buffer.begin() + confirmed_bytes + ); + auto wav = cloud_build_wav(confirmed_audio.data(), confirmed_audio.size()); + std::string b64 = cloud_base64_encode(wav.data(), wav.size()); + cloud_job_id = handle->next_cloud_job_id++; + CACTUS_LOG_INFO( + "cloud_handoff", + "Queued stream cloud job id=" << cloud_job_id + << " confirmed_chars=" << confirmed.size() + << " audio_bytes=" << confirmed_audio.size()); + handle->pending_cloud_jobs.push_back({ + cloud_job_id, + std::async(std::launch::async, cloud_transcribe_request, b64, confirmed, 15L, nullptr) + }); + } + + handle->stream_audio_offset_sec += static_cast(confirmed_bytes) / (16000.0f * 2.0f); + handle->audio_buffer.erase( + handle->audio_buffer.begin(), + handle->audio_buffer.begin() + confirmed_bytes + ); + } + + confirmed = confirmed_delta; + handle->previous_cloud_handoff = false; + handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); + } else { + handle->previous_cloud_handoff = json_bool(json_str, "cloud_handoff"); + } + + if (pending_tail.empty()) { + handle->previous_parakeet_pending_ticks = 0; + } else if (pending_equivalent) { + handle->previous_parakeet_pending_ticks = std::min( + handle->previous_parakeet_pending_ticks + 1, 1000000); + } else { + handle->previous_parakeet_pending_ticks = 1; + } + handle->previous_parakeet_pending = pending_tail; + if (pending_tail.empty()) { + handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); + } + handle->previous_segments = current_segments; + } else { + size_t confirmed_segments = 0; + float confirmed_end_sec = 0.0f; + for (size_t i = 0; i < std::min(handle->previous_segments.size(), current_segments.size()); ++i) { + if (handle->previous_segments[i].text != current_segments[i].text) { + break; + } + confirmed_segments = i + 1; + confirmed_end_sec = std::min(handle->previous_segments[i].end, current_segments[i].end); + } + + if (confirmed_segments > 0 && confirmed_end_sec > 0.0f) { + buffer_duration_ms = confirmed_end_sec * 1000.0; + + for (size_t i = 0; i < confirmed_segments; ++i) { + if (!confirmed.empty()) confirmed += ' '; + confirmed += handle->previous_segments[i].text; + } + + if (!handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); + } + + if (chunk_decode_tokens > 0.0) { + handle->stream_total_tokens += static_cast(std::round(chunk_decode_tokens)); + handle->stream_cumulative_tokens += static_cast(std::round(chunk_decode_tokens)); + } + + if (!handle->stream_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_first_token_ms = std::chrono::duration_cast(now - handle->stream_start).count(); + handle->stream_first_token_seen = true; + } + + if (!handle->stream_session_first_token_seen) { + auto now = std::chrono::steady_clock::now(); + handle->stream_session_first_token_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); + handle->stream_session_first_token_seen = true; + } + + const size_t confirmed_bytes = std::min( + static_cast(confirmed_end_sec * 16000.0) * 2, + handle->audio_buffer.size() + ); + + if (handle->previous_cloud_handoff && !confirmed.empty()) { + cloud_handoff_triggered = true; + std::vector confirmed_audio( + handle->audio_buffer.begin(), + handle->audio_buffer.begin() + confirmed_bytes + ); + auto wav = cloud_build_wav(confirmed_audio.data(), confirmed_audio.size()); + std::string b64 = cloud_base64_encode(wav.data(), wav.size()); + cloud_job_id = handle->next_cloud_job_id++; + CACTUS_LOG_INFO( + "cloud_handoff", + "Queued stream cloud job id=" << cloud_job_id + << " confirmed_chars=" << confirmed.size() + << " audio_bytes=" << confirmed_audio.size()); + handle->pending_cloud_jobs.push_back({ + cloud_job_id, + std::async(std::launch::async, cloud_transcribe_request, b64, confirmed, 15L, nullptr) + }); + } + + handle->stream_audio_offset_sec += confirmed_end_sec; + handle->audio_buffer.erase( + handle->audio_buffer.begin(), + handle->audio_buffer.begin() + confirmed_bytes + ); + constexpr float kSegmentEpsilonSec = 0.02f; + size_t current_tail_start = 0; + while (current_tail_start < current_segments.size() && + current_segments[current_tail_start].end <= confirmed_end_sec + kSegmentEpsilonSec) { + ++current_tail_start; + } + handle->previous_segments = std::vector( + current_segments.begin() + current_tail_start, current_segments.end()); + handle->previous_cloud_handoff = false; + } else { + handle->previous_segments = current_segments; + handle->previous_cloud_handoff = json_bool(json_str, "cloud_handoff"); + } + } + + for (auto it = handle->pending_cloud_jobs.begin(); it != handle->pending_cloud_jobs.end(); ) { + if (it->result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { + CloudResponse job_result = it->result.get(); + CACTUS_LOG_INFO( + "cloud_handoff", + "Completed stream cloud job id=" << it->id + << " used_cloud=" << (job_result.used_cloud ? "true" : "false") + << " error=" << (job_result.error.empty() ? "none" : job_result.error) + << " transcript_chars=" << job_result.transcript.size()); + handle->completed_cloud_results.push_back({it->id, std::move(job_result)}); + it = handle->pending_cloud_jobs.erase(it); + } else { + ++it; + } + } + + if (!handle->completed_cloud_results.empty()) { + cloud_result_job_id = handle->completed_cloud_results.front().first; + cloud_result = handle->completed_cloud_results.front().second; + handle->completed_cloud_results.erase(handle->completed_cloud_results.begin()); + + if (!cloud_result.api_key_hash.empty()) { + cactus::telemetry::setCloudKey(cloud_result.api_key_hash.c_str()); + } + } + + constexpr int STREAM_TOKENS_CAP = 20000; + constexpr double STREAM_DURATION_CAP_MS = 600000.0; + auto now = std::chrono::steady_clock::now(); + double elapsed_ms = std::chrono::duration_cast(now - handle->stream_start).count(); + if (handle->stream_total_tokens >= STREAM_TOKENS_CAP || elapsed_ms >= STREAM_DURATION_CAP_MS) { + double period_tps = (elapsed_ms > 0.0) ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms : 0.0; + + double cumulative_elapsed_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); + double cumulative_tps = (cumulative_elapsed_ms > 0.0) ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / cumulative_elapsed_ms : 0.0; + + cactus::telemetry::recordStreamTranscription( + handle->model_handle->model_name.c_str(), + true, + handle->stream_first_token_ms, + period_tps, + elapsed_ms, + handle->stream_total_tokens, + handle->stream_session_first_token_ms, + cumulative_tps, + cumulative_elapsed_ms, + handle->stream_cumulative_tokens, + "" + ); + + handle->stream_start = std::chrono::steady_clock::now(); + handle->stream_first_token_seen = false; + handle->stream_first_token_ms = 0.0; + handle->stream_total_tokens = 0; + } + + std::string pending_output = response; + if (is_parakeet && !handle->previous_parakeet_pending.empty()) { + pending_output = handle->previous_parakeet_pending; + } + + std::string json_response = build_stream_response( + json_str, + json_string(json_str, "error"), + confirmed, + pending_output, + serialize_segments_with_offset(current_segments, segments_offset_sec), + cloud_handoff_triggered, + buffer_duration_ms, + cloud_job_id, + cloud_result_job_id, + cloud_result + ); + + if (json_response.length() >= buffer_size) { + last_error_message = "Response buffer too small"; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); + return -1; + } + + std::strcpy(response_buffer, json_response.c_str()); + return static_cast(json_response.length()); + } catch (const std::exception& e) { + last_error_message = "Exception during stream_transcribe_process: " + std::string(e.what()); + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response(e.what(), response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, e.what()); + return -1; + } catch (...) { + last_error_message = "Unknown exception during stream transcription processing"; + CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); + handle_error_response("Unknown error during stream processing", response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, "Unknown error during stream processing"); + return -1; + } +} + +int cactus_stream_transcribe_stop( + cactus_stream_transcribe_t stream, + char* response_buffer, + size_t buffer_size +) { + if (!stream) { + last_error_message = "Stream not initialized."; + CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); + return -1; + } + + auto* handle = static_cast(stream); + auto clear_stream_vocab_bias = [&]() { + if (handle->has_custom_vocabulary_bias && + handle->model_handle && + handle->model_handle->model) { + handle->model_handle->model->clear_vocab_bias(); + } + }; + + if (!response_buffer || buffer_size == 0) { + clear_stream_vocab_bias(); + delete handle; + return 0; + } + + try { + const auto model_type = handle->model_handle->model->get_config().model_type; + bool is_moonshine = model_type == cactus::engine::Config::ModelType::MOONSHINE; + bool is_parakeet_tdt = + model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; + bool is_parakeet = + model_type == cactus::engine::Config::ModelType::PARAKEET || + is_parakeet_tdt; + bool is_tinyllama = model_type == cactus::engine::Config::ModelType::GEMMA4; + + std::string final_confirmed; + if (is_parakeet_tdt) { + final_confirmed = handle->parakeet_committed_text; + if (!(handle->parakeet_tdt_chunked_stream && handle->parakeet_tdt_decode_context.initialized) && + !handle->previous_parakeet_pending.empty()) { + const std::string normalized_pending = parakeet_strip_recent_committed_prefix( + final_confirmed, handle->previous_parakeet_pending); + const std::string pending_delta = parakeet_emit_delta( + final_confirmed, normalized_pending); + if (!pending_delta.empty()) { + if (!final_confirmed.empty()) final_confirmed += " "; + final_confirmed += pending_delta; + } + } + } else if (is_parakeet) { + final_confirmed = handle->parakeet_committed_text; + if (!handle->previous_parakeet_pending.empty()) { + const std::string normalized_pending = parakeet_strip_recent_committed_prefix( + final_confirmed, handle->previous_parakeet_pending); + const std::string pending_delta = parakeet_emit_delta( + final_confirmed, normalized_pending); + if (!pending_delta.empty()) { + if (!final_confirmed.empty()) final_confirmed += " "; + final_confirmed += pending_delta; + } + } + } else { + for (const auto& seg : handle->previous_segments) { + if (!final_confirmed.empty()) final_confirmed += ' '; + final_confirmed += seg.text; + } + } + + if (is_parakeet_tdt && + handle->parakeet_tdt_chunked_stream && + handle->parakeet_tdt_decode_context.initialized) { + auto& ctx = handle->parakeet_tdt_decode_context; + auto* tdt_model = static_cast( + handle->model_handle->model.get()); + const size_t total_samples = ctx.audio_samples.size(); + if (total_samples > ctx.samples_decoded_up_to) { + constexpr size_t kTdtLeftContextSamples = 8 * 16000; + const size_t window_start_sample = ctx.samples_decoded_up_to > kTdtLeftContextSamples + ? ctx.samples_decoded_up_to - kTdtLeftContextSamples + : 0; + std::vector window_audio( + ctx.audio_samples.begin() + window_start_sample, + ctx.audio_samples.end()); + + auto cfg = get_parakeet_spectrogram_config(); + const size_t waveform_samples = window_audio.size(); + apply_preemphasis(window_audio, 0.97f); + window_audio = ctx.audio_processor.compute_spectrogram(window_audio, cfg); + normalize_parakeet_log_mel(window_audio, ctx.mel_bins); + size_t valid_frames = waveform_samples / cfg.hop_length; + if (valid_frames == 0) valid_frames = 1; + trim_mel_frames(window_audio, ctx.mel_bins, valid_frames); + + const uint32_t subsampling = + std::max(1, handle->model_handle->model->get_config().subsampling_factor); + const size_t samples_per_enc_frame = cfg.hop_length * subsampling; + const size_t context_samples = ctx.samples_decoded_up_to - window_start_sample; + const size_t decode_start_frame = context_samples / samples_per_enc_frame; + const size_t total_window_samples = total_samples - window_start_sample; + const size_t decode_end_frame = total_window_samples / samples_per_enc_frame; + + if (decode_end_frame > decode_start_frame) { + cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; + { + std::lock_guard lock(handle->model_handle->model_mutex); + cactus::telemetry::setStreamMode(true); + try { + chunk_decode = tdt_model->decode_chunk_stream( + window_audio, + decode_start_frame, + decode_start_frame, + decode_end_frame, + ctx.decoder_state); + cactus_reset(handle->model_handle); + } catch (...) { + cactus::telemetry::setStreamMode(false); + throw; + } + cactus::telemetry::setStreamMode(false); + } + + std::string confirmed_flush = suppress_unwanted_text(chunk_decode.confirmed_text); + std::string pending_flush = suppress_unwanted_text(chunk_decode.pending_text); + if (!confirmed_flush.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(confirmed_flush, handle->custom_vocabulary); + } + if (!pending_flush.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(pending_flush, handle->custom_vocabulary); + } + parakeet_append_monotonic_text(final_confirmed, confirmed_flush); + parakeet_append_monotonic_text(final_confirmed, pending_flush); + } + } else if (!handle->previous_parakeet_pending.empty()) { + parakeet_append_monotonic_text(final_confirmed, handle->previous_parakeet_pending); + } + } + + if (!handle->audio_buffer.empty()) { + std::string whisper_prompt = "<|startoftranscript|><|" + handle->options.language + "|><|transcribe|><|notimestamps|>"; + const char* transcribe_prompt = + is_tinyllama ? "Transcribe the audio." : + (is_moonshine || is_parakeet) ? "" : + whisper_prompt.c_str(); + std::string effective_transcribe_options_json = handle->transcribe_options_json; + if (is_parakeet && !is_parakeet_tdt) { + effective_transcribe_options_json = ensure_json_bool_option( + effective_transcribe_options_json, "use_vad", false); + } + + if (is_parakeet_tdt) { + const bool use_chunked_stop = + handle->parakeet_tdt_chunked_stream && + !handle->parakeet_tdt_chunked_blocked_until_pause && + (handle->parakeet_tdt_decoder_state.initialized || + !handle->parakeet_tdt_resume_checkpoints.empty() || + !handle->parakeet_tdt_pending_raw.empty()); + if (use_chunked_stop) { + final_confirmed = handle->parakeet_committed_text; + constexpr double kParakeetTdtLeftContextSec = 2.0; + const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); + if (handle->parakeet_chunk_cursor_bytes < handle->audio_buffer.size()) { + const size_t chunk_start_bytes = handle->parakeet_chunk_cursor_bytes; + const size_t chunk_end_bytes = handle->audio_buffer.size(); + const size_t window_start_bytes = chunk_start_bytes > left_context_bytes + ? chunk_start_bytes - left_context_bytes + : 0; + cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; + cactus::engine::ParakeetTDTModel::ChunkStreamState chunk_state; + if (run_parakeet_tdt_chunk_decode( + handle, + window_start_bytes, + chunk_end_bytes, + chunk_start_bytes, + chunk_end_bytes, + chunk_decode, + chunk_state)) { + std::string confirmed_chunk = suppress_unwanted_text(chunk_decode.confirmed_text); + std::string pending_chunk = suppress_unwanted_text(chunk_decode.pending_text); + if (!confirmed_chunk.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(confirmed_chunk, handle->custom_vocabulary); + } + if (!pending_chunk.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(pending_chunk, handle->custom_vocabulary); + } + parakeet_append_monotonic_text(final_confirmed, confirmed_chunk); + handle->parakeet_tdt_pending_raw = + strip_unwanted_text_preserving_edges(chunk_decode.pending_text); + handle->parakeet_chunk_cursor_bytes = chunk_end_bytes; + handle->parakeet_tdt_decoder_state = chunk_state; + handle->parakeet_committed_until_sec = std::max( + handle->parakeet_committed_until_sec, + static_cast( + handle->stream_audio_offset_sec + + pcm_bytes_to_seconds( + std::min( + chunk_end_bytes, + std::max( + window_start_bytes, + window_start_bytes + seconds_to_pcm_bytes( + chunk_decode.resume_end_sec)))))); + } + } else if (!handle->previous_parakeet_pending.empty()) { + const std::string normalized_pending = parakeet_strip_recent_committed_prefix( + final_confirmed, handle->previous_parakeet_pending); + const std::string pending_delta = parakeet_emit_delta( + final_confirmed, normalized_pending); + if (!pending_delta.empty()) { + if (!final_confirmed.empty()) final_confirmed += " "; + final_confirmed += pending_delta; + } + } + if (!handle->parakeet_tdt_pending_raw.empty()) { + std::string pending_flush = suppress_unwanted_text( + handle->parakeet_tdt_pending_raw); + if (!pending_flush.empty() && !handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction( + pending_flush, handle->custom_vocabulary); + } + parakeet_append_monotonic_text(final_confirmed, pending_flush); + handle->parakeet_tdt_pending_raw.clear(); + } + handle->parakeet_committed_text = final_confirmed; + } else { + constexpr double kParakeetTdtLeftContextSec = 2.0; + constexpr float kParakeetTdtCommitEpsilonSec = 0.03f; + const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); + const size_t flush_start_bytes = handle->parakeet_onset_active + ? std::min(handle->parakeet_onset_start_bytes, handle->audio_buffer.size()) + : (handle->parakeet_chunk_cursor_bytes > left_context_bytes + ? handle->parakeet_chunk_cursor_bytes - left_context_bytes + : 0); + const std::string tdt_vad_options_json = ensure_json_bool_option( + handle->transcribe_options_json, "use_vad", true); + const std::string tdt_no_vad_options_json = ensure_json_bool_option( + handle->transcribe_options_json, "use_vad", false); + + StreamWindowDecodeResult flush_decode; + bool flush_ok = run_stream_window_transcribe( + handle, + transcribe_prompt, + tdt_vad_options_json, + handle->audio_buffer.data() + flush_start_bytes, + handle->audio_buffer.size() - flush_start_bytes, + flush_decode); + if (flush_ok && flush_decode.response.empty() && flush_decode.segments.empty()) { + flush_ok = run_stream_window_transcribe( + handle, + transcribe_prompt, + tdt_no_vad_options_json, + handle->audio_buffer.data() + flush_start_bytes, + handle->audio_buffer.size() - flush_start_bytes, + flush_decode); + } + if (flush_ok) { + const float flush_offset_sec = static_cast( + handle->stream_audio_offset_sec + pcm_bytes_to_seconds(flush_start_bytes)); + float flushed_until_sec = handle->parakeet_committed_until_sec; + std::string flushed_text = join_segments_by_end_time( + flush_decode.segments, + flush_offset_sec, + handle->parakeet_committed_until_sec + kParakeetTdtCommitEpsilonSec, + std::numeric_limits::max(), + &flushed_until_sec); + if (flushed_text.empty()) { + flushed_text = parakeet_strip_recent_committed_prefix( + final_confirmed, flush_decode.response); + } + const std::string effective_flush_delta = + parakeet_emit_delta(final_confirmed, flushed_text); + if (!effective_flush_delta.empty()) { + if (!final_confirmed.empty()) final_confirmed += " "; + final_confirmed += effective_flush_delta; + } + if (flushed_until_sec > handle->parakeet_committed_until_sec) { + handle->parakeet_committed_until_sec = flushed_until_sec; + } + } else { + CACTUS_LOG_WARN( + "stream_transcribe_stop", + "Final flush transcription failed; returning previously confirmed transcript."); + } + } + } else { + cactus::telemetry::setStreamMode(true); + const int flush_result = cactus_transcribe( + handle->model_handle, + nullptr, + transcribe_prompt, + handle->transcribe_response_buffer, + sizeof(handle->transcribe_response_buffer), + effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), + nullptr, + nullptr, + handle->audio_buffer.data(), + handle->audio_buffer.size()); + cactus::telemetry::setStreamMode(false); + + cactus_reset(handle->model_handle); + + if (flush_result >= 0) { + std::string flush_json(handle->transcribe_response_buffer); + std::string flushed_text = suppress_unwanted_text(json_string(flush_json, "response")); + if (flushed_text.empty()) { + auto flush_segments = parse_segments(flush_json); + for (const auto& seg : flush_segments) { + if (!flushed_text.empty()) flushed_text += ' '; + flushed_text += seg.text; + } + } + if (!flushed_text.empty()) { + if (is_parakeet) { + const std::string normalized_flush = parakeet_strip_recent_committed_prefix( + final_confirmed, flushed_text); + const std::string effective_flush_delta = + parakeet_emit_delta(final_confirmed, normalized_flush); + if (!effective_flush_delta.empty()) { + if (!final_confirmed.empty()) final_confirmed += " "; + final_confirmed += effective_flush_delta; + } + } else { + final_confirmed = flushed_text; + } + } + } else { + CACTUS_LOG_WARN( + "stream_transcribe_stop", + "Final flush transcription failed; returning previously confirmed segments."); + } + } + } + + if (!handle->custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(final_confirmed, handle->custom_vocabulary); + } + + std::string json_response = "{\"success\":true,\"confirmed\":\"" + + escape_json(final_confirmed) + "\"}"; + + auto now = std::chrono::steady_clock::now(); + double elapsed_ms = std::chrono::duration_cast(now - handle->stream_start).count(); + double period_tps = (elapsed_ms > 0.0) ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms : 0.0; + + double cumulative_elapsed_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); + double cumulative_tps = (cumulative_elapsed_ms > 0.0) ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / cumulative_elapsed_ms : 0.0; + + cactus::telemetry::recordStreamTranscription( + handle->model_handle->model_name.c_str(), + true, + handle->stream_first_token_ms, + period_tps, + elapsed_ms, + handle->stream_total_tokens, + handle->stream_session_first_token_ms, + cumulative_tps, + cumulative_elapsed_ms, + handle->stream_cumulative_tokens, + "" + ); + + if (json_response.length() >= buffer_size) { + last_error_message = "Response buffer too small"; + CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, last_error_message.c_str()); + clear_stream_vocab_bias(); + delete handle; + return -1; + } + + std::strcpy(response_buffer, json_response.c_str()); + clear_stream_vocab_bias(); + delete handle; + return static_cast(json_response.length()); + } catch (const std::exception& e) { + last_error_message = "Exception during stream_transcribe_stop: " + std::string(e.what()); + CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); + handle_error_response(e.what(), response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, e.what()); + clear_stream_vocab_bias(); + delete handle; + return -1; + } catch (...) { + last_error_message = "Unknown exception during stream transcription stop"; + CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); + handle_error_response("Unknown error during stream stop", response_buffer, buffer_size); + cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, "Unknown error during stream stop"); + clear_stream_vocab_bias(); + delete handle; + return -1; + } +} + +} diff --git a/cactus-transpile/ffi/cactus_telemetry.cpp b/cactus-transpile/ffi/cactus_telemetry.cpp new file mode 100644 index 000000000..1844f0e22 --- /dev/null +++ b/cactus-transpile/ffi/cactus_telemetry.cpp @@ -0,0 +1,22 @@ +#include "cactus_ffi.h" +#include "telemetry/telemetry.h" + +extern "C" { + +void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version) { + cactus::telemetry::setTelemetryEnvironment(framework, cache_location, version); +} + +void cactus_set_app_id(const char* app_id) { + cactus::telemetry::setAppId(app_id); +} + +void cactus_telemetry_flush(void) { + cactus::telemetry::flush(); +} + +void cactus_telemetry_shutdown(void) { + cactus::telemetry::shutdown(); +} + +} diff --git a/cactus-transpile/ffi/cactus_transcribe.cpp b/cactus-transpile/ffi/cactus_transcribe.cpp new file mode 100644 index 000000000..539c0f155 --- /dev/null +++ b/cactus-transpile/ffi/cactus_transcribe.cpp @@ -0,0 +1,829 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include "../models/model.h" +#include "telemetry/telemetry.h" +#include "../../libs/audio/wav.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; +using cactus::audio::WHISPER_TARGET_FRAMES; +using cactus::audio::WHISPER_SAMPLE_RATE; +using cactus::audio::apply_preemphasis; +using cactus::audio::get_parakeet_spectrogram_config; +using cactus::audio::get_whisper_spectrogram_config; +using cactus::audio::init_whisper_mel_filters; +using cactus::audio::normalize_parakeet_log_mel; +using cactus::audio::normalize_whisper_mel; +using cactus::audio::trim_mel_frames; +using cactus::audio::get_htk_spectrogram_config; +using cactus::audio::get_gemma4_audio_spectrogram_config; +using cactus::audio::transpose_mel_to_frame_major; + +static constexpr size_t WHISPER_MAX_DECODER_POSITIONS = 448; +static constexpr size_t MAX_CHUNK_SAMPLES = WHISPER_SAMPLE_RATE * 30; +static constexpr size_t MAX_CONTEXT_WORDS = 64; + +static std::string extract_whisper_language_code(std::string token_text) { + const size_t open = token_text.find("<|"); + if (open == std::string::npos) return ""; + + const size_t close = token_text.find("|>", open + 2); + if (close == std::string::npos || close <= open + 2) return ""; + + std::string inner = token_text.substr(open + 2, close - (open + 2)); + std::transform(inner.begin(), inner.end(), inner.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + + static const std::vector non_language_tokens = { + "startoftranscript", "transcribe", "translate", "notimestamps", + "endoftext", "nospeech", "nocaptions", "sot", "sot_prev", "sot_lm" + }; + for (const auto& reserved : non_language_tokens) { + if (inner == reserved) return ""; + } + + if (inner.size() < 2 || inner.size() > 8) return ""; + for (char c : inner) { + const bool ok = (c >= 'a' && c <= 'z') || c == '-'; + if (!ok) return ""; + } + return inner; +} + +static bool is_terminal_transcription_piece(const std::string& piece) { + return piece == "<|endoftext|>" || + piece == "<|endoftranscript|>" || + piece == "" || + piece == ""; +} + +extern "C" { + +int cactus_transcribe( + cactus_model_t model, + const char* audio_file_path, + const char* prompt, + char* response_buffer, + size_t buffer_size, + const char* options_json, + cactus_token_callback callback, + void* user_data, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +) { + if (validate_audio_params("transcribe", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) + return -1; + + try { + auto start_time = std::chrono::high_resolution_clock::now(); + auto* handle = static_cast(model); + std::lock_guard lock(handle->model_mutex); + handle->should_stop = false; + + float cloud_handoff_threshold = handle->model->get_config().default_cloud_handoff_threshold; + const std::string opts = options_json ? options_json : ""; + InferenceOptions options = parse_inference_options_json(opts); + { + size_t pos = opts.find("\"cloud_handoff_threshold\""); + if (pos != std::string::npos) { + pos = opts.find(':', pos); + if (pos != std::string::npos) { + ++pos; + while (pos < opts.size() && std::isspace(static_cast(opts[pos]))) ++pos; + try { + cloud_handoff_threshold = std::stof(opts.c_str() + pos); + } catch (...) {} + } + } + } + + const char* force_handoff_env = std::getenv("CACTUS_FORCE_HANDOFF"); + if (force_handoff_env && force_handoff_env[0] == '1' && force_handoff_env[1] == '\0') { + cloud_handoff_threshold = 0.0001f; + } + + const bool request_has_custom_vocabulary_options = + opts.find("\"custom_vocabulary\"") != std::string::npos || + opts.find("\"vocabulary_boost\"") != std::string::npos; + const bool apply_request_scoped_vocabulary_bias = + request_has_custom_vocabulary_options && !handle->model->has_vocab_bias(); + + struct ScopedVocabularyBiasReset { + Model* model; + bool clear_on_exit; + ~ScopedVocabularyBiasReset() { + if (clear_on_exit && model) { + model->clear_vocab_bias(); + } + } + } scoped_vocabulary_bias_reset{ + handle->model.get(), + apply_request_scoped_vocabulary_bias + }; + std::vector custom_vocabulary; + float vocabulary_boost_unused = 5.0f; + parse_custom_vocabulary_options(opts, custom_vocabulary, vocabulary_boost_unused); + + if (apply_request_scoped_vocabulary_bias) { + apply_custom_vocabulary_options(handle->model.get(), opts); + } + + if (request_has_custom_vocabulary_options && + options.temperature == 0.0f && options.top_p <= 0.0f && options.top_k == 0) { + // Keep deterministic decoding while ensuring bias is applied in sampling. + options.top_k = 1; + } + + bool is_whisper = handle->model->get_config().model_type == cactus::engine::Config::ModelType::WHISPER; + bool is_moonshine = handle->model->get_config().model_type == cactus::engine::Config::ModelType::MOONSHINE; + bool is_parakeet_tdt = handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; + bool is_parakeet = + handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET || + handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; + bool is_gemma4 = handle->model->get_config().model_type == cactus::engine::Config::ModelType::GEMMA4; + + std::vector audio_samples; + if (audio_file_path == nullptr) { + auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); + audio_samples = resample_to_16k_fp32(waveform_fp32, WHISPER_SAMPLE_RATE); + } else { + AudioFP32 audio = load_wav(audio_file_path); + audio_samples = resample_to_16k_fp32(audio.samples, audio.sample_rate); + } + + if (opts.find("\"max_tokens\"") == std::string::npos) { + const float audio_length_sec = static_cast(audio_samples.size()) / static_cast(WHISPER_SAMPLE_RATE); + const float tps = is_parakeet ? 30.0f : (is_gemma4 ? 30.0f : 20.0f); + const size_t estimated = static_cast(audio_length_sec * tps); + options.max_tokens = std::max(estimated, 100); + } + + if (is_gemma4) { + if (audio_samples.empty()) { + handle_error_response("No audio input provided", response_buffer, buffer_size); + cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "No audio input"); + return -1; + } + + const auto& model_config = handle->model->get_config(); + uint32_t audio_token_id = model_config.audio_token_id; + if (audio_token_id == 0) { + CACTUS_LOG_WARN("transcribe", "audio_token_id not set in config, using default 258881"); + audio_token_id = 258881; + } + + auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, model_config); + std::vector& audio_features = audio_prep.features; + size_t num_soft_tokens = audio_prep.num_soft_tokens; + + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) { + CACTUS_LOG_ERROR("transcribe", "Tokenizer unavailable"); + handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); + cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Tokenizer unavailable"); + return -1; + } + + std::string task_text = (prompt[0] != '\0') ? std::string(prompt) : "Transcribe the audio."; + auto prefix_tokens = tokenizer->encode("<|turn>user\n" + task_text + "<|audio>"); + auto suffix_tokens = tokenizer->encode("\n<|turn>model\n"); + + std::vector tokens; + tokens.reserve(prefix_tokens.size() + num_soft_tokens + suffix_tokens.size()); + tokens.insert(tokens.end(), prefix_tokens.begin(), prefix_tokens.end()); + for (size_t j = 0; j < num_soft_tokens; j++) + tokens.push_back(audio_token_id); + tokens.insert(tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); + + std::vector> stop_token_sequences = {{ tokenizer->get_eos_token() }}; + auto append_stop = [&](const char* stop_text) { + std::vector seq = tokenizer->encode(stop_text); + if (!seq.empty()) + stop_token_sequences.push_back(std::move(seq)); + }; + append_stop(""); + append_stop(""); + append_stop(""); + + const size_t prompt_token_count = tokens.size(); + double time_to_first_token = 0.0; + size_t completion_tokens = 0; + std::string final_text; + float total_entropy_sum = 0.0f; + float max_token_entropy_norm = 0.0f; + std::vector generated_tokens; + generated_tokens.reserve(options.max_tokens); + + for (size_t i = 0; i < options.max_tokens; ++i) { + if (handle->should_stop) break; + + float token_entropy = 0.0f; + uint32_t next_token = handle->model->decode_with_audio( + tokens, audio_features, + options.temperature, options.top_p, options.top_k, + "", &token_entropy, + options.min_p, options.repetition_penalty + ); + + if (completion_tokens == 0) [[unlikely]] { + auto t_first = std::chrono::high_resolution_clock::now(); + time_to_first_token = std::chrono::duration_cast(t_first - start_time).count() / 1000.0; + } + + total_entropy_sum += token_entropy; + if (token_entropy > max_token_entropy_norm) max_token_entropy_norm = token_entropy; + + generated_tokens.emplace_back(next_token); + if (matches_stop_sequence(generated_tokens, stop_token_sequences)) + break; + + std::string piece = tokenizer->decode({ next_token }); + tokens.emplace_back(next_token); + completion_tokens++; + final_text += piece; + if (callback) callback(piece.c_str(), next_token, user_data); + } + + cactus_reset(model); + + float mean_entropy = completion_tokens > 0 ? total_entropy_sum / static_cast(completion_tokens) : 0.0f; + float confidence = 1.0f - mean_entropy; + + auto end_time = std::chrono::high_resolution_clock::now(); + double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + double prefill_tps = time_to_first_token > 0 ? (prompt_token_count * 1000.0) / time_to_first_token : 0.0; + double decode_time_ms = std::max(0.0, total_time_ms - time_to_first_token); + double decode_tps = (completion_tokens > 1 && decode_time_ms > 0.0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; + + if (!final_text.empty() && final_text[0] == ' ') + final_text.erase(0, 1); + + // Strip thinking channel content: <|channel>thought... + { + auto chan_start = final_text.find("<|channel>"); + auto chan_end = final_text.find(""); + if (chan_start != std::string::npos && chan_end != std::string::npos && chan_end > chan_start) { + size_t after = chan_end + std::string("").length(); + final_text = final_text.substr(after); + if (!final_text.empty() && final_text[0] == ' ') + final_text.erase(0, 1); + } + } + + const bool cloud_handoff = !final_text.empty() && final_text.length() > 5 && + cloud_handoff_threshold > 0.0f && max_token_entropy_norm > cloud_handoff_threshold; + + std::string json = construct_response_json(final_text, {}, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_token_count, completion_tokens, confidence, cloud_handoff); + + if (json.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); + return -1; + } + + cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, time_to_first_token, decode_tps, total_time_ms, static_cast(completion_tokens), get_ram_usage_mb(), ""); + std::strcpy(response_buffer, json.c_str()); + return static_cast(json.size()); + } + + auto to_sec = [](size_t samples) { + return static_cast(samples) / static_cast(WHISPER_SAMPLE_RATE); + }; + + struct AudioChunk { + std::vector audio; + float start_sec; + float end_sec; + std::vector> anchors; + }; + std::vector audio_chunks; + + if (options.use_vad) { + auto* vad = static_cast(handle->vad_model.get()); + auto vad_segments = vad->get_speech_timestamps(audio_samples, {}); + audio_chunks.reserve(vad_segments.size()); + + std::vector current; + std::vector> current_anchors; + size_t chunk_start_sample = 0; + size_t chunk_end_sample = 0; + float concat_cursor = 0.0f; + for (const auto& seg : vad_segments) { + size_t end = std::min(seg.end, audio_samples.size()); + if (current.size() + (end - seg.start) > MAX_CHUNK_SAMPLES) { + audio_chunks.emplace_back(std::move(current), to_sec(chunk_start_sample), to_sec(chunk_end_sample), std::move(current_anchors)); + current.clear(); + current_anchors.clear(); + concat_cursor = 0.0f; + } + if (current.empty()) chunk_start_sample = seg.start; + chunk_end_sample = end; + current_anchors.emplace_back(concat_cursor, to_sec(seg.start)); + concat_cursor += to_sec(end - seg.start); + current.insert( + current.end(), + audio_samples.begin() + seg.start, + audio_samples.begin() + end + ); + } + + if (!current.empty()) { + audio_chunks.emplace_back(std::move(current), to_sec(chunk_start_sample), to_sec(chunk_end_sample), std::move(current_anchors)); + } + + if (audio_chunks.empty()) { + CACTUS_LOG_DEBUG("transcribe", "VAD detected only silence, returning empty transcription"); + auto vad_end_time = std::chrono::high_resolution_clock::now(); + double vad_total_time_ms = std::chrono::duration_cast(vad_end_time - start_time).count() / 1000.0; + std::string json = construct_response_json("", {}, 0.0, vad_total_time_ms, 0.0, 0.0, 0, 0, 1.0f); + if (json.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); + return -1; + } + cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, 0.0, 0.0, vad_total_time_ms, 0, get_ram_usage_mb(), ""); + std::strcpy(response_buffer, json.c_str()); + return static_cast(json.size()); + } + } else { + audio_chunks.reserve((audio_samples.size() + MAX_CHUNK_SAMPLES - 1) / MAX_CHUNK_SAMPLES); + for (size_t start = 0; start < audio_samples.size(); start += MAX_CHUNK_SAMPLES) { + size_t end = std::min(start + MAX_CHUNK_SAMPLES, audio_samples.size()); + audio_chunks.emplace_back(std::vector(audio_samples.begin() + start, audio_samples.begin() + end), to_sec(start), to_sec(end)); + } + } + + auto cfg = is_parakeet ? get_parakeet_spectrogram_config() : get_whisper_spectrogram_config(); + size_t mel_bins = std::max(1, static_cast(handle->model->get_config().num_mel_bins)); + const bool is_whisper_v3 = is_whisper && mel_bins > 80; + AudioProcessor ap; + if (is_parakeet) { + ap.init_mel_filters(cfg.n_fft / 2 + 1, mel_bins, 0.0f, 8000.0f, WHISPER_SAMPLE_RATE); + } else if (!is_moonshine) { + init_whisper_mel_filters(ap, cfg, mel_bins); + } + + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) { + CACTUS_LOG_ERROR("transcribe", "Tokenizer unavailable"); + handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); + cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Tokenizer unavailable"); + return -1; + } + + std::vector initial_tokens = tokenizer->encode( + prompt && prompt[0] != '\0' + ? std::string(prompt) + : (is_whisper ? "<|startoftranscript|>" : std::string()) + ); + if (initial_tokens.empty() && !is_moonshine && !is_parakeet) { + CACTUS_LOG_ERROR("transcribe", "Decoder input tokens empty after encoding prompt"); + handle_error_response("Decoder input tokens empty", response_buffer, buffer_size); + cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Decoder input tokens empty"); + return -1; + } + + float max_tps = handle->model->get_config().default_max_tps; + if (max_tps < 0) max_tps = 100; + + std::vector> stop_token_sequences = {{ tokenizer->get_eos_token() }}; + auto append_exact_stop_sequence = [&](const char* stop_text) { + std::vector seq = tokenizer->encode(stop_text); + if (!seq.empty() && tokenizer->decode(seq) == stop_text) { + stop_token_sequences.push_back(std::move(seq)); + } + }; + append_exact_stop_sequence("<|endoftext|>"); + append_exact_stop_sequence("<|endoftranscript|>"); + append_exact_stop_sequence(""); + append_exact_stop_sequence(""); + + double time_to_first_token = 0.0; + size_t completion_tokens = 0; + std::string final_text; + float total_entropy_sum = 0.0f; + float max_token_entropy_norm = 0.0f; + + const std::regex whisper_timestamp_re(R"(<\|(\d+(?:\.\d+)?)\|>)"); + std::vector segments; + + auto sop = tokenizer->encode("<|startofprev|>"); + auto sot = tokenizer->encode("<|startoftranscript|>"); + auto zero = tokenizer->encode("<|0.00|>"); + auto notimestamps = tokenizer->encode("<|notimestamps|>"); + const bool has_notimestamps = !notimestamps.empty() && + std::search(initial_tokens.begin(), initial_tokens.end(), + notimestamps.begin(), notimestamps.end()) != initial_tokens.end(); + auto sot_it = std::search(initial_tokens.begin(), initial_tokens.end(), sot.begin(), sot.end()); + const auto sot_begin = sot_it != initial_tokens.end() ? sot_it : initial_tokens.begin(); + + for (auto& audio_chunk : audio_chunks) { + if (handle->should_stop || completion_tokens >= options.max_tokens) break; + + std::vector chunk_audio = std::move(audio_chunk.audio); + const float audio_chunk_length_sec = static_cast(chunk_audio.size()) / static_cast(WHISPER_SAMPLE_RATE); + + std::vector tokens; + if (final_text.empty() || is_parakeet || is_moonshine) { + tokens = initial_tokens; + } else { + size_t word_count = 0, pos = final_text.size(); + while (pos > 0 && word_count < MAX_CONTEXT_WORDS) { + while (pos > 0 && std::isspace((unsigned char)final_text[pos - 1])) --pos; + while (pos > 0 && !std::isspace((unsigned char)final_text[pos - 1])) --pos; + ++word_count; + } + tokens = sop; + auto ctx = tokenizer->encode(final_text.substr(pos)); + tokens.insert(tokens.end(), ctx.begin(), ctx.end()); + tokens.insert(tokens.end(), sot_begin, initial_tokens.end()); + } + + if (is_whisper && !has_notimestamps && !zero.empty() && (tokens.empty() || tokens.back() != zero.back())) { + tokens.insert(tokens.end(), zero.begin(), zero.end()); + } + + if (!is_moonshine) { + if (is_parakeet) { + size_t waveform_samples = chunk_audio.size(); + apply_preemphasis(chunk_audio, 0.97f); + chunk_audio = ap.compute_spectrogram(chunk_audio, cfg); + normalize_parakeet_log_mel(chunk_audio, mel_bins); + size_t valid_frames = waveform_samples / cfg.hop_length; + if (valid_frames == 0) valid_frames = 1; + trim_mel_frames(chunk_audio, mel_bins, valid_frames); + } else { + std::vector mel = ap.compute_spectrogram(chunk_audio, cfg); + chunk_audio = normalize_whisper_mel(mel, mel_bins, is_whisper_v3); + } + } + + if (chunk_audio.empty()) { + CACTUS_LOG_DEBUG("transcribe", "Chunk audio features empty, skipping"); + continue; + } + + CACTUS_LOG_DEBUG("transcribe", "Chunk audio features size: " << chunk_audio.size()); + + size_t chunk_max_tokens = options.max_tokens - completion_tokens; + if (!is_parakeet) { + size_t max_allowed = tokens.size() < WHISPER_MAX_DECODER_POSITIONS ? + WHISPER_MAX_DECODER_POSITIONS - tokens.size() : 0; + if (chunk_max_tokens > max_allowed) chunk_max_tokens = max_allowed; + } + size_t max_tps_tokens = std::max(1, static_cast(audio_chunk_length_sec * max_tps)); + if (chunk_max_tokens > max_tps_tokens) chunk_max_tokens = max_tps_tokens; + + tokens.reserve(tokens.size() + chunk_max_tokens); + std::vector generated_tokens; + generated_tokens.reserve(chunk_max_tokens); + + float segment_start = audio_chunk.start_sec; + float segment_end = audio_chunk.end_sec; + float orig_offset = audio_chunk.start_sec; + size_t anchor_idx = 0; + std::string segment_text; + + for (size_t i = 0; i < chunk_max_tokens; ++i) { + if (handle->should_stop) break; + + float token_entropy = 0.0f; + float tok_time_start = 0.0f; + float tok_time_end = 0.0f; + uint32_t next_token = handle->model->decode_with_audio( + tokens, + chunk_audio, + options.temperature, + options.top_p, + options.top_k, + "", + &token_entropy, + options.min_p, + options.repetition_penalty, + is_parakeet_tdt ? &tok_time_start : nullptr, + is_parakeet_tdt ? &tok_time_end : nullptr + ); + + if (completion_tokens == 0) [[unlikely]] { + auto t_first = std::chrono::high_resolution_clock::now(); + time_to_first_token = std::chrono::duration_cast(t_first - start_time).count() / 1000.0; + } + + total_entropy_sum += token_entropy; + if (token_entropy > max_token_entropy_norm) max_token_entropy_norm = token_entropy; + + generated_tokens.emplace_back(next_token); + if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { + break; + } + + std::string piece = tokenizer->decode({ next_token }); + if (is_terminal_transcription_piece(piece)) { + break; + } + tokens.emplace_back(next_token); + completion_tokens++; + + std::smatch timestamp_match; + if (is_parakeet_tdt) { + while (anchor_idx + 1 < audio_chunk.anchors.size() && audio_chunk.anchors[anchor_idx + 1].first <= tok_time_end) { + ++anchor_idx; + orig_offset = audio_chunk.anchors[anchor_idx].second - audio_chunk.anchors[anchor_idx].first; + } + const bool new_word = !piece.empty() && piece[0] == ' '; + if (new_word && !segment_text.empty()) { + final_text += segment_text; + segments.emplace_back( + segment_start, + segment_end, + segment_text[0] == ' ' + ? (segment_text.erase(0, 1), std::move(segment_text)) + : std::move(segment_text) + ); + segment_text.clear(); + } + if (segment_text.empty()) segment_start = tok_time_start + orig_offset; + segment_text += piece; + segment_end = tok_time_end + orig_offset; + if (callback) callback(piece.c_str(), next_token, user_data); + continue; + } else if (std::regex_match(piece, timestamp_match, whisper_timestamp_re)) { + const float ts_sec = std::stof(timestamp_match[1].str()); + while (anchor_idx + 1 < audio_chunk.anchors.size() && audio_chunk.anchors[anchor_idx + 1].first <= ts_sec) { + ++anchor_idx; + orig_offset = audio_chunk.anchors[anchor_idx].second - audio_chunk.anchors[anchor_idx].first; + } + segment_end = std::min(ts_sec + orig_offset, audio_chunk.end_sec); + if (!segment_text.empty()) { + final_text += segment_text; + segments.emplace_back( + segment_start, + segment_end, + segment_text[0] == ' ' + ? (segment_text.erase(0, 1), std::move(segment_text)) + : std::move(segment_text) + ); + segment_text.clear(); + } + segment_start = segment_end; + continue; + } + + segment_text += piece; + if (callback) callback(piece.c_str(), next_token, user_data); + } + + if (!segment_text.empty()) { + final_text += segment_text; + segments.emplace_back( + segment_start, + segment_end, + segment_text[0] == ' ' + ? (segment_text.erase(0, 1), std::move(segment_text)) + : std::move(segment_text) + ); + } + + cactus_reset(model); + } + + float mean_entropy = completion_tokens > 0 ? total_entropy_sum / static_cast(completion_tokens) : 0.0f; + float confidence = 1.0f - mean_entropy; + + auto end_time = std::chrono::high_resolution_clock::now(); + double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + size_t prompt_tokens = initial_tokens.size(); + double prefill_tps = time_to_first_token > 0 ? (prompt_tokens * 1000.0) / time_to_first_token : 0.0; + double decode_time_ms = std::max(0.0, total_time_ms - time_to_first_token); + double decode_tps = (completion_tokens > 1 && decode_time_ms > 0.0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; + + if (!final_text.empty() && final_text[0] == ' ') final_text.erase(0, 1); + + if (!custom_vocabulary.empty()) { + apply_vocabulary_spelling_correction(final_text, custom_vocabulary); + for (auto& seg : segments) { + apply_vocabulary_spelling_correction(seg.text, custom_vocabulary); + } + } + + const bool cloud_handoff = !final_text.empty() && final_text.length() > 5 && + cloud_handoff_threshold > 0.0f && max_token_entropy_norm > cloud_handoff_threshold; + + std::string json = construct_response_json(final_text, {}, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_tokens, completion_tokens, confidence, cloud_handoff, "", segments); + + if (json.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); + return -1; + } + + cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, time_to_first_token, decode_tps, total_time_ms, static_cast(completion_tokens), get_ram_usage_mb(), ""); + + std::strcpy(response_buffer, json.c_str()); + + return static_cast(json.size()); + } + catch (const std::exception& e) { + CACTUS_LOG_ERROR("transcribe", "Exception: " << e.what()); + handle_error_response(e.what(), response_buffer, buffer_size); + cactus::telemetry::recordTranscription(model ? static_cast(model)->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, e.what()); + return -1; + } + catch (...) { + CACTUS_LOG_ERROR("transcribe", "Unknown exception during transcription"); + handle_error_response("Unknown error in transcribe", response_buffer, buffer_size); + cactus::telemetry::recordTranscription(model ? static_cast(model)->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, "Unknown error in transcribe"); + return -1; + } +} + +int cactus_detect_language( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +) { + if (!model) { + std::string error_msg = last_error_message.empty() ? "Model not initialized." : last_error_message; + CACTUS_LOG_ERROR("detect_language", error_msg); + handle_error_response(error_msg, response_buffer, buffer_size); + return -1; + } + if (!response_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR("detect_language", "Invalid parameters: response_buffer or buffer_size"); + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { + CACTUS_LOG_ERROR("detect_language", "No audio input provided"); + handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); + return -1; + } + if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { + CACTUS_LOG_ERROR("detect_language", "Both audio_file_path and pcm_buffer provided"); + handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); + return -1; + } + if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { + CACTUS_LOG_ERROR("detect_language", "Invalid pcm_buffer_size: " << pcm_buffer_size); + handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); + return -1; + } + + try { + auto start_time = std::chrono::high_resolution_clock::now(); + auto* handle = static_cast(model); + std::lock_guard lock(handle->model_mutex); + handle->should_stop = false; + + if (handle->model->get_config().model_type != cactus::engine::Config::ModelType::WHISPER) { + handle_error_response("Language detection currently requires a Whisper model", response_buffer, buffer_size); + return -1; + } + + InferenceOptions options = parse_inference_options_json(options_json ? options_json : ""); + + std::vector audio_buffer; + if (audio_file_path == nullptr) { + const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); + size_t num_samples = pcm_buffer_size / 2; + + std::vector waveform_fp32(num_samples); + for (size_t i = 0; i < num_samples; i++) { + waveform_fp32[i] = static_cast(pcm_samples[i]) / 32768.0f; + } + audio_buffer = resample_to_16k_fp32(waveform_fp32, WHISPER_SAMPLE_RATE); + } else { + AudioFP32 audio = load_wav(audio_file_path); + audio_buffer = resample_to_16k_fp32(audio.samples, audio.sample_rate); + } + + if (options.use_vad && handle->vad_model) { + auto* vad = static_cast(handle->vad_model.get()); + auto segments = vad->get_speech_timestamps(audio_buffer, {}); + + std::vector speech_audio; + for (const auto& segment : segments) { + speech_audio.insert( + speech_audio.end(), + audio_buffer.begin() + segment.start, + audio_buffer.begin() + std::min(segment.end, audio_buffer.size()) + ); + } + audio_buffer = std::move(speech_audio); + } + + if (audio_buffer.empty()) { + handle_error_response("No speech detected in audio input", response_buffer, buffer_size); + return -1; + } + + AudioProcessor ap; + auto cfg = get_whisper_spectrogram_config(); + const size_t mel_bins = std::max(1, static_cast(handle->model->get_config().num_mel_bins)); + init_whisper_mel_filters(ap, cfg, mel_bins); + std::vector mel = ap.compute_spectrogram(audio_buffer, cfg); + std::vector features = normalize_whisper_mel(mel, mel_bins, mel_bins > 80); + if (features.empty()) { + handle_error_response("Computed audio features are empty", response_buffer, buffer_size); + return -1; + } + + auto* tokenizer = handle->model->get_tokenizer(); + if (!tokenizer) { + handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); + return -1; + } + + std::vector decode_tokens = tokenizer->encode("<|startoftranscript|>"); + if (decode_tokens.empty()) { + handle_error_response("Failed to encode Whisper start token", response_buffer, buffer_size); + return -1; + } + + handle->model->reset_cache(); + float entropy = 1.0f; + uint32_t token_id = 0; + std::string token_text; + std::string language; + + // Whisper may emit one or more control tokens before the language token. + // Decode a few initial tokens and select the first valid language token. + constexpr size_t kMaxLanguageProbeSteps = 4; + for (size_t step = 0; step < kMaxLanguageProbeSteps; ++step) { + float step_entropy = 1.0f; + const uint32_t step_token_id = handle->model->decode_with_audio( + decode_tokens, features, 0.0f, 0.0f, 0, "", &step_entropy, + 0.0f, 1.0f + ); + const std::string step_token_text = tokenizer->decode({step_token_id}); + const std::string step_language = extract_whisper_language_code(step_token_text); + + token_id = step_token_id; + token_text = step_token_text; + entropy = step_entropy; + + if (!step_language.empty()) { + language = step_language; + break; + } + + decode_tokens.push_back(step_token_id); + } + handle->model->reset_cache(); + + if (language.empty()) { + language = "unknown"; + } + + float confidence = 1.0f - entropy; + if (confidence < 0.0f) confidence = 0.0f; + if (confidence > 1.0f) confidence = 1.0f; + + auto end_time = std::chrono::high_resolution_clock::now(); + double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"language\":\"" << escape_json_string(language) << "\","; + json << "\"language_token\":\"" << escape_json_string(token_text) << "\","; + json << "\"token_id\":" << token_id << ","; + json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; + json << "\"entropy\":" << std::fixed << std::setprecision(4) << entropy << ","; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); + json << "}"; + + std::string response = json.str(); + if (response.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + + std::strcpy(response_buffer, response.c_str()); + return static_cast(response.size()); + } catch (const std::exception& e) { + CACTUS_LOG_ERROR("detect_language", "Exception: " << e.what()); + handle_error_response(e.what(), response_buffer, buffer_size); + return -1; + } catch (...) { + CACTUS_LOG_ERROR("detect_language", "Unknown exception during language detection"); + handle_error_response("Unknown error in detect_language", response_buffer, buffer_size); + return -1; + } +} + +} diff --git a/cactus-transpile/ffi/cactus_utils.h b/cactus-transpile/ffi/cactus_utils.h new file mode 100644 index 000000000..925a7c99c --- /dev/null +++ b/cactus-transpile/ffi/cactus_utils.h @@ -0,0 +1,1850 @@ +#ifndef CACTUS_UTILS_H +#define CACTUS_UTILS_H + +#include "../engine/engine.h" +#include "../models/model.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#elif defined(_WIN32) +#include +#include +#elif defined(__linux__) || defined(__ANDROID__) +#include +#endif + +inline size_t get_memory_footprint_bytes() { +#ifdef __APPLE__ + task_vm_info_data_t vm_info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count) == KERN_SUCCESS) + return vm_info.phys_footprint; + +#elif defined(_WIN32) + PROCESS_MEMORY_COUNTERS_EX pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) + return pmc.PrivateUsage; + +#elif defined(__linux__) || defined(__ANDROID__) + std::ifstream statm("/proc/self/statm"); + if (statm.is_open()) { + size_t size, resident; + statm >> size >> resident; + return resident * sysconf(_SC_PAGESIZE); + } +#endif + return 0; +} + +inline double get_ram_usage_mb() { + return get_memory_footprint_bytes() / (1024.0 * 1024.0); +} + +struct CactusModelHandle { + std::unique_ptr model; + std::unique_ptr vad_model; + std::atomic should_stop; + std::vector processed_tokens; + struct ProcessedImage { + std::string path; + long long last_modified_timestamp = 0; + + bool operator==(const ProcessedImage& other) const { + return path == other.path && last_modified_timestamp == other.last_modified_timestamp; + } + }; + + std::vector> processed_images; + std::mutex model_mutex; + std::string model_name; + std::unique_ptr corpus_index; + std::string corpus_dir; + size_t corpus_embedding_dim = 0; + std::vector> tool_embeddings; + std::vector tool_texts; + + CactusModelHandle() : should_stop(false) {} +}; + +extern std::string last_error_message; + +bool matches_stop_sequence(const std::vector& generated_tokens, + const std::vector>& stop_sequences); + +std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query); + +namespace cactus { +namespace audio { + +static constexpr size_t WHISPER_TARGET_FRAMES = 3000; +static constexpr int WHISPER_SAMPLE_RATE = 16000; + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_whisper_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 400; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "reflect"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1e-10f; + cfg.log_mel = "log10"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_parakeet_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 5.960464477539063e-08f; // 2^-24 guard value used by HF Parakeet. + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = false; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_htk_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 321; + cfg.frame_length = 320; + cfg.fft_override = 1024; + cfg.hop_length = 160; + cfg.power = 1.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 0.001f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 0.001f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_gemma4_audio_spectrogram_config( + const cactus::engine::Config& model_config) { + auto cfg = get_htk_spectrogram_config(); + cfg.fft_override = model_config.audio_fft_length; + cfg.mel_floor_additive = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_wespeaker_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1.1754944e-38f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1.1754944e-38f; + cfg.remove_dc_offset = true; + cfg.preemphasis = 0.97f; + cfg.hann_periodic = false; + cfg.window_a0 = 0.54f; + return cfg; +} + +// Whisper v1/v2: 80 mel bins, HTK. Whisper v3: 128 mel bins, Slaney, 512-FFT, no DC removal. +inline void init_whisper_mel_filters(cactus::engine::AudioProcessor& ap, + cactus::engine::AudioProcessor::SpectrogramConfig& cfg, + size_t mel_bins) { + const size_t num_mel_filters = std::max(1, mel_bins); + const bool is_v3 = mel_bins > 80; + if (is_v3) { + cfg.fft_override = 512; + cfg.remove_dc_offset = false; + } + const size_t fft_len = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + const size_t num_frequency_bins = fft_len / 2 + 1; + if (is_v3) { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE, "slaney", "slaney"); + } else { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE); + } +} + +// use_mel_floor_padding=true pads short audio with the normalized mel floor (required for v3). +inline std::vector normalize_whisper_mel(std::vector& mel, size_t n_mels, + bool use_mel_floor_padding = false) { + if (mel.empty() || n_mels == 0) return mel; + size_t n_frames = mel.size() / n_mels; + + float max_val = -std::numeric_limits::infinity(); + for (float v : mel) if (v > max_val) max_val = v; + + float min_allowed = max_val - 8.0f; + for (float& v : mel) { + if (v < min_allowed) v = min_allowed; + v = (v + 4.0f) * 0.25f; + } + + if (n_frames != WHISPER_TARGET_FRAMES) { + float pad_val = use_mel_floor_padding ? (min_allowed + 4.0f) * 0.25f : 0.0f; + std::vector fixed(n_mels * WHISPER_TARGET_FRAMES, pad_val); + size_t copy_frames = std::min(n_frames, WHISPER_TARGET_FRAMES); + for (size_t m = 0; m < n_mels; ++m) { + const float* src = &mel[m * n_frames]; + float* dst = &fixed[m * WHISPER_TARGET_FRAMES]; + std::copy(src, src + copy_frames, dst); + } + return fixed; + } + return std::move(mel); +} + +inline std::vector transpose_mel_to_frame_major(const std::vector& mel, + size_t num_mels, size_t num_frames) { + std::vector transposed(num_frames * num_mels); + for (size_t m = 0; m < num_mels; m++) { + for (size_t t = 0; t < num_frames; t++) { + transposed[t * num_mels + m] = mel[m * num_frames + t]; + } + } + return transposed; +} + +inline void apply_preemphasis(std::vector& waveform, float coefficient = 0.97f) { + if (waveform.size() < 2 || coefficient == 0.0f) { + return; + } + for (size_t i = waveform.size() - 1; i > 0; --i) { + waveform[i] -= coefficient * waveform[i - 1]; + } +} + +inline void normalize_parakeet_log_mel(std::vector& mel, size_t num_mels, float epsilon = 1e-5f) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + const size_t num_frames = mel.size() / num_mels; + if (num_frames == 0) { + return; + } + + for (size_t m = 0; m < num_mels; ++m) { + const size_t base = m * num_frames; + float mean = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + mean += mel[base + t]; + } + mean /= static_cast(num_frames); + + float variance = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + const float d = mel[base + t] - mean; + variance += d * d; + } + const float denom = static_cast(std::max(1, num_frames - 1)); + const float inv_std = 1.0f / std::sqrt((variance / denom) + epsilon); + for (size_t t = 0; t < num_frames; ++t) { + mel[base + t] = (mel[base + t] - mean) * inv_std; + } + } +} + +inline void trim_mel_frames(std::vector& mel, size_t num_mels, size_t valid_frames) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + size_t total_frames = mel.size() / num_mels; + if (valid_frames == 0 || valid_frames >= total_frames) { + return; + } + std::vector trimmed(num_mels * valid_frames); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &mel[m * total_frames]; + float* dst = &trimmed[m * valid_frames]; + std::copy(src, src + valid_frames, dst); + } + mel.swap(trimmed); +} + +struct AudioPreprocessResult { + std::vector features; + size_t num_frames = 0; + size_t num_soft_tokens = 0; +}; + +inline AudioPreprocessResult preprocess_audio_for_gemma4( + std::vector audio_samples, + const cactus::engine::Config& model_config +) { + AudioPreprocessResult result; + if (audio_samples.empty()) return result; + + size_t pad_amt = 320 - (audio_samples.size() % 320); + if (pad_amt < 320) + audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); + + size_t mel_bins = model_config.audio_input_feat_size; + auto cfg = get_gemma4_audio_spectrogram_config(model_config); + + size_t semicausal_pad = cfg.frame_length / 2; + audio_samples.insert(audio_samples.begin(), semicausal_pad, 0.0f); + + cactus::engine::AudioProcessor ap; + size_t fft_for_mel = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + ap.init_mel_filters(fft_for_mel / 2 + 1, mel_bins, 0.0f, 8000.0f, 16000, + nullptr, "htk"); + std::vector mel = ap.compute_spectrogram(audio_samples, cfg); + + result.num_frames = mel.size() / mel_bins; + result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); + + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + + return result; +} + +inline std::vector pcm_buffer_to_float_samples( + const uint8_t* pcm_buffer, size_t pcm_buffer_size +) { + const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); + size_t num_samples = pcm_buffer_size / 2; + std::vector waveform_fp32(num_samples); + constexpr float inv_32768 = 1.0f / 32768.0f; + for (size_t i = 0; i < num_samples; i++) + waveform_fp32[i] = static_cast(pcm_samples[i]) * inv_32768; + return waveform_fp32; +} + +} // namespace audio +} // namespace cactus + +namespace cactus { +namespace ffi { + +inline bool env_flag_enabled(const char* key) { + const char* value = std::getenv(key); + return value && value[0] != '\0' && !(value[0] == '0' && value[1] == '\0'); +} + +inline std::string generateUUID() { +#ifdef __APPLE__ + uuid_t uuid; + uuid_generate_random(uuid); + char uuid_str[37]; + uuid_unparse_lower(uuid, uuid_str); + return std::string(uuid_str); +#else + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_int_distribution<> dis(0, 15); + static std::uniform_int_distribution<> dis2(8, 11); + + std::stringstream ss; + ss << std::hex; + for (int i = 0; i < 8; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 4; i++) ss << dis(gen); + ss << "-4"; + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + ss << dis2(gen); + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 12; i++) ss << dis(gen); + return ss.str(); +#endif +} + +struct ToolFunction { + std::string name; + std::string description; + std::unordered_map parameters; +}; + +struct InferenceOptions { + float temperature = 0.0f; + float top_p = 0.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + float confidence_threshold = 0.7f; + size_t top_k = 0; + size_t max_tokens = 100; + size_t tool_rag_top_k = 2; + size_t cloud_timeout_ms = 15000; + std::vector stop_sequences; + bool force_tools = false; + bool include_stop_sequences = false; + bool use_vad = true; + bool telemetry_enabled = true; + bool auto_handoff = true; + bool handoff_with_images = true; + bool enable_thinking_if_supported = true; +}; + +} // namespace ffi +} // namespace cactus + +std::vector select_relevant_tools( + CactusModelHandle* handle, + const std::string& query, + const std::vector& all_tools, + size_t top_k); + +#include "gemma_tools.h" + +namespace cactus { +namespace ffi { + +inline std::string escape_json_string(const std::string& s) { + std::ostringstream o; + for (char c : s) { + if (c == '"') o << "\\\""; + else if (c == '\n') o << "\\n"; + else if (c == '\r') o << "\\r"; + else if (c == '\t') o << "\\t"; + else if (c == '\\') o << "\\\\"; + else o << c; + } + return o.str(); +} + + +inline std::string trim_string(const std::string& s) { + size_t start = 0; + while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; + size_t end = s.size(); + while (end > start && std::isspace(static_cast(s[end - 1]))) --end; + return s.substr(start, end - start); +} + +inline size_t find_matching_delimiter(const std::string& s, size_t pos, char open, char close) { + int depth = 1; + pos++; + while (pos < s.length() && depth > 0) { + if (s[pos] == open) depth++; + else if (s[pos] == close) depth--; + else if (s[pos] == '"') { + pos++; + while (pos < s.length() && s[pos] != '"') { + if (s[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + return pos; +} + +inline std::string env_or_default(const char* key, const char* fallback) { + const char* v = std::getenv(key); + if (v && v[0] != '\0') return std::string(v); + return std::string(fallback); +} + +inline std::string json_string_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return {}; + + size_t i = pos + pattern.size(); + while (i < json.size() && std::isspace(static_cast(json[i]))) i++; + if (i >= json.size() || json[i] != '"') return {}; + ++i; + + std::string out; + out.reserve(128); + while (i < json.size()) { + char c = json[i++]; + if (c == '"') return out; + if (c == '\\' && i < json.size()) { + char e = json[i++]; + switch (e) { + case '"': out.push_back('"'); break; + case '\\': out.push_back('\\'); break; + case '/': out.push_back('/'); break; + case 'b': out.push_back('\b'); break; + case 'f': out.push_back('\f'); break; + case 'n': out.push_back('\n'); break; + case 'r': out.push_back('\r'); break; + case 't': out.push_back('\t'); break; + default: out.push_back(e); break; + } + continue; + } + out.push_back(c); + } + return {}; +} + +inline std::string json_array_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return "[]"; + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return "[]"; + + int depth = 1; + size_t end = start + 1; + while (end < json.size() && depth > 0) { + if (json[end] == '[') depth++; + else if (json[end] == ']') depth--; + end++; + } + return json.substr(start, end - start); +} + +inline std::vector split_json_array(const std::string& array_json) { + std::vector out; + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) i++; + if (i + 1 >= array_json.size() || array_json[i] != '{') break; + + size_t start = i; + int depth = 0; + bool in_str = false; + bool esc = false; + for (; i < array_json.size(); ++i) { + char c = array_json[i]; + if (in_str) { + if (esc) esc = false; + else if (c == '\\') esc = true; + else if (c == '"') in_str = false; + continue; + } + if (c == '"') { in_str = true; continue; } + if (c == '{') depth++; + if (c == '}') { + depth--; + if (depth == 0) { + out.push_back(array_json.substr(start, i - start + 1)); + i++; + break; + } + } + } + } + return out; +} + +inline std::string serialize_tools_json(const std::vector& tools) { + if (tools.empty()) return ""; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < tools.size(); ++i) { + if (i > 0) oss << ","; + oss << "{\"type\":\"function\",\"function\":{"; + oss << "\"name\":\"" << escape_json_string(tools[i].name) << "\","; + oss << "\"description\":\"" << escape_json_string(tools[i].description) << "\""; + auto it = tools[i].parameters.find("schema"); + if (it != tools[i].parameters.end()) { + oss << ",\"parameters\":" << it->second; + } + oss << "}}"; + } + oss << "]"; + return oss.str(); +} + +namespace json_sorted { + +inline void skip_ws(const std::string& s, size_t& p) { + while (p < s.size() && std::isspace(static_cast(s[p]))) p++; +} + +inline std::string parse_string(const std::string& s, size_t& p) { + std::string r = "\""; + p++; + while (p < s.size()) { + if (s[p] == '\\') { + r += s[p++]; + if (p < s.size()) r += s[p++]; + } else if (s[p] == '"') { + r += '"'; + p++; + return r; + } else { + r += s[p++]; + } + } + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p); + +inline std::string parse_object(const std::string& s, size_t& p) { + p++; + std::map entries; + skip_ws(s, p); + while (p < s.size() && s[p] != '}') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + std::string key = parse_string(s, p); + skip_ws(s, p); + if (p < s.size() && s[p] == ':') p++; + skip_ws(s, p); + std::string val = parse_value(s, p); + entries[key] = val; + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "{"; + bool first = true; + for (const auto& kv : entries) { + if (!first) r += ", "; + r += kv.first + ": " + kv.second; + first = false; + } + r += "}"; + return r; +} + +inline std::string parse_array(const std::string& s, size_t& p) { + p++; + std::vector items; + skip_ws(s, p); + while (p < s.size() && s[p] != ']') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + items.push_back(parse_value(s, p)); + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "["; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) r += ", "; + r += items[i]; + } + r += "]"; + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p) { + skip_ws(s, p); + if (p >= s.size()) return ""; + if (s[p] == '"') return parse_string(s, p); + if (s[p] == '{') return parse_object(s, p); + if (s[p] == '[') return parse_array(s, p); + size_t start = p; + while (p < s.size() && s[p] != ',' && s[p] != '}' && s[p] != ']' && !std::isspace(static_cast(s[p]))) p++; + return s.substr(start, p - start); +} + +inline std::string reformat(const std::string& json) { + size_t p = 0; + return parse_value(json, p); +} + +} // namespace json_sorted + +inline std::string serialize_tools_for_template(const std::vector& tools) { + if (tools.empty()) return ""; + std::string result; + for (const auto& tool : tools) { + std::map func_fields; + func_fields["\"description\""] = "\"" + escape_json_string(tool.description) + "\""; + func_fields["\"name\""] = "\"" + escape_json_string(tool.name) + "\""; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + func_fields["\"parameters\""] = json_sorted::reformat(it->second); + } + std::string func_json = "{"; + bool first = true; + for (const auto& kv : func_fields) { + if (!first) func_json += ", "; + func_json += kv.first + ": " + kv.second; + first = false; + } + func_json += "}"; + result += "\n{\"function\": " + func_json + ", \"type\": \"function\"}"; + } + return result; +} + +inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) { + std::ostringstream json; + json << "{"; + json << "\"success\":false,"; + json << "\"error\":\"" << escape_json_string(error_message) << "\","; + json << "\"cloud_handoff\":false,"; + json << "\"response\":null,"; + json << "\"function_calls\":[],"; + json << "\"confidence\":0.0,"; + json << "\"time_to_first_token_ms\":0.0,"; + json << "\"total_time_ms\":0.0,"; + json << "\"prefill_tps\":0.0,"; + json << "\"decode_tps\":0.0,"; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":0,"; + json << "\"decode_tokens\":0,"; + json << "\"total_tokens\":0"; + json << "}"; + std::string error_json = json.str(); + if (response_buffer && error_json.length() < buffer_size) { + std::strcpy(response_buffer, error_json.c_str()); + } +} + +inline std::vector parse_messages_json(const std::string& json, + std::vector& out_image_paths, + std::vector* out_audio_paths = nullptr) { + std::vector messages; + out_image_paths.clear(); + if (out_audio_paths) out_audio_paths->clear(); + + size_t pos = json.find('['); + if (pos == std::string::npos) { + throw std::runtime_error("Invalid JSON: expected array"); + } + + pos = json.find('{', pos); + while (pos != std::string::npos) { + cactus::engine::ChatMessage msg; + + size_t obj_start = pos; + int brace_count = 1; + size_t obj_end = obj_start + 1; + while (obj_end < json.length() && brace_count > 0) { + if (json[obj_end] == '{') brace_count++; + else if (json[obj_end] == '}') brace_count--; + obj_end++; + } + + size_t role_pos = json.find("\"role\"", pos); + if (role_pos == std::string::npos || role_pos >= obj_end) break; + + size_t role_start = json.find('"', role_pos + 6) + 1; + size_t role_end = json.find('"', role_start); + msg.role = json.substr(role_start, role_end - role_start); + + size_t content_pos = json.find("\"content\"", role_end); + if (content_pos != std::string::npos && content_pos < obj_end) { + size_t content_start = json.find('"', content_pos + 9) + 1; + size_t content_end = content_start; + + while (content_end < json.length()) { + content_end = json.find('"', content_end); + if (content_end == std::string::npos) break; + if (json[content_end - 1] != '\\') break; + content_end++; + } + + msg.content = json.substr(content_start, content_end - content_start); + + size_t escape_pos = 0; + while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\n"); + escape_pos += 1; + } + escape_pos = 0; + while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\""); + escape_pos += 1; + } + } + + auto parse_path_array = [&](const char* key, std::vector& dest, + std::vector* out_paths) { + size_t key_pos = json.find(key, pos); + if (key_pos == std::string::npos || key_pos >= obj_end) return; + size_t array_start = json.find('[', key_pos); + if (array_start == std::string::npos || array_start >= obj_end) return; + size_t array_end = json.find(']', array_start); + if (array_end == std::string::npos || array_end >= obj_end) return; + size_t cur = array_start; + while (true) { + cur = json.find('"', cur + 1); + if (cur == std::string::npos || cur >= array_end) break; + size_t str_start = cur + 1; + size_t str_end = json.find('"', str_start); + if (str_end == std::string::npos || str_end > array_end) break; + std::string path = std::filesystem::absolute( + std::filesystem::path(json.substr(str_start, str_end - str_start))).string(); + dest.push_back(path); + if (out_paths) out_paths->push_back(path); + cur = str_end; + } + }; + + parse_path_array("\"images\"", msg.images, &out_image_paths); + parse_path_array("\"audio\"", msg.audio, out_audio_paths); + + if (msg.role == "tool") { + size_t name_pos = json.find("\"name\"", obj_start); + if (name_pos != std::string::npos && name_pos < obj_end) { + size_t name_quote = json.find('"', name_pos + 6); + if (name_quote != std::string::npos && name_quote < obj_end) { + size_t name_start = name_quote + 1; + size_t name_end = json.find('"', name_start); + if (name_end != std::string::npos && name_end < obj_end) { + msg.name = json.substr(name_start, name_end - name_start); + } + } + } + } + + size_t tool_calls_pos = json.find("\"tool_calls\"", obj_start); + if (tool_calls_pos != std::string::npos && tool_calls_pos < obj_end) { + size_t tool_calls_arr_start = json.find('[', tool_calls_pos); + if (tool_calls_arr_start != std::string::npos && tool_calls_arr_start < obj_end) { + size_t tool_calls_arr_end = find_matching_delimiter(json, tool_calls_arr_start, '[', ']'); + + size_t search_pos = tool_calls_arr_start; + while (true) { + size_t func_pos = json.find("\"function\"", search_pos); + if (func_pos == std::string::npos || func_pos >= tool_calls_arr_end) break; + + size_t func_obj_start = json.find('{', func_pos + 10); + if (func_obj_start == std::string::npos || func_obj_start >= tool_calls_arr_end) break; + + size_t func_obj_end = find_matching_delimiter(json, func_obj_start, '{', '}'); + + cactus::engine::ToolCallInfo tool_call; + + size_t fn_name_pos = json.find("\"name\"", func_obj_start); + if (fn_name_pos != std::string::npos && fn_name_pos < func_obj_end) { + size_t fn_name_quote = json.find('"', fn_name_pos + 6); + if (fn_name_quote != std::string::npos && fn_name_quote < func_obj_end) { + size_t fn_name_start = fn_name_quote + 1; + size_t fn_name_end = json.find('"', fn_name_start); + if (fn_name_end != std::string::npos && fn_name_end < func_obj_end) { + tool_call.name = json.substr(fn_name_start, fn_name_end - fn_name_start); + } + } + } + + size_t args_pos = json.find("\"arguments\"", func_obj_start); + if (args_pos != std::string::npos && args_pos < func_obj_end) { + size_t colon_pos = json.find(':', args_pos + 11); + if (colon_pos != std::string::npos && colon_pos < func_obj_end) { + size_t args_start = colon_pos + 1; + while (args_start < json.length() && std::isspace(static_cast(json[args_start]))) args_start++; + + if (args_start < func_obj_end && json[args_start] == '{') { + size_t args_end = find_matching_delimiter(json, args_start, '{', '}'); + tool_call.arguments = json.substr(args_start, args_end - args_start); + } else if (args_start < func_obj_end && json[args_start] == '"') { + size_t str_start = args_start + 1; + size_t str_end = str_start; + while (str_end < json.length() && json[str_end] != '"') { + if (json[str_end] == '\\') str_end++; + str_end++; + } + tool_call.arguments = json.substr(str_start, str_end - str_start); + } + } + } + + if (!tool_call.name.empty()) { + msg.tool_calls.push_back(tool_call); + } + search_pos = func_obj_end; + } + } + } + + messages.push_back(msg); + + pos = json.find('{', obj_end); + } + + return messages; +} + +inline std::vector parse_tools_json(const std::string& json) { + std::vector tools; + + if (json.empty()) return tools; + + size_t pos = json.find('['); + if (pos == std::string::npos) return tools; + + pos = json.find("\"function\"", pos); + while (pos != std::string::npos) { + ToolFunction tool; + + size_t name_pos = json.find("\"name\"", pos); + if (name_pos != std::string::npos) { + size_t name_start = json.find('"', name_pos + 6) + 1; + size_t name_end = json.find('"', name_start); + tool.name = json.substr(name_start, name_end - name_start); + } + + size_t desc_pos = json.find("\"description\"", pos); + if (desc_pos != std::string::npos) { + size_t desc_start = json.find('"', desc_pos + 13) + 1; + size_t desc_end = json.find('"', desc_start); + tool.description = json.substr(desc_start, desc_end - desc_start); + } + + size_t params_pos = json.find("\"parameters\"", pos); + if (params_pos != std::string::npos) { + size_t params_start = json.find('{', params_pos); + if (params_start != std::string::npos) { + int brace_count = 1; + size_t params_end = params_start + 1; + while (params_end < json.length() && brace_count > 0) { + if (json[params_end] == '{') brace_count++; + else if (json[params_end] == '}') brace_count--; + params_end++; + } + tool.parameters["schema"] = json.substr(params_start, params_end - params_start); + } + } + + tools.push_back(tool); + + pos = json.find("\"function\"", name_pos); + } + + return tools; +} + +inline bool try_parse_json_float(const std::string& json, const std::string& key, float& out_value) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return false; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + + size_t end = start; + while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; + + try { + out_value = std::stof(json.substr(start, end - start)); + return true; + } catch (...) { + return false; + } +} + +inline std::vector parse_json_string_array_field(const std::string& json, const std::string& key) { + std::vector out; + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return out; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return out; + + int depth = 1; + bool in_string = false; + bool escaped = false; + size_t end = start + 1; + + while (end < json.size() && depth > 0) { + char c = json[end]; + if (in_string) { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == '"') in_string = false; + } else { + if (c == '"') in_string = true; + else if (c == '[') depth++; + else if (c == ']') depth--; + } + ++end; + } + + if (depth != 0) return out; + const std::string array_json = json.substr(start, end - start); + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) { + ++i; + } + if (i + 1 >= array_json.size() || array_json[i] == ']') break; + if (array_json[i] != '"') break; + + ++i; + std::string value; + bool escaped = false; + while (i < array_json.size()) { + char c = array_json[i++]; + if (escaped) { + switch (c) { + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + case '/': value.push_back('/'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + default: value.push_back(c); break; + } + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') { + out.push_back(value); + break; + } + value.push_back(c); + } + } + + return out; +} + +inline void parse_custom_vocabulary_options(const std::string& json, + std::vector& custom_vocabulary, + float& vocabulary_boost) { + custom_vocabulary.clear(); + vocabulary_boost = 5.0f; + if (json.empty()) return; + + float parsed_boost = vocabulary_boost; + if (try_parse_json_float(json, "vocabulary_boost", parsed_boost)) { + vocabulary_boost = std::clamp(parsed_boost, 0.0f, 20.0f); + } + + custom_vocabulary = parse_json_string_array_field(json, "custom_vocabulary"); +} + +inline std::unordered_map build_token_bias_map(const std::vector>& tokenized_entries, + float vocabulary_boost) { + std::unordered_map vocab_bias; + const float clamped_boost = std::clamp(vocabulary_boost, 0.0f, 20.0f); + if (clamped_boost == 0.0f) return vocab_bias; + + for (const auto& token_ids : tokenized_entries) { + for (uint32_t token_id : token_ids) { + float& entry = vocab_bias[token_id]; + if (entry < clamped_boost) { + entry = clamped_boost; + } + } + } + + return vocab_bias; +} + +inline std::unordered_map build_custom_vocabulary_bias(cactus::engine::Tokenizer* tokenizer, + const std::vector& custom_vocabulary, + float vocabulary_boost) { + if (!tokenizer || custom_vocabulary.empty()) return {}; + std::vector> tokenized_entries; + tokenized_entries.reserve(custom_vocabulary.size()); + + for (const auto& word : custom_vocabulary) { + if (word.empty()) continue; + tokenized_entries.push_back(tokenizer->encode(word)); + } + + return build_token_bias_map(tokenized_entries, vocabulary_boost); +} + +inline void apply_custom_vocabulary_options(cactus::engine::Model* model, const std::string& json) { + if (!model) return; + + std::vector custom_vocabulary; + float vocabulary_boost = 5.0f; + parse_custom_vocabulary_options(json, custom_vocabulary, vocabulary_boost); + model->set_vocab_bias(build_custom_vocabulary_bias(model->get_tokenizer(), custom_vocabulary, vocabulary_boost)); +} + +inline size_t levenshtein_ci(const std::string& a, const std::string& b) { + const size_t m = a.size(), n = b.size(); + std::vector prev(n + 1), curr(n + 1); + for (size_t j = 0; j <= n; ++j) prev[j] = j; + for (size_t i = 1; i <= m; ++i) { + curr[0] = i; + for (size_t j = 1; j <= n; ++j) { + const bool match = std::tolower(static_cast(a[i - 1])) == + std::tolower(static_cast(b[j - 1])); + curr[j] = std::min({prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (match ? 0 : 1)}); + } + std::swap(prev, curr); + } + return prev[n]; +} + +inline std::string collapse_spaces(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + if (c != ' ') out += c; + } + return out; +} + +inline void apply_vocabulary_spelling_correction( + std::string& text, + const std::vector& custom_vocabulary) +{ + if (custom_vocabulary.empty() || text.empty()) return; + + struct VocabEntry { + const std::string* original; + std::string collapsed; + }; + std::vector vocab_entries; + vocab_entries.reserve(custom_vocabulary.size()); + for (const auto& v : custom_vocabulary) { + vocab_entries.push_back({&v, collapse_spaces(v)}); + } + + struct Token { std::string text; bool is_word; }; + std::vector tokens; + size_t pos = 0; + while (pos < text.size()) { + if (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-') { + size_t start = pos; + while (pos < text.size() && (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-')) { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), true}); + } else { + size_t start = pos; + while (pos < text.size() && !std::isalnum(static_cast(text[pos])) && + text[pos] != '\'' && text[pos] != '-') { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), false}); + } + } + + std::vector word_indices; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i].is_word) word_indices.push_back(i); + } + + std::vector consumed(tokens.size(), false); + + auto strip_suffix = [](const std::string& word) -> std::pair { + if (word.size() >= 3 && word.substr(word.size() - 2) == "'s") { + return {word.substr(0, word.size() - 2), "'s"}; + } + if (word.size() >= 3 && word.substr(word.size() - 2) == "'t") { + return {word.substr(0, word.size() - 2), "'t"}; + } + if (word.size() >= 4 && word.back() == 's' && + word[word.size() - 2] != 's' && // avoid stripping from "boss", "class" + std::isalpha(static_cast(word[word.size() - 2]))) { + return {word.substr(0, word.size() - 1), "s"}; + } + return {word, ""}; + }; + + size_t wi = 0; + while (wi < word_indices.size()) { + size_t best_dist = std::numeric_limits::max(); + const std::string* best_match = nullptr; + size_t best_window = 0; + size_t best_first_token = 0; + size_t best_last_token = 0; + std::string best_suffix; + + for (size_t window = std::min(3, word_indices.size() - wi); window >= 1; --window) { + std::string window_collapsed; + const size_t first_tok = word_indices[wi]; + const size_t last_tok = word_indices[wi + window - 1]; + for (size_t w = 0; w < window; ++w) { + window_collapsed += tokens[word_indices[wi + w]].text; + } + + if (window == 1 && window_collapsed.size() < 3) break; + + auto [stem, suffix] = strip_suffix(window_collapsed); + const std::string* candidates[] = {&window_collapsed, &stem}; + const std::string suffixes[] = {"", suffix}; + const size_t num_candidates = suffix.empty() ? 1 : 2; + + for (size_t ci = 0; ci < num_candidates; ++ci) { + const std::string& candidate = *candidates[ci]; + if (candidate.empty()) continue; + + for (const auto& entry : vocab_entries) { + const size_t wlen = candidate.size(); + const size_t vlen = entry.collapsed.size(); + + const size_t len_diff = wlen > vlen ? wlen - vlen : vlen - wlen; + const size_t max_dist = std::max(1, std::min(wlen, vlen) / 3); + if (len_diff > max_dist) continue; + + const size_t dist = levenshtein_ci(candidate, entry.collapsed); + + // For single-edit corrections, require first char match to prevent + // false positives like "vortex" → "Cortex". + if (dist == 1 && window == 1) { + const bool first_char_match = + std::tolower(static_cast(candidate[0])) == + std::tolower(static_cast(entry.collapsed[0])); + if (!first_char_match) continue; + } + + if (dist <= max_dist && dist < best_dist) { + best_dist = dist; + best_match = entry.original; + best_window = window; + best_first_token = first_tok; + best_last_token = last_tok; + best_suffix = suffixes[ci]; + } + } + } + + if (best_dist == 0) break; + } + + // Allow dist==0 for multi-word merges where word boundaries changed. + const bool should_replace = best_match && + best_dist != std::numeric_limits::max() && + (best_dist > 0 || best_window > 1); + + if (should_replace) { + tokens[best_first_token].text = *best_match + best_suffix; + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + consumed[t] = true; + } + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + if (t > 0) consumed[t - 1] = consumed[t - 1] || !tokens[t - 1].is_word; + } + wi += best_window; + } else { + ++wi; + } + } + + std::string result; + result.reserve(text.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + if (!consumed[i]) { + result += tokens[i].text; + } + } + + text = std::move(result); +} + +inline InferenceOptions parse_inference_options_json(const std::string& json) { + InferenceOptions options; + + if (json.empty()) return options; + + size_t pos = json.find("\"temperature\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.temperature = std::stof(json.substr(pos)); + } + + pos = json.find("\"top_p\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_p = std::stof(json.substr(pos)); + } + + float parsed_min_p = options.min_p; + if (try_parse_json_float(json, "min_p", parsed_min_p)) { + options.min_p = std::clamp(parsed_min_p, 0.0f, 1.0f); + } + + float parsed_rep_penalty = options.repetition_penalty; + if (try_parse_json_float(json, "repetition_penalty", parsed_rep_penalty)) { + if (std::isfinite(parsed_rep_penalty) && parsed_rep_penalty > 0.0f) { + options.repetition_penalty = parsed_rep_penalty; + } + } + + pos = json.find("\"top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"max_tokens\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.max_tokens = std::stoul(json.substr(pos)); + } + + pos = json.find("\"force_tools\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.force_tools = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"tool_rag_top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.tool_rag_top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"confidence_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.confidence_threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"include_stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.include_stop_sequences = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"use_vad\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.use_vad = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"telemetry_enabled\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.telemetry_enabled = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"auto_handoff\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.auto_handoff = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"cloud_timeout_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.cloud_timeout_ms = std::stoul(json.substr(pos)); + } + + pos = json.find("\"handoff_with_images\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.handoff_with_images = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"enable_thinking_if_supported\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find('[', pos); + if (pos != std::string::npos) { + size_t end_pos = json.find(']', pos); + size_t seq_pos = json.find('"', pos); + + while (seq_pos != std::string::npos && seq_pos < end_pos) { + size_t seq_start = seq_pos + 1; + size_t seq_end = json.find('"', seq_start); + if (seq_end != std::string::npos) { + options.stop_sequences.push_back(json.substr(seq_start, seq_end - seq_start)); + } + seq_pos = json.find('"', seq_end + 1); + } + } + } + + return options; +} + +static inline std::string trim_lfm2_slice(const std::string& value, size_t begin, size_t end) { + return trim_string(value.substr(begin, end - begin)); +} + +static inline void append_lfm2_call(const std::string& entry, + std::vector& function_calls) { + if (entry.empty()) return; + + std::string trimmed_entry = trim_lfm2_slice(entry, 0, entry.size()); + if (trimmed_entry.empty()) return; + + size_t paren_pos = trimmed_entry.find('('); + if (paren_pos == std::string::npos) return; + + std::string func_name = trim_lfm2_slice(trimmed_entry, 0, paren_pos); + std::string args_str = trim_lfm2_slice(trimmed_entry, paren_pos + 1, trimmed_entry.size()); + + if (!args_str.empty() && args_str.back() == ')') { + args_str.pop_back(); + args_str = trim_lfm2_slice(args_str, 0, args_str.size()); + } + + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{"; + + size_t arg_pos = 0; + bool first_arg = true; + while (arg_pos < args_str.length()) { + while (arg_pos < args_str.length() && std::isspace(static_cast(args_str[arg_pos]))) { + arg_pos++; + } + + size_t eq_pos = args_str.find('=', arg_pos); + if (eq_pos == std::string::npos) break; + + std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos); + + size_t val_start = eq_pos + 1; + size_t val_end = val_start; + + if (val_start < args_str.length() && args_str[val_start] == '"') { + val_start++; + val_end = args_str.find('"', val_start); + if (val_end == std::string::npos) break; + } else { + val_end = args_str.find(',', val_start); + if (val_end == std::string::npos) val_end = args_str.length(); + } + + std::string arg_value = args_str.substr(val_start, val_end - val_start); + + if (!first_arg) json_call += ","; + json_call += "\"" + arg_name + "\":\"" + arg_value + "\""; + first_arg = false; + + arg_pos = args_str.find(',', val_end); + if (arg_pos != std::string::npos) { + arg_pos++; + } else { + break; + } + } + + json_call += "}}"; + function_calls.push_back(json_call); +} + +inline void parse_function_calls_from_response(const std::string& response_text, + std::string& regular_response, + std::vector& function_calls) { + regular_response = response_text; + function_calls.clear(); + + gemma::parse_function_calls(regular_response, function_calls); + + const std::string QWEN_TOOL_START = ""; + const std::string QWEN_TOOL_END = ""; + size_t qwen_start_pos = 0; + + while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) { + size_t content_start = qwen_start_pos + QWEN_TOOL_START.length(); + size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start); + + size_t erase_end; + std::string json_content; + + if (qwen_end_pos != std::string::npos) { + json_content = regular_response.substr(content_start, qwen_end_pos - content_start); + erase_end = qwen_end_pos + QWEN_TOOL_END.length(); + } else { + json_content = regular_response.substr(content_start); + erase_end = regular_response.length(); + } + + size_t first = json_content.find_first_not_of(" \t\n\r"); + size_t last = json_content.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) { + json_content = json_content.substr(first, last - first + 1); + } + + if (json_content.size() > 2 && json_content.find("\"name\"") != std::string::npos) { + // Unwrap array wrapper if present: [{"name":...}] -> {"name":...} + if (json_content[0] == '[') { + size_t obj_start = json_content.find('{'); + size_t obj_end = json_content.rfind('}'); + if (obj_start != std::string::npos && obj_end != std::string::npos && obj_end > obj_start) { + json_content = json_content.substr(obj_start, obj_end - obj_start + 1); + } + } + if (json_content[0] == '{') { + size_t depth = 0; + bool in_string = false; + bool escaped = false; + size_t end_pos = 0; + for (size_t c = 0; c < json_content.size(); c++) { + char ch = json_content[c]; + if (escaped) { escaped = false; continue; } + if (ch == '\\' && in_string) { escaped = true; continue; } + if (ch == '"') { in_string = !in_string; continue; } + if (!in_string) { + if (ch == '{') depth++; + else if (ch == '}' && --depth == 0) { end_pos = c + 1; break; } + } + } + if (end_pos > 0) { + function_calls.push_back(json_content.substr(0, end_pos)); + } + } + } + + regular_response.erase(qwen_start_pos, erase_end - qwen_start_pos); + } + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + size_t lfm2_start_pos = 0; + + while ((lfm2_start_pos = regular_response.find(TOOL_CALL_START, lfm2_start_pos)) != std::string::npos) { + size_t content_start = lfm2_start_pos + TOOL_CALL_START.length(); + size_t tool_end_pos = regular_response.find(TOOL_CALL_END, content_start); + + if (tool_end_pos != std::string::npos) { + std::string tool_content = regular_response.substr(content_start, tool_end_pos - content_start); + std::string content = tool_content; + size_t trim_start = 0; + while (trim_start < content.size() && std::isspace(static_cast(content[trim_start]))) { + trim_start++; + } + + if (trim_start < content.size()) { + size_t trim_end = content.size() - 1; + while (trim_end > trim_start && std::isspace(static_cast(content[trim_end]))) { + trim_end--; + } + content = content.substr(trim_start, trim_end - trim_start + 1); + } else { + content.clear(); + } + + if (!content.empty() && content.front() == '[' && content.back() == ']') { + std::string inner = content.substr(1, content.size() - 2); + + size_t inner_first = inner.find_first_not_of(" \t\n\r"); + if (inner_first != std::string::npos && inner[inner_first] == '{') { + size_t pos = inner_first; + while (pos < inner.size()) { + if (inner[pos] == '{') { + int brace_depth = 1; + size_t obj_start = pos; + pos++; + while (pos < inner.size() && brace_depth > 0) { + if (inner[pos] == '{') brace_depth++; + else if (inner[pos] == '}') brace_depth--; + pos++; + } + if (brace_depth == 0) { + std::string json_obj = inner.substr(obj_start, pos - obj_start); + if (json_obj.find("\"name\"") != std::string::npos) { + function_calls.push_back(json_obj); + } + } + } else { + pos++; + } + } + } else { + size_t start = 0; + int paren_depth = 0; + + for (size_t i = 0; i < inner.size(); ++i) { + char c = inner[i]; + if (c == '(') { + paren_depth++; + } else if (c == ')' && paren_depth > 0) { + paren_depth--; + } else if (c == ',' && paren_depth == 0) { + append_lfm2_call(inner.substr(start, i - start), function_calls); + start = i + 1; + } + } + + if (start < inner.size()) { + append_lfm2_call(inner.substr(start), function_calls); + } + } + } else if (!content.empty()) { + append_lfm2_call(content, function_calls); + } + + regular_response.erase(lfm2_start_pos, tool_end_pos + TOOL_CALL_END.length() - lfm2_start_pos); + } else { + break; + } + } + + const char* FUNCTION_CALL_MARKER = "\"function_call\""; + size_t search_pos = 0; + const size_t text_len = regular_response.length(); + + while (search_pos < text_len) { + size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos); + if (marker_pos == std::string::npos) break; + + size_t json_start = regular_response.find('{', marker_pos); + if (json_start == std::string::npos) break; + + int brace_count = 1; + size_t json_end = json_start + 1; + while (json_end < text_len && brace_count > 0) { + char c = regular_response[json_end]; + brace_count += (c == '{') - (c == '}'); + json_end++; + } + + if (brace_count == 0) { + function_calls.push_back(regular_response.substr(json_start, json_end - json_start)); + regular_response = regular_response.substr(0, marker_pos); + size_t last_bracket = regular_response.rfind('{'); + if(last_bracket != std::string::npos) { + regular_response = regular_response.substr(0, last_bracket); + } + } + search_pos = json_end; + } +} + +inline std::vector> find_channel_token_ranges( + const std::vector& tokens, size_t offset, + uint32_t channel_open_id, uint32_t channel_close_id) { + std::vector> ranges; + size_t pos = 0; + while (pos < tokens.size()) { + if (tokens[pos] != channel_open_id) { + pos++; + continue; + } + + size_t block_start = pos; + pos++; + while (pos < tokens.size() && tokens[pos] != channel_close_id) { + pos++; + } + if (pos < tokens.size()) { + pos++; + } + ranges.push_back({offset + block_start, pos - block_start}); + } + return ranges; +} + +inline void strip_tag_blocks(std::string& text, std::string& extracted, + const std::string& open_tag, const std::string& close_tag) { + std::string result; + size_t pos = 0; + + size_t first_close = text.find(close_tag); + size_t first_open = text.find(open_tag); + if (first_close != std::string::npos && + (first_open == std::string::npos || first_close < first_open)) { + extracted += text.substr(0, first_close); + pos = first_close + close_tag.size(); + } + + while (pos < text.size()) { + size_t open_pos = text.find(open_tag, pos); + if (open_pos == std::string::npos) { + result += text.substr(pos); + break; + } + result += text.substr(pos, open_pos - pos); + size_t content_start = open_pos + open_tag.size(); + size_t close_pos = text.find(close_tag, content_start); + if (close_pos == std::string::npos) { + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start); + break; + } + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start, close_pos - content_start); + pos = close_pos + close_tag.size(); + } + text = result; +} + +inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { + thinking.clear(); + content = input; + + auto trim = [](std::string& s) { + size_t first = s.find_first_not_of(" \t\n\r"); + size_t last = s.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) + s = s.substr(first, last - first + 1); + else + s.clear(); + }; + + if (content.find("<|channel>") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "<|channel>", ""); + } else if (content.find("") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "", ""); + } else { + return; + } + + trim(thinking); + trim(content); +} + +struct TranscriptSegment { + float start; + float end; + std::string text; +}; + +inline std::string construct_response_json(const std::string& regular_response, + const std::vector& function_calls, + double time_to_first_token, + double total_time_ms, + double prefill_tps, + double decode_tps, + size_t prompt_tokens, + size_t completion_tokens, + float confidence = 0.0f, + bool cloud_handoff = false, + const std::string& thinking = "", + const std::vector& segments = {}) { + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; + json << "\"response\":\"" << escape_json_string(regular_response) << "\","; + if (!thinking.empty()) { + json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; + } + json << "\"function_calls\":["; + for (size_t i = 0; i < function_calls.size(); ++i) { + if (i > 0) json << ","; + json << function_calls[i]; + } + json << "],"; + json << "\"segments\":["; + for (size_t i = 0; i < segments.size(); ++i) { + if (i > 0) json << ","; + json << "{\"start\":" << std::fixed << std::setprecision(3) << segments[i].start + << ",\"end\":" << std::fixed << std::setprecision(3) << segments[i].end + << ",\"text\":\"" << escape_json_string(segments[i].text) << "\"}"; + } + json << "],"; + json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; + json << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << time_to_first_token << ","; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; + json << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":" << prompt_tokens << ","; + json << "\"decode_tokens\":" << completion_tokens << ","; + json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); + json << "}"; + return json.str(); +} + +inline std::string serialize_function_calls(const std::vector& calls) { + if (calls.empty()) return "[]"; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < calls.size(); ++i) { + if (i > 0) oss << ","; + oss << calls[i]; + } + oss << "]"; + return oss.str(); +} + +inline int validate_audio_params( + const char* component, + void* model, + char* response_buffer, size_t buffer_size, + const char* audio_file_path, + const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + if (!model) { + std::string err = last_error_message.empty() ? "Model not initialized." : last_error_message; + CACTUS_LOG_ERROR(component, err); + handle_error_response(err, response_buffer, buffer_size); + return -1; + } + if (!response_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR(component, "Invalid parameters: response_buffer or buffer_size"); + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { + CACTUS_LOG_ERROR(component, "No audio input provided"); + handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); + return -1; + } + if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { + CACTUS_LOG_ERROR(component, "Both audio_file_path and pcm_buffer provided"); + handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); + return -1; + } + if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { + CACTUS_LOG_ERROR(component, "Invalid pcm_buffer_size"); + handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); + return -1; + } + return 0; +} + +inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + const int16_t* samples = reinterpret_cast(pcm_buffer); + size_t n = pcm_buffer_size / 2; + std::vector out(n); + for (size_t i = 0; i < n; ++i) + out[i] = static_cast(samples[i]) / 32768.0f; + return out; +} + +} // namespace ffi +} // namespace cactus + +#ifdef __cplusplus +extern "C" { +#endif + +const char* cactus_get_last_error(); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_UTILS_H diff --git a/cactus-transpile/ffi/cactus_vad.cpp b/cactus-transpile/ffi/cactus_vad.cpp new file mode 100644 index 000000000..10b2e99c3 --- /dev/null +++ b/cactus-transpile/ffi/cactus_vad.cpp @@ -0,0 +1,176 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" +#include "../models/model.h" +#include "../../libs/audio/wav.h" +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; + + +static SileroVADModel::SpeechTimestampsOptions parse_speech_timestamps_options(const std::string& json) { + SileroVADModel::SpeechTimestampsOptions options; + + if (json.empty()) return options; + + size_t pos = json.find("\"threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"neg_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.neg_threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"min_speech_duration_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.min_speech_duration_ms = std::stoi(json.substr(pos)); + } + + pos = json.find("\"max_speech_duration_s\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.max_speech_duration_s = std::stof(json.substr(pos)); + } + + pos = json.find("\"min_silence_duration_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.min_silence_duration_ms = std::stoi(json.substr(pos)); + } + + pos = json.find("\"speech_pad_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.speech_pad_ms = std::stoi(json.substr(pos)); + } + + pos = json.find("\"window_size_samples\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.window_size_samples = std::stoi(json.substr(pos)); + } + + pos = json.find("\"min_silence_at_max_speech\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.min_silence_at_max_speech = std::stoi(json.substr(pos)); + } + + pos = json.find("\"use_max_poss_sil_at_max_speech\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + std::string value = json.substr(pos, 5); + options.use_max_poss_sil_at_max_speech = (value.find("true") != std::string::npos); + } + + pos = json.find("\"sampling_rate\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(json[pos])) pos++; + options.sampling_rate = std::stoi(json.substr(pos)); + } + + return options; +} + + +extern "C" { + +int cactus_vad( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +) { + if (validate_audio_params("vad", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) + return -1; + + try { + auto start_time = std::chrono::high_resolution_clock::now(); + auto* handle = static_cast(model); + auto* vad_model = static_cast(handle->model.get()); + + SileroVADModel::SpeechTimestampsOptions options = parse_speech_timestamps_options(options_json ? options_json : ""); + + std::vector audio; + if (audio_file_path == nullptr) { + audio = pcm_to_float(pcm_buffer, pcm_buffer_size); + } else { + AudioFP32 wav_audio = load_wav(audio_file_path); + audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); + } + + if (audio.empty()) { + last_error_message = "Failed to load audio or audio is empty"; + CACTUS_LOG_ERROR("vad", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } + + auto segments = vad_model->get_speech_timestamps(audio, options); + + auto end_time = std::chrono::high_resolution_clock::now(); + double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"segments\":["; + + for (size_t i = 0; i < segments.size(); ++i) { + if (i > 0) json << ","; + json << "{"; + json << "\"start\":" << segments[i].start << ","; + json << "\"end\":" << segments[i].end; + json << "}"; + } + + json << "],"; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); + json << "}"; + + std::string response = json.str(); + if (response.length() >= buffer_size) { + last_error_message = "Response buffer too small"; + CACTUS_LOG_ERROR("vad", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } + + std::strcpy(response_buffer, response.c_str()); + return static_cast(response.length()); + + } catch (const std::exception& e) { + last_error_message = "Exception during VAD processing: " + std::string(e.what()); + CACTUS_LOG_ERROR("vad", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } catch (...) { + last_error_message = "Unknown exception during VAD processing"; + CACTUS_LOG_ERROR("vad", last_error_message); + handle_error_response(last_error_message, response_buffer, buffer_size); + return -1; + } +} + +} diff --git a/cactus-transpile/ffi/gemma_tools.h b/cactus-transpile/ffi/gemma_tools.h new file mode 100644 index 000000000..f0f9fe26d --- /dev/null +++ b/cactus-transpile/ffi/gemma_tools.h @@ -0,0 +1,576 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gemma { + +inline std::string to_upper(const std::string& s) { + std::string result = s; + for (auto& c : result) c = std::toupper(c); + return result; +} + +inline std::string escape(const std::string& s) { + return "" + s + ""; +} + +inline void skip_whitespace(const std::string& json, size_t& pos) { + while (pos < json.length() && std::isspace(json[pos])) pos++; +} + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.length()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.length()) pos++; + return value; +} + +std::string format_argument(const std::string& json, size_t& pos, bool escape_keys); +std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/); + +inline std::string format_argument(const std::string& json, size_t& pos, bool escape_keys = true) { + skip_whitespace(json, pos); + if (pos >= json.length()) return ""; + + char c = json[pos]; + + if (c == '"') { + pos++; + std::string value = extract_json_string(json, pos); + return escape(value); + } else if (c == '{') { + std::string result = "{"; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + if (escape_keys) { + result += escape(key) + ":" + value; + } else { + result += key + ":" + value; + } + } + result += "}"; + return result; + } else if (c == '[') { + std::string result = "["; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == ']') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + result += value; + } + result += "]"; + return result; + } else if (json.compare(pos, 4, "true") == 0) { + pos += 4; + return "true"; + } else if (json.compare(pos, 5, "false") == 0) { + pos += 5; + return "false"; + } else if (json.compare(pos, 4, "null") == 0) { + pos += 4; + return "null"; + } else { + size_t start = pos; + while (pos < json.length() && (std::isdigit(json[pos]) || json[pos] == '.' || + json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E')) { + pos++; + } + return json.substr(start, pos - start); + } +} + +inline std::map parse_json_object_raw(const std::string& json, size_t& pos) { + std::map result; + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] != '{') return result; + pos++; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + skip_whitespace(json, pos); + + size_t value_start = pos; + if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + pos++; + } else if (json[pos] == '{') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '{') depth++; + else if (json[pos] == '}') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else if (json[pos] == '[') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '[') depth++; + else if (json[pos] == ']') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else { + while (pos < json.length() && json[pos] != ',' && json[pos] != '}') pos++; + } + result[key] = json.substr(value_start, pos - value_start); + } + return result; +} + +inline std::string get_json_string_value(const std::string& json, size_t pos) { + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == '"') { + pos++; + return extract_json_string(json, pos); + } + return ""; +} + +inline std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/) { + static const std::set standard_keys = {"description", "type", "properties", "required", "nullable"}; + + size_t pos = 0; + auto properties = parse_json_object_raw(properties_json, pos); + + std::string result; + bool first = true; + + for (const auto& [key, value_json] : properties) { + if (standard_keys.count(key)) continue; + + if (!first) result += ","; + first = false; + + size_t prop_pos = 0; + auto prop_obj = parse_json_object_raw(value_json, prop_pos); + + result += key + ":{"; + + if (prop_obj.count("description")) { + std::string desc = get_json_string_value(prop_obj["description"], 0); + result += "description:" + escape(desc); + } + + std::string type_val; + if (prop_obj.count("type")) { + type_val = get_json_string_value(prop_obj["type"], 0); + } + + if (to_upper(type_val) == "STRING") { + if (prop_obj.count("enum")) { + size_t enum_pos = 0; + std::string enum_formatted = format_argument(prop_obj["enum"], enum_pos, true); + result += ",enum:" + enum_formatted; + } + } else if (to_upper(type_val) == "OBJECT") { + if (prop_obj.count("properties")) { + std::string nested_required; + if (prop_obj.count("required")) { + nested_required = prop_obj["required"]; + } + result += ",properties:{" + format_parameters(prop_obj["properties"], nested_required) + "}"; + } + if (prop_obj.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(prop_obj["required"], req_pos); + if (req_pos < prop_obj["required"].length() && prop_obj["required"][req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < prop_obj["required"].length()) { + skip_whitespace(prop_obj["required"], req_pos); + if (prop_obj["required"][req_pos] == ']') break; + if (prop_obj["required"][req_pos] == ',') { req_pos++; continue; } + if (prop_obj["required"][req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(prop_obj["required"], req_pos); + if (!req_first) req_items += ","; + req_first = false; + req_items += escape(req_item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + } else if (to_upper(type_val) == "ARRAY") { + if (prop_obj.count("items")) { + result += ",items:{"; + size_t items_pos = 0; + auto items_obj = parse_json_object_raw(prop_obj["items"], items_pos); + bool items_first = true; + + for (const auto& [item_key, item_value] : items_obj) { + if (!items_first) result += ","; + items_first = false; + + if (item_key == "properties") { + std::string items_required; + if (items_obj.count("required")) { + items_required = items_obj["required"]; + } + result += "properties:{" + format_parameters(item_value, items_required) + "}"; + } else if (item_key == "required") { + result += "required:["; + size_t req_pos = 0; + skip_whitespace(item_value, req_pos); + if (req_pos < item_value.length() && item_value[req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < item_value.length()) { + skip_whitespace(item_value, req_pos); + if (item_value[req_pos] == ']') break; + if (item_value[req_pos] == ',') { req_pos++; continue; } + if (item_value[req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(item_value, req_pos); + if (!req_first) result += ","; + req_first = false; + result += escape(req_item); + } + } + } + result += "]"; + } else if (item_key == "type") { + std::string item_type = get_json_string_value(item_value, 0); + result += "type:" + escape(to_upper(item_type)); + } else { + size_t val_pos = 0; + result += item_key + ":" + format_argument(item_value, val_pos, true); + } + } + result += "}"; + } + } + + if (!type_val.empty()) { + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + return result; +} + +inline std::string format_function_declaration(const std::string& name, + const std::string& description, + const std::string& params_json) { + std::string result = "declaration:" + name + "{"; + result += "description:" + escape(description); + + if (!params_json.empty()) { + result += ",parameters:{"; + + size_t pos = 0; + auto params = parse_json_object_raw(params_json, pos); + + if (params.count("properties")) { + std::string required_json; + if (params.count("required")) { + required_json = params["required"]; + } + result += "properties:{" + format_parameters(params["properties"], required_json) + "}"; + } + + if (params.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(params["required"], req_pos); + if (req_pos < params["required"].length() && params["required"][req_pos] == '[') { + req_pos++; + bool first = true; + while (req_pos < params["required"].length()) { + skip_whitespace(params["required"], req_pos); + if (params["required"][req_pos] == ']') break; + if (params["required"][req_pos] == ',') { req_pos++; continue; } + if (params["required"][req_pos] == '"') { + req_pos++; + std::string item = extract_json_string(params["required"], req_pos); + if (!first) req_items += ","; + first = false; + req_items += escape(item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + + if (params.count("type")) { + std::string type_val = get_json_string_value(params["type"], 0); + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + result += "}"; + return result; +} + +template +inline std::string format_tools(const std::vector& tools, bool use_pipe_tags = false) { + if (tools.empty()) return ""; + + const char* decl_start = use_pipe_tags ? "<|tool>" : ""; + const char* decl_end = use_pipe_tags ? "" : ""; + + std::string result; + for (const auto& tool : tools) { + result += decl_start; + std::string params_json; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + params_json = it->second; + } + + result += format_function_declaration(tool.name, tool.description, params_json); + result += decl_end; + } + return result; +} + + +inline size_t match_quote_tag(const std::string& s, size_t pos) { + if (s.compare(pos, 8, "") == 0) return 8; + if (s.compare(pos, 5, "<|\"|>") == 0) return 5; + return 0; +} + +inline size_t find_quote_tag(const std::string& s, size_t pos) { + size_t e = s.find("", pos); + size_t t = s.find("<|\"|>", pos); + if (e == std::string::npos) return t; + if (t == std::string::npos) return e; + return std::min(e, t); +} + +inline std::string unescape(const std::string& s) { + const std::string ESCAPE_TAG = ""; + std::string result = s; + size_t pos = 0; + while ((pos = result.find(ESCAPE_TAG, pos)) != std::string::npos) { + result.erase(pos, ESCAPE_TAG.length()); + } + return result; +} + +inline std::string args_to_json(const std::string& args_content) { + std::string result = "{"; + size_t pos = 0; + bool first = true; + + if (!args_content.empty() && args_content[0] == '{') pos = 1; + + while (pos < args_content.length()) { + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + if (pos >= args_content.length() || args_content[pos] == '}') break; + if (args_content[pos] == ',') { pos++; continue; } + + size_t key_start = pos; + while (pos < args_content.length() && args_content[pos] != ':') pos++; + std::string key = args_content.substr(key_start, pos - key_start); + if (pos < args_content.length()) pos++; + + std::string value; + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + + if (pos < args_content.length()) { + size_t qtag_len = match_quote_tag(args_content, pos); + if (qtag_len > 0) { + pos += qtag_len; + size_t val_end = find_quote_tag(args_content, pos); + if (val_end != std::string::npos) { + value = "\"" + args_content.substr(pos, val_end - pos) + "\""; + pos = val_end + match_quote_tag(args_content, val_end); + } + } else if (args_content[pos] == '{') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '{') depth++; + else if (args_content[pos] == '}') depth--; + pos++; + } + value = args_to_json(args_content.substr(start, pos - start)); + } else if (args_content[pos] == '[') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '[') depth++; + else if (args_content[pos] == ']') depth--; + pos++; + } + std::string arr_content = args_content.substr(start + 1, pos - start - 2); + value = "["; + size_t arr_pos = 0; + bool first_item = true; + while (arr_pos < arr_content.length()) { + while (arr_pos < arr_content.length() && (std::isspace(arr_content[arr_pos]) || arr_content[arr_pos] == ',')) arr_pos++; + if (arr_pos >= arr_content.length()) break; + + if (!first_item) value += ","; + first_item = false; + + size_t aq_len = match_quote_tag(arr_content, arr_pos); + if (aq_len > 0) { + arr_pos += aq_len; + size_t end = find_quote_tag(arr_content, arr_pos); + if (end != std::string::npos) { + value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; + arr_pos = end + match_quote_tag(arr_content, end); + } + } else { + size_t end = arr_content.find_first_of(",]", arr_pos); + if (end == std::string::npos) end = arr_content.length(); + value += arr_content.substr(arr_pos, end - arr_pos); + arr_pos = end; + } + } + value += "]"; + } else { + size_t val_start = pos; + while (pos < args_content.length() && args_content[pos] != ',' && args_content[pos] != '}') { + pos++; + } + value = args_content.substr(val_start, pos - val_start); + while (!value.empty() && std::isspace(value.back())) value.pop_back(); + } + } + + if (!first) result += ","; + first = false; + result += "\"" + key + "\":" + value; + } + + result += "}"; + return result; +} + +inline void parse_function_calls(std::string& response, std::vector& function_calls) { + + const std::string CALL_START = (response.find("<|tool_call>") != std::string::npos) + ? "<|tool_call>" : ""; + const std::string CALL_END = (CALL_START == "<|tool_call>") + ? "" : ""; + size_t pos = 0; + + while ((pos = response.find(CALL_START, pos)) != std::string::npos) { + size_t content_start = pos + CALL_START.length(); + size_t call_end_pos = response.find(CALL_END, content_start); + + size_t content_end = (call_end_pos != std::string::npos) ? call_end_pos : response.length(); + std::string call_content = response.substr(content_start, content_end - content_start); + + if (call_content.compare(0, 5, "call:") == 0) { + size_t brace_pos = call_content.find('{'); + + if (brace_pos == std::string::npos) { + size_t sep_pos = call_content.find_first_of(", ", 5); + if (sep_pos != std::string::npos) { + std::string func_name = call_content.substr(5, sep_pos - 5); + size_t args_start = sep_pos + 1; + while (args_start < call_content.length() && + (call_content[args_start] == ' ' || call_content[args_start] == ',')) { + args_start++; + } + std::string args_content = "{" + call_content.substr(args_start); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } else { + std::string func_name = call_content.substr(5, brace_pos - 5); + std::string args_content = call_content.substr(brace_pos); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } + + size_t erase_end = (call_end_pos != std::string::npos) ? + call_end_pos + CALL_END.length() : response.length(); + response.erase(pos, erase_end - pos); + } +} + +} // namespace gemma \ No newline at end of file diff --git a/cactus-transpile/graph/graph.h b/cactus-transpile/graph/graph.h new file mode 100644 index 000000000..4c387e582 --- /dev/null +++ b/cactus-transpile/graph/graph.h @@ -0,0 +1,805 @@ +#ifndef GRAPH_H +#define GRAPH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { + +enum class LogLevel { + DEBUG = 0, + INFO = 1, + WARN = 2, + ERROR = 3, + NONE = 4 +}; + +class Logger { +public: + static Logger& instance() { + static Logger logger; + return logger; + } + + void set_level(LogLevel level) { min_level_ = level; } + LogLevel get_level() const { return min_level_; } + + + void set_callback(std::function cb) { + std::lock_guard lock(mutex_); + callback_ = cb; + } + + void log(LogLevel level, const std::string& component, const std::string& message) { + if (level < min_level_) return; + + std::lock_guard lock(mutex_); + + if (callback_) { + callback_(level, component, message); + } else { + std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl; + } + + if (level == LogLevel::ERROR) { + last_error_ = "[" + component + "] " + message; + } + } + + const std::string& last_error() const { return last_error_; } + void clear_error() { last_error_.clear(); } + +private: + Logger() : min_level_(LogLevel::WARN) {} + + static const char* level_string(LogLevel level) { + switch (level) { + case LogLevel::DEBUG: return "DEBUG"; + case LogLevel::INFO: return "INFO"; + case LogLevel::WARN: return "WARN"; + case LogLevel::ERROR: return "ERROR"; + default: return "?"; + } + } + + LogLevel min_level_; + std::mutex mutex_; + std::string last_error_; + std::function callback_; +}; + +} // namespace cactus + +#define CACTUS_LOG(level, component, msg) \ + do { \ + if (static_cast(level) >= static_cast(cactus::Logger::instance().get_level())) { \ + std::ostringstream _cactus_log_ss; \ + _cactus_log_ss << msg; \ + cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \ + } \ + } while(0) + +#define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg) +#define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg) +#define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) +#define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) + +namespace GraphFile { + class MappedFile; + struct SerializedGraph; +} + +enum class Precision { + INT8, + FP16, + FP32, + INT4 +}; + +enum class ComputeBackend { + CPU, + NPU +}; + +enum class Activation { + SILU, + GELU, + GELU_ERF, + RELU, + SIGMOID, + TANH +}; + +enum class OpType { + INPUT, PRECISION_CAST, + ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE, + ABS, POW, FLATTEN, VIEW, + MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, + BILINEAR_INTERPOLATION, + SUM, MEAN, VARIANCE, MIN, MAX, + RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, CONV1D_SAME_DEPTHWISE_K9, CONV1D_POINTWISE, CONV2D_K3S2P1, CONV2D_DEPTHWISE_K3S2P1, CONV2D_POINTWISE_1X1, GLU, BATCHNORM, + SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN, SCALAR_LOG, + RELU, SILU, GELU, GELU_ERF, SIGMOID, TANH, + SAMPLE, CONCAT, CAT, + SCATTER_TOPK, + TOPK, LAYERNORM, GROUPNORM, + MOE_LAYER, + INDEX, + PERSISTENT, + QUANTIZE_ACTIVATIONS, + LSTM_CELL, + GATED_DELTANET_DECODE, + GATED_DELTANET_PREFILL, + STFT, + ALTUP_PREDICT, + ALTUP_CORRECT, + GAUSSIAN_TOPK, + MAXPOOL1D, + BILSTM_SEQUENCE, + LEAKY_RELU, + CONV2D_K3S1P1, + STATS_POOL, + WEIGHTED_STATS_POOL, + NOT_EQUAL, + SCALAR_NOT_EQUAL, + SCALAR_FLOOR_DIVIDE, + MASKED_SELECT_PREFIX, + MASKED_SCATTER, + EXPAND, + CONV2D, + CUMSUM +}; + +struct PrecisionTraits { + static constexpr size_t size_of(Precision prec) { + switch (prec) { + case Precision::INT8: return 1; + case Precision::FP16: return 2; + case Precision::FP32: return 4; + case Precision::INT4: return 1; + } + return 1; + } + + static constexpr size_t packed_size_of(Precision prec, size_t count) { + switch (prec) { + case Precision::INT4: return (count + 1) / 2; + default: return count * size_of(prec); + } + } + + static size_t byte_offset_of(Precision prec, size_t element_offset) { + switch (prec) { + case Precision::INT4: + assert(element_offset % 32 == 0 && "INT4 byte offset must be group-aligned (multiple of 32)"); + return element_offset / 2; + default: return element_offset * size_of(prec); + } + } + + static constexpr bool is_integer(Precision prec) { + switch (prec) { + case Precision::INT8: return true; + case Precision::INT4: return true; + case Precision::FP16: return false; + case Precision::FP32: return false; + } + return true; + } + + static constexpr bool is_floating_point(Precision prec) { + switch (prec) { + case Precision::INT8: return false; + case Precision::INT4: return false; + case Precision::FP16: return true; + case Precision::FP32: return true; + } + return false; + } +}; + +namespace Quantization { + void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); + void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); + void fp16_to_fp32(const __fp16* src, float* dst, size_t count); + void fp32_to_fp16(const float* src, __fp16* dst, size_t count); + void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); + void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +} + +struct TensorConfig { + Precision default_precision = Precision::INT8; + Precision compute_precision = Precision::INT8; + Precision output_precision = Precision::INT8; + bool auto_mixed_precision = false; + + static TensorConfig& global(); +}; + +struct BroadcastInfo { + std::vector output_shape; + bool needs_broadcasting; + + static BroadcastInfo compute(const std::vector& lhs, const std::vector& rhs); +}; + +class BufferPool; + +struct BufferDesc { + std::vector shape; + size_t total_size; + size_t byte_size; + std::unique_ptr data; + void* external_data; + char* pooled_data; + Precision precision; + + size_t group_size = 0; + size_t num_groups = 0; + void* scales_data = nullptr; + std::unique_ptr owned_scales; + + bool is_interleaved = false; + size_t original_N = 0; + + void* activation_scales_data = nullptr; + std::unique_ptr owned_activation_scales; + size_t num_rows_for_activation_scales = 0; + + BufferDesc(); + BufferDesc(const std::vector& s, Precision prec = Precision::INT8); + ~BufferDesc(); + + BufferDesc(BufferDesc&& other) noexcept; + BufferDesc& operator=(BufferDesc&& other) noexcept; + + BufferDesc(const BufferDesc&) = delete; + BufferDesc& operator=(const BufferDesc&) = delete; + + void* get_data(); + const void* get_data() const; + + template + T* data_as() { return static_cast(get_data()); } + + template + const T* data_as() const { return static_cast(get_data()); } + + const __fp16* scales_as_fp16() const { + return reinterpret_cast(scales_data); + } + + bool is_grouped_int8() const { + return precision == Precision::INT8 && group_size > 0; + } + + bool is_grouped_int4() const { + return precision == Precision::INT4 && group_size > 0; + } + + void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) { + group_size = gs; + num_groups = ng; + scales_data = scales_ptr; + } + + void set_interleaved(bool interleaved, size_t orig_n) { + is_interleaved = interleaved; + original_N = orig_n; + } + + bool has_activation_scales() const { + return activation_scales_data != nullptr && num_rows_for_activation_scales > 0; + } + const float* activation_scales_as_float() const { + return reinterpret_cast(activation_scales_data); + } + float* activation_scales_as_float() { + return reinterpret_cast(activation_scales_data); + } + void allocate_activation_scales(size_t num_rows) { + num_rows_for_activation_scales = num_rows; + owned_activation_scales = std::make_unique(num_rows * sizeof(float)); + activation_scales_data = owned_activation_scales.get(); + } + void set_activation_scales(void* scales_ptr, size_t num_rows) { + activation_scales_data = scales_ptr; + num_rows_for_activation_scales = num_rows; + } + + void allocate(); + void allocate_from_pool(BufferPool& pool); + void release_to_pool(BufferPool& pool); + void set_external(void* ptr); +}; + +struct OpParams { + float scalar = 0.0f; + float scale = 1.0f; + float theta = 10000.0f; + float epsilon = 1e-6f; + int axis = -1; + bool pretransposed_rhs = false; + size_t position_offset = 0; + size_t slice_start = 0; + size_t slice_length = 0; + size_t window_size = 0; + bool is_causal = true; + bool attention_mask_is_additive = false; + float logit_cap = 0.0f; + std::vector new_shape; + std::vector permutation; + Precision output_precision = Precision::INT8; + BroadcastInfo broadcast_info; + ComputeBackend backend = ComputeBackend::CPU; + + size_t dilation = 1; + size_t stride = 1; + size_t padding = 0; + float temperature = 1.0f; + float top_p = 1.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + size_t top_k = 0; + size_t random_seed = 0; + + size_t index_value = 0; + size_t num_classes = 0; + size_t num_groups = 0; + size_t dst_height = 0; + size_t dst_width = 0; + bool align_corners = true; + bool normalize_routing = false; + size_t num_experts = 0; + size_t num_experts_per_tok = 0; + bool moe_gated = true; + Activation activation = Activation::SILU; + + std::vector bias_values; + std::vector bias_indices; + + const int8_t* cached_keys_int8 = nullptr; + const int8_t* cached_values_int8 = nullptr; + const float* cached_k_scales = nullptr; + const float* cached_v_scales = nullptr; + size_t cache_seq_len = 0; + size_t num_kv_heads = 0; + size_t head_dim = 0; + size_t num_fft_bins = 0; + size_t chunk_size = 0; + size_t num_altup_inputs = 0; + size_t v_head_dim = 0; + size_t kernel_size = 0; +}; + +struct GraphNode { + size_t id; + OpType op_type; + std::vector input_ids; + BufferDesc output_buffer; + OpParams params; + + GraphNode(size_t node_id, OpType type); +}; + +using nodes_vector = std::vector>; +using node_index_map_t = std::unordered_map; + +inline const BufferDesc& get_input(const GraphNode& node, size_t idx, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + return nodes[node_index_map.at(node.input_ids[idx])]->output_buffer; +} + +struct AxisDims { + size_t outer, axis_size, inner; + static AxisDims from_shape(const std::vector& shape, size_t axis) { + AxisDims d; + d.outer = 1; + for (size_t i = 0; i < axis; i++) d.outer *= shape[i]; + d.axis_size = shape[axis]; + d.inner = 1; + for (size_t i = axis + 1; i < shape.size(); i++) d.inner *= shape[i]; + return d; + } +}; + +template +void dispatch_binary_op(OpType op, const T* lhs, const T* rhs, T* output, size_t count); + +template +void dispatch_unary_op(OpType op, const T* input, T* output, size_t count, float param = 0.0f); + +void compute_node_optimized(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_cumsum_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_masked_select_prefix_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_masked_scatter_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_expand_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_conv2d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); + +void shrink_thread_local_buffers(); +class BufferPool { +public: + BufferPool() = default; + ~BufferPool() = default; + + BufferPool(const BufferPool&) = delete; + BufferPool& operator=(const BufferPool&) = delete; + BufferPool(BufferPool&&) noexcept = default; + BufferPool& operator=(BufferPool&&) noexcept = default; + + char* acquire(size_t byte_size); + void release(char* ptr, size_t byte_size); + void clear(); + + size_t active_bytes() const { return active_bytes_; } + size_t pool_bytes() const { return pool_bytes_; } + size_t peak_bytes() const { return peak_bytes_; } + +private: + std::unordered_map>> free_buffers_; + size_t active_bytes_ = 0; + size_t pool_bytes_ = 0; + size_t peak_bytes_ = 0; + + size_t round_up_size(size_t size) const; +}; + +namespace ValidationUtils { + void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name); + void validate_precision(Precision actual, Precision required, const std::string& op_name); + void validate_input_count(size_t actual, size_t required, const std::string& op_name); +} + + +class CactusGraph { +public: + CactusGraph(); + ~CactusGraph() = default; + + CactusGraph(const CactusGraph&) = delete; + CactusGraph& operator=(const CactusGraph&) = delete; + CactusGraph(CactusGraph&&) noexcept = default; + CactusGraph& operator=(CactusGraph&&) noexcept = default; + + struct DebugNodeEntry { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + void save(const std::string& path); + static CactusGraph load(const std::string& path); + + size_t input(const std::vector& shape, Precision precision = Precision::INT8); + size_t precision_cast(size_t input, Precision target_precision); + size_t quantize_activations(size_t input); + + size_t add(size_t input1, size_t input2); + size_t add_clipped(size_t input1, size_t input2); + size_t subtract(size_t input1, size_t input2); + size_t multiply(size_t input1, size_t input2); + size_t divide(size_t input1, size_t input2); + size_t not_equal(size_t input1, size_t input2); + + size_t scalar_add(size_t input, float value); + size_t scalar_subtract(size_t input, float value); + size_t scalar_multiply(size_t input, float value); + size_t scalar_divide(size_t input, float value); + size_t scalar_floor_divide(size_t input, float value); + size_t scalar_not_equal(size_t input, float value); + size_t scalar_exp(size_t input); + size_t scalar_sqrt(size_t input); + size_t scalar_cos(size_t input); + size_t scalar_sin(size_t input); + size_t scalar_log(size_t input); + + size_t relu(size_t input); + size_t silu(size_t input); + size_t gelu(size_t input); + size_t gelu_erf(size_t input); + size_t sigmoid(size_t input); + size_t tanh(size_t input); + size_t glu(size_t input, int axis = -1); + + size_t abs(size_t input); + size_t pow(size_t input, float exponent); + size_t view(size_t input, const std::vector& new_shape); + size_t flatten(size_t input, int start_dim = 0, int end_dim = -1); + + size_t matmul( + size_t input1, + size_t input2, + bool pretransposed_rhs = false, + ComputeBackend backend = ComputeBackend::CPU, + Precision output_precision = Precision::INT8 + ); + size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU); + size_t transposeN(size_t input, const std::vector& permutation, ComputeBackend backend = ComputeBackend::CPU); + size_t reshape(size_t input, const std::vector& new_shape); + size_t expand(size_t input, const std::vector& new_shape); + size_t slice(size_t input, int axis, size_t start, size_t length); + size_t index(size_t input, size_t index_value, int dim); + size_t masked_select_prefix(size_t input, size_t mask); + size_t masked_scatter(size_t input, size_t mask, size_t source); + + size_t sum(size_t input, int axis); + size_t mean(size_t input, int axis); + size_t variance(size_t input, int axis); + size_t min(size_t input, int axis); + size_t max(size_t input, int axis); + size_t cumsum(size_t input, int axis); + + size_t gather(size_t embeddings, size_t indices, int axis = 0); + size_t mmap_embeddings(const std::string& filename); + size_t mmap_weights(const std::string& filename); + void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr); + void set_interleaved(size_t node_id, bool interleaved, size_t original_N); + + void release_weight_pages(size_t node_id); + void prefetch_weight_pages(size_t node_id); + void release_all_weight_pages(); + size_t embedding(const std::string& filename, size_t indices); + size_t embedding(size_t embedding_tensor, size_t indices); + size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners = true); + + size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f); + size_t layernorm(size_t input, size_t weight, float epsilon = 1e-5f); // No bias version + size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); + size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); + size_t topk(size_t input, size_t k); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w3_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation = Activation::SILU, + size_t per_expert_scale = 0); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation); + size_t rms_norm(size_t input, size_t weight, float epsilon = 1e-5f); + size_t rope(size_t input, float theta, size_t position_offset = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t rope_gptj(size_t input, float theta, size_t position_offset = 0, size_t rot_dim = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t softmax(size_t input, int axis = -1); + size_t attention(size_t query, size_t key, size_t value, float scale, bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU); + size_t attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, + bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU, + bool additive_mask = false, size_t position_offset = 0, size_t window_size = 0, + float logit_cap = 0.0f); + size_t rel_pos_bias(size_t query, size_t relative_key, float scale); + + size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, + const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, + size_t window_size = 0, size_t v_head_dim = 0); + + size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); + size_t conv1d_k3(size_t input, size_t weight, size_t stride); + size_t conv1d_k7s3(size_t input, size_t weight, size_t bias); + size_t conv1d(size_t input, size_t weight, size_t stride); + size_t conv1d(size_t input, size_t weight, size_t bias, size_t stride); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias); + size_t conv1d_pointwise(size_t input, size_t weight); + size_t conv1d_pointwise(size_t input, size_t weight, size_t bias); + size_t conv2d_k3s2p1(size_t input, size_t weight); + size_t conv2d_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_pointwise_1x1(size_t input, size_t weight); + size_t conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias); + size_t conv2d(size_t input, size_t weight, size_t stride = 1, size_t padding = 0, size_t dilation = 1, size_t groups = 1); + size_t conv2d(size_t input, size_t weight, size_t bias, size_t stride, size_t padding, size_t dilation, size_t groups); + + size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh); + size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, float scale = 0.0f); + size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, size_t chunk_size = 64, float scale = 0.0f); + size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins); + + size_t altup_predict(size_t coefs, const size_t* streams, size_t num_streams); + size_t altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions); + + size_t gaussian_topk(size_t input, float ppf); + + size_t maxpool1d(size_t input, size_t kernel_size, size_t stride); + size_t leaky_relu(size_t input, float negative_slope = 0.01f); + size_t bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, + size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd); + size_t conv2d_k3s1p1(size_t input, size_t weight); + size_t conv2d_k3s1p1(size_t input, size_t weight, size_t bias); + size_t stats_pool(size_t input); + size_t weighted_stats_pool(size_t input, size_t weights); + + size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20, + const std::unordered_map& logit_bias = {}); + size_t sample_with_options(size_t logits, float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, const std::unordered_map& logit_bias = {}); + + size_t concat(size_t input1, size_t input2, int axis = 0); + size_t cat(const std::vector& inputs, int axis); + size_t scatter_topk(size_t indices, size_t values, size_t num_classes); + + void set_input(size_t node_id, const void* data, Precision precision); + void set_external_input(size_t node_id, void* data, Precision precision); + void* get_output(size_t node_id); + + void execute(const std::string& profile_file = ""); + void hard_reset(); + void soft_reset(); + void soft_reset_keep_pool(); + void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; } + + void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + const std::vector& get_debug_nodes() const; + void clear_debug_nodes(); + + size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); + const BufferDesc& get_output_buffer(size_t node_id) const; + void allocate_buffers(); + size_t get_node_count() const; + + size_t persistent(size_t source_node); + bool is_populated(size_t persistent_node_id) const; + void invalidate_persistent(size_t persistent_node_id); + + std::vector> nodes_; + std::unordered_map node_index_map_; + +private: + static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); + size_t next_node_id_; + std::vector> mapped_files_; + std::unordered_map weight_cache_; + std::unordered_map node_to_mapped_file_; + std::vector debug_nodes_; + BufferPool buffer_pool_; + bool prefill_mode_ = false; + + std::unordered_set persistent_node_ids_; + std::unordered_set populated_node_ids_; +}; + + +namespace GraphFile { + struct LoadedNode { + size_t node_id; + std::vector shape; + Precision precision; + size_t byte_size; + }; + + struct GraphHeader { + uint32_t magic; + uint32_t version; + uint32_t node_count; + uint32_t flags = 0; + }; + + struct NodeEntry { + uint32_t index; // serialized node index 0..n-1 + OpType op_type; + std::vector inputs; + std::vector output_shape; + Precision precision; + OpParams params; + }; + + struct SerializedGraph { + GraphHeader header; + std::vector nodes; + std::vector graph_inputs; // IDs of serialized inputs + std::vector graph_outputs; // IDs of serialized outputs + }; + + SerializedGraph load_graph(const std::string& filename); + void save_graph(const CactusGraph& graph, const std::string& filename); + + void save_node(CactusGraph& graph, size_t node_id, const std::string& filename); + + class MappedFile { + public: + MappedFile(const std::string& filename); + ~MappedFile(); + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + + const std::vector& shape() const; + Precision precision() const; + size_t byte_size() const; + + size_t group_size() const { return group_size_; } + size_t num_groups() const { return num_groups_; } + const void* scales_data() const; + + bool is_interleaved() const { return is_interleaved_; } + size_t original_N() const { return original_N_; } + + void* data(); + const void* data() const; + + template + const T* typed_data() const; + + void release_pages(); + void prefetch_pages(); + + private: + int fd_; + void* mapped_data_; + size_t file_size_, data_offset_; + std::vector shape_; + Precision precision_; + size_t byte_size_; + size_t group_size_ = 0; + size_t num_groups_ = 0; + size_t scales_offset_ = 0; + size_t scales_bytes_ = 0; + uint32_t alignment_ = 32; + + bool is_interleaved_ = false; + size_t original_N_ = 0; + + void parse_header(); + void apply_madvise_hints(); + }; +} + +#endif diff --git a/cactus-transpile/graph/graph_builder.cpp b/cactus-transpile/graph/graph_builder.cpp new file mode 100644 index 000000000..0710a1485 --- /dev/null +++ b/cactus-transpile/graph/graph_builder.cpp @@ -0,0 +1,1879 @@ +#include "graph.h" +#include +#include +#include +#include + +namespace { + +size_t conv2d_output_dim(size_t input, size_t kernel, size_t stride, size_t padding, size_t dilation) { + if (stride == 0) { + throw std::runtime_error("conv2d stride must be > 0"); + } + if (dilation == 0) { + throw std::runtime_error("conv2d dilation must be > 0"); + } + const size_t effective_kernel = dilation * (kernel - 1) + 1; + if (input + 2 * padding < effective_kernel) { + throw std::runtime_error("conv2d input spatial dimensions are too small for the requested kernel"); + } + return (input + 2 * padding - effective_kernel) / stride + 1; +} + +void validate_conv2d_weight_shape( + const BufferDesc& xin, + const BufferDesc& w, + size_t groups, + const char* op_name +) { + if (xin.shape.size() != 4) { + throw std::runtime_error(std::string(op_name) + " expects input [N, C_in, H, W]"); + } + if (w.shape.size() != 4) { + throw std::runtime_error(std::string(op_name) + " weight must be [C_out, C_in/groups, K_h, K_w]"); + } + if (groups == 0) { + throw std::runtime_error(std::string(op_name) + " groups must be > 0"); + } + + const size_t c_in = xin.shape[1]; + const size_t c_out = w.shape[0]; + const size_t c_in_per_group = w.shape[1]; + if ((c_in % groups) != 0) { + throw std::runtime_error(std::string(op_name) + " input channels must be divisible by groups"); + } + if ((c_out % groups) != 0) { + throw std::runtime_error(std::string(op_name) + " output channels must be divisible by groups"); + } + if (c_in_per_group * groups != c_in) { + throw std::runtime_error(std::string(op_name) + " weight must match [C_out, C_in/groups, K_h, K_w]"); + } + if (xin.shape[2] == 0 || xin.shape[3] == 0) { + throw std::runtime_error(std::string(op_name) + " input spatial dimensions must be > 0"); + } +} + +int normalize_reduction_axis(const BufferDesc& input_buffer, int axis, const char* op_name) { + if (input_buffer.shape.empty()) { + if (axis == 0 || axis == -1) { + return 0; + } + throw std::runtime_error(std::string(op_name) + " axis out of range for scalar input"); + } + + int actual_axis = axis; + if (actual_axis < 0) { + actual_axis += static_cast(input_buffer.shape.size()); + } + if (actual_axis < 0 || static_cast(actual_axis) >= input_buffer.shape.size()) { + throw std::runtime_error(std::string(op_name) + " axis out of range"); + } + return actual_axis; +} + +std::vector reduction_output_shape(const BufferDesc& input_buffer, int axis) { + std::vector output_shape = input_buffer.shape; + output_shape.erase(output_shape.begin() + axis); + if (output_shape.empty()) { + output_shape = {1}; + } + return output_shape; +} + +} // namespace + +size_t CactusGraph::input(const std::vector& shape, Precision precision) { + return add_node(OpType::INPUT, {}, shape, {.output_precision = precision}); +} + +size_t CactusGraph::add(size_t input1, size_t input2) { + const auto& lhs_buffer = get_output_buffer(input1); + const auto& rhs_buffer = get_output_buffer(input2); + BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); + OpParams params{.broadcast_info = broadcast_info}; + + return add_node(OpType::ADD, {input1, input2}, broadcast_info.output_shape, params); +} + +size_t CactusGraph::add_clipped(size_t input1, size_t input2) { + const auto& lhs_buffer = get_output_buffer(input1); + const auto& rhs_buffer = get_output_buffer(input2); + + BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); + OpParams params{.broadcast_info = broadcast_info}; + + return add_node(OpType::ADD_CLIPPED, {input1, input2}, broadcast_info.output_shape, params); +} + +size_t CactusGraph::subtract(size_t input1, size_t input2) { + const auto& lhs_buffer = get_output_buffer(input1); + const auto& rhs_buffer = get_output_buffer(input2); + + BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); + OpParams params{.broadcast_info = broadcast_info}; + + return add_node(OpType::SUBTRACT, {input1, input2}, broadcast_info.output_shape, params); +} + +size_t CactusGraph::multiply(size_t input1, size_t input2) { + const auto& lhs_buffer = get_output_buffer(input1); + const auto& rhs_buffer = get_output_buffer(input2); + + BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); + OpParams params{.broadcast_info = broadcast_info}; + + return add_node(OpType::MULTIPLY, {input1, input2}, broadcast_info.output_shape, params); +} + +size_t CactusGraph::divide(size_t input1, size_t input2) { + const auto& lhs_buffer = get_output_buffer(input1); + const auto& rhs_buffer = get_output_buffer(input2); + + BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); + OpParams params{.broadcast_info = broadcast_info}; + + return add_node(OpType::DIVIDE, {input1, input2}, broadcast_info.output_shape, params); +} + +size_t CactusGraph::not_equal(size_t input1, size_t input2) { + const auto& lhs_buffer = get_output_buffer(input1); + const auto& rhs_buffer = get_output_buffer(input2); + + BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); + OpParams params{.broadcast_info = broadcast_info, .output_precision = Precision::FP16}; + + return add_node(OpType::NOT_EQUAL, {input1, input2}, broadcast_info.output_shape, params); +} + +size_t CactusGraph::abs(size_t input) { + const auto& input_buffer = get_output_buffer(input); + OpParams params{.output_precision = input_buffer.precision}; + return add_node(OpType::ABS, {input}, input_buffer.shape, params); +} + +size_t CactusGraph::pow(size_t input, float exponent) { + const auto& input_buffer = get_output_buffer(input); + OpParams params{.scalar = exponent, .output_precision = input_buffer.precision}; + return add_node(OpType::POW, {input}, input_buffer.shape, params); +} + +size_t CactusGraph::view(size_t input, const std::vector& new_shape) { + const auto& input_buffer = get_output_buffer(input); + + size_t input_elements = 1; + for (size_t dim : input_buffer.shape) { + input_elements *= dim; + } + + size_t new_elements = 1; + for (size_t dim : new_shape) { + new_elements *= dim; + } + + if (input_elements != new_elements) { + throw std::runtime_error("View operation requires total number of elements to remain the same"); + } + + OpParams params{.new_shape = new_shape}; + return add_node(OpType::VIEW, {input}, new_shape, params); +} + +size_t CactusGraph::flatten(size_t input, int start_dim, int end_dim) { + const auto& input_buffer = get_output_buffer(input); + const auto& shape = input_buffer.shape; + size_t rank = shape.size(); + + if (start_dim < 0) start_dim += rank; + if (end_dim < 0) end_dim += rank; + + if (start_dim < 0 || static_cast(start_dim) >= rank || + end_dim < 0 || static_cast(end_dim) >= rank || + start_dim > end_dim) { + throw std::runtime_error("Invalid start_dim or end_dim for flatten operation"); + } + + std::vector output_shape; + + for (int i = 0; i < start_dim; ++i) { + output_shape.push_back(shape[i]); + } + + size_t flattened_dim = 1; + for (int i = start_dim; i <= end_dim; ++i) { + flattened_dim *= shape[i]; + } + output_shape.push_back(flattened_dim); + + for (size_t i = end_dim + 1; i < rank; ++i) { + output_shape.push_back(shape[i]); + } + + OpParams params{.new_shape = output_shape}; + return add_node(OpType::FLATTEN, {input}, output_shape, params); +} + +size_t CactusGraph::matmul( + size_t input1, + size_t input2, + bool pretransposed_rhs, + ComputeBackend backend, + Precision output_precision +) { + const auto& lhs_buffer = get_output_buffer(input1); + const auto& rhs_buffer = get_output_buffer(input2); + + if (lhs_buffer.shape.size() != 2 || rhs_buffer.shape.size() != 2) { + throw std::invalid_argument("Matrix multiplication requires 2D tensors"); + } + + size_t M = lhs_buffer.shape[0]; + size_t K = lhs_buffer.shape[1]; + size_t rhs_K = pretransposed_rhs ? rhs_buffer.shape[1] : rhs_buffer.shape[0]; + + // For interleaved weights, use original_N (shape stores padded N) + size_t N; + if (rhs_buffer.is_interleaved && rhs_buffer.original_N > 0) { + N = rhs_buffer.original_N; + } else { + N = pretransposed_rhs ? rhs_buffer.shape[0] : rhs_buffer.shape[1]; + } + + if (K != rhs_K) { + std::cout << "Matrix dimensions incompatible for multiplication: " + << "lhs=[" << M << "," << K << "] rhs=[" << rhs_buffer.shape[0] << "," << rhs_buffer.shape[1] << "]" + << " pretransposed=" << pretransposed_rhs + << " (K=" << K << " != rhs_K=" << rhs_K << ")" << std::endl; + throw std::invalid_argument("Matrix dimensions incompatible for multiplication"); + } + + std::vector output_shape = {M, N}; + OpParams params{ + .pretransposed_rhs = pretransposed_rhs, + .output_precision = output_precision, + .backend = backend, + }; + return add_node(OpType::MATMUL, {input1, input2}, output_shape, params); +} + +size_t CactusGraph::transpose(size_t input, ComputeBackend backend) { + const auto& input_buffer = get_output_buffer(input); + std::vector output_shape = input_buffer.shape; + + if (output_shape.size() >= 2) { + std::swap(output_shape[output_shape.size()-2], output_shape[output_shape.size()-1]); + } + + std::vector permutation; + for (size_t i = 0; i < input_buffer.shape.size(); ++i) { + permutation.push_back(i); + } + if (permutation.size() >= 2) { + std::swap(permutation[permutation.size()-2], permutation[permutation.size()-1]); + } + + OpParams params{.permutation = permutation, .backend = backend}; + return add_node(OpType::TRANSPOSE, {input}, output_shape, params); +} + +size_t CactusGraph::transposeN(size_t input, const std::vector& permutation, ComputeBackend backend) { + const auto& input_buffer = get_output_buffer(input); + if (permutation.size() != input_buffer.shape.size()) { + throw std::runtime_error("transposeN permutation size must match tensor rank"); + } + std::vector output_shape(permutation.size()); + for (size_t i = 0; i < permutation.size(); ++i) { + size_t p = permutation[i]; + if (p >= input_buffer.shape.size()) { + throw std::runtime_error("transposeN permutation index out of range"); + } + output_shape[i] = input_buffer.shape[p]; + } + OpParams params{.permutation = permutation, .backend = backend}; + return add_node(OpType::TRANSPOSE, {input}, output_shape, params); +} + +size_t CactusGraph::reshape(size_t input, const std::vector& new_shape) { + OpParams params{.new_shape = new_shape}; + return add_node(OpType::RESHAPE, {input}, new_shape, params); +} + +size_t CactusGraph::expand(size_t input, const std::vector& new_shape) { + const auto& input_buffer = get_output_buffer(input); + const auto& input_shape = input_buffer.shape; + + if (new_shape.size() < input_shape.size()) { + throw std::invalid_argument("expand cannot reduce rank"); + } + + const size_t rank_delta = new_shape.size() - input_shape.size(); + for (size_t axis = 0; axis < new_shape.size(); ++axis) { + const size_t input_dim = axis < rank_delta ? 1 : input_shape[axis - rank_delta]; + const size_t output_dim = new_shape[axis]; + if (input_dim != 1 && input_dim != output_dim) { + throw std::invalid_argument("expand target shape is not broadcast-compatible with input"); + } + } + + OpParams params{}; + params.output_precision = input_buffer.precision; + return add_node(OpType::EXPAND, {input}, new_shape, params); +} + +size_t CactusGraph::index(size_t input, size_t index_value, int dim) { + const auto& input_buffer = get_output_buffer(input); + const auto& shape = input_buffer.shape; + + if (shape.empty()) { + throw std::invalid_argument("Cannot index a scalar tensor"); + } + + int actual_dim = dim; + if (actual_dim < 0) { + actual_dim += static_cast(shape.size()); + } + + if (actual_dim < 0 || static_cast(actual_dim) >= shape.size()) { + throw std::invalid_argument("Index dimension out of bounds"); + } + + if (index_value >= shape[actual_dim]) { + throw std::invalid_argument("Index value " + std::to_string(index_value) + + " out of bounds for dimension " + std::to_string(actual_dim) + + " with size " + std::to_string(shape[actual_dim])); + } + + std::vector output_shape; + for (size_t i = 0; i < shape.size(); ++i) { + if (static_cast(i) != actual_dim) { + output_shape.push_back(shape[i]); + } + } + + if (output_shape.empty()) { + output_shape = {1}; + } + + OpParams params{.axis = actual_dim, .output_precision = input_buffer.precision, .index_value = index_value}; + return add_node(OpType::INDEX, {input}, output_shape, params); +} + +size_t CactusGraph::masked_select_prefix(size_t input, size_t mask) { + const auto& input_buffer = get_output_buffer(input); + const auto& mask_buffer = get_output_buffer(mask); + + if (mask_buffer.shape.empty()) { + throw std::invalid_argument("masked_select_prefix requires a non-scalar mask"); + } + if (input_buffer.shape.size() < mask_buffer.shape.size()) { + throw std::invalid_argument("masked_select_prefix mask rank must not exceed input rank"); + } + for (size_t i = 0; i < mask_buffer.shape.size(); ++i) { + if (input_buffer.shape[i] != mask_buffer.shape[i]) { + throw std::invalid_argument("masked_select_prefix mask shape must match the input prefix shape"); + } + } + + std::vector output_shape; + output_shape.push_back(mask_buffer.total_size); + for (size_t i = mask_buffer.shape.size(); i < input_buffer.shape.size(); ++i) { + output_shape.push_back(input_buffer.shape[i]); + } + + OpParams params{}; + params.output_precision = input_buffer.precision; + return add_node(OpType::MASKED_SELECT_PREFIX, {input, mask}, output_shape, params); +} + +size_t CactusGraph::masked_scatter(size_t input, size_t mask, size_t source) { + const auto& input_buffer = get_output_buffer(input); + const auto& mask_buffer = get_output_buffer(mask); + const auto& source_buffer = get_output_buffer(source); + + if (input_buffer.shape != mask_buffer.shape) { + throw std::invalid_argument("masked_scatter mask shape must match input shape"); + } + if (input_buffer.precision != source_buffer.precision) { + throw std::invalid_argument("masked_scatter source precision must match input precision"); + } + + OpParams params{}; + params.output_precision = input_buffer.precision; + return add_node(OpType::MASKED_SCATTER, {input, mask, source}, input_buffer.shape, params); +} + +size_t CactusGraph::sum(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + const int actual_axis = normalize_reduction_axis(input_buffer, axis, "sum"); + const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); + OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; + return add_node(OpType::SUM, {input}, output_shape, params); +} + +size_t CactusGraph::mean(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + const int actual_axis = normalize_reduction_axis(input_buffer, axis, "mean"); + const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); + OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; + return add_node(OpType::MEAN, {input}, output_shape, params); +} + +size_t CactusGraph::variance(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + const int actual_axis = normalize_reduction_axis(input_buffer, axis, "variance"); + const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); + OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; + return add_node(OpType::VARIANCE, {input}, output_shape, params); +} + +size_t CactusGraph::min(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + const int actual_axis = normalize_reduction_axis(input_buffer, axis, "min"); + const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); + OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; + return add_node(OpType::MIN, {input}, output_shape, params); +} + +size_t CactusGraph::max(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + const int actual_axis = normalize_reduction_axis(input_buffer, axis, "max"); + const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); + OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; + return add_node(OpType::MAX, {input}, output_shape, params); +} + +size_t CactusGraph::cumsum(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + const int actual_axis = normalize_reduction_axis(input_buffer, axis, "cumsum"); + OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; + return add_node(OpType::CUMSUM, {input}, input_buffer.shape, params); +} + +size_t CactusGraph::rms_norm(size_t input, size_t weight, float epsilon) { + OpParams params{.epsilon = epsilon}; + return add_node(OpType::RMS_NORM, {input, weight}, {}, params); +} + +size_t CactusGraph::rope(size_t input, float theta, size_t position_offset, ComputeBackend backend) { + OpParams params{.theta = theta, .position_offset = position_offset, .backend = backend}; + return add_node(OpType::ROPE, {input}, {}, params); +} + +size_t CactusGraph::softmax(size_t input, int axis) { + OpParams params{.axis = axis}; + return add_node(OpType::SOFTMAX, {input}, {}, params); +} + +size_t CactusGraph::topk(size_t input, size_t k) { + const auto& input_buffer = get_output_buffer(input); + + if (input_buffer.shape.empty()) { + throw std::runtime_error("TopK requires non-empty input tensor"); + } + + std::vector output_shape = {2, input_buffer.shape[0], k}; + OpParams params{.output_precision = Precision::FP32, .top_k = k}; + + return add_node(OpType::TOPK, {input}, output_shape, params); +} + +size_t CactusGraph::moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w3_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation, + size_t per_expert_scale) { + const auto& hidden_buffer = get_output_buffer(hidden); + const auto& routing_buffer = get_output_buffer(routing_probs); + const auto& topk_buffer = get_output_buffer(topk_indices); + + if (hidden_buffer.shape.size() != 2) { + throw std::runtime_error("moe_layer expects [tokens, hidden_dim] for hidden"); + } + if (routing_buffer.shape.size() != 2 || topk_buffer.shape.size() != 2) { + throw std::runtime_error("moe_layer expects 2D routing_probs and topk_indices"); + } + if (routing_buffer.shape[0] != hidden_buffer.shape[0] || topk_buffer.shape[0] != hidden_buffer.shape[0]) { + throw std::runtime_error("moe_layer token dimension mismatch across inputs"); + } + if (w1_weights.size() != num_experts || w3_weights.size() != num_experts || w2_weights.size() != num_experts) { + throw std::runtime_error("moe_layer expects num_experts weight tensors for each of w1, w3, w2"); + } + + std::vector input_ids; + input_ids.reserve(3 + 3 * num_experts + 1); + input_ids.push_back(hidden); + input_ids.push_back(routing_probs); + input_ids.push_back(topk_indices); + for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w1_weights[i]); + for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w3_weights[i]); + for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w2_weights[i]); + if (per_expert_scale != 0) input_ids.push_back(per_expert_scale); + + OpParams params; + params.num_experts = num_experts; + params.num_experts_per_tok = num_experts_per_tok; + params.normalize_routing = normalize_routing; + params.epsilon = epsilon; + params.scalar = routed_scaling_factor; + params.output_precision = hidden_buffer.precision; + params.activation = activation; + params.moe_gated = true; + + return add_node(OpType::MOE_LAYER, input_ids, hidden_buffer.shape, params); +} + +size_t CactusGraph::moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation) { + const auto& hidden_buffer = get_output_buffer(hidden); + const auto& routing_buffer = get_output_buffer(routing_probs); + const auto& topk_buffer = get_output_buffer(topk_indices); + + if (hidden_buffer.shape.size() != 2) { + throw std::runtime_error("moe_layer expects [tokens, hidden_dim] for hidden"); + } + if (routing_buffer.shape.size() != 2 || topk_buffer.shape.size() != 2) { + throw std::runtime_error("moe_layer expects 2D routing_probs and topk_indices"); + } + if (routing_buffer.shape[0] != hidden_buffer.shape[0] || topk_buffer.shape[0] != hidden_buffer.shape[0]) { + throw std::runtime_error("moe_layer token dimension mismatch across inputs"); + } + if (w1_weights.size() != num_experts || w2_weights.size() != num_experts) { + throw std::runtime_error("moe_layer expects num_experts weight tensors for each of w1, w2"); + } + + std::vector input_ids; + input_ids.reserve(3 + 2 * num_experts); + input_ids.push_back(hidden); + input_ids.push_back(routing_probs); + input_ids.push_back(topk_indices); + for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w1_weights[i]); + for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w2_weights[i]); + + OpParams params; + params.num_experts = num_experts; + params.num_experts_per_tok = num_experts_per_tok; + params.normalize_routing = normalize_routing; + params.epsilon = epsilon; + params.scalar = routed_scaling_factor; + params.output_precision = hidden_buffer.precision; + params.moe_gated = false; + params.activation = activation; + + return add_node(OpType::MOE_LAYER, input_ids, hidden_buffer.shape, params); +} + +size_t CactusGraph::layernorm(size_t input, size_t weight, size_t bias, float epsilon) { + OpParams params{.epsilon = epsilon}; + return add_node(OpType::LAYERNORM, {input, weight, bias}, {}, params); +} + +size_t CactusGraph::layernorm(size_t input, size_t weight, float epsilon) { + OpParams params{.epsilon = epsilon}; + return add_node(OpType::LAYERNORM, {input, weight}, {}, params); +} + +size_t CactusGraph::groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups, float epsilon) { + OpParams params{.epsilon = epsilon, .num_groups = num_groups}; + return add_node(OpType::GROUPNORM, {input, weight, bias}, {}, params); +} + +size_t CactusGraph::batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis, float epsilon) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + const auto& rm = get_output_buffer(running_mean); + const auto& rv = get_output_buffer(running_var); + + if (xin.shape.empty()) { + throw std::runtime_error("batchnorm expects non-scalar input"); + } + + int actual_axis = axis; + if (actual_axis < 0) { + actual_axis += static_cast(xin.shape.size()); + } + if (actual_axis < 0 || static_cast(actual_axis) >= xin.shape.size()) { + throw std::runtime_error("batchnorm axis out of range"); + } + + const size_t C = xin.shape[static_cast(actual_axis)]; + if (w.total_size != C || b.total_size != C || rm.total_size != C || rv.total_size != C) { + throw std::runtime_error("batchnorm parameter size mismatch"); + } + + OpParams params{.epsilon = epsilon, .axis = actual_axis}; + params.output_precision = xin.precision; + return add_node(OpType::BATCHNORM, {input, weight, bias, running_mean, running_var}, xin.shape, params); +} + +size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, bool is_causal, ComputeBackend backend) { + OpParams params{.scale = scale, .is_causal = is_causal, .backend = backend}; + const auto& qs = get_output_buffer(query).shape; + const auto& vs = get_output_buffer(value).shape; + return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); +} + +size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend) { + OpParams params{.scale = scale, .position_offset = position_offset, .backend = backend}; + const auto& qs = get_output_buffer(query).shape; + const auto& vs = get_output_buffer(value).shape; + return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); +} + +size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend) { + OpParams params{.scale = scale, .position_offset = position_offset, .window_size = window_size, .backend = backend}; + const auto& qs = get_output_buffer(query).shape; + const auto& vs = get_output_buffer(value).shape; + return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); +} + +size_t CactusGraph::attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, + bool is_causal, ComputeBackend backend, bool additive_mask, + size_t position_offset, size_t window_size, float logit_cap) { + OpParams params{ + .scale = scale, + .position_offset = position_offset, + .window_size = window_size, + .is_causal = is_causal, + .backend = backend + }; + params.attention_mask_is_additive = additive_mask; + params.logit_cap = logit_cap; + const auto& qs = get_output_buffer(query).shape; + const auto& vs = get_output_buffer(value).shape; + return add_node(OpType::ATTENTION, {query, key, value, mask}, {qs[0], qs[1], qs[2], vs[3]}, params); +} + +size_t CactusGraph::rel_pos_bias(size_t query, size_t relative_key, float scale) { + const auto& q = get_output_buffer(query); + const auto& r = get_output_buffer(relative_key); + + if (q.shape.size() != 4) { + throw std::runtime_error("rel_pos_bias expects query [B, T, H, D]"); + } + if (r.shape.size() != 4) { + throw std::runtime_error("rel_pos_bias expects relative_key [B, R, H, D]"); + } + + const size_t B = q.shape[0]; + const size_t T = q.shape[1]; + const size_t H = q.shape[2]; + const size_t D = q.shape[3]; + const size_t R = r.shape[1]; + + if (r.shape[0] != 1 && r.shape[0] != B) { + throw std::runtime_error("rel_pos_bias relative_key batch must be 1 or match query batch"); + } + if (r.shape[2] != H || r.shape[3] != D) { + throw std::runtime_error("rel_pos_bias expects matching [H, D] between query and relative_key"); + } + if (R < (2 * T - 1)) { + throw std::runtime_error("rel_pos_bias requires relative_key length >= 2*T-1"); + } + + OpParams params{.scale = scale, .output_precision = q.precision}; + return add_node(OpType::REL_POS_BIAS, {query, relative_key}, {B, H, T, T}, params); +} + +size_t CactusGraph::attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, + const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, + size_t window_size, size_t v_head_dim) { + OpParams params; + params.scale = scale; + params.position_offset = position_offset; + params.window_size = window_size; + params.cached_keys_int8 = cached_keys; + params.cached_values_int8 = cached_values; + params.cached_k_scales = k_scales; + params.cached_v_scales = v_scales; + params.cache_seq_len = cache_len; + params.num_kv_heads = num_kv_heads; + params.head_dim = head_dim; + params.v_head_dim = v_head_dim; + std::vector out_shape; + if (v_head_dim != 0 && v_head_dim != head_dim) { + const auto& q_buf = get_output_buffer(query); + out_shape = {q_buf.shape[0], q_buf.shape[1], q_buf.shape[2], v_head_dim}; + } + return add_node(OpType::ATTENTION_INT8_HYBRID, {query, key_new, value_new}, out_shape, params); +} + +size_t CactusGraph::conv1d_causal(size_t input, size_t weight, size_t, size_t dilation) { + OpParams params{.dilation = dilation}; + return add_node(OpType::CONV1D_CAUSAL, {input, weight}, {}, params); +} + +size_t CactusGraph::conv1d_k3(size_t input, size_t weight, size_t stride) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 3) throw std::runtime_error("conv1d_k3 expects N,C,L"); + if (w.shape.size() != 3) throw std::runtime_error("weight must be [C_out,C_in,3]"); + if (w.shape[1] != xin.shape[1]) throw std::runtime_error("C_in mismatch in conv1d_k3"); + if (w.shape[2] != 3) throw std::runtime_error("K=3 expected in conv1d_k3"); + + const size_t N = xin.shape[0]; + const size_t L = xin.shape[2]; + const size_t C_out= w.shape[0]; + const size_t K = w.shape[2]; + + const size_t pad = 1; + const size_t L_out = (L + 2 * pad - K) / stride + 1; + + OpParams params{}; + params.stride = stride; + params.output_precision = xin.precision; + + std::vector out_shape{N, C_out, L_out}; + return add_node(OpType::CONV1D_K3, {input, weight}, out_shape, params); +} + +size_t CactusGraph::conv1d_k7s3(size_t input, size_t weight, size_t bias) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + if (xin.shape.size() != 3) throw std::runtime_error("conv1d_k7s3 expects N,C,L"); + if (w.shape.size() != 3) throw std::runtime_error("weight must be [C_in, 7, C_out]"); + if (w.shape[0] != xin.shape[1]) throw std::runtime_error("C_in mismatch in conv1d_k7s3"); + if (w.shape[1] != 7) throw std::runtime_error("K=7 expected in conv1d_k7s3"); + + size_t C_out = w.shape[2]; + if (b.total_size != C_out) throw std::runtime_error("Bias size mismatch"); + + const size_t N = xin.shape[0]; + const size_t L = xin.shape[2]; + const size_t K = 7; + const size_t stride = 3; + + const size_t L_out = (L < K) ? 0 : (L - K) / stride + 1; + + OpParams params{}; + params.stride = stride; + params.output_precision = xin.precision; + + std::vector out_shape{N, C_out, L_out}; + return add_node(OpType::CONV1D_K7S3, {input, weight, bias}, out_shape, params); +} + +size_t CactusGraph::conv1d(size_t input, size_t weight, size_t stride) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 3) throw std::runtime_error("conv1d expects N,C,L"); + if (w.shape.size() != 3) throw std::runtime_error("conv1d weight expects [C_out, C_in, K]"); + + size_t N = xin.shape[0]; + size_t C_out = w.shape[0]; + size_t L = xin.shape[2]; + size_t K = w.shape[2]; + size_t L_out = (L - K) / stride + 1; + + OpParams params{.stride = stride}; + return add_node(OpType::CONV1D, {input, weight}, {N, C_out, L_out}, params); +} + +size_t CactusGraph::conv1d(size_t input, size_t weight, size_t bias, size_t stride) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + if (xin.shape.size() != 3) throw std::runtime_error("conv1d expects N,C,L"); + if (w.shape.size() != 3) throw std::runtime_error("conv1d weight expects [C_out, C_in, K]"); + + size_t N = xin.shape[0]; + size_t C_out = w.shape[0]; + size_t L = xin.shape[2]; + size_t K = w.shape[2]; + size_t L_out = (L - K) / stride + 1; + if (b.total_size != C_out) throw std::runtime_error("conv1d bias size mismatch"); + + OpParams params{.output_precision = xin.precision, .stride = stride}; + return add_node(OpType::CONV1D, {input, weight, bias}, {N, C_out, L_out}, params); +} + +size_t CactusGraph::conv1d_same_depthwise_k9(size_t input, size_t weight) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 3) { + throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); + } + + const size_t N = xin.shape[0]; + const size_t L = xin.shape[1]; + const size_t C = xin.shape[2]; + + if (w.shape.size() == 2) { + if (w.shape[0] != C || w.shape[1] != 9) { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); + } + } else if (w.shape.size() == 3) { + if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 9) { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); + } + } else { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); + } + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV1D_SAME_DEPTHWISE_K9, {input, weight}, {N, L, C}, params); +} + +size_t CactusGraph::conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + if (xin.shape.size() != 3) { + throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); + } + + const size_t N = xin.shape[0]; + const size_t L = xin.shape[1]; + const size_t C = xin.shape[2]; + + if (w.shape.size() == 2) { + if (w.shape[0] != C || w.shape[1] != 9) { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); + } + } else if (w.shape.size() == 3) { + if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 9) { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); + } + } else { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); + } + + if (b.total_size != C) { + throw std::runtime_error("conv1d_same_depthwise_k9 bias size mismatch"); + } + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV1D_SAME_DEPTHWISE_K9, {input, weight, bias}, {N, L, C}, params); +} + +size_t CactusGraph::conv1d_pointwise(size_t input, size_t weight) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 3) { + throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); + } + + const size_t N = xin.shape[0]; + const size_t L = xin.shape[1]; + const size_t C_in = xin.shape[2]; + size_t C_out = 0; + + if (w.shape.size() == 2) { + C_out = w.shape[0]; + if (w.shape[1] != C_in) { + throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); + } + } else if (w.shape.size() == 3) { + C_out = w.shape[0]; + if (w.shape[1] != C_in || w.shape[2] != 1) { + throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); + } + } else { + throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); + } + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV1D_POINTWISE, {input, weight}, {N, L, C_out}, params); +} + +size_t CactusGraph::conv1d_pointwise(size_t input, size_t weight, size_t bias) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + if (xin.shape.size() != 3) { + throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); + } + + const size_t C_in = xin.shape[2]; + size_t C_out = 0; + + if (w.shape.size() == 2) { + C_out = w.shape[0]; + if (w.shape[1] != C_in) { + throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); + } + } else if (w.shape.size() == 3) { + C_out = w.shape[0]; + if (w.shape[1] != C_in || w.shape[2] != 1) { + throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); + } + } else { + throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); + } + + if (b.total_size != C_out) { + throw std::runtime_error("conv1d_pointwise bias size mismatch"); + } + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV1D_POINTWISE, {input, weight, bias}, {xin.shape[0], xin.shape[1], C_out}, params); +} + +size_t CactusGraph::conv2d_k3s2p1(size_t input, size_t weight) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); + } + if (w.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s2p1 weight must be [C_out, C_in, 3, 3]"); + } + + const size_t N = xin.shape[0]; + const size_t C_in = xin.shape[1]; + const size_t H = xin.shape[2]; + const size_t W = xin.shape[3]; + const size_t C_out = w.shape[0]; + + if (w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { + throw std::runtime_error("conv2d_k3s2p1 weight must match [C_out, C_in, 3, 3]"); + } + if (H == 0 || W == 0) { + throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); + } + + const size_t H_out = (H - 1) / 2 + 1; + const size_t W_out = (W - 1) / 2 + 1; + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV2D_K3S2P1, {input, weight}, {N, C_out, H_out, W_out}, params); +} + +size_t CactusGraph::conv2d_k3s2p1(size_t input, size_t weight, size_t bias) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + if (xin.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); + } + + const size_t C_in = xin.shape[1]; + const size_t H = xin.shape[2]; + const size_t W = xin.shape[3]; + const size_t C_out = w.shape[0]; + + if (w.shape.size() != 4 || w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { + throw std::runtime_error("conv2d_k3s2p1 weight must be [C_out, C_in, 3, 3]"); + } + if (b.total_size != C_out) { + throw std::runtime_error("conv2d_k3s2p1 bias size mismatch"); + } + if (H == 0 || W == 0) { + throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); + } + + const size_t N = xin.shape[0]; + const size_t H_out = (H - 1) / 2 + 1; + const size_t W_out = (W - 1) / 2 + 1; + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV2D_K3S2P1, {input, weight, bias}, {N, C_out, H_out, W_out}, params); +} + +size_t CactusGraph::conv2d_depthwise_k3s2p1(size_t input, size_t weight) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 4) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); + } + + const size_t N = xin.shape[0]; + const size_t C = xin.shape[1]; + const size_t H = xin.shape[2]; + const size_t W = xin.shape[3]; + + if (w.shape.size() == 3) { + if (w.shape[0] != C || w.shape[1] != 3 || w.shape[2] != 3) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); + } + } else if (w.shape.size() == 4) { + if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 3 || w.shape[3] != 3) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); + } + } else { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); + } + + if (H == 0 || W == 0) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); + } + + const size_t H_out = (H - 1) / 2 + 1; + const size_t W_out = (W - 1) / 2 + 1; + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV2D_DEPTHWISE_K3S2P1, {input, weight}, {N, C, H_out, W_out}, params); +} + +size_t CactusGraph::conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + if (xin.shape.size() != 4) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); + } + + const size_t C = xin.shape[1]; + const size_t H = xin.shape[2]; + const size_t W = xin.shape[3]; + + if (w.shape.size() == 3) { + if (w.shape[0] != C || w.shape[1] != 3 || w.shape[2] != 3) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); + } + } else if (w.shape.size() == 4) { + if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 3 || w.shape[3] != 3) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); + } + } else { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); + } + + if (b.total_size != C) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 bias size mismatch"); + } + if (H == 0 || W == 0) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); + } + + const size_t N = xin.shape[0]; + const size_t H_out = (H - 1) / 2 + 1; + const size_t W_out = (W - 1) / 2 + 1; + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV2D_DEPTHWISE_K3S2P1, {input, weight, bias}, {N, C, H_out, W_out}, params); +} + +size_t CactusGraph::conv2d_pointwise_1x1(size_t input, size_t weight) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 4) { + throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); + } + + const size_t N = xin.shape[0]; + const size_t C_in = xin.shape[1]; + const size_t H = xin.shape[2]; + const size_t W = xin.shape[3]; + size_t C_out = 0; + + if (w.shape.size() == 2) { + C_out = w.shape[0]; + if (w.shape[1] != C_in) { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); + } + } else if (w.shape.size() == 4) { + C_out = w.shape[0]; + if (w.shape[1] != C_in || w.shape[2] != 1 || w.shape[3] != 1) { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); + } + } else { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); + } + + if (H == 0 || W == 0) { + throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); + } + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV2D_POINTWISE_1X1, {input, weight}, {N, C_out, H, W}, params); +} + +size_t CactusGraph::conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + if (xin.shape.size() != 4) { + throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); + } + + const size_t C_in = xin.shape[1]; + const size_t H = xin.shape[2]; + const size_t W = xin.shape[3]; + size_t C_out = 0; + + if (w.shape.size() == 2) { + C_out = w.shape[0]; + if (w.shape[1] != C_in) { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); + } + } else if (w.shape.size() == 4) { + C_out = w.shape[0]; + if (w.shape[1] != C_in || w.shape[2] != 1 || w.shape[3] != 1) { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); + } + } else { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); + } + + if (b.total_size != C_out) { + throw std::runtime_error("conv2d_pointwise_1x1 bias size mismatch"); + } + if (H == 0 || W == 0) { + throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); + } + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV2D_POINTWISE_1X1, {input, weight, bias}, {xin.shape[0], C_out, H, W}, params); +} + +size_t CactusGraph::conv2d(size_t input, size_t weight, size_t stride, size_t padding, size_t dilation, size_t groups) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + validate_conv2d_weight_shape(xin, w, groups, "conv2d"); + + const size_t n = xin.shape[0]; + const size_t c_out = w.shape[0]; + const size_t h_out = conv2d_output_dim(xin.shape[2], w.shape[2], stride, padding, dilation); + const size_t w_out = conv2d_output_dim(xin.shape[3], w.shape[3], stride, padding, dilation); + + OpParams params{}; + params.output_precision = xin.precision; + params.stride = stride; + params.padding = padding; + params.dilation = dilation; + params.num_groups = groups; + return add_node(OpType::CONV2D, {input, weight}, {n, c_out, h_out, w_out}, params); +} + +size_t CactusGraph::conv2d(size_t input, size_t weight, size_t bias, size_t stride, size_t padding, size_t dilation, size_t groups) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + validate_conv2d_weight_shape(xin, w, groups, "conv2d"); + + const size_t c_out = w.shape[0]; + if (b.total_size != c_out) { + throw std::runtime_error("conv2d bias size mismatch"); + } + + const size_t n = xin.shape[0]; + const size_t h_out = conv2d_output_dim(xin.shape[2], w.shape[2], stride, padding, dilation); + const size_t w_out = conv2d_output_dim(xin.shape[3], w.shape[3], stride, padding, dilation); + + OpParams params{}; + params.output_precision = xin.precision; + params.stride = stride; + params.padding = padding; + params.dilation = dilation; + params.num_groups = groups; + return add_node(OpType::CONV2D, {input, weight, bias}, {n, c_out, h_out, w_out}, params); +} + +size_t CactusGraph::lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh) { + const auto& h_buffer = get_output_buffer(h_prev); + std::vector output_shape = {h_buffer.shape[0], h_buffer.shape[1], 2}; + return add_node(OpType::LSTM_CELL, {input, h_prev, c_prev, weight_ih, weight_hh, bias_ih, bias_hh}, output_shape, {}); +} + +size_t CactusGraph::gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, float scale) { + const auto& q = get_output_buffer(query); + const auto& k = get_output_buffer(key); + const auto& v = get_output_buffer(value); + const auto& g = get_output_buffer(gate_log); + const auto& b = get_output_buffer(beta); + const auto& s = get_output_buffer(initial_state); + + if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { + throw std::runtime_error("gated_deltanet_decode expects query/key/value rank 4 [B, T, H, D]"); + } + if (g.shape.size() != 3 || b.shape.size() != 3) { + throw std::runtime_error("gated_deltanet_decode expects gate_log/beta rank 3 [B, T, H]"); + } + if (s.shape.size() != 4) { + throw std::runtime_error("gated_deltanet_decode expects initial_state rank 4 [B, K, H, V]"); + } + + const size_t B = q.shape[0]; + const size_t T = q.shape[1]; + const size_t Hq = q.shape[2]; + const size_t K = q.shape[3]; + const size_t Hv = v.shape[2]; + const size_t V = v.shape[3]; + if (T != 1) { + throw std::runtime_error("gated_deltanet_decode expects sequence length T=1"); + } + auto is_supported_precision = [](Precision p) { + return p == Precision::FP16 || p == Precision::FP32; + }; + if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || + !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || + !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { + throw std::runtime_error("gated_deltanet_decode requires FP16/FP32 inputs"); + } + + float op_scale = scale; + if (op_scale == 0.0f) { + op_scale = 1.0f / std::sqrt(static_cast(K)); + } + + OpParams params; + params.scale = op_scale; + params.num_kv_heads = Hq; + params.output_precision = Precision::FP16; + return add_node(OpType::GATED_DELTANET_DECODE, + {query, key, value, gate_log, beta, initial_state}, + {B, static_cast(1 + K), Hv, V}, + params); +} + +size_t CactusGraph::gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, size_t chunk_size, float scale) { + const auto& q = get_output_buffer(query); + const auto& k = get_output_buffer(key); + const auto& v = get_output_buffer(value); + const auto& g = get_output_buffer(gate_log); + const auto& b = get_output_buffer(beta); + const auto& s = get_output_buffer(initial_state); + + if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { + throw std::runtime_error("gated_deltanet_prefill expects query/key/value rank 4 [B, T, H, D]"); + } + if (g.shape.size() != 3 || b.shape.size() != 3) { + throw std::runtime_error("gated_deltanet_prefill expects gate_log/beta rank 3 [B, T, H]"); + } + if (s.shape.size() != 4) { + throw std::runtime_error("gated_deltanet_prefill expects initial_state rank 4 [B, K, H, V]"); + } + + const size_t B = q.shape[0]; + const size_t T = q.shape[1]; + const size_t Hq = q.shape[2]; + const size_t K = q.shape[3]; + const size_t Hv = v.shape[2]; + const size_t V = v.shape[3]; + auto is_supported_precision = [](Precision p) { + return p == Precision::FP16 || p == Precision::FP32; + }; + if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || + !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || + !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { + throw std::runtime_error("gated_deltanet_prefill requires FP16/FP32 inputs"); + } + + if (chunk_size == 0) { + chunk_size = 64; + } + + float op_scale = scale; + if (op_scale == 0.0f) { + op_scale = 1.0f / std::sqrt(static_cast(K)); + } + + OpParams params; + params.scale = op_scale; + params.chunk_size = chunk_size; + params.num_kv_heads = Hq; + params.output_precision = Precision::FP16; + return add_node(OpType::GATED_DELTANET_PREFILL, + {query, key, value, gate_log, beta, initial_state}, + {B, T + K, Hv, V}, + params); +} + +size_t CactusGraph::stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 3) throw std::runtime_error("stft expects N,C,L input"); + if (w.shape.size() != 3) throw std::runtime_error("stft weight expects [C_out, C_in, K]"); + + size_t N = xin.shape[0]; + size_t L = xin.shape[2]; + size_t K = w.shape[2]; + size_t L_out = (L - K) / stride + 1; + + OpParams params{}; + params.stride = stride; + params.num_fft_bins = num_fft_bins; + + return add_node(OpType::STFT, {input, weight}, {N, 2 * num_fft_bins, L_out}, params); +} + +size_t CactusGraph::concat(size_t input1, size_t input2, int axis) { + const auto& buffer1 = get_output_buffer(input1); + const auto& buffer2 = get_output_buffer(input2); + + if (buffer1.shape.size() != buffer2.shape.size()) { + throw std::runtime_error("Concat requires inputs with same number of dimensions"); + } + + std::vector output_shape = buffer1.shape; + size_t ndims = output_shape.size(); + + if (axis < 0) axis += ndims; + if (axis < 0 || static_cast(axis) >= ndims) { + throw std::runtime_error("Invalid axis for concat operation"); + } + + for (size_t i = 0; i < ndims; ++i) { + if (i != static_cast(axis) && buffer1.shape[i] != buffer2.shape[i]) { + throw std::runtime_error("Concat inputs must have same shape except on concat axis"); + } + } + + output_shape[axis] = buffer1.shape[axis] + buffer2.shape[axis]; + + OpParams params; + params.axis = axis; + return add_node(OpType::CONCAT, {input1, input2}, output_shape, params); +} + +size_t CactusGraph::cat(const std::vector& inputs, int axis) { + if (inputs.empty()) { + throw std::runtime_error("Cat requires at least one input"); + } + + const auto& first_buffer = get_output_buffer(inputs[0]); + std::vector output_shape = first_buffer.shape; + size_t ndims = output_shape.size(); + + if (axis < 0) axis += ndims; + if (axis < 0 || static_cast(axis) >= ndims) { + throw std::runtime_error("Invalid axis for cat operation"); + } + + for (size_t i = 1; i < inputs.size(); ++i) { + const auto& buffer = get_output_buffer(inputs[i]); + if (buffer.shape.size() != ndims) { + throw std::runtime_error("All inputs to cat must have same number of dimensions"); + } + for (size_t d = 0; d < ndims; ++d) { + if (d != static_cast(axis) && buffer.shape[d] != output_shape[d]) { + throw std::runtime_error("All inputs to cat must have same shape except on cat axis"); + } + } + output_shape[axis] += buffer.shape[axis]; + } + + OpParams params; + params.axis = axis; + return add_node(OpType::CAT, inputs, output_shape, params); +} + +size_t CactusGraph::scatter_topk(size_t indices, size_t values, size_t num_classes) { + const auto& indices_buffer = get_output_buffer(indices); + const auto& values_buffer = get_output_buffer(values); + + if (indices_buffer.shape != values_buffer.shape) { + throw std::runtime_error("ScatterTopK requires indices and values with identical shapes"); + } + if (indices_buffer.shape.size() != 2) { + throw std::runtime_error("ScatterTopK currently supports 2D tensors [batch, top_k]"); + } + if (indices_buffer.precision != Precision::FP32 || values_buffer.precision != Precision::FP32) { + throw std::runtime_error("ScatterTopK expects FP32 indices and values"); + } + + std::vector output_shape = {num_classes, indices_buffer.shape[0]}; + OpParams params{.output_precision = Precision::FP32, .num_classes = num_classes}; + return add_node(OpType::SCATTER_TOPK, {indices, values}, output_shape, params); +} + +size_t CactusGraph::sample(size_t logits, float temperature, float top_p, size_t top_k, + const std::unordered_map& logit_bias) { + return this->sample_with_options(logits, temperature, top_p, 0.15f, 1.1f, top_k, logit_bias); +} + +size_t CactusGraph::sample_with_options(size_t logits, float temperature, float top_p, + float min_p, float repetition_penalty, size_t top_k, + const std::unordered_map& logit_bias) { + const auto& logits_buffer = get_output_buffer(logits); + + if (logits_buffer.shape.empty()) { + throw std::runtime_error("Sample requires non-empty logits tensor"); + } + + OpParams params; + params.temperature = temperature; + params.top_p = top_p; + params.min_p = min_p; + params.repetition_penalty = repetition_penalty; + params.top_k = top_k; + params.random_seed = std::chrono::high_resolution_clock::now().time_since_epoch().count(); + params.output_precision = Precision::FP32; + + if (!logit_bias.empty()) { + params.bias_indices.reserve(logit_bias.size()); + params.bias_values.reserve(logit_bias.size()); + for (const auto& [idx, val] : logit_bias) { + params.bias_indices.push_back(idx); + params.bias_values.push_back(val); + } + } + + std::vector output_shape = {1}; + return add_node(OpType::SAMPLE, {logits}, output_shape, params); +} + +size_t CactusGraph::scalar_add(size_t input, float value) { + OpParams params{}; + params.scalar = value; + params.output_precision = get_output_buffer(input).precision; + return add_node(OpType::SCALAR_ADD, {input}, {}, params); +} + +size_t CactusGraph::scalar_subtract(size_t input, float value) { + OpParams params{}; + params.scalar = value; + params.output_precision = get_output_buffer(input).precision; + return add_node(OpType::SCALAR_SUBTRACT, {input}, {}, params); +} + +size_t CactusGraph::scalar_multiply(size_t input, float value) { + OpParams params{}; + params.scalar = value; + params.output_precision = get_output_buffer(input).precision; + return add_node(OpType::SCALAR_MULTIPLY, {input}, {}, params); +} + +size_t CactusGraph::scalar_divide(size_t input, float value) { + OpParams params{}; + params.scalar = value; + params.output_precision = get_output_buffer(input).precision; + return add_node(OpType::SCALAR_DIVIDE, {input}, {}, params); +} + +size_t CactusGraph::scalar_floor_divide(size_t input, float value) { + OpParams params{}; + params.scalar = value; + params.output_precision = get_output_buffer(input).precision; + return add_node(OpType::SCALAR_FLOOR_DIVIDE, {input}, {}, params); +} + +size_t CactusGraph::scalar_not_equal(size_t input, float value) { + OpParams params{}; + params.scalar = value; + params.output_precision = Precision::FP16; + return add_node(OpType::SCALAR_NOT_EQUAL, {input}, {}, params); +} + +size_t CactusGraph::scalar_exp(size_t input) { + return add_node(OpType::SCALAR_EXP, {input}, {}); +} + +size_t CactusGraph::scalar_sqrt(size_t input) { + return add_node(OpType::SCALAR_SQRT, {input}, {}); +} + +size_t CactusGraph::scalar_cos(size_t input) { + return add_node(OpType::SCALAR_COS, {input}, {}); +} + +size_t CactusGraph::scalar_sin(size_t input) { + return add_node(OpType::SCALAR_SIN, {input}, {}); +} + +size_t CactusGraph::scalar_log(size_t input) { + return add_node(OpType::SCALAR_LOG, {input}, {}); +} + +size_t CactusGraph::relu(size_t input) { + return add_node(OpType::RELU, {input}, {}); +} + +size_t CactusGraph::silu(size_t input) { + return add_node(OpType::SILU, {input}, {}); +} + +size_t CactusGraph::gelu(size_t input) { + return add_node(OpType::GELU, {input}, {}); +} + +size_t CactusGraph::gelu_erf(size_t input) { + return add_node(OpType::GELU_ERF, {input}, {}); +} + +size_t CactusGraph::sigmoid(size_t input) { + return add_node(OpType::SIGMOID, {input}, {}); +} + +size_t CactusGraph::tanh(size_t input) { + return add_node(OpType::TANH, {input}, {}); +} + +size_t CactusGraph::glu(size_t input, int axis) { + const auto& xin = get_output_buffer(input); + if (xin.shape.empty()) { + throw std::runtime_error("glu expects non-scalar input"); + } + + int actual_axis = axis; + if (actual_axis < 0) { + actual_axis += static_cast(xin.shape.size()); + } + if (actual_axis < 0 || static_cast(actual_axis) >= xin.shape.size()) { + throw std::runtime_error("glu axis out of range"); + } + + std::vector out_shape = xin.shape; + const size_t axis_size = out_shape[static_cast(actual_axis)]; + if ((axis_size % 2) != 0) { + throw std::runtime_error("glu split dimension must be even"); + } + out_shape[static_cast(actual_axis)] = axis_size / 2; + + OpParams params{}; + params.axis = actual_axis; + params.output_precision = xin.precision; + return add_node(OpType::GLU, {input}, out_shape, params); +} + +size_t CactusGraph::rope_gptj(size_t input, float theta, size_t position_offset, size_t rot_dim, ComputeBackend backend) { + OpParams params; + params.theta = theta; + params.position_offset = position_offset; + params.scalar = static_cast(rot_dim); + params.backend = backend; + return add_node(OpType::ROPE_GPTJ, {input}, {}, params); +} + +size_t CactusGraph::gather(size_t tensor, size_t indices, int axis) { + const auto& tensor_buffer = get_output_buffer(tensor); + const auto& idx_shape = get_output_buffer(indices).shape; + + if (tensor_buffer.shape.empty()) { + throw std::runtime_error("Cannot gather from scalar tensor"); + } + + int actual_axis = axis; + if (actual_axis < 0) { + actual_axis += static_cast(tensor_buffer.shape.size()); + } + if (actual_axis < 0 || actual_axis >= static_cast(tensor_buffer.shape.size())) { + throw std::runtime_error("Gather axis out of range"); + } + + std::vector output_shape; + if (idx_shape.size() == tensor_buffer.shape.size()) { + output_shape = idx_shape; + } else { + if (actual_axis != 0) { + throw std::runtime_error("Lower-rank gather only supports axis 0"); + } + output_shape = idx_shape; + for (size_t i = 1; i < tensor_buffer.shape.size(); i++) { + output_shape.push_back(tensor_buffer.shape[i]); + } + } + + OpParams params; + params.axis = actual_axis; + params.output_precision = tensor_buffer.precision; + + return add_node(OpType::GATHER, {tensor, indices}, output_shape, params); +} + +size_t CactusGraph::slice(size_t input, int axis, size_t start, size_t length) { + const auto& input_buffer = get_output_buffer(input); + if (input_buffer.shape.empty()) { + throw std::runtime_error("Cannot slice a scalar tensor"); + } + + size_t axis_index = static_cast(axis); + size_t axis_size = input_buffer.shape[axis_index]; + + if (start + length > axis_size) { + throw std::runtime_error("Slice range extends beyond axis size"); + } + + std::vector output_shape = input_buffer.shape; + output_shape[axis_index] = length; + + OpParams params; + params.axis = axis_index; + params.slice_start = start; + params.slice_length = length; + params.output_precision = input_buffer.precision; + + return add_node(OpType::SLICE, {input}, output_shape, params); +} + +size_t CactusGraph::embedding(size_t embedding_tensor, size_t indices) { + const auto& emb_buffer = get_output_buffer(embedding_tensor); + const auto& idx_shape = get_output_buffer(indices).shape; + + if (emb_buffer.shape.size() != 2) { + std::cerr << "Error: Embedding tensor " << embedding_tensor << " has invalid shape: ["; + for(auto d : emb_buffer.shape) std::cerr << d << ","; + std::cerr << "]. OpType=" << (int)nodes_[node_index_map_[embedding_tensor]]->op_type + << " ExtData=" << emb_buffer.external_data << std::endl; + throw std::runtime_error("Embedding tensor must be 2D [vocab_size, hidden_dim]"); + } + + std::vector output_shape = idx_shape; + output_shape.push_back(emb_buffer.shape[1]); + + OpParams params; + params.output_precision = Precision::FP16; + return add_node(OpType::EMBEDDING, {embedding_tensor, indices}, output_shape, params); +} + +size_t CactusGraph::bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners) { + const auto& pos_embeds_buffer = get_output_buffer(pos_embeds); + size_t embed_dim = pos_embeds_buffer.shape[1]; + std::vector output_shape = {dst_height * dst_width, embed_dim}; + + OpParams params; + params.dst_height = dst_height; + params.dst_width = dst_width; + params.align_corners = align_corners; + params.output_precision = Precision::FP16; + + return add_node(OpType::BILINEAR_INTERPOLATION, {pos_embeds}, output_shape, params); +} + +size_t CactusGraph::precision_cast(size_t input, Precision target_precision) { + OpParams params{}; + params.output_precision = target_precision; + return add_node(OpType::PRECISION_CAST, {input}, {}, params); +} + +size_t CactusGraph::quantize_activations(size_t input) { + const auto& input_buffer = get_output_buffer(input); + + if (input_buffer.precision != Precision::FP16) { + throw std::invalid_argument("quantize_activations requires FP16 input"); + } + + OpParams params{}; + params.output_precision = Precision::INT8; + return add_node(OpType::QUANTIZE_ACTIVATIONS, {input}, input_buffer.shape, params); +} + +size_t CactusGraph::add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params) { + auto node = std::make_unique(next_node_id_, op_type); + node->input_ids = inputs; + node->params = params; + + std::vector result_shape = output_shape; + if (result_shape.empty() && !inputs.empty()) { + result_shape = nodes_[node_index_map_[inputs[0]]]->output_buffer.shape; + } + + Precision result_precision = params.output_precision; + if (op_type == OpType::PRECISION_CAST) { + result_precision = params.output_precision; + } else if (result_precision == Precision::INT8 && !inputs.empty()) { + result_precision = nodes_[node_index_map_[inputs[0]]]->output_buffer.precision; + } + + node->output_buffer = BufferDesc(result_shape, result_precision); + + size_t node_id = next_node_id_++; + node_index_map_[node_id] = nodes_.size(); + nodes_.push_back(std::move(node)); + + return node_id; +} + +const BufferDesc& CactusGraph::get_output_buffer(size_t node_id) const { + return nodes_[node_index_map_.at(node_id)]->output_buffer; +} + +size_t CactusGraph::persistent(size_t source_node) { + const auto& source_buffer = get_output_buffer(source_node); + OpParams params; + params.output_precision = source_buffer.precision; + size_t node_id = add_node(OpType::PERSISTENT, {source_node}, source_buffer.shape, params); + persistent_node_ids_.insert(node_id); + return node_id; +} + +bool CactusGraph::is_populated(size_t persistent_node_id) const { + return populated_node_ids_.count(persistent_node_id) > 0; +} + +void CactusGraph::invalidate_persistent(size_t persistent_node_id) { + populated_node_ids_.erase(persistent_node_id); + persistent_node_ids_.erase(persistent_node_id); +} + +size_t CactusGraph::altup_predict(size_t coefs, const size_t* streams, size_t num_streams) { + const auto& stream0_buf = get_output_buffer(streams[0]); + + size_t seq_len = stream0_buf.shape[0]; + size_t hidden_dim = stream0_buf.shape[1]; + + std::vector input_ids = {coefs}; + for (size_t i = 0; i < num_streams; i++) + input_ids.push_back(streams[i]); + + OpParams params; + params.num_altup_inputs = num_streams; + return add_node(OpType::ALTUP_PREDICT, input_ids, {num_streams * seq_len, hidden_dim}, params); +} + +size_t CactusGraph::altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions) { + const auto& pred0_buf = get_output_buffer(predictions[0]); + + size_t seq_len = pred0_buf.shape[0]; + size_t hidden_dim = pred0_buf.shape[1]; + + std::vector input_ids = {coefs, innovation}; + for (size_t i = 0; i < num_predictions; i++) + input_ids.push_back(predictions[i]); + + OpParams params; + params.num_altup_inputs = num_predictions; + return add_node(OpType::ALTUP_CORRECT, input_ids, {num_predictions * seq_len, hidden_dim}, params); +} + +size_t CactusGraph::gaussian_topk(size_t input, float ppf) { + const auto& in_buf = get_output_buffer(input); + OpParams params; + params.scalar = ppf; + return add_node(OpType::GAUSSIAN_TOPK, {input}, in_buf.shape, params); +} + +size_t CactusGraph::leaky_relu(size_t input, float negative_slope) { + const auto& in_buf = get_output_buffer(input); + OpParams params; + params.scalar = negative_slope; + return add_node(OpType::LEAKY_RELU, {input}, in_buf.shape, params); +} + +size_t CactusGraph::bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, + size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd) { + const auto& in_buf = get_output_buffer(input); + const auto& w_ih = get_output_buffer(w_ih_fwd); + size_t batch = in_buf.shape[0]; + size_t seq_len = in_buf.shape[1]; + size_t hidden_size = w_ih.shape[0] / 4; + return add_node(OpType::BILSTM_SEQUENCE, + {input, w_ih_fwd, w_hh_fwd, b_ih_fwd, b_hh_fwd, w_ih_bwd, w_hh_bwd, b_ih_bwd, b_hh_bwd}, + {batch, seq_len, 2 * hidden_size}, {}); +} + +size_t CactusGraph::maxpool1d(size_t input, size_t kernel_size, size_t stride) { + const auto& in_buf = get_output_buffer(input); + size_t batch = in_buf.shape[0]; + size_t channels = in_buf.shape[1]; + size_t input_length = in_buf.shape[2]; + size_t output_length = (input_length - kernel_size) / stride + 1; + + OpParams params; + params.kernel_size = kernel_size; + params.stride = stride; + return add_node(OpType::MAXPOOL1D, {input}, {batch, channels, output_length}, params); +} + +size_t CactusGraph::conv2d_k3s1p1(size_t input, size_t weight) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + + if (xin.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); + } + if (w.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s1p1 weight must be [C_out, C_in, 3, 3]"); + } + + const size_t N = xin.shape[0]; + const size_t C_in = xin.shape[1]; + const size_t H = xin.shape[2]; + const size_t W = xin.shape[3]; + const size_t C_out = w.shape[0]; + + if (w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { + throw std::runtime_error("conv2d_k3s1p1 weight must match [C_out, C_in, 3, 3]"); + } + if (H == 0 || W == 0) { + throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); + } + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV2D_K3S1P1, {input, weight}, {N, C_out, H, W}, params); +} + +size_t CactusGraph::conv2d_k3s1p1(size_t input, size_t weight, size_t bias) { + const auto& xin = get_output_buffer(input); + const auto& w = get_output_buffer(weight); + const auto& b = get_output_buffer(bias); + + if (xin.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); + } + + const size_t N = xin.shape[0]; + const size_t C_in = xin.shape[1]; + const size_t H = xin.shape[2]; + const size_t W = xin.shape[3]; + + if (w.shape.size() != 4 || w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { + throw std::runtime_error("conv2d_k3s1p1 weight must be [C_out, C_in, 3, 3]"); + } + const size_t C_out = w.shape[0]; + if (b.total_size != C_out) { + throw std::runtime_error("conv2d_k3s1p1 bias size mismatch"); + } + if (H == 0 || W == 0) { + throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); + } + + OpParams params{}; + params.output_precision = xin.precision; + return add_node(OpType::CONV2D_K3S1P1, {input, weight, bias}, {N, C_out, H, W}, params); +} + +size_t CactusGraph::stats_pool(size_t input) { + const auto& xin = get_output_buffer(input); + size_t batch = xin.shape[0]; + size_t features = 1; + for (size_t i = 1; i < xin.shape.size() - 1; ++i) features *= xin.shape[i]; + return add_node(OpType::STATS_POOL, {input}, {batch, features * 2}); +} + +size_t CactusGraph::weighted_stats_pool(size_t input, size_t weights) { + const auto& xin = get_output_buffer(input); + size_t batch = xin.shape[0]; + size_t features = 1; + for (size_t i = 1; i < xin.shape.size() - 1; ++i) features *= xin.shape[i]; + return add_node(OpType::WEIGHTED_STATS_POOL, {input, weights}, {batch, features * 2}); +} diff --git a/cactus-transpile/graph/graph_core.cpp b/cactus-transpile/graph/graph_core.cpp new file mode 100644 index 000000000..f418da72b --- /dev/null +++ b/cactus-transpile/graph/graph_core.cpp @@ -0,0 +1,265 @@ +#include "graph.h" +#include +#include +#include + +size_t BufferPool::round_up_size(size_t size) const { + constexpr size_t ALIGNMENT = 64; + constexpr size_t MIN_BUCKET = 1024; + + if (size < MIN_BUCKET) return MIN_BUCKET; + + size_t aligned = (size + ALIGNMENT - 1) & ~(ALIGNMENT - 1); + + size_t bucket = MIN_BUCKET; + while (bucket < aligned) { + bucket *= 2; + } + return bucket; +} + +char* BufferPool::acquire(size_t byte_size) { + if (byte_size == 0) return nullptr; + + size_t bucket_size = round_up_size(byte_size); + + auto it = free_buffers_.find(bucket_size); + if (it != free_buffers_.end() && !it->second.empty()) { + auto buffer = std::move(it->second.back()); + it->second.pop_back(); + pool_bytes_ -= bucket_size; + active_bytes_ += bucket_size; + if (active_bytes_ > peak_bytes_) { + peak_bytes_ = active_bytes_; + } + return buffer.release(); + } + + active_bytes_ += bucket_size; + if (active_bytes_ > peak_bytes_) { + peak_bytes_ = active_bytes_; + } + return new char[bucket_size]; +} + +void BufferPool::release(char* ptr, size_t byte_size) { + if (!ptr || byte_size == 0) return; + + size_t bucket_size = round_up_size(byte_size); + active_bytes_ -= bucket_size; + pool_bytes_ += bucket_size; + + free_buffers_[bucket_size].push_back(std::unique_ptr(ptr)); +} + +void BufferPool::clear() { + free_buffers_.clear(); + pool_bytes_ = 0; +} + +BufferDesc::BufferDesc() + : total_size(0), byte_size(0), external_data(nullptr), pooled_data(nullptr), + precision(Precision::FP16) {} + +BufferDesc::BufferDesc(const std::vector& s, Precision prec) + : shape(s), external_data(nullptr), pooled_data(nullptr), precision(prec) { + total_size = 1; + for (size_t dim : shape) total_size *= dim; + byte_size = PrecisionTraits::packed_size_of(prec, total_size); +} + +BufferDesc::~BufferDesc() { + if (pooled_data) { + delete[] pooled_data; + pooled_data = nullptr; + } +} + +BufferDesc::BufferDesc(BufferDesc&& other) noexcept + : shape(std::move(other.shape)), + total_size(other.total_size), + byte_size(other.byte_size), + data(std::move(other.data)), + external_data(other.external_data), + pooled_data(other.pooled_data), + precision(other.precision), + group_size(other.group_size), + num_groups(other.num_groups), + scales_data(other.scales_data), + owned_scales(std::move(other.owned_scales)), + is_interleaved(other.is_interleaved), + original_N(other.original_N), + activation_scales_data(other.activation_scales_data), + owned_activation_scales(std::move(other.owned_activation_scales)), + num_rows_for_activation_scales(other.num_rows_for_activation_scales) { + other.total_size = 0; + other.byte_size = 0; + other.external_data = nullptr; + other.pooled_data = nullptr; + other.group_size = 0; + other.num_groups = 0; + other.scales_data = nullptr; + other.is_interleaved = false; + other.original_N = 0; + other.activation_scales_data = nullptr; + other.num_rows_for_activation_scales = 0; +} + +BufferDesc& BufferDesc::operator=(BufferDesc&& other) noexcept { + if (this != &other) { + if (pooled_data) { + delete[] pooled_data; + } + + shape = std::move(other.shape); + total_size = other.total_size; + byte_size = other.byte_size; + data = std::move(other.data); + external_data = other.external_data; + pooled_data = other.pooled_data; + precision = other.precision; + group_size = other.group_size; + num_groups = other.num_groups; + scales_data = other.scales_data; + owned_scales = std::move(other.owned_scales); + is_interleaved = other.is_interleaved; + original_N = other.original_N; + activation_scales_data = other.activation_scales_data; + owned_activation_scales = std::move(other.owned_activation_scales); + num_rows_for_activation_scales = other.num_rows_for_activation_scales; + + other.total_size = 0; + other.byte_size = 0; + other.external_data = nullptr; + other.pooled_data = nullptr; + other.group_size = 0; + other.num_groups = 0; + other.scales_data = nullptr; + other.is_interleaved = false; + other.original_N = 0; + other.activation_scales_data = nullptr; + other.num_rows_for_activation_scales = 0; + } + return *this; +} + +void* BufferDesc::get_data() { + if (external_data) return external_data; + if (pooled_data) return pooled_data; + return data.get(); +} + +const void* BufferDesc::get_data() const { + if (external_data) return external_data; + if (pooled_data) return pooled_data; + return data.get(); +} + +void BufferDesc::allocate() { + if (!data && !external_data && !pooled_data) { + data = std::make_unique(byte_size); + } +} + +void BufferDesc::allocate_from_pool(BufferPool& pool) { + if (!data && !external_data && !pooled_data && byte_size > 0) { + pooled_data = pool.acquire(byte_size); + } +} + +void BufferDesc::release_to_pool(BufferPool& pool) { + if (pooled_data && byte_size > 0) { + pool.release(pooled_data, byte_size); + pooled_data = nullptr; + } +} + +void BufferDesc::set_external(void* ptr) { + external_data = ptr; + data.reset(); + pooled_data = nullptr; +} + +// GraphNode implementation +GraphNode::GraphNode(size_t node_id, OpType type) : id(node_id), op_type(type) {} + +// TensorConfig implementation +TensorConfig& TensorConfig::global() { + static TensorConfig instance; + return instance; +} + +// BroadcastInfo implementation +BroadcastInfo BroadcastInfo::compute(const std::vector& lhs, const std::vector& rhs) { + BroadcastInfo info; + size_t max_dims = std::max(lhs.size(), rhs.size()); + info.output_shape.resize(max_dims); + + for (size_t i = 0; i < max_dims; ++i) { + size_t lhs_dim = i < lhs.size() ? lhs[lhs.size() - 1 - i] : 1; + size_t rhs_dim = i < rhs.size() ? rhs[rhs.size() - 1 - i] : 1; + + if (lhs_dim != rhs_dim && lhs_dim != 1 && rhs_dim != 1) { + throw std::invalid_argument("Shapes are not compatible for broadcasting"); + } + + info.output_shape[max_dims - 1 - i] = std::max(lhs_dim, rhs_dim); + } + + info.needs_broadcasting = (lhs != info.output_shape || rhs != info.output_shape); + return info; +} + +namespace ValidationUtils { + void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name) { + if (shape.size() != required_dims) { + throw std::runtime_error(op_name + " requires " + std::to_string(required_dims) + + "D tensor, got " + std::to_string(shape.size()) + "D tensor"); + } + } + + void validate_precision(Precision actual, Precision required, const std::string& op_name) { + if (actual != required) { + std::string actual_str = (actual == Precision::INT8) ? "INT8" : "FP32"; + std::string required_str = (required == Precision::INT8) ? "INT8" : "FP32"; + throw std::runtime_error(op_name + " requires " + required_str + " precision, got " + actual_str); + } + } + + void validate_input_count(size_t actual, size_t required, const std::string& op_name) { + if (actual < required) { + throw std::runtime_error(op_name + " requires " + std::to_string(required) + + " inputs, got " + std::to_string(actual) + " inputs"); + } + } +} + +CactusGraph::CactusGraph() : next_node_id_(1) {} + +size_t CactusGraph::get_node_count() const { + return nodes_.size(); +} + +void CactusGraph::register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) { + debug_nodes_.push_back({layer_idx, name, node_id}); +} + +void CactusGraph::capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) { + register_debug_node(layer_idx, name, node_id); +} + +const std::vector& CactusGraph::get_debug_nodes() const { + return debug_nodes_; +} + +void CactusGraph::clear_debug_nodes() { + debug_nodes_.clear(); +} + +void CactusGraph::allocate_buffers() { + for (auto& node : nodes_) { + if (node->op_type != OpType::INPUT) { + node->output_buffer.allocate(); + } + } +} diff --git a/cactus-transpile/graph/graph_execute.cpp b/cactus-transpile/graph/graph_execute.cpp new file mode 100644 index 000000000..985c43aae --- /dev/null +++ b/cactus-transpile/graph/graph_execute.cpp @@ -0,0 +1,948 @@ +#include "graph.h" +#include "../kernel/kernel_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using ComputeFn = void(*)(GraphNode&, const nodes_vector&, const node_index_map_t&); + +#define DECLARE_COMPUTE(name) \ + extern void name(GraphNode&, const nodes_vector&, const node_index_map_t&) + +DECLARE_COMPUTE(compute_binary_op_node); +DECLARE_COMPUTE(compute_unary_op_node); +DECLARE_COMPUTE(compute_activation_node); +DECLARE_COMPUTE(compute_reduce_node); +DECLARE_COMPUTE(compute_cumsum_node); +DECLARE_COMPUTE(compute_reshape_node); +DECLARE_COMPUTE(compute_precision_cast_node); +DECLARE_COMPUTE(compute_matmul_node); +DECLARE_COMPUTE(compute_rms_norm_node); +DECLARE_COMPUTE(compute_rope_node); +DECLARE_COMPUTE(compute_softmax_node); +DECLARE_COMPUTE(compute_attention_node); +DECLARE_COMPUTE(compute_attention_int8_hybrid_node); +DECLARE_COMPUTE(compute_rel_pos_bias_node); +DECLARE_COMPUTE(compute_layernorm_node); +DECLARE_COMPUTE(compute_conv1d_causal_node); +DECLARE_COMPUTE(compute_conv1d_k3_node); +DECLARE_COMPUTE(compute_conv1d_k7s3_node); +DECLARE_COMPUTE(compute_conv1d_node); +DECLARE_COMPUTE(compute_conv1d_same_depthwise_k9_node); +DECLARE_COMPUTE(compute_conv1d_pointwise_node); +DECLARE_COMPUTE(compute_conv2d_k3s2p1_node); +DECLARE_COMPUTE(compute_conv2d_depthwise_k3s2p1_node); +DECLARE_COMPUTE(compute_conv2d_pointwise_1x1_node); +DECLARE_COMPUTE(compute_glu_node); +DECLARE_COMPUTE(compute_batchnorm_node); +DECLARE_COMPUTE(compute_groupnorm_node); +DECLARE_COMPUTE(compute_rope_gptj_node); +DECLARE_COMPUTE(compute_lstm_cell_node); +DECLARE_COMPUTE(compute_gated_deltanet_decode_node); +DECLARE_COMPUTE(compute_gated_deltanet_prefill_node); +DECLARE_COMPUTE(compute_stft_node); +DECLARE_COMPUTE(compute_altup_predict_node); +DECLARE_COMPUTE(compute_altup_correct_node); +DECLARE_COMPUTE(compute_gaussian_topk_node); +DECLARE_COMPUTE(compute_maxpool1d_node); +DECLARE_COMPUTE(compute_bilstm_sequence_node); +DECLARE_COMPUTE(compute_conv2d_k3s1p1_node); +DECLARE_COMPUTE(compute_conv2d_node); +DECLARE_COMPUTE(compute_stats_pool_node); +DECLARE_COMPUTE(compute_transpose_node); +DECLARE_COMPUTE(compute_gather_node); +DECLARE_COMPUTE(compute_slice_node); +DECLARE_COMPUTE(compute_embedding_node); +DECLARE_COMPUTE(compute_concat_node); +DECLARE_COMPUTE(compute_cat_node); +DECLARE_COMPUTE(compute_index_node); +DECLARE_COMPUTE(compute_bilinear_interpolation_node); +DECLARE_COMPUTE(compute_sample_node); +DECLARE_COMPUTE(compute_topk_node); +DECLARE_COMPUTE(compute_scatter_topk_node); +DECLARE_COMPUTE(compute_moe_layer_node); +DECLARE_COMPUTE(compute_persistent_node); +DECLARE_COMPUTE(compute_quantize_activations_node); +extern void shrink_thread_local_buffers(); +#undef DECLARE_COMPUTE + +static const std::unordered_map dispatch_table = { + {OpType::ADD, compute_binary_op_node}, + {OpType::ADD_CLIPPED, compute_binary_op_node}, + {OpType::SUBTRACT, compute_binary_op_node}, + {OpType::MULTIPLY, compute_binary_op_node}, + {OpType::DIVIDE, compute_binary_op_node}, + {OpType::NOT_EQUAL, compute_binary_op_node}, + {OpType::SCALAR_ADD, compute_unary_op_node}, + {OpType::SCALAR_SUBTRACT, compute_unary_op_node}, + {OpType::SCALAR_MULTIPLY, compute_unary_op_node}, + {OpType::SCALAR_DIVIDE, compute_unary_op_node}, + {OpType::SCALAR_NOT_EQUAL, compute_unary_op_node}, + {OpType::SCALAR_FLOOR_DIVIDE, compute_unary_op_node}, + {OpType::SCALAR_EXP, compute_unary_op_node}, + {OpType::SCALAR_SQRT, compute_unary_op_node}, + {OpType::SCALAR_COS, compute_unary_op_node}, + {OpType::SCALAR_SIN, compute_unary_op_node}, + {OpType::SCALAR_LOG, compute_unary_op_node}, + {OpType::ABS, compute_unary_op_node}, + {OpType::POW, compute_unary_op_node}, + {OpType::RELU, compute_activation_node}, + {OpType::SILU, compute_activation_node}, + {OpType::GELU, compute_activation_node}, + {OpType::GELU_ERF, compute_activation_node}, + {OpType::SIGMOID, compute_activation_node}, + {OpType::TANH, compute_activation_node}, + {OpType::LEAKY_RELU, compute_activation_node}, + {OpType::SUM, compute_reduce_node}, + {OpType::MEAN, compute_reduce_node}, + {OpType::VARIANCE, compute_reduce_node}, + {OpType::MIN, compute_reduce_node}, + {OpType::MAX, compute_reduce_node}, + {OpType::CUMSUM, compute_cumsum_node}, + {OpType::FLATTEN, compute_reshape_node}, + {OpType::VIEW, compute_reshape_node}, + {OpType::RESHAPE, compute_reshape_node}, + {OpType::PRECISION_CAST, compute_precision_cast_node}, + {OpType::MATMUL, compute_matmul_node}, + {OpType::RMS_NORM, compute_rms_norm_node}, + {OpType::LAYERNORM, compute_layernorm_node}, + {OpType::GROUPNORM, compute_groupnorm_node}, + {OpType::BATCHNORM, compute_batchnorm_node}, + {OpType::ROPE, compute_rope_node}, + {OpType::ROPE_GPTJ, compute_rope_gptj_node}, + {OpType::SOFTMAX, compute_softmax_node}, + {OpType::ATTENTION, compute_attention_node}, + {OpType::ATTENTION_INT8_HYBRID, compute_attention_int8_hybrid_node}, + {OpType::REL_POS_BIAS, compute_rel_pos_bias_node}, + {OpType::CONV1D_CAUSAL, compute_conv1d_causal_node}, + {OpType::CONV1D_K3, compute_conv1d_k3_node}, + {OpType::CONV1D_K7S3, compute_conv1d_k7s3_node}, + {OpType::CONV1D, compute_conv1d_node}, + {OpType::CONV1D_SAME_DEPTHWISE_K9, compute_conv1d_same_depthwise_k9_node}, + {OpType::CONV1D_POINTWISE, compute_conv1d_pointwise_node}, + {OpType::CONV2D_K3S2P1, compute_conv2d_k3s2p1_node}, + {OpType::CONV2D_DEPTHWISE_K3S2P1, compute_conv2d_depthwise_k3s2p1_node}, + {OpType::CONV2D_POINTWISE_1X1, compute_conv2d_pointwise_1x1_node}, + {OpType::CONV2D_K3S1P1, compute_conv2d_k3s1p1_node}, + {OpType::GLU, compute_glu_node}, + {OpType::TRANSPOSE, compute_transpose_node}, + {OpType::GATHER, compute_gather_node}, + {OpType::SLICE, compute_slice_node}, + {OpType::EMBEDDING, compute_embedding_node}, + {OpType::CONCAT, compute_concat_node}, + {OpType::CAT, compute_cat_node}, + {OpType::INDEX, compute_index_node}, + {OpType::MASKED_SELECT_PREFIX, compute_masked_select_prefix_node}, + {OpType::MASKED_SCATTER, compute_masked_scatter_node}, + {OpType::EXPAND, compute_expand_node}, + {OpType::CONV2D, compute_conv2d_node}, + {OpType::BILINEAR_INTERPOLATION, compute_bilinear_interpolation_node}, + {OpType::SAMPLE, compute_sample_node}, + {OpType::TOPK, compute_topk_node}, + {OpType::SCATTER_TOPK, compute_scatter_topk_node}, + {OpType::MOE_LAYER, compute_moe_layer_node}, + {OpType::PERSISTENT, compute_persistent_node}, + {OpType::QUANTIZE_ACTIVATIONS, compute_quantize_activations_node}, + {OpType::LSTM_CELL, compute_lstm_cell_node}, + {OpType::GATED_DELTANET_DECODE, compute_gated_deltanet_decode_node}, + {OpType::GATED_DELTANET_PREFILL, compute_gated_deltanet_prefill_node}, + {OpType::STFT, compute_stft_node}, + {OpType::ALTUP_PREDICT, compute_altup_predict_node}, + {OpType::ALTUP_CORRECT, compute_altup_correct_node}, + {OpType::GAUSSIAN_TOPK, compute_gaussian_topk_node}, + {OpType::MAXPOOL1D, compute_maxpool1d_node}, + {OpType::BILSTM_SEQUENCE, compute_bilstm_sequence_node}, + {OpType::STATS_POOL, compute_stats_pool_node}, + {OpType::WEIGHTED_STATS_POOL, compute_weighted_stats_pool_node}, +}; + +static const char* get_op_name(OpType op) { + switch (op) { + case OpType::INPUT: return "INPUT"; + case OpType::PRECISION_CAST: return "PRECISION_CAST"; + case OpType::ADD: return "ADD"; + case OpType::ADD_CLIPPED: return "ADD_CLIPPED"; + case OpType::SUBTRACT: return "SUBTRACT"; + case OpType::MULTIPLY: return "MULTIPLY"; + case OpType::DIVIDE: return "DIVIDE"; + case OpType::ABS: return "ABS"; + case OpType::POW: return "POW"; + case OpType::FLATTEN: return "FLATTEN"; + case OpType::VIEW: return "VIEW"; + case OpType::MATMUL: return "MATMUL"; + case OpType::TRANSPOSE: return "TRANSPOSE"; + case OpType::RESHAPE: return "RESHAPE"; + case OpType::SLICE: return "SLICE"; + case OpType::GATHER: return "GATHER"; + case OpType::EMBEDDING: return "EMBEDDING"; + case OpType::BILINEAR_INTERPOLATION: return "BILINEAR_INTERPOLATION"; + case OpType::SUM: return "SUM"; + case OpType::MEAN: return "MEAN"; + case OpType::VARIANCE: return "VARIANCE"; + case OpType::MIN: return "MIN"; + case OpType::MAX: return "MAX"; + case OpType::CUMSUM: return "CUMSUM"; + case OpType::RMS_NORM: return "RMS_NORM"; + case OpType::ROPE: return "ROPE"; + case OpType::ROPE_GPTJ: return "ROPE_GPTJ"; + case OpType::SOFTMAX: return "SOFTMAX"; + case OpType::ATTENTION: return "ATTENTION"; + case OpType::ATTENTION_INT8_HYBRID: return "ATTENTION_INT8_HYBRID"; + case OpType::REL_POS_BIAS: return "REL_POS_BIAS"; + case OpType::CONV1D_CAUSAL: return "CONV1D_CAUSAL"; + case OpType::CONV1D_K3: return "CONV1D_K3"; + case OpType::CONV1D_K7S3: return "CONV1D_K7S3"; + case OpType::CONV1D: return "CONV1D"; + case OpType::CONV1D_SAME_DEPTHWISE_K9: return "CONV1D_SAME_DEPTHWISE_K9"; + case OpType::CONV1D_POINTWISE: return "CONV1D_POINTWISE"; + case OpType::CONV2D_K3S2P1: return "CONV2D_K3S2P1"; + case OpType::CONV2D_DEPTHWISE_K3S2P1: return "CONV2D_DEPTHWISE_K3S2P1"; + case OpType::CONV2D_POINTWISE_1X1: return "CONV2D_POINTWISE_1X1"; + case OpType::GLU: return "GLU"; + case OpType::BATCHNORM: return "BATCHNORM"; + case OpType::SCALAR_ADD: return "SCALAR_ADD"; + case OpType::SCALAR_SUBTRACT: return "SCALAR_SUBTRACT"; + case OpType::SCALAR_MULTIPLY: return "SCALAR_MULTIPLY"; + case OpType::SCALAR_DIVIDE: return "SCALAR_DIVIDE"; + case OpType::SCALAR_EXP: return "SCALAR_EXP"; + case OpType::SCALAR_SQRT: return "SCALAR_SQRT"; + case OpType::SCALAR_COS: return "SCALAR_COS"; + case OpType::SCALAR_SIN: return "SCALAR_SIN"; + case OpType::SCALAR_LOG: return "SCALAR_LOG"; + case OpType::RELU: return "RELU"; + case OpType::SILU: return "SILU"; + case OpType::GELU: return "GELU"; + case OpType::GELU_ERF: return "GELU_ERF"; + case OpType::SIGMOID: return "SIGMOID"; + case OpType::TANH: return "TANH"; + case OpType::SAMPLE: return "SAMPLE"; + case OpType::CONCAT: return "CONCAT"; + case OpType::CAT: return "CAT"; + case OpType::SCATTER_TOPK: return "SCATTER_TOPK"; + case OpType::TOPK: return "TOPK"; + case OpType::LAYERNORM: return "LAYERNORM"; + case OpType::GROUPNORM: return "GROUPNORM"; + case OpType::MOE_LAYER: return "MOE_LAYER"; + case OpType::INDEX: return "INDEX"; + case OpType::PERSISTENT: return "PERSISTENT"; + case OpType::QUANTIZE_ACTIVATIONS: return "QUANTIZE_ACTIVATIONS"; + case OpType::LSTM_CELL: return "LSTM_CELL"; + case OpType::GATED_DELTANET_DECODE: return "GATED_DELTANET_DECODE"; + case OpType::GATED_DELTANET_PREFILL: return "GATED_DELTANET_PREFILL"; + case OpType::STFT: return "STFT"; + case OpType::ALTUP_PREDICT: return "ALTUP_PREDICT"; + case OpType::ALTUP_CORRECT: return "ALTUP_CORRECT"; + case OpType::GAUSSIAN_TOPK: return "GAUSSIAN_TOPK"; + case OpType::MAXPOOL1D: return "MAXPOOL1D"; + case OpType::BILSTM_SEQUENCE: return "BILSTM_SEQUENCE"; + case OpType::LEAKY_RELU: return "LEAKY_RELU"; + case OpType::CONV2D_K3S1P1: return "CONV2D_K3S1P1"; + case OpType::STATS_POOL: return "STATS_POOL"; + case OpType::WEIGHTED_STATS_POOL: return "WEIGHTED_STATS_POOL"; + case OpType::NOT_EQUAL: return "NOT_EQUAL"; + case OpType::SCALAR_NOT_EQUAL: return "SCALAR_NOT_EQUAL"; + case OpType::SCALAR_FLOOR_DIVIDE: return "SCALAR_FLOOR_DIVIDE"; + case OpType::MASKED_SELECT_PREFIX: return "MASKED_SELECT_PREFIX"; + case OpType::MASKED_SCATTER: return "MASKED_SCATTER"; + case OpType::EXPAND: return "EXPAND"; + case OpType::CONV2D: return "CONV2D"; + } + return "UNKNOWN_OP"; +} + +static std::string format_shape(const std::vector& shape) { + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < shape.size(); ++i) { + if (i > 0) { + oss << ","; + } + oss << shape[i]; + } + oss << "]"; + return oss.str(); +} + +static std::string format_node_debug(const GraphNode& node, const nodes_vector& nodes, const node_index_map_t& node_index_map) { + std::ostringstream oss; + oss << "node_id=" << node.id + << " op=" << get_op_name(node.op_type) + << " output_shape=" << format_shape(node.output_buffer.shape) + << " output_elems=" << node.output_buffer.total_size; + if (!node.input_ids.empty()) { + oss << " inputs=["; + for (size_t i = 0; i < node.input_ids.size(); ++i) { + if (i > 0) { + oss << "; "; + } + const size_t input_id = node.input_ids[i]; + oss << input_id; + auto it = node_index_map.find(input_id); + if (it != node_index_map.end()) { + const auto& input_node = nodes[it->second]; + oss << ":" << get_op_name(input_node->op_type) + << ":" << format_shape(input_node->output_buffer.shape) + << ":" << input_node->output_buffer.total_size; + } + } + oss << "]"; + } + return oss.str(); +} + +void compute_node_optimized(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& node_index_map) { + if (node.op_type == OpType::INPUT) return; + + auto it = dispatch_table.find(node.op_type); + if (it != dispatch_table.end()) { + it->second(node, nodes, node_index_map); + } else { + throw std::runtime_error("Unknown operation type: " + std::to_string(static_cast(node.op_type))); + } +} + +void CactusGraph::set_input(size_t node_id, const void* data, Precision) { + auto it = node_index_map_.find(node_id); + if (it == node_index_map_.end()) { + throw std::out_of_range("Unknown input node id: " + std::to_string(node_id)); + } + + auto& node = *nodes_[it->second]; + if (node.op_type != OpType::INPUT) { + throw std::invalid_argument("Can only set data on input nodes"); + } + + if (!node.output_buffer.data && !node.output_buffer.external_data) { + node.output_buffer.allocate(); + } + + std::memcpy(node.output_buffer.get_data(), data, node.output_buffer.byte_size); +} + +void CactusGraph::set_external_input(size_t node_id, void* data, Precision) { + auto it = node_index_map_.find(node_id); + if (it == node_index_map_.end()) { + throw std::out_of_range("Unknown input node id: " + std::to_string(node_id)); + } + + auto& node = *nodes_[it->second]; + if (node.op_type != OpType::INPUT) { + throw std::invalid_argument("Can only set data on input nodes"); + } + + node.output_buffer.set_external(data); +} + +void* CactusGraph::get_output(size_t node_id) { + auto it = node_index_map_.find(node_id); + if (it == node_index_map_.end()) { + throw std::out_of_range("Unknown output node id: " + std::to_string(node_id)); + } + + auto& buffer = nodes_[it->second]->output_buffer; + if (!buffer.get_data()) { + buffer.allocate(); + } + return buffer.get_data(); +} + +void CactusGraph::execute(const std::string& profile_file) { + std::vector last_use(nodes_.size(), 0); + for (size_t i = 0; i < nodes_.size(); ++i) { + for (size_t input_id : nodes_[i]->input_ids) { + auto it = node_index_map_.find(input_id); + if (it != node_index_map_.end()) { + last_use[it->second] = std::max(last_use[it->second], i); + } + } + } + + BufferPool& pool = buffer_pool_; + + auto get_env_int = [](const char* name, int fallback) -> int { + const char* val = std::getenv(name); + return val ? std::atoi(val) : fallback; + }; + + auto get_env_str = [](const char* name) -> std::string { + const char* val = std::getenv(name); + return val ? std::string(val) : std::string(); + }; + + bool capture_to_stdout = get_env_int("CACTUS_CAPTURE_STDOUT", 0) != 0; + std::string capture_file_path = get_env_str("CACTUS_CAPTURE_FILE"); + bool capture_requested = get_env_int("CACTUS_CAPTURE_ENABLE", 0) != 0; + std::string capture_dir = get_env_str("CACTUS_CAPTURE_DIR"); + bool trace_nodes = get_env_int("CACTUS_TRACE_NODES", 0) != 0; + + if (!capture_requested) { + capture_requested = capture_to_stdout || !capture_file_path.empty() || !capture_dir.empty(); + } else if (capture_file_path.empty() && !capture_to_stdout && capture_dir.empty()) { + capture_to_stdout = true; + } + + size_t capture_preview_count = static_cast(get_env_int("CACTUS_CAPTURE_PREVIEW_COUNT", 8)); + size_t capture_max_elements = static_cast(get_env_int("CACTUS_CAPTURE_MAX_ELEMENTS", 65536)); + + std::string env_profile = get_env_str("CACTUS_PROFILE_FILE"); + if (env_profile.empty()) env_profile = get_env_str("CACTUS_PROFILE"); + + std::string target_profile = profile_file; + if (target_profile.empty() && !env_profile.empty()) { + target_profile = env_profile; + } + + bool enable_profiling = !target_profile.empty(); + bool to_stdout = (target_profile == "stdout" || target_profile == "-"); + + std::ofstream profile_out; + std::ostream* out = &std::cout; + + if (enable_profiling && !to_stdout) { + profile_out.open(target_profile, std::ios::app); + if (profile_out.is_open()) { + out = &profile_out; + } + } + + auto total_start = std::chrono::high_resolution_clock::now(); + + if (enable_profiling) { + *out << "=== Graph Execution Profile ===" << std::endl; + *out << std::left << std::setw(24) << "Operation" + << std::setw(12) << "Time (ms)" + << std::setw(20) << "Output Shape" + << "Backend" << std::endl; + *out << std::string(72, '-') << std::endl; + } + + for (size_t node_idx = 0; node_idx < nodes_.size(); ++node_idx) { + auto& node = nodes_[node_idx]; + + if (node->op_type != OpType::INPUT) { + node->output_buffer.allocate_from_pool(pool); + } + + if (trace_nodes) { + std::cerr << "[graph:node_begin] idx=" << node_idx + << " " << format_node_debug(*node, nodes_, node_index_map_) + << std::endl; + } + + if (enable_profiling && node->op_type != OpType::INPUT) { + auto start = std::chrono::high_resolution_clock::now(); + + try { + compute_node_optimized(*node, nodes_, node_index_map_); + } catch (const std::exception& e) { + throw std::runtime_error( + std::string(e.what()) + " | " + format_node_debug(*node, nodes_, node_index_map_) + ); + } + + if (node->op_type == OpType::PERSISTENT) { + populated_node_ids_.insert(node->id); + } + + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start); + double ms = duration.count() / 1000.0; + + std::string shape_str = "["; + for (size_t i = 0; i < node->output_buffer.shape.size(); ++i) { + if (i > 0) shape_str += ","; + shape_str += std::to_string(node->output_buffer.shape[i]); + } + shape_str += "]"; + + std::string values_str = ""; + if (node->output_buffer.get_data()) { + size_t num_values = std::min(size_t(5), node->output_buffer.total_size); + values_str = " values=["; + + if (node->output_buffer.precision == Precision::FP32) { + if (node->op_type == OpType::SAMPLE) { + uint32_t* uint32_data = reinterpret_cast(node->output_buffer.get_data()); + for (size_t i = 0; i < num_values; ++i) { + if (i > 0) values_str += ","; + values_str += std::to_string(uint32_data[i]); + } + } else { + float* float_data = reinterpret_cast(node->output_buffer.get_data()); + for (size_t i = 0; i < num_values; ++i) { + if (i > 0) values_str += ","; + values_str += std::to_string(float_data[i]).substr(0, 6); + } + } + } else if (node->output_buffer.precision == Precision::FP16) { + __fp16* fp16_data = reinterpret_cast<__fp16*>(node->output_buffer.get_data()); + for (size_t i = 0; i < num_values; ++i) { + if (i > 0) values_str += ","; + values_str += std::to_string(static_cast(fp16_data[i])).substr(0, 6); + } + } else if (node->output_buffer.precision == Precision::INT8) { + int8_t* int8_data = reinterpret_cast(node->output_buffer.get_data()); + for (size_t i = 0; i < num_values; ++i) { + if (i > 0) values_str += ","; + values_str += std::to_string(static_cast(int8_data[i])); + } + } + + if (node->output_buffer.total_size > 5) { + values_str += ",..."; + } + values_str += "]"; + } + + std::string weights_str = ""; + if ((node->op_type == OpType::RMS_NORM || node->op_type == OpType::MATMUL || + node->op_type == OpType::GATHER || node->op_type == OpType::EMBEDDING || + node->op_type == OpType::ATTENTION || node->op_type == OpType::CONCAT) && + node->input_ids.size() >= 2) { + const auto& weight_node = nodes_[node_index_map_.at(node->input_ids[1])]; + if (weight_node->output_buffer.get_data()) { + size_t num_values = std::min(size_t(5), weight_node->output_buffer.total_size); + weights_str = " weights=["; + + if (weight_node->output_buffer.precision == Precision::FP32) { + const float* float_data = weight_node->output_buffer.data_as(); + for (size_t i = 0; i < num_values; ++i) { + if (i > 0) weights_str += ","; + weights_str += std::to_string(float_data[i]).substr(0, 6); + } + } else if (weight_node->output_buffer.precision == Precision::FP16) { + const __fp16* fp16_data = weight_node->output_buffer.data_as<__fp16>(); + for (size_t i = 0; i < num_values; ++i) { + if (i > 0) weights_str += ","; + weights_str += std::to_string(static_cast(fp16_data[i])).substr(0, 6); + } + } else if (weight_node->output_buffer.precision == Precision::INT8) { + const int8_t* int8_data = weight_node->output_buffer.data_as(); + for (size_t i = 0; i < num_values; ++i) { + if (i > 0) weights_str += ","; + weights_str += std::to_string(static_cast(int8_data[i])); + } + } else if (weight_node->output_buffer.precision == Precision::INT4) { + const uint8_t* packed = weight_node->output_buffer.data_as(); + int8x16_t high, low; + unpack_int4_as_int8x16x2(packed, high, low); + int8_t low_lanes[16], high_lanes[16]; + vst1q_s8(low_lanes, low); + vst1q_s8(high_lanes, high); + + for (size_t i = 0; i < num_values; ++i) { + if (i > 0) weights_str += ","; + int8_t val = (i < 16) ? low_lanes[i] : high_lanes[i - 16]; + weights_str += std::to_string(static_cast(val)); + } + } + + if (weight_node->output_buffer.total_size > 5) { + weights_str += ",..."; + } + weights_str += "]"; + } + } + + *out << std::left << std::setw(24) << get_op_name(node->op_type) + << std::setw(12) << std::fixed << std::setprecision(3) << ms + << std::setw(20) << shape_str + << values_str << weights_str << std::endl; + } else { + try { + compute_node_optimized(*node, nodes_, node_index_map_); + } catch (const std::exception& e) { + throw std::runtime_error( + std::string(e.what()) + " | " + format_node_debug(*node, nodes_, node_index_map_) + ); + } + + if (node->op_type == OpType::PERSISTENT) { + populated_node_ids_.insert(node->id); + } + } + + if (trace_nodes) { + std::cerr << "[graph:node_end] idx=" << node_idx + << " node_id=" << node->id + << " op=" << get_op_name(node->op_type) + << std::endl; + } + } + + if (trace_nodes) { + std::cerr << "[graph:after_node_loop]" << std::endl; + } + + std::unique_ptr capture_file_stream; + std::vector capture_outputs; + + if (trace_nodes) { + std::cerr << "[graph:after_capture_decl]" << std::endl; + } + + if (capture_requested) { + if (capture_to_stdout) { + capture_outputs.push_back(&std::cout); + } + + if (!capture_file_path.empty()) { + std::filesystem::path capture_path(capture_file_path); + if (capture_path.has_parent_path()) { + std::error_code ec; + std::filesystem::create_directories(capture_path.parent_path(), ec); + } + + auto stream_ptr = std::make_unique(capture_path, std::ios::out | std::ios::app); + if (stream_ptr->is_open()) { + capture_outputs.push_back(stream_ptr.get()); + capture_file_stream = std::move(stream_ptr); + } else { + std::cerr << "Failed to open capture file: " << capture_path << std::endl; + } + } + + if (!capture_dir.empty()) { + std::filesystem::path dir_path(capture_dir); + std::error_code ec; + std::filesystem::create_directories(dir_path, ec); + } + + if (capture_outputs.empty() && capture_dir.empty()) { + capture_requested = false; + } + } + + if (capture_requested) { + auto precision_to_string = [](Precision p) -> const char* { + switch (p) { + case Precision::FP32: return "FP32"; + case Precision::FP16: return "FP16"; + case Precision::INT8: return "INT8"; + default: return "UNKNOWN"; + } + }; + + auto format_double = [](double value) { + std::ostringstream oss; + oss << std::fixed << std::setprecision(6) << value; + return oss.str(); + }; + + auto now = std::chrono::system_clock::now(); + std::time_t now_time = std::chrono::system_clock::to_time_t(now); + std::tm time_info{}; +#if defined(_WIN32) + localtime_s(&time_info, &now_time); +#else + localtime_r(&now_time, &time_info); +#endif + + auto write_header = [&](std::ostream& stream) { + stream << "=== Graph Debug Capture ===" << std::endl; + stream << "Timestamp: " << std::put_time(&time_info, "%Y-%m-%d %H:%M:%S") << std::endl; + stream << "Captured nodes: " << debug_nodes_.size() << std::endl; + stream << std::string(60, '-') << std::endl; + }; + + auto write_separator = [](std::ostream& stream) { + stream << std::string(60, '-') << std::endl; + }; + + if (debug_nodes_.empty()) { + for (auto* stream : capture_outputs) { + write_header(*stream); + *stream << "No debug nodes registered on this graph." << std::endl; + write_separator(*stream); + stream->flush(); + } + } else { + for (auto* stream : capture_outputs) { + write_header(*stream); + } + + for (const auto& entry : debug_nodes_) { + auto node_it = node_index_map_.find(entry.node_id); + const GraphNode* node_ptr = nullptr; + if (node_it != node_index_map_.end()) { + node_ptr = nodes_[node_it->second].get(); + } + + if (!node_ptr) { + for (auto* stream : capture_outputs) { + *stream << "Layer " << entry.layer_idx << " - " << entry.name + << " (node " << entry.node_id << ")" << std::endl; + *stream << " Data: " << std::endl; + write_separator(*stream); + } + continue; + } + + const BufferDesc& buffer = node_ptr->output_buffer; + const void* data_ptr = buffer.get_data(); + size_t total_size = buffer.total_size; + + std::ostringstream shape_ss; + shape_ss << "["; + for (size_t i = 0; i < buffer.shape.size(); ++i) { + if (i > 0) { + shape_ss << ","; + } + shape_ss << buffer.shape[i]; + } + shape_ss << "]"; + std::string shape_str = shape_ss.str(); + + bool has_data = data_ptr != nullptr && total_size > 0; + size_t elements_to_process = total_size; + bool truncated = false; + if (has_data && elements_to_process > capture_max_elements && capture_max_elements > 0) { + elements_to_process = capture_max_elements; + truncated = true; + } + + std::vector preview_values; + if (capture_preview_count > 0) { + preview_values.reserve(std::min(capture_preview_count, elements_to_process)); + } + + double min_val = std::numeric_limits::infinity(); + double max_val = -std::numeric_limits::infinity(); + long double sum = 0.0L; + long double sum_sq = 0.0L; + + if (has_data && elements_to_process > 0) { + auto accumulate = [&](float value, size_t index) { + double v = static_cast(value); + min_val = std::min(min_val, v); + max_val = std::max(max_val, v); + sum += static_cast(value); + sum_sq += static_cast(value) * static_cast(value); + if (capture_preview_count > 0 && index < capture_preview_count) { + preview_values.push_back(value); + } + }; + + if (buffer.precision == Precision::FP32) { + const float* typed = static_cast(data_ptr); + for (size_t i = 0; i < elements_to_process; ++i) { + accumulate(typed[i], i); + } + } else if (buffer.precision == Precision::FP16) { + const __fp16* typed = reinterpret_cast(data_ptr); + for (size_t i = 0; i < elements_to_process; ++i) { + accumulate(static_cast(typed[i]), i); + } + } else if (buffer.precision == Precision::INT8) { + const int8_t* typed = reinterpret_cast(data_ptr); + for (size_t i = 0; i < elements_to_process; ++i) { + accumulate(static_cast(typed[i]), i); + } + } else if (buffer.precision == Precision::INT4) { + assert(elements_to_process % 32 == 0 && "INT4 precision capture requires element count to be multiple of 32"); + const uint8_t* packed_ptr = reinterpret_cast(data_ptr); + for (size_t i = 0; i < elements_to_process; i+=32) { + int8x16_t high, low; + unpack_int4_as_int8x16x2(packed_ptr + i / 2, high, low); + int8_t high_lanes[16], low_lanes[16]; + vst1q_s8(high_lanes, high); + vst1q_s8(low_lanes, low); + + for (size_t j = 0; j < 16; ++j) { + accumulate(static_cast(low_lanes[j]), i + j); + } + for (size_t j = 0; j < 16; ++j) { + accumulate(static_cast(high_lanes[j]), i + 16 + j); + } + } + } else { + has_data = false; + } + } else { + has_data = false; + } + + if (!capture_dir.empty() && has_data) { + std::string safe_name = entry.name; + std::string filename = capture_dir + "/" + safe_name + ".bin"; + std::ofstream bin_file(filename, std::ios::binary); + if (bin_file.is_open()) { + size_t bytes_to_write = buffer.byte_size; + if (truncated) { + bytes_to_write = PrecisionTraits::packed_size_of(buffer.precision, elements_to_process); + } + bin_file.write(reinterpret_cast(data_ptr), bytes_to_write); + } + } + + size_t processed_count = has_data ? elements_to_process : 0; + long double mean_ld = processed_count > 0 ? sum / processed_count : 0.0L; + long double variance_ld = processed_count > 0 ? (sum_sq / processed_count) - (mean_ld * mean_ld) : 0.0L; + if (variance_ld < 0.0L) { + variance_ld = 0.0L; + } + double mean_val = static_cast(mean_ld); + double stddev_val = processed_count > 0 ? std::sqrt(static_cast(variance_ld)) : 0.0; + + std::ostringstream preview_ss; + if (capture_preview_count > 0 && !preview_values.empty()) { + preview_ss << "["; + for (size_t i = 0; i < preview_values.size(); ++i) { + if (i > 0) { + preview_ss << ", "; + } + preview_ss << format_double(static_cast(preview_values[i])); + } + if (processed_count > preview_values.size()) { + if (!preview_values.empty()) { + preview_ss << ", ..."; + } else { + preview_ss << "..."; + } + } + preview_ss << "]"; + } + + for (auto* stream : capture_outputs) { + *stream << "Layer " << entry.layer_idx << " - " << entry.name + << " (node " << entry.node_id << ")" << std::endl; + *stream << " Shape: " << shape_str << " Precision: " << precision_to_string(buffer.precision) << std::endl; + if (!has_data) { + *stream << " Data: " << std::endl; + } else { + *stream << " Stats: min=" << format_double(min_val) + << " max=" << format_double(max_val) + << " mean=" << format_double(mean_val) + << " std=" << format_double(stddev_val) << std::endl; + if (truncated || processed_count < total_size) { + *stream << " Note: stats computed on first " << processed_count + << " of " << total_size << " values" << std::endl; + } + if (capture_preview_count > 0 && !preview_values.empty()) { + *stream << " Preview: " << preview_ss.str() << std::endl; + } + } + write_separator(*stream); + } + } + + for (auto* stream : capture_outputs) { + stream->flush(); + } + } + } + + if (enable_profiling) { + auto total_end = std::chrono::high_resolution_clock::now(); + auto total_duration = std::chrono::duration_cast(total_end - total_start); + double total_ms = total_duration.count() / 1000.0; + + *out << std::string(72, '-') << std::endl; + *out << "Total execution time: " << std::fixed << std::setprecision(3) << total_ms << " ms" << std::endl; + *out << "================================" << std::endl; + + if (profile_out.is_open()) { + profile_out.close(); + } + } + + if (trace_nodes) { + std::cerr << "[graph:execute_end]" << std::endl; + } +} + +void CactusGraph::hard_reset() { + nodes_.clear(); + node_index_map_.clear(); + mapped_files_.clear(); + weight_cache_.clear(); + next_node_id_ = 1; + debug_nodes_.clear(); + buffer_pool_.clear(); +} + +void CactusGraph::soft_reset() { + std::set cached_node_ids; + for (const auto& cache_entry : weight_cache_) { + cached_node_ids.insert(cache_entry.second); + } + + for (size_t pid : persistent_node_ids_) { + cached_node_ids.insert(pid); + } + + size_t max_preserved_id = 0; + for (const auto& node : nodes_) { + if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || + cached_node_ids.count(node->id)) { + max_preserved_id = std::max(max_preserved_id, node->id); + } + } + + auto preserved_nodes = std::move(nodes_); + auto preserved_index_map = std::move(node_index_map_); + + nodes_.clear(); + node_index_map_.clear(); + + for (auto& node : preserved_nodes) { + if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || + cached_node_ids.count(node->id)) { + size_t index = nodes_.size(); + node_index_map_[node->id] = index; + nodes_.push_back(std::move(node)); + } + } + + next_node_id_ = max_preserved_id + 1; + debug_nodes_.clear(); + if (!prefill_mode_) { + buffer_pool_.clear(); + shrink_thread_local_buffers(); + } +} + +void CactusGraph::soft_reset_keep_pool() { + std::set cached_node_ids; + for (const auto& cache_entry : weight_cache_) { + cached_node_ids.insert(cache_entry.second); + } + + for (size_t pid : persistent_node_ids_) { + cached_node_ids.insert(pid); + } + + size_t max_preserved_id = 0; + for (const auto& node : nodes_) { + if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || + cached_node_ids.count(node->id)) { + max_preserved_id = std::max(max_preserved_id, node->id); + } + } + + auto preserved_nodes = std::move(nodes_); + + nodes_.clear(); + node_index_map_.clear(); + + for (auto& node : preserved_nodes) { + if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || + cached_node_ids.count(node->id)) { + size_t index = nodes_.size(); + node_index_map_[node->id] = index; + nodes_.push_back(std::move(node)); + } + } + + next_node_id_ = max_preserved_id + 1; + debug_nodes_.clear(); +} diff --git a/cactus-transpile/graph/graph_io.cpp b/cactus-transpile/graph/graph_io.cpp new file mode 100644 index 000000000..418258c3c --- /dev/null +++ b/cactus-transpile/graph/graph_io.cpp @@ -0,0 +1,809 @@ +#include "graph.h" +#include "graph_param_io.h" +#include +#include +#include +#include +#include +#include +#include + +namespace { + constexpr uint32_t fourcc(char a, char b, char c, char d) { + return static_cast(static_cast(a)) | + (static_cast(static_cast(b)) << 8) | + (static_cast(static_cast(c)) << 16) | + (static_cast(static_cast(d)) << 24); + } + + constexpr uint32_t CACTUS_MAGIC = 0x54434143; + constexpr uint32_t CACTUS_GRAPH_MAGIC = fourcc('C', 'G', 'R', 'F'); + constexpr uint32_t FLAG_HAS_SCALES = 1 << 0; + constexpr uint32_t FLAG_INTERLEAVED = 1 << 3; + constexpr size_t HEADER_SIZE = 84; + + inline size_t align_offset(size_t offset, size_t alignment) { + size_t remainder = offset % alignment; + if (remainder == 0) return offset; + return offset + (alignment - remainder); + } + + inline void write_u32(std::ostream& out, uint32_t v) { + out.write(reinterpret_cast(&v), sizeof(v)); + } + + inline void write_u64(std::ostream& out, uint64_t v) { + out.write(reinterpret_cast(&v), sizeof(v)); + } + + void write_size_vector(std::ostream& out, const std::vector& values) { + uint32_t size = static_cast(values.size()); + write_u32(out, size); + for (size_t v : values) { + write_u64(out, static_cast(v)); + } + } + + void write_u32_vector(std::ostream& out, const std::vector& values) { + uint32_t size = static_cast(values.size()); + write_u32(out, size); + for (uint32_t v : values) { + write_u32(out, v); + } + } + + std::vector compute_leaf_outputs(const GraphFile::SerializedGraph& sg) { + std::vector referenced(sg.nodes.size(), false); + + for (const auto& node : sg.nodes) { + for (uint32_t input_idx : node.inputs) { + if (input_idx >= referenced.size()) { + throw std::runtime_error("Graph save failed: input index out of range"); + } + referenced[input_idx] = true; + } + } + + std::vector outputs; + outputs.reserve(sg.nodes.size()); + + for (uint32_t i = 0; i < sg.nodes.size(); ++i) { + if (!referenced[i]) { + outputs.push_back(i); + } + } + + return outputs; + } + + void write_serialized_graph(std::ostream& out, const GraphFile::SerializedGraph& sg) { + write_u32(out, sg.header.magic); + write_u32(out, sg.header.version); + write_u32(out, sg.header.node_count); + write_u32(out, sg.header.flags); + + write_u32_vector(out, sg.graph_inputs); + write_u32_vector(out, sg.graph_outputs); + + for (const auto& node : sg.nodes) { + write_u32(out, node.index); + write_u32(out, static_cast(node.op_type)); + write_u32_vector(out, node.inputs); + write_size_vector(out, node.output_shape); + write_u32(out, static_cast(node.precision)); + GraphParamIO::write_op_params(out, node.op_type, node.params); + } + + if (!out) { + throw std::runtime_error("Error writing serialized graph"); + } + } + + // read helpers + uint32_t read_u32(std::istream& in) { + uint32_t v = 0; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) { + throw std::runtime_error("Unexpected EOF while reading uint32"); + } + return v; + } + + uint64_t read_u64(std::istream& in) { + uint64_t v = 0; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) { + throw std::runtime_error("Unexpected EOF while reading uint64"); + } + return v; + } + + std::vector read_u32_vector(std::istream& in) { + uint32_t count = read_u32(in); + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; ++i) { + values.push_back(read_u32(in)); + } + return values; + } + + std::vector read_size_vector(std::istream& in) { + uint32_t count = read_u32(in); + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; ++i) { + values.push_back(static_cast(read_u64(in))); + } + return values; + } + + bool is_binary_broadcast_op(OpType op_type) { + switch (op_type) { + case OpType::ADD: + case OpType::ADD_CLIPPED: + case OpType::SUBTRACT: + case OpType::MULTIPLY: + case OpType::DIVIDE: + case OpType::NOT_EQUAL: + return true; + default: + return false; + } + } + + void populate_derived_params(CactusGraph& graph, + const GraphFile::NodeEntry& node_entry, + const std::vector& runtime_inputs, + OpParams& params) { + if (is_binary_broadcast_op(node_entry.op_type)) { + if (runtime_inputs.size() != 2) { + throw std::runtime_error("Graph file corrupted: binary op missing inputs"); + } + const auto& lhs = graph.get_output_buffer(runtime_inputs[0]); + const auto& rhs = graph.get_output_buffer(runtime_inputs[1]); + params.broadcast_info = BroadcastInfo::compute(lhs.shape, rhs.shape); + } + } + + GraphFile::GraphHeader read_graph_header(std::istream& in) { + GraphFile::GraphHeader header; + header.magic = read_u32(in); + header.version = read_u32(in); + header.node_count = read_u32(in); + header.flags = read_u32(in); + + if (header.magic != CACTUS_GRAPH_MAGIC) { + throw std::runtime_error("Invalid graph file: bad magic"); + } + if (header.version != 4) { + throw std::runtime_error("Unsupported graph file version: " + + std::to_string(header.version)); + } + + return header; + } + + GraphFile::NodeEntry read_node_entry(std::istream& in) { + GraphFile::NodeEntry node; + node.index = read_u32(in); + uint32_t op_type_val = read_u32(in); + if (op_type_val > static_cast(OpType::CUMSUM)) { + throw std::runtime_error("Graph file corrupted: invalid op type"); + } + node.op_type = static_cast(op_type_val); + node.inputs = read_u32_vector(in); + node.output_shape = read_size_vector(in); + uint32_t precision_val = read_u32(in); + if (precision_val > static_cast(Precision::INT4)) { + throw std::runtime_error("Graph file corrupted: invalid precision"); + } + node.precision = static_cast(precision_val); + GraphParamIO::read_op_params(in, node.op_type, node.params); + return node; + } + + +} // namespace + +void CactusGraph::save(const std::string& path) { + GraphFile::save_graph(*this, path); +} + +CactusGraph CactusGraph::from_serialized(const GraphFile::SerializedGraph& sg) { + CactusGraph graph; + std::vector runtime_ids; + runtime_ids.reserve(sg.nodes.size()); + + if (sg.nodes.size() != sg.header.node_count) { + throw std::runtime_error("Graph file corrupted: node count mismatch"); + } + + for (size_t i = 0; i < sg.nodes.size(); ++i) { + const auto& node_entry = sg.nodes[i]; + + if (node_entry.index != i) { + throw std::runtime_error("Graph file corrupted: node indices must be dense and ordered"); + } + std::vector runtime_inputs; + runtime_inputs.reserve(node_entry.inputs.size()); + + for (uint32_t serialized_input_idx : node_entry.inputs) { + if (serialized_input_idx >= runtime_ids.size()) { + throw std::runtime_error( + "Graph file corrupted: input refers to a node that has not been reconstructed yet" + ); + } + runtime_inputs.push_back(runtime_ids[serialized_input_idx]); + } + + size_t new_node_id = 0; + + if (node_entry.op_type == OpType::INPUT) { + new_node_id = graph.input(node_entry.output_shape, node_entry.precision); + } + else { + OpParams params = node_entry.params; + params.output_precision = node_entry.precision; + populate_derived_params(graph, node_entry, runtime_inputs, params); + new_node_id = graph.add_node(node_entry.op_type, runtime_inputs, node_entry.output_shape, params); + + if (node_entry.op_type == OpType::PERSISTENT) { + graph.persistent_node_ids_.insert(new_node_id); + } + } + runtime_ids.push_back(new_node_id); + } + return graph; +} + +CactusGraph CactusGraph::load(const std::string& path) { + GraphFile::SerializedGraph sg = GraphFile::load_graph(path); + return from_serialized(sg); +} + +size_t CactusGraph::mmap_embeddings(const std::string& filename) { + auto mapped_file = std::make_unique(filename); + + const auto& shape = mapped_file->shape(); + if (shape.size() != 2) { + throw std::runtime_error("Memory-mapped embeddings must be 2D [vocab_size, embedding_dim]"); + } + + Precision precision = mapped_file->precision(); + + size_t node_id = input(shape, precision); + set_external_input(node_id, const_cast(mapped_file->data()), precision); + + if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { + set_grouped_scales(node_id, mapped_file->group_size(), mapped_file->num_groups(), + const_cast(mapped_file->scales_data())); + + if (mapped_file->is_interleaved()) { + auto& buffer = nodes_[node_index_map_.at(node_id)]->output_buffer; + buffer.set_interleaved(true, mapped_file->original_N()); + } + } + + size_t file_idx = mapped_files_.size(); + mapped_files_.push_back(std::move(mapped_file)); + node_to_mapped_file_[node_id] = file_idx; + weight_cache_[filename] = node_id; + return node_id; +} + +size_t CactusGraph::mmap_weights(const std::string& filename) { + auto it = weight_cache_.find(filename); + if (it != weight_cache_.end()) { + return it->second; + } + + auto mapped_file = std::make_unique(filename); + + const auto& shape = mapped_file->shape(); + Precision precision = mapped_file->precision(); + + size_t node_id = input(shape, precision); + set_external_input(node_id, const_cast(mapped_file->data()), precision); + + if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { + set_grouped_scales(node_id, mapped_file->group_size(), mapped_file->num_groups(), + const_cast(mapped_file->scales_data())); + + if (mapped_file->is_interleaved()) { + auto& buffer = nodes_[node_index_map_.at(node_id)]->output_buffer; + buffer.set_interleaved(true, mapped_file->original_N()); + } + } + + size_t file_idx = mapped_files_.size(); + mapped_files_.push_back(std::move(mapped_file)); + node_to_mapped_file_[node_id] = file_idx; + weight_cache_[filename] = node_id; + return node_id; +} + +void CactusGraph::release_weight_pages(size_t node_id) { + auto it = node_to_mapped_file_.find(node_id); + if (it != node_to_mapped_file_.end() && it->second < mapped_files_.size()) { + mapped_files_[it->second]->release_pages(); + } +} + +void CactusGraph::prefetch_weight_pages(size_t node_id) { + auto it = node_to_mapped_file_.find(node_id); + if (it != node_to_mapped_file_.end() && it->second < mapped_files_.size()) { + mapped_files_[it->second]->prefetch_pages(); + } +} + +void CactusGraph::release_all_weight_pages() { + for (auto& mf : mapped_files_) { + if (mf) mf->release_pages(); + } +} + +void CactusGraph::set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr) { + auto it = node_index_map_.find(node_id); + if (it != node_index_map_.end()) { + nodes_[it->second]->output_buffer.set_grouped_scales(group_size, num_groups, scales_ptr); + } +} + +void CactusGraph::set_interleaved(size_t node_id, bool interleaved, size_t original_N) { + auto it = node_index_map_.find(node_id); + if (it != node_index_map_.end()) { + nodes_[it->second]->output_buffer.set_interleaved(interleaved, original_N); + } +} + +size_t CactusGraph::embedding(const std::string& filename, size_t indices) { + auto mapped_file = std::make_unique(filename); + + const auto& shape = mapped_file->shape(); + if (shape.size() != 2) { + throw std::runtime_error("Embedding file must contain 2D tensor [vocab_size, hidden_dim]"); + } + + Precision precision = mapped_file->precision(); + size_t embeddings_node = input(shape, precision); + set_external_input(embeddings_node, const_cast(mapped_file->data()), precision); + + if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { + set_grouped_scales(embeddings_node, mapped_file->group_size(), mapped_file->num_groups(), + const_cast(mapped_file->scales_data())); + + if (mapped_file->is_interleaved()) { + auto& buffer = nodes_[node_index_map_.at(embeddings_node)]->output_buffer; + buffer.set_interleaved(true, mapped_file->original_N()); + } + } + + mapped_files_.push_back(std::move(mapped_file)); + + const auto& idx_shape = get_output_buffer(indices).shape; + std::vector output_shape = idx_shape; + output_shape.push_back(shape[1]); + + OpParams params; + params.output_precision = Precision::FP16; + return add_node(OpType::EMBEDDING, {embeddings_node, indices}, output_shape, params); +} + + +namespace GraphFile { + +void save_graph(const CactusGraph& graph, + const std::string& filename) { + + std::ofstream out(filename, std::ios::binary); + if (!out) { + throw std::runtime_error("Cannot open file for writing: " + filename); + } + + std::unordered_map id_to_index; + id_to_index.reserve(graph.nodes_.size()); + + for (uint32_t i = 0; i < graph.nodes_.size(); ++i) { + id_to_index[graph.nodes_[i]->id] = i; + } + + SerializedGraph sg; + sg.header.magic = CACTUS_GRAPH_MAGIC; + sg.header.version = 4; + sg.header.node_count = static_cast(graph.nodes_.size()); + sg.header.flags = 0; + + sg.nodes.reserve(graph.nodes_.size()); + + for (uint32_t i = 0; i < graph.nodes_.size(); ++i) { + const auto& node = graph.nodes_[i]; + + NodeEntry entry; + entry.index = i; + entry.op_type = node->op_type; + entry.output_shape = node->output_buffer.shape; + entry.precision = node->output_buffer.precision; + entry.params = node->params; + + entry.inputs.reserve(node->input_ids.size()); + for (size_t input_id : node->input_ids) { + auto it = id_to_index.find(input_id); + if (it == id_to_index.end()) { + throw std::runtime_error("Graph save failed: missing input id mapping"); + } + entry.inputs.push_back(it->second); + } + + if (node->op_type == OpType::INPUT) { + sg.graph_inputs.push_back(entry.index); + } + + sg.nodes.push_back(std::move(entry)); + } + + sg.graph_outputs = compute_leaf_outputs(sg); + + write_serialized_graph(out, sg); + + if (!out) { + throw std::runtime_error("Error writing graph data to file: " + filename); + } +} + +SerializedGraph load_graph(const std::string& filename) { + std::ifstream in(filename, std::ios::binary); + if (!in) { + throw std::runtime_error("Cannot open file for reading: " + filename); + } + SerializedGraph sg; + sg.header = read_graph_header(in); + sg.graph_inputs = read_u32_vector(in); + sg.graph_outputs = read_u32_vector(in); + sg.nodes.reserve(sg.header.node_count); + for (uint32_t i = 0; i < sg.header.node_count; ++i) { + sg.nodes.push_back(read_node_entry(in)); + } + + if (sg.nodes.size() != sg.header.node_count) { + throw std::runtime_error("Graph file corrupted: node count mismatch"); + } + + for (uint32_t i = 0; i < sg.nodes.size(); ++i) { + const auto& node = sg.nodes[i]; + + if (node.index != i) { + throw std::runtime_error("Graph file corrupted: node indices must be dense and ordered"); + } + + for (uint32_t input_idx : node.inputs) { + if (input_idx >= sg.nodes.size()) { + throw std::runtime_error("Graph file corrupted: input index out of range"); + } + } + } + + for (uint32_t input_idx : sg.graph_inputs) { + if (input_idx >= sg.nodes.size()) { + throw std::runtime_error("Graph file corrupted: graph input index out of range"); + } + } + + for (uint32_t output_idx : sg.graph_outputs) { + if (output_idx >= sg.nodes.size()) { + throw std::runtime_error("Graph file corrupted: graph output index out of range"); + } + } + + return sg; +} + +void save_node(CactusGraph& graph, size_t node_id, const std::string& filename) { + graph.execute(); + void* data = graph.get_output(node_id); + + const auto& buffer = graph.get_output_buffer(node_id); + const auto& shape = buffer.shape; + Precision precision = buffer.precision; + + std::ofstream file(filename, std::ios::binary); + if (!file) { + throw std::runtime_error("Cannot open file for writing: " + filename); + } + + size_t total_elements = 1; + for (size_t dim : shape) { + total_elements *= dim; + } + + size_t byte_size = PrecisionTraits::packed_size_of(precision, total_elements); + + bool has_scales = PrecisionTraits::is_integer(precision) && buffer.group_size > 0 && buffer.scales_data; + size_t N = shape.size() >= 1 ? shape[0] : 1; + size_t scales_bytes = has_scales ? (N * buffer.num_groups * sizeof(__fp16)) : 0; + + uint32_t ndim = static_cast(shape.size()); + uint32_t flags = has_scales ? FLAG_HAS_SCALES : 0; + if (buffer.is_interleaved) { + flags |= FLAG_INTERLEAVED; + } + uint32_t alignment = 32; + + uint32_t magic = CACTUS_MAGIC; + file.write(reinterpret_cast(&magic), sizeof(magic)); + file.write(reinterpret_cast(&flags), sizeof(flags)); + file.write(reinterpret_cast(&alignment), sizeof(alignment)); + file.write(reinterpret_cast(&ndim), sizeof(ndim)); + + for (uint32_t i = 0; i < 4; i++) { + uint64_t dim_val = (i < shape.size()) ? static_cast(shape[i]) : 0; + file.write(reinterpret_cast(&dim_val), sizeof(dim_val)); + } + + uint32_t prec_val = static_cast(precision); + file.write(reinterpret_cast(&prec_val), sizeof(prec_val)); + + uint64_t data_bytes = static_cast(byte_size); + uint64_t scales_bytes_val = static_cast(scales_bytes); + file.write(reinterpret_cast(&data_bytes), sizeof(data_bytes)); + file.write(reinterpret_cast(&scales_bytes_val), sizeof(scales_bytes_val)); + + uint32_t group_size = has_scales ? static_cast(buffer.group_size) : 0; + uint32_t num_groups = has_scales ? static_cast(buffer.num_groups) : 0; + file.write(reinterpret_cast(&group_size), sizeof(group_size)); + file.write(reinterpret_cast(&num_groups), sizeof(num_groups)); + + uint64_t original_N = buffer.is_interleaved ? buffer.original_N : N; + file.write(reinterpret_cast(&original_N), sizeof(original_N)); + + size_t header_end = HEADER_SIZE; + size_t aligned_header = align_offset(header_end, alignment); + size_t header_padding = aligned_header - header_end; + for (size_t i = 0; i < header_padding; i++) { + char zero = 0; + file.write(&zero, 1); + } + + if (has_scales) { + file.write(static_cast(buffer.scales_data), scales_bytes); + + size_t scales_end = aligned_header + scales_bytes; + size_t data_start = align_offset(scales_end, alignment); + size_t scales_padding = data_start - scales_end; + for (size_t i = 0; i < scales_padding; i++) { + char zero = 0; + file.write(&zero, 1); + } + } + + file.write(static_cast(data), byte_size); + + if (!file) { + throw std::runtime_error("Error writing node data to file: " + filename); + } +} + +// MappedFile implementation + +MappedFile::MappedFile(const std::string& filename) + : fd_(-1), mapped_data_(nullptr), file_size_(0), data_offset_(0) { + fd_ = open(filename.c_str(), O_RDONLY); + if (fd_ == -1) { + throw std::runtime_error("Cannot open file for mapping: " + filename); + } + + struct stat st; + if (fstat(fd_, &st) == -1) { + close(fd_); + throw std::runtime_error("Cannot get file size: " + filename); + } + file_size_ = static_cast(st.st_size); + + mapped_data_ = mmap(nullptr, file_size_, PROT_READ, MAP_SHARED, fd_, 0); + if (mapped_data_ == MAP_FAILED) { + close(fd_); + throw std::runtime_error("Cannot map file: " + filename); + } + + close(fd_); + fd_ = -1; + + parse_header(); + apply_madvise_hints(); +} + +MappedFile::~MappedFile() { + if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { + madvise(mapped_data_, file_size_, MADV_DONTNEED); + munmap(mapped_data_, file_size_); + mapped_data_ = nullptr; + } + if (fd_ != -1) { + close(fd_); + fd_ = -1; + } +} + +MappedFile::MappedFile(MappedFile&& other) noexcept + : fd_(other.fd_), mapped_data_(other.mapped_data_), file_size_(other.file_size_), + data_offset_(other.data_offset_), shape_(std::move(other.shape_)), + precision_(other.precision_), byte_size_(other.byte_size_), + group_size_(other.group_size_), num_groups_(other.num_groups_), + scales_offset_(other.scales_offset_), scales_bytes_(other.scales_bytes_), + alignment_(other.alignment_), + is_interleaved_(other.is_interleaved_), + original_N_(other.original_N_) { + other.fd_ = -1; + other.mapped_data_ = nullptr; + other.file_size_ = 0; + other.is_interleaved_ = false; + other.original_N_ = 0; +} + +MappedFile& MappedFile::operator=(MappedFile&& other) noexcept { + if (this != &other) { + if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { + munmap(mapped_data_, file_size_); + } + if (fd_ != -1) { + close(fd_); + } + + fd_ = other.fd_; + mapped_data_ = other.mapped_data_; + file_size_ = other.file_size_; + data_offset_ = other.data_offset_; + shape_ = std::move(other.shape_); + precision_ = other.precision_; + byte_size_ = other.byte_size_; + group_size_ = other.group_size_; + num_groups_ = other.num_groups_; + scales_offset_ = other.scales_offset_; + scales_bytes_ = other.scales_bytes_; + alignment_ = other.alignment_; + is_interleaved_ = other.is_interleaved_; + original_N_ = other.original_N_; + other.fd_ = -1; + other.mapped_data_ = nullptr; + other.file_size_ = 0; + other.is_interleaved_ = false; + other.original_N_ = 0; + } + return *this; +} + +const std::vector& MappedFile::shape() const { + return shape_; +} + +Precision MappedFile::precision() const { + return precision_; +} + +size_t MappedFile::byte_size() const { + return byte_size_; +} + +const void* MappedFile::scales_data() const { + return static_cast(mapped_data_) + scales_offset_; +} + +void* MappedFile::data() { + return static_cast(mapped_data_) + data_offset_; +} + +const void* MappedFile::data() const { + return static_cast(mapped_data_) + data_offset_; +} + +template +const T* MappedFile::typed_data() const { + return static_cast(data()); +} + +void MappedFile::parse_header() { + if (file_size_ < HEADER_SIZE) { + throw std::runtime_error("File too small: insufficient data for header"); + } + + const char* ptr = static_cast(mapped_data_); + size_t offset = 0; + + uint32_t magic = *reinterpret_cast(ptr + offset); + offset += sizeof(uint32_t); + if (magic != CACTUS_MAGIC) { + throw std::runtime_error("Invalid tensor file: missing CACT magic number"); + } + + uint32_t flags = *reinterpret_cast(ptr + offset); + offset += sizeof(uint32_t); + is_interleaved_ = (flags & FLAG_INTERLEAVED) != 0; + + alignment_ = *reinterpret_cast(ptr + offset); + offset += sizeof(uint32_t); + if (alignment_ == 0) alignment_ = 1; + + uint32_t ndim = *reinterpret_cast(ptr + offset); + offset += sizeof(uint32_t); + + shape_.clear(); + for (uint32_t i = 0; i < 4; i++) { + uint64_t dim_val = *reinterpret_cast(ptr + offset); + offset += sizeof(uint64_t); + if (i < ndim && dim_val > 0) { + shape_.push_back(static_cast(dim_val)); + } + } + + uint32_t prec_val = *reinterpret_cast(ptr + offset); + precision_ = static_cast(prec_val); + offset += sizeof(uint32_t); + + byte_size_ = *reinterpret_cast(ptr + offset); + offset += sizeof(uint64_t); + + scales_bytes_ = *reinterpret_cast(ptr + offset); + offset += sizeof(uint64_t); + + group_size_ = *reinterpret_cast(ptr + offset); + offset += sizeof(uint32_t); + + num_groups_ = *reinterpret_cast(ptr + offset); + offset += sizeof(uint32_t); + + original_N_ = *reinterpret_cast(ptr + offset); + offset += sizeof(uint64_t); + + size_t aligned_header = align_offset(HEADER_SIZE, alignment_); + + if (scales_bytes_ > 0) { + scales_offset_ = aligned_header; + size_t scales_end = scales_offset_ + scales_bytes_; + data_offset_ = align_offset(scales_end, alignment_); + } else { + scales_offset_ = 0; + data_offset_ = aligned_header; + } + + if (data_offset_ + byte_size_ > file_size_) { + throw std::runtime_error("File corrupted: data extends beyond file size"); + } + +} + +void MappedFile::apply_madvise_hints() { + if (scales_bytes_ > 0 && scales_offset_ > 0) { + madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_WILLNEED); + } + + madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_SEQUENTIAL); + + if (byte_size_ > 1024 * 1024) { + madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_WILLNEED); + } +} + +void MappedFile::release_pages() { + if (mapped_data_ == nullptr || mapped_data_ == MAP_FAILED) return; + + if (scales_bytes_ > 0 && scales_offset_ > 0) { + madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_DONTNEED); + } + madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_DONTNEED); +} + +void MappedFile::prefetch_pages() { + if (mapped_data_ == nullptr || mapped_data_ == MAP_FAILED) return; + + if (scales_bytes_ > 0 && scales_offset_ > 0) { + madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_WILLNEED); + } + madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_WILLNEED); +} + +template const int8_t* MappedFile::typed_data() const; +template const float* MappedFile::typed_data() const; +template const uint16_t* MappedFile::typed_data() const; +template const uint8_t* MappedFile::typed_data() const; + +} // namespace GraphFile diff --git a/cactus-transpile/graph/graph_ops_math.cpp b/cactus-transpile/graph/graph_ops_math.cpp new file mode 100644 index 000000000..ee49bfd56 --- /dev/null +++ b/cactus-transpile/graph/graph_ops_math.cpp @@ -0,0 +1,737 @@ +#include "graph.h" +#include "../kernel/kernel.h" +#include +#include +#include + +namespace Quantization { + void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale) { + cactus_int8_to_fp32(src, dst, count, scale); + } + + void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale) { + cactus_fp32_to_int8(src, dst, count, scale); + } + + void fp16_to_fp32(const __fp16* src, float* dst, size_t count) { + cactus_fp16_to_fp32(src, dst, count); + } + + void fp32_to_fp16(const float* src, __fp16* dst, size_t count) { + cactus_fp32_to_fp16(src, dst, count); + } + + void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale) { + cactus_int8_to_fp16(src, dst, count, scale); + } + + void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale) { + cactus_fp16_to_int8(src, dst, count, scale); + } +} + +static std::vector compute_strides(const std::vector& shape, const std::vector& target_shape) { + std::vector strides(target_shape.size()); + + size_t shape_offset = target_shape.size() - shape.size(); + + for (size_t i = 0; i < target_shape.size(); ++i) { + if (i < shape_offset) { + strides[i] = 0; + } else { + size_t dim_idx = i - shape_offset; + if (shape[dim_idx] == 1) { + strides[i] = 0; + } else { + strides[i] = 1; + for (size_t j = dim_idx + 1; j < shape.size(); ++j) { + strides[i] *= shape[j]; + } + } + } + } + + return strides; +} + +void dispatch_binary_op_f16(OpType op, const __fp16* lhs, const __fp16* rhs, __fp16* output, size_t count) { + switch (op) { + case OpType::ADD: + cactus_add_f16(lhs, rhs, output, count); + break; + case OpType::ADD_CLIPPED: + cactus_add_f16_clipped(lhs, rhs, output, count); + break; + case OpType::SUBTRACT: + cactus_subtract_f16(lhs, rhs, output, count); + break; + case OpType::MULTIPLY: + cactus_multiply_f16(lhs, rhs, output, count); + break; + case OpType::DIVIDE: + cactus_divide_f16(lhs, rhs, output, count); + break; + case OpType::NOT_EQUAL: + cactus_not_equal_f16(lhs, rhs, output, count); + break; + default: + break; + } +} + +static float dispatch_binary_op_scalar_fp32(OpType op, float lhs, float rhs) { + switch (op) { + case OpType::ADD: + return lhs + rhs; + case OpType::ADD_CLIPPED: { + constexpr float FP16_MAX = 65500.0f; + float value = lhs + rhs; + return std::fmin(std::fmax(value, -FP16_MAX), FP16_MAX); + } + case OpType::SUBTRACT: + return lhs - rhs; + case OpType::MULTIPLY: + return lhs * rhs; + case OpType::DIVIDE: + return lhs / rhs; + case OpType::NOT_EQUAL: + return lhs != rhs ? 1.0f : 0.0f; + default: + throw std::runtime_error("unsupported FP32 binary op"); + } +} + +static void dispatch_binary_op_fp32(OpType op, const float* lhs, const float* rhs, float* output, size_t count) { + for (size_t i = 0; i < count; ++i) { + output[i] = dispatch_binary_op_scalar_fp32(op, lhs[i], rhs[i]); + } +} + +static void dispatch_binary_op_fp32_to_fp16( + OpType op, + const float* lhs, + const float* rhs, + __fp16* output, + size_t count +) { + for (size_t i = 0; i < count; ++i) { + output[i] = static_cast<__fp16>(dispatch_binary_op_scalar_fp32(op, lhs[i], rhs[i])); + } +} + +static void dispatch_binary_op_f16_to_fp32( + OpType op, + const __fp16* lhs, + const __fp16* rhs, + float* output, + size_t count +) { + for (size_t i = 0; i < count; ++i) { + output[i] = dispatch_binary_op_scalar_fp32( + op, + static_cast(lhs[i]), + static_cast(rhs[i]) + ); + } +} + +static void dispatch_binary_op_broadcast_fp32( + OpType op, + const float* lhs, + const float* rhs, + float* output, + const std::vector& lhs_strides, + const std::vector& rhs_strides, + const std::vector& output_shape +) { + size_t output_size = 1; + for (size_t dim : output_shape) { + output_size *= dim; + } + + for (size_t linear_index = 0; linear_index < output_size; ++linear_index) { + size_t remainder = linear_index; + size_t lhs_index = 0; + size_t rhs_index = 0; + for (size_t dim = output_shape.size(); dim-- > 0;) { + const size_t coord = remainder % output_shape[dim]; + remainder /= output_shape[dim]; + lhs_index += coord * lhs_strides[dim]; + rhs_index += coord * rhs_strides[dim]; + } + output[linear_index] = dispatch_binary_op_scalar_fp32(op, lhs[lhs_index], rhs[rhs_index]); + } +} + +static void dispatch_binary_op_broadcast_fp32_to_fp16( + OpType op, + const float* lhs, + const float* rhs, + __fp16* output, + const std::vector& lhs_strides, + const std::vector& rhs_strides, + const std::vector& output_shape +) { + size_t output_size = 1; + for (size_t dim : output_shape) { + output_size *= dim; + } + + for (size_t linear_index = 0; linear_index < output_size; ++linear_index) { + size_t remainder = linear_index; + size_t lhs_index = 0; + size_t rhs_index = 0; + for (size_t dim = output_shape.size(); dim-- > 0;) { + const size_t coord = remainder % output_shape[dim]; + remainder /= output_shape[dim]; + lhs_index += coord * lhs_strides[dim]; + rhs_index += coord * rhs_strides[dim]; + } + output[linear_index] = static_cast<__fp16>( + dispatch_binary_op_scalar_fp32(op, lhs[lhs_index], rhs[rhs_index]) + ); + } +} + +static void dispatch_binary_op_broadcast_f16_to_fp32( + OpType op, + const __fp16* lhs, + const __fp16* rhs, + float* output, + const std::vector& lhs_strides, + const std::vector& rhs_strides, + const std::vector& output_shape +) { + size_t output_size = 1; + for (size_t dim : output_shape) { + output_size *= dim; + } + + for (size_t linear_index = 0; linear_index < output_size; ++linear_index) { + size_t remainder = linear_index; + size_t lhs_index = 0; + size_t rhs_index = 0; + for (size_t dim = output_shape.size(); dim-- > 0;) { + const size_t coord = remainder % output_shape[dim]; + remainder /= output_shape[dim]; + lhs_index += coord * lhs_strides[dim]; + rhs_index += coord * rhs_strides[dim]; + } + output[linear_index] = dispatch_binary_op_scalar_fp32( + op, + static_cast(lhs[lhs_index]), + static_cast(rhs[rhs_index]) + ); + } +} + +void dispatch_unary_op_f16(OpType op, const __fp16* input, __fp16* output, size_t count, float param) { + ScalarOpType scalar_op; + switch (op) { + case OpType::SCALAR_ADD: scalar_op = ScalarOpType::ADD; break; + case OpType::SCALAR_SUBTRACT: scalar_op = ScalarOpType::SUBTRACT; break; + case OpType::SCALAR_MULTIPLY: scalar_op = ScalarOpType::MULTIPLY; break; + case OpType::SCALAR_DIVIDE: scalar_op = ScalarOpType::DIVIDE; break; + case OpType::SCALAR_NOT_EQUAL: scalar_op = ScalarOpType::NOT_EQUAL; break; + case OpType::SCALAR_EXP: scalar_op = ScalarOpType::EXP; break; + case OpType::SCALAR_SQRT: scalar_op = ScalarOpType::SQRT; break; + case OpType::SCALAR_COS: scalar_op = ScalarOpType::COS; break; + case OpType::SCALAR_SIN: scalar_op = ScalarOpType::SIN; break; + case OpType::SCALAR_LOG: scalar_op = ScalarOpType::LOG; break; + case OpType::SCALAR_FLOOR_DIVIDE: scalar_op = ScalarOpType::FLOOR_DIVIDE; break; + case OpType::ABS: scalar_op = ScalarOpType::ABS; break; + case OpType::POW: scalar_op = ScalarOpType::POW; break; + default: return; + } + + cactus_scalar_op_f16(input, output, count, param, scalar_op); +} + +static float dispatch_unary_op_scalar_fp32(OpType op, float input, float param) { + switch (op) { + case OpType::SCALAR_ADD: + return input + param; + case OpType::SCALAR_SUBTRACT: + return input - param; + case OpType::SCALAR_MULTIPLY: + return input * param; + case OpType::SCALAR_DIVIDE: + return input / param; + case OpType::SCALAR_NOT_EQUAL: + return input != param ? 1.0f : 0.0f; + case OpType::SCALAR_EXP: + return std::exp(input); + case OpType::SCALAR_SQRT: + return std::sqrt(input); + case OpType::SCALAR_COS: + return std::cos(input); + case OpType::SCALAR_SIN: + return std::sin(input); + case OpType::SCALAR_LOG: + return std::log(input); + case OpType::SCALAR_FLOOR_DIVIDE: + return std::floor(input / param); + case OpType::ABS: + return std::fabs(input); + case OpType::POW: + return std::pow(input, param); + default: + throw std::runtime_error("unsupported FP32 unary op"); + } +} + +static void dispatch_unary_op_fp32(OpType op, const float* input, float* output, size_t count, float param) { + for (size_t i = 0; i < count; ++i) { + output[i] = dispatch_unary_op_scalar_fp32(op, input[i], param); + } +} + +static void dispatch_unary_op_fp32_to_fp16( + OpType op, + const float* input, + __fp16* output, + size_t count, + float param +) { + for (size_t i = 0; i < count; ++i) { + output[i] = static_cast<__fp16>(dispatch_unary_op_scalar_fp32(op, input[i], param)); + } +} + +static void dispatch_unary_op_f16_to_fp32( + OpType op, + const __fp16* input, + float* output, + size_t count, + float param +) { + for (size_t i = 0; i < count; ++i) { + output[i] = dispatch_unary_op_scalar_fp32(op, static_cast(input[i]), param); + } +} + +void compute_binary_op_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& lhs = get_input(node, 0, nodes, node_index_map); + const auto& rhs = get_input(node, 1, nodes, node_index_map); + const Precision output_precision = node.output_buffer.precision; + + if (lhs.precision != rhs.precision) { + throw std::runtime_error("Binary operations require matching input precision"); + } + + if (lhs.precision != Precision::FP16 && lhs.precision != Precision::FP32) { + throw std::runtime_error( + "Binary operations only support FP16/FP32 precision (got " + + std::to_string(static_cast(lhs.precision)) + ")" + ); + } + + if (node.params.broadcast_info.needs_broadcasting) { + std::vector lhs_strides = compute_strides(lhs.shape, node.params.broadcast_info.output_shape); + std::vector rhs_strides = compute_strides(rhs.shape, node.params.broadcast_info.output_shape); + + if (lhs.precision == Precision::FP16) { + if (output_precision == Precision::FP16) { + switch (node.op_type) { + case OpType::ADD: + case OpType::ADD_CLIPPED: + cactus_add_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + lhs_strides.data(), rhs_strides.data(), + node.params.broadcast_info.output_shape.data(), + node.params.broadcast_info.output_shape.size()); + break; + case OpType::SUBTRACT: + cactus_subtract_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + lhs_strides.data(), rhs_strides.data(), + node.params.broadcast_info.output_shape.data(), + node.params.broadcast_info.output_shape.size()); + break; + case OpType::MULTIPLY: + cactus_multiply_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + lhs_strides.data(), rhs_strides.data(), + node.params.broadcast_info.output_shape.data(), + node.params.broadcast_info.output_shape.size()); + break; + case OpType::DIVIDE: + cactus_divide_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + lhs_strides.data(), rhs_strides.data(), + node.params.broadcast_info.output_shape.data(), + node.params.broadcast_info.output_shape.size()); + break; + case OpType::NOT_EQUAL: + cactus_not_equal_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + lhs_strides.data(), rhs_strides.data(), + node.params.broadcast_info.output_shape.data(), + node.params.broadcast_info.output_shape.size()); + break; + default: + break; + } + } else if (output_precision == Precision::FP32) { + dispatch_binary_op_broadcast_f16_to_fp32( + node.op_type, + lhs.data_as<__fp16>(), + rhs.data_as<__fp16>(), + node.output_buffer.data_as(), + lhs_strides, + rhs_strides, + node.params.broadcast_info.output_shape + ); + } else { + throw std::runtime_error("Binary broadcast FP16 inputs only support FP16/FP32 outputs"); + } + } else { + if (output_precision == Precision::FP32) { + dispatch_binary_op_broadcast_fp32( + node.op_type, + lhs.data_as(), + rhs.data_as(), + node.output_buffer.data_as(), + lhs_strides, + rhs_strides, + node.params.broadcast_info.output_shape + ); + } else if (output_precision == Precision::FP16) { + dispatch_binary_op_broadcast_fp32_to_fp16( + node.op_type, + lhs.data_as(), + rhs.data_as(), + node.output_buffer.data_as<__fp16>(), + lhs_strides, + rhs_strides, + node.params.broadcast_info.output_shape + ); + } else { + throw std::runtime_error("Binary broadcast FP32 inputs only support FP16/FP32 outputs"); + } + } + } else { + if (lhs.precision == Precision::FP16) { + if (output_precision == Precision::FP16) { + dispatch_binary_op_f16(node.op_type, lhs.data_as<__fp16>(), + rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + } else if (output_precision == Precision::FP32) { + dispatch_binary_op_f16_to_fp32( + node.op_type, + lhs.data_as<__fp16>(), + rhs.data_as<__fp16>(), + node.output_buffer.data_as(), + node.output_buffer.total_size + ); + } else { + throw std::runtime_error("Binary FP16 inputs only support FP16/FP32 outputs"); + } + } else { + if (output_precision == Precision::FP32) { + dispatch_binary_op_fp32(node.op_type, lhs.data_as(), + rhs.data_as(), node.output_buffer.data_as(), + node.output_buffer.total_size); + } else if (output_precision == Precision::FP16) { + dispatch_binary_op_fp32_to_fp16( + node.op_type, + lhs.data_as(), + rhs.data_as(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size + ); + } else { + throw std::runtime_error("Binary FP32 inputs only support FP16/FP32 outputs"); + } + } + } +} + +void compute_unary_op_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input = get_input(node, 0, nodes, node_index_map); + const Precision output_precision = node.output_buffer.precision; + + if (input.precision != Precision::FP16 && input.precision != Precision::FP32) { + throw std::runtime_error("Scalar operations only support FP16/FP32 precision"); + } + + if (input.precision == Precision::FP16) { + if (output_precision == Precision::FP16) { + dispatch_unary_op_f16(node.op_type, input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size, node.params.scalar); + } else if (output_precision == Precision::FP32) { + dispatch_unary_op_f16_to_fp32( + node.op_type, + input.data_as<__fp16>(), + node.output_buffer.data_as(), + node.output_buffer.total_size, + node.params.scalar + ); + } else { + throw std::runtime_error("Scalar FP16 inputs only support FP16/FP32 outputs"); + } + } else { + if (output_precision == Precision::FP32) { + dispatch_unary_op_fp32(node.op_type, input.data_as(), + node.output_buffer.data_as(), + node.output_buffer.total_size, node.params.scalar); + } else if (output_precision == Precision::FP16) { + dispatch_unary_op_fp32_to_fp16( + node.op_type, + input.data_as(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size, + node.params.scalar + ); + } else { + throw std::runtime_error("Scalar FP32 inputs only support FP16/FP32 outputs"); + } + } +} + +void compute_activation_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input = get_input(node, 0, nodes, node_index_map); + + if (input.precision != Precision::FP16) { + throw std::runtime_error("Activation operations only support FP16 precision"); + } + + switch (node.op_type) { + case OpType::RELU: + cactus_relu_f16(input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + break; + case OpType::SILU: + cactus_silu_f16(input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + break; + case OpType::GELU: + cactus_gelu_f16(input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + break; + case OpType::GELU_ERF: + cactus_gelu_f16_erf(input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + break; + case OpType::SIGMOID: + cactus_sigmoid_f16(input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + break; + case OpType::TANH: + cactus_tanh_f16(input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size); + break; + case OpType::LEAKY_RELU: + cactus_leaky_relu_f16(input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + node.output_buffer.total_size, node.params.scalar); + break; + default: + break; + } +} + +void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + int axis = node.params.axis; + + if (input_buffer.precision != Precision::FP16 && input_buffer.precision != Precision::FP32) { + throw std::runtime_error("Reduction operations only support FP16 and FP32 precision"); + } + + if (axis == -1) { + if (input_buffer.precision == Precision::FP32) { + switch (node.op_type) { + case OpType::SUM: { + double result = cactus_sum_all_f32(input_buffer.data_as(), input_buffer.total_size); + node.output_buffer.data_as()[0] = static_cast(result); + break; + } + case OpType::MEAN: { + double result = cactus_mean_all_f32(input_buffer.data_as(), input_buffer.total_size); + node.output_buffer.data_as()[0] = static_cast(result); + break; + } + case OpType::VARIANCE: { + double result = cactus_variance_all_f32(input_buffer.data_as(), input_buffer.total_size); + node.output_buffer.data_as()[0] = static_cast(result); + break; + } + case OpType::MIN: + node.output_buffer.data_as()[0] = cactus_min_all_f32(input_buffer.data_as(), input_buffer.total_size); + break; + case OpType::MAX: + node.output_buffer.data_as()[0] = cactus_max_all_f32(input_buffer.data_as(), input_buffer.total_size); + break; + default: + break; + } + } else { + switch (node.op_type) { + case OpType::SUM: { + double result = cactus_sum_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); + node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); + break; + } + case OpType::MEAN: { + double result = cactus_mean_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); + node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); + break; + } + case OpType::VARIANCE: { + double result = cactus_variance_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); + node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); + break; + } + case OpType::MIN: { + __fp16 result = cactus_min_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); + node.output_buffer.data_as<__fp16>()[0] = result; + break; + } + case OpType::MAX: { + __fp16 result = cactus_max_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); + node.output_buffer.data_as<__fp16>()[0] = result; + break; + } + default: + break; + } + } + } else { + auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); + + if (input_buffer.precision == Precision::FP32) { + switch (node.op_type) { + case OpType::SUM: + cactus_sum_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), + dims.outer, dims.axis_size, dims.inner); + break; + case OpType::MEAN: + cactus_mean_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), + dims.outer, dims.axis_size, dims.inner); + break; + case OpType::VARIANCE: + cactus_variance_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), + dims.outer, dims.axis_size, dims.inner); + break; + case OpType::MIN: + cactus_min_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), + dims.outer, dims.axis_size, dims.inner); + break; + case OpType::MAX: + cactus_max_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), + dims.outer, dims.axis_size, dims.inner); + break; + default: + break; + } + } else { + switch (node.op_type) { + case OpType::SUM: + cactus_sum_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + dims.outer, dims.axis_size, dims.inner); + break; + case OpType::MEAN: + cactus_mean_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + dims.outer, dims.axis_size, dims.inner); + break; + case OpType::VARIANCE: + cactus_variance_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + dims.outer, dims.axis_size, dims.inner); + break; + case OpType::MIN: + cactus_min_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + dims.outer, dims.axis_size, dims.inner); + break; + case OpType::MAX: + cactus_max_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + dims.outer, dims.axis_size, dims.inner); + break; + default: + break; + } + } + } +} + +void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + + size_t input_total_elements = input_buffer.total_size; + size_t output_total_elements = node.output_buffer.total_size; + + if (input_total_elements != output_total_elements) { + throw std::runtime_error("Reshape operation: input elements (" + std::to_string(input_total_elements) + + ") must match output elements (" + std::to_string(output_total_elements) + ")"); + } + + std::memcpy(node.output_buffer.get_data(), input_buffer.get_data(), input_buffer.byte_size); +} + +template +static void dequant_grouped_int8(const int8_t* src, OutT* dst, const __fp16* scales, + const std::vector& shape, size_t group_size) { + if (shape.size() == 2) { + size_t N = shape[0], K = shape[1]; + size_t num_groups = K / group_size; + for (size_t row = 0; row < N; ++row) { + for (size_t col = 0; col < K; ++col) { + size_t idx = row * K + col; + float scale = static_cast(scales[row * num_groups + col / group_size]); + dst[idx] = static_cast(static_cast(src[idx]) * scale); + } + } + } else if (shape.size() == 1) { + size_t K = shape[0]; + for (size_t col = 0; col < K; ++col) { + float scale = static_cast(scales[col / group_size]); + dst[col] = static_cast(static_cast(src[col]) * scale); + } + } +} + +void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buf = get_input(node, 0, nodes, node_index_map); + + if (input_buf.precision == node.output_buffer.precision) { + std::memcpy(node.output_buffer.get_data(), input_buf.get_data(), input_buf.byte_size); + return; + } + + size_t count = input_buf.total_size; + + if (input_buf.precision == Precision::INT8 && node.output_buffer.precision == Precision::FP32) { + if (input_buf.is_grouped_int8()) { + dequant_grouped_int8(input_buf.data_as(), node.output_buffer.data_as(), + input_buf.scales_as_fp16(), input_buf.shape, input_buf.group_size); + } else { + Quantization::int8_to_fp32(input_buf.data_as(), node.output_buffer.data_as(), count, 1.0f); + } + } else if (input_buf.precision == Precision::FP32 && node.output_buffer.precision == Precision::INT8) { + Quantization::fp32_to_int8(input_buf.data_as(), node.output_buffer.data_as(), count, 1.0f); + } else if (input_buf.precision == Precision::FP16 && node.output_buffer.precision == Precision::FP32) { + Quantization::fp16_to_fp32(input_buf.data_as<__fp16>(), node.output_buffer.data_as(), count); + } else if (input_buf.precision == Precision::FP32 && node.output_buffer.precision == Precision::FP16) { + Quantization::fp32_to_fp16(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), count); + } else if (input_buf.precision == Precision::INT8 && node.output_buffer.precision == Precision::FP16) { + if (input_buf.is_grouped_int8()) { + dequant_grouped_int8<__fp16>(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), + input_buf.scales_as_fp16(), input_buf.shape, input_buf.group_size); + } else { + Quantization::int8_to_fp16(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), count, 1.0f); + } + } else if (input_buf.precision == Precision::FP16 && node.output_buffer.precision == Precision::INT8) { + Quantization::fp16_to_int8(input_buf.data_as<__fp16>(), node.output_buffer.data_as(), count, 1.0f); + } else { + throw std::runtime_error("Unsupported precision conversion from " + + std::to_string(static_cast(input_buf.precision)) + + " to " + std::to_string(static_cast(node.output_buffer.precision))); + } +} diff --git a/cactus-transpile/graph/graph_ops_nn.cpp b/cactus-transpile/graph/graph_ops_nn.cpp new file mode 100644 index 000000000..21799bc52 --- /dev/null +++ b/cactus-transpile/graph/graph_ops_nn.cpp @@ -0,0 +1,2754 @@ +#include "graph.h" +#include "../kernel/kernel.h" +#include "../kernel/kernel_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + thread_local std::vector<__fp16> transpose_buffer_fp16; + thread_local std::vector transpose_buffer_fp32; + thread_local std::vector quant_activation_buffer; + thread_local std::vector quant_scales_buffer; + + thread_local const __fp16* cached_quant_src = nullptr; + thread_local size_t cached_quant_M = 0; + thread_local size_t cached_quant_K = 0; + + void ensure_transpose_buffer_fp16(size_t required_size) { + if (transpose_buffer_fp16.size() < required_size) { + transpose_buffer_fp16.resize(required_size); + } + } + + void ensure_transpose_buffer_fp32(size_t required_size) { + if (transpose_buffer_fp32.size() < required_size) { + transpose_buffer_fp32.resize(required_size); + } + } + + void ensure_quant_buffers(size_t M, size_t K) { + size_t required_data = M * K; + if (quant_activation_buffer.size() < required_data) { + quant_activation_buffer.resize(required_data); + } + if (quant_scales_buffer.size() < M) { + quant_scales_buffer.resize(M); + } + } + + void matmul_fp16_inputs_to_fp32( + const __fp16* lhs, + const __fp16* rhs_transposed, + float* output, + size_t M, + size_t K, + size_t N + ) { + CactusThreading::parallel_for( + M, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_row, size_t end_row) { + for (size_t m = start_row; m < end_row; ++m) { + const __fp16* lhs_row = lhs + (m * K); + float* out_row = output + (m * N); + for (size_t n = 0; n < N; ++n) { + const __fp16* rhs_row = rhs_transposed + (n * K); + float acc = 0.0f; + for (size_t k = 0; k < K; ++k) { + acc += static_cast(lhs_row[k]) * static_cast(rhs_row[k]); + } + out_row[n] = acc; + } + } + } + ); + } + + void matmul_fp16_inputs_to_fp16( + const __fp16* lhs, + const __fp16* rhs_transposed, + __fp16* output, + size_t M, + size_t K, + size_t N + ) { + CactusThreading::parallel_for( + M, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_row, size_t end_row) { + for (size_t m = start_row; m < end_row; ++m) { + const __fp16* lhs_row = lhs + (m * K); + __fp16* out_row = output + (m * N); + for (size_t n = 0; n < N; ++n) { + const __fp16* rhs_row = rhs_transposed + (n * K); + float acc = 0.0f; + for (size_t k = 0; k < K; ++k) { + acc += static_cast(lhs_row[k]) * static_cast(rhs_row[k]); + } + out_row[n] = static_cast<__fp16>(acc); + } + } + } + ); + } + + void matmul_fp32_inputs_to_fp32( + const float* lhs, + const float* rhs_transposed, + float* output, + size_t M, + size_t K, + size_t N + ) { + CactusThreading::parallel_for( + M, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_row, size_t end_row) { + for (size_t m = start_row; m < end_row; ++m) { + const float* lhs_row = lhs + (m * K); + float* out_row = output + (m * N); + for (size_t n = 0; n < N; ++n) { + const float* rhs_row = rhs_transposed + (n * K); + float acc = 0.0f; + for (size_t k = 0; k < K; ++k) { + acc += lhs_row[k] * rhs_row[k]; + } + out_row[n] = acc; + } + } + } + ); + } + + void matmul_fp32_inputs_to_fp16( + const float* lhs, + const float* rhs_transposed, + __fp16* output, + size_t M, + size_t K, + size_t N + ) { + CactusThreading::parallel_for( + M, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_row, size_t end_row) { + for (size_t m = start_row; m < end_row; ++m) { + const float* lhs_row = lhs + (m * K); + __fp16* out_row = output + (m * N); + for (size_t n = 0; n < N; ++n) { + const float* rhs_row = rhs_transposed + (n * K); + float acc = 0.0f; + for (size_t k = 0; k < K; ++k) { + acc += lhs_row[k] * rhs_row[k]; + } + out_row[n] = static_cast<__fp16>(acc); + } + } + } + ); + } + + void quantize_activations_fp16_to_int8(const __fp16* src, int8_t* dst, float* scales, size_t M, size_t K) { + if (src == cached_quant_src && M == cached_quant_M && K == cached_quant_K) { + return; + } + + constexpr size_t PARALLEL_THRESHOLD = 16; + + if (M >= PARALLEL_THRESHOLD) { + CactusThreading::parallel_for(M, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst, scales, K](size_t m_start, size_t m_end) { + for (size_t m = m_start; m < m_end; m++) { + float max_abs = cactus_fp16_max_abs(src + m * K, K); + float scale = max_abs / 127.0f; + if (scale < 1e-10f) scale = 1e-10f; + scales[m] = scale; + cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); + } + }); + } else { + for (size_t m = 0; m < M; m++) { + float max_abs = cactus_fp16_max_abs(src + m * K, K); + float scale = max_abs / 127.0f; + if (scale < 1e-10f) scale = 1e-10f; + scales[m] = scale; + cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); + } + } + + cached_quant_src = src; + cached_quant_M = M; + cached_quant_K = K; + } + + const __fp16* as_fp16_ptr(const BufferDesc& buffer, std::vector<__fp16>& scratch) { + if (buffer.precision == Precision::FP16) { + return buffer.data_as<__fp16>(); + } + if (buffer.precision == Precision::FP32) { + scratch.resize(buffer.total_size); + cactus_fp32_to_fp16(buffer.data_as(), scratch.data(), buffer.total_size); + return scratch.data(); + } + throw std::runtime_error("GATED_DELTANET unsupported precision (expected FP16/FP32)"); + } + + const float* as_fp32_ptr(const BufferDesc& buffer, std::vector& scratch) { + if (buffer.precision == Precision::FP32) { + return buffer.data_as(); + } + if (buffer.precision == Precision::FP16) { + scratch.resize(buffer.total_size); + cactus_fp16_to_fp32(buffer.data_as<__fp16>(), scratch.data(), buffer.total_size); + return scratch.data(); + } + throw std::runtime_error("matmul unsupported precision (expected FP16/FP32)"); + } + + void transpose_2d_fp32(const float* source, float* destination, size_t num_rows, size_t num_cols) { + CactusThreading::parallel_for( + num_rows, + CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_row, size_t end_row) { + for (size_t row = start_row; row < end_row; ++row) { + for (size_t col = 0; col < num_cols; ++col) { + destination[col * num_rows + row] = source[row * num_cols + col]; + } + } + } + ); + } + + void validate_gated_deltanet_inputs( + const BufferDesc& q, + const BufferDesc& k, + const BufferDesc& v, + const BufferDesc& g, + const BufferDesc& b, + const BufferDesc& s) { + auto is_supported_precision = [](Precision p) { + return p == Precision::FP16 || p == Precision::FP32; + }; + if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || + !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || + !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { + throw std::runtime_error("GATED_DELTANET requires FP16/FP32 inputs"); + } + + if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { + throw std::runtime_error("GATED_DELTANET expects query/key/value rank 4 [B, T, H, D]"); + } + if (g.shape.size() != 3 || b.shape.size() != 3) { + throw std::runtime_error("GATED_DELTANET expects gate_log/beta rank 3 [B, T, H]"); + } + if (s.shape.size() != 4) { + throw std::runtime_error("GATED_DELTANET expects state rank 4 [B, K, H, V]"); + } + + const size_t B = q.shape[0]; + const size_t T = q.shape[1]; + const size_t Hq = q.shape[2]; + const size_t K = q.shape[3]; + + if (k.shape[0] != B || k.shape[1] != T || k.shape[2] != Hq || k.shape[3] != K) { + throw std::runtime_error("GATED_DELTANET query/key shape mismatch"); + } + if (v.shape[0] != B || v.shape[1] != T) { + throw std::runtime_error("GATED_DELTANET value shape mismatch"); + } + const size_t Hv = v.shape[2]; + if (g.shape[0] != B || g.shape[1] != T || g.shape[2] != Hv || + b.shape[0] != B || b.shape[1] != T || b.shape[2] != Hv) { + throw std::runtime_error("GATED_DELTANET gate_log/beta shape mismatch"); + } + if (Hq == 0 || Hv == 0 || (Hv % Hq) != 0) { + throw std::runtime_error("GATED_DELTANET expects value heads divisible by q/k heads"); + } + const size_t V = v.shape[3]; + if (s.shape[0] != B || s.shape[1] != K || s.shape[2] != Hv || s.shape[3] != V) { + throw std::runtime_error("GATED_DELTANET state shape mismatch"); + } + } + + +} + +void shrink_thread_local_buffers() { + std::vector<__fp16>().swap(transpose_buffer_fp16); + std::vector().swap(quant_activation_buffer); + std::vector().swap(quant_scales_buffer); + cached_quant_src = nullptr; + cached_quant_M = 0; + cached_quant_K = 0; +} + +void compute_quantize_activations_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& shape = input_buffer.shape; + + if (input_buffer.precision != Precision::FP16) { + throw std::runtime_error("QUANTIZE_ACTIVATIONS requires FP16 input"); + } + + if (shape.size() < 2) { + throw std::runtime_error("QUANTIZE_ACTIVATIONS requires at least 2D tensor"); + } + + size_t K = shape.back(); + size_t M = 1; + for (size_t i = 0; i < shape.size() - 1; ++i) { + M *= shape[i]; + } + + if (!node.output_buffer.has_activation_scales() || + node.output_buffer.num_rows_for_activation_scales != M) { + node.output_buffer.allocate_activation_scales(M); + } + + const __fp16* src = input_buffer.data_as<__fp16>(); + int8_t* dst = node.output_buffer.data_as(); + float* scales = node.output_buffer.activation_scales_as_float(); + + constexpr size_t PARALLEL_THRESHOLD = 16; + + if (M >= PARALLEL_THRESHOLD) { + CactusThreading::parallel_for(M, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst, scales, K](size_t m_start, size_t m_end) { + for (size_t m = m_start; m < m_end; m++) { + float max_abs = cactus_fp16_max_abs(src + m * K, K); + float scale = max_abs / 127.0f; + if (scale < 1e-10f) scale = 1e-10f; + scales[m] = scale; + cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); + } + }); + } else { + for (size_t m = 0; m < M; m++) { + float max_abs = cactus_fp16_max_abs(src + m * K, K); + float scale = max_abs / 127.0f; + if (scale < 1e-10f) scale = 1e-10f; + scales[m] = scale; + cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); + } + } +} + +void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& lhs_buffer = get_input(node, 0, nodes, node_index_map); + const auto& rhs_buffer = get_input(node, 1, nodes, node_index_map); + const auto& lhs_shape = lhs_buffer.shape; + const auto& rhs_shape = rhs_buffer.shape; + + size_t M = lhs_shape[lhs_shape.size() - 2]; + size_t K = lhs_shape[lhs_shape.size() - 1]; + size_t N; + if (rhs_buffer.is_interleaved && rhs_buffer.original_N > 0) { + N = rhs_buffer.original_N; + } else { + N = node.params.pretransposed_rhs ? + rhs_shape[rhs_shape.size() - 2] : rhs_shape[rhs_shape.size() - 1]; + } + + bool pretransposed_rhs = node.params.pretransposed_rhs; + + ComputeBackend backend = node.params.backend; + + if (backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU matrix multiplication not yet implemented"); + } + + const bool lhs_is_prequantized_int8 = (lhs_buffer.precision == Precision::INT8 && + lhs_buffer.has_activation_scales()); + + if (PrecisionTraits::is_integer(rhs_buffer.precision) && rhs_buffer.group_size > 0) { + const int8_t* rhs = rhs_buffer.data_as(); + const __fp16* rhs_scales = rhs_buffer.scales_as_fp16(); + if (node.output_buffer.precision != Precision::FP16) { + throw std::runtime_error("Quantized matmul currently only supports FP16 outputs"); + } + __fp16* output = node.output_buffer.data_as<__fp16>(); + + if (!pretransposed_rhs) { + throw std::runtime_error("Group-wise quantized matmul requires pretransposed weights"); + } + + const int8_t* lhs_int8; + const float* lhs_scales; + + if (lhs_is_prequantized_int8) { + lhs_int8 = lhs_buffer.data_as(); + lhs_scales = lhs_buffer.activation_scales_as_float(); + } else if (lhs_buffer.precision == Precision::FP16) { + ensure_quant_buffers(M, K); + cached_quant_src = nullptr; + quantize_activations_fp16_to_int8(lhs_buffer.data_as<__fp16>(), quant_activation_buffer.data(), + quant_scales_buffer.data(), M, K); + lhs_int8 = quant_activation_buffer.data(); + lhs_scales = quant_scales_buffer.data(); + } else { + throw std::runtime_error("Quantized matmul requires INT8 (pre-quantized) or FP16 activations"); + } + + cactus_matmul_integer(rhs_buffer.precision, + lhs_int8, lhs_scales, + rhs, rhs_scales, output, + M, K, N, rhs_buffer.group_size); + } else { + const bool fp16_fast_path = + lhs_buffer.precision == Precision::FP16 && rhs_buffer.precision == Precision::FP16; + if (fp16_fast_path) { + const __fp16* lhs = lhs_buffer.data_as<__fp16>(); + const __fp16* rhs = rhs_buffer.data_as<__fp16>(); + + if (pretransposed_rhs) { + if (node.output_buffer.precision == Precision::FP16) { + matmul_fp16_inputs_to_fp16(lhs, rhs, node.output_buffer.data_as<__fp16>(), M, K, N); + } else if (node.output_buffer.precision == Precision::FP32) { + matmul_fp16_inputs_to_fp32(lhs, rhs, node.output_buffer.data_as(), M, K, N); + } else { + throw std::runtime_error("FP16 matmul only supports FP16/FP32 outputs"); + } + } else { + size_t transpose_size = rhs_shape[0] * rhs_shape[1]; + ensure_transpose_buffer_fp16(transpose_size); + + cactus_transpose_2d_f16(rhs, transpose_buffer_fp16.data(), + rhs_shape[0], rhs_shape[1], 0, rhs_shape[0]); + if (node.output_buffer.precision == Precision::FP16) { + matmul_fp16_inputs_to_fp16(lhs, transpose_buffer_fp16.data(), node.output_buffer.data_as<__fp16>(), M, K, N); + } else if (node.output_buffer.precision == Precision::FP32) { + matmul_fp16_inputs_to_fp32(lhs, transpose_buffer_fp16.data(), node.output_buffer.data_as(), M, K, N); + } else { + throw std::runtime_error("FP16 matmul only supports FP16/FP32 outputs"); + } + } + } else { + std::vector lhs_fp32_scratch; + std::vector rhs_fp32_scratch; + const float* lhs = as_fp32_ptr(lhs_buffer, lhs_fp32_scratch); + const float* rhs = as_fp32_ptr(rhs_buffer, rhs_fp32_scratch); + const float* rhs_transposed = rhs; + + if (!pretransposed_rhs) { + size_t transpose_size = rhs_shape[0] * rhs_shape[1]; + ensure_transpose_buffer_fp32(transpose_size); + transpose_2d_fp32(rhs, transpose_buffer_fp32.data(), rhs_shape[0], rhs_shape[1]); + rhs_transposed = transpose_buffer_fp32.data(); + } + + if (node.output_buffer.precision == Precision::FP32) { + matmul_fp32_inputs_to_fp32(lhs, rhs_transposed, node.output_buffer.data_as(), M, K, N); + } else if (node.output_buffer.precision == Precision::FP16) { + matmul_fp32_inputs_to_fp16(lhs, rhs_transposed, node.output_buffer.data_as<__fp16>(), M, K, N); + } else { + throw std::runtime_error("FP32 matmul only supports FP16/FP32 outputs"); + } + } + } +} + +namespace { + thread_local std::vector<__fp16> moe_compact_hidden_buf; + thread_local std::vector<__fp16> moe_gate_buf; + thread_local std::vector<__fp16> moe_up_buf; + thread_local std::vector<__fp16> moe_expert_out_buf; + thread_local std::vector moe_lhs_q_buf; + thread_local std::vector moe_lhs_scales_buf; + thread_local std::vector moe_expert_offsets_buf; + thread_local std::vector moe_expert_tokens_buf; + thread_local std::vector moe_routing_denom_buf; + + void ensure_moe_buffers(size_t max_tokens, size_t hidden_dim, size_t intermediate_dim, + size_t num_experts, size_t top_k) { + size_t hidden_size = max_tokens * hidden_dim; + size_t inter_size = max_tokens * intermediate_dim; + if (moe_compact_hidden_buf.size() < hidden_size) moe_compact_hidden_buf.resize(hidden_size); + if (moe_gate_buf.size() < inter_size) moe_gate_buf.resize(inter_size); + if (moe_up_buf.size() < inter_size) moe_up_buf.resize(inter_size); + if (moe_expert_out_buf.size() < hidden_size) moe_expert_out_buf.resize(hidden_size); + size_t max_k = std::max(hidden_dim, intermediate_dim); + size_t quant_size = max_tokens * max_k; + if (moe_lhs_q_buf.size() < quant_size) moe_lhs_q_buf.resize(quant_size); + if (moe_lhs_scales_buf.size() < max_tokens) moe_lhs_scales_buf.resize(max_tokens); + size_t total_assignments = max_tokens * top_k; + if (moe_expert_offsets_buf.size() < num_experts + 1) moe_expert_offsets_buf.resize(num_experts + 1); + if (moe_expert_tokens_buf.size() < total_assignments) moe_expert_tokens_buf.resize(total_assignments); + if (moe_routing_denom_buf.size() < max_tokens) moe_routing_denom_buf.resize(max_tokens); + } + + void moe_matmul(const __fp16* lhs, + size_t M, + size_t K, + const BufferDesc& rhs_buffer, + __fp16* output, + size_t N, + bool lhs_prequantized = false) { + if (rhs_buffer.precision == Precision::FP16) { + cactus_matmul_f16(lhs, rhs_buffer.data_as<__fp16>(), output, M, K, N); + return; + } + + if (PrecisionTraits::is_integer(rhs_buffer.precision) && rhs_buffer.group_size > 0) { + int8_t* lhs_q = moe_lhs_q_buf.data(); + float* lhs_scales = moe_lhs_scales_buf.data(); + if (!lhs_prequantized) { + for (size_t row = 0; row < M; ++row) { + float scale = cactus_fp16_max_abs(lhs + row * K, K) / 127.0f; + if (scale < 1e-10f) scale = 1e-10f; + lhs_scales[row] = scale; + cactus_fp16_to_int8(lhs + row * K, lhs_q + row * K, K, scale); + } + } + cactus_matmul_integer(rhs_buffer.precision, + lhs_q, lhs_scales, + rhs_buffer.data_as(), + rhs_buffer.scales_as_fp16(), + output, M, K, N, rhs_buffer.group_size); + return; + } + + throw std::runtime_error("moe_layer only supports FP16 or grouped INT4/INT8 expert weights"); + } +} + +void compute_moe_layer_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const size_t num_experts = node.params.num_experts; + const size_t top_k = node.params.num_experts_per_tok; + const bool normalize_routing = node.params.normalize_routing; + const float eps = node.params.epsilon; + const float routed_scaling_factor = node.params.scalar; + const bool gated = node.params.moe_gated; + const Activation activation = node.params.activation; + const size_t base_inputs = gated ? (3 + 3 * num_experts) : (3 + 2 * num_experts); + bool has_per_expert_scale = node.input_ids.size() == base_inputs + 1; + if (node.input_ids.size() != base_inputs && node.input_ids.size() != base_inputs + 1) { + throw std::runtime_error("moe_layer expects " + std::to_string(base_inputs) + " or " + std::to_string(base_inputs + 1) + " inputs, got " + std::to_string(node.input_ids.size())); + } + + const auto& hidden_buffer = get_input(node, 0, nodes, node_index_map); + const auto& routing_buffer = get_input(node, 1, nodes, node_index_map); + const auto& topk_idx_buffer = get_input(node, 2, nodes, node_index_map); + + if (hidden_buffer.precision != Precision::FP16 || node.output_buffer.precision != Precision::FP16) { + throw std::runtime_error("moe_layer expects FP16 hidden/output"); + } + if (topk_idx_buffer.precision != Precision::FP32) { + throw std::runtime_error("moe_layer expects FP32 topk indices"); + } + + const __fp16* expert_scales_fp16 = nullptr; + if (has_per_expert_scale) { + const auto& scale_buffer = get_input(node, base_inputs, nodes, node_index_map); + if (scale_buffer.precision != Precision::FP16) { + throw std::runtime_error("moe_layer expects FP16 per_expert_scale"); + } + expert_scales_fp16 = scale_buffer.data_as<__fp16>(); + } + + const size_t token_count = hidden_buffer.shape[0]; + const size_t hidden_dim = hidden_buffer.shape[1]; + const size_t total_num_experts = routing_buffer.shape[1]; + + const auto& w1_0_buffer = get_input(node, 3, nodes, node_index_map); + const size_t expert_intermediate_dim = w1_0_buffer.shape[0]; + + const auto* hidden = hidden_buffer.data_as<__fp16>(); + auto* output = node.output_buffer.data_as<__fp16>(); + const auto* topk_idx = topk_idx_buffer.data_as(); + const auto* routing_fp16 = routing_buffer.precision == Precision::FP16 ? routing_buffer.data_as<__fp16>() : nullptr; + const auto* routing_fp32 = routing_buffer.precision == Precision::FP32 ? routing_buffer.data_as() : nullptr; + + auto routing_prob = [&](size_t tok, size_t exp) -> float { + const size_t offset = tok * total_num_experts + exp; + if (routing_fp16) return static_cast(routing_fp16[offset]); + return routing_fp32[offset]; + }; + + ensure_moe_buffers(token_count, hidden_dim, expert_intermediate_dim, num_experts, top_k); + + size_t* expert_offsets = moe_expert_offsets_buf.data(); + size_t* expert_tokens_flat = moe_expert_tokens_buf.data(); + + std::memset(expert_offsets, 0, (num_experts + 1) * sizeof(size_t)); + for (size_t tok = 0; tok < token_count; ++tok) { + for (size_t k = 0; k < top_k; ++k) { + float raw_idx = topk_idx[tok * top_k + k]; + if (!std::isfinite(raw_idx)) { + throw std::runtime_error("moe_layer got non-finite expert index"); + } + size_t idx = static_cast(raw_idx + 0.5f); + if (idx >= num_experts) { + throw std::runtime_error("moe_layer got expert index out of range"); + } + expert_offsets[idx + 1]++; + } + } + + for (size_t e = 0; e < num_experts; ++e) { + expert_offsets[e + 1] += expert_offsets[e]; + } + + thread_local std::vector moe_write_cursors; + if (moe_write_cursors.size() < num_experts) moe_write_cursors.resize(num_experts); + std::memcpy(moe_write_cursors.data(), expert_offsets, num_experts * sizeof(size_t)); + + for (size_t tok = 0; tok < token_count; ++tok) { + for (size_t k = 0; k < top_k; ++k) { + size_t idx = static_cast(topk_idx[tok * top_k + k] + 0.5f); + expert_tokens_flat[moe_write_cursors[idx]++] = tok; + } + } + + float* routing_denom = moe_routing_denom_buf.data(); + if (normalize_routing) { + for (size_t tok = 0; tok < token_count; ++tok) { + float sum_probs = 0.0f; + for (size_t k = 0; k < top_k; ++k) { + size_t idx = static_cast(topk_idx[tok * top_k + k] + 0.5f); + sum_probs += routing_prob(tok, idx); + } + routing_denom[tok] = sum_probs + eps; + } + } + + std::memset(output, 0, token_count * hidden_dim * sizeof(__fp16)); + + for (size_t expert_idx = 0; expert_idx < num_experts; ++expert_idx) { + const size_t start = expert_offsets[expert_idx]; + const size_t end = expert_offsets[expert_idx + 1]; + if (start == end) continue; + + const size_t selected_count = end - start; + const size_t* selected_tokens = expert_tokens_flat + start; + + const auto& w1_buffer = get_input(node, 3 + expert_idx, nodes, node_index_map); + const auto& w2_buffer = gated + ? get_input(node, 3 + 2 * num_experts + expert_idx, nodes, node_index_map) + : get_input(node, 3 + num_experts + expert_idx, nodes, node_index_map); + + __fp16* compact_hidden = moe_compact_hidden_buf.data(); + for (size_t i = 0; i < selected_count; ++i) { + std::memcpy(compact_hidden + i * hidden_dim, + hidden + selected_tokens[i] * hidden_dim, + hidden_dim * sizeof(__fp16)); + } + + __fp16* gate = moe_gate_buf.data(); + __fp16* up = moe_up_buf.data(); + __fp16* expert_out = moe_expert_out_buf.data(); + + moe_matmul(compact_hidden, selected_count, hidden_dim, w1_buffer, gate, expert_intermediate_dim); + const bool w1_was_int8 = w1_buffer.is_grouped_int8(); + + switch (activation) { + case Activation::GELU: + cactus_gelu_f16(gate, gate, selected_count * expert_intermediate_dim); + break; + case Activation::GELU_ERF: + cactus_gelu_f16_erf(gate, gate, selected_count * expert_intermediate_dim); + break; + case Activation::RELU: + cactus_relu_f16(gate, gate, selected_count * expert_intermediate_dim); + break; + case Activation::SILU: + default: + cactus_silu_f16(gate, gate, selected_count * expert_intermediate_dim); + break; + } + + if (gated) { + const auto& w3_buffer = get_input(node, 3 + num_experts + expert_idx, nodes, node_index_map); + moe_matmul(compact_hidden, selected_count, hidden_dim, w3_buffer, up, expert_intermediate_dim, w1_was_int8); + cactus_multiply_f16(gate, up, gate, selected_count * expert_intermediate_dim); + } + + moe_matmul(gate, selected_count, expert_intermediate_dim, w2_buffer, expert_out, hidden_dim); + + for (size_t i = 0; i < selected_count; ++i) { + const size_t tok = selected_tokens[i]; + float expert_prob = routing_prob(tok, expert_idx); + if (expert_prob <= 0.0f) continue; + + float route_weight = expert_prob; + if (normalize_routing) { + route_weight = expert_prob / routing_denom[tok]; + } + route_weight *= routed_scaling_factor; + if (expert_scales_fp16) { + route_weight *= static_cast(expert_scales_fp16[expert_idx]); + } + + auto* out_row = output + tok * hidden_dim; + const auto* expert_row = expert_out + i * hidden_dim; + cactus_add_scaled_f16(out_row, expert_row, out_row, hidden_dim, route_weight); + } + } +} + +void compute_rms_norm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& weight_buffer = get_input(node, 1, nodes, node_index_map); + + if (input_buffer.shape.size() != 2) { + throw std::runtime_error("RMS normalization requires 2D input tensor [batch_size, dims], got " + + std::to_string(input_buffer.shape.size()) + "D tensor"); + } + + size_t batch_size = input_buffer.shape[0]; + size_t dims = input_buffer.shape[1]; + + if (input_buffer.precision != Precision::FP16) { + throw std::runtime_error("RMS normalization only supports FP16 precision"); + } + + cactus_rms_norm_f16(input_buffer.data_as<__fp16>(), weight_buffer.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), batch_size, dims, node.params.epsilon); +} + +void compute_rope_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU RoPE operation not yet implemented"); + } + + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& shape = input_buffer.shape; + + if (shape.size() < 4) { + throw std::runtime_error("RoPE operation requires 4D tensor with shape [batch, seq_len, num_heads, head_dim], got " + + std::to_string(shape.size()) + "D tensor"); + } + + if (input_buffer.precision != Precision::FP16 || node.output_buffer.precision != Precision::FP16) { + throw std::runtime_error("RoPE operation only supports FP16 precision"); + } + + size_t batch_size = shape[0]; + size_t seq_len = shape[1]; + size_t num_heads = shape[2]; + size_t head_dim = shape[3]; + + cactus_rope_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, num_heads, head_dim, node.params.position_offset, node.params.theta); +} + +void compute_softmax_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& shape = input_buffer.shape; + + if (shape.size() < 2) { + throw std::runtime_error("Softmax operation requires at least 2D tensor, got " + + std::to_string(shape.size()) + "D tensor"); + } + + if (input_buffer.precision != Precision::FP16) { + throw std::runtime_error("Softmax operation only supports FP16 precision"); + } + + size_t batch_size = 1; + for (size_t i = 0; i < shape.size() - 1; i++) { + batch_size *= shape[i]; + } + size_t vocab_size = shape[shape.size() - 1]; + + cactus_softmax_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + batch_size, 1, vocab_size); +} + +void compute_rel_pos_bias_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.input_ids.size() != 2) { + throw std::runtime_error("REL_POS_BIAS requires 2 inputs (query, relative_key)"); + } + + const auto& q_buffer = get_input(node, 0, nodes, node_index_map); + const auto& r_buffer = get_input(node, 1, nodes, node_index_map); + auto& y_buffer = node.output_buffer; + + if (q_buffer.shape.size() != 4) { + throw std::runtime_error("REL_POS_BIAS query must be [B, T, H, D]"); + } + if (r_buffer.shape.size() != 4) { + throw std::runtime_error("REL_POS_BIAS relative_key must be [B, R, H, D]"); + } + if (q_buffer.precision != Precision::FP16 || r_buffer.precision != Precision::FP16) { + throw std::runtime_error("REL_POS_BIAS currently only supports FP16 tensors"); + } + + const size_t B = q_buffer.shape[0]; + const size_t T = q_buffer.shape[1]; + const size_t H = q_buffer.shape[2]; + const size_t D = q_buffer.shape[3]; + const size_t Rb = r_buffer.shape[0]; + const size_t R = r_buffer.shape[1]; + + if (Rb != 1 && Rb != B) { + throw std::runtime_error("REL_POS_BIAS relative_key batch must be 1 or match query batch"); + } + if (r_buffer.shape[2] != H || r_buffer.shape[3] != D) { + throw std::runtime_error("REL_POS_BIAS expects matching [H, D] between query and relative_key"); + } + if (R < (2 * T - 1)) { + throw std::runtime_error("REL_POS_BIAS requires relative_key length >= 2*T-1"); + } + + const __fp16* q = q_buffer.data_as<__fp16>(); + const __fp16* r = r_buffer.data_as<__fp16>(); + __fp16* y = y_buffer.data_as<__fp16>(); + + const float scale = node.params.scale; + + const size_t q_batch_stride = T * H * D; + const size_t r_batch_stride = R * H * D; + const size_t y_batch_stride = H * T * T; + const size_t q_head_stride = D; + const size_t r_head_stride = D; + const size_t q_time_stride = H * D; + const size_t r_time_stride = H * D; + + CactusThreading::parallel_for(B * H * T, CactusThreading::Thresholds::ATTENTION, + [&](size_t start_idx, size_t end_idx) { + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t b = work_idx / (H * T); + const size_t rem = work_idx % (H * T); + const size_t h = rem / T; + const size_t t = rem % T; + + const size_t rb = (Rb == 1) ? 0 : b; + const __fp16* q_vec = q + b * q_batch_stride + t * q_time_stride + h * q_head_stride; + const __fp16* r_base = r + rb * r_batch_stride + h * r_head_stride; + __fp16* y_row = y + b * y_batch_stride + h * (T * T) + t * T; + + for (size_t j = 0; j < T; ++j) { + const size_t rel_idx = (T - 1) - t + j; + const __fp16* r_vec = r_base + rel_idx * r_time_stride; + + float acc = 0.0f; + for (size_t d = 0; d < D; ++d) { + acc += static_cast(q_vec[d]) * static_cast(r_vec[d]); + } + y_row[j] = static_cast<__fp16>(acc * scale); + } + } + }); +} + +void compute_attention_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU attention operation not yet implemented"); + } + + if (node.input_ids.size() < 3 || node.input_ids.size() > 4) { + throw std::runtime_error("Attention operation requires 3 or 4 inputs (query, key, value[, mask]), got " + + std::to_string(node.input_ids.size()) + " inputs"); + } + + const auto& query_buffer = get_input(node, 0, nodes, node_index_map); + const auto& key_buffer = get_input(node, 1, nodes, node_index_map); + const auto& value_buffer = get_input(node, 2, nodes, node_index_map); + const BufferDesc* mask_buffer = nullptr; + if (node.input_ids.size() == 4) { + mask_buffer = &get_input(node, 3, nodes, node_index_map); + } + const auto& q_shape = query_buffer.shape; + const auto& k_shape = key_buffer.shape; + + if (q_shape.size() < 4) { + throw std::runtime_error("Attention operation requires 4D tensors [batch, seq_len, num_heads, head_dim], got " + + std::to_string(q_shape.size()) + "D tensor"); + } + + if (query_buffer.precision != Precision::FP16) { + throw std::runtime_error("Attention operation only supports FP16 precision"); + } + + size_t batch_size = q_shape[0]; + size_t seq_len = q_shape[1]; + size_t num_q_heads = q_shape[2]; + size_t head_dim = q_shape[3]; + size_t num_kv_heads = k_shape[2]; + size_t kv_seq_len = key_buffer.shape[1]; + size_t v_head_dim = value_buffer.shape[3]; + bool mask_per_head = false; + const __fp16* mask_ptr = nullptr; + + if (mask_buffer) { + if (mask_buffer->precision != Precision::FP16) { + throw std::runtime_error("Attention mask tensor must be FP16"); + } + + if (mask_buffer->shape.size() == 3) { + if (mask_buffer->shape[0] != batch_size || + mask_buffer->shape[1] != seq_len || + mask_buffer->shape[2] != kv_seq_len) { + throw std::runtime_error("Attention mask [B, T, S] shape mismatch"); + } + mask_per_head = false; + } else if (mask_buffer->shape.size() == 4) { + if (mask_buffer->shape[0] != batch_size || + mask_buffer->shape[2] != seq_len || + mask_buffer->shape[3] != kv_seq_len) { + throw std::runtime_error("Attention mask [B, H, T, S] shape mismatch"); + } + if (mask_buffer->shape[1] == 1) { + mask_per_head = false; + } else if (mask_buffer->shape[1] == num_q_heads) { + mask_per_head = true; + } else { + throw std::runtime_error("Attention mask [B, H, T, S] head dimension mismatch"); + } + } else { + throw std::runtime_error("Attention mask must be rank 3 or 4"); + } + + mask_ptr = mask_buffer->data_as<__fp16>(); + } + + cactus_attention_f16(query_buffer.data_as<__fp16>(), key_buffer.data_as<__fp16>(), + value_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, kv_seq_len, num_q_heads, num_kv_heads, head_dim, node.params.scale, mask_ptr, + node.params.position_offset, node.params.window_size, node.params.is_causal, + node.params.attention_mask_is_additive, mask_per_head, v_head_dim, node.params.logit_cap); +} + +void compute_attention_int8_hybrid_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& query_buffer = get_input(node, 0, nodes, node_index_map); + const auto& key_new_buffer = get_input(node, 1, nodes, node_index_map); + const auto& value_new_buffer = get_input(node, 2, nodes, node_index_map); + const auto& q_shape = query_buffer.shape; + + if (q_shape.size() < 4) { + throw std::runtime_error("ATTENTION_INT8_HYBRID requires 4D query tensor"); + } + + size_t batch_size = q_shape[0]; + size_t seq_len = q_shape[1]; + size_t num_q_heads = q_shape[2]; + size_t head_dim = node.params.head_dim; + size_t v_head_dim = node.params.v_head_dim; + size_t num_kv_heads = node.params.num_kv_heads; + size_t cache_len = node.params.cache_seq_len; + size_t new_len = key_new_buffer.shape[1]; + + cactus_attention_hybrid_int8_fp16( + query_buffer.data_as<__fp16>(), + node.params.cached_keys_int8, + node.params.cached_values_int8, + node.params.cached_k_scales, + node.params.cached_v_scales, + key_new_buffer.data_as<__fp16>(), + value_new_buffer.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, cache_len, new_len, + num_q_heads, num_kv_heads, head_dim, + node.params.scale, node.params.position_offset, true, + node.params.window_size, KV_QUANT_GROUP_SIZE, v_head_dim + ); +} + +void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& weight_buffer = get_input(node, 1, nodes, node_index_map); + bool has_bias = node.input_ids.size() > 2; + float epsilon = node.params.epsilon; + + if (input_buffer.shape.empty()) { + throw std::runtime_error("LayerNorm requires non-empty input tensor"); + } + + size_t feature_size = input_buffer.shape.back(); + size_t batch_size = input_buffer.total_size / feature_size; + + if (weight_buffer.total_size != feature_size) { + throw std::runtime_error("LayerNorm weight size mismatch with input feature dimension"); + } + + using BufferDesc = std::remove_reference_t; + const BufferDesc* bias_buffer_ptr = nullptr; + if (has_bias) { + const auto& bias_buffer = get_input(node, 2, nodes, node_index_map); + if (bias_buffer.total_size != feature_size) { + throw std::runtime_error("LayerNorm bias size mismatch with input feature dimension"); + } + bias_buffer_ptr = &bias_buffer; + } + + if (input_buffer.precision == Precision::FP16 && + weight_buffer.precision == Precision::FP16 && + node.output_buffer.precision == Precision::FP16 && + (!has_bias || bias_buffer_ptr->precision == Precision::FP16)) { + cactus_layer_norm_f16( + input_buffer.data_as<__fp16>(), + weight_buffer.data_as<__fp16>(), + has_bias ? bias_buffer_ptr->data_as<__fp16>() : nullptr, + node.output_buffer.data_as<__fp16>(), + batch_size, + feature_size, + epsilon); + return; + } + + std::vector input_float(input_buffer.total_size); + std::vector weight_float(feature_size); + std::vector bias_float(feature_size, 0.0f); + + if (input_buffer.precision == Precision::INT8) { + throw std::runtime_error("LayerNorm currently does not support INT8 input"); + } else if (input_buffer.precision == Precision::FP16) { + const __fp16* input_fp16 = input_buffer.data_as<__fp16>(); + for (size_t i = 0; i < input_buffer.total_size; ++i) { + input_float[i] = static_cast(input_fp16[i]); + } + } else { + std::memcpy(input_float.data(), input_buffer.data_as(), input_buffer.total_size * sizeof(float)); + } + + if (weight_buffer.precision == Precision::INT8) { + throw std::runtime_error("LayerNorm currently does not support INT8 weight"); + } else if (weight_buffer.precision == Precision::FP16) { + const __fp16* weight_fp16 = weight_buffer.data_as<__fp16>(); + for (size_t i = 0; i < feature_size; ++i) { + weight_float[i] = static_cast(weight_fp16[i]); + } + } else { + std::memcpy(weight_float.data(), weight_buffer.data_as(), feature_size * sizeof(float)); + } + + if (has_bias) { + const auto& bias_buffer = *bias_buffer_ptr; + if (bias_buffer.precision == Precision::INT8) { + throw std::runtime_error("LayerNorm currently does not support INT8 bias"); + } else if (bias_buffer.precision == Precision::FP16) { + const __fp16* bias_fp16 = bias_buffer.data_as<__fp16>(); + for (size_t i = 0; i < feature_size; ++i) { + bias_float[i] = static_cast(bias_fp16[i]); + } + } else { + std::memcpy(bias_float.data(), bias_buffer.data_as(), feature_size * sizeof(float)); + } + } + + std::vector output_float(input_buffer.total_size); + for (size_t b = 0; b < batch_size; ++b) { + const float* input_row = input_float.data() + b * feature_size; + float* output_row = output_float.data() + b * feature_size; + + float mean = 0.0f; + for (size_t i = 0; i < feature_size; ++i) { + mean += input_row[i]; + } + mean /= feature_size; + + float variance = 0.0f; + for (size_t i = 0; i < feature_size; ++i) { + float diff = input_row[i] - mean; + variance += diff * diff; + } + variance /= feature_size; + + float std_inv = 1.0f / std::sqrt(variance + epsilon); + for (size_t i = 0; i < feature_size; ++i) { + output_row[i] = (input_row[i] - mean) * std_inv * weight_float[i] + bias_float[i]; + } + } + + if (node.output_buffer.precision == Precision::INT8) { + throw std::runtime_error("LayerNorm currently does not support INT8 output"); + } else if (node.output_buffer.precision == Precision::FP16) { + __fp16* output_fp16 = node.output_buffer.data_as<__fp16>(); + for (size_t i = 0; i < input_buffer.total_size; ++i) { + output_fp16[i] = static_cast<__fp16>(output_float[i]); + } + } else { + std::memcpy(node.output_buffer.data_as(), output_float.data(), input_buffer.total_size * sizeof(float)); + } +} + +void compute_conv1d_causal_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU causal convolution operation not yet implemented"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + auto& Y = node.output_buffer; + + if (X.shape.size() != 3) { + throw std::runtime_error("Causal conv requires 3D input [batch, seq_len, in_channels]"); + } + if (W.shape.size() != 3) { + throw std::runtime_error("Weight must be 3D"); + } + + const size_t N = X.shape[0]; + const size_t L = X.shape[1]; + const size_t C_in = X.shape[2]; + const size_t W0 = W.shape[0]; + const size_t W1 = W.shape[1]; + const size_t W2 = W.shape[2]; + const size_t dil = node.params.dilation; + if (dil < 1) throw std::runtime_error("dilation must be >= 1"); + + size_t M = 1; + size_t C_out = 0; + const bool standard_layout = (W1 == 1); + const bool transposed_layout = (W2 == 1); + if ((!standard_layout && !transposed_layout) || (W0 % C_in != 0)) { + throw std::runtime_error("Only depthwise causal convolution is supported currently"); + } + const size_t K = standard_layout ? W2 : W1; + M = W0 / C_in; + C_out = C_in * M; + + Y.shape = { N, L, C_out }; + Y.precision = X.precision; + + auto transpose_depthwise_weights_fp16 = [&](const __fp16* src) { + std::vector<__fp16> transposed(W0 * K); + for (size_t oc = 0; oc < W0; ++oc) { + for (size_t k = 0; k < K; ++k) { + transposed[oc * K + k] = src[(oc * W1 + k) * W2]; + } + } + return transposed; + }; + + if (W.precision == Precision::INT8) { + const size_t W_size = W0 * W1 * W2; + const int8_t* W_int8 = W.data_as(); + + std::vector<__fp16> W_fp16(W_size); + + if (W.is_grouped_int8()) { + const __fp16* scales = W.scales_as_fp16(); + const size_t K_total = W1 * K; + const size_t group_size = W.group_size; + const size_t num_groups = K_total / group_size; + + for (size_t row = 0; row < W0; ++row) { + for (size_t col = 0; col < K_total; ++col) { + size_t idx = row * K_total + col; + size_t group_idx = col / group_size; + float scale = static_cast(scales[row * num_groups + group_idx]); + W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); + } + } + } else { + for (size_t i = 0; i < W_size; ++i) { + W_fp16[i] = static_cast<__fp16>(W_int8[i]); + } + } + + if (transposed_layout && !standard_layout) { + auto fixed = transpose_depthwise_weights_fp16(W_fp16.data()); + cactus_conv1d_causal_depthwise_f16( + X.data_as<__fp16>(), fixed.data(), Y.data_as<__fp16>(), + N, L, C_in, K, dil); + } else { + cactus_conv1d_causal_depthwise_f16( + X.data_as<__fp16>(), W_fp16.data(), Y.data_as<__fp16>(), + N, L, C_in, K, dil); + } + } else if (W.precision == Precision::FP16) { + if (transposed_layout && !standard_layout) { + auto fixed = transpose_depthwise_weights_fp16(W.data_as<__fp16>()); + cactus_conv1d_causal_depthwise_f16( + X.data_as<__fp16>(), fixed.data(), Y.data_as<__fp16>(), + N, L, C_in, K, dil); + } else { + cactus_conv1d_causal_depthwise_f16( + X.data_as<__fp16>(), W.data_as<__fp16>(), Y.data_as<__fp16>(), + N, L, C_in, K, dil); + } + } else { + throw std::runtime_error("Depthwise causal conv supports INT8/FP16 weights"); + } +} + +void compute_conv1d_k3_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU causal convolution operation not yet implemented"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + auto& Y = node.output_buffer; + + if (X.shape.size() != 3) + throw std::runtime_error("Conv requires 3D input [N, C_in, L]!"); + + if (W.shape.size() != 3) + throw std::runtime_error("Weight must be [C_out, C_in, 3]!"); + + const size_t N = X.shape[0]; + const size_t C_in = X.shape[1]; + const size_t L = X.shape[2]; + + const size_t C_out = W.shape[0]; + const size_t K = W.shape[2]; + const size_t stride = node.params.stride; + + if (K != 3) + throw std::runtime_error("Conv1d_k3 only supports K=3!"); + + size_t L_out = ((L - 1) / stride) + 1; + Y.shape = { N, C_out, L_out }; + Y.precision = X.precision; + + if (X.precision != Precision::FP16) { + throw std::runtime_error("Conv1d_k3 only supports FP16 activations"); + } + + if (W.precision == Precision::INT8) { + const size_t W_size = C_out * C_in * K; + const int8_t* W_int8 = W.data_as(); + + std::vector<__fp16> W_fp16(W_size); + + if (W.is_grouped_int8()) { + const __fp16* scales = W.scales_as_fp16(); + const size_t K_total = C_in * K; + const size_t group_size = W.group_size; + const size_t num_groups = K_total / group_size; + + for (size_t row = 0; row < C_out; ++row) { + for (size_t col = 0; col < K_total; ++col) { + size_t idx = row * K_total + col; + size_t group_idx = col / group_size; + float scale = static_cast(scales[row * num_groups + group_idx]); + W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); + } + } + } else { + for (size_t i = 0; i < W_size; ++i) { + W_fp16[i] = static_cast<__fp16>(W_int8[i]); + } + } + + cactus_conv1d_f16_k3( + X.data_as<__fp16>(), + W_fp16.data(), + Y.data_as<__fp16>(), + N, L, C_in, C_out, stride + ); + } else if (W.precision == Precision::FP16) { + cactus_conv1d_f16_k3( + X.data_as<__fp16>(), + W.data_as<__fp16>(), + Y.data_as<__fp16>(), + N, L, C_in, C_out, stride + ); + } else { + throw std::runtime_error("Conv1d_k3 only supports FP16 and INT8 weights"); + } +} + +void compute_conv1d_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + + auto& Y = node.output_buffer; + + if (X.shape.size() != 3) { + throw std::runtime_error("conv1d expects input [N, C_in, L]"); + } + if (W.shape.size() != 3) { + throw std::runtime_error("conv1d weight must be [C_out, C_in, K]"); + } + + const size_t N = X.shape[0]; + const size_t C_in = X.shape[1]; + const size_t L = X.shape[2]; + const size_t C_out = W.shape[0]; + const size_t K = W.shape[2]; + const size_t stride = node.params.stride; + + if (W.shape[1] != C_in) { + throw std::runtime_error("conv1d weight C_in mismatch"); + } + + if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { + throw std::runtime_error("Conv1d only supports FP16"); + } + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->total_size != C_out) { + throw std::runtime_error("conv1d bias size mismatch"); + } + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C_out); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv1d bias only supports FP16/FP32"); + } + } + + cactus_conv1d_f16(X.data_as<__fp16>(), W.data_as<__fp16>(), bias_ptr, + Y.data_as<__fp16>(), N, L, C_in, C_out, K, stride); +} + +void compute_conv1d_same_depthwise_k9_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU conv1d_same_depthwise_k9 operation not yet implemented"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + auto& Y = node.output_buffer; + + if (X.shape.size() != 3) { + throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); + } + if (X.precision != Precision::FP16) { + throw std::runtime_error("conv1d_same_depthwise_k9 only supports FP16 activations"); + } + + const size_t N = X.shape[0]; + const size_t L = X.shape[1]; + const size_t C = X.shape[2]; + const size_t K = 9; + + if (W.shape.size() == 2) { + if (W.shape[0] != C || W.shape[1] != K) { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); + } + } else if (W.shape.size() == 3) { + if (W.shape[0] != C || W.shape[1] != 1 || W.shape[2] != K) { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); + } + } else { + throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); + } + + Y.shape = {N, L, C}; + Y.precision = Precision::FP16; + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->total_size != C) { + throw std::runtime_error("conv1d_same_depthwise_k9 bias size mismatch"); + } + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv1d_same_depthwise_k9 bias only supports FP16/FP32"); + } + } + + if (W.precision == Precision::FP16) { + cactus_conv1d_same_depthwise_f16_k9( + X.data_as<__fp16>(), + W.data_as<__fp16>(), + bias_ptr, + Y.data_as<__fp16>(), + N, L, C + ); + return; + } + + if (W.precision == Precision::INT8) { + const size_t W_size = C * K; + const int8_t* W_int8 = W.data_as(); + std::vector<__fp16> W_fp16(W_size); + + if (W.is_grouped_int8()) { + const __fp16* scales = W.scales_as_fp16(); + const size_t K_total = K; + const size_t group_size = W.group_size; + if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { + throw std::runtime_error("Grouped INT8 conv1d_same_depthwise_k9 requires valid per-group scales"); + } + + const size_t num_groups = K_total / group_size; + for (size_t row = 0; row < C; ++row) { + for (size_t col = 0; col < K_total; ++col) { + size_t idx = row * K_total + col; + size_t group_idx = col / group_size; + float scale = static_cast(scales[row * num_groups + group_idx]); + W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); + } + } + } else { + for (size_t i = 0; i < W_size; ++i) { + W_fp16[i] = static_cast<__fp16>(W_int8[i]); + } + } + + cactus_conv1d_same_depthwise_f16_k9( + X.data_as<__fp16>(), + W_fp16.data(), + bias_ptr, + Y.data_as<__fp16>(), + N, L, C + ); + return; + } + + throw std::runtime_error("conv1d_same_depthwise_k9 only supports FP16/INT8 weights"); +} + +void compute_conv2d_k3s2p1_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU conv2d_k3s2p1 operation not yet implemented"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + auto& Y = node.output_buffer; + + if (X.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); + } + if (X.precision != Precision::FP16) { + throw std::runtime_error("conv2d_k3s2p1 only supports FP16 activations"); + } + + const size_t N = X.shape[0]; + const size_t C_in = X.shape[1]; + const size_t H = X.shape[2]; + const size_t W_in = X.shape[3]; + const size_t C_out = Y.shape[1]; + + if (H == 0 || W_in == 0) { + throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); + } + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C_out); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv2d_k3s2p1 bias only supports FP16/FP32"); + } + } + + if (W.precision == Precision::FP16) { + if (W.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s2p1 FP16 weight must be [C_out, C_in, 3, 3]"); + } + cactus_conv2d_f16_k3s2p1_nchw( + X.data_as<__fp16>(), + W.data_as<__fp16>(), + bias_ptr, + Y.data_as<__fp16>(), + N, C_in, H, W_in, C_out + ); + return; + } + + if (W.precision == Precision::INT8) { + const size_t K_total = C_in * 9; + const size_t W_size = C_out * K_total; + const int8_t* W_int8 = W.data_as(); + std::vector<__fp16> W_fp16(W_size); + + if (W.is_grouped_int8()) { + const __fp16* scales = W.scales_as_fp16(); + const size_t group_size = W.group_size; + if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { + throw std::runtime_error("Grouped INT8 conv2d_k3s2p1 requires valid per-group scales"); + } + + const size_t num_groups = K_total / group_size; + for (size_t row = 0; row < C_out; ++row) { + for (size_t col = 0; col < K_total; ++col) { + size_t idx = row * K_total + col; + size_t group_idx = col / group_size; + float scale = static_cast(scales[row * num_groups + group_idx]); + W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); + } + } + } else { + for (size_t i = 0; i < W_size; ++i) { + W_fp16[i] = static_cast<__fp16>(W_int8[i]); + } + } + + cactus_conv2d_f16_k3s2p1_nchw( + X.data_as<__fp16>(), + W_fp16.data(), + bias_ptr, + Y.data_as<__fp16>(), + N, C_in, H, W_in, C_out + ); + return; + } + + throw std::runtime_error("conv2d_k3s2p1 only supports FP16/INT8 weights"); +} + +void compute_conv2d_depthwise_k3s2p1_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU conv2d_depthwise_k3s2p1 operation not yet implemented"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + auto& Y = node.output_buffer; + + if (X.shape.size() != 4) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); + } + if (X.precision != Precision::FP16) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 only supports FP16 activations"); + } + + const size_t N = X.shape[0]; + const size_t C = X.shape[1]; + const size_t H = X.shape[2]; + const size_t W_in = X.shape[3]; + if (H == 0 || W_in == 0) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); + } + + if (W.shape.size() == 3) { + if (W.shape[0] != C || W.shape[1] != 3 || W.shape[2] != 3) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); + } + } else if (W.shape.size() == 4) { + if (W.shape[0] != C || W.shape[1] != 1 || W.shape[2] != 3 || W.shape[3] != 3) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); + } + } else { + throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); + } + + const size_t H_out = (H - 1) / 2 + 1; + const size_t W_out = (W_in - 1) / 2 + 1; + Y.shape = {N, C, H_out, W_out}; + Y.precision = Precision::FP16; + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->total_size != C) { + throw std::runtime_error("conv2d_depthwise_k3s2p1 bias size mismatch"); + } + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv2d_depthwise_k3s2p1 bias only supports FP16/FP32"); + } + } + + if (W.precision == Precision::FP16) { + cactus_conv2d_depthwise_f16_k3s2p1_nchw( + X.data_as<__fp16>(), + W.data_as<__fp16>(), + bias_ptr, + Y.data_as<__fp16>(), + N, C, H, W_in + ); + return; + } + + if (W.precision == Precision::INT8) { + const size_t K_total = 9; + const size_t W_size = C * K_total; + const int8_t* W_int8 = W.data_as(); + std::vector<__fp16> W_fp16(W_size); + + if (W.is_grouped_int8()) { + const __fp16* scales = W.scales_as_fp16(); + const size_t group_size = W.group_size; + if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { + throw std::runtime_error("Grouped INT8 conv2d_depthwise_k3s2p1 requires valid per-group scales"); + } + + const size_t num_groups = K_total / group_size; + for (size_t row = 0; row < C; ++row) { + for (size_t col = 0; col < K_total; ++col) { + size_t idx = row * K_total + col; + size_t group_idx = col / group_size; + float scale = static_cast(scales[row * num_groups + group_idx]); + W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); + } + } + } else { + for (size_t i = 0; i < W_size; ++i) { + W_fp16[i] = static_cast<__fp16>(W_int8[i]); + } + } + + cactus_conv2d_depthwise_f16_k3s2p1_nchw( + X.data_as<__fp16>(), + W_fp16.data(), + bias_ptr, + Y.data_as<__fp16>(), + N, C, H, W_in + ); + return; + } + + throw std::runtime_error("conv2d_depthwise_k3s2p1 only supports FP16/INT8 weights"); +} + +void compute_conv2d_pointwise_1x1_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU conv2d_pointwise_1x1 operation not yet implemented"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + auto& Y = node.output_buffer; + + if (X.shape.size() != 4) { + throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); + } + if (X.precision != Precision::FP16) { + throw std::runtime_error("conv2d_pointwise_1x1 only supports FP16 activations"); + } + + const size_t N = X.shape[0]; + const size_t C_in = X.shape[1]; + const size_t H = X.shape[2]; + const size_t W_in = X.shape[3]; + if (H == 0 || W_in == 0) { + throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); + } + + size_t C_out = 0; + if (W.shape.size() == 2) { + C_out = W.shape[0]; + if (W.shape[1] != C_in) { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); + } + } else if (W.shape.size() == 4) { + C_out = W.shape[0]; + if (W.shape[1] != C_in || W.shape[2] != 1 || W.shape[3] != 1) { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); + } + } else { + throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); + } + + Y.shape = {N, C_out, H, W_in}; + Y.precision = Precision::FP16; + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->total_size != C_out) { + throw std::runtime_error("conv2d_pointwise_1x1 bias size mismatch"); + } + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C_out); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv2d_pointwise_1x1 bias only supports FP16/FP32"); + } + } + + if (W.precision == Precision::FP16) { + cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + X.data_as<__fp16>(), + W.data_as<__fp16>(), + bias_ptr, + Y.data_as<__fp16>(), + N, C_in, H, W_in, C_out + ); + return; + } + + if (W.precision == Precision::INT8) { + const size_t K_total = C_in; + const size_t W_size = C_out * K_total; + const int8_t* W_int8 = W.data_as(); + std::vector<__fp16> W_fp16(W_size); + + if (W.is_grouped_int8()) { + const __fp16* scales = W.scales_as_fp16(); + const size_t group_size = W.group_size; + if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { + throw std::runtime_error("Grouped INT8 conv2d_pointwise_1x1 requires valid per-group scales"); + } + + const size_t num_groups = K_total / group_size; + for (size_t row = 0; row < C_out; ++row) { + for (size_t col = 0; col < K_total; ++col) { + size_t idx = row * K_total + col; + size_t group_idx = col / group_size; + float scale = static_cast(scales[row * num_groups + group_idx]); + W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); + } + } + } else { + for (size_t i = 0; i < W_size; ++i) { + W_fp16[i] = static_cast<__fp16>(W_int8[i]); + } + } + + cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + X.data_as<__fp16>(), + W_fp16.data(), + bias_ptr, + Y.data_as<__fp16>(), + N, C_in, H, W_in, C_out + ); + return; + } + + throw std::runtime_error("conv2d_pointwise_1x1 only supports FP16/INT8 weights"); +} + +void compute_conv1d_pointwise_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU conv1d_pointwise operation not yet implemented"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + auto& Y = node.output_buffer; + + if (X.shape.size() != 3) { + throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); + } + if (X.precision != Precision::FP16) { + throw std::runtime_error("conv1d_pointwise only supports FP16 activations"); + } + + const size_t N = X.shape[0]; + const size_t L = X.shape[1]; + const size_t C_in = X.shape[2]; + + size_t C_out = 0; + if (W.shape.size() == 2) { + C_out = W.shape[0]; + if (W.shape[1] != C_in) { + throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); + } + } else if (W.shape.size() == 3) { + C_out = W.shape[0]; + if (W.shape[1] != C_in || W.shape[2] != 1) { + throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); + } + } else { + throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); + } + + Y.shape = {N, L, C_out}; + Y.precision = Precision::FP16; + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->total_size != C_out) { + throw std::runtime_error("conv1d_pointwise bias size mismatch"); + } + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C_out); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv1d_pointwise bias only supports FP16/FP32"); + } + } + + if (W.precision == Precision::FP16) { + cactus_conv1d_pointwise_f16_gemm( + X.data_as<__fp16>(), + W.data_as<__fp16>(), + bias_ptr, + Y.data_as<__fp16>(), + N, L, C_in, C_out + ); + return; + } + + if (W.precision == Precision::INT8) { + const size_t K_total = C_in; + const size_t W_size = C_out * K_total; + const int8_t* W_int8 = W.data_as(); + std::vector<__fp16> W_fp16(W_size); + + if (W.is_grouped_int8()) { + const __fp16* scales = W.scales_as_fp16(); + const size_t group_size = W.group_size; + if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { + throw std::runtime_error("Grouped INT8 conv1d_pointwise requires valid per-group scales"); + } + + const size_t num_groups = K_total / group_size; + for (size_t row = 0; row < C_out; ++row) { + for (size_t col = 0; col < K_total; ++col) { + size_t idx = row * K_total + col; + size_t group_idx = col / group_size; + float scale = static_cast(scales[row * num_groups + group_idx]); + W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); + } + } + } else { + for (size_t i = 0; i < W_size; ++i) { + W_fp16[i] = static_cast<__fp16>(W_int8[i]); + } + } + + cactus_conv1d_pointwise_f16_gemm( + X.data_as<__fp16>(), + W_fp16.data(), + bias_ptr, + Y.data_as<__fp16>(), + N, L, C_in, C_out + ); + return; + } + + throw std::runtime_error("conv1d_pointwise only supports FP16/INT8 weights"); +} + +void compute_glu_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& X = get_input(node, 0, nodes, node_index_map); + auto& Y = node.output_buffer; + + if (X.shape.empty()) { + throw std::runtime_error("GLU expects non-scalar input"); + } + + int axis = node.params.axis; + if (axis < 0) axis += static_cast(X.shape.size()); + if (axis < 0 || static_cast(axis) >= X.shape.size()) { + throw std::runtime_error("GLU axis out of range"); + } + + const size_t axis_size = X.shape[static_cast(axis)]; + if ((axis_size % 2) != 0) { + throw std::runtime_error("GLU split dimension must be even"); + } + const size_t split = axis_size / 2; + + size_t outer = 1; + for (int i = 0; i < axis; ++i) { + outer *= X.shape[static_cast(i)]; + } + size_t inner = 1; + for (size_t i = static_cast(axis) + 1; i < X.shape.size(); ++i) { + inner *= X.shape[i]; + } + + std::vector out_shape = X.shape; + out_shape[static_cast(axis)] = split; + Y.shape = out_shape; + Y.precision = X.precision; + + if (X.precision == Precision::FP16) { + cactus_glu_f16(X.data_as<__fp16>(), Y.data_as<__fp16>(), outer, split, inner); + return; + } + + if (X.precision == Precision::FP32) { + cactus_glu_f32(X.data_as(), Y.data_as(), outer, split, inner); + return; + } + + throw std::runtime_error("GLU only supports FP16/FP32"); +} + +void compute_batchnorm_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.input_ids.size() != 5) { + throw std::runtime_error("BatchNorm expects 5 inputs: input, weight, bias, running_mean, running_var"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const auto& B = get_input(node, 2, nodes, node_index_map); + const auto& RM = get_input(node, 3, nodes, node_index_map); + const auto& RV = get_input(node, 4, nodes, node_index_map); + auto& Y = node.output_buffer; + + if (X.shape.empty()) { + throw std::runtime_error("BatchNorm expects non-scalar input"); + } + + int axis = node.params.axis; + if (axis < 0) axis += static_cast(X.shape.size()); + if (axis < 0 || static_cast(axis) >= X.shape.size()) { + throw std::runtime_error("BatchNorm axis out of range"); + } + + const size_t C = X.shape[static_cast(axis)]; + if (W.total_size != C || B.total_size != C || RM.total_size != C || RV.total_size != C) { + throw std::runtime_error("BatchNorm parameter size mismatch"); + } + + auto load_1d_float = [C](const BufferDesc& buf, const char* name) -> std::vector { + if (buf.total_size != C) { + throw std::runtime_error(std::string("BatchNorm parameter size mismatch for ") + name); + } + std::vector out(C); + if (buf.precision == Precision::FP16) { + const __fp16* p = buf.data_as<__fp16>(); + for (size_t i = 0; i < C; ++i) out[i] = static_cast(p[i]); + } else if (buf.precision == Precision::FP32) { + std::memcpy(out.data(), buf.data_as(), C * sizeof(float)); + } else { + throw std::runtime_error(std::string("BatchNorm parameter ") + name + " must be FP16 or FP32"); + } + return out; + }; + + const std::vector gamma = load_1d_float(W, "weight"); + const std::vector beta = load_1d_float(B, "bias"); + const std::vector mean = load_1d_float(RM, "running_mean"); + const std::vector var = load_1d_float(RV, "running_var"); + + size_t outer = 1; + for (int i = 0; i < axis; ++i) { + outer *= X.shape[static_cast(i)]; + } + size_t inner = 1; + for (size_t i = static_cast(axis) + 1; i < X.shape.size(); ++i) { + inner *= X.shape[i]; + } + + Y.shape = X.shape; + Y.precision = X.precision; + + if (X.precision == Precision::FP16) { + cactus_batchnorm_f16( + X.data_as<__fp16>(), + gamma.data(), + beta.data(), + mean.data(), + var.data(), + Y.data_as<__fp16>(), + outer, + C, + inner, + node.params.epsilon + ); + return; + } + + if (X.precision == Precision::FP32) { + cactus_batchnorm_f32( + X.data_as(), + gamma.data(), + beta.data(), + mean.data(), + var.data(), + Y.data_as(), + outer, + C, + inner, + node.params.epsilon + ); + return; + } + + throw std::runtime_error("BatchNorm only supports FP16/FP32 activations"); +} + +void compute_stft_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + auto& Y = node.output_buffer; + + const size_t N = X.shape[0]; + const size_t C_in = X.shape[1]; + const size_t L = X.shape[2]; + const size_t C_out = W.shape[0]; + const size_t K = W.shape[2]; + const size_t stride = node.params.stride; + const size_t num_fft_bins = node.params.num_fft_bins; + + if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { + throw std::runtime_error("stft only supports FP16"); + } + + cactus_stft_f16(X.data_as<__fp16>(), W.data_as<__fp16>(), + Y.data_as<__fp16>(), N, L, C_in, C_out, K, stride, num_fft_bins); +} + +void compute_conv1d_k7s3_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + + auto& Y = node.output_buffer; + + const size_t N = X.shape[0]; + const size_t C_in = X.shape[1]; + const size_t L = X.shape[2]; + + if (W.shape.size() != 3) throw std::runtime_error("Weight must be 3D"); + const size_t C_in_W = W.shape[0]; + const size_t K = W.shape[1]; + const size_t C_out = W.shape[2]; + const size_t stride = node.params.stride; + + if (C_in != C_in_W) throw std::runtime_error("Channel mismatch in conv1d_k7s3"); + if (K != 7 || stride != 3) throw std::runtime_error("conv1d_k7s3 requires K=7, stride=3"); + + if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { + throw std::runtime_error("Conv1d specialized only supports FP16"); + } + + size_t L_out = (L < 7) ? 0 : (L - 7) / 3 + 1; + Y.shape = {N, C_out, L_out}; + Y.precision = Precision::FP16; + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->total_size != C_out) { + throw std::runtime_error("conv1d_k7s3 bias size mismatch"); + } + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C_out); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv1d_k7s3 bias only supports FP16/FP32"); + } + } + + cactus_conv1d_f16_k7s3_oc8( + X.data_as<__fp16>(), + W.data_as<__fp16>(), + bias_ptr, + Y.data_as<__fp16>(), + N, L, C_in, C_out + ); +} + +void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.input_ids.size() != 6) { + throw std::runtime_error("GATED_DELTANET_DECODE expects 6 inputs"); + } + + const auto& q = get_input(node, 0, nodes, node_index_map); + const auto& k = get_input(node, 1, nodes, node_index_map); + const auto& v = get_input(node, 2, nodes, node_index_map); + const auto& g = get_input(node, 3, nodes, node_index_map); + const auto& b = get_input(node, 4, nodes, node_index_map); + const auto& s = get_input(node, 5, nodes, node_index_map); + + validate_gated_deltanet_inputs(q, k, v, g, b, s); + if (q.shape[1] != 1) { + throw std::runtime_error("GATED_DELTANET_DECODE expects T=1"); + } + + const size_t B = q.shape[0]; + const size_t Hq = q.shape[2]; + const size_t K = q.shape[3]; + const size_t Hv = v.shape[2]; + const size_t V = v.shape[3]; + const size_t qk_heads_from_params = node.params.num_kv_heads; + if (qk_heads_from_params != 0 && qk_heads_from_params != Hq) { + throw std::runtime_error("GATED_DELTANET_DECODE num_qk_heads param mismatch"); + } + + std::vector<__fp16> q_cast; + std::vector<__fp16> k_cast; + std::vector<__fp16> v_cast; + std::vector<__fp16> g_cast; + std::vector<__fp16> b_cast; + std::vector<__fp16> s_cast; + const __fp16* q_data = as_fp16_ptr(q, q_cast); + const __fp16* k_data = as_fp16_ptr(k, k_cast); + const __fp16* v_data = as_fp16_ptr(v, v_cast); + const __fp16* g_data = as_fp16_ptr(g, g_cast); + const __fp16* b_data = as_fp16_ptr(b, b_cast); + const __fp16* s_data = as_fp16_ptr(s, s_cast); + __fp16* out = node.output_buffer.data_as<__fp16>(); + + cactus_gated_deltanet_decode_f16( + q_data, k_data, v_data, g_data, b_data, s_data, out, + B, Hq, Hv, K, V, node.params.scale); +} + +void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.input_ids.size() != 6) { + throw std::runtime_error("GATED_DELTANET_PREFILL expects 6 inputs"); + } + + const auto& q = get_input(node, 0, nodes, node_index_map); + const auto& k = get_input(node, 1, nodes, node_index_map); + const auto& v = get_input(node, 2, nodes, node_index_map); + const auto& g = get_input(node, 3, nodes, node_index_map); + const auto& b = get_input(node, 4, nodes, node_index_map); + const auto& s = get_input(node, 5, nodes, node_index_map); + + validate_gated_deltanet_inputs(q, k, v, g, b, s); + + const size_t B = q.shape[0]; + const size_t T = q.shape[1]; + const size_t Hq = q.shape[2]; + const size_t K = q.shape[3]; + const size_t Hv = v.shape[2]; + const size_t V = v.shape[3]; + const size_t qk_heads_from_params = node.params.num_kv_heads; + if (qk_heads_from_params != 0 && qk_heads_from_params != Hq) { + throw std::runtime_error("GATED_DELTANET_PREFILL num_qk_heads param mismatch"); + } + + std::vector<__fp16> q_cast; + std::vector<__fp16> k_cast; + std::vector<__fp16> v_cast; + std::vector<__fp16> g_cast; + std::vector<__fp16> b_cast; + std::vector<__fp16> s_cast; + const __fp16* q_data = as_fp16_ptr(q, q_cast); + const __fp16* k_data = as_fp16_ptr(k, k_cast); + const __fp16* v_data = as_fp16_ptr(v, v_cast); + const __fp16* g_data = as_fp16_ptr(g, g_cast); + const __fp16* b_data = as_fp16_ptr(b, b_cast); + const __fp16* s_data = as_fp16_ptr(s, s_cast); + __fp16* out = node.output_buffer.data_as<__fp16>(); + + cactus_gated_deltanet_prefill_f16( + q_data, k_data, v_data, g_data, b_data, s_data, out, + B, T, Hq, Hv, K, V, node.params.chunk_size, node.params.scale); +} + +void compute_rope_gptj_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& shape = input_buffer.shape; + + size_t batch_size = shape[0]; + size_t seq_len = shape[1]; + size_t num_heads = shape[2]; + size_t head_dim = shape[3]; + size_t rot_dim = static_cast(node.params.scalar); + + cactus_gpt_j_rope_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, num_heads, head_dim, rot_dim, + node.params.position_offset, node.params.theta); +} + +void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& input = get_input(node, 0, nodes, node_index_map); + const auto& weight = get_input(node, 1, nodes, node_index_map); + const auto& bias = get_input(node, 2, nodes, node_index_map); + float epsilon = node.params.epsilon; + + size_t batch_size = input.shape[0]; + size_t channels = input.shape[1]; + size_t spatial_size = 1; + for (size_t i = 2; i < input.shape.size(); ++i) spatial_size *= input.shape[i]; + + size_t num_groups = node.params.num_groups; + if (num_groups == 0) num_groups = 32; + + if (channels % num_groups != 0) { + throw std::runtime_error("GroupNorm: channels must be divisible by num_groups"); + } + + size_t channels_per_group = channels / num_groups; + + const __fp16* src = input.data_as<__fp16>(); + const __fp16* w = weight.data_as<__fp16>(); + const __fp16* b = bias.data_as<__fp16>(); + __fp16* dst = node.output_buffer.data_as<__fp16>(); + + for (size_t n = 0; n < batch_size; ++n) { + for (size_t g = 0; g < num_groups; ++g) { + float sum = 0.0f, sum_sq = 0.0f; + size_t count = 0; + + for (size_t c = 0; c < channels_per_group; ++c) { + size_t ch = g * channels_per_group + c; + for (size_t s = 0; s < spatial_size; ++s) { + size_t idx = n * channels * spatial_size + ch * spatial_size + s; + float val = static_cast(src[idx]); + sum += val; + sum_sq += val * val; + count++; + } + } + + float mean = sum / count; + float var = (sum_sq / count) - (mean * mean); + float inv_std = 1.0f / std::sqrt(var + epsilon); + + for (size_t c = 0; c < channels_per_group; ++c) { + size_t ch = g * channels_per_group + c; + float wt = static_cast(w[ch]); + float bi = static_cast(b[ch]); + + for (size_t s = 0; s < spatial_size; ++s) { + size_t idx = n * channels * spatial_size + ch * spatial_size + s; + float val = static_cast(src[idx]); + dst[idx] = static_cast<__fp16>((val - mean) * inv_std * wt + bi); + } + } + } + } +} + +void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& h_prev_buffer = get_input(node, 1, nodes, node_index_map); + const auto& c_prev_buffer = get_input(node, 2, nodes, node_index_map); + const auto& weight_ih_buffer = get_input(node, 3, nodes, node_index_map); + const auto& weight_hh_buffer = get_input(node, 4, nodes, node_index_map); + const auto& bias_ih_buffer = get_input(node, 5, nodes, node_index_map); + const auto& bias_hh_buffer = get_input(node, 6, nodes, node_index_map); + + if (input_buffer.precision != Precision::FP16 || h_prev_buffer.precision != Precision::FP16 || + c_prev_buffer.precision != Precision::FP16 || weight_ih_buffer.precision != Precision::FP16 || + weight_hh_buffer.precision != Precision::FP16 || bias_ih_buffer.precision != Precision::FP16 || + bias_hh_buffer.precision != Precision::FP16) { + throw std::runtime_error("LSTM cell requires all inputs to be FP16"); + } + + if (input_buffer.shape.size() != 2 || h_prev_buffer.shape.size() != 2 || c_prev_buffer.shape.size() != 2) { + throw std::runtime_error("LSTM cell input/state shapes must be 2D [batch, features]"); + } + + const size_t batch_size = input_buffer.shape[0]; + const size_t input_size = input_buffer.shape[1]; + const size_t hidden_size = h_prev_buffer.shape[1]; + + const __fp16* x_input = input_buffer.data_as<__fp16>(); + const __fp16* h_prev = h_prev_buffer.data_as<__fp16>(); + const __fp16* c_prev = c_prev_buffer.data_as<__fp16>(); + const __fp16* weight_ih = weight_ih_buffer.data_as<__fp16>(); + const __fp16* weight_hh = weight_hh_buffer.data_as<__fp16>(); + const __fp16* bias_ih = bias_ih_buffer.data_as<__fp16>(); + const __fp16* bias_hh = bias_hh_buffer.data_as<__fp16>(); + + node.output_buffer.shape = {batch_size, hidden_size, 2}; + node.output_buffer.total_size = batch_size * hidden_size * 2; + node.output_buffer.precision = Precision::FP16; + node.output_buffer.allocate(); + + std::vector<__fp16> h_new_temp(batch_size * hidden_size); + std::vector<__fp16> c_new_temp(batch_size * hidden_size); + + cactus_lstm_cell_f16( + x_input, h_prev, c_prev, + weight_ih, weight_hh, + bias_ih, bias_hh, + h_new_temp.data(), c_new_temp.data(), + batch_size, input_size, hidden_size + ); + + __fp16* output = node.output_buffer.data_as<__fp16>(); + for (size_t b = 0; b < batch_size; ++b) { + for (size_t i = 0; i < hidden_size; ++i) { + const size_t idx = b * hidden_size + i; + output[b * hidden_size * 2 + i * 2] = h_new_temp[idx]; + output[b * hidden_size * 2 + i * 2 + 1] = c_new_temp[idx]; + } + } +} + +void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + size_t n = node.params.num_altup_inputs; + const auto& coefs_buf = get_input(node, 0, nodes, node_index_map); + + std::vector stream_ptrs(n); + for (size_t i = 0; i < n; i++) { + stream_ptrs[i] = get_input(node, 1 + i, nodes, node_index_map).data_as<__fp16>(); + } + + const auto& stream0_buf = get_input(node, 1, nodes, node_index_map); + size_t seq_len = stream0_buf.shape[0]; + size_t hidden_dim = stream0_buf.shape[1]; + + cactus_altup_predict_f16( + coefs_buf.data_as<__fp16>(), + stream_ptrs.data(), + node.output_buffer.data_as<__fp16>(), + n, seq_len, hidden_dim); +} + +void compute_gaussian_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buf = get_input(node, 0, nodes, node_index_map); + const __fp16* input = input_buf.data_as<__fp16>(); + __fp16* output = node.output_buffer.data_as<__fp16>(); + + size_t rows = input_buf.shape[0]; + size_t cols = input_buf.shape[1]; + float ppf = node.params.scalar; + + cactus_gaussian_topk_f16(input, output, rows, cols, ppf); +} + +void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + size_t n = node.params.num_altup_inputs; + const auto& coefs_buf = get_input(node, 0, nodes, node_index_map); + const auto& innov_buf = get_input(node, 1, nodes, node_index_map); + + std::vector pred_ptrs(n); + for (size_t i = 0; i < n; i++) { + pred_ptrs[i] = get_input(node, 2 + i, nodes, node_index_map).data_as<__fp16>(); + } + + size_t seq_len = innov_buf.shape[0]; + size_t hidden_dim = innov_buf.shape[1]; + + cactus_altup_correct_f16( + coefs_buf.data_as<__fp16>(), + innov_buf.data_as<__fp16>(), + pred_ptrs.data(), + node.output_buffer.data_as<__fp16>(), + n, seq_len, hidden_dim); +} + +void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& input = get_input(node, 0, nodes, node_index_map); + const auto& w_ih_fwd = get_input(node, 1, nodes, node_index_map); + const auto& w_hh_fwd = get_input(node, 2, nodes, node_index_map); + const auto& b_ih_fwd = get_input(node, 3, nodes, node_index_map); + const auto& b_hh_fwd = get_input(node, 4, nodes, node_index_map); + const auto& w_ih_bwd = get_input(node, 5, nodes, node_index_map); + const auto& w_hh_bwd = get_input(node, 6, nodes, node_index_map); + const auto& b_ih_bwd = get_input(node, 7, nodes, node_index_map); + const auto& b_hh_bwd = get_input(node, 8, nodes, node_index_map); + + size_t batch_size = input.shape[0]; + size_t seq_len = input.shape[1]; + size_t input_size = input.shape[2]; + size_t hidden_size = w_ih_fwd.shape[0] / 4; + + cactus_bilstm_sequence_f16( + input.data_as<__fp16>(), + w_ih_fwd.data_as<__fp16>(), w_hh_fwd.data_as<__fp16>(), + b_ih_fwd.data_as<__fp16>(), b_hh_fwd.data_as<__fp16>(), + w_ih_bwd.data_as<__fp16>(), w_hh_bwd.data_as<__fp16>(), + b_ih_bwd.data_as<__fp16>(), b_hh_bwd.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, input_size, hidden_size); +} + +void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& input = get_input(node, 0, nodes, node_index_map); + + size_t batch_size = input.shape[0]; + size_t channels = input.shape[1]; + size_t input_length = input.shape[2]; + size_t kernel_size = node.params.kernel_size; + size_t stride = node.params.stride; + + cactus_maxpool1d_f16( + input.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + batch_size, channels, input_length, + kernel_size, stride); +} + +void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + auto& Y = node.output_buffer; + + if (X.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); + } + if (X.precision != Precision::FP16) { + throw std::runtime_error("conv2d_k3s1p1 only supports FP16 activations"); + } + + const size_t N = X.shape[0]; + const size_t C_in = X.shape[1]; + const size_t H = X.shape[2]; + const size_t W_in = X.shape[3]; + const size_t C_out = Y.shape[1]; + + if (H == 0 || W_in == 0) { + throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); + } + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C_out); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv2d_k3s1p1 bias only supports FP16/FP32"); + } + } + + if (W.precision == Precision::FP16) { + if (W.shape.size() != 4) { + throw std::runtime_error("conv2d_k3s1p1 FP16 weight must be [C_out, C_in, 3, 3]"); + } + cactus_conv2d_f16_k3s1p1_nchw( + X.data_as<__fp16>(), W.data_as<__fp16>(), bias_ptr, + Y.data_as<__fp16>(), + N, C_in, H, W_in, C_out); + return; + } + + throw std::runtime_error("conv2d_k3s1p1 only supports FP16 weights"); +} + +void compute_conv2d_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU conv2d operation not yet implemented"); + } + + const auto& X = get_input(node, 0, nodes, node_index_map); + const auto& W = get_input(node, 1, nodes, node_index_map); + const BufferDesc* B = nullptr; + if (node.input_ids.size() >= 3) { + B = &get_input(node, 2, nodes, node_index_map); + } + auto& Y = node.output_buffer; + + if (X.shape.size() != 4) { + throw std::runtime_error("conv2d expects input [N, C_in, H, W]"); + } + if (W.shape.size() != 4) { + throw std::runtime_error("conv2d weight must be [C_out, C_in/groups, K_h, K_w]"); + } + if (X.precision != Precision::FP16) { + throw std::runtime_error("conv2d only supports FP16 activations"); + } + + const size_t stride = node.params.stride; + const size_t padding = node.params.padding; + const size_t dilation = node.params.dilation; + const size_t groups = node.params.num_groups; + if (stride == 0) { + throw std::runtime_error("conv2d stride must be > 0"); + } + if (dilation == 0) { + throw std::runtime_error("conv2d dilation must be > 0"); + } + if (groups == 0) { + throw std::runtime_error("conv2d groups must be > 0"); + } + + const size_t N = X.shape[0]; + const size_t C_in = X.shape[1]; + const size_t H_in = X.shape[2]; + const size_t W_in = X.shape[3]; + const size_t C_out = W.shape[0]; + const size_t C_in_per_group = W.shape[1]; + const size_t K_h = W.shape[2]; + const size_t K_w = W.shape[3]; + + if ((C_in % groups) != 0) { + throw std::runtime_error("conv2d input channels must be divisible by groups"); + } + if ((C_out % groups) != 0) { + throw std::runtime_error("conv2d output channels must be divisible by groups"); + } + if (C_in_per_group * groups != C_in) { + throw std::runtime_error("conv2d weight must match [C_out, C_in/groups, K_h, K_w]"); + } + + const size_t effective_kh = dilation * (K_h - 1) + 1; + const size_t effective_kw = dilation * (K_w - 1) + 1; + if (H_in + 2 * padding < effective_kh || W_in + 2 * padding < effective_kw) { + throw std::runtime_error("conv2d input spatial dimensions are too small for the requested kernel"); + } + + const size_t H_out_expected = (H_in + 2 * padding - effective_kh) / stride + 1; + const size_t W_out_expected = (W_in + 2 * padding - effective_kw) / stride + 1; + if (Y.shape.size() != 4 || Y.shape[0] != N || Y.shape[1] != C_out || + Y.shape[2] != H_out_expected || Y.shape[3] != W_out_expected) { + throw std::runtime_error("conv2d output buffer shape mismatch"); + } + Y.precision = Precision::FP16; + + const __fp16* x_ptr = X.data_as<__fp16>(); + __fp16* y_ptr = Y.data_as<__fp16>(); + + auto finite_check_enabled = []() -> bool { + static const bool enabled = []() { + const char* env = std::getenv("CACTUS_CHECK_FINITE"); + return env != nullptr && env[0] != '\0' && env[0] != '0'; + }(); + return enabled; + }; + + auto throw_nonfinite_fp16 = [&](const char* label, const __fp16* data, size_t count) { + if (!finite_check_enabled() || data == nullptr) { + return; + } + for (size_t i = 0; i < count; ++i) { + const float value = static_cast(data[i]); + if (!std::isfinite(value)) { + throw std::runtime_error( + std::string("conv2d detected non-finite ") + label + + " at flat index " + std::to_string(i) + + " value=" + std::to_string(value) + ); + } + } + }; + + const __fp16* bias_ptr = nullptr; + std::vector<__fp16> bias_fp16; + if (B) { + if (B->total_size != C_out) { + throw std::runtime_error("conv2d bias size mismatch"); + } + if (B->precision == Precision::FP16) { + bias_ptr = B->data_as<__fp16>(); + } else if (B->precision == Precision::FP32) { + bias_fp16.resize(C_out); + cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); + bias_ptr = bias_fp16.data(); + } else { + throw std::runtime_error("conv2d bias only supports FP16/FP32"); + } + } + + const size_t kernel_elems_per_out = C_in_per_group * K_h * K_w; + const __fp16* w_ptr = nullptr; + std::vector<__fp16> w_fp16; + if (W.precision == Precision::FP16) { + w_ptr = W.data_as<__fp16>(); + } else if (W.precision == Precision::INT8) { + const int8_t* w_int8 = W.data_as(); + w_fp16.resize(C_out * kernel_elems_per_out); + if (W.is_grouped_int8()) { + const __fp16* scales = W.scales_as_fp16(); + const size_t group_size = W.group_size; + if (group_size == 0 || (kernel_elems_per_out % group_size) != 0 || scales == nullptr) { + throw std::runtime_error("Grouped INT8 conv2d requires valid per-group scales"); + } + const size_t num_qgroups = kernel_elems_per_out / group_size; + for (size_t oc = 0; oc < C_out; ++oc) { + for (size_t col = 0; col < kernel_elems_per_out; ++col) { + const size_t idx = oc * kernel_elems_per_out + col; + const size_t qgroup = col / group_size; + const float scale = static_cast(scales[oc * num_qgroups + qgroup]); + w_fp16[idx] = static_cast<__fp16>(w_int8[idx] * scale); + } + } + } else { + for (size_t i = 0; i < w_fp16.size(); ++i) { + w_fp16[i] = static_cast<__fp16>(w_int8[i]); + } + } + w_ptr = w_fp16.data(); + } else { + throw std::runtime_error("conv2d only supports FP16/INT8 weights"); + } + + const size_t C_out_per_group = C_out / groups; + const size_t C_in_group = C_in / groups; + const size_t H_out = Y.shape[2]; + const size_t W_out = Y.shape[3]; + const size_t total_work = N * C_out; + + throw_nonfinite_fp16("input activation", x_ptr, X.total_size); + throw_nonfinite_fp16("weight", w_ptr, C_out * kernel_elems_per_out); + if (bias_ptr != nullptr) { + throw_nonfinite_fp16("bias", bias_ptr, C_out); + } + + CactusThreading::parallel_for( + total_work, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t work_start, size_t work_end) { + for (size_t work_idx = work_start; work_idx < work_end; ++work_idx) { + const size_t n = work_idx / C_out; + const size_t oc = work_idx % C_out; + const size_t group = oc / C_out_per_group; + const size_t ic_base = group * C_in_group; + const size_t w_oc_base = oc * kernel_elems_per_out; + + for (size_t oh = 0; oh < H_out; ++oh) { + for (size_t ow = 0; ow < W_out; ++ow) { + float acc = bias_ptr ? static_cast(bias_ptr[oc]) : 0.0f; + const size_t ih_origin = oh * stride; + const size_t iw_origin = ow * stride; + + for (size_t ic_local = 0; ic_local < C_in_group; ++ic_local) { + const size_t ic = ic_base + ic_local; + const size_t w_ic_base = w_oc_base + ic_local * K_h * K_w; + + for (size_t kh = 0; kh < K_h; ++kh) { + const long long ih = static_cast(ih_origin) + + static_cast(kh * dilation) - + static_cast(padding); + if (ih < 0 || ih >= static_cast(H_in)) { + continue; + } + + for (size_t kw = 0; kw < K_w; ++kw) { + const long long iw = static_cast(iw_origin) + + static_cast(kw * dilation) - + static_cast(padding); + if (iw < 0 || iw >= static_cast(W_in)) { + continue; + } + + const size_t x_idx = ((n * C_in + ic) * H_in + static_cast(ih)) * W_in + static_cast(iw); + const size_t w_idx = w_ic_base + kh * K_w + kw; + acc += static_cast(x_ptr[x_idx]) * static_cast(w_ptr[w_idx]); + } + } + } + + const size_t y_idx = ((n * C_out + oc) * H_out + oh) * W_out + ow; + y_ptr[y_idx] = static_cast<__fp16>(acc); + } + } + } + } + ); + + throw_nonfinite_fp16("output activation", y_ptr, Y.total_size); +} + +void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& input = get_input(node, 0, nodes, node_index_map); + const __fp16* src = input.data_as<__fp16>(); + __fp16* dst = node.output_buffer.data_as<__fp16>(); + + size_t batch = input.shape[0]; + size_t total_per_batch = input.total_size / batch; + size_t T = input.shape.back(); + size_t features = total_per_batch / T; + + for (size_t b = 0; b < batch; ++b) { + const __fp16* batch_src = src + b * total_per_batch; + __fp16* batch_dst = dst + b * features * 2; + + for (size_t f = 0; f < features; ++f) { + float sum = 0.0f, sum_sq = 0.0f; + for (size_t t = 0; t < T; ++t) { + float v = static_cast(batch_src[f * T + t]); + sum += v; + sum_sq += v * v; + } + float mean = sum / static_cast(T); + float var = T > 1 ? (sum_sq - static_cast(T) * mean * mean) / static_cast(T - 1) : 0.0f; + float std_val = sqrtf(fmaxf(var, 0.0f)); + batch_dst[f] = static_cast<__fp16>(mean); + batch_dst[features + f] = static_cast<__fp16>(std_val); + } + } +} + +void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& input = get_input(node, 0, nodes, node_index_map); + const auto& weight_buf = get_input(node, 1, nodes, node_index_map); + const __fp16* src = input.data_as<__fp16>(); + const float* weights = weight_buf.data_as(); + __fp16* dst = node.output_buffer.data_as<__fp16>(); + + size_t batch = input.shape[0]; + size_t total_per_batch = input.total_size / batch; + size_t T = input.shape.back(); + size_t features = total_per_batch / T; + + constexpr float eps = 1e-8f; + + for (size_t b = 0; b < batch; ++b) { + const __fp16* batch_src = src + b * total_per_batch; + const float* batch_w = weights + b * T; + __fp16* batch_dst = dst + b * features * 2; + + float v1 = 0.0f, v2 = 0.0f; + for (size_t t = 0; t < T; ++t) { + float w = batch_w[t]; + v1 += w; + v2 += w * w; + } + float v1_safe = v1 + eps; + float var_denom = v1_safe - v2 / v1_safe + eps; + + for (size_t f = 0; f < features; ++f) { + float wsum = 0.0f; + for (size_t t = 0; t < T; ++t) { + wsum += static_cast(batch_src[f * T + t]) * batch_w[t]; + } + float mean = wsum / v1_safe; + + float wvar = 0.0f; + for (size_t t = 0; t < T; ++t) { + float dx = static_cast(batch_src[f * T + t]) - mean; + wvar += batch_w[t] * dx * dx; + } + float std_val = sqrtf(fmaxf(wvar / var_denom, 0.0f)); + + batch_dst[f] = static_cast<__fp16>(mean); + batch_dst[features + f] = static_cast<__fp16>(std_val); + } + } +} diff --git a/cactus-transpile/graph/graph_ops_sample.cpp b/cactus-transpile/graph/graph_ops_sample.cpp new file mode 100644 index 000000000..26f6b621f --- /dev/null +++ b/cactus-transpile/graph/graph_ops_sample.cpp @@ -0,0 +1,134 @@ +#include "graph.h" +#include "../kernel/kernel.h" +#include +#include +#include +#include + +void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& logits_buffer = get_input(node, 0, nodes, node_index_map); + + float temperature = node.params.temperature; + float top_p = node.params.top_p; + float min_p = node.params.min_p; + float repetition_penalty = node.params.repetition_penalty; + size_t top_k = node.params.top_k; + size_t random_seed = node.params.random_seed; + + const float* bias_values = node.params.bias_values.empty() ? nullptr : node.params.bias_values.data(); + const uint32_t* bias_indices = node.params.bias_indices.empty() ? nullptr : node.params.bias_indices.data(); + size_t bias_count = node.params.bias_values.size(); + + if (logits_buffer.shape.size() != 2) { + throw std::runtime_error("Sample expects 2D logits tensor [seq_len, vocab_size]"); + } + + size_t seq_len = logits_buffer.shape[0]; + size_t vocab_size = logits_buffer.shape[1]; + size_t last_token_offset = (seq_len - 1) * vocab_size; + + if (logits_buffer.precision == Precision::FP16) { + const __fp16* logits_fp16 = logits_buffer.data_as<__fp16>(); + cactus_sample_f16_ex(logits_fp16 + last_token_offset, node.output_buffer.data_as(), + vocab_size, temperature, top_p, min_p, repetition_penalty, top_k, random_seed, + bias_values, bias_indices, bias_count); + } else { + const float* logits_fp32 = logits_buffer.data_as(); + cactus_sample_f32_ex(logits_fp32 + last_token_offset, node.output_buffer.data_as(), + vocab_size, temperature, top_p, min_p, repetition_penalty, top_k, random_seed, + bias_values, bias_indices, bias_count); + } +} + +void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + if (input_buffer.shape.size() != 2) { + throw std::runtime_error("TopK currently only supports 2D tensors [batch, features]"); + } + + size_t k = node.params.top_k; + size_t batch_size = input_buffer.shape[0]; + size_t feature_size = input_buffer.shape[1]; + size_t block_size = batch_size * k; + + std::vector input_float(input_buffer.total_size); + if (input_buffer.precision == Precision::INT8) { + throw std::runtime_error("TopK currently does not support INT8 input"); + } else if (input_buffer.precision == Precision::FP16) { + const __fp16* input_fp16 = input_buffer.data_as<__fp16>(); + for (size_t i = 0; i < input_buffer.total_size; ++i) { + input_float[i] = static_cast(input_fp16[i]); + } + } else { + const float* input_fp32 = input_buffer.data_as(); + std::memcpy(input_float.data(), input_fp32, input_buffer.total_size * sizeof(float)); + } + + float* output = node.output_buffer.data_as(); + + for (size_t b = 0; b < batch_size; ++b) { + const float* row = input_float.data() + b * feature_size; + + std::vector> indexed_values(feature_size); + for (size_t i = 0; i < feature_size; ++i) { + indexed_values[i] = {i, row[i]}; + } + + std::partial_sort(indexed_values.begin(), + indexed_values.begin() + k, + indexed_values.end(), + [](const auto& a, const auto& b) { return a.second > b.second; }); + + float* idx_out_row = output + b * k; + float* val_out_row = output + block_size + b * k; + for (size_t i = 0; i < k; ++i) { + idx_out_row[i] = static_cast(indexed_values[i].first); + val_out_row[i] = indexed_values[i].second; + } + } +} + +void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& indices_buffer = get_input(node, 0, nodes, node_index_map); + const auto& values_buffer = get_input(node, 1, nodes, node_index_map); + + if (indices_buffer.shape != values_buffer.shape) { + throw std::runtime_error("ScatterTopK requires indices and values with identical shapes"); + } + if (indices_buffer.shape.size() != 2) { + throw std::runtime_error("ScatterTopK currently supports 2D tensors"); + } + + size_t batch_size = indices_buffer.shape[0]; + size_t top_k = indices_buffer.shape[1]; + size_t num_classes = node.params.num_classes; + + if (num_classes == 0) { + throw std::runtime_error("ScatterTopK requires num_classes > 0"); + } + + float* output = node.output_buffer.data_as(); + std::fill(output, output + num_classes * batch_size, 0.0f); + + if (indices_buffer.precision != Precision::FP32 || values_buffer.precision != Precision::FP32) { + throw std::runtime_error("ScatterTopK currently expects FP32 inputs"); + } + + const float* indices_data = indices_buffer.data_as(); + const float* values_data = values_buffer.data_as(); + + for (size_t b = 0; b < batch_size; ++b) { + for (size_t k = 0; k < top_k; ++k) { + float raw_index = indices_data[b * top_k + k]; + if (!std::isfinite(raw_index)) { + throw std::runtime_error("ScatterTopK index is not finite"); + } + size_t expert_index = static_cast(raw_index + 0.5f); + if (expert_index >= num_classes) { + throw std::runtime_error("ScatterTopK index out of range"); + } + float weight = values_data[b * top_k + k]; + output[expert_index * batch_size + b] = weight; + } + } +} diff --git a/cactus-transpile/graph/graph_ops_tensor.cpp b/cactus-transpile/graph/graph_ops_tensor.cpp new file mode 100644 index 000000000..c852ad480 --- /dev/null +++ b/cactus-transpile/graph/graph_ops_tensor.cpp @@ -0,0 +1,813 @@ +#include "graph.h" +#include "../kernel/kernel.h" +#include "../kernel/kernel_utils.h" +#include +#include +#include + +template +static void transpose_scalar_nd( + const T* source, + T* destination, + const std::vector& shape, + const std::vector& permutation +) { + const size_t ndim = shape.size(); + std::vector output_shape(ndim, 0); + for (size_t axis = 0; axis < ndim; ++axis) { + output_shape[axis] = shape[permutation[axis]]; + } + + std::vector input_strides(ndim, 1); + for (size_t axis = ndim; axis-- > 1;) { + input_strides[axis - 1] = input_strides[axis] * shape[axis]; + } + + std::vector output_strides(ndim, 1); + for (size_t axis = ndim; axis-- > 1;) { + output_strides[axis - 1] = output_strides[axis] * output_shape[axis]; + } + + size_t total = 1; + for (size_t dim : output_shape) { + total *= dim; + } + + for (size_t out_idx = 0; out_idx < total; ++out_idx) { + size_t remaining = out_idx; + size_t src_idx = 0; + for (size_t axis = 0; axis < ndim; ++axis) { + const size_t coord = remaining / output_strides[axis]; + remaining %= output_strides[axis]; + src_idx += coord * input_strides[permutation[axis]]; + } + destination[out_idx] = source[src_idx]; + } +} + +void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + if (node.params.backend == ComputeBackend::NPU) { + throw std::runtime_error("NPU transpose operation not yet implemented"); + } + + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& permutation = node.params.permutation; + if (input_buffer.precision == Precision::FP16) { + const __fp16* input = input_buffer.data_as<__fp16>(); + __fp16* output = node.output_buffer.data_as<__fp16>(); + cactus_transpose_f16( + input, + output, + input_buffer.shape.data(), + permutation.data(), + permutation.size(), + 0, + input_buffer.total_size + ); + return; + } + + if (input_buffer.precision == Precision::FP32) { + const float* input = input_buffer.data_as(); + float* output = node.output_buffer.data_as(); + transpose_scalar_nd(input, output, input_buffer.shape, permutation); + return; + } + + throw std::runtime_error("Transpose only supports FP16 and FP32 precision"); +} + +void compute_gather_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& tensor_buffer = get_input(node, 0, nodes, node_index_map); + const auto& indices_buffer = get_input(node, 1, nodes, node_index_map); + const int axis = node.params.axis; + + if (indices_buffer.shape.size() == tensor_buffer.shape.size()) { + if (tensor_buffer.precision == Precision::INT4) { + throw std::runtime_error("Generic gather does not support INT4 tensors"); + } + + const size_t rank = tensor_buffer.shape.size(); + if (rank == 0) { + throw std::runtime_error("Cannot gather from scalar tensor"); + } + + std::vector input_strides(rank, 1); + std::vector output_strides(rank, 1); + for (size_t i = rank; i-- > 1;) { + input_strides[i - 1] = input_strides[i] * tensor_buffer.shape[i]; + output_strides[i - 1] = output_strides[i] * indices_buffer.shape[i]; + } + + auto read_index = [&](size_t flat_index) -> size_t { + if (indices_buffer.precision == Precision::INT8) { + const int8_t* indices = indices_buffer.data_as(); + return static_cast(indices[flat_index]); + } + if (indices_buffer.precision == Precision::FP16) { + const __fp16* indices = indices_buffer.data_as<__fp16>(); + return static_cast(indices[flat_index]); + } + const float* indices = indices_buffer.data_as(); + return static_cast(indices[flat_index]); + }; + + const size_t total = node.output_buffer.total_size; + const size_t bytes_per_scalar = PrecisionTraits::packed_size_of(tensor_buffer.precision, 1); + const char* input_data = static_cast(tensor_buffer.get_data()); + char* output_data = static_cast(node.output_buffer.get_data()); + + for (size_t flat_output = 0; flat_output < total; ++flat_output) { + size_t remainder = flat_output; + size_t input_offset = 0; + + const size_t gathered_index = read_index(flat_output); + if (gathered_index >= tensor_buffer.shape[static_cast(axis)]) { + throw std::runtime_error( + "Gather index " + std::to_string(gathered_index) + + " out of bounds for dimension " + std::to_string(tensor_buffer.shape[static_cast(axis)]) + ); + } + + remainder = flat_output; + for (size_t dim = 0; dim < rank; ++dim) { + const size_t coord = remainder / output_strides[dim]; + remainder %= output_strides[dim]; + const size_t input_coord = (static_cast(dim) == axis) ? gathered_index : coord; + input_offset += input_coord * input_strides[dim]; + } + + std::memcpy( + output_data + flat_output * bytes_per_scalar, + input_data + PrecisionTraits::byte_offset_of(tensor_buffer.precision, input_offset), + bytes_per_scalar + ); + } + return; + } + + size_t first_dim = tensor_buffer.shape[0]; + size_t element_size = 1; + for (size_t i = 1; i < tensor_buffer.shape.size(); i++) { + element_size *= tensor_buffer.shape[i]; + } + + size_t num_indices = indices_buffer.total_size; + size_t bytes_per_element = PrecisionTraits::packed_size_of(tensor_buffer.precision, element_size); + + if (PrecisionTraits::is_integer(tensor_buffer.precision)) { + const char* tensor_data = static_cast(tensor_buffer.get_data()); + char* output = static_cast(node.output_buffer.get_data()); + Precision prec = tensor_buffer.precision; + + const bool is_grouped = tensor_buffer.group_size > 0; + __fp16* gathered_scales = nullptr; + const __fp16* src_scales = nullptr; + size_t num_groups = 0; + + if (is_grouped) { + num_groups = tensor_buffer.num_groups; + src_scales = tensor_buffer.scales_as_fp16(); + size_t scales_bytes = num_indices * num_groups * sizeof(__fp16); + node.output_buffer.owned_scales = std::make_unique(scales_bytes); + gathered_scales = reinterpret_cast<__fp16*>(node.output_buffer.owned_scales.get()); + } + + const int8_t* indices = indices_buffer.data_as(); + for (size_t i = 0; i < num_indices; i++) { + size_t idx = static_cast(indices[i]); + if (idx >= first_dim) { + throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); + } + std::memcpy(output + PrecisionTraits::byte_offset_of(prec, i * element_size), + tensor_data + PrecisionTraits::byte_offset_of(prec, idx * element_size), + bytes_per_element); + if (is_grouped) { + for (size_t g = 0; g < num_groups; g++) { + gathered_scales[i * num_groups + g] = src_scales[idx * num_groups + g]; + } + } + } + + if (is_grouped) { + node.output_buffer.group_size = tensor_buffer.group_size; + node.output_buffer.num_groups = num_groups; + node.output_buffer.scales_data = gathered_scales; + } + } else if (tensor_buffer.precision == Precision::FP16) { + const __fp16* tensor_data = tensor_buffer.data_as<__fp16>(); + __fp16* output = node.output_buffer.data_as<__fp16>(); + + if (indices_buffer.precision == Precision::INT8) { + const int8_t* indices = indices_buffer.data_as(); + for (size_t i = 0; i < num_indices; i++) { + size_t idx = static_cast(indices[i]); + if (idx >= first_dim) { + throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); + } + std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); + } + } else { + const float* indices = indices_buffer.data_as(); + for (size_t i = 0; i < num_indices; i++) { + size_t idx = static_cast(indices[i]); + if (idx >= first_dim) { + throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); + } + std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); + } + } + } else { + const float* tensor_data = tensor_buffer.data_as(); + float* output = node.output_buffer.data_as(); + + if (indices_buffer.precision == Precision::INT8) { + const int8_t* indices = indices_buffer.data_as(); + for (size_t i = 0; i < num_indices; i++) { + size_t idx = static_cast(indices[i]); + if (idx >= first_dim) { + throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); + } + std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); + } + } else { + const float* indices = indices_buffer.data_as(); + for (size_t i = 0; i < num_indices; i++) { + size_t idx = static_cast(indices[i]); + if (idx >= first_dim) { + throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); + } + std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); + } + } + } +} + +void compute_slice_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + auto* input_node = nodes[node_index_map.at(node.input_ids[0])].get(); + auto& input_buffer = input_node->output_buffer; + + const size_t axis_index = static_cast(node.params.axis); + + const size_t axis_size = input_buffer.shape[axis_index]; + const size_t slice_start = node.params.slice_start; + size_t slice_length = node.params.slice_length; + + if (slice_length == 0) { + slice_length = axis_size - slice_start; + } + + if (axis_index == 0) { + auto dims0 = AxisDims::from_shape(input_buffer.shape, 0); + + auto* base_ptr = static_cast(input_buffer.get_data()); + if (!base_ptr) { + throw std::runtime_error("Slice input buffer is not available"); + } + + const size_t byte_offset = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims0.inner); + + node.output_buffer.set_external(base_ptr + byte_offset); + node.output_buffer.precision = input_buffer.precision; + + if (input_buffer.is_grouped_int8()) { + size_t num_groups = input_buffer.num_groups; + size_t scales_bytes = slice_length * num_groups * sizeof(__fp16); + node.output_buffer.owned_scales = std::make_unique(scales_bytes); + __fp16* sliced_scales = reinterpret_cast<__fp16*>(node.output_buffer.owned_scales.get()); + const __fp16* input_scales = input_buffer.scales_as_fp16(); + + for (size_t i = 0; i < slice_length; i++) { + for (size_t g = 0; g < num_groups; g++) { + sliced_scales[i * num_groups + g] = input_scales[(slice_start + i) * num_groups + g]; + } + } + + node.output_buffer.group_size = input_buffer.group_size; + node.output_buffer.num_groups = num_groups; + node.output_buffer.scales_data = sliced_scales; + } + return; + } + + const char* input_ptr = static_cast(input_buffer.get_data()); + if (!input_ptr) { + throw std::runtime_error("Slice input buffer is not available"); + } + + auto dims = AxisDims::from_shape(input_buffer.shape, axis_index); + + node.output_buffer.external_data = nullptr; + node.output_buffer.allocate(); + node.output_buffer.precision = input_buffer.precision; + + auto* output_ptr = static_cast(node.output_buffer.get_data()); + if (!output_ptr) { + throw std::runtime_error("Slice output buffer could not be allocated"); + } + + const size_t copy_block_elements = slice_length * dims.inner; + const size_t axis_stride_elements = axis_size * dims.inner; + const size_t copy_block_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, copy_block_elements); + const size_t axis_stride_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, axis_stride_elements); + + for (size_t outer = 0; outer < dims.outer; ++outer) { + const char* src = input_ptr + outer * axis_stride_bytes + PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims.inner); + char* dst = output_ptr + outer * copy_block_bytes; + std::memcpy(dst, src, copy_block_bytes); + } +} + +void compute_embedding_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& embeddings_buffer = get_input(node, 0, nodes, node_index_map); + const auto& indices_buffer = get_input(node, 1, nodes, node_index_map); + + size_t hidden_dim = embeddings_buffer.shape[1]; + size_t num_indices = indices_buffer.total_size; + size_t vocab_size = embeddings_buffer.original_N > 0 + ? embeddings_buffer.original_N + : embeddings_buffer.shape[0]; + + std::vector indices_float; + const float* indices_ptr; + if (indices_buffer.precision == Precision::FP32) { + indices_ptr = indices_buffer.data_as(); + } else { + indices_float.resize(num_indices); + const int8_t* int_indices = indices_buffer.data_as(); + for (size_t i = 0; i < num_indices; i++) { + indices_float[i] = static_cast(int_indices[i]); + } + indices_ptr = indices_float.data(); + } + + __fp16* output = node.output_buffer.data_as<__fp16>(); + + Precision emb_prec = embeddings_buffer.precision; + if (PrecisionTraits::is_integer(emb_prec) && embeddings_buffer.group_size > 0) { + const int8_t* embeddings = embeddings_buffer.data_as(); + const __fp16* scales = embeddings_buffer.scales_as_fp16(); + size_t group_size = embeddings_buffer.group_size; + size_t num_groups = embeddings_buffer.num_groups; + + static const uint8_t gather_indices[4][16] = { + {0, 1, 2, 3, 16, 17, 18, 19, 32, 33, 34, 35, 48, 49, 50, 51}, // lane 0 + {4, 5, 6, 7, 20, 21, 22, 23, 36, 37, 38, 39, 52, 53, 54, 55}, // lane 1 + {8, 9, 10, 11, 24, 25, 26, 27, 40, 41, 42, 43, 56, 57, 58, 59}, // lane 2 + {12, 13, 14, 15, 28, 29, 30, 31, 44, 45, 46, 47, 60, 61, 62, 63} // lane 3 + }; + + auto load_table = [emb_prec](const int8_t* base) -> int8x16x4_t { + if (emb_prec == Precision::INT4) { + const uint8_t* ubase = reinterpret_cast(base); + int8x16_t low_a, high_a, low_b, high_b; + unpack_int4_as_int8x16x2(ubase, high_a, low_a); + unpack_int4_as_int8x16x2(ubase + 16, high_b, low_b); + return {low_a, high_a, low_b, high_b}; + } + return vld1q_s8_x4(base); + }; + + for (size_t i = 0; i < num_indices; i++) { + size_t idx = static_cast(indices_ptr[i]); + if (idx >= vocab_size) { + throw std::runtime_error("Embedding index out of bounds: " + std::to_string(idx) + " >= " + std::to_string(vocab_size)); + } + + size_t block = idx / 4; + size_t lane = idx % 4; + __fp16* out_row = output + i * hidden_dim; + + uint8x16_t indices_vec = vld1q_u8(gather_indices[lane]); + + for (size_t g = 0; g < num_groups; g++) { + float scale = (float)scales[(block * num_groups + g) * 4 + lane]; + float32x4_t scale_vec = vdupq_n_f32(scale); + + size_t k_start = g * group_size; + size_t k_end = std::min(k_start + group_size, hidden_dim); + + const int8_t* group_base = embeddings + PrecisionTraits::byte_offset_of(emb_prec, (block * hidden_dim + k_start) * 4); + + size_t k = k_start; + for (; k + 16 <= k_end; k += 16) { + const int8_t* chunk_base = group_base + PrecisionTraits::byte_offset_of(emb_prec, (k - k_start) * 4); + int8x16x4_t table = load_table(chunk_base); + + int8x16_t values = vqtbl4q_s8(table, indices_vec); + + int16x8_t lo16 = vmovl_s8(vget_low_s8(values)); + int16x8_t hi16 = vmovl_s8(vget_high_s8(values)); + + float32x4_t f0 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(lo16))), scale_vec); + float32x4_t f1 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(lo16))), scale_vec); + float32x4_t f2 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(hi16))), scale_vec); + float32x4_t f3 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(hi16))), scale_vec); + + vst1q_f16(out_row + k, vcombine_f16(vcvt_f16_f32(f0), vcvt_f16_f32(f1))); + vst1q_f16(out_row + k + 8, vcombine_f16(vcvt_f16_f32(f2), vcvt_f16_f32(f3))); + } + + if (k < k_end) + assert (embeddings_buffer.precision != Precision::INT4 && "grouped INT4 embeddings must have hidden_dim that is a multiple of 32"); + for (; k < k_end; k++) { + size_t k_group = k / 4; + size_t k_within = k % 4; + int8_t val = embeddings[(block * (hidden_dim / 4) + k_group) * 16 + lane * 4 + k_within]; + out_row[k] = static_cast<__fp16>(val * scale); + } + } + } + } else if (embeddings_buffer.precision == Precision::FP16) { + const __fp16* embeddings = embeddings_buffer.data_as<__fp16>(); + for (size_t i = 0; i < num_indices; i++) { + size_t idx = static_cast(indices_ptr[i]); + if (idx >= vocab_size) { + throw std::runtime_error("Embedding index out of bounds: " + std::to_string(idx) + " >= " + std::to_string(vocab_size)); + } + std::memcpy(output + i * hidden_dim, embeddings + idx * hidden_dim, hidden_dim * sizeof(__fp16)); + } + } else { + throw std::runtime_error("Embedding requires interleaved grouped INT4/INT8 or FP16"); + } +} + +static size_t product_range(const std::vector& shape, size_t start, size_t end) { + size_t product = 1; + for (size_t i = start; i < end; ++i) { + product *= shape[i]; + } + return product; +} + +static void concat_same_precision_tensors( + const std::vector& inputs, + BufferDesc& output, + int axis +) { + if (inputs.empty()) { + throw std::runtime_error("concat requires at least one input tensor"); + } + if (axis < 0) { + throw std::runtime_error("concat does not support negative axis"); + } + + const auto& first = *inputs.front(); + if (first.precision == Precision::INT4 || first.is_grouped_int8()) { + throw std::runtime_error("concat does not support packed/grouped integer tensors"); + } + + const size_t rank = first.shape.size(); + if (static_cast(axis) >= rank) { + throw std::runtime_error("concat axis out of range"); + } + + const size_t element_bytes = PrecisionTraits::size_of(first.precision); + const size_t outer = product_range(first.shape, 0, static_cast(axis)); + const size_t inner = product_range(first.shape, static_cast(axis) + 1, rank); + const size_t output_axis = output.shape[static_cast(axis)]; + char* output_ptr = static_cast(output.get_data()); + + for (size_t outer_idx = 0; outer_idx < outer; ++outer_idx) { + size_t output_axis_offset = 0; + for (const BufferDesc* input : inputs) { + if (input->precision != first.precision) { + throw std::runtime_error("concat requires all input precisions to match"); + } + if (input->shape.size() != rank) { + throw std::runtime_error("concat requires all input ranks to match"); + } + if (input->precision == Precision::INT4 || input->is_grouped_int8()) { + throw std::runtime_error("concat does not support packed/grouped integer tensors"); + } + + const size_t axis_size = input->shape[static_cast(axis)]; + const size_t chunk_elements = axis_size * inner; + const size_t output_base = ((outer_idx * output_axis) + output_axis_offset) * inner * element_bytes; + const size_t input_base = outer_idx * chunk_elements * element_bytes; + std::memcpy( + output_ptr + output_base, + static_cast(input->get_data()) + input_base, + chunk_elements * element_bytes + ); + output_axis_offset += axis_size; + } + } +} + +void compute_concat_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input1_buffer = get_input(node, 0, nodes, node_index_map); + const auto& input2_buffer = get_input(node, 1, nodes, node_index_map); + std::vector inputs = {&input1_buffer, &input2_buffer}; + concat_same_precision_tensors(inputs, node.output_buffer, node.params.axis); +} + +void compute_cat_node( + GraphNode& node, + const std::vector>& nodes, + const std::unordered_map& node_index_map +) { + if (node.params.axis < 0) { + throw std::runtime_error("Cat operation does not support negative axis"); + } + if (node.input_ids.size() < 2) { + throw std::runtime_error("Cat operation requires at least 2 input tensors"); + } + + std::vector inputs; + inputs.reserve(node.input_ids.size()); + + for (size_t i = 0; i < node.input_ids.size(); i++) { + const auto& buffer = get_input(node, i, nodes, node_index_map); + inputs.push_back(&buffer); + } + concat_same_precision_tensors(inputs, node.output_buffer, node.params.axis); +} + +void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& input_shape = input_buffer.shape; + + int dim = node.params.axis; + size_t index_value = node.params.index_value; + + const char* input_data = static_cast(input_buffer.get_data()); + char* output_data = static_cast(node.output_buffer.get_data()); + + if (dim == 0) { + size_t slice_size = input_buffer.total_size / input_shape[0]; + size_t offset_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, index_value * slice_size); + node.output_buffer.set_external(const_cast(input_data) + offset_bytes); + return; + } + + std::vector input_strides(input_shape.size()); + input_strides[input_shape.size() - 1] = 1; + for (int i = static_cast(input_shape.size()) - 2; i >= 0; --i) { + input_strides[i] = input_strides[i + 1] * input_shape[i + 1]; + } + + size_t slice_size = input_strides[dim]; + size_t outer_size = input_buffer.total_size / input_strides[dim - 1]; + size_t dim_stride = input_strides[dim]; + size_t block_size = dim_stride * input_shape[dim]; + + size_t output_idx = 0; + for (size_t outer_idx = 0; outer_idx < outer_size; ++outer_idx) { + size_t input_base = outer_idx * block_size + index_value * dim_stride; + + char* output_offset_bytes = output_data + PrecisionTraits::byte_offset_of(input_buffer.precision, output_idx); + const char* input_offset_bytes = input_data + PrecisionTraits::byte_offset_of(input_buffer.precision, input_base); + size_t length = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_size); + std::memcpy(output_offset_bytes, input_offset_bytes, length); + + output_idx += slice_size; + } +} + +static bool mask_element_is_true(const BufferDesc& mask_buffer, size_t index) { + switch (mask_buffer.precision) { + case Precision::FP16: + return static_cast(mask_buffer.data_as<__fp16>()[index]) != 0.0f; + case Precision::FP32: + return mask_buffer.data_as()[index] != 0.0f; + case Precision::INT8: + return mask_buffer.data_as()[index] != 0; + default: + throw std::runtime_error("Unsupported mask precision for masked select/scatter"); + } +} + +void compute_masked_select_prefix_node( + GraphNode& node, + const std::vector>& nodes, + const std::unordered_map& node_index_map +) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& mask_buffer = get_input(node, 1, nodes, node_index_map); + + if (input_buffer.precision == Precision::INT4 || input_buffer.is_grouped_int8()) { + throw std::runtime_error("masked_select_prefix does not support packed/grouped integer tensors"); + } + if (mask_buffer.shape.empty()) { + throw std::runtime_error("masked_select_prefix requires non-scalar mask"); + } + if (input_buffer.shape.size() < mask_buffer.shape.size()) { + throw std::runtime_error("masked_select_prefix mask rank must not exceed input rank"); + } + for (size_t i = 0; i < mask_buffer.shape.size(); ++i) { + if (input_buffer.shape[i] != mask_buffer.shape[i]) { + throw std::runtime_error("masked_select_prefix mask shape must match the input prefix shape"); + } + } + + const size_t prefix_elems = mask_buffer.total_size; + const size_t trailing_elems = input_buffer.total_size / prefix_elems; + const size_t block_bytes = trailing_elems * PrecisionTraits::size_of(input_buffer.precision); + const char* input_ptr = static_cast(input_buffer.get_data()); + char* output_ptr = static_cast(node.output_buffer.get_data()); + std::memset(output_ptr, 0, node.output_buffer.byte_size); + + size_t packed_row = 0; + for (size_t prefix_idx = 0; prefix_idx < prefix_elems; ++prefix_idx) { + if (!mask_element_is_true(mask_buffer, prefix_idx)) { + continue; + } + std::memcpy( + output_ptr + (packed_row * block_bytes), + input_ptr + (prefix_idx * block_bytes), + block_bytes + ); + packed_row++; + } +} + +void compute_masked_scatter_node( + GraphNode& node, + const std::vector>& nodes, + const std::unordered_map& node_index_map +) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& mask_buffer = get_input(node, 1, nodes, node_index_map); + const auto& source_buffer = get_input(node, 2, nodes, node_index_map); + + if (input_buffer.precision == Precision::INT4 || source_buffer.precision == Precision::INT4) { + throw std::runtime_error("masked_scatter does not support INT4 tensors"); + } + if (input_buffer.shape != mask_buffer.shape) { + throw std::runtime_error("masked_scatter mask shape must match input shape"); + } + if (input_buffer.precision != source_buffer.precision) { + throw std::runtime_error("masked_scatter source precision must match input precision"); + } + + const size_t element_bytes = PrecisionTraits::size_of(input_buffer.precision); + const char* input_ptr = static_cast(input_buffer.get_data()); + const char* source_ptr = static_cast(source_buffer.get_data()); + char* output_ptr = static_cast(node.output_buffer.get_data()); + std::memcpy(output_ptr, input_ptr, node.output_buffer.byte_size); + + size_t source_index = 0; + for (size_t element_idx = 0; element_idx < mask_buffer.total_size; ++element_idx) { + if (!mask_element_is_true(mask_buffer, element_idx)) { + continue; + } + if (source_index >= source_buffer.total_size) { + throw std::runtime_error("masked_scatter source does not contain enough elements"); + } + std::memcpy( + output_ptr + (element_idx * element_bytes), + source_ptr + (source_index * element_bytes), + element_bytes + ); + source_index++; + } +} + +void compute_expand_node( + GraphNode& node, + const std::vector>& nodes, + const std::unordered_map& node_index_map +) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + const auto& input_shape = input_buffer.shape; + const auto& output_shape = node.output_buffer.shape; + + if (input_buffer.precision == Precision::INT4 || input_buffer.is_grouped_int8()) { + throw std::runtime_error("expand does not support packed/grouped integer tensors"); + } + if (output_shape.size() < input_shape.size()) { + throw std::runtime_error("expand cannot reduce rank"); + } + + const size_t rank_delta = output_shape.size() - input_shape.size(); + std::vector padded_input_shape(output_shape.size(), 1); + for (size_t axis = 0; axis < input_shape.size(); ++axis) { + padded_input_shape[rank_delta + axis] = input_shape[axis]; + } + for (size_t axis = 0; axis < output_shape.size(); ++axis) { + if (padded_input_shape[axis] != 1 && padded_input_shape[axis] != output_shape[axis]) { + throw std::runtime_error("expand target shape is not broadcast-compatible with input"); + } + } + + std::vector input_strides(output_shape.size(), 1); + std::vector output_strides(output_shape.size(), 1); + for (size_t axis = output_shape.size(); axis-- > 1;) { + input_strides[axis - 1] = input_strides[axis] * padded_input_shape[axis]; + output_strides[axis - 1] = output_strides[axis] * output_shape[axis]; + } + + const size_t element_bytes = PrecisionTraits::size_of(input_buffer.precision); + const char* input_ptr = static_cast(input_buffer.get_data()); + char* output_ptr = static_cast(node.output_buffer.get_data()); + for (size_t output_index = 0; output_index < node.output_buffer.total_size; ++output_index) { + size_t remainder = output_index; + size_t input_index = 0; + for (size_t axis = 0; axis < output_shape.size(); ++axis) { + const size_t coord = remainder / output_strides[axis]; + remainder %= output_strides[axis]; + const size_t input_coord = padded_input_shape[axis] == 1 ? 0 : coord; + input_index += input_coord * input_strides[axis]; + } + std::memcpy( + output_ptr + (output_index * element_bytes), + input_ptr + (input_index * element_bytes), + element_bytes + ); + } +} + +void compute_cumsum_node( + GraphNode& node, + const std::vector>& nodes, + const std::unordered_map& node_index_map +) { + const auto& input_buffer = get_input(node, 0, nodes, node_index_map); + + if (input_buffer.precision != Precision::FP16 && input_buffer.precision != Precision::FP32) { + throw std::runtime_error("cumsum only supports FP16 and FP32 precision"); + } + + const int axis = node.params.axis; + if (axis < 0 || static_cast(axis) >= input_buffer.shape.size()) { + throw std::runtime_error("cumsum axis out of range"); + } + + auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); + if (input_buffer.precision == Precision::FP32) { + cactus_cumsum_axis_f32( + input_buffer.data_as(), + node.output_buffer.data_as(), + dims.outer, + dims.axis_size, + dims.inner + ); + return; + } + + cactus_cumsum_axis_f16( + input_buffer.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + dims.outer, + dims.axis_size, + dims.inner + ); +} + +void compute_bilinear_interpolation_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + const auto& pos_embeds_buffer = get_input(node, 0, nodes, node_index_map); + + size_t total_pos_embeds = pos_embeds_buffer.shape[0]; + size_t embed_dim = pos_embeds_buffer.shape[1]; + + size_t src_height = static_cast(std::sqrt(total_pos_embeds)); + size_t src_width = src_height; + + size_t dst_height = node.params.dst_height; + size_t dst_width = node.params.dst_width; + bool align_corners = node.params.align_corners; + + __fp16* output = node.output_buffer.data_as<__fp16>(); + + if (pos_embeds_buffer.precision == Precision::FP16) { + const __fp16* input = pos_embeds_buffer.data_as<__fp16>(); + cactus_bilinear_interpolation_f16(input, output, src_height, src_width, embed_dim, + dst_height, dst_width, align_corners); + } + else if (pos_embeds_buffer.precision == Precision::INT8) { + std::vector<__fp16> input_fp16(total_pos_embeds * embed_dim); + cactus_int8_to_fp16(pos_embeds_buffer.data_as(), input_fp16.data(), + total_pos_embeds * embed_dim); + cactus_bilinear_interpolation_f16(input_fp16.data(), output, src_height, src_width, embed_dim, + dst_height, dst_width, align_corners); + } + else { + throw std::runtime_error("BILINEAR_INTERPOLATION only supports INT8 and FP16 input precision"); + } +} + +void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { + if (node.input_ids.empty()) { + return; + } + + auto it = node_index_map.find(node.input_ids[0]); + + if (it != node_index_map.end()) { + const auto& input_buffer = nodes[it->second]->output_buffer; + + if (!node.output_buffer.get_data()) { + node.output_buffer.allocate(); + } + + std::memcpy(node.output_buffer.get_data(), + input_buffer.get_data(), + input_buffer.byte_size); + } else { + if (node.output_buffer.get_data()) { + return; + } + throw std::runtime_error("PERSISTENT node input not found and not populated - this should not happen"); + } +} diff --git a/cactus-transpile/graph/graph_param_io.cpp b/cactus-transpile/graph/graph_param_io.cpp new file mode 100644 index 000000000..64af5ca19 --- /dev/null +++ b/cactus-transpile/graph/graph_param_io.cpp @@ -0,0 +1,394 @@ +#include "graph_param_io.h" + +#include +#include +#include +#include + +namespace { + +void write_u32(std::ostream& out, uint32_t v) { + out.write(reinterpret_cast(&v), sizeof(v)); +} + +void write_u64(std::ostream& out, uint64_t v) { + out.write(reinterpret_cast(&v), sizeof(v)); +} + +void write_i32(std::ostream& out, int32_t v) { + out.write(reinterpret_cast(&v), sizeof(v)); +} + +void write_f32(std::ostream& out, float v) { + out.write(reinterpret_cast(&v), sizeof(v)); +} + +uint32_t read_u32(std::istream& in) { + uint32_t v = 0; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) throw std::runtime_error("Unexpected EOF while reading uint32"); + return v; +} + +uint64_t read_u64(std::istream& in) { + uint64_t v = 0; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) throw std::runtime_error("Unexpected EOF while reading uint64"); + return v; +} + +int32_t read_i32(std::istream& in) { + int32_t v = 0; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) throw std::runtime_error("Unexpected EOF while reading int32"); + return v; +} + +float read_f32(std::istream& in) { + float v = 0.0f; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) throw std::runtime_error("Unexpected EOF while reading float"); + return v; +} + +void write_size_vector(std::ostream& out, const std::vector& values) { + write_u32(out, static_cast(values.size())); + for (size_t v : values) write_u64(out, static_cast(v)); +} + +std::vector read_size_vector(std::istream& in) { + uint32_t count = read_u32(in); + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; ++i) values.push_back(static_cast(read_u64(in))); + return values; +} + +void write_u32_vector(std::ostream& out, const std::vector& values) { + write_u32(out, static_cast(values.size())); + for (uint32_t v : values) write_u32(out, v); +} + +std::vector read_u32_vector(std::istream& in) { + uint32_t count = read_u32(in); + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; ++i) values.push_back(read_u32(in)); + return values; +} + +void write_f32_vector(std::ostream& out, const std::vector& values) { + write_u32(out, static_cast(values.size())); + for (float v : values) write_f32(out, v); +} + +std::vector read_f32_vector(std::istream& in) { + uint32_t count = read_u32(in); + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; ++i) values.push_back(read_f32(in)); + return values; +} + +enum class ParamField : uint32_t { + Scalar = 1, + Axis, + NewShape, + PretransposedRhs, + Backend, + SliceStart, + SliceLength, + Epsilon, + NumGroups, + IndexValue, + Permutation, + Scale, + Theta, + PositionOffset, + WindowSize, + IsCausal, + AttentionMaskIsAdditive, + LogitCap, + TopK, + Temperature, + TopP, + MinP, + RepetitionPenalty, + RandomSeed, + BiasIndices, + BiasValues, + NormalizeRouting, + NumExperts, + NumExpertsPerTok, + MoeGated, + Activation, + NumKvHeads, + ChunkSize, + DstHeight, + DstWidth, + AlignCorners, + NumClasses, + NumAltupInputs, + KernelSize, + Stride, + Dilation, + NumFftBins, + CacheSeqLen, + HeadDim, + VHeadDim, + CachedKeysInt8Ptr, + CachedValuesInt8Ptr, + CachedKScalesPtr, + CachedVScalesPtr, + Padding, +}; + +enum class FieldPersistence { + Persistent, + RuntimeOnly, + Derived, +}; + +struct FieldSpec { + ParamField field; + FieldPersistence persistence; +}; + +using Schema = std::vector; + +const Schema& op_schema(OpType op_type) { + static const Schema kEmpty{}; + static const std::unordered_map kSchemas = { + {OpType::POW, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_ADD, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_SUBTRACT, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_MULTIPLY, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_DIVIDE, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_NOT_EQUAL, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_FLOOR_DIVIDE, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SOFTMAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::SUM, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::MEAN, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::VARIANCE, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::MIN, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::MAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::CUMSUM, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::INDEX, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::IndexValue, FieldPersistence::Persistent}}}, + {OpType::GATHER, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::CONCAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::CAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::VIEW, {{ParamField::NewShape, FieldPersistence::Persistent}}}, + {OpType::RESHAPE, {{ParamField::NewShape, FieldPersistence::Persistent}}}, + {OpType::FLATTEN, {{ParamField::NewShape, FieldPersistence::Persistent}}}, + {OpType::MATMUL, {{ParamField::PretransposedRhs, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::TRANSPOSE, {{ParamField::Permutation, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::SLICE, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::SliceStart, FieldPersistence::Persistent}, {ParamField::SliceLength, FieldPersistence::Persistent}}}, + {OpType::RMS_NORM, {{ParamField::Epsilon, FieldPersistence::Persistent}}}, + {OpType::LAYERNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}}}, + {OpType::GROUPNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::NumGroups, FieldPersistence::Persistent}}}, + {OpType::BATCHNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::ROPE, {{ParamField::Theta, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::ROPE_GPTJ, {{ParamField::Theta, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::Scalar, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::TOPK, {{ParamField::TopK, FieldPersistence::Persistent}}}, + {OpType::ATTENTION, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::WindowSize, FieldPersistence::Persistent}, {ParamField::IsCausal, FieldPersistence::Persistent}, {ParamField::AttentionMaskIsAdditive, FieldPersistence::Persistent}, {ParamField::LogitCap, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::REL_POS_BIAS, {{ParamField::Scale, FieldPersistence::Persistent}}}, + {OpType::ATTENTION_INT8_HYBRID, { + {ParamField::Scale, FieldPersistence::Persistent}, + {ParamField::PositionOffset, FieldPersistence::Persistent}, + {ParamField::WindowSize, FieldPersistence::Persistent}, + {ParamField::NumKvHeads, FieldPersistence::Persistent}, + {ParamField::CacheSeqLen, FieldPersistence::Persistent}, + {ParamField::HeadDim, FieldPersistence::Persistent}, + {ParamField::VHeadDim, FieldPersistence::Persistent}, + {ParamField::CachedKeysInt8Ptr, FieldPersistence::RuntimeOnly}, + {ParamField::CachedValuesInt8Ptr, FieldPersistence::RuntimeOnly}, + {ParamField::CachedKScalesPtr, FieldPersistence::RuntimeOnly}, + {ParamField::CachedVScalesPtr, FieldPersistence::RuntimeOnly}, + }}, + {OpType::MOE_LAYER, {{ParamField::Scalar, FieldPersistence::Persistent}, {ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::NormalizeRouting, FieldPersistence::Persistent}, {ParamField::NumExperts, FieldPersistence::Persistent}, {ParamField::NumExpertsPerTok, FieldPersistence::Persistent}, {ParamField::MoeGated, FieldPersistence::Persistent}, {ParamField::Activation, FieldPersistence::Persistent}}}, + {OpType::GATED_DELTANET_DECODE, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::NumKvHeads, FieldPersistence::Persistent}}}, + {OpType::GATED_DELTANET_PREFILL, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::NumKvHeads, FieldPersistence::Persistent}, {ParamField::ChunkSize, FieldPersistence::Persistent}}}, + {OpType::STFT, {{ParamField::Stride, FieldPersistence::Persistent}, {ParamField::NumFftBins, FieldPersistence::Persistent}}}, + {OpType::BILINEAR_INTERPOLATION, {{ParamField::DstHeight, FieldPersistence::Persistent}, {ParamField::DstWidth, FieldPersistence::Persistent}, {ParamField::AlignCorners, FieldPersistence::Persistent}}}, + {OpType::SCATTER_TOPK, {{ParamField::NumClasses, FieldPersistence::Persistent}}}, + {OpType::ALTUP_PREDICT, {{ParamField::NumAltupInputs, FieldPersistence::Persistent}}}, + {OpType::ALTUP_CORRECT, {{ParamField::NumAltupInputs, FieldPersistence::Persistent}}}, + {OpType::MAXPOOL1D, {{ParamField::KernelSize, FieldPersistence::Persistent}, {ParamField::Stride, FieldPersistence::Persistent}}}, + {OpType::CONV1D_CAUSAL, {{ParamField::Dilation, FieldPersistence::Persistent}}}, + {OpType::CONV1D_K3, {{ParamField::Stride, FieldPersistence::Persistent}}}, + {OpType::CONV1D_K7S3, {{ParamField::Stride, FieldPersistence::Persistent}}}, + {OpType::CONV1D, {{ParamField::Stride, FieldPersistence::Persistent}}}, + {OpType::GLU, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::CONV2D, { + {ParamField::Stride, FieldPersistence::Persistent}, + {ParamField::Padding, FieldPersistence::Persistent}, + {ParamField::Dilation, FieldPersistence::Persistent}, + {ParamField::NumGroups, FieldPersistence::Persistent}, + }}, + {OpType::SAMPLE, {{ParamField::Temperature, FieldPersistence::Persistent}, {ParamField::TopP, FieldPersistence::Persistent}, {ParamField::MinP, FieldPersistence::Persistent}, {ParamField::RepetitionPenalty, FieldPersistence::Persistent}, {ParamField::TopK, FieldPersistence::Persistent}, {ParamField::RandomSeed, FieldPersistence::Persistent}, {ParamField::BiasIndices, FieldPersistence::Persistent}, {ParamField::BiasValues, FieldPersistence::Persistent}}}, + }; + + auto it = kSchemas.find(op_type); + return it == kSchemas.end() ? kEmpty : it->second; +} + +void write_field(std::ostream& out, ParamField field, const OpParams& params) { + switch (field) { + case ParamField::Scalar: write_f32(out, params.scalar); break; + case ParamField::Axis: write_i32(out, static_cast(params.axis)); break; + case ParamField::NewShape: write_size_vector(out, params.new_shape); break; + case ParamField::PretransposedRhs: write_u32(out, params.pretransposed_rhs ? 1u : 0u); break; + case ParamField::Backend: write_u32(out, static_cast(params.backend)); break; + case ParamField::SliceStart: write_u64(out, static_cast(params.slice_start)); break; + case ParamField::SliceLength: write_u64(out, static_cast(params.slice_length)); break; + case ParamField::Epsilon: write_f32(out, params.epsilon); break; + case ParamField::NumGroups: write_u64(out, static_cast(params.num_groups)); break; + case ParamField::IndexValue: write_u64(out, static_cast(params.index_value)); break; + case ParamField::Permutation: write_size_vector(out, params.permutation); break; + case ParamField::Scale: write_f32(out, params.scale); break; + case ParamField::Theta: write_f32(out, params.theta); break; + case ParamField::PositionOffset: write_u64(out, static_cast(params.position_offset)); break; + case ParamField::WindowSize: write_u64(out, static_cast(params.window_size)); break; + case ParamField::IsCausal: write_u32(out, params.is_causal ? 1u : 0u); break; + case ParamField::AttentionMaskIsAdditive: write_u32(out, params.attention_mask_is_additive ? 1u : 0u); break; + case ParamField::LogitCap: write_f32(out, params.logit_cap); break; + case ParamField::TopK: write_u64(out, static_cast(params.top_k)); break; + case ParamField::Temperature: write_f32(out, params.temperature); break; + case ParamField::TopP: write_f32(out, params.top_p); break; + case ParamField::MinP: write_f32(out, params.min_p); break; + case ParamField::RepetitionPenalty: write_f32(out, params.repetition_penalty); break; + case ParamField::RandomSeed: write_u64(out, static_cast(params.random_seed)); break; + case ParamField::BiasIndices: write_u32_vector(out, params.bias_indices); break; + case ParamField::BiasValues: write_f32_vector(out, params.bias_values); break; + case ParamField::NormalizeRouting: write_u32(out, params.normalize_routing ? 1u : 0u); break; + case ParamField::NumExperts: write_u64(out, static_cast(params.num_experts)); break; + case ParamField::NumExpertsPerTok: write_u64(out, static_cast(params.num_experts_per_tok)); break; + case ParamField::MoeGated: write_u32(out, params.moe_gated ? 1u : 0u); break; + case ParamField::Activation: write_u32(out, static_cast(params.activation)); break; + case ParamField::NumKvHeads: write_u64(out, static_cast(params.num_kv_heads)); break; + case ParamField::ChunkSize: write_u64(out, static_cast(params.chunk_size)); break; + case ParamField::DstHeight: write_u64(out, static_cast(params.dst_height)); break; + case ParamField::DstWidth: write_u64(out, static_cast(params.dst_width)); break; + case ParamField::AlignCorners: write_u32(out, params.align_corners ? 1u : 0u); break; + case ParamField::NumClasses: write_u64(out, static_cast(params.num_classes)); break; + case ParamField::NumAltupInputs: write_u64(out, static_cast(params.num_altup_inputs)); break; + case ParamField::KernelSize: write_u64(out, static_cast(params.kernel_size)); break; + case ParamField::Stride: write_u64(out, static_cast(params.stride)); break; + case ParamField::Dilation: write_u64(out, static_cast(params.dilation)); break; + case ParamField::NumFftBins: write_u64(out, static_cast(params.num_fft_bins)); break; + case ParamField::CacheSeqLen: write_u64(out, static_cast(params.cache_seq_len)); break; + case ParamField::HeadDim: write_u64(out, static_cast(params.head_dim)); break; + case ParamField::VHeadDim: write_u64(out, static_cast(params.v_head_dim)); break; + case ParamField::CachedKeysInt8Ptr: + case ParamField::CachedValuesInt8Ptr: + case ParamField::CachedKScalesPtr: + case ParamField::CachedVScalesPtr: + throw std::runtime_error("Attempted to serialize runtime-only pointer field"); + case ParamField::Padding: write_u64(out, static_cast(params.padding)); break; + } +} + +void read_field(std::istream& in, ParamField field, OpParams& params) { + switch (field) { + case ParamField::Scalar: params.scalar = read_f32(in); break; + case ParamField::Axis: params.axis = static_cast(read_i32(in)); break; + case ParamField::NewShape: params.new_shape = read_size_vector(in); break; + case ParamField::PretransposedRhs: params.pretransposed_rhs = (read_u32(in) != 0); break; + case ParamField::Backend: { + uint32_t backend_val = read_u32(in); + if (backend_val > static_cast(ComputeBackend::NPU)) { + throw std::runtime_error("Graph file corrupted: invalid backend"); + } + params.backend = static_cast(backend_val); + break; + } + case ParamField::SliceStart: params.slice_start = static_cast(read_u64(in)); break; + case ParamField::SliceLength: params.slice_length = static_cast(read_u64(in)); break; + case ParamField::Epsilon: params.epsilon = read_f32(in); break; + case ParamField::NumGroups: params.num_groups = static_cast(read_u64(in)); break; + case ParamField::IndexValue: params.index_value = static_cast(read_u64(in)); break; + case ParamField::Permutation: params.permutation = read_size_vector(in); break; + case ParamField::Scale: params.scale = read_f32(in); break; + case ParamField::Theta: params.theta = read_f32(in); break; + case ParamField::PositionOffset: params.position_offset = static_cast(read_u64(in)); break; + case ParamField::WindowSize: params.window_size = static_cast(read_u64(in)); break; + case ParamField::IsCausal: params.is_causal = (read_u32(in) != 0); break; + case ParamField::AttentionMaskIsAdditive: params.attention_mask_is_additive = (read_u32(in) != 0); break; + case ParamField::LogitCap: params.logit_cap = read_f32(in); break; + case ParamField::TopK: params.top_k = static_cast(read_u64(in)); break; + case ParamField::Temperature: params.temperature = read_f32(in); break; + case ParamField::TopP: params.top_p = read_f32(in); break; + case ParamField::MinP: params.min_p = read_f32(in); break; + case ParamField::RepetitionPenalty: params.repetition_penalty = read_f32(in); break; + case ParamField::RandomSeed: params.random_seed = static_cast(read_u64(in)); break; + case ParamField::BiasIndices: params.bias_indices = read_u32_vector(in); break; + case ParamField::BiasValues: params.bias_values = read_f32_vector(in); break; + case ParamField::NormalizeRouting: params.normalize_routing = (read_u32(in) != 0); break; + case ParamField::NumExperts: params.num_experts = static_cast(read_u64(in)); break; + case ParamField::NumExpertsPerTok: params.num_experts_per_tok = static_cast(read_u64(in)); break; + case ParamField::MoeGated: params.moe_gated = (read_u32(in) != 0); break; + case ParamField::Activation: params.activation = static_cast(read_u32(in)); break; + case ParamField::NumKvHeads: params.num_kv_heads = static_cast(read_u64(in)); break; + case ParamField::ChunkSize: params.chunk_size = static_cast(read_u64(in)); break; + case ParamField::DstHeight: params.dst_height = static_cast(read_u64(in)); break; + case ParamField::DstWidth: params.dst_width = static_cast(read_u64(in)); break; + case ParamField::AlignCorners: params.align_corners = (read_u32(in) != 0); break; + case ParamField::NumClasses: params.num_classes = static_cast(read_u64(in)); break; + case ParamField::NumAltupInputs: params.num_altup_inputs = static_cast(read_u64(in)); break; + case ParamField::KernelSize: params.kernel_size = static_cast(read_u64(in)); break; + case ParamField::Stride: params.stride = static_cast(read_u64(in)); break; + case ParamField::Dilation: params.dilation = static_cast(read_u64(in)); break; + case ParamField::NumFftBins: params.num_fft_bins = static_cast(read_u64(in)); break; + case ParamField::CacheSeqLen: params.cache_seq_len = static_cast(read_u64(in)); break; + case ParamField::HeadDim: params.head_dim = static_cast(read_u64(in)); break; + case ParamField::VHeadDim: params.v_head_dim = static_cast(read_u64(in)); break; + case ParamField::CachedKeysInt8Ptr: + case ParamField::CachedValuesInt8Ptr: + case ParamField::CachedKScalesPtr: + case ParamField::CachedVScalesPtr: + throw std::runtime_error("Graph file corrupted: runtime-only field serialized"); + case ParamField::Padding: params.padding = static_cast(read_u64(in)); break; + } +} + +} // namespace + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params) { + const auto& schema = op_schema(op_type); + std::vector fields; + fields.reserve(schema.size()); + for (const auto& spec : schema) { + if (spec.persistence == FieldPersistence::Persistent) { + fields.push_back(spec.field); + } + } + + write_u32(out, static_cast(fields.size())); + for (ParamField field : fields) { + write_u32(out, static_cast(field)); + write_field(out, field, params); + } +} + +void read_op_params(std::istream& in, OpType op_type, OpParams& params) { + uint32_t field_count = read_u32(in); + const auto& schema = op_schema(op_type); + std::unordered_set allowed; + for (const auto& spec : schema) { + if (spec.persistence == FieldPersistence::Persistent) { + allowed.insert(static_cast(spec.field)); + } + } + + for (uint32_t i = 0; i < field_count; ++i) { + uint32_t raw_field = read_u32(in); + if (allowed.count(raw_field) == 0) { + throw std::runtime_error("Graph file corrupted: field not allowed for op"); + } + read_field(in, static_cast(raw_field), params); + } +} + +} // namespace GraphParamIO diff --git a/cactus-transpile/graph/graph_param_io.h b/cactus-transpile/graph/graph_param_io.h new file mode 100644 index 000000000..0362ea8c5 --- /dev/null +++ b/cactus-transpile/graph/graph_param_io.h @@ -0,0 +1,11 @@ +#pragma once + +#include "graph.h" +#include + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); +void read_op_params(std::istream& in, OpType op_type, OpParams& params); + +} // namespace GraphParamIO diff --git a/cactus-transpile/kernel/kernel.h b/cactus-transpile/kernel/kernel.h new file mode 100644 index 000000000..8141ce43c --- /dev/null +++ b/cactus-transpile/kernel/kernel.h @@ -0,0 +1,488 @@ +#ifndef KERNEL_H +#define KERNEL_H + +#include +#include + +enum class Precision; + +enum class ScalarOpType { + ADD, + SUBTRACT, + MULTIPLY, + DIVIDE, + NOT_EQUAL, + ABS, + EXP, + POW, + SQRT, + COS, + SIN, + LOG, + FLOOR_DIVIDE +}; + +constexpr size_t KV_QUANT_GROUP_SIZE = 32; + +void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale); +void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_not_equal_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); + +void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_not_equal_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); + +void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type); + +void cactus_gemv_int8(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int8_i8mm(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8_i8mm(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_integer(Precision precision, + const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c, + size_t M, size_t K, size_t N); + +void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, + size_t num_rows, size_t num_cols, size_t start_row, size_t end_row); +void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, + const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx); + +double cactus_sum_all_f16(const __fp16* data, size_t num_elements); +void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); +double cactus_sum_all_f32(const float* data, size_t num_elements); +void cactus_sum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_mean_all_f16(const __fp16* data, size_t num_elements); +void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); +double cactus_mean_all_f32(const float* data, size_t num_elements); +void cactus_mean_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_variance_all_f16(const __fp16* data, size_t num_elements); +void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); +double cactus_variance_all_f32(const float* data, size_t num_elements); +void cactus_variance_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements); +void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); +float cactus_min_all_f32(const float* data, size_t num_elements); +void cactus_min_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements); +void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); +float cactus_max_all_f32(const float* data, size_t num_elements); +void cactus_max_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); +void cactus_cumsum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); +void cactus_cumsum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); + +void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output, + size_t batch_size, size_t dims, float eps); + +void cactus_layer_norm_f16(const __fp16* input, const __fp16* weight, const __fp16* bias, + __fp16* output, size_t batch_size, size_t dims, float eps); + +void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t start_pos, float theta); + +void cactus_gpt_j_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t rot_dim, size_t start_pos, float theta); + +void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, + size_t seq_len, size_t vocab_size); + +void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope); + +void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_glu_f16( + const __fp16* input, + __fp16* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_glu_f32( + const float* input, + float* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_batchnorm_f16( + const __fp16* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + __fp16* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_batchnorm_f32( + const float* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + float* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output, + size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, + size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, + bool is_causal = true, bool mask_is_additive = false, bool mask_per_head = false, + size_t v_head_dim = 0, float logit_cap = 0.0f); + +void cactus_attention_hybrid_int8_fp16( + const __fp16* queries, + const int8_t* keys_cached, + const int8_t* values_cached, + const float* k_scales, + const float* v_scales, + const __fp16* keys_new, + const __fp16* values_new, + __fp16* output, + size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len, + size_t num_q_heads, size_t num_kv_heads, size_t head_dim, + float scale, size_t position_offset = 0, bool is_causal = true, size_t window_size = 0, + size_t group_size = KV_QUANT_GROUP_SIZE, size_t v_head_dim = 0); + +void cactus_gated_deltanet_decode_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + float scale); + +void cactus_gated_deltanet_prefill_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + size_t requested_chunk_size, + float scale); + +void cactus_conv1d_causal_depthwise_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C, + size_t K, + size_t dilation); + +void cactus_conv1d_f16_k3( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t stride +); + +void cactus_conv1d_f16( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t K, + size_t stride +); + +void cactus_stft_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, size_t stride, + size_t num_fft_bins +); + +void cactus_conv1d_f16_k7s3_oc8( + const __fp16* input, + const __fp16* Wpack, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_conv1d_same_depthwise_f16_k9( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C +); + +void cactus_conv2d_f16_k3s1p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_depthwise_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C, + size_t H, + size_t W +); + +void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv1d_pointwise_f16_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, + size_t dst_height, size_t dst_width, bool align_corners = true); + +void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, + const size_t* shape1, const size_t* shape2, const size_t* output_shape, + size_t ndims, int axis); +void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, + const size_t* output_shape, size_t num_inputs, size_t rank, int axis); + +void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); +void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count); +void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count); +void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +float cactus_fp16_max_abs(const __fp16* src, size_t count); + +void cactus_quantize_kv_fp16_to_int8( + const __fp16* src, + int8_t* dst, + float* scales, + size_t seq_len, size_t kv_heads, size_t head_dim, + size_t group_size = KV_QUANT_GROUP_SIZE); + +inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) { + size_t num_groups = (head_dim + group_size - 1) / group_size; + return seq_len * kv_heads * num_groups; +} + +void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count); + +void cactus_gaussian_topk_f16( + const __fp16* input, + __fp16* output, + size_t rows, + size_t cols, + float ppf); + +void cactus_altup_predict_f16( + const __fp16* coefs, + const __fp16* const* streams, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_altup_correct_f16( + const __fp16* coefs, + const __fp16* innovation, + const __fp16* const* predictions, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_lstm_cell_f16( + const __fp16* x_input, + const __fp16* h_prev, + const __fp16* c_prev, + const __fp16* weight_ih, + const __fp16* weight_hh, + const __fp16* bias_ih, + const __fp16* bias_hh, + __fp16* h_new, + __fp16* c_new, + size_t batch_size, + size_t input_size, + size_t hidden_size +); + +void cactus_bilstm_sequence_f16( + const __fp16* input, + const __fp16* weight_ih_fwd, + const __fp16* weight_hh_fwd, + const __fp16* bias_ih_fwd, + const __fp16* bias_hh_fwd, + const __fp16* weight_ih_bwd, + const __fp16* weight_hh_bwd, + const __fp16* bias_ih_bwd, + const __fp16* bias_hh_bwd, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t input_size, + size_t hidden_size +); + +void cactus_maxpool1d_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t channels, + size_t input_length, + size_t kernel_size, + size_t stride +); + +#endif diff --git a/cactus-transpile/kernel/kernel_altup.cpp b/cactus-transpile/kernel/kernel_altup.cpp new file mode 100644 index 000000000..f15bf9030 --- /dev/null +++ b/cactus-transpile/kernel/kernel_altup.cpp @@ -0,0 +1,180 @@ +#include "kernel.h" +#include +#include +#include +#include + +void cactus_gaussian_topk_f16( + const __fp16* input, + __fp16* output, + size_t rows, + size_t cols, + float ppf +) { + static constexpr float MAX_SAFE_ABS = 240.0f; + + for (size_t r = 0; r < rows; r++) { + const __fp16* row_in = input + r * cols; + __fp16* row_out = output + r * cols; + + // Pass 1: compute mean and max absolute deviation + float32x4_t sum_vec = vdupq_n_f32(0.0f); + size_t d = 0; + for (; d + 8 <= cols; d += 8) { + float16x8_t v = vld1q_f16(row_in + d); + sum_vec = vaddq_f32(sum_vec, vcvt_f32_f16(vget_low_f16(v))); + sum_vec = vaddq_f32(sum_vec, vcvt_f32_f16(vget_high_f16(v))); + } + float sum = vaddvq_f32(sum_vec); + for (; d < cols; d++) { + sum += static_cast(row_in[d]); + } + float mu = sum / static_cast(cols); + + float32x4_t mu_vec = vdupq_n_f32(mu); + float32x4_t max_abs_vec = vdupq_n_f32(0.0f); + d = 0; + for (; d + 8 <= cols; d += 8) { + float16x8_t v = vld1q_f16(row_in + d); + float32x4_t lo = vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), mu_vec); + float32x4_t hi = vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), mu_vec); + max_abs_vec = vmaxq_f32(max_abs_vec, vabsq_f32(lo)); + max_abs_vec = vmaxq_f32(max_abs_vec, vabsq_f32(hi)); + } + float max_abs = vmaxvq_f32(max_abs_vec); + for (; d < cols; d++) { + float diff = static_cast(row_in[d]) - mu; + float a = diff < 0.0f ? -diff : diff; + if (a > max_abs) max_abs = a; + } + + // Safe scaling: scale = max(max_abs / 240, 1) + float scale = max_abs * (1.0f / MAX_SAFE_ABS); + if (scale < 1.0f) scale = 1.0f; + float inv_scale = 1.0f / scale; + + // Pass 2: compute variance of scaled deviations + float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); + float32x4_t var_sum_vec = vdupq_n_f32(0.0f); + d = 0; + for (; d + 8 <= cols; d += 8) { + float16x8_t v = vld1q_f16(row_in + d); + float32x4_t lo = vmulq_f32(vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), mu_vec), inv_scale_vec); + float32x4_t hi = vmulq_f32(vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), mu_vec), inv_scale_vec); + var_sum_vec = vfmaq_f32(var_sum_vec, lo, lo); + var_sum_vec = vfmaq_f32(var_sum_vec, hi, hi); + } + float var_sum = vaddvq_f32(var_sum_vec); + for (; d < cols; d++) { + float ds = (static_cast(row_in[d]) - mu) * inv_scale; + var_sum += ds * ds; + } + float variance = var_sum / static_cast(cols); + float sigma = sqrtf(variance) * scale; + + // cutoff = mu + ppf * sigma, output = relu(input - cutoff) + float cutoff = mu + ppf * sigma; + float32x4_t cutoff_vec = vdupq_n_f32(cutoff); + float32x4_t zero_vec = vdupq_n_f32(0.0f); + d = 0; + for (; d + 8 <= cols; d += 8) { + float16x8_t v = vld1q_f16(row_in + d); + float32x4_t lo = vmaxq_f32(vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), cutoff_vec), zero_vec); + float32x4_t hi = vmaxq_f32(vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), cutoff_vec), zero_vec); + vst1q_f16(row_out + d, vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi))); + } + for (; d < cols; d++) { + float val = static_cast(row_in[d]) - cutoff; + row_out[d] = static_cast<__fp16>(val > 0.0f ? val : 0.0f); + } + } +} + +void cactus_altup_predict_f16( + const __fp16* coefs, + const __fp16* const* streams, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim +) { + if (n > 8) throw std::runtime_error("cactus_altup_predict_f16 expects n <= 8"); + const size_t coef_stride = n * n; + + for (size_t i = 0; i < n; i++) { + for (size_t s = 0; s < seq_len; s++) { + const __fp16* coef_row = coefs + s * coef_stride + i * n; + __fp16* out_row = output + (i * seq_len + s) * hidden_dim; + const __fp16* src_i = streams[i] + s * hidden_dim; + + float c[8]; + for (size_t j = 0; j < n; j++) { + c[j] = static_cast(coef_row[j]); + } + + size_t d = 0; + for (; d + 8 <= hidden_dim; d += 8) { + float32x4_t acc_lo = vcvt_f32_f16(vget_low_f16(vld1q_f16(src_i + d))); + float32x4_t acc_hi = vcvt_f32_f16(vget_high_f16(vld1q_f16(src_i + d))); + + for (size_t j = 0; j < n; j++) { + float16x8_t sj = vld1q_f16(streams[j] + s * hidden_dim + d); + float32x4_t cj = vdupq_n_f32(c[j]); + acc_lo = vfmaq_f32(acc_lo, vcvt_f32_f16(vget_low_f16(sj)), cj); + acc_hi = vfmaq_f32(acc_hi, vcvt_f32_f16(vget_high_f16(sj)), cj); + } + + vst1q_f16(out_row + d, vcombine_f16(vcvt_f16_f32(acc_lo), vcvt_f16_f32(acc_hi))); + } + for (; d < hidden_dim; d++) { + float acc = static_cast(src_i[d]); + for (size_t j = 0; j < n; j++) { + acc += c[j] * static_cast(streams[j][s * hidden_dim + d]); + } + out_row[d] = static_cast<__fp16>(acc); + } + } + } +} + +void cactus_altup_correct_f16( + const __fp16* coefs, + const __fp16* innovation, + const __fp16* const* predictions, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim +) { + for (size_t i = 0; i < n; i++) { + for (size_t s = 0; s < seq_len; s++) { + float ci = static_cast(coefs[s * n + i]); + float32x4_t ci_vec = vdupq_n_f32(ci); + + const __fp16* pred_row = predictions[i] + s * hidden_dim; + const __fp16* innov_row = innovation + s * hidden_dim; + __fp16* out_row = output + (i * seq_len + s) * hidden_dim; + + size_t d = 0; + for (; d + 8 <= hidden_dim; d += 8) { + float16x8_t pred = vld1q_f16(pred_row + d); + float16x8_t innov = vld1q_f16(innov_row + d); + + float32x4_t p_lo = vcvt_f32_f16(vget_low_f16(pred)); + float32x4_t p_hi = vcvt_f32_f16(vget_high_f16(pred)); + float32x4_t i_lo = vcvt_f32_f16(vget_low_f16(innov)); + float32x4_t i_hi = vcvt_f32_f16(vget_high_f16(innov)); + + float32x4_t out_lo = vfmaq_f32(p_lo, i_lo, ci_vec); + float32x4_t out_hi = vfmaq_f32(p_hi, i_hi, ci_vec); + + vst1q_f16(out_row + d, vcombine_f16(vcvt_f16_f32(out_lo), vcvt_f16_f32(out_hi))); + } + for (; d < hidden_dim; d++) { + out_row[d] = static_cast<__fp16>( + static_cast(pred_row[d]) + ci * static_cast(innov_row[d]) + ); + } + } + } +} diff --git a/cactus-transpile/kernel/kernel_attention.cpp b/cactus-transpile/kernel/kernel_attention.cpp new file mode 100644 index 000000000..1ff483802 --- /dev/null +++ b/cactus-transpile/kernel/kernel_attention.cpp @@ -0,0 +1,1365 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +inline size_t sliding_window_kv_start(size_t absolute_q_pos, size_t window_size) { + if (window_size == 0) { + return 0; + } + const size_t visible_tokens = absolute_q_pos + 1; + if (visible_tokens <= window_size) { + return 0; + } + return visible_tokens - window_size; +} + +inline bool sliding_window_masks_kv(size_t absolute_q_pos, size_t kv_pos, size_t window_size) { + return window_size > 0 && kv_pos < absolute_q_pos && (absolute_q_pos - kv_pos) >= window_size; +} + +} // namespace + +#ifdef __APPLE__ +#include +#endif + +#ifdef __APPLE__ +static void cactus_attention_f16_accelerate( + const __fp16* queries, + const __fp16* keys, + const __fp16* values, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t kv_seq_len, + size_t num_q_heads, + size_t num_kv_heads, + size_t head_dim, + size_t v_head_dim, + float scale, + size_t position_offset, + bool is_causal +) { + constexpr size_t BLOCK_SIZE = 64; + + const size_t group_size = num_q_heads / num_kv_heads; + const size_t q_batch_stride = seq_len * num_q_heads * head_dim; + const size_t k_batch_stride = kv_seq_len * num_kv_heads * head_dim; + const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; + const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; + const size_t q_seq_stride = num_q_heads * head_dim; + const size_t k_seq_stride = num_kv_heads * head_dim; + const size_t v_seq_stride = num_kv_heads * v_head_dim; + const size_t o_seq_stride = num_q_heads * v_head_dim; + + static constexpr CactusThreading::ParallelConfig ATTENTION_BATCHED{1, 1}; + CactusThreading::parallel_for(batch_size * num_q_heads, ATTENTION_BATCHED, + [&](size_t start, size_t end) { + + std::vector Q_f32(seq_len * head_dim); + std::vector K_f32(BLOCK_SIZE * head_dim); + std::vector V_f32(BLOCK_SIZE * v_head_dim); + std::vector scores(seq_len * BLOCK_SIZE); + std::vector acc(seq_len * v_head_dim); + std::vector row_max(seq_len); + std::vector row_sum(seq_len); + + for (size_t work = start; work < end; ++work) { + const size_t batch = work / num_q_heads; + const size_t q_head = work % num_q_heads; + const size_t kv_head = q_head / group_size; + + for (size_t q = 0; q < seq_len; ++q) { + const __fp16* q_src = queries + batch*q_batch_stride + q*q_seq_stride + q_head*head_dim; + float* q_dst = Q_f32.data() + q * head_dim; + for (size_t d = 0; d < head_dim; d += 8) { + float16x8_t v = vld1q_f16(q_src + d); + vst1q_f32(q_dst + d, vcvt_f32_f16(vget_low_f16(v))); + vst1q_f32(q_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); + } + } + + std::fill(row_max.begin(), row_max.begin() + seq_len, -INFINITY); + std::fill(row_sum.begin(), row_sum.begin() + seq_len, 0.0f); + memset(acc.data(), 0, seq_len * v_head_dim * sizeof(float)); + + for (size_t kv0 = 0; kv0 < kv_seq_len; kv0 += BLOCK_SIZE) { + const size_t block_len = std::min(BLOCK_SIZE, kv_seq_len - kv0); + + size_t q_start = 0; + size_t active_rows = seq_len; + if (is_causal) { + if (kv0 > position_offset) { + q_start = kv0 - position_offset; + } + if (q_start >= seq_len) continue; + active_rows = seq_len - q_start; + } + + for (size_t i = 0; i < block_len; ++i) { + const __fp16* k_src = keys + batch*k_batch_stride + (kv0+i)*k_seq_stride + kv_head*head_dim; + float* k_dst = K_f32.data() + i * head_dim; + for (size_t d = 0; d < head_dim; d += 8) { + float16x8_t v = vld1q_f16(k_src + d); + vst1q_f32(k_dst + d, vcvt_f32_f16(vget_low_f16(v))); + vst1q_f32(k_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); + } + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + (int)active_rows, (int)block_len, (int)head_dim, + scale, + Q_f32.data() + q_start * head_dim, (int)head_dim, + K_f32.data(), (int)head_dim, + 0.0f, + scores.data(), (int)block_len); + + for (size_t r = 0; r < active_rows; ++r) { + const size_t q_pos = q_start + r; + const size_t abs_q = position_offset + q_pos; + float* s_row = scores.data() + r * block_len; + + size_t valid_len = block_len; + if (is_causal) { + if (abs_q < kv0) { + memset(s_row, 0, block_len * sizeof(float)); + continue; + } + valid_len = std::min(block_len, abs_q - kv0 + 1); + } + + float32x4_t vmax = vdupq_n_f32(-INFINITY); + size_t j = 0; + for (; j + 4 <= valid_len; j += 4) { + vmax = vmaxq_f32(vmax, vld1q_f32(s_row + j)); + } + float block_max = vmaxvq_f32(vmax); + for (; j < valid_len; ++j) { + block_max = std::max(block_max, s_row[j]); + } + + float prev_max = row_max[q_pos]; + float new_max = std::max(prev_max, block_max); + float scale_old = expf(prev_max - new_max); + + if (prev_max != -INFINITY) { + float* acc_row = acc.data() + q_pos * v_head_dim; + float32x4_t sv = vdupq_n_f32(scale_old); + for (size_t d = 0; d < v_head_dim; d += 4) { + float32x4_t a = vld1q_f32(acc_row + d); + vst1q_f32(acc_row + d, vmulq_f32(a, sv)); + } + } + row_sum[q_pos] = row_sum[q_pos] * scale_old; + row_max[q_pos] = new_max; + + float32x4_t vsum = vdupq_n_f32(0.0f); + float32x4_t vnmax = vdupq_n_f32(new_max); + float32x4_t log2e = vdupq_n_f32(1.442695f); + j = 0; + for (; j + 4 <= valid_len; j += 4) { + float32x4_t x = vmulq_f32(vsubq_f32(vld1q_f32(s_row + j), vnmax), log2e); + float32x4_t x_floor = vrndmq_f32(x); + int32x4_t xi = vcvtq_s32_f32(x_floor); + float32x4_t xf = vsubq_f32(x, x_floor); + float32x4_t t = vfmaq_n_f32(vdupq_n_f32(0.2246932f), xf, 0.0789673f); + t = vfmaq_f32(vdupq_n_f32(0.6963248f), t, xf); + float32x4_t y = vfmaq_f32(vdupq_n_f32(0.9999003f), t, xf); + xi = vshlq_n_s32(vaddq_s32(xi, vdupq_n_s32(127)), 23); + y = vmulq_f32(y, vreinterpretq_f32_s32(xi)); + uint32x4_t underflow = vcltq_f32(x, vdupq_n_f32(-126.0f)); + y = vbslq_f32(underflow, vdupq_n_f32(0.0f), y); + vst1q_f32(s_row + j, y); + vsum = vaddq_f32(vsum, y); + } + float block_sum = vaddvq_f32(vsum); + for (; j < valid_len; ++j) { + s_row[j] = expf(s_row[j] - new_max); + block_sum += s_row[j]; + } + if (valid_len < block_len) { + memset(s_row + valid_len, 0, (block_len - valid_len) * sizeof(float)); + } + row_sum[q_pos] += block_sum; + } + + for (size_t i = 0; i < block_len; ++i) { + const __fp16* v_src = values + batch*v_batch_stride + (kv0+i)*v_seq_stride + kv_head*v_head_dim; + float* v_dst = V_f32.data() + i * v_head_dim; + for (size_t d = 0; d < v_head_dim; d += 8) { + float16x8_t v = vld1q_f16(v_src + d); + vst1q_f32(v_dst + d, vcvt_f32_f16(vget_low_f16(v))); + vst1q_f32(v_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); + } + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + (int)active_rows, (int)v_head_dim, (int)block_len, + 1.0f, + scores.data(), (int)block_len, + V_f32.data(), (int)v_head_dim, + 1.0f, + acc.data() + q_start * v_head_dim, (int)v_head_dim); + } + + for (size_t q = 0; q < seq_len; ++q) { + __fp16* o = output + batch*o_batch_stride + q*o_seq_stride + q_head*v_head_dim; + float sum = row_sum[q]; + if (sum == 0.0f) { + memset(o, 0, v_head_dim * sizeof(__fp16)); + continue; + } + float inv = 1.0f / sum; + float32x4_t invv = vdupq_n_f32(inv); + float* acc_row = acc.data() + q * v_head_dim; + for (size_t d = 0; d < v_head_dim; d += 8) { + float32x4_t a0 = vmulq_f32(vld1q_f32(acc_row + d), invv); + float32x4_t a1 = vmulq_f32(vld1q_f32(acc_row + d + 4), invv); + vst1q_f16(o + d, vcombine_f16(vcvt_f16_f32(a0), vcvt_f16_f32(a1))); + } + } + } + }); +} +#endif + +static inline void cactus_attention_f16_fast( + const __fp16* queries, + const __fp16* keys, + const __fp16* values, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t kv_seq_len, + size_t num_q_heads, + size_t num_kv_heads, + size_t head_dim, + float scale, + size_t position_offset, + bool is_causal, + size_t window_size, + size_t v_head_dim +) { + constexpr size_t BLOCK_SIZE = 32; + const size_t qk_nblocks = head_dim / 8; + const size_t v_nblocks = v_head_dim / 8; + +#ifdef __APPLE__ + if (seq_len >= 64 && window_size == 0) { + cactus_attention_f16_accelerate( + queries, keys, values, output, + batch_size, seq_len, kv_seq_len, + num_q_heads, num_kv_heads, head_dim, v_head_dim, + scale, position_offset, is_causal + ); + return; + } +#endif + + const size_t group_size = num_q_heads / num_kv_heads; + const size_t q_batch_stride = seq_len * num_q_heads * head_dim; + const size_t kv_batch_stride = kv_seq_len * num_kv_heads * head_dim; + const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; + const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; + const size_t q_seq_stride = num_q_heads * head_dim; + const size_t kv_seq_stride = num_kv_heads * head_dim; + const size_t v_seq_stride = num_kv_heads * v_head_dim; + const size_t o_seq_stride = num_q_heads * v_head_dim; + + CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, + [&](size_t start, size_t end) { + + float block_scores[BLOCK_SIZE]; + std::vector acc_lo(v_nblocks), acc_hi(v_nblocks); + + for (size_t work = start; work < end; ++work) { + const size_t batch = work / (num_q_heads * seq_len); + const size_t rem = work % (num_q_heads * seq_len); + const size_t q_head = rem / seq_len; + const size_t q_pos = rem % seq_len; + const size_t kv_head = q_head / group_size; + + const __fp16* q = queries + batch*q_batch_stride + q_pos*q_seq_stride + q_head*head_dim; + __fp16* o = output + batch*o_batch_stride + q_pos*o_seq_stride + q_head*v_head_dim; + + for (size_t i = 0; i < v_nblocks; i++) { + acc_lo[i] = vdupq_n_f32(0.f); + acc_hi[i] = vdupq_n_f32(0.f); + } + + float running_max = -INFINITY; + float running_sum = 0.f; + + const size_t abs_q = position_offset + q_pos; + size_t kv_end = is_causal ? std::min(kv_seq_len, abs_q + 1) : kv_seq_len; + size_t kv_start = sliding_window_kv_start(abs_q, window_size); + + for (size_t kv0 = kv_start; kv0 < kv_end; kv0 += BLOCK_SIZE) { + const size_t kv1 = std::min(kv0 + BLOCK_SIZE, kv_end); + float block_max = -INFINITY; + + for (size_t i = kv0; i < kv1; i++) { + float32x4_t s0 = vdupq_n_f32(0.f); + float32x4_t s1 = vdupq_n_f32(0.f); + + const __fp16* k = keys + batch*kv_batch_stride + i*kv_seq_stride + kv_head*head_dim; + + for (size_t d = 0; d < qk_nblocks; d++) { + float16x8_t qv = vld1q_f16(q + d*8); + float16x8_t kv = vld1q_f16(k + d*8); + + float32x4_t ql = vcvt_f32_f16(vget_low_f16(qv)); + float32x4_t qh = vcvt_f32_f16(vget_high_f16(qv)); + float32x4_t kl = vcvt_f32_f16(vget_low_f16(kv)); + float32x4_t kh = vcvt_f32_f16(vget_high_f16(kv)); + + s0 = vfmaq_f32(s0, ql, kl); + s1 = vfmaq_f32(s1, qh, kh); + } + + float score = vaddvq_f32(vaddq_f32(s0, s1)) * scale; + block_scores[i - kv0] = score; + block_max = std::max(block_max, score); + } + + float current_block_scale = 1.0f; + if (block_max > running_max) { + float scale_correction = expf(running_max - block_max); + running_sum *= scale_correction; + + for (size_t d = 0; d < v_nblocks; d++) { + acc_lo[d] = vmulq_n_f32(acc_lo[d], scale_correction); + acc_hi[d] = vmulq_n_f32(acc_hi[d], scale_correction); + } + running_max = block_max; + } else { + current_block_scale = expf(block_max - running_max); + } + + float block_sum = 0.f; + for (size_t i = 0; i < kv1 - kv0; i++) { + block_scores[i] = expf(block_scores[i] - block_max); + block_sum += block_scores[i]; + } + + for (size_t i = 0; i < kv1 - kv0; i++) { + const float attn_weight = block_scores[i] * current_block_scale; + if (attn_weight == 0.f) continue; + + const __fp16* v = values + batch*v_batch_stride + (kv0+i)*v_seq_stride + kv_head*v_head_dim; + float32x4_t wv = vdupq_n_f32(attn_weight); + + for (size_t d = 0; d < v_nblocks; d++) { + float16x8_t vv = vld1q_f16(v + d*8); + acc_lo[d] = vfmaq_f32(acc_lo[d], vcvt_f32_f16(vget_low_f16(vv)), wv); + acc_hi[d] = vfmaq_f32(acc_hi[d], vcvt_f32_f16(vget_high_f16(vv)), wv); + } + } + + running_sum += block_sum * current_block_scale; + } + + if (running_sum == 0.f) { + memset(o, 0, v_head_dim * sizeof(__fp16)); + continue; + } + + float inv = 1.f / running_sum; + float32x4_t invv = vdupq_n_f32(inv); + + for (size_t d = 0; d < v_nblocks; d++) { + float16x8_t out = vcombine_f16( + vcvt_f16_f32(vmulq_f32(acc_lo[d], invv)), + vcvt_f16_f32(vmulq_f32(acc_hi[d], invv)) + ); + vst1q_f16(o + d*8, out); + } + } + }); +} + +void cactus_attention_f16( + const __fp16* queries, + const __fp16* keys, + const __fp16* values, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t kv_seq_len, + size_t num_q_heads, + size_t num_kv_heads, + size_t head_dim, + float scale, + const __fp16* mask, + size_t position_offset, + size_t window_size, + bool is_causal, + bool mask_is_additive, + bool mask_per_head, + size_t v_head_dim, + float logit_cap +) { + if (v_head_dim == 0) v_head_dim = head_dim; + if (scale == 0.0f) { + scale = 1.0f / sqrtf(static_cast(head_dim)); + } + + if (mask == nullptr && head_dim % 8 == 0 && v_head_dim % 8 == 0 && logit_cap == 0.0f) { + cactus_attention_f16_fast( + queries, keys, values, output, + batch_size, seq_len, kv_seq_len, + num_q_heads, num_kv_heads, head_dim, + scale, position_offset, is_causal, window_size, v_head_dim + ); + return; + } + + constexpr size_t VECTOR_WIDTH = 8; + constexpr size_t BLOCK_SIZE = 32; + const size_t head_dim_aligned = (head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; + const size_t v_head_dim_aligned = (v_head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; + + const size_t group_size = num_q_heads / num_kv_heads; + + const size_t q_batch_stride = seq_len * num_q_heads * head_dim; + const size_t kv_batch_stride = kv_seq_len * num_kv_heads * head_dim; + const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; + const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; + const size_t q_seq_stride = num_q_heads * head_dim; + const size_t kv_seq_stride = num_kv_heads * head_dim; + const size_t v_seq_stride = num_kv_heads * v_head_dim; + const size_t o_seq_stride = num_q_heads * v_head_dim; + const size_t mask_batch_stride = mask + ? (mask_per_head ? (num_q_heads * seq_len * kv_seq_len) : (seq_len * kv_seq_len)) + : 0; + + CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, + [=](size_t start_idx, size_t end_idx) { + std::vector block_scores(BLOCK_SIZE); + std::vector output_accum_low(v_head_dim_aligned / VECTOR_WIDTH * 2); + std::vector output_accum_high(v_head_dim_aligned / VECTOR_WIDTH * 2); + + const size_t v_tail_dims = v_head_dim - v_head_dim_aligned; + std::vector output_accum_tail(v_tail_dims, 0.0f); + + const float NEG_INF = -std::numeric_limits::infinity(); + const size_t used_vec_blocks = v_head_dim_aligned / VECTOR_WIDTH; + + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t batch_idx = work_idx / (num_q_heads * seq_len); + const size_t remainder = work_idx % (num_q_heads * seq_len); + const size_t q_head_idx = remainder / seq_len; + const size_t q_pos = remainder % seq_len; + + const size_t kv_head_idx = q_head_idx / group_size; + + const __fp16* Q_base = queries + batch_idx * q_batch_stride; + const __fp16* K_base = keys + batch_idx * kv_batch_stride; + const __fp16* V_base = values + batch_idx * v_batch_stride; + __fp16* O_base = output + batch_idx * o_batch_stride; + const __fp16* M = mask ? (mask + batch_idx * mask_batch_stride) : nullptr; + const __fp16* q_vec = Q_base + q_pos * q_seq_stride + q_head_idx * head_dim; + __fp16* o_vec = O_base + q_pos * o_seq_stride + q_head_idx * v_head_dim; + + float running_max = -std::numeric_limits::infinity(); + float running_sum = 0.0f; + + for (size_t i = 0; i < output_accum_low.size(); ++i) { + output_accum_low[i] = vdupq_n_f32(0.0f); + output_accum_high[i] = vdupq_n_f32(0.0f); + } + for (size_t i = 0; i < v_tail_dims; ++i) { + output_accum_tail[i] = 0.0f; + } + + const bool is_decode = (q_pos == seq_len - 1) && seq_len > 1; + const size_t absolute_q_pos = position_offset + q_pos; + + size_t kv_start = 0; + size_t kv_end = kv_seq_len; + + if (window_size > 0 && window_size < kv_seq_len) { + kv_start = sliding_window_kv_start(absolute_q_pos, window_size); + if (is_causal) { + kv_end = std::min(kv_end, absolute_q_pos + 1); + } + } else if (is_causal) { + kv_end = std::min(kv_end, absolute_q_pos + 1); + } + + for (size_t kv_block_start = kv_start; kv_block_start < kv_end; kv_block_start += BLOCK_SIZE) { + const size_t kv_block_end = std::min(kv_block_start + BLOCK_SIZE, kv_end); + const size_t block_size = kv_block_end - kv_block_start; + + float block_max = -std::numeric_limits::infinity(); + + if (!is_decode && is_causal && kv_block_start > absolute_q_pos) { + for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { + block_scores[kv_idx] = NEG_INF; + } + continue; + } + + for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { + const size_t kv_pos = kv_block_start + kv_idx; + + if (!is_decode && is_causal && kv_pos > absolute_q_pos) { + block_scores[kv_idx] = NEG_INF; + continue; + } + + const __fp16* k_vec = K_base + kv_pos * kv_seq_stride + kv_head_idx * head_dim; + + if (kv_idx + 1 < block_size) { + const __fp16* next_k_vec = K_base + (kv_pos + 1) * kv_seq_stride + kv_head_idx * head_dim; + __builtin_prefetch(next_k_vec, 0, 1); + } + + float32x4_t score_accum_low = vdupq_n_f32(0.0f); + float32x4_t score_accum_high = vdupq_n_f32(0.0f); + + for (size_t dim_block = 0; dim_block < head_dim_aligned; dim_block += VECTOR_WIDTH) { + float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); + float16x8_t k_vec_f16 = vld1q_f16(&k_vec[dim_block]); + + float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); + float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); + float32x4_t k_low = vcvt_f32_f16(vget_low_f16(k_vec_f16)); + float32x4_t k_high = vcvt_f32_f16(vget_high_f16(k_vec_f16)); + + score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); + score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); + } + + float score = vaddvq_f32(vaddq_f32(score_accum_low, score_accum_high)); + + for (size_t dim = head_dim_aligned; dim < head_dim; ++dim) { + score += static_cast(q_vec[dim]) * static_cast(k_vec[dim]); + } + + score *= scale; + + size_t absolute_q_pos = position_offset + q_pos; + + if (is_causal && kv_pos > absolute_q_pos) { + score = NEG_INF; + } + else if (sliding_window_masks_kv(absolute_q_pos, kv_pos, window_size)) { + score = NEG_INF; + } + else if (M) { + const size_t mask_index = mask_per_head + ? ((q_head_idx * seq_len + q_pos) * kv_seq_len + kv_pos) + : (q_pos * kv_seq_len + kv_pos); + const float mask_value = static_cast(M[mask_index]); + if (mask_is_additive) { + if (!std::isfinite(mask_value)) { + score = NEG_INF; + } else { + score += mask_value; + } + } else if (mask_value == 0.0f) { + score = NEG_INF; + } + } + + if (logit_cap > 0.0f && std::isfinite(score)) { + score = logit_cap * tanhf(score / logit_cap); + } + + block_scores[kv_idx] = score; + block_max = std::max(block_max, score); + } + + float current_block_scale = 1.0f; + + if (block_max > NEG_INF) { + if (block_max > running_max) { + float scale_correction = expf(running_max - block_max); + running_sum *= scale_correction; + + for (size_t i = 0; i < used_vec_blocks; ++i) { + output_accum_low[i] = vmulq_n_f32(output_accum_low[i], scale_correction); + output_accum_high[i] = vmulq_n_f32(output_accum_high[i], scale_correction); + } + for (size_t i = 0; i < v_tail_dims; ++i) { + output_accum_tail[i] *= scale_correction; + } + running_max = block_max; + } else { + current_block_scale = expf(block_max - running_max); + } + } + + float block_sum = 0.0f; + const size_t vec_size = (block_size / 4) * 4; + + for (size_t kv_idx = 0; kv_idx < vec_size; kv_idx += 4) { + float32x4_t scores = vld1q_f32(&block_scores[kv_idx]); + uint32x4_t inf_mask = vceqq_f32(scores, vdupq_n_f32(NEG_INF)); + + float32x4_t x = vsubq_f32(scores, vdupq_n_f32(block_max)); + x = vmulq_n_f32(x, 1.442695f); + float32x4_t x_floor = vrndmq_f32(x); + int32x4_t xi = vcvtq_s32_f32(x_floor); + float32x4_t xf = vsubq_f32(x, x_floor); + + float32x4_t t = vfmaq_n_f32(vdupq_n_f32(0.2246932f), xf, 0.0789673f); + t = vfmaq_f32(vdupq_n_f32(0.6963248f), t, xf); + float32x4_t y = vfmaq_f32(vdupq_n_f32(0.9999003f), t, xf); + + xi = vaddq_s32(xi, vdupq_n_s32(127)); + xi = vshlq_n_s32(xi, 23); + y = vmulq_f32(y, vreinterpretq_f32_s32(xi)); + + uint32x4_t underflow_mask = vcltq_f32(x, vdupq_n_f32(-126.0f)); + uint32x4_t zero_mask = vorrq_u32(inf_mask, underflow_mask); + y = vbslq_f32(zero_mask, vdupq_n_f32(0.0f), y); + + vst1q_f32(&block_scores[kv_idx], y); + block_sum += vaddvq_f32(y); + } + + for (size_t kv_idx = vec_size; kv_idx < block_size; ++kv_idx) { + if (block_scores[kv_idx] != NEG_INF) { + block_scores[kv_idx] = expf(block_scores[kv_idx] - block_max); + block_sum += block_scores[kv_idx]; + } else { + block_scores[kv_idx] = 0.0f; + } + } + + for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { + const float attn_weight = block_scores[kv_idx] * current_block_scale; + if (attn_weight == 0.0f) continue; + + const size_t kv_pos = kv_block_start + kv_idx; + const __fp16* v_vec = V_base + kv_pos * v_seq_stride + kv_head_idx * v_head_dim; + + const float32x4_t weight_vec = vdupq_n_f32(attn_weight); + + for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { + float16x8_t v_vec_f16 = vld1q_f16(&v_vec[dim_block]); + float32x4_t v_low = vcvt_f32_f16(vget_low_f16(v_vec_f16)); + float32x4_t v_high = vcvt_f32_f16(vget_high_f16(v_vec_f16)); + + size_t idx = dim_block / VECTOR_WIDTH; + output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); + output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); + } + + for (size_t dim = v_head_dim_aligned; dim < v_head_dim; ++dim) { + float val = attn_weight * static_cast(v_vec[dim]); + output_accum_tail[dim - v_head_dim_aligned] += val; + } + } + + running_sum += block_sum * current_block_scale; + } + + if (running_sum > 0.0f) { + const float inv_sum = 1.0f / running_sum; + const float32x4_t inv_sum_vec = vdupq_n_f32(inv_sum); + + for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { + size_t idx = dim_block / VECTOR_WIDTH; + float32x4_t final_low = vmulq_f32(output_accum_low[idx], inv_sum_vec); + float32x4_t final_high = vmulq_f32(output_accum_high[idx], inv_sum_vec); + + float16x4_t low_f16 = vcvt_f16_f32(final_low); + float16x4_t high_f16 = vcvt_f16_f32(final_high); + float16x8_t combined = vcombine_f16(low_f16, high_f16); + + vst1q_f16(&o_vec[dim_block], combined); + } + + for (size_t dim = v_head_dim_aligned; dim < v_head_dim; ++dim) { + o_vec[dim] = static_cast<__fp16>(output_accum_tail[dim - v_head_dim_aligned] * inv_sum); + } + } else { + for (size_t dim = 0; dim < v_head_dim; ++dim) { + o_vec[dim] = static_cast<__fp16>(0.0f); + } + } + } + }); +} + +void cactus_attention_hybrid_int8_fp16( + const __fp16* queries, + const int8_t* keys_cached, + const int8_t* values_cached, + const float* k_scales, + const float* v_scales, + const __fp16* keys_new, + const __fp16* values_new, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t cache_len, + size_t new_len, + size_t num_q_heads, + size_t num_kv_heads, + size_t head_dim, + float scale, + size_t position_offset, + bool is_causal, + size_t window_size, + size_t quant_group_size, + size_t v_head_dim +) { + if (v_head_dim == 0) v_head_dim = head_dim; + if (scale == 0.0f) { + scale = 1.0f / sqrtf(static_cast(head_dim)); + } + + const size_t kv_seq_len = cache_len + new_len; + + constexpr size_t VECTOR_WIDTH = 8; + constexpr size_t BLOCK_SIZE = 32; + const size_t head_dim_aligned = (head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; + const size_t v_head_dim_aligned = (v_head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; + + const size_t gqa_group_size = num_q_heads / num_kv_heads; + const size_t num_quant_groups_k = (head_dim + quant_group_size - 1) / quant_group_size; + const size_t num_quant_groups_v = (v_head_dim + quant_group_size - 1) / quant_group_size; + + const size_t q_batch_stride = seq_len * num_q_heads * head_dim; + const size_t k_cached_batch_stride = cache_len * num_kv_heads * head_dim; + const size_t v_cached_batch_stride = cache_len * num_kv_heads * v_head_dim; + const size_t k_new_batch_stride = new_len * num_kv_heads * head_dim; + const size_t v_new_batch_stride = new_len * num_kv_heads * v_head_dim; + const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; + const size_t q_seq_stride = num_q_heads * head_dim; + const size_t k_seq_stride = num_kv_heads * head_dim; + const size_t v_seq_stride = num_kv_heads * v_head_dim; + const size_t o_seq_stride = num_q_heads * v_head_dim; + + CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, + [=](size_t start_idx, size_t end_idx) { + std::vector block_scores(BLOCK_SIZE); + std::vector output_accum_low(v_head_dim_aligned / VECTOR_WIDTH * 2); + std::vector output_accum_high(v_head_dim_aligned / VECTOR_WIDTH * 2); + + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t batch_idx = work_idx / (num_q_heads * seq_len); + const size_t remainder = work_idx % (num_q_heads * seq_len); + const size_t q_head_idx = remainder / seq_len; + const size_t q_pos = remainder % seq_len; + + const size_t kv_head_idx = q_head_idx / gqa_group_size; + + const __fp16* Q_base = queries + batch_idx * q_batch_stride; + const int8_t* K_cached_base = keys_cached + batch_idx * k_cached_batch_stride; + const int8_t* V_cached_base = values_cached + batch_idx * v_cached_batch_stride; + const __fp16* K_new_base = keys_new + batch_idx * k_new_batch_stride; + const __fp16* V_new_base = values_new + batch_idx * v_new_batch_stride; + __fp16* O_base = output + batch_idx * o_batch_stride; + + const __fp16* q_vec = Q_base + q_pos * q_seq_stride + q_head_idx * head_dim; + __fp16* o_vec = O_base + q_pos * o_seq_stride + q_head_idx * v_head_dim; + + float running_max = -std::numeric_limits::infinity(); + float running_sum = 0.0f; + + for (size_t i = 0; i < output_accum_low.size(); ++i) { + output_accum_low[i] = vdupq_n_f32(0.0f); + output_accum_high[i] = vdupq_n_f32(0.0f); + } + + const size_t absolute_q_pos = position_offset + q_pos; + size_t kv_end = is_causal ? std::min(kv_seq_len, cache_len + q_pos + 1) : kv_seq_len; + + size_t kv_start = sliding_window_kv_start(absolute_q_pos, window_size); + + constexpr size_t SINK_SIZE = 4; + const size_t cache_abs_offset = (position_offset >= cache_len) ? (position_offset - cache_len) : 0; + + const size_t kv_block_start0 = (window_size > 0 && kv_start > 0) ? 0 + : (kv_start / BLOCK_SIZE) * BLOCK_SIZE; + + for (size_t kv_block_start = kv_block_start0; kv_block_start < kv_end; kv_block_start += BLOCK_SIZE) { + const size_t kv_block_end = std::min(kv_block_start + BLOCK_SIZE, kv_end); + const size_t block_size = kv_block_end - kv_block_start; + + float block_max = -std::numeric_limits::infinity(); + + for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { + const size_t kv_pos = kv_block_start + kv_idx; + + bool window_masked = false; + if (window_size > 0 && kv_start > 0) { + if (kv_pos < cache_len) { + if (cache_abs_offset == 0 || kv_pos >= SINK_SIZE) { + window_masked = (cache_abs_offset + kv_pos < kv_start); + } + } else { + window_masked = (kv_pos + cache_abs_offset < kv_start); + } + } + + if ((is_causal && kv_pos > absolute_q_pos) || window_masked) { + block_scores[kv_idx] = -std::numeric_limits::infinity(); + continue; + } + + float32x4_t score_accum_low = vdupq_n_f32(0.0f); + float32x4_t score_accum_high = vdupq_n_f32(0.0f); + + if (kv_pos < cache_len) { + const int8_t* k_vec = K_cached_base + kv_pos * k_seq_stride + kv_head_idx * head_dim; + const float* k_scale_base = k_scales + (kv_pos * num_kv_heads + kv_head_idx) * num_quant_groups_k; + + for (size_t quant_group = 0; quant_group < num_quant_groups_k; quant_group++) { + const size_t dim_base = quant_group * quant_group_size; + const float k_scale = k_scale_base[quant_group]; + const float32x4_t k_scale_vec = vdupq_n_f32(k_scale); + + #pragma unroll + for (size_t i = 0; i < 4; i++) { + const size_t dim_block = dim_base + i * VECTOR_WIDTH; + if (dim_block >= head_dim_aligned) break; + + float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); + float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); + float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); + + int8x8_t k_vec_i8 = vld1_s8(&k_vec[dim_block]); + int16x8_t k_vec_i16 = vmovl_s8(k_vec_i8); + float32x4_t k_low = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(k_vec_i16))), k_scale_vec); + float32x4_t k_high = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(k_vec_i16))), k_scale_vec); + + score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); + score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); + } + } + } else { + const size_t new_pos = kv_pos - cache_len; + const __fp16* k_vec = K_new_base + new_pos * k_seq_stride + kv_head_idx * head_dim; + + for (size_t dim_block = 0; dim_block < head_dim_aligned; dim_block += VECTOR_WIDTH) { + float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); + float16x8_t k_vec_f16 = vld1q_f16(&k_vec[dim_block]); + + float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); + float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); + float32x4_t k_low = vcvt_f32_f16(vget_low_f16(k_vec_f16)); + float32x4_t k_high = vcvt_f32_f16(vget_high_f16(k_vec_f16)); + + score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); + score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); + } + } + + float score = vaddvq_f32(vaddq_f32(score_accum_low, score_accum_high)) * scale; + block_scores[kv_idx] = score; + block_max = std::max(block_max, score); + } + + float current_block_scale = 1.0f; + if (block_max > -std::numeric_limits::infinity()) { + if (block_max > running_max) { + float scale_correction = expf(running_max - block_max); + running_sum *= scale_correction; + + for (size_t i = 0; i < output_accum_low.size() / 2; ++i) { + output_accum_low[i] = vmulq_n_f32(output_accum_low[i], scale_correction); + output_accum_high[i] = vmulq_n_f32(output_accum_high[i], scale_correction); + } + running_max = block_max; + } else { + current_block_scale = expf(block_max - running_max); + } + } + + float block_sum = 0.0f; + for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { + if (block_scores[kv_idx] != -std::numeric_limits::infinity()) { + block_scores[kv_idx] = expf(block_scores[kv_idx] - block_max); + block_sum += block_scores[kv_idx]; + } else { + block_scores[kv_idx] = 0.0f; + } + } + + for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { + const float attn_weight = block_scores[kv_idx] * current_block_scale; + if (attn_weight == 0.0f) continue; + + const size_t kv_pos = kv_block_start + kv_idx; + const float32x4_t weight_vec = vdupq_n_f32(attn_weight); + + if (kv_pos < cache_len) { + const int8_t* v_vec = V_cached_base + kv_pos * v_seq_stride + kv_head_idx * v_head_dim; + const float* v_scale_base = v_scales + (kv_pos * num_kv_heads + kv_head_idx) * num_quant_groups_v; + + for (size_t quant_group = 0; quant_group < num_quant_groups_v; quant_group++) { + const size_t dim_base = quant_group * quant_group_size; + const float v_scale = v_scale_base[quant_group]; + const float32x4_t v_scale_vec = vdupq_n_f32(v_scale); + + #pragma unroll + for (size_t i = 0; i < 4; i++) { + const size_t dim_block = dim_base + i * VECTOR_WIDTH; + if (dim_block >= v_head_dim_aligned) break; + + int8x8_t v_vec_i8 = vld1_s8(&v_vec[dim_block]); + int16x8_t v_vec_i16 = vmovl_s8(v_vec_i8); + float32x4_t v_low = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_vec_i16))), v_scale_vec); + float32x4_t v_high = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_vec_i16))), v_scale_vec); + + size_t idx = dim_block / VECTOR_WIDTH; + output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); + output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); + } + } + } else { + const size_t new_pos = kv_pos - cache_len; + const __fp16* v_vec = V_new_base + new_pos * v_seq_stride + kv_head_idx * v_head_dim; + + for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { + float16x8_t v_vec_f16 = vld1q_f16(&v_vec[dim_block]); + float32x4_t v_low = vcvt_f32_f16(vget_low_f16(v_vec_f16)); + float32x4_t v_high = vcvt_f32_f16(vget_high_f16(v_vec_f16)); + + size_t idx = dim_block / VECTOR_WIDTH; + output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); + output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); + } + } + } + + running_sum += block_sum * current_block_scale; + } + + if (running_sum > 0.0f) { + const float inv_sum = 1.0f / running_sum; + const float32x4_t inv_sum_vec = vdupq_n_f32(inv_sum); + + for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { + size_t idx = dim_block / VECTOR_WIDTH; + float32x4_t final_low = vmulq_f32(output_accum_low[idx], inv_sum_vec); + float32x4_t final_high = vmulq_f32(output_accum_high[idx], inv_sum_vec); + + float16x4_t low_f16 = vcvt_f16_f32(final_low); + float16x4_t high_f16 = vcvt_f16_f32(final_high); + float16x8_t combined = vcombine_f16(low_f16, high_f16); + + vst1q_f16(&o_vec[dim_block], combined); + } + } else { + for (size_t dim = 0; dim < v_head_dim; ++dim) { + o_vec[dim] = static_cast<__fp16>(0.0f); + } + } + } + }); +} + +void cactus_rms_norm_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t batch_size, + size_t dims, + float eps +) { + constexpr size_t SIMD_WIDTH = 8; + constexpr size_t UNROLL_FACTOR = 2; + constexpr size_t TILE_SIZE = SIMD_WIDTH * UNROLL_FACTOR; + + for (size_t b = 0; b < batch_size; ++b) { + const __fp16* input_row = input + b * dims; + __fp16* output_row = output + b * dims; + + float32x4_t sum_squares_vec[UNROLL_FACTOR * 2]; + for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { + sum_squares_vec[u] = vdupq_n_f32(0.0f); + } + + size_t i = 0; + const size_t tile_end = (dims >= TILE_SIZE) ? dims - TILE_SIZE + 1 : 0; + + for (; i < tile_end; i += TILE_SIZE) { + for (size_t u = 0; u < UNROLL_FACTOR; u++) { + float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); + float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); + float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); + sum_squares_vec[u * 2] = vfmaq_f32(sum_squares_vec[u * 2], input_low, input_low); + sum_squares_vec[u * 2 + 1] = vfmaq_f32(sum_squares_vec[u * 2 + 1], input_high, input_high); + } + } + + const size_t simd_end = (dims >= SIMD_WIDTH) ? dims - SIMD_WIDTH + 1 : 0; + for (; i < simd_end; i += SIMD_WIDTH) { + float16x8_t input_vec = vld1q_f16(&input_row[i]); + float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); + float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); + sum_squares_vec[0] = vfmaq_f32(sum_squares_vec[0], input_low, input_low); + sum_squares_vec[1] = vfmaq_f32(sum_squares_vec[1], input_high, input_high); + } + + float32x4_t total_sum = sum_squares_vec[0]; + for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { + total_sum = vaddq_f32(total_sum, sum_squares_vec[u]); + } + float sum_squares = vaddvq_f32(total_sum); + + for (; i < dims; ++i) { + float val = static_cast(input_row[i]); + sum_squares += val * val; + } + + float rms = sqrtf(sum_squares / static_cast(dims) + eps); + float inv_rms = 1.0f / rms; + float16x8_t inv_rms_vec = vdupq_n_f16(static_cast<__fp16>(inv_rms)); + + i = 0; + for (; i < tile_end; i += TILE_SIZE) { + for (size_t u = 0; u < UNROLL_FACTOR; u++) { + float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); + float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); + float16x8_t norm_vec = vmulq_f16(vmulq_f16(input_vec, inv_rms_vec), weight_vec); + vst1q_f16(&output_row[i + u * SIMD_WIDTH], norm_vec); + } + } + + for (; i < simd_end; i += SIMD_WIDTH) { + float16x8_t input_vec = vld1q_f16(&input_row[i]); + float16x8_t weight_vec = vld1q_f16(&weight[i]); + float16x8_t norm_vec = vmulq_f16(vmulq_f16(input_vec, inv_rms_vec), weight_vec); + vst1q_f16(&output_row[i], norm_vec); + } + + for (; i < dims; ++i) { + output_row[i] = static_cast<__fp16>(static_cast(input_row[i]) * inv_rms * static_cast(weight[i])); + } + } +} + +void cactus_layer_norm_f16( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t batch_size, + size_t dims, + float eps +) { + constexpr size_t SIMD_WIDTH = 8; + constexpr size_t UNROLL_FACTOR = 3; + constexpr size_t TILE_SIZE = SIMD_WIDTH * UNROLL_FACTOR; + + const size_t tile_end = (dims >= TILE_SIZE) ? dims - TILE_SIZE + 1 : 0; + const size_t simd_end = (dims >= SIMD_WIDTH) ? dims - SIMD_WIDTH + 1 : 0; + + for (size_t b = 0; b < batch_size; ++b) { + const __fp16* input_row = input + b * dims; + __fp16* output_row = output + b * dims; + + float32x4_t sum_input_vec[UNROLL_FACTOR * 2]; + float32x4_t sum_squares_vec[UNROLL_FACTOR * 2]; + for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { + sum_input_vec[u] = vdupq_n_f32(0.0f); + sum_squares_vec[u] = vdupq_n_f32(0.0f); + } + + size_t i = 0; + + for (; i < tile_end; i += TILE_SIZE) { + for (size_t u = 0; u < UNROLL_FACTOR; u++) { + float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); + float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); + float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); + + sum_input_vec[u * 2] = vaddq_f32(sum_input_vec[u * 2], input_low); + sum_input_vec[u * 2 + 1] = vaddq_f32(sum_input_vec[u * 2 + 1], input_high); + + sum_squares_vec[u * 2] = vfmaq_f32(sum_squares_vec[u * 2], input_low, input_low); + sum_squares_vec[u * 2 + 1] = vfmaq_f32(sum_squares_vec[u * 2 + 1], input_high, input_high); + } + } + + for (; i < simd_end; i += SIMD_WIDTH) { + float16x8_t input_vec = vld1q_f16(&input_row[i]); + float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); + float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); + sum_input_vec[0] = vaddq_f32(sum_input_vec[0], input_low); + sum_input_vec[1] = vaddq_f32(sum_input_vec[1], input_high); + sum_squares_vec[0] = vfmaq_f32(sum_squares_vec[0], input_low, input_low); + sum_squares_vec[1] = vfmaq_f32(sum_squares_vec[1], input_high, input_high); + } + + float32x4_t total_sum_inputs = sum_input_vec[0]; + float32x4_t total_sum_squares = sum_squares_vec[0]; + for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { + total_sum_inputs = vaddq_f32(total_sum_inputs, sum_input_vec[u]); + total_sum_squares = vaddq_f32(total_sum_squares, sum_squares_vec[u]); + } + + float sum_inputs = vaddvq_f32(total_sum_inputs); + float sum_squares = vaddvq_f32(total_sum_squares); + for (; i < dims; ++i) { + float val = static_cast(input_row[i]); + sum_inputs += val; + sum_squares += val * val; + } + + float mean = sum_inputs / static_cast(dims); + float mean_squares = sum_squares / static_cast(dims); + float variance = mean_squares - mean * mean; + if (variance < 0.0f) variance = 0.0f; + float inv_std = 1.0f / sqrtf(variance + eps); + + float16x8_t mean_vec = vdupq_n_f16(static_cast<__fp16>(mean)); + float16x8_t inv_std_vec = vdupq_n_f16(static_cast<__fp16>(inv_std)); + + i = 0; + if (bias) { + for (; i < tile_end; i += TILE_SIZE) { + for (size_t u = 0; u < UNROLL_FACTOR; u++) { + float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); + float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); + float16x8_t bias_vec = vld1q_f16(&bias[i + u * SIMD_WIDTH]); + float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); + out_vec = vaddq_f16(out_vec, bias_vec); + vst1q_f16(&output_row[i + u * SIMD_WIDTH], out_vec); + } + } + + for (; i < simd_end; i += SIMD_WIDTH) { + float16x8_t input_vec = vld1q_f16(&input_row[i]); + float16x8_t weight_vec = vld1q_f16(&weight[i]); + float16x8_t bias_vec = vld1q_f16(&bias[i]); + float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); + out_vec = vaddq_f16(out_vec, bias_vec); + vst1q_f16(&output_row[i], out_vec); + } + + for (; i < dims; ++i) { + output_row[i] = static_cast<__fp16>((static_cast(input_row[i]) - mean) * inv_std * static_cast(weight[i]) + static_cast(bias[i])); + } + } else { + for (; i < tile_end; i += TILE_SIZE) { + for (size_t u = 0; u < UNROLL_FACTOR; u++) { + float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); + float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); + float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); + vst1q_f16(&output_row[i + u * SIMD_WIDTH], out_vec); + } + } + + for (; i < simd_end; i += SIMD_WIDTH) { + float16x8_t input_vec = vld1q_f16(&input_row[i]); + float16x8_t weight_vec = vld1q_f16(&weight[i]); + float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); + vst1q_f16(&output_row[i], out_vec); + } + + for (; i < dims; ++i) { + output_row[i] = static_cast<__fp16>((static_cast(input_row[i]) - mean) * inv_std * static_cast(weight[i])); + } + } + } +} + +namespace CactusRoPEF16 { + +struct RoPECacheF16 { + std::vector<__fp16> cos_table; + std::vector<__fp16> sin_table; + size_t max_seq_len; + size_t head_dim; + float theta; + bool initialized; + + RoPECacheF16() : max_seq_len(0), head_dim(0), theta(0.0f), initialized(false) {} +}; + +static thread_local RoPECacheF16 rope_cache_f16; + +void precompute_rope_tables_f16(size_t seq_len, size_t head_dim, float theta) { + if (rope_cache_f16.initialized && + rope_cache_f16.max_seq_len >= seq_len && + rope_cache_f16.head_dim == head_dim && + rope_cache_f16.theta == theta) { + return; + } + + const size_t half_dim = head_dim / 2; + const size_t table_size = seq_len * half_dim; + + rope_cache_f16.cos_table.resize(table_size); + rope_cache_f16.sin_table.resize(table_size); + + for (size_t pos = 0; pos < seq_len; ++pos) { + const float pos_float = static_cast(pos); + for (size_t i = 0; i < half_dim; ++i) { + const float freq = 1.0f / powf(theta, (2.0f * i) / head_dim); + const float angle = pos_float * freq; + + const size_t idx = pos * half_dim + i; + rope_cache_f16.cos_table[idx] = static_cast<__fp16>(cosf(angle)); + rope_cache_f16.sin_table[idx] = static_cast<__fp16>(sinf(angle)); + } + } + + rope_cache_f16.max_seq_len = seq_len; + rope_cache_f16.head_dim = head_dim; + rope_cache_f16.theta = theta; + rope_cache_f16.initialized = true; +} + +} + +void cactus_rope_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t num_heads, + size_t head_dim, + size_t start_pos, + float theta +) { + const size_t half_dim = head_dim / 2; + + CactusRoPEF16::precompute_rope_tables_f16(seq_len + start_pos, head_dim, theta); + + const __fp16* cos_cache = CactusRoPEF16::rope_cache_f16.cos_table.data() + start_pos * half_dim; + const __fp16* sin_cache = CactusRoPEF16::rope_cache_f16.sin_table.data() + start_pos * half_dim; + + CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_idx, size_t end_idx) { + for (size_t idx = start_idx; idx < end_idx; ++idx) { + const size_t batch_idx = idx / seq_len; + const size_t seq_idx = idx % seq_len; + + for (size_t head_idx = 0; head_idx < num_heads; ++head_idx) { + const size_t offset = ((batch_idx * seq_len + seq_idx) * num_heads + head_idx) * head_dim; + const __fp16* input_ptr = input + offset; + __fp16* output_ptr = output + offset; + + const __fp16* cos_ptr = cos_cache + seq_idx * half_dim; + const __fp16* sin_ptr = sin_cache + seq_idx * half_dim; + + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_half_dim = (half_dim / SIMD_WIDTH) * SIMD_WIDTH; + + for (size_t i = 0; i < vectorized_half_dim; i += SIMD_WIDTH) { + float16x8_t cos_vec = vld1q_f16(&cos_ptr[i]); + float16x8_t sin_vec = vld1q_f16(&sin_ptr[i]); + + float16x8_t x_first_half = vld1q_f16(&input_ptr[i]); + float16x8_t x_second_half = vld1q_f16(&input_ptr[i + half_dim]); + + float16x8_t first_result = vfmsq_f16(vmulq_f16(x_first_half, cos_vec), x_second_half, sin_vec); + float16x8_t second_result = vfmaq_f16(vmulq_f16(x_second_half, cos_vec), x_first_half, sin_vec); + + vst1q_f16(&output_ptr[i], first_result); + vst1q_f16(&output_ptr[i + half_dim], second_result); + } + + for (size_t i = vectorized_half_dim; i < half_dim; ++i) { + const __fp16 cos_val = cos_ptr[i]; + const __fp16 sin_val = sin_ptr[i]; + + const __fp16 x_first_half = input_ptr[i]; + const __fp16 x_second_half = input_ptr[i + half_dim]; + + output_ptr[i] = x_first_half * cos_val - x_second_half * sin_val; + + output_ptr[i + half_dim] = x_second_half * cos_val + x_first_half * sin_val; + } + } + } + }); +} + +void cactus_gpt_j_rope_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t num_heads, + size_t head_dim, + size_t rot_dim, + size_t start_pos, + float theta +) { + const size_t half_rot_dim = rot_dim / 2; + + CactusRoPEF16::precompute_rope_tables_f16(seq_len + start_pos, rot_dim, theta); + + const __fp16* cos_cache = CactusRoPEF16::rope_cache_f16.cos_table.data() + start_pos * half_rot_dim; + const __fp16* sin_cache = CactusRoPEF16::rope_cache_f16.sin_table.data() + start_pos * half_rot_dim; + + CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_idx, size_t end_idx) { + for (size_t idx = start_idx; idx < end_idx; ++idx) { + const size_t batch_idx = idx / seq_len; + const size_t seq_idx = idx % seq_len; + + for (size_t head_idx = 0; head_idx < num_heads; ++head_idx) { + const size_t offset = ((batch_idx * seq_len + seq_idx) * num_heads + head_idx) * head_dim; + const __fp16* input_ptr = input + offset; + __fp16* output_ptr = output + offset; + + const __fp16* cos_ptr = cos_cache + seq_idx * half_rot_dim; + const __fp16* sin_ptr = sin_cache + seq_idx * half_rot_dim; + + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_half_rot_dim = (half_rot_dim / SIMD_WIDTH) * SIMD_WIDTH; + + for (size_t i = 0; i < vectorized_half_rot_dim; i += SIMD_WIDTH) { + float16x8_t cos_vec = vld1q_f16(&cos_ptr[i]); + float16x8_t sin_vec = vld1q_f16(&sin_ptr[i]); + + float16x8x2_t x_vec = vld2q_f16(&input_ptr[2*i]); + float16x8_t x_first_half = x_vec.val[0]; + float16x8_t x_second_half = x_vec.val[1]; + + float16x8_t first_result = vfmsq_f16(vmulq_f16(x_first_half, cos_vec), x_second_half, sin_vec); + float16x8_t second_result = vfmaq_f16(vmulq_f16(x_second_half, cos_vec), x_first_half, sin_vec); + + float16x8x2_t t; + t.val[0] = first_result; + t.val[1] = second_result; + vst2q_f16(&output_ptr[2*i], t); + } + + for (size_t i = vectorized_half_rot_dim; i < half_rot_dim; ++i) { + const __fp16 cos_val = cos_ptr[i]; + const __fp16 sin_val = sin_ptr[i]; + + const __fp16 x_first_half = input_ptr[2*i]; + const __fp16 x_second_half = input_ptr[2*i + 1]; + + output_ptr[2*i] = x_first_half * cos_val - x_second_half * sin_val; + + output_ptr[2*i + 1] = x_second_half * cos_val + x_first_half * sin_val; + } + + constexpr size_t TAIL_SIMD_WIDTH = 8; + size_t copy_idx = rot_dim; + const size_t copy_end_vec = (head_dim / TAIL_SIMD_WIDTH) * TAIL_SIMD_WIDTH; + + for (; copy_idx + TAIL_SIMD_WIDTH <= copy_end_vec; copy_idx += TAIL_SIMD_WIDTH) { + float16x8_t v = vld1q_f16(&input_ptr[copy_idx]); + vst1q_f16(&output_ptr[copy_idx], v); + } + for (; copy_idx < head_dim; ++copy_idx) { + output_ptr[copy_idx] = input_ptr[copy_idx]; + } + } + } + }); +} diff --git a/cactus-transpile/kernel/kernel_bilstm.cpp b/cactus-transpile/kernel/kernel_bilstm.cpp new file mode 100644 index 000000000..2b4ea2321 --- /dev/null +++ b/cactus-transpile/kernel/kernel_bilstm.cpp @@ -0,0 +1,370 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#endif + +static void apply_lstm_gates_f32( + const float* __restrict gates, + float* __restrict c, + float* __restrict h, + size_t hidden_size +) { + const float32x4_t one = vdupq_n_f32(1.0f); + size_t i = 0; + + for (; i + 4 <= hidden_size; i += 4) { + float32x4_t i_raw = vld1q_f32(gates + i); + float32x4_t f_raw = vld1q_f32(gates + hidden_size + i); + float32x4_t g_raw = vld1q_f32(gates + 2 * hidden_size + i); + float32x4_t o_raw = vld1q_f32(gates + 3 * hidden_size + i); + + float32x4_t i_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_raw)))); + float32x4_t f_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_raw)))); + float32x4_t g_act = fast_tanh_f32x4(g_raw); + float32x4_t o_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_raw)))); + + float32x4_t c_prev = vld1q_f32(c + i); + float32x4_t c_new = vfmaq_f32(vmulq_f32(f_act, c_prev), i_act, g_act); + vst1q_f32(c + i, c_new); + + vst1q_f32(h + i, vmulq_f32(o_act, fast_tanh_f32x4(c_new))); + } + + for (; i < hidden_size; ++i) { + float ig = 1.0f / (1.0f + expf(-gates[i])); + float fg = 1.0f / (1.0f + expf(-gates[hidden_size + i])); + float gg = tanhf(gates[2 * hidden_size + i]); + float og = 1.0f / (1.0f + expf(-gates[3 * hidden_size + i])); + float cv = fg * c[i] + ig * gg; + c[i] = cv; + h[i] = og * tanhf(cv); + } +} + +#ifndef __APPLE__ +static void apply_lstm_gates_f16( + const __fp16* __restrict gates, + __fp16* __restrict c, + __fp16* __restrict h, + size_t hidden_size +) { + constexpr size_t W = 8; + const size_t simd_end = (hidden_size / W) * W; + const float32x4_t one = vdupq_n_f32(1.0f); + + for (size_t i = 0; i < simd_end; i += W) { + float16x8_t i_raw = vld1q_f16(gates + i); + float16x8_t f_raw = vld1q_f16(gates + hidden_size + i); + float16x8_t g_raw = vld1q_f16(gates + 2 * hidden_size + i); + float16x8_t o_raw = vld1q_f16(gates + 3 * hidden_size + i); + + float32x4_t il = vcvt_f32_f16(vget_low_f16(i_raw)); + float32x4_t ih = vcvt_f32_f16(vget_high_f16(i_raw)); + float16x8_t i_act = vcombine_f16( + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(il))))), + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(ih)))))); + + float32x4_t fl = vcvt_f32_f16(vget_low_f16(f_raw)); + float32x4_t fh = vcvt_f32_f16(vget_high_f16(f_raw)); + float16x8_t f_act = vcombine_f16( + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(fl))))), + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(fh)))))); + + float32x4_t gl = vcvt_f32_f16(vget_low_f16(g_raw)); + float32x4_t gh = vcvt_f32_f16(vget_high_f16(g_raw)); + float16x8_t g_act = vcombine_f16( + vcvt_f16_f32(fast_tanh_f32x4(gl)), + vcvt_f16_f32(fast_tanh_f32x4(gh))); + + float32x4_t ol = vcvt_f32_f16(vget_low_f16(o_raw)); + float32x4_t oh = vcvt_f32_f16(vget_high_f16(o_raw)); + float16x8_t o_act = vcombine_f16( + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(ol))))), + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(oh)))))); + + float16x8_t c_prev = vld1q_f16(c + i); + float16x8_t c_new = vfmaq_f16(vmulq_f16(f_act, c_prev), i_act, g_act); + vst1q_f16(c + i, c_new); + + float32x4_t cl = vcvt_f32_f16(vget_low_f16(c_new)); + float32x4_t ch = vcvt_f32_f16(vget_high_f16(c_new)); + float16x8_t c_tanh = vcombine_f16( + vcvt_f16_f32(fast_tanh_f32x4(cl)), + vcvt_f16_f32(fast_tanh_f32x4(ch))); + vst1q_f16(h + i, vmulq_f16(o_act, c_tanh)); + } + + for (size_t i = simd_end; i < hidden_size; ++i) { + float ig = 1.0f / (1.0f + expf(-static_cast(gates[i]))); + float fg = 1.0f / (1.0f + expf(-static_cast(gates[hidden_size + i]))); + float gg = tanhf(static_cast(gates[2 * hidden_size + i])); + float og = 1.0f / (1.0f + expf(-static_cast(gates[3 * hidden_size + i]))); + float cv = fg * static_cast(c[i]) + ig * gg; + c[i] = static_cast<__fp16>(cv); + h[i] = static_cast<__fp16>(og * tanhf(cv)); + } +} + +static inline float hsum_f16x8_f32(float16x8_t v) { + float16x4_t lo = vget_low_f16(v); + float16x4_t hi = vget_high_f16(v); + float16x4_t s4 = vadd_f16(lo, hi); + float16x4_t s2 = vadd_f16(s4, vext_f16(s4, s4, 2)); + float16x4_t s1 = vadd_f16(s2, vext_f16(s2, s2, 1)); + return static_cast(vget_lane_f16(s1, 0)); +} + +static void lstm_gemv_f16_neon( + const __fp16* __restrict x, + const __fp16* __restrict W, + const __fp16* __restrict bias, + __fp16* __restrict out, + size_t K, size_t N +) { + const size_t K16 = (K / 16) * 16; + const size_t K8 = (K / 8) * 8; + size_t n = 0; + for (; n + 4 <= N; n += 4) { + const __fp16* w0 = W + n * K; + const __fp16* w1 = W + (n + 1) * K; + const __fp16* w2 = W + (n + 2) * K; + const __fp16* w3 = W + (n + 3) * K; + float16x8_t a0 = vdupq_n_f16(0); + float16x8_t a1 = vdupq_n_f16(0); + float16x8_t a2 = vdupq_n_f16(0); + float16x8_t a3 = vdupq_n_f16(0); + for (size_t k = 0; k < K16; k += 16) { + float16x8_t xlo = vld1q_f16(x + k); + float16x8_t xhi = vld1q_f16(x + k + 8); + a0 = vfmaq_f16(a0, xlo, vld1q_f16(w0 + k)); + a0 = vfmaq_f16(a0, xhi, vld1q_f16(w0 + k + 8)); + a1 = vfmaq_f16(a1, xlo, vld1q_f16(w1 + k)); + a1 = vfmaq_f16(a1, xhi, vld1q_f16(w1 + k + 8)); + a2 = vfmaq_f16(a2, xlo, vld1q_f16(w2 + k)); + a2 = vfmaq_f16(a2, xhi, vld1q_f16(w2 + k + 8)); + a3 = vfmaq_f16(a3, xlo, vld1q_f16(w3 + k)); + a3 = vfmaq_f16(a3, xhi, vld1q_f16(w3 + k + 8)); + } + for (size_t k = K16; k < K8; k += 8) { + float16x8_t xv = vld1q_f16(x + k); + a0 = vfmaq_f16(a0, xv, vld1q_f16(w0 + k)); + a1 = vfmaq_f16(a1, xv, vld1q_f16(w1 + k)); + a2 = vfmaq_f16(a2, xv, vld1q_f16(w2 + k)); + a3 = vfmaq_f16(a3, xv, vld1q_f16(w3 + k)); + } + float s0 = hsum_f16x8_f32(a0); + float s1 = hsum_f16x8_f32(a1); + float s2 = hsum_f16x8_f32(a2); + float s3 = hsum_f16x8_f32(a3); + for (size_t k = K8; k < K; ++k) { + float xv = static_cast(x[k]); + s0 += xv * static_cast(w0[k]); + s1 += xv * static_cast(w1[k]); + s2 += xv * static_cast(w2[k]); + s3 += xv * static_cast(w3[k]); + } + out[n] = static_cast<__fp16>(s0 + static_cast(bias[n])); + out[n + 1] = static_cast<__fp16>(s1 + static_cast(bias[n + 1])); + out[n + 2] = static_cast<__fp16>(s2 + static_cast(bias[n + 2])); + out[n + 3] = static_cast<__fp16>(s3 + static_cast(bias[n + 3])); + } + for (; n < N; ++n) { + const __fp16* w = W + n * K; + float16x8_t acc = vdupq_n_f16(0); + for (size_t k = 0; k < K8; k += 8) + acc = vfmaq_f16(acc, vld1q_f16(x + k), vld1q_f16(w + k)); + float s = hsum_f16x8_f32(acc); + for (size_t k = K8; k < K; ++k) + s += static_cast(x[k]) * static_cast(w[k]); + out[n] = static_cast<__fp16>(s + static_cast(bias[n])); + } +} +#endif + +static inline void fp16_to_fp32_neon(const __fp16* src, float* dst, size_t n) { + size_t i = 0; + for (; i + 8 <= n; i += 8) { + float16x8_t v = vld1q_f16(src + i); + vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(v))); + vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(v))); + } + for (; i < n; ++i) + dst[i] = static_cast(src[i]); +} + +static inline void fp32_to_fp16_neon(const float* src, __fp16* dst, size_t n) { + size_t i = 0; + for (; i + 8 <= n; i += 8) { + float32x4_t lo = vld1q_f32(src + i); + float32x4_t hi = vld1q_f32(src + i + 4); + vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi))); + } + for (; i < n; ++i) + dst[i] = static_cast<__fp16>(src[i]); +} + +void cactus_bilstm_sequence_f16( + const __fp16* input, + const __fp16* weight_ih_fwd, + const __fp16* weight_hh_fwd, + const __fp16* bias_ih_fwd, + const __fp16* bias_hh_fwd, + const __fp16* weight_ih_bwd, + const __fp16* weight_hh_bwd, + const __fp16* bias_ih_bwd, + const __fp16* bias_hh_bwd, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t input_size, + size_t hidden_size +) { + const size_t gate_size = 4 * hidden_size; + const size_t combined_K = input_size + hidden_size; + const size_t output_size = 2 * hidden_size; + + std::vector bias_fwd_f32(gate_size); + std::vector bias_bwd_f32(gate_size); + for (size_t g = 0; g < gate_size; ++g) { + bias_fwd_f32[g] = static_cast(bias_ih_fwd[g]) + static_cast(bias_hh_fwd[g]); + bias_bwd_f32[g] = static_cast(bias_ih_bwd[g]) + static_cast(bias_hh_bwd[g]); + } + +#ifdef __APPLE__ + std::vector W_fwd_f32(gate_size * combined_K); + std::vector W_bwd_f32(gate_size * combined_K); + for (size_t g = 0; g < gate_size; ++g) { + fp16_to_fp32_neon(weight_ih_fwd + g * input_size, + W_fwd_f32.data() + g * combined_K, input_size); + fp16_to_fp32_neon(weight_hh_fwd + g * hidden_size, + W_fwd_f32.data() + g * combined_K + input_size, hidden_size); + fp16_to_fp32_neon(weight_ih_bwd + g * input_size, + W_bwd_f32.data() + g * combined_K, input_size); + fp16_to_fp32_neon(weight_hh_bwd + g * hidden_size, + W_bwd_f32.data() + g * combined_K + input_size, hidden_size); + } + + for (size_t b = 0; b < batch_size; ++b) { + const __fp16* batch_in = input + b * seq_len * input_size; + __fp16* batch_out = output + b * seq_len * output_size; + + auto& pool = CactusThreading::get_thread_pool(); + auto bwd_future = pool.enqueue([&, batch_in, batch_out]() { + std::vector xh(combined_K); + std::vector gates(gate_size); + std::vector h(hidden_size, 0.0f); + std::vector c(hidden_size, 0.0f); + + for (size_t idx = 0; idx < seq_len; ++idx) { + const size_t t = seq_len - 1 - idx; + fp16_to_fp32_neon(batch_in + t * input_size, xh.data(), input_size); + memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(float)); + + memcpy(gates.data(), bias_bwd_f32.data(), gate_size * sizeof(float)); + cblas_sgemv(CblasRowMajor, CblasNoTrans, + static_cast(gate_size), static_cast(combined_K), + 1.0f, W_bwd_f32.data(), static_cast(combined_K), + xh.data(), 1, 1.0f, gates.data(), 1); + + apply_lstm_gates_f32(gates.data(), c.data(), h.data(), hidden_size); + + fp32_to_fp16_neon(h.data(), batch_out + t * output_size + hidden_size, hidden_size); + } + }); + + { + std::vector xh(combined_K); + std::vector gates(gate_size); + std::vector h(hidden_size, 0.0f); + std::vector c(hidden_size, 0.0f); + + for (size_t t = 0; t < seq_len; ++t) { + fp16_to_fp32_neon(batch_in + t * input_size, xh.data(), input_size); + memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(float)); + + memcpy(gates.data(), bias_fwd_f32.data(), gate_size * sizeof(float)); + cblas_sgemv(CblasRowMajor, CblasNoTrans, + static_cast(gate_size), static_cast(combined_K), + 1.0f, W_fwd_f32.data(), static_cast(combined_K), + xh.data(), 1, 1.0f, gates.data(), 1); + + apply_lstm_gates_f32(gates.data(), c.data(), h.data(), hidden_size); + + fp32_to_fp16_neon(h.data(), batch_out + t * output_size, hidden_size); + } + } + + bwd_future.get(); + } + +#else + std::vector<__fp16> W_fwd_f16(gate_size * combined_K); + std::vector<__fp16> W_bwd_f16(gate_size * combined_K); + for (size_t g = 0; g < gate_size; ++g) { + memcpy(W_fwd_f16.data() + g * combined_K, + weight_ih_fwd + g * input_size, input_size * sizeof(__fp16)); + memcpy(W_fwd_f16.data() + g * combined_K + input_size, + weight_hh_fwd + g * hidden_size, hidden_size * sizeof(__fp16)); + memcpy(W_bwd_f16.data() + g * combined_K, + weight_ih_bwd + g * input_size, input_size * sizeof(__fp16)); + memcpy(W_bwd_f16.data() + g * combined_K + input_size, + weight_hh_bwd + g * hidden_size, hidden_size * sizeof(__fp16)); + } + + std::vector<__fp16> bias_fwd_f16(gate_size); + std::vector<__fp16> bias_bwd_f16(gate_size); + for (size_t g = 0; g < gate_size; ++g) { + bias_fwd_f16[g] = static_cast<__fp16>(bias_fwd_f32[g]); + bias_bwd_f16[g] = static_cast<__fp16>(bias_bwd_f32[g]); + } + + for (size_t b = 0; b < batch_size; ++b) { + const __fp16* batch_in = input + b * seq_len * input_size; + __fp16* batch_out = output + b * seq_len * output_size; + + auto& pool = CactusThreading::get_thread_pool(); + auto bwd_future = pool.enqueue([&, batch_in, batch_out]() { + std::vector<__fp16> xh(combined_K); + std::vector<__fp16> gates(gate_size); + std::vector<__fp16> h(hidden_size, static_cast<__fp16>(0)); + std::vector<__fp16> c(hidden_size, static_cast<__fp16>(0)); + + for (size_t idx = 0; idx < seq_len; ++idx) { + const size_t t = seq_len - 1 - idx; + memcpy(xh.data(), batch_in + t * input_size, input_size * sizeof(__fp16)); + memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(__fp16)); + lstm_gemv_f16_neon(xh.data(), W_bwd_f16.data(), bias_bwd_f16.data(), + gates.data(), combined_K, gate_size); + apply_lstm_gates_f16(gates.data(), c.data(), h.data(), hidden_size); + memcpy(batch_out + t * output_size + hidden_size, + h.data(), hidden_size * sizeof(__fp16)); + } + }); + + { + std::vector<__fp16> xh(combined_K); + std::vector<__fp16> gates(gate_size); + std::vector<__fp16> h(hidden_size, static_cast<__fp16>(0)); + std::vector<__fp16> c(hidden_size, static_cast<__fp16>(0)); + + for (size_t t = 0; t < seq_len; ++t) { + memcpy(xh.data(), batch_in + t * input_size, input_size * sizeof(__fp16)); + memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(__fp16)); + lstm_gemv_f16_neon(xh.data(), W_fwd_f16.data(), bias_fwd_f16.data(), + gates.data(), combined_K, gate_size); + apply_lstm_gates_f16(gates.data(), c.data(), h.data(), hidden_size); + memcpy(batch_out + t * output_size, + h.data(), hidden_size * sizeof(__fp16)); + } + } + + bwd_future.get(); + } +#endif +} diff --git a/cactus-transpile/kernel/kernel_blas.cpp b/cactus-transpile/kernel/kernel_blas.cpp new file mode 100644 index 000000000..fba63880b --- /dev/null +++ b/cactus-transpile/kernel/kernel_blas.cpp @@ -0,0 +1,639 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include +#include +#include + +static inline size_t compute_linear_index(const size_t* coords, const size_t* strides, size_t ndim) { + size_t index = 0; + for (size_t i = 0; i < ndim; ++i) { + index += coords[i] * strides[i]; + } + return index; +} + +static inline void increment_coords(size_t* coords, const size_t* shape, size_t ndim) { + for (int i = ndim - 1; i >= 0; --i) { + coords[i]++; + if (coords[i] < shape[i]) { + break; + } + coords[i] = 0; + } +} + +enum class BroadcastOp { ADD, SUB, MUL, DIV, NE }; + +template +static inline float16x8_t broadcast_op_vec(float16x8_t a, float16x8_t b) { + if constexpr (Op == BroadcastOp::ADD) return vaddq_f16(a, b); + else if constexpr (Op == BroadcastOp::SUB) return vsubq_f16(a, b); + else if constexpr (Op == BroadcastOp::MUL) return vmulq_f16(a, b); + else if constexpr (Op == BroadcastOp::DIV) return vdivq_f16(a, b); + else return vreinterpretq_f16_u16(vdupq_n_u16(0)); +} + +template +static inline __fp16 broadcast_op_scalar(__fp16 a, __fp16 b) { + if constexpr (Op == BroadcastOp::ADD) return a + b; + else if constexpr (Op == BroadcastOp::SUB) return a - b; + else if constexpr (Op == BroadcastOp::MUL) return a * b; + else if constexpr (Op == BroadcastOp::DIV) return a / b; + else return static_cast<__fp16>(a != b ? 1.0f : 0.0f); +} + +template +static void broadcast_op_optimized(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim) { + size_t total_elements = 1; + for (size_t i = 0; i < ndim; ++i) { + total_elements *= output_shape[i]; + } + + if (total_elements == 0) return; + + size_t inner_size = output_shape[ndim - 1]; + bool a_inner_contiguous = (a_strides[ndim - 1] == 1) || (a_strides[ndim - 1] == 0); + bool b_inner_contiguous = (b_strides[ndim - 1] == 1) || (b_strides[ndim - 1] == 0); + bool a_inner_broadcast = (a_strides[ndim - 1] == 0); + bool b_inner_broadcast = (b_strides[ndim - 1] == 0); + + size_t outer_size = total_elements / inner_size; + + if constexpr (Op == BroadcastOp::NE) { + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_outer, size_t end_outer) { + std::vector coords(ndim, 0); + + size_t tmp = start_outer; + for (int i = ndim - 2; i >= 0; --i) { + coords[i] = tmp % output_shape[i]; + tmp /= output_shape[i]; + } + + for (size_t outer_idx = start_outer; outer_idx < end_outer; ++outer_idx) { + size_t a_base = 0, b_base = 0; + for (size_t i = 0; i < ndim - 1; ++i) { + a_base += coords[i] * a_strides[i]; + b_base += coords[i] * b_strides[i]; + } + + __fp16* out_ptr = output + outer_idx * inner_size; + for (size_t i = 0; i < inner_size; ++i) { + const size_t a_idx = a_base + (a_inner_broadcast ? 0 : i); + const size_t b_idx = b_base + (b_inner_broadcast ? 0 : i); + out_ptr[i] = static_cast<__fp16>(a[a_idx] != b[b_idx] ? 1.0f : 0.0f); + } + + increment_coords(coords.data(), output_shape, ndim); + } + }); + return; + } + + if (a_inner_contiguous && b_inner_contiguous && inner_size >= 8) { + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_outer, size_t end_outer) { + std::vector coords(ndim, 0); + + size_t tmp = start_outer; + for (int i = ndim - 2; i >= 0; --i) { + coords[i] = tmp % output_shape[i]; + tmp /= output_shape[i]; + } + + for (size_t outer_idx = start_outer; outer_idx < end_outer; ++outer_idx) { + size_t a_base = 0, b_base = 0; + for (size_t i = 0; i < ndim - 1; ++i) { + a_base += coords[i] * a_strides[i]; + b_base += coords[i] * b_strides[i]; + } + + __fp16* out_ptr = output + outer_idx * inner_size; + const size_t vec_end = (inner_size / 8) * 8; + + const bool use_stream = total_elements >= STREAMING_STORE_THRESHOLD; + + if (a_inner_broadcast && b_inner_broadcast) { + __fp16 result = broadcast_op_scalar(a[a_base], b[b_base]); + float16x8_t result_vec = vdupq_n_f16(result); + if (use_stream) { + for (size_t i = 0; i < vec_end; i += 8) { + stream_store_f16x8(out_ptr + i, result_vec); + } + } else { + for (size_t i = 0; i < vec_end; i += 8) { + vst1q_f16(out_ptr + i, result_vec); + } + } + for (size_t i = vec_end; i < inner_size; ++i) { + out_ptr[i] = result; + } + } else if (a_inner_broadcast) { + float16x8_t a_vec = vdupq_n_f16(a[a_base]); + const __fp16* b_ptr = b + b_base; + if (use_stream) { + for (size_t i = 0; i < vec_end; i += 8) { + float16x8_t b_vec = vld1q_f16(b_ptr + i); + stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); + } + } else { + for (size_t i = 0; i < vec_end; i += 8) { + float16x8_t b_vec = vld1q_f16(b_ptr + i); + vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); + } + } + for (size_t i = vec_end; i < inner_size; ++i) { + out_ptr[i] = broadcast_op_scalar(a[a_base], b_ptr[i]); + } + } else if (b_inner_broadcast) { + const __fp16* a_ptr = a + a_base; + float16x8_t b_vec = vdupq_n_f16(b[b_base]); + if (use_stream) { + for (size_t i = 0; i < vec_end; i += 8) { + float16x8_t a_vec = vld1q_f16(a_ptr + i); + stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); + } + } else { + for (size_t i = 0; i < vec_end; i += 8) { + float16x8_t a_vec = vld1q_f16(a_ptr + i); + vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); + } + } + for (size_t i = vec_end; i < inner_size; ++i) { + out_ptr[i] = broadcast_op_scalar(a_ptr[i], b[b_base]); + } + } else { + const __fp16* a_ptr = a + a_base; + const __fp16* b_ptr = b + b_base; + if (use_stream) { + for (size_t i = 0; i < vec_end; i += 8) { + float16x8_t a_vec = vld1q_f16(a_ptr + i); + float16x8_t b_vec = vld1q_f16(b_ptr + i); + stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); + } + } else { + for (size_t i = 0; i < vec_end; i += 8) { + float16x8_t a_vec = vld1q_f16(a_ptr + i); + float16x8_t b_vec = vld1q_f16(b_ptr + i); + vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); + } + } + for (size_t i = vec_end; i < inner_size; ++i) { + out_ptr[i] = broadcast_op_scalar(a_ptr[i], b_ptr[i]); + } + } + + for (int i = ndim - 2; i >= 0; --i) { + coords[i]++; + if (coords[i] < output_shape[i]) break; + coords[i] = 0; + } + } + }); + } else { + CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + std::vector coords(ndim); + + size_t tmp = start_idx; + for (int i = ndim - 1; i >= 0; --i) { + coords[i] = tmp % output_shape[i]; + tmp /= output_shape[i]; + } + + for (size_t linear_idx = start_idx; linear_idx < end_idx; ++linear_idx) { + size_t a_idx = compute_linear_index(coords.data(), a_strides, ndim); + size_t b_idx = compute_linear_index(coords.data(), b_strides, ndim); + + output[linear_idx] = broadcast_op_scalar(a[a_idx], b[b_idx]); + + increment_coords(coords.data(), output_shape, ndim); + } + }); + } +} + +void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { + const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + constexpr size_t SIMD_WIDTH = 8; + constexpr size_t UNROLL = 4; + const size_t unrolled_end = start_idx + ((end_idx - start_idx) / (SIMD_WIDTH * UNROLL)) * (SIMD_WIDTH * UNROLL); + const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; + + if (use_streaming) { + for (size_t i = start_idx; i < unrolled_end; i += SIMD_WIDTH * UNROLL) { + __builtin_prefetch(&a[i + 256], 0, 0); + __builtin_prefetch(&b[i + 256], 0, 0); + float16x8_t v0 = vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i])); + float16x8_t v1 = vaddq_f16(vld1q_f16(&a[i + 8]), vld1q_f16(&b[i + 8])); + float16x8_t v2 = vaddq_f16(vld1q_f16(&a[i + 16]), vld1q_f16(&b[i + 16])); + float16x8_t v3 = vaddq_f16(vld1q_f16(&a[i + 24]), vld1q_f16(&b[i + 24])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + stream_store_f16x8(&output[i + 16], v2); + stream_store_f16x8(&output[i + 24], v3); + } + for (size_t i = unrolled_end; i < vectorized_end; i += SIMD_WIDTH) { + stream_store_f16x8(&output[i], vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i]))); + } + } else { + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + vst1q_f16(&output[i], vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i]))); + } + } + + for (size_t i = vectorized_end; i < end_idx; ++i) { + output[i] = a[i] + b[i]; + } + }); +} + +void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; + + constexpr float FP16_MAX = 65500.0f; + const float32x4_t max_val = vdupq_n_f32(FP16_MAX); + const float32x4_t min_val = vdupq_n_f32(-FP16_MAX); + + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t a_vec = vld1q_f16(&a[i]); + float16x8_t b_vec = vld1q_f16(&b[i]); + + float32x4_t a_low = vcvt_f32_f16(vget_low_f16(a_vec)); + float32x4_t a_high = vcvt_f32_f16(vget_high_f16(a_vec)); + float32x4_t b_low = vcvt_f32_f16(vget_low_f16(b_vec)); + float32x4_t b_high = vcvt_f32_f16(vget_high_f16(b_vec)); + + float32x4_t result_low = vaddq_f32(a_low, b_low); + float32x4_t result_high = vaddq_f32(a_high, b_high); + + result_low = vminq_f32(vmaxq_f32(result_low, min_val), max_val); + result_high = vminq_f32(vmaxq_f32(result_high, min_val), max_val); + + float16x4_t result_low_f16 = vcvt_f16_f32(result_low); + float16x4_t result_high_f16 = vcvt_f16_f32(result_high); + float16x8_t result_vec = vcombine_f16(result_low_f16, result_high_f16); + + vst1q_f16(&output[i], result_vec); + } + + for (size_t i = vectorized_end; i < end_idx; ++i) { + float result = static_cast(a[i]) + static_cast(b[i]); + result = std::fmin(std::fmax(result, -FP16_MAX), FP16_MAX); + output[i] = static_cast<__fp16>(result); + } + }); +} + +void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { + const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; + + if (use_streaming) { + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t a_vec = vld1q_f16(&a[i]); + float16x8_t b_vec = vld1q_f16(&b[i]); + stream_store_f16x8(&output[i], vsubq_f16(a_vec, b_vec)); + } + } else { + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t a_vec = vld1q_f16(&a[i]); + float16x8_t b_vec = vld1q_f16(&b[i]); + vst1q_f16(&output[i], vsubq_f16(a_vec, b_vec)); + } + } + + for (size_t i = vectorized_end; i < end_idx; ++i) { + output[i] = a[i] - b[i]; + } + }); +} + +void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { + const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; + + if (use_streaming) { + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t a_vec = vld1q_f16(&a[i]); + float16x8_t b_vec = vld1q_f16(&b[i]); + stream_store_f16x8(&output[i], vmulq_f16(a_vec, b_vec)); + } + } else { + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t a_vec = vld1q_f16(&a[i]); + float16x8_t b_vec = vld1q_f16(&b[i]); + vst1q_f16(&output[i], vmulq_f16(a_vec, b_vec)); + } + } + + for (size_t i = vectorized_end; i < end_idx; ++i) { + output[i] = a[i] * b[i]; + } + }); +} + +void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale) { + constexpr size_t SIMD_WIDTH = 8; + const float32x4_t vscale = vdupq_n_f32(scale); + const size_t vec_end = (num_elements / SIMD_WIDTH) * SIMD_WIDTH; + + for (size_t i = 0; i < vec_end; i += SIMD_WIDTH) { + float16x8_t base_vec = vld1q_f16(base + i); + float16x8_t src_vec = vld1q_f16(src + i); + + float32x4_t base_lo = vcvt_f32_f16(vget_low_f16(base_vec)); + float32x4_t base_hi = vcvt_f32_f16(vget_high_f16(base_vec)); + float32x4_t src_lo = vcvt_f32_f16(vget_low_f16(src_vec)); + float32x4_t src_hi = vcvt_f32_f16(vget_high_f16(src_vec)); + + float32x4_t result_lo = vfmaq_f32(base_lo, src_lo, vscale); + float32x4_t result_hi = vfmaq_f32(base_hi, src_hi, vscale); + + vst1q_f16(output + i, vcombine_f16(vcvt_f16_f32(result_lo), vcvt_f16_f32(result_hi))); + } + for (size_t i = vec_end; i < num_elements; ++i) { + output[i] = static_cast<__fp16>(static_cast(base[i]) + + static_cast(src[i]) * scale); + } +} + +void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { + const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; + + if (use_streaming) { + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t a_vec = vld1q_f16(&a[i]); + float16x8_t b_vec = vld1q_f16(&b[i]); + stream_store_f16x8(&output[i], vdivq_f16(a_vec, b_vec)); + } + } else { + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t a_vec = vld1q_f16(&a[i]); + float16x8_t b_vec = vld1q_f16(&b[i]); + vst1q_f16(&output[i], vdivq_f16(a_vec, b_vec)); + } + } + + for (size_t i = vectorized_end; i < end_idx; ++i) { + output[i] = a[i] / b[i]; + } + }); +} + +void cactus_not_equal_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start_idx, size_t end_idx) { + for (size_t i = start_idx; i < end_idx; ++i) { + output[i] = static_cast<__fp16>(a[i] != b[i] ? 1.0f : 0.0f); + } + }); +} + +void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim) { + broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); +} + +void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim) { + broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); +} + +void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim) { + broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); +} + +void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim) { + broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); +} + +void cactus_not_equal_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim) { + broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); +} + +void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, + const size_t* shape1, const size_t* shape2, const size_t* output_shape, + size_t ndims, int axis) { + if (axis < 0) axis += ndims; + + size_t outer_size = 1; + for (size_t i = 0; i < static_cast(axis); ++i) { + outer_size *= output_shape[i]; + } + + size_t inner_size = 1; + for (size_t i = axis + 1; i < ndims; ++i) { + inner_size *= output_shape[i]; + } + + size_t axis_size1 = shape1[axis]; + size_t axis_size2 = shape2[axis]; + + size_t input1_stride = axis_size1 * inner_size; + size_t input2_stride = axis_size2 * inner_size; + size_t output_stride = (axis_size1 + axis_size2) * inner_size; + + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start, size_t end) { + for (size_t outer = start; outer < end; ++outer) { + const __fp16* in1_ptr = input1 + outer * input1_stride; + const __fp16* in2_ptr = input2 + outer * input2_stride; + __fp16* out_ptr = output + outer * output_stride; + + size_t copy_size1 = axis_size1 * inner_size; + std::memcpy(out_ptr, in1_ptr, copy_size1 * sizeof(__fp16)); + + size_t copy_size2 = axis_size2 * inner_size; + std::memcpy(out_ptr + copy_size1, in2_ptr, copy_size2 * sizeof(__fp16)); + } + }); +} + +void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, + const size_t* output_shape, size_t num_inputs, size_t rank, int axis) { + if (axis < 0) axis += rank; + + size_t outer_size = 1; + for (size_t i = 0; i < static_cast(axis); ++i) { + outer_size *= output_shape[i]; + } + + size_t inner_size = 1; + for (size_t i = axis + 1; i < rank; ++i) { + inner_size *= output_shape[i]; + } + + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start, size_t end) { + for (size_t outer = start; outer < end; ++outer) { + __fp16* out_ptr = output + outer * inner_size * output_shape[axis]; + size_t offset = 0; + + for (size_t input_idx = 0; input_idx < num_inputs; ++input_idx) { + const __fp16* in_ptr = inputs[input_idx] + outer * inner_size * input_shapes[input_idx][axis]; + size_t copy_size = input_shapes[input_idx][axis] * inner_size; + std::memcpy(out_ptr + offset, in_ptr, copy_size * sizeof(__fp16)); + offset += copy_size; + } + } + }); +} + +void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, size_t num_rows, size_t num_cols, size_t start_row, size_t end_row) { + constexpr size_t TILE_SIZE = 32; + constexpr size_t VECTOR_WIDTH = 8; + + for (size_t row_tile_start = start_row; row_tile_start < end_row; row_tile_start += TILE_SIZE) { + const size_t row_tile_end = std::min(row_tile_start + TILE_SIZE, end_row); + + for (size_t col_tile_start = 0; col_tile_start < num_cols; col_tile_start += TILE_SIZE) { + const size_t col_tile_end = std::min(col_tile_start + TILE_SIZE, num_cols); + + for (size_t row_block = row_tile_start; row_block < row_tile_end; row_block += VECTOR_WIDTH) { + const size_t row_block_end = std::min(row_block + VECTOR_WIDTH, row_tile_end); + + for (size_t col_block = col_tile_start; col_block < col_tile_end; col_block += VECTOR_WIDTH) { + const size_t col_block_end = std::min(col_block + VECTOR_WIDTH, col_tile_end); + + if (row_block_end - row_block >= 8 && col_block_end - col_block >= 8) { + float16x8_t rows[8]; + for (int i = 0; i < 8; i++) { + if (row_block + i < row_block_end) { + rows[i] = vld1q_f16(&source[(row_block + i) * num_cols + col_block]); + } else { + rows[i] = vdupq_n_f16(0.0f); + } + } + + float16x8x2_t r01 = vtrnq_f16(rows[0], rows[1]); + float16x8x2_t r23 = vtrnq_f16(rows[2], rows[3]); + float16x8x2_t r45 = vtrnq_f16(rows[4], rows[5]); + float16x8x2_t r67 = vtrnq_f16(rows[6], rows[7]); + + float32x4x2_t r0123_lo = vtrnq_f32( + vreinterpretq_f32_f16(r01.val[0]), + vreinterpretq_f32_f16(r23.val[0])); + float32x4x2_t r0123_hi = vtrnq_f32( + vreinterpretq_f32_f16(r01.val[1]), + vreinterpretq_f32_f16(r23.val[1])); + float32x4x2_t r4567_lo = vtrnq_f32( + vreinterpretq_f32_f16(r45.val[0]), + vreinterpretq_f32_f16(r67.val[0])); + float32x4x2_t r4567_hi = vtrnq_f32( + vreinterpretq_f32_f16(r45.val[1]), + vreinterpretq_f32_f16(r67.val[1])); + + float16x8_t col0 = vreinterpretq_f16_f32(vcombine_f32( + vget_low_f32(r0123_lo.val[0]), vget_low_f32(r4567_lo.val[0]))); + float16x8_t col1 = vreinterpretq_f16_f32(vcombine_f32( + vget_low_f32(r0123_hi.val[0]), vget_low_f32(r4567_hi.val[0]))); + float16x8_t col2 = vreinterpretq_f16_f32(vcombine_f32( + vget_low_f32(r0123_lo.val[1]), vget_low_f32(r4567_lo.val[1]))); + float16x8_t col3 = vreinterpretq_f16_f32(vcombine_f32( + vget_low_f32(r0123_hi.val[1]), vget_low_f32(r4567_hi.val[1]))); + float16x8_t col4 = vreinterpretq_f16_f32(vcombine_f32( + vget_high_f32(r0123_lo.val[0]), vget_high_f32(r4567_lo.val[0]))); + float16x8_t col5 = vreinterpretq_f16_f32(vcombine_f32( + vget_high_f32(r0123_hi.val[0]), vget_high_f32(r4567_hi.val[0]))); + float16x8_t col6 = vreinterpretq_f16_f32(vcombine_f32( + vget_high_f32(r0123_lo.val[1]), vget_high_f32(r4567_lo.val[1]))); + float16x8_t col7 = vreinterpretq_f16_f32(vcombine_f32( + vget_high_f32(r0123_hi.val[1]), vget_high_f32(r4567_hi.val[1]))); + + float16x8_t cols[8] = {col0, col1, col2, col3, col4, col5, col6, col7}; + for (int c = 0; c < 8 && col_block + c < col_block_end; c++) { + if (col_block + c < num_cols) { + if (row_block_end - row_block >= 8) { + vst1q_f16(&destination[(col_block + c) * num_rows + row_block], cols[c]); + } else { + __fp16 temp[8]; + vst1q_f16(temp, cols[c]); + for (size_t i = 0; i < row_block_end - row_block; ++i) { + destination[(col_block + c) * num_rows + row_block + i] = temp[i]; + } + } + } + } + } else { + for (size_t row = row_block; row < row_block_end; row++) { + for (size_t col = col_block; col < col_block_end; col++) { + destination[col * num_rows + row] = source[row * num_cols + col]; + } + } + } + } + } + } + } +} + +void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx) { + if (ndim == 2 && permutation[0] == 1 && permutation[1] == 0) { + size_t num_rows = shape[0]; + size_t num_cols = shape[1]; + + constexpr size_t THRESHOLD = 8192; + constexpr size_t TILE_ROWS = 32; + if (num_rows * num_cols >= THRESHOLD) { + const size_t num_row_blocks = (num_rows + TILE_ROWS - 1) / TILE_ROWS; + + CactusThreading::parallel_for(num_row_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [=](size_t start_block, size_t end_block) { + for (size_t block_idx = start_block; block_idx < end_block; ++block_idx) { + size_t start_row = block_idx * TILE_ROWS; + size_t end_row = std::min(start_row + TILE_ROWS, num_rows); + + cactus_transpose_2d_f16(source, destination, num_rows, num_cols, start_row, end_row); + } + }); + } else { + cactus_transpose_2d_f16(source, destination, num_rows, num_cols, 0, num_rows); + } + } else { + for (size_t idx = start_idx; idx < end_idx; ++idx) { + size_t src_idx = 0; + size_t tmp_idx = idx; + + for (size_t i = 0; i < ndim; ++i) { + size_t coord = tmp_idx % shape[permutation[ndim - 1 - i]]; + tmp_idx /= shape[permutation[ndim - 1 - i]]; + + size_t stride = 1; + for (size_t j = permutation[ndim - 1 - i] + 1; j < ndim; ++j) { + stride *= shape[j]; + } + src_idx += coord * stride; + } + + destination[idx] = source[src_idx]; + } + } +} diff --git a/cactus-transpile/kernel/kernel_conv.cpp b/cactus-transpile/kernel/kernel_conv.cpp new file mode 100644 index 000000000..c1d6b4f52 --- /dev/null +++ b/cactus-transpile/kernel/kernel_conv.cpp @@ -0,0 +1,1663 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#endif + +constexpr size_t T_TILE_F16 = 2; +constexpr size_t ACCELERATE_K_THRESHOLD = 32; +constexpr size_t ACCELERATE_L_THRESHOLD = 128; + +namespace { + +void fp16_rowmajor_gemm_rhs_transposed_to_fp32( + const __fp16* lhs, + const __fp16* rhs, + float* output, + size_t M, + size_t K, + size_t N +) { + if (M == 0 || K == 0 || N == 0) { + return; + } + +#ifdef __APPLE__ + std::vector lhs_f32(M * K); + std::vector rhs_f32(N * K); + for (size_t i = 0; i < lhs_f32.size(); ++i) { + lhs_f32[i] = static_cast(lhs[i]); + } + for (size_t i = 0; i < rhs_f32.size(); ++i) { + rhs_f32[i] = static_cast(rhs[i]); + } + + cblas_sgemm( + CblasRowMajor, + CblasNoTrans, + CblasTrans, + static_cast(M), + static_cast(N), + static_cast(K), + 1.0f, + lhs_f32.data(), + static_cast(K), + rhs_f32.data(), + static_cast(K), + 0.0f, + output, + static_cast(N) + ); + return; +#endif + + CactusThreading::parallel_for( + M, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t row_start, size_t row_end) { + for (size_t m = row_start; m < row_end; ++m) { + const __fp16* lhs_row = lhs + m * K; + float* out_row = output + m * N; + for (size_t n = 0; n < N; ++n) { + const __fp16* rhs_row = rhs + n * K; + float acc = 0.0f; + for (size_t k = 0; k < K; ++k) { + acc += static_cast(lhs_row[k]) * static_cast(rhs_row[k]); + } + out_row[n] = acc; + } + } + } + ); +} + +} // namespace + +#ifdef __APPLE__ +static void conv1d_causal_depthwise_f16_accelerate( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, size_t C, size_t K, size_t dilation) +{ + const size_t in_bs = L * C; + const size_t out_bs = L * C; + CactusThreading::parallel_for_2d(N, C, CactusThreading::Thresholds::ATTENTION, [&](size_t n, size_t c) { + const __fp16* Xb = input + n * in_bs; + __fp16* Yb = output + n * out_bs; + const __fp16* Wc = weight + c * K; + + if (dilation == 1) { + + size_t i = 0; + for(; i + 7 < L; i += 8){ + float16x8_t acc = vdupq_n_f16(0.0f); + + for (size_t k = 0; k < K; ++k) { + float16x8_t input_vec = vld1q_f16(&Xb[(i + k) * C + c]); + float16x8_t weight_vec = vdupq_n_f16(Wc[k]); + acc = vfmaq_f16(acc, input_vec, weight_vec); + } + vst1q_f16(&Yb[i * C + c], acc); + } + + for (; i < L; ++i) { + float acc = 0.0f; + for (size_t k = 0; k < K; ++k) { + acc += float(Xb[(i + k) * C + c]) * float(Wc[k]); + } + Yb[i * C + c] = (__fp16)acc; + } + } else { + + for (size_t i = 0; i < L; ++i) { + float acc = 0.0f; + + for (size_t k = 0; k < K; ++k) { + size_t idx = i + k * dilation; + acc += float(Xb[idx * C + c]) * float(Wc[k]); + } + + Yb[i * C + c] = (__fp16)acc; + } + } + }); +} +#endif + +void cactus_conv1d_causal_depthwise_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, size_t C, size_t K, size_t dilation) +{ +#ifdef __APPLE__ + if (K >= ACCELERATE_K_THRESHOLD && L >= ACCELERATE_L_THRESHOLD) { + conv1d_causal_depthwise_f16_accelerate(input, weight, output, N, L, C, K, dilation); + return; + } +#endif + + const size_t in_bs = L * C; + const size_t out_bs = L * C; + + CactusThreading::parallel_for_2d(N, C, CactusThreading::Thresholds::ATTENTION, [&](size_t n, size_t c) { + const __fp16* Xb = input + n * in_bs; + __fp16* Yb = output + n * out_bs; + + std::vector wrev(K); + const __fp16* Wc = weight + c * K; + for (size_t k = 0; k < K; ++k) wrev[k] = (float)Wc[K - 1 - k]; + + for (size_t t0 = 0; t0 < L; t0 += T_TILE_F16) { + const size_t t1 = std::min(t0 + 1, L - 1); + + float32x4_t vacc0 = vdupq_n_f32(0.f); + float32x4_t vacc1 = vdupq_n_f32(0.f); + + size_t k = 0; + for (; k + 8 <= K; k += 8) { + + float x0_0=0, x1_0=0, x2_0=0, x3_0=0; + float x0_1=0, x1_1=0, x2_1=0, x3_1=0; + { + ptrdiff_t a0=(ptrdiff_t)t0-(ptrdiff_t)((k+0)*dilation); + ptrdiff_t a1=(ptrdiff_t)t0-(ptrdiff_t)((k+1)*dilation); + ptrdiff_t a2=(ptrdiff_t)t0-(ptrdiff_t)((k+2)*dilation); + ptrdiff_t a3=(ptrdiff_t)t0-(ptrdiff_t)((k+3)*dilation); + if (a0>=0) x0_0 = (float)Xb[(size_t)a0*C + c]; + if (a1>=0) x1_0 = (float)Xb[(size_t)a1*C + c]; + if (a2>=0) x2_0 = (float)Xb[(size_t)a2*C + c]; + if (a3>=0) x3_0 = (float)Xb[(size_t)a3*C + c]; + + ptrdiff_t b0=(ptrdiff_t)t1-(ptrdiff_t)((k+0)*dilation); + ptrdiff_t b1=(ptrdiff_t)t1-(ptrdiff_t)((k+1)*dilation); + ptrdiff_t b2=(ptrdiff_t)t1-(ptrdiff_t)((k+2)*dilation); + ptrdiff_t b3=(ptrdiff_t)t1-(ptrdiff_t)((k+3)*dilation); + if (b0>=0) x0_1 = (float)Xb[(size_t)b0*C + c]; + if (b1>=0) x1_1 = (float)Xb[(size_t)b1*C + c]; + if (b2>=0) x2_1 = (float)Xb[(size_t)b2*C + c]; + if (b3>=0) x3_1 = (float)Xb[(size_t)b3*C + c]; + } + float32x4_t xv0 = {x0_0,x1_0,x2_0,x3_0}; + float32x4_t yv0 = {x0_1,x1_1,x2_1,x3_1}; + float32x4_t wv0 = {wrev[k+0],wrev[k+1],wrev[k+2],wrev[k+3]}; + vacc0 = vfmaq_f32(vacc0, xv0, wv0); + vacc1 = vfmaq_f32(vacc1, yv0, wv0); + + float a0_0=0, a1_0=0, a2_0=0, a3_0=0; + float a0_1=0, a1_1=0, a2_1=0, a3_1=0; + { + ptrdiff_t a0i=(ptrdiff_t)t0-(ptrdiff_t)((k+4)*dilation); + ptrdiff_t a1i=(ptrdiff_t)t0-(ptrdiff_t)((k+5)*dilation); + ptrdiff_t a2i=(ptrdiff_t)t0-(ptrdiff_t)((k+6)*dilation); + ptrdiff_t a3i=(ptrdiff_t)t0-(ptrdiff_t)((k+7)*dilation); + if (a0i>=0) a0_0 = (float)Xb[(size_t)a0i*C + c]; + if (a1i>=0) a1_0 = (float)Xb[(size_t)a1i*C + c]; + if (a2i>=0) a2_0 = (float)Xb[(size_t)a2i*C + c]; + if (a3i>=0) a3_0 = (float)Xb[(size_t)a3i*C + c]; + + ptrdiff_t b0i=(ptrdiff_t)t1-(ptrdiff_t)((k+4)*dilation); + ptrdiff_t b1i=(ptrdiff_t)t1-(ptrdiff_t)((k+5)*dilation); + ptrdiff_t b2i=(ptrdiff_t)t1-(ptrdiff_t)((k+6)*dilation); + ptrdiff_t b3i=(ptrdiff_t)t1-(ptrdiff_t)((k+7)*dilation); + if (b0i>=0) a0_1 = (float)Xb[(size_t)b0i*C + c]; + if (b1i>=0) a1_1 = (float)Xb[(size_t)b1i*C + c]; + if (b2i>=0) a2_1 = (float)Xb[(size_t)b2i*C + c]; + if (b3i>=0) a3_1 = (float)Xb[(size_t)b3i*C + c]; + } + float32x4_t xv1 = {a0_0,a1_0,a2_0,a3_0}; + float32x4_t yv1 = {a0_1,a1_1,a2_1,a3_1}; + float32x4_t wv1 = {wrev[k+4],wrev[k+5],wrev[k+6],wrev[k+7]}; + vacc0 = vfmaq_f32(vacc0, xv1, wv1); + vacc1 = vfmaq_f32(vacc1, yv1, wv1); + } + + float acc0 = vaddvq_f32(vacc0); + float acc1 = vaddvq_f32(vacc1); + + for (; k < K; ++k) { + ptrdiff_t a=(ptrdiff_t)t0-(ptrdiff_t)(k*dilation); + if (a>=0) acc0 += wrev[k] * (float)Xb[(size_t)a*C + c]; + ptrdiff_t b=(ptrdiff_t)t1-(ptrdiff_t)(k*dilation); + if (b>=0) acc1 += wrev[k] * (float)Xb[(size_t)b*C + c]; + } + + Yb[t0*C + c] = (__fp16)acc0; + if (t0 + 1 < L) Yb[t1*C + c] = (__fp16)acc1; + } + }); +} + +void cactus_conv1d_f16_k3( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t stride +){ + const size_t out_len = ((L - 1) / stride) + 1; + const size_t in_bs = C_in * L; + const size_t out_bs = C_out * out_len; + + const size_t total_compute = N * C_out * out_len * C_in * 3; + CactusThreading::ParallelConfig config = (total_compute < 100000) + ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} + : CactusThreading::Thresholds::ATTENTION; + + CactusThreading::parallel_for_2d(N, C_out, config, [&](size_t n, size_t oc) { + const __fp16* Xb = input + n * in_bs; + __fp16* Yoc = output + n * out_bs + oc * out_len; + const __fp16* Woc = weight + oc * (C_in * 3); + + for (size_t out_idx = 0; out_idx < out_len; out_idx += 2) { + const size_t out_t0 = out_idx; + const bool have_t1 = (out_idx + 1) < out_len; + const size_t out_t1 = have_t1 ? (out_idx + 1) : out_idx; + + const size_t t0 = out_t0 * stride; + const size_t t1 = have_t1 ? (out_t1 * stride) : t0; + + float32x4_t acc0 = vdupq_n_f32(0.f); + float32x4_t acc1 = vdupq_n_f32(0.f); + + size_t ic = 0; + for (; ic + 16 <= C_in; ic += 16) { + for (size_t u = 0; u < 16; ++u) { + const __fp16* Xc = Xb + (ic + u) * L; + const __fp16* Wc = Woc + (ic + u) * 3; + + const float16x8_t wv = { + Wc[0], Wc[1], Wc[2], (__fp16)0, + Wc[0], Wc[1], Wc[2], (__fp16)0 + }; + + const ptrdiff_t tm0 = (ptrdiff_t)t0 - 1; + const ptrdiff_t tp0 = (ptrdiff_t)t0 + 1; + const ptrdiff_t tm1 = (ptrdiff_t)t1 - 1; + const ptrdiff_t tp1 = (ptrdiff_t)t1 + 1; + + const __fp16 x0m = (tm0 >= 0) ? Xc[tm0] : (__fp16)0; + const __fp16 x00 = Xc[t0]; + const __fp16 x0p = (tp0 < (ptrdiff_t)L) ? Xc[tp0] : (__fp16)0; + + __fp16 x1m = 0, x10 = 0, x1p = 0; + if (have_t1) { + x1m = (tm1 >= 0) ? Xc[tm1] : (__fp16)0; + x10 = Xc[t1]; + x1p = (tp1 < (ptrdiff_t)L) ? Xc[tp1] : (__fp16)0; + } + + const float16x8_t xv = { + x0m, x00, x0p, (__fp16)0, + x1m, x10, x1p, (__fp16)0 + }; + + const float16x4_t xv0_h = vget_low_f16(xv); + const float16x4_t wv0_h = vget_low_f16(wv); + acc0 = vfmaq_f32(acc0, vcvt_f32_f16(xv0_h), vcvt_f32_f16(wv0_h)); + + if (have_t1) { + const float16x4_t xv1_h = vget_high_f16(xv); + const float16x4_t wv1_h = vget_high_f16(wv); + acc1 = vfmaq_f32(acc1, vcvt_f32_f16(xv1_h), vcvt_f32_f16(wv1_h)); + } + } + } + + for (; ic < C_in; ++ic) { + const __fp16* Xc = Xb + ic * L; + const __fp16* Wc = Woc + ic * 3; + + const float16x8_t wv = { + Wc[0], Wc[1], Wc[2], (__fp16)0, + Wc[0], Wc[1], Wc[2], (__fp16)0 + }; + + const ptrdiff_t tm0 = (ptrdiff_t)t0 - 1; + const ptrdiff_t tp0 = (ptrdiff_t)t0 + 1; + const ptrdiff_t tm1 = (ptrdiff_t)t1 - 1; + const ptrdiff_t tp1 = (ptrdiff_t)t1 + 1; + + const __fp16 x0m = (tm0 >= 0) ? Xc[tm0] : (__fp16)0; + const __fp16 x00 = Xc[t0]; + const __fp16 x0p = (tp0 < (ptrdiff_t)L) ? Xc[tp0] : (__fp16)0; + + __fp16 x1m = 0, x10 = 0, x1p = 0; + if (have_t1) { + x1m = (tm1 >= 0) ? Xc[tm1] : (__fp16)0; + x10 = Xc[t1]; + x1p = (tp1 < (ptrdiff_t)L) ? Xc[tp1] : (__fp16)0; + } + + const float16x8_t xv = { + x0m, x00, x0p, (__fp16)0, + x1m, x10, x1p, (__fp16)0 + }; + + const float16x4_t xv0_h = vget_low_f16(xv); + const float16x4_t wv0_h = vget_low_f16(wv); + acc0 = vfmaq_f32(acc0, vcvt_f32_f16(xv0_h), vcvt_f32_f16(wv0_h)); + + if (have_t1) { + const float16x4_t xv1_h = vget_high_f16(xv); + const float16x4_t wv1_h = vget_high_f16(wv); + acc1 = vfmaq_f32(acc1, vcvt_f32_f16(xv1_h), vcvt_f32_f16(wv1_h)); + } + } + + float32x2_t s0 = vadd_f32(vget_low_f32(acc0), vget_high_f32(acc0)); + float sum0 = vget_lane_f32(s0, 0) + vget_lane_f32(s0, 1); + Yoc[out_t0] = (__fp16)sum0; + + if (have_t1) { + float32x2_t s1 = vadd_f32(vget_low_f32(acc1), vget_high_f32(acc1)); + float sum1 = vget_lane_f32(s1, 0) + vget_lane_f32(s1, 1); + Yoc[out_t1] = (__fp16)sum1; + } + } + }); +} + +#ifdef __APPLE__ +static void conv1d_f16_accelerate( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, + size_t stride +){ + const size_t out_len = ((L - K) / stride) + 1; + const size_t in_bs = C_in * L; + const size_t out_bs = C_out * out_len; + + const size_t total_compute = N * C_out * out_len * C_in * K; + CactusThreading::ParallelConfig config = (total_compute < 100000) + ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} + : CactusThreading::Thresholds::ATTENTION; + + CactusThreading::parallel_for_2d( + N, C_out, config, + [&](size_t n, size_t oc) { + + const __fp16* Xb = input + n * in_bs; + __fp16* Yoc = output + n * out_bs + oc * out_len; + const __fp16* Woc = weight + oc * (C_in * K); + const float b = bias ? (float)bias[oc] : 0.f; + + std::vector out_f32(out_len, b); + std::vector input_f32(L); + std::vector weight_f32(K); + + for (size_t ic = 0; ic < C_in; ++ic) { + const __fp16* Xc = Xb + ic * L; + const __fp16* Wc = Woc + ic * K; + + for (size_t i = 0; i < L; ++i) input_f32[i] = (float)Xc[i]; + for (size_t k = 0; k < K; ++k) weight_f32[k] = (float)Wc[k]; + + if (stride == 1) { + std::vector conv_out(out_len); + vDSP_conv(input_f32.data(), 1, weight_f32.data(), 1, + conv_out.data(), 1, out_len, K); + vDSP_vadd(out_f32.data(), 1, conv_out.data(), 1, + out_f32.data(), 1, out_len); + } else { + std::vector full_conv(L - K + 1); + vDSP_conv(input_f32.data(), 1, weight_f32.data(), 1, + full_conv.data(), 1, L - K + 1, K); + for (size_t out_t = 0; out_t < out_len; ++out_t) { + out_f32[out_t] += full_conv[out_t * stride]; + } + } + } + + for (size_t out_t = 0; out_t < out_len; ++out_t) { + Yoc[out_t] = (__fp16)out_f32[out_t]; + } + }); +} +#endif + +static void conv1d_f16_neon( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, + size_t stride +){ + const size_t out_len = ((L - K) / stride) + 1; + const size_t in_bs = C_in * L; + const size_t out_bs = C_out * out_len; + + const size_t total_compute = N * C_out * out_len * C_in * K; + CactusThreading::ParallelConfig config = (total_compute < 100000) + ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} + : CactusThreading::Thresholds::ATTENTION; + + CactusThreading::parallel_for_2d( + N, C_out, config, + [&](size_t n, size_t oc) { + + const __fp16* Xb = input + n * in_bs; + __fp16* Yoc = output + n * out_bs + oc * out_len; + const __fp16* Woc = weight + oc * (C_in * K); + const float b = bias ? (float)bias[oc] : 0.f; + + for (size_t out_t = 0; out_t < out_len; ++out_t) { + const size_t t = out_t * stride; + float sum = b; + + for (size_t ic = 0; ic < C_in; ++ic) { + const __fp16* Xc = Xb + ic * L + t; + const __fp16* Wc = Woc + ic * K; + + float32x4_t acc0 = vdupq_n_f32(0.f); + float32x4_t acc1 = vdupq_n_f32(0.f); + + size_t k = 0; + + for (; k + 8 <= K; k += 8) { + const float16x8_t xv = vld1q_f16(Xc + k); + const float16x8_t wv = vld1q_f16(Wc + k); + + acc0 = vfmaq_f32(acc0, + vcvt_f32_f16(vget_low_f16(xv)), + vcvt_f32_f16(vget_low_f16(wv))); + acc1 = vfmaq_f32(acc1, + vcvt_f32_f16(vget_high_f16(xv)), + vcvt_f32_f16(vget_high_f16(wv))); + } + + float32x2_t s1 = vadd_f32(vget_low_f32(acc0), vget_high_f32(acc0)); + sum += vget_lane_f32(s1, 0) + vget_lane_f32(s1, 1); + + float32x2_t s2 = vadd_f32(vget_low_f32(acc1), vget_high_f32(acc1)); + sum += vget_lane_f32(s2, 0) + vget_lane_f32(s2, 1); + + for (; k < K; ++k) { + sum += (float)Xc[k] * (float)Wc[k]; + } + } + + Yoc[out_t] = (__fp16)sum; + } + }); +} + +#ifdef __APPLE__ +static void conv1d_f16_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, + size_t stride +) { + const size_t out_len = (L - K) / stride + 1; + const size_t col_K = C_in * K; + + std::vector W_f32(C_out * col_K); + for (size_t i = 0; i < C_out * col_K; ++i) + W_f32[i] = static_cast(weight[i]); + + std::vector bias_f32; + if (bias) { + bias_f32.resize(C_out); + for (size_t i = 0; i < C_out; ++i) + bias_f32[i] = static_cast(bias[i]); + } + + std::vector col(col_K * out_len); + std::vector Y_f32(C_out * out_len); + + for (size_t n = 0; n < N; ++n) { + const __fp16* Xn = input + n * C_in * L; + __fp16* Yn = output + n * C_out * out_len; + + if (stride == 1) { + for (size_t ic = 0; ic < C_in; ++ic) { + const __fp16* Xc = Xn + ic * L; + for (size_t k = 0; k < K; ++k) { + float* dst = col.data() + (ic * K + k) * out_len; + const __fp16* src = Xc + k; + size_t t = 0; + for (; t + 8 <= out_len; t += 8) { + float16x8_t v = vld1q_f16(src + t); + vst1q_f32(dst + t, vcvt_f32_f16(vget_low_f16(v))); + vst1q_f32(dst + t + 4, vcvt_f32_f16(vget_high_f16(v))); + } + for (; t < out_len; ++t) + dst[t] = static_cast(src[t]); + } + } + } else { + for (size_t ic = 0; ic < C_in; ++ic) { + const __fp16* Xc = Xn + ic * L; + for (size_t k = 0; k < K; ++k) { + float* dst = col.data() + (ic * K + k) * out_len; + for (size_t t = 0; t < out_len; ++t) + dst[t] = static_cast(Xc[t * stride + k]); + } + } + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + static_cast(C_out), static_cast(out_len), static_cast(col_K), + 1.0f, W_f32.data(), static_cast(col_K), + col.data(), static_cast(out_len), + 0.0f, Y_f32.data(), static_cast(out_len)); + + if (bias) { + for (size_t oc = 0; oc < C_out; ++oc) { + float b = bias_f32[oc]; + const float* src = Y_f32.data() + oc * out_len; + __fp16* dst = Yn + oc * out_len; + size_t t = 0; + for (; t + 8 <= out_len; t += 8) { + float32x4_t v0 = vaddq_f32(vld1q_f32(src + t), vdupq_n_f32(b)); + float32x4_t v1 = vaddq_f32(vld1q_f32(src + t + 4), vdupq_n_f32(b)); + vst1q_f16(dst + t, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); + } + for (; t < out_len; ++t) + dst[t] = static_cast<__fp16>(src[t] + b); + } + } else { + for (size_t oc = 0; oc < C_out; ++oc) { + const float* src = Y_f32.data() + oc * out_len; + __fp16* dst = Yn + oc * out_len; + size_t t = 0; + for (; t + 8 <= out_len; t += 8) { + float32x4_t v0 = vld1q_f32(src + t); + float32x4_t v1 = vld1q_f32(src + t + 4); + vst1q_f16(dst + t, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); + } + for (; t < out_len; ++t) + dst[t] = static_cast<__fp16>(src[t]); + } + } + } +} +#endif + +void cactus_conv1d_f16( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, + size_t stride +){ +#ifdef __APPLE__ + if (stride > 1 || K < ACCELERATE_K_THRESHOLD) { + conv1d_f16_gemm(input, weight, bias, output, N, L, C_in, C_out, K, stride); + return; + } + if (K >= ACCELERATE_K_THRESHOLD && L >= ACCELERATE_L_THRESHOLD) { + conv1d_f16_accelerate(input, weight, bias, output, N, L, C_in, C_out, K, stride); + return; + } +#endif + conv1d_f16_neon(input, weight, bias, output, N, L, C_in, C_out, K, stride); +} + +inline void conv1d_k7s3_oc8_t4( + const __fp16* Xb, + const __fp16* Wpack, + const __fp16* bias, + __fp16* Yb, + size_t L, + size_t out_len, + size_t C_in, + size_t C_out, + size_t out_t, + size_t oc0 +){ + float32x4_t acc[4][2]; + float32x4_t b0 = vdupq_n_f32(0.f); + float32x4_t b1 = vdupq_n_f32(0.f); + if (bias) { + float16x8_t bv = vld1q_f16(bias + oc0); + b0 = vcvt_f32_f16(vget_low_f16(bv)); + b1 = vcvt_f32_f16(vget_high_f16(bv)); + } + + for (int j = 0; j < 4; ++j) { + acc[j][0] = b0; + acc[j][1] = b1; + } + + const size_t t_base = out_t * 3; + + for (size_t ic = 0; ic < C_in; ++ic) { + const __fp16* Wic = Wpack + (ic * 7) * C_out + oc0; + const __fp16* Xic = Xb + ic * L + t_base; + + for (int k = 0; k < 7; ++k) { + float16x8_t w_half = vld1q_f16(Wic + k * C_out); + float32x4_t w0 = vcvt_f32_f16(vget_low_f16(w_half)); + float32x4_t w1 = vcvt_f32_f16(vget_high_f16(w_half)); + for (int j = 0; j < 4; ++j) { + float x_val = (float)Xic[j * 3 + k]; + float32x4_t xv = vdupq_n_f32(x_val); + + acc[j][0] = vfmaq_f32(acc[j][0], w0, xv); + acc[j][1] = vfmaq_f32(acc[j][1], w1, xv); + } + } + } + + float tmp0[4], tmp1[4]; + for(int j=0; j<4; ++j) { + vst1q_f32(tmp0, acc[j][0]); + vst1q_f32(tmp1, acc[j][1]); + + for(int i=0; i<4; ++i) { + Yb[(oc0 + i) * out_len + out_t + j] = (__fp16)tmp0[i]; + } + for(int i=0; i<4; ++i) { + Yb[(oc0 + 4 + i) * out_len + out_t + j] = (__fp16)tmp1[i]; + } + } +} + +inline void conv1d_k7s3_oc8_scalar( + const __fp16* Xb, + const __fp16* Wpack, + const __fp16* bias, + __fp16* Yb, + size_t L, + size_t out_len, + size_t C_in, + size_t C_out, + size_t out_t, + size_t oc0 +){ + float32x4_t acc0 = vdupq_n_f32(0.f); + float32x4_t acc1 = vdupq_n_f32(0.f); + + if (bias) { + float16x8_t bv = vld1q_f16(bias + oc0); + acc0 = vcvt_f32_f16(vget_low_f16(bv)); + acc1 = vcvt_f32_f16(vget_high_f16(bv)); + } + + const size_t t_base = out_t * 3; + + for (size_t ic = 0; ic < C_in; ++ic) { + const __fp16* Wic = Wpack + (ic * 7) * C_out + oc0; + const __fp16* Xic = Xb + ic * L + t_base; + for (int k = 0; k < 7; ++k) { + float x_val = (float)Xic[k]; + float32x4_t xv = vdupq_n_f32(x_val); + + float16x8_t w_half = vld1q_f16(Wic + k * C_out); + acc0 = vfmaq_f32(acc0, vcvt_f32_f16(vget_low_f16(w_half)), xv); + acc1 = vfmaq_f32(acc1, vcvt_f32_f16(vget_high_f16(w_half)), xv); + } + } + + float tmp[8]; + vst1q_f32(tmp, acc0); + vst1q_f32(tmp+4, acc1); + + for(int i=0; i<8; ++i) { + if (oc0 + i < C_out) { + Yb[(oc0 + i) * out_len + out_t] = (__fp16)tmp[i]; + } + } +} + +void cactus_conv1d_f16_k7s3_oc8( + const __fp16* input, + const __fp16* Wpack, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out) +{ + if (L < 7) return; + size_t out_len = (L - 7) / 3 + 1; + size_t num_oc_blocks = (C_out + 7) / 8; + + CactusThreading::parallel_for_2d(N, num_oc_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, [=](size_t n, size_t ob) { + size_t oc0 = ob * 8; + const __fp16* Xb = input + n * (C_in * L); + __fp16* Yb = output + n * (C_out * out_len); + + size_t out_t = 0; + for (; out_t + 4 <= out_len; out_t += 4) { + conv1d_k7s3_oc8_t4(Xb, Wpack, bias, Yb, L, out_len, C_in, C_out, out_t, oc0); + } + + for (; out_t < out_len; ++out_t) { + conv1d_k7s3_oc8_scalar(Xb, Wpack, bias, Yb, L, out_len, C_in, C_out, out_t, oc0); + } + }); +} + + + +void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, + size_t dst_height, size_t dst_width, bool align_corners) +{ + float scale_h, scale_w; + if (align_corners) { + scale_h = (src_height > 1 && dst_height > 1) + ? static_cast(src_height - 1) / static_cast(dst_height - 1) + : 0.0f; + scale_w = (src_width > 1 && dst_width > 1) + ? static_cast(src_width - 1) / static_cast(dst_width - 1) + : 0.0f; + } else { + scale_h = (dst_height > 0) + ? static_cast(src_height) / static_cast(dst_height) + : 0.0f; + scale_w = (dst_width > 0) + ? static_cast(src_width) / static_cast(dst_width) + : 0.0f; + } + + for (size_t dst_y = 0; dst_y < dst_height; ++dst_y) { + for (size_t dst_x = 0; dst_x < dst_width; ++dst_x) { + float src_y_float, src_x_float; + if (align_corners) { + src_y_float = static_cast(dst_y) * scale_h; + src_x_float = static_cast(dst_x) * scale_w; + } else { + src_y_float = (static_cast(dst_y) + 0.5f) * scale_h - 0.5f; + src_x_float = (static_cast(dst_x) + 0.5f) * scale_w - 0.5f; + if (src_y_float < 0.0f) src_y_float = 0.0f; + if (src_x_float < 0.0f) src_x_float = 0.0f; + const float max_y = static_cast(src_height) - 1.0f; + const float max_x = static_cast(src_width) - 1.0f; + if (src_y_float > max_y) src_y_float = max_y; + if (src_x_float > max_x) src_x_float = max_x; + } + + int y0 = static_cast(std::floor(src_y_float)); + int x0 = static_cast(std::floor(src_x_float)); + + int y1 = ((y0 + 1) < static_cast(src_height)) ? (y0 + 1) : (static_cast(src_height) - 1); + int x1 = ((x0 + 1) < static_cast(src_width)) ? (x0 + 1) : (static_cast(src_width) - 1); + + float dy = src_y_float - y0; + float dx = src_x_float - x0; + + float w00 = (1.0f - dx) * (1.0f - dy); + float w01 = dx * (1.0f - dy); + float w10 = (1.0f - dx) * dy; + float w11 = dx * dy; + + size_t idx00 = (y0 * static_cast(src_width) + x0) * static_cast(embed_dim); + size_t idx01 = (y0 * static_cast(src_width) + x1) * static_cast(embed_dim); + size_t idx10 = (y1 * static_cast(src_width) + x0) * static_cast(embed_dim); + size_t idx11 = (y1 * static_cast(src_width) + x1) * static_cast(embed_dim); + + size_t out_idx = (dst_y * dst_width + dst_x) * embed_dim; + + size_t d = 0; + for(; d + 8 <= embed_dim; d += 8){ + float16x8_t v00 = vld1q_f16(input + idx00 + d); + float16x8_t v01 = vld1q_f16(input + idx01 + d); + float16x8_t v10 = vld1q_f16(input + idx10 + d); + float16x8_t v11 = vld1q_f16(input + idx11 + d); + + float16x8_t result = vfmaq_f16( + vfmaq_f16( + vfmaq_f16(vdupq_n_f16(0.0f), v00, vdupq_n_f16(w00)), + v01, vdupq_n_f16(w01)), + v10, vdupq_n_f16(w10)); + result = vfmaq_f16(result, v11, vdupq_n_f16(w11)); + + vst1q_f16(output + out_idx + d, result); + } + + for (; d < embed_dim; ++d) { + output[out_idx + d] = + input[idx00 + d] * w00 + + input[idx01 + d] * w01 + + input[idx10 + d] * w10 + + input[idx11 + d] * w11; + } + } + } +} + +void cactus_stft_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t /*C_out*/, + size_t K, size_t stride, + size_t num_fft_bins +) { + const size_t out_len = ((L - K) / stride) + 1; + const size_t in_bs = C_in * L; + const size_t out_bs = 2 * num_fft_bins * out_len; + + for (size_t n = 0; n < N; ++n) { + const __fp16* Xb = input + n * in_bs; + + for (size_t bin = 0; bin < num_fft_bins; ++bin) { + const __fp16* Wr = weight + bin * (C_in * K); + const __fp16* Wi = weight + (bin + num_fft_bins) * (C_in * K); + + for (size_t out_t = 0; out_t < out_len; ++out_t) { + const size_t t = out_t * stride; + float sum_real = 0.0f; + float sum_imag = 0.0f; + + for (size_t ic = 0; ic < C_in; ++ic) { + const __fp16* Xc = Xb + ic * L + t; + const __fp16* Wrc = Wr + ic * K; + const __fp16* Wic = Wi + ic * K; + + float32x4_t acc_r0 = vdupq_n_f32(0.f); + float32x4_t acc_r1 = vdupq_n_f32(0.f); + float32x4_t acc_i0 = vdupq_n_f32(0.f); + float32x4_t acc_i1 = vdupq_n_f32(0.f); + + size_t k = 0; + for (; k + 8 <= K; k += 8) { + const float16x8_t xv = vld1q_f16(Xc + k); + const float16x8_t wr = vld1q_f16(Wrc + k); + const float16x8_t wi = vld1q_f16(Wic + k); + + acc_r0 = vfmaq_f32(acc_r0, vcvt_f32_f16(vget_low_f16(xv)), vcvt_f32_f16(vget_low_f16(wr))); + acc_r1 = vfmaq_f32(acc_r1, vcvt_f32_f16(vget_high_f16(xv)), vcvt_f32_f16(vget_high_f16(wr))); + acc_i0 = vfmaq_f32(acc_i0, vcvt_f32_f16(vget_low_f16(xv)), vcvt_f32_f16(vget_low_f16(wi))); + acc_i1 = vfmaq_f32(acc_i1, vcvt_f32_f16(vget_high_f16(xv)), vcvt_f32_f16(vget_high_f16(wi))); + } + + sum_real += vaddvq_f32(acc_r0) + vaddvq_f32(acc_r1); + sum_imag += vaddvq_f32(acc_i0) + vaddvq_f32(acc_i1); + + for (; k < K; ++k) { + float x = (float)Xc[k]; + sum_real += x * (float)Wrc[k]; + sum_imag += x * (float)Wic[k]; + } + } + + output[n * out_bs + bin * out_len + out_t] = (__fp16)sum_real; + output[n * out_bs + (bin + num_fft_bins) * out_len + out_t] = (__fp16)sum_imag; + } + } + } +} + +void cactus_conv1d_same_depthwise_f16_k9( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, size_t L, size_t C) +{ + constexpr int K = 9; + + const size_t bs = L * C; + + CactusThreading::parallel_for_2d( + N, C, CactusThreading::Thresholds::ATTENTION, + [&](size_t n, size_t c) { + + const __fp16* Xb = input + n * bs; + __fp16* Yb = output + n * bs; + + const __fp16* Wc = weight + c * K; + const float b = bias ? (float)bias[c] : 0.0f; + + const float w0 = (float)Wc[0]; + const float w1 = (float)Wc[1]; + const float w2 = (float)Wc[2]; + const float w3 = (float)Wc[3]; + const float w4 = (float)Wc[4]; + const float w5 = (float)Wc[5]; + const float w6 = (float)Wc[6]; + const float w7 = (float)Wc[7]; + const float w8 = (float)Wc[8]; + + const float32x4_t wv0 = {w0, w1, w2, w3}; + const float32x4_t wv1 = {w4, w5, w6, w7}; + + for (size_t t0 = 0; t0 < L; t0 += T_TILE_F16) { + const bool have_t1 = (t0 + 1 < L); + const size_t t1 = have_t1 ? (t0 + 1) : t0; + + float x0_0=0, x1_0=0, x2_0=0, x3_0=0; + float x4_0=0, x5_0=0, x6_0=0, x7_0=0; + float x8_0=0; + + { + const ptrdiff_t i0 = (ptrdiff_t)t0 - 4; + const ptrdiff_t i1 = (ptrdiff_t)t0 - 3; + const ptrdiff_t i2 = (ptrdiff_t)t0 - 2; + const ptrdiff_t i3 = (ptrdiff_t)t0 - 1; + if (i0 >= 0 && i0 < (ptrdiff_t)L) x0_0 = (float)Xb[(size_t)i0 * C + c]; + if (i1 >= 0 && i1 < (ptrdiff_t)L) x1_0 = (float)Xb[(size_t)i1 * C + c]; + if (i2 >= 0 && i2 < (ptrdiff_t)L) x2_0 = (float)Xb[(size_t)i2 * C + c]; + if (i3 >= 0 && i3 < (ptrdiff_t)L) x3_0 = (float)Xb[(size_t)i3 * C + c]; + + const ptrdiff_t j4 = (ptrdiff_t)t0 + 0; + const ptrdiff_t j5 = (ptrdiff_t)t0 + 1; + const ptrdiff_t j6 = (ptrdiff_t)t0 + 2; + const ptrdiff_t j7 = (ptrdiff_t)t0 + 3; + const ptrdiff_t j8 = (ptrdiff_t)t0 + 4; + if (j4 >= 0 && j4 < (ptrdiff_t)L) x4_0 = (float)Xb[(size_t)j4 * C + c]; + if (j5 >= 0 && j5 < (ptrdiff_t)L) x5_0 = (float)Xb[(size_t)j5 * C + c]; + if (j6 >= 0 && j6 < (ptrdiff_t)L) x6_0 = (float)Xb[(size_t)j6 * C + c]; + if (j7 >= 0 && j7 < (ptrdiff_t)L) x7_0 = (float)Xb[(size_t)j7 * C + c]; + if (j8 >= 0 && j8 < (ptrdiff_t)L) x8_0 = (float)Xb[(size_t)j8 * C + c]; + } + + float32x4_t acc0v = vdupq_n_f32(0.f); + const float32x4_t xv0_0 = {x0_0, x1_0, x2_0, x3_0}; + const float32x4_t xv1_0 = {x4_0, x5_0, x6_0, x7_0}; + acc0v = vfmaq_f32(acc0v, xv0_0, wv0); + acc0v = vfmaq_f32(acc0v, xv1_0, wv1); + float acc0 = vaddvq_f32(acc0v) + (w8 * x8_0) + b; + + float acc1 = 0.f; + if (have_t1) { + float x0_1=0, x1_1=0, x2_1=0, x3_1=0; + float x4_1=0, x5_1=0, x6_1=0, x7_1=0; + float x8_1=0; + + { + const ptrdiff_t i0 = (ptrdiff_t)t1 - 4; + const ptrdiff_t i1 = (ptrdiff_t)t1 - 3; + const ptrdiff_t i2 = (ptrdiff_t)t1 - 2; + const ptrdiff_t i3 = (ptrdiff_t)t1 - 1; + if (i0 >= 0 && i0 < (ptrdiff_t)L) x0_1 = (float)Xb[(size_t)i0 * C + c]; + if (i1 >= 0 && i1 < (ptrdiff_t)L) x1_1 = (float)Xb[(size_t)i1 * C + c]; + if (i2 >= 0 && i2 < (ptrdiff_t)L) x2_1 = (float)Xb[(size_t)i2 * C + c]; + if (i3 >= 0 && i3 < (ptrdiff_t)L) x3_1 = (float)Xb[(size_t)i3 * C + c]; + + const ptrdiff_t j4 = (ptrdiff_t)t1 + 0; + const ptrdiff_t j5 = (ptrdiff_t)t1 + 1; + const ptrdiff_t j6 = (ptrdiff_t)t1 + 2; + const ptrdiff_t j7 = (ptrdiff_t)t1 + 3; + const ptrdiff_t j8 = (ptrdiff_t)t1 + 4; + if (j4 >= 0 && j4 < (ptrdiff_t)L) x4_1 = (float)Xb[(size_t)j4 * C + c]; + if (j5 >= 0 && j5 < (ptrdiff_t)L) x5_1 = (float)Xb[(size_t)j5 * C + c]; + if (j6 >= 0 && j6 < (ptrdiff_t)L) x6_1 = (float)Xb[(size_t)j6 * C + c]; + if (j7 >= 0 && j7 < (ptrdiff_t)L) x7_1 = (float)Xb[(size_t)j7 * C + c]; + if (j8 >= 0 && j8 < (ptrdiff_t)L) x8_1 = (float)Xb[(size_t)j8 * C + c]; + } + + float32x4_t acc1v = vdupq_n_f32(0.f); + const float32x4_t xv0_1 = {x0_1, x1_1, x2_1, x3_1}; + const float32x4_t xv1_1 = {x4_1, x5_1, x6_1, x7_1}; + acc1v = vfmaq_f32(acc1v, xv0_1, wv0); + acc1v = vfmaq_f32(acc1v, xv1_1, wv1); + acc1 = vaddvq_f32(acc1v) + (w8 * x8_1) + b; + } + + Yb[t0 * C + c] = (__fp16)acc0; + if (have_t1) Yb[t1 * C + c] = (__fp16)acc1; + } + }); +} + + +void cactus_conv2d_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, size_t H, size_t W, + size_t C_out +){ + if (H + 2 < 3 || W + 2 < 3) return; + const size_t H_out = (H - 1) / 2 + 1; + const size_t W_out = (W - 1) / 2 + 1; + +#ifdef __APPLE__ + { + const size_t spatial = H_out * W_out; + const size_t col_K = C_in * 9; + std::vector W_f32(C_out * col_K); + for (size_t oc = 0; oc < C_out; ++oc) + for (size_t ic = 0; ic < C_in; ++ic) + for (size_t kh = 0; kh < 3; ++kh) + for (size_t kw = 0; kw < 3; ++kw) + W_f32[oc * col_K + ic * 9 + kh * 3 + kw] = + static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); + + std::vector bias_f32(C_out, 0.0f); + if (bias) + for (size_t i = 0; i < C_out; ++i) + bias_f32[i] = static_cast(bias[i]); + + std::vector col(col_K * spatial); + std::vector Y_f32(C_out * spatial); + + for (size_t n = 0; n < N; ++n) { + const __fp16* Xn = input + n * C_in * H * W; + __fp16* Yn = output + n * C_out * spatial; + + for (size_t ic = 0; ic < C_in; ++ic) { + for (size_t kh = 0; kh < 3; ++kh) { + for (size_t kw = 0; kw < 3; ++kw) { + float* dst = col.data() + (ic * 9 + kh * 3 + kw) * spatial; + for (size_t oh = 0; oh < H_out; ++oh) { + ptrdiff_t ih = static_cast(oh) * 2 + static_cast(kh) - 1; + float* dst_row = dst + oh * W_out; + if (ih < 0 || ih >= static_cast(H)) { + memset(dst_row, 0, W_out * sizeof(float)); + continue; + } + const __fp16* src_row = Xn + ic * H * W + static_cast(ih) * W; + const ptrdiff_t iw_offset = static_cast(kw) - 1; + for (size_t ow = 0; ow < W_out; ++ow) { + ptrdiff_t iw = static_cast(ow) * 2 + iw_offset; + dst_row[ow] = (iw < 0 || iw >= static_cast(W)) + ? 0.0f : static_cast(src_row[iw]); + } + } + } + } + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + static_cast(C_out), static_cast(spatial), static_cast(col_K), + 1.0f, W_f32.data(), static_cast(col_K), + col.data(), static_cast(spatial), + 0.0f, Y_f32.data(), static_cast(spatial)); + + for (size_t oc = 0; oc < C_out; ++oc) { + const float b = bias_f32[oc]; + const float* src = Y_f32.data() + oc * spatial; + __fp16* dst = Yn + oc * spatial; + size_t i = 0; + for (; i + 8 <= spatial; i += 8) { + float32x4_t v0 = vaddq_f32(vld1q_f32(src + i), vdupq_n_f32(b)); + float32x4_t v1 = vaddq_f32(vld1q_f32(src + i + 4), vdupq_n_f32(b)); + vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); + } + for (; i < spatial; ++i) + dst[i] = static_cast<__fp16>(src[i] + b); + } + } + return; + } +#endif + + auto in_idx = [&](size_t n, size_t ic, size_t ih, size_t iw) -> size_t { + return ((n * C_in + ic) * H + ih) * W + iw; + }; + auto w_idx = [&](size_t oc, size_t ic, size_t kh, size_t kw) -> size_t { + return (((oc * C_in + ic) * 3 + kh) * 3 + kw); + }; + auto out_idx = [&](size_t n, size_t oc, size_t oh, size_t ow) -> size_t { + return ((n * C_out + oc) * H_out + oh) * W_out + ow; + }; + + constexpr size_t OW_VEC = 4; + + const size_t total_compute = N * C_out * H_out * W_out * C_in * 9; + CactusThreading::ParallelConfig cfg = + (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} + : CactusThreading::Thresholds::ATTENTION; + + CactusThreading::parallel_for_2d(N, C_out, cfg, [&](size_t n, size_t oc) { + const float b0 = bias ? (float)bias[oc] : 0.0f; + + for (size_t oh = 0; oh < H_out; ++oh) { + const ptrdiff_t ih0 = (ptrdiff_t)oh * 2 - 1; + + const bool h_interior = (ih0 >= 0) && ((size_t)(ih0 + 2) < H); + + for (size_t ow = 0; ow < W_out; ) { + const size_t rem = W_out - ow; + + const bool do_vec = (rem >= OW_VEC); + const bool w_interior_vec = + do_vec && + (ow >= 1) && + (((ow + (OW_VEC - 1)) * 2 + 1) < W); + + float32x4_t vacc = vdupq_n_f32(b0); + + if (h_interior && w_interior_vec) { + for (size_t ic = 0; ic < C_in; ++ic) { + + const __fp16* row0 = input + in_idx(n, ic, (size_t)(ih0 + 0), 0); + const __fp16* row1 = input + in_idx(n, ic, (size_t)(ih0 + 1), 0); + const __fp16* row2 = input + in_idx(n, ic, (size_t)(ih0 + 2), 0); + const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; + { + const float w00 = (float)weight[w_idx(oc, ic, 0, 0)]; + const float w01 = (float)weight[w_idx(oc, ic, 0, 1)]; + const float w02 = (float)weight[w_idx(oc, ic, 0, 2)]; + + float x0 = (float)row0[(size_t)(iw_base0 + 0)]; + float x1 = (float)row0[(size_t)(iw_base0 + 2)]; + float x2 = (float)row0[(size_t)(iw_base0 + 4)]; + float x3 = (float)row0[(size_t)(iw_base0 + 6)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w00)); + + x0 = (float)row0[(size_t)(iw_base0 + 1)]; + x1 = (float)row0[(size_t)(iw_base0 + 3)]; + x2 = (float)row0[(size_t)(iw_base0 + 5)]; + x3 = (float)row0[(size_t)(iw_base0 + 7)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w01)); + + x0 = (float)row0[(size_t)(iw_base0 + 2)]; + x1 = (float)row0[(size_t)(iw_base0 + 4)]; + x2 = (float)row0[(size_t)(iw_base0 + 6)]; + x3 = (float)row0[(size_t)(iw_base0 + 8)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w02)); + } + + { + const float w10 = (float)weight[w_idx(oc, ic, 1, 0)]; + const float w11 = (float)weight[w_idx(oc, ic, 1, 1)]; + const float w12 = (float)weight[w_idx(oc, ic, 1, 2)]; + const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; + + float x0 = (float)row1[(size_t)(iw_base0 + 0)]; + float x1 = (float)row1[(size_t)(iw_base0 + 2)]; + float x2 = (float)row1[(size_t)(iw_base0 + 4)]; + float x3 = (float)row1[(size_t)(iw_base0 + 6)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w10)); + + x0 = (float)row1[(size_t)(iw_base0 + 1)]; + x1 = (float)row1[(size_t)(iw_base0 + 3)]; + x2 = (float)row1[(size_t)(iw_base0 + 5)]; + x3 = (float)row1[(size_t)(iw_base0 + 7)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w11)); + + x0 = (float)row1[(size_t)(iw_base0 + 2)]; + x1 = (float)row1[(size_t)(iw_base0 + 4)]; + x2 = (float)row1[(size_t)(iw_base0 + 6)]; + x3 = (float)row1[(size_t)(iw_base0 + 8)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w12)); + } + + { + const float w20 = (float)weight[w_idx(oc, ic, 2, 0)]; + const float w21 = (float)weight[w_idx(oc, ic, 2, 1)]; + const float w22 = (float)weight[w_idx(oc, ic, 2, 2)]; + const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; + + float x0 = (float)row2[(size_t)(iw_base0 + 0)]; + float x1 = (float)row2[(size_t)(iw_base0 + 2)]; + float x2 = (float)row2[(size_t)(iw_base0 + 4)]; + float x3 = (float)row2[(size_t)(iw_base0 + 6)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w20)); + + x0 = (float)row2[(size_t)(iw_base0 + 1)]; + x1 = (float)row2[(size_t)(iw_base0 + 3)]; + x2 = (float)row2[(size_t)(iw_base0 + 5)]; + x3 = (float)row2[(size_t)(iw_base0 + 7)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w21)); + + x0 = (float)row2[(size_t)(iw_base0 + 2)]; + x1 = (float)row2[(size_t)(iw_base0 + 4)]; + x2 = (float)row2[(size_t)(iw_base0 + 6)]; + x3 = (float)row2[(size_t)(iw_base0 + 8)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w22)); + } + } + + __fp16* Y = output + out_idx(n, oc, oh, ow); + const float16x4_t yv = vcvt_f16_f32(vacc); + vst1_f16(Y, yv); + + ow += OW_VEC; + continue; + } + + const size_t tile = do_vec ? OW_VEC : 1; + float acc_s[OW_VEC] = {b0, b0, b0, b0}; + + for (size_t ic = 0; ic < C_in; ++ic) { + for (size_t kh = 0; kh < 3; ++kh) { + const ptrdiff_t ih = ih0 + (ptrdiff_t)kh; + if (ih < 0 || ih >= (ptrdiff_t)H) continue; + + const __fp16* row = input + in_idx(n, ic, (size_t)ih, 0); + + for (size_t kw = 0; kw < 3; ++kw) { + const float wv = (float)weight[w_idx(oc, ic, kh, kw)]; + for (size_t t = 0; t < tile; ++t) { + const size_t ow_t = ow + t; + const ptrdiff_t iw = (ptrdiff_t)ow_t * 2 - 1 + (ptrdiff_t)kw; + if (iw < 0 || iw >= (ptrdiff_t)W) continue; + acc_s[t] += (float)row[(size_t)iw] * wv; + } + } + } + } + + __fp16* Y = output + out_idx(n, oc, oh, ow); + for (size_t t = 0; t < tile; ++t) { + Y[t] = (__fp16)acc_s[t]; + } + ow += tile; + } + } + }); +} + +void cactus_conv2d_depthwise_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C, + size_t H, + size_t W +){ + if (H + 2 < 3 || W + 2 < 3) return; + const size_t H_out = (H - 1) / 2 + 1; + const size_t W_out = (W - 1) / 2 + 1; + + auto in_idx = [&](size_t n, size_t c, size_t ih, size_t iw) -> size_t { + return ((n * C + c) * H + ih) * W + iw; + }; + auto out_idx = [&](size_t n, size_t c, size_t oh, size_t ow) -> size_t { + return ((n * C + c) * H_out + oh) * W_out + ow; + }; + + constexpr size_t OW_VEC = 4; + const size_t total_compute = N * C * H_out * W_out * 9; + CactusThreading::ParallelConfig cfg = + (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} + : CactusThreading::Thresholds::ATTENTION; + + CactusThreading::parallel_for_2d(N, C, cfg, [&](size_t n, size_t c) { + const __fp16* Wc = weight + c * 9; + const float w00 = (float)Wc[0], w01 = (float)Wc[1], w02 = (float)Wc[2]; + const float w10 = (float)Wc[3], w11 = (float)Wc[4], w12 = (float)Wc[5]; + const float w20 = (float)Wc[6], w21 = (float)Wc[7], w22 = (float)Wc[8]; + const float b0 = bias ? (float)bias[c] : 0.0f; + + for (size_t oh = 0; oh < H_out; ++oh) { + const ptrdiff_t ih0 = (ptrdiff_t)oh * 2 - 1; + const bool h_interior = (ih0 >= 0) && ((size_t)(ih0 + 2) < H); + + for (size_t ow = 0; ow < W_out; ) { + const size_t rem = W_out - ow; + const bool do_vec = (rem >= OW_VEC); + const bool w_interior_vec = + do_vec && + (ow >= 1) && + (((ow + (OW_VEC - 1)) * 2 + 1) < W); + + if (h_interior && w_interior_vec) { + const __fp16* row0 = input + in_idx(n, c, (size_t)(ih0 + 0), 0); + const __fp16* row1 = input + in_idx(n, c, (size_t)(ih0 + 1), 0); + const __fp16* row2 = input + in_idx(n, c, (size_t)(ih0 + 2), 0); + const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; + + float32x4_t vacc = vdupq_n_f32(b0); + float x0, x1, x2, x3; + + x0 = (float)row0[(size_t)(iw_base0 + 0)]; + x1 = (float)row0[(size_t)(iw_base0 + 2)]; + x2 = (float)row0[(size_t)(iw_base0 + 4)]; + x3 = (float)row0[(size_t)(iw_base0 + 6)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w00)); + + x0 = (float)row0[(size_t)(iw_base0 + 1)]; + x1 = (float)row0[(size_t)(iw_base0 + 3)]; + x2 = (float)row0[(size_t)(iw_base0 + 5)]; + x3 = (float)row0[(size_t)(iw_base0 + 7)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w01)); + + x0 = (float)row0[(size_t)(iw_base0 + 2)]; + x1 = (float)row0[(size_t)(iw_base0 + 4)]; + x2 = (float)row0[(size_t)(iw_base0 + 6)]; + x3 = (float)row0[(size_t)(iw_base0 + 8)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w02)); + + x0 = (float)row1[(size_t)(iw_base0 + 0)]; + x1 = (float)row1[(size_t)(iw_base0 + 2)]; + x2 = (float)row1[(size_t)(iw_base0 + 4)]; + x3 = (float)row1[(size_t)(iw_base0 + 6)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w10)); + + x0 = (float)row1[(size_t)(iw_base0 + 1)]; + x1 = (float)row1[(size_t)(iw_base0 + 3)]; + x2 = (float)row1[(size_t)(iw_base0 + 5)]; + x3 = (float)row1[(size_t)(iw_base0 + 7)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w11)); + + x0 = (float)row1[(size_t)(iw_base0 + 2)]; + x1 = (float)row1[(size_t)(iw_base0 + 4)]; + x2 = (float)row1[(size_t)(iw_base0 + 6)]; + x3 = (float)row1[(size_t)(iw_base0 + 8)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w12)); + + x0 = (float)row2[(size_t)(iw_base0 + 0)]; + x1 = (float)row2[(size_t)(iw_base0 + 2)]; + x2 = (float)row2[(size_t)(iw_base0 + 4)]; + x3 = (float)row2[(size_t)(iw_base0 + 6)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w20)); + + x0 = (float)row2[(size_t)(iw_base0 + 1)]; + x1 = (float)row2[(size_t)(iw_base0 + 3)]; + x2 = (float)row2[(size_t)(iw_base0 + 5)]; + x3 = (float)row2[(size_t)(iw_base0 + 7)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w21)); + + x0 = (float)row2[(size_t)(iw_base0 + 2)]; + x1 = (float)row2[(size_t)(iw_base0 + 4)]; + x2 = (float)row2[(size_t)(iw_base0 + 6)]; + x3 = (float)row2[(size_t)(iw_base0 + 8)]; + vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w22)); + + __fp16* Y = output + out_idx(n, c, oh, ow); + vst1_f16(Y, vcvt_f16_f32(vacc)); + ow += OW_VEC; + continue; + } + + const size_t tile = do_vec ? OW_VEC : 1; + float acc_s[OW_VEC] = {b0, b0, b0, b0}; + for (size_t t = 0; t < tile; ++t) { + const size_t ow_t = ow + t; + for (size_t kh = 0; kh < 3; ++kh) { + const ptrdiff_t ih = ih0 + (ptrdiff_t)kh; + if (ih < 0 || ih >= (ptrdiff_t)H) continue; + + const __fp16* row = input + in_idx(n, c, (size_t)ih, 0); + for (size_t kw = 0; kw < 3; ++kw) { + const ptrdiff_t iw = (ptrdiff_t)ow_t * 2 - 1 + (ptrdiff_t)kw; + if (iw < 0 || iw >= (ptrdiff_t)W) continue; + const float wv = (float)Wc[kh * 3 + kw]; + acc_s[t] += (float)row[(size_t)iw] * wv; + } + } + } + + __fp16* Y = output + out_idx(n, c, oh, ow); + for (size_t t = 0; t < tile; ++t) { + Y[t] = (__fp16)acc_s[t]; + } + ow += tile; + } + } + }); +} + +void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +){ + const size_t HW = H * W; + if (HW == 0 || N == 0) return; + + CactusThreading::parallel_for( + N, + CactusThreading::Thresholds::ATTENTION, + [&](size_t n_start, size_t n_end) { + std::vector<__fp16> packed_in(HW * C_in); + std::vector packed_out(HW * C_out); + + for (size_t n = n_start; n < n_end; ++n) { + const __fp16* Xn = input + n * C_in * HW; + __fp16* Yn = output + n * C_out * HW; + + for (size_t hw = 0; hw < HW; ++hw) { + const size_t h = hw / W; + const size_t w = hw - h * W; + __fp16* row = packed_in.data() + hw * C_in; + for (size_t ic = 0; ic < C_in; ++ic) { + row[ic] = Xn[(ic * H + h) * W + w]; + } + } + + fp16_rowmajor_gemm_rhs_transposed_to_fp32( + packed_in.data(), + weight, + packed_out.data(), + HW, + C_in, + C_out + ); + + for (size_t hw = 0; hw < HW; ++hw) { + const size_t h = hw / W; + const size_t w = hw - h * W; + const float* row = packed_out.data() + hw * C_out; + for (size_t oc = 0; oc < C_out; ++oc) { + float outv = row[oc]; + if (bias) outv += (float)bias[oc]; + Yn[(oc * H + h) * W + w] = (__fp16)outv; + } + } + } + }); +} + +void cactus_conv1d_pointwise_f16_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +){ + const size_t M = N * L; + if (M == 0) return; + + std::vector output_f32(M * C_out); + fp16_rowmajor_gemm_rhs_transposed_to_fp32( + input, + weight, + output_f32.data(), + M, + C_in, + C_out + ); + + CactusThreading::parallel_for( + M, + CactusThreading::Thresholds::ATTENTION, + [&](size_t row_start, size_t row_end) { + for (size_t m = row_start; m < row_end; ++m) { + const float* src_row = output_f32.data() + m * C_out; + __fp16* dst_row = output + m * C_out; + for (size_t oc = 0; oc < C_out; ++oc) { + float value = src_row[oc]; + if (bias) { + value += static_cast(bias[oc]); + } + dst_row[oc] = static_cast<__fp16>(value); + } + } + } + ); +} + +void cactus_conv2d_f16_k3s1p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, size_t H, size_t W, + size_t C_out +) { + const size_t H_out = H; + const size_t W_out = W; + +#ifdef __APPLE__ + const size_t col_K = C_in * 9; + std::vector W_f32(C_out * col_K); + for (size_t oc = 0; oc < C_out; ++oc) { + for (size_t ic = 0; ic < C_in; ++ic) { + for (size_t kh = 0; kh < 3; ++kh) { + for (size_t kw = 0; kw < 3; ++kw) { + W_f32[oc * col_K + ic * 9 + kh * 3 + kw] = + static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); + } + } + } + } + + std::vector bias_f32(C_out, 0.0f); + if (bias) { + for (size_t i = 0; i < C_out; ++i) + bias_f32[i] = static_cast(bias[i]); + } + + std::vector col(col_K * H_out * W_out); + std::vector Y_f32(C_out * H_out * W_out); + + for (size_t n = 0; n < N; ++n) { + const __fp16* Xn = input + n * C_in * H * W; + __fp16* Yn = output + n * C_out * H_out * W_out; + + for (size_t ic = 0; ic < C_in; ++ic) { + for (size_t kh = 0; kh < 3; ++kh) { + for (size_t kw = 0; kw < 3; ++kw) { + float* dst = col.data() + (ic * 9 + kh * 3 + kw) * H_out * W_out; + for (size_t oh = 0; oh < H_out; ++oh) { + ptrdiff_t ih = static_cast(oh) + static_cast(kh) - 1; + float* dst_row = dst + oh * W_out; + if (ih < 0 || ih >= static_cast(H)) { + memset(dst_row, 0, W_out * sizeof(float)); + continue; + } + const __fp16* src_row = Xn + ic * H * W + static_cast(ih) * W; + const ptrdiff_t iw_offset = static_cast(kw) - 1; + size_t ow_start = 0, ow_end = W_out; + if (iw_offset < 0) { dst_row[0] = 0.0f; ow_start = 1; } + if (iw_offset > 0) { dst_row[W_out - 1] = 0.0f; ow_end = W_out - 1; } + size_t ow = ow_start; + for (; ow + 8 <= ow_end; ow += 8) { + float16x8_t v = vld1q_f16(src_row + ow + iw_offset); + vst1q_f32(dst_row + ow, vcvt_f32_f16(vget_low_f16(v))); + vst1q_f32(dst_row + ow + 4, vcvt_f32_f16(vget_high_f16(v))); + } + for (; ow < ow_end; ++ow) + dst_row[ow] = static_cast(src_row[ow + iw_offset]); + } + } + } + } + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + static_cast(C_out), static_cast(H_out * W_out), static_cast(col_K), + 1.0f, W_f32.data(), static_cast(col_K), + col.data(), static_cast(H_out * W_out), + 0.0f, Y_f32.data(), static_cast(H_out * W_out)); + + for (size_t oc = 0; oc < C_out; ++oc) { + float b = bias_f32[oc]; + const float* src = Y_f32.data() + oc * H_out * W_out; + __fp16* dst = Yn + oc * H_out * W_out; + size_t i = 0; + for (; i + 8 <= H_out * W_out; i += 8) { + float32x4_t v0 = vaddq_f32(vld1q_f32(src + i), vdupq_n_f32(b)); + float32x4_t v1 = vaddq_f32(vld1q_f32(src + i + 4), vdupq_n_f32(b)); + vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); + } + for (; i < H_out * W_out; ++i) + dst[i] = static_cast<__fp16>(src[i] + b); + } + } + +#else + const size_t total_compute = N * C_out * H_out * W_out * C_in * 9; + CactusThreading::ParallelConfig cfg = + (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} + : CactusThreading::Thresholds::ATTENTION; + + CactusThreading::parallel_for_2d(N, C_out, cfg, [&](size_t n, size_t oc) { + const float b0 = bias ? static_cast(bias[oc]) : 0.0f; + for (size_t oh = 0; oh < H_out; ++oh) { + for (size_t ow = 0; ow < W_out; ++ow) { + float acc = b0; + for (size_t ic = 0; ic < C_in; ++ic) { + for (size_t kh = 0; kh < 3; ++kh) { + for (size_t kw = 0; kw < 3; ++kw) { + const ptrdiff_t ih = static_cast(oh) + kh - 1; + const ptrdiff_t iw = static_cast(ow) + kw - 1; + if (ih >= 0 && ih < static_cast(H) && + iw >= 0 && iw < static_cast(W)) { + acc += static_cast(input[((n * C_in + ic) * H + ih) * W + iw]) * + static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); + } + } + } + } + output[((n * C_out + oc) * H_out + oh) * W_out + ow] = static_cast<__fp16>(acc); + } + } + }); +#endif +} + +void cactus_maxpool1d_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t channels, + size_t input_length, + size_t kernel_size, + size_t stride +) { + const size_t output_length = (input_length - kernel_size) / stride + 1; + + CactusThreading::parallel_for(batch_size * channels, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t start, size_t end) { + for (size_t bc = start; bc < end; ++bc) { + const size_t b = bc / channels; + const size_t c = bc % channels; + + const __fp16* src = input + b * channels * input_length + c * input_length; + __fp16* dst = output + b * channels * output_length + c * output_length; + + for (size_t i = 0; i < output_length; ++i) { + const size_t in_start = i * stride; + float max_val = -std::numeric_limits::infinity(); + for (size_t k = 0; k < kernel_size; ++k) { + float val = static_cast(src[in_start + k]); + if (val > max_val) max_val = val; + } + dst[i] = static_cast<__fp16>(max_val); + } + } + }); +} diff --git a/cactus-transpile/kernel/kernel_deltanet.cpp b/cactus-transpile/kernel/kernel_deltanet.cpp new file mode 100644 index 000000000..4972bc49b --- /dev/null +++ b/cactus-transpile/kernel/kernel_deltanet.cpp @@ -0,0 +1,677 @@ +#include "kernel.h" +#include "kernel_utils.h" + +#include +#include +#include +#include +#include + +namespace { + +inline float safe_exp_gate(float gate_log) { + const float clamped = std::max(-20.0f, std::min(6.0f, gate_log)); + return std::exp(clamped); +} + +inline void fold_state_scale(std::vector& state, float factor) { + const float32x4_t f4 = vdupq_n_f32(factor); + size_t i = 0; + for (; i + 8 <= state.size(); i += 8) { + vst1q_f32(&state[i], vmulq_f32(vld1q_f32(&state[i]), f4)); + vst1q_f32(&state[i + 4], vmulq_f32(vld1q_f32(&state[i + 4]), f4)); + } + for (; i < state.size(); ++i) state[i] *= factor; +} + +struct GatedDeltaChunkScratch { + std::vector state; + std::vector m_chunk; + std::vector q_chunk; + std::vector k_chunk; + std::vector v_chunk; + std::vector g_chunk; + std::vector beta_chunk; + std::vector n0_proj; + std::vector delta_chunk; + std::vector gram; + std::vector p_prev; + std::vector p_curr; + std::vector inv_p_curr; + std::vector coeff; + std::vector out_acc; + + void ensure(size_t k_dim, size_t v_dim, size_t c_max) { + const size_t kv = k_dim * v_dim; + if (state.size() < kv) state.resize(kv); + if (m_chunk.size() < kv) m_chunk.resize(kv); + if (q_chunk.size() < c_max * k_dim) q_chunk.resize(c_max * k_dim); + if (k_chunk.size() < c_max * k_dim) k_chunk.resize(c_max * k_dim); + if (v_chunk.size() < c_max * v_dim) v_chunk.resize(c_max * v_dim); + if (g_chunk.size() < c_max) g_chunk.resize(c_max); + if (beta_chunk.size() < c_max) beta_chunk.resize(c_max); + if (n0_proj.size() < c_max * v_dim) n0_proj.resize(c_max * v_dim); + if (delta_chunk.size() < c_max * v_dim) delta_chunk.resize(c_max * v_dim); + if (gram.size() < c_max * c_max) gram.resize(c_max * c_max); + if (p_prev.size() < c_max) p_prev.resize(c_max); + if (p_curr.size() < c_max) p_curr.resize(c_max); + if (inv_p_curr.size() < c_max) inv_p_curr.resize(c_max); + if (coeff.size() < c_max * c_max) coeff.resize(c_max * c_max); + if (out_acc.size() < v_dim) out_acc.resize(v_dim); + } +}; + +thread_local GatedDeltaChunkScratch g_gated_deltanet_chunk_scratch; + +inline size_t tuned_gated_deltanet_chunk_size(size_t requested_chunk, size_t k_dim, size_t v_dim) { + size_t chunk = requested_chunk == 0 ? 64 : requested_chunk; + + const char* env_chunk = std::getenv("CACTUS_GATED_DELTANET_CHUNK_SIZE"); + if (env_chunk != nullptr) { + long parsed = std::strtol(env_chunk, nullptr, 10); + if (parsed > 1) { + chunk = static_cast(parsed); + } + } else { + (void)k_dim; + (void)v_dim; + chunk = std::min(chunk, 16); + } + + return std::max(2, chunk); +} + +void gated_deltanet_step( + const __fp16* q_ptr, + const __fp16* k_ptr, + const __fp16* v_ptr, + float gate_log, + float beta, + float scale, + size_t k_dim, + size_t v_dim, + std::vector& state, + float& state_scale, + std::vector& proj, + std::vector& delta, + __fp16* out_ptr) { + const float gate_log_safe = std::isfinite(gate_log) ? gate_log : -20.0f; + const float beta_safe = std::isfinite(beta) ? std::min(1.0f, std::max(0.0f, beta)) : 0.0f; + + if (proj.size() != v_dim) { + proj.assign(v_dim, 0.0f); + } + if (delta.size() != v_dim) { + delta.assign(v_dim, 0.0f); + } + + const float prev_scale = state_scale; + std::fill(proj.begin(), proj.end(), 0.0f); + + for (size_t kd = 0; kd < k_dim; ++kd) { + if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 0, 3); + const float k_val = static_cast(k_ptr[kd]); + const float* state_row = state.data() + kd * v_dim; + size_t vd = 0; + const float32x4_t k4 = vdupq_n_f32(k_val); + for (; vd + 4 <= v_dim; vd += 4) { + float32x4_t p = vld1q_f32(proj.data() + vd); + float32x4_t s = vld1q_f32(state_row + vd); + p = vfmaq_f32(p, s, k4); + vst1q_f32(proj.data() + vd, p); + } + for (; vd < v_dim; ++vd) { + proj[vd] += state_row[vd] * k_val; + } + } + { + size_t vd = 0; + const float32x4_t scale4 = vdupq_n_f32(prev_scale); + for (; vd + 4 <= v_dim; vd += 4) { + float32x4_t p = vld1q_f32(proj.data() + vd); + p = vmulq_f32(p, scale4); + vst1q_f32(proj.data() + vd, p); + } + for (; vd < v_dim; ++vd) { + proj[vd] *= prev_scale; + } + } + + for (size_t vd = 0; vd < v_dim; ++vd) { + const float v_val = static_cast(v_ptr[vd]); + delta[vd] = (v_val - proj[vd]) * beta_safe; + } + + state_scale = prev_scale * safe_exp_gate(gate_log_safe); + if (!std::isfinite(state_scale) || + std::fabs(state_scale) < 1e-8f || + std::fabs(state_scale) > 1e8f) { + fold_state_scale(state, state_scale); + state_scale = 1.0f; + } + const float inv_scale = 1.0f / state_scale; + + for (size_t kd = 0; kd < k_dim; ++kd) { + if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 1, 3); + const float k_scaled = static_cast(k_ptr[kd]) * inv_scale; + float* state_row = state.data() + kd * v_dim; + size_t vd = 0; + const float32x4_t k4 = vdupq_n_f32(k_scaled); + for (; vd + 4 <= v_dim; vd += 4) { + float32x4_t s = vld1q_f32(state_row + vd); + float32x4_t d = vld1q_f32(delta.data() + vd); + s = vfmaq_f32(s, d, k4); + vst1q_f32(state_row + vd, s); + } + for (; vd < v_dim; ++vd) { + state_row[vd] += k_scaled * delta[vd]; + } + for (size_t vd = 0; vd < v_dim; ++vd) { + if (!std::isfinite(state_row[vd])) { + state_row[vd] = 0.0f; + } + } + } + + std::fill(proj.begin(), proj.end(), 0.0f); + for (size_t kd = 0; kd < k_dim; ++kd) { + if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 0, 3); + const float q_val = static_cast(q_ptr[kd]); + const float* state_row = state.data() + kd * v_dim; + size_t vd = 0; + const float32x4_t q4 = vdupq_n_f32(q_val); + for (; vd + 4 <= v_dim; vd += 4) { + float32x4_t o = vld1q_f32(proj.data() + vd); + float32x4_t s = vld1q_f32(state_row + vd); + o = vfmaq_f32(o, s, q4); + vst1q_f32(proj.data() + vd, o); + } + for (; vd < v_dim; ++vd) { + proj[vd] += state_row[vd] * q_val; + } + } + + const float out_scale = state_scale * scale; + for (size_t vd = 0; vd < v_dim; ++vd) { + float acc = proj[vd]; + if (!std::isfinite(acc)) { + acc = 0.0f; + } + out_ptr[vd] = static_cast<__fp16>(acc * out_scale); + } +} + +void gated_deltanet_prefill_old_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + float scale) { + const size_t out_seq = T + K; + const size_t qk_repeat = Hv / Hq; + + CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t bh_start, size_t bh_end) { + std::vector state(K * V); + std::vector proj(V); + std::vector delta(V); + + for (size_t bh = bh_start; bh < bh_end; ++bh) { + const size_t batch = bh / Hv; + const size_t v_head = bh % Hv; + const size_t qk_head = v_head / qk_repeat; + + for (size_t kd = 0; kd < K; ++kd) { + for (size_t vd = 0; vd < V; ++vd) { + const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V + vd); + state[kd * V + vd] = static_cast(s_data[s_idx]); + } + } + + float state_scale = 1.0f; + for (size_t t = 0; t < T; ++t) { + const size_t qkv_k_base = (((batch * T + t) * Hq + qk_head) * K); + const size_t qkv_v_base = (((batch * T + t) * Hv + v_head) * V); + const size_t gb_idx = ((batch * T + t) * Hv + v_head); + + gated_deltanet_step( + q_data + qkv_k_base, + k_data + qkv_k_base, + v_data + qkv_v_base, + static_cast(g_data[gb_idx]), + static_cast(b_data[gb_idx]), + scale, + K, + V, + state, + state_scale, + proj, + delta, + out + (((batch * out_seq + t) * Hv + v_head) * V)); + } + + fold_state_scale(state, state_scale); + for (size_t kd = 0; kd < K; ++kd) { + for (size_t vd = 0; vd < V; ++vd) { + const size_t out_idx = (((batch * out_seq + (T + kd)) * Hv + v_head) * V + vd); + out[out_idx] = static_cast<__fp16>(state[kd * V + vd]); + } + } + } + }); +} + +void gated_deltanet_prefill_chunked_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + size_t chunk_size, + float scale) { + const size_t out_seq = T + K; + const size_t qk_repeat = Hv / Hq; + constexpr float kMinAbs = 1e-12f; + + CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t bh_start, size_t bh_end) { + auto& ws = g_gated_deltanet_chunk_scratch; + ws.ensure(K, V, chunk_size); + + float* state = ws.state.data(); + float* m_chunk = ws.m_chunk.data(); + float* q_chunk = ws.q_chunk.data(); + float* k_chunk = ws.k_chunk.data(); + float* v_chunk = ws.v_chunk.data(); + float* g_chunk = ws.g_chunk.data(); + float* beta_chunk = ws.beta_chunk.data(); + float* n0_proj = ws.n0_proj.data(); + float* delta_chunk = ws.delta_chunk.data(); + float* gram = ws.gram.data(); + float* p_prev = ws.p_prev.data(); + float* p_curr = ws.p_curr.data(); + float* inv_p_curr = ws.inv_p_curr.data(); + float* coeff = ws.coeff.data(); + float* out_acc = ws.out_acc.data(); + + for (size_t bh = bh_start; bh < bh_end; ++bh) { + const size_t batch = bh / Hv; + const size_t v_head = bh % Hv; + const size_t qk_head = v_head / qk_repeat; + + for (size_t kd = 0; kd < K; ++kd) { + const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V); + cactus_fp16_to_fp32(s_data + s_idx, state + kd * V, V); + } + + for (size_t t0 = 0; t0 < T; t0 += chunk_size) { + const size_t C = std::min(chunk_size, T - t0); + const size_t CV = C * V; + const size_t CC = C * C; + + std::fill(n0_proj, n0_proj + CV, 0.0f); + std::fill(delta_chunk, delta_chunk + CV, 0.0f); + std::fill(gram, gram + CC, 0.0f); + std::fill(coeff, coeff + CC, 0.0f); + std::fill(m_chunk, m_chunk + (K * V), 0.0f); + + // Fused fp16→fp32 conversion + n0_proj: k_chunk stays in L1 + for (size_t ct = 0; ct < C; ++ct) { + const size_t t = t0 + ct; + const size_t qkv_k_base = (((batch * T + t) * Hq + qk_head) * K); + const size_t qkv_v_base = (((batch * T + t) * Hv + v_head) * V); + const size_t gb_idx = ((batch * T + t) * Hv + v_head); + + // Inline fp16→fp32 for q + { + const __fp16* src = q_data + qkv_k_base; + float* dst = q_chunk + ct * K; + size_t i = 0; + for (; i + 8 <= K; i += 8) { + float16x8_t in = vld1q_f16(src + i); + vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); + vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); + } + for (; i < K; ++i) dst[i] = static_cast(src[i]); + } + + // Inline fp16→fp32 for k + { + const __fp16* src = k_data + qkv_k_base; + float* dst = k_chunk + ct * K; + size_t i = 0; + for (; i + 8 <= K; i += 8) { + float16x8_t in = vld1q_f16(src + i); + vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); + vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); + } + for (; i < K; ++i) dst[i] = static_cast(src[i]); + } + + // Inline fp16→fp32 for v + { + const __fp16* src = v_data + qkv_v_base; + float* dst = v_chunk + ct * V; + size_t i = 0; + for (; i + 8 <= V; i += 8) { + float16x8_t in = vld1q_f16(src + i); + vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); + vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); + } + for (; i < V; ++i) dst[i] = static_cast(src[i]); + } + + const float gate_log = static_cast(g_data[gb_idx]); + const float beta = static_cast(b_data[gb_idx]); + g_chunk[ct] = safe_exp_gate(std::isfinite(gate_log) ? gate_log : -20.0f); + beta_chunk[ct] = std::isfinite(beta) ? std::min(1.0f, std::max(0.0f, beta)) : 0.0f; + + // n0_proj fused here so k_chunk[ct] is still in L1 + const float* k_row = k_chunk + ct * K; + float* p_row = n0_proj + ct * V; + for (size_t kd = 0; kd < K; ++kd) { + const float kval = k_row[kd]; + const float* s_row = state + kd * V; + size_t vd = 0; + const float32x4_t kv4 = vdupq_n_f32(kval); + for (; vd + 8 <= V; vd += 8) { + float32x4_t p4 = vld1q_f32(p_row + vd); + const float32x4_t slo = vld1q_f32(s_row + vd); + const float32x4_t shi = vld1q_f32(s_row + vd + 4); + p4 = vfmaq_f32(p4, slo, kv4); + vst1q_f32(p_row + vd, p4); + p4 = vld1q_f32(p_row + vd + 4); + p4 = vfmaq_f32(p4, shi, kv4); + vst1q_f32(p_row + vd + 4, p4); + } + for (; vd < V; ++vd) { + p_row[vd] += kval * s_row[vd]; + } + } + } + + for (size_t t = 0; t < C; ++t) { + const float* kt = k_chunk + t * K; + for (size_t j = 0; j < t; ++j) { + const float* kj = k_chunk + j * K; + float dot = 0.0f; + size_t kd = 0; + float32x4_t acc4 = vdupq_n_f32(0.0f); + for (; kd + 8 <= K; kd += 8) { + const float32x4_t al = vld1q_f32(kt + kd); + const float32x4_t ah = vld1q_f32(kt + kd + 4); + const float32x4_t bl = vld1q_f32(kj + kd); + const float32x4_t bh = vld1q_f32(kj + kd + 4); + acc4 = vfmaq_f32(acc4, al, bl); + acc4 = vfmaq_f32(acc4, ah, bh); + } + dot += vaddvq_f32(acc4); + for (; kd < K; ++kd) { + dot += kt[kd] * kj[kd]; + } + gram[t * C + j] = dot; + } + } + + float running_p = 1.0f; + for (size_t t = 0; t < C; ++t) { + p_prev[t] = running_p; + running_p *= g_chunk[t]; + p_curr[t] = running_p; + inv_p_curr[t] = (std::fabs(running_p) > kMinAbs) ? (1.0f / running_p) : 0.0f; + } + + for (size_t t = 0; t < C; ++t) { + const float pp = p_prev[t]; + float* coeff_row = coeff + t * C; + for (size_t j = 0; j < t; ++j) { + const float inv_pc = inv_p_curr[j]; + coeff_row[j] = inv_pc == 0.0f ? 0.0f : (pp * inv_pc * gram[t * C + j]); + } + } + + for (size_t t = 0; t < C; ++t) { + const float pp = p_prev[t]; + const float bt = beta_chunk[t]; + const float* n0_row = n0_proj + t * V; + const float* v_row = v_chunk + t * V; + float* d_row = delta_chunk + t * V; + + size_t vd = 0; + const float32x4_t pp4 = vdupq_n_f32(pp); + const float32x4_t bt4 = vdupq_n_f32(bt); + for (; vd + 8 <= V; vd += 8) { + float32x4_t acc_lo = vmulq_f32(vld1q_f32(n0_row + vd), pp4); + float32x4_t acc_hi = vmulq_f32(vld1q_f32(n0_row + vd + 4), pp4); + for (size_t j = 0; j < t; ++j) { + const float c = coeff[t * C + j]; + const float* dj = delta_chunk + j * V + vd; + acc_lo = vmlaq_n_f32(acc_lo, vld1q_f32(dj), c); + acc_hi = vmlaq_n_f32(acc_hi, vld1q_f32(dj + 4), c); + } + vst1q_f32(d_row + vd, vmulq_f32(vsubq_f32(vld1q_f32(v_row + vd), acc_lo), bt4)); + vst1q_f32(d_row + vd + 4, vmulq_f32(vsubq_f32(vld1q_f32(v_row + vd + 4), acc_hi), bt4)); + } + for (; vd < V; ++vd) { + float acc = pp * n0_row[vd]; + for (size_t j = 0; j < t; ++j) { + acc += coeff[t * C + j] * delta_chunk[j * V + vd]; + } + d_row[vd] = bt * (v_row[vd] - acc); + } + } + + running_p = 1.0f; + for (size_t t = 0; t < C; ++t) { + const float gt = g_chunk[t]; + const float* k_row = k_chunk + t * K; + const float* q_row = q_chunk + t * K; + const float* d_row = delta_chunk + t * V; + + running_p *= gt; + const float p_run = running_p; + + for (size_t kd = 0; kd < K; ++kd) { + const float kval = k_row[kd]; + float* m_row = m_chunk + kd * V; + size_t vd = 0; + const float32x4_t gt4 = vdupq_n_f32(gt); + const float32x4_t kv4 = vdupq_n_f32(kval); + for (; vd + 8 <= V; vd += 8) { + float32x4_t ml = vld1q_f32(m_row + vd); + float32x4_t mh = vld1q_f32(m_row + vd + 4); + const float32x4_t dl = vld1q_f32(d_row + vd); + const float32x4_t dh = vld1q_f32(d_row + vd + 4); + ml = vmulq_f32(ml, gt4); + mh = vmulq_f32(mh, gt4); + ml = vfmaq_f32(ml, dl, kv4); + mh = vfmaq_f32(mh, dh, kv4); + vst1q_f32(m_row + vd, ml); + vst1q_f32(m_row + vd + 4, mh); + } + for (; vd < V; ++vd) { + m_row[vd] = gt * m_row[vd] + kval * d_row[vd]; + } + } + + const size_t out_base = (((batch * out_seq + (t0 + t)) * Hv + v_head) * V); + std::fill(out_acc, out_acc + V, 0.0f); + for (size_t kd = 0; kd < K; ++kd) { + const float qval = q_row[kd]; + const float* s_row = state + kd * V; + const float* m_row = m_chunk + kd * V; + size_t vd = 0; + const float32x4_t q4 = vdupq_n_f32(qval); + const float32x4_t p4 = vdupq_n_f32(p_run); + for (; vd + 8 <= V; vd += 8) { + float32x4_t acc4 = vld1q_f32(out_acc + vd); + float32x4_t acc4h = vld1q_f32(out_acc + vd + 4); + const float32x4_t sl = vld1q_f32(s_row + vd); + const float32x4_t sh = vld1q_f32(s_row + vd + 4); + const float32x4_t ml = vld1q_f32(m_row + vd); + const float32x4_t mh = vld1q_f32(m_row + vd + 4); + const float32x4_t nl = vfmaq_f32(ml, sl, p4); + const float32x4_t nh = vfmaq_f32(mh, sh, p4); + acc4 = vfmaq_f32(acc4, nl, q4); + acc4h = vfmaq_f32(acc4h, nh, q4); + vst1q_f32(out_acc + vd, acc4); + vst1q_f32(out_acc + vd + 4, acc4h); + } + for (; vd < V; ++vd) { + out_acc[vd] += qval * (p_run * s_row[vd] + m_row[vd]); + } + } + for (size_t vd = 0; vd < V; ++vd) { + out[out_base + vd] = static_cast<__fp16>(out_acc[vd] * scale); + } + } + + const float p_end = running_p; + for (size_t kd = 0; kd < K; ++kd) { + float* s_row = state + kd * V; + const float* m_row = m_chunk + kd * V; + size_t vd = 0; + const float32x4_t p4 = vdupq_n_f32(p_end); + for (; vd + 8 <= V; vd += 8) { + float32x4_t sl = vld1q_f32(s_row + vd); + float32x4_t sh = vld1q_f32(s_row + vd + 4); + const float32x4_t ml = vld1q_f32(m_row + vd); + const float32x4_t mh = vld1q_f32(m_row + vd + 4); + sl = vfmaq_f32(ml, sl, p4); + sh = vfmaq_f32(mh, sh, p4); + vst1q_f32(s_row + vd, sl); + vst1q_f32(s_row + vd + 4, sh); + } + for (; vd < V; ++vd) { + s_row[vd] = p_end * s_row[vd] + m_row[vd]; + } + } + } + + for (size_t kd = 0; kd < K; ++kd) { + const size_t out_idx = (((batch * out_seq + (T + kd)) * Hv + v_head) * V); + cactus_fp32_to_fp16(state + kd * V, out + out_idx, V); + } + } + }); +} + +} // namespace + +void cactus_gated_deltanet_decode_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + float scale) { + const size_t qk_repeat = Hv / Hq; + const size_t out_seq = 1 + K; + + CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t bh_start, size_t bh_end) { + std::vector state(K * V); + std::vector proj(V); + std::vector delta(V); + + for (size_t bh = bh_start; bh < bh_end; ++bh) { + const size_t batch = bh / Hv; + const size_t v_head = bh % Hv; + const size_t qk_head = v_head / qk_repeat; + + for (size_t kd = 0; kd < K; ++kd) { + for (size_t vd = 0; vd < V; ++vd) { + const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V + vd); + state[kd * V + vd] = static_cast(s_data[s_idx]); + } + } + + float state_scale = 1.0f; + const size_t qk_base = ((batch * Hq + qk_head) * K); + const size_t v_base = ((batch * Hv + v_head) * V); + const float gate_log = static_cast(g_data[batch * Hv + v_head]); + const float beta = static_cast(b_data[batch * Hv + v_head]); + + gated_deltanet_step( + q_data + qk_base, + k_data + qk_base, + v_data + v_base, + gate_log, + beta, + scale, + K, + V, + state, + state_scale, + proj, + delta, + out + (((batch * out_seq) * Hv + v_head) * V)); + + fold_state_scale(state, state_scale); + for (size_t kd = 0; kd < K; ++kd) { + for (size_t vd = 0; vd < V; ++vd) { + const size_t out_idx = (((batch * out_seq + (1 + kd)) * Hv + v_head) * V + vd); + out[out_idx] = static_cast<__fp16>(state[kd * V + vd]); + } + } + } + }); +} + +void cactus_gated_deltanet_prefill_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + size_t requested_chunk_size, + float scale) { + const char* force_old = std::getenv("CACTUS_GATED_DELTANET_PREFILL_OLD"); + if (force_old != nullptr && std::atoi(force_old) != 0) { + gated_deltanet_prefill_old_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, + B, T, Hq, Hv, K, V, scale); + return; + } + + if (requested_chunk_size <= 1) { + gated_deltanet_prefill_old_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, + B, T, Hq, Hv, K, V, scale); + return; + } + + const size_t chunk_size = tuned_gated_deltanet_chunk_size(requested_chunk_size, K, V); + gated_deltanet_prefill_chunked_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, + B, T, Hq, Hv, K, V, chunk_size, scale); +} diff --git a/cactus-transpile/kernel/kernel_i8mm.cpp b/cactus-transpile/kernel/kernel_i8mm.cpp new file mode 100644 index 000000000..c8c976173 --- /dev/null +++ b/cactus-transpile/kernel/kernel_i8mm.cpp @@ -0,0 +1,189 @@ +#include + +#ifndef __ARM_FEATURE_MATMUL_INT8 +#error "kernel_i8mm.cpp must be compiled with I8MM enabled (e.g. -march=armv8.2-a+fp16+simd+dotprod+i8mm)" +#endif + +#include "kernel.h" +#include "kernel_utils.h" +#include +#include + +void cactus_gemv_int8_i8mm( + const int8_t* A, + const float A_scale, + const int8_t* B, + const __fp16* B_scales, + __fp16* C, + size_t K, size_t N, + size_t group_size +) { + if (K == 0 || N == 0) return; + + const size_t num_groups = K / group_size; + const size_t N_blocks = (N + 3) / 4; + + auto process_blocks = [=](size_t block_start, size_t block_end) { + for (size_t n_block = block_start; n_block < block_end; ++n_block) { + const size_t n_start = n_block * 4; + const size_t actual_n = std::min(size_t(4), N - n_start); + + float32x4_t running_sum = vdupq_n_f32(0.0f); + + for (size_t g = 0; g < num_groups; g++) { + const size_t k_base = g * group_size; + const int8_t* a_ptr = A + k_base; + const int8_t* b_ptr = B + (n_block * K + k_base) * 4; + + __builtin_prefetch(b_ptr + group_size * 8, 0, 3); + + int32x4_t acc_01 = vdupq_n_s32(0); + int32x4_t acc_23 = vdupq_n_s32(0); + + for (size_t k_off = 0; k_off < group_size; k_off += 8) { + int32x4_t chunk0 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4)); + int32x4_t chunk1 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4 + 16)); + + int8x16_t b01 = vreinterpretq_s8_s32(vzip1q_s32(chunk0, chunk1)); + int8x16_t b23 = vreinterpretq_s8_s32(vzip2q_s32(chunk0, chunk1)); + + int8x8_t a8 = vld1_s8(a_ptr + k_off); + int8x16_t a_dup = vcombine_s8(a8, a8); + + acc_01 = vmmlaq_s32(acc_01, a_dup, b01); + acc_23 = vmmlaq_s32(acc_23, a_dup, b23); + } + + float32x2_t r01 = vcvt_f32_s32(vget_low_s32(acc_01)); + float32x2_t r23 = vcvt_f32_s32(vget_low_s32(acc_23)); + float32x4_t group_result = vcombine_f32(r01, r23); + + const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; + float32x4_t scales = vcvt_f32_f16(vld1_f16(scale_ptr)); + running_sum = vmlaq_f32(running_sum, group_result, scales); + } + + float32x4_t result = vmulq_n_f32(running_sum, A_scale); + float16x4_t result_f16 = vcvt_f16_f32(result); + + if (actual_n == 4) { + vst1_f16(C + n_start, result_f16); + } else { + for (size_t ni = 0; ni < actual_n; ni++) { + C[n_start + ni] = vget_lane_f16(result_f16, 0); + result_f16 = vext_f16(result_f16, result_f16, 1); + } + } + } + }; + + auto& pool = CactusThreading::get_thread_pool(); + size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); + num_threads = std::min(num_threads, N_blocks); + + if (num_threads <= 1) { + process_blocks(0, N_blocks); + } else { + pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); + pool.wait_all(); + } +} + +void cactus_gemm_int8_i8mm( + const int8_t* A, + const float* A_scales, + const int8_t* B, + const __fp16* B_scales, + __fp16* C, + size_t M, size_t K, size_t N, + size_t group_size +) { + if (M == 0 || K == 0 || N == 0) return; + + const size_t num_groups = K / group_size; + const size_t N_blocks = (N + 3) / 4; + const size_t M_pairs = (M + 1) / 2; + const size_t total_tiles = M_pairs * N_blocks; + + CactusThreading::parallel_gemm_tiles(M, total_tiles, + [=](size_t tile_start, size_t tile_end) { + for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { + const size_t m_pair = tile_idx / N_blocks; + const size_t n_block = tile_idx % N_blocks; + const size_t m0 = m_pair * 2; + const size_t m1 = std::min(m0 + 1, M - 1); + const size_t n_start = n_block * 4; + const size_t actual_n = std::min(size_t(4), N - n_start); + + const int8_t* a_row0 = A + m0 * K; + const int8_t* a_row1 = A + m1 * K; + + float32x4_t sum_row0 = vdupq_n_f32(0.0f); + float32x4_t sum_row1 = vdupq_n_f32(0.0f); + + for (size_t g = 0; g < num_groups; g++) { + const size_t k_base = g * group_size; + const int8_t* b_ptr = B + (n_block * K + k_base) * 4; + + __builtin_prefetch(b_ptr + group_size * 8, 0, 3); + + int32x4_t acc_01 = vdupq_n_s32(0); + int32x4_t acc_23 = vdupq_n_s32(0); + + for (size_t k_off = 0; k_off < group_size; k_off += 8) { + int32x4_t chunk0 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4)); + int32x4_t chunk1 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4 + 16)); + + int8x16_t b01 = vreinterpretq_s8_s32(vzip1q_s32(chunk0, chunk1)); + int8x16_t b23 = vreinterpretq_s8_s32(vzip2q_s32(chunk0, chunk1)); + + int8x8_t a0_8 = vld1_s8(a_row0 + k_base + k_off); + int8x8_t a1_8 = vld1_s8(a_row1 + k_base + k_off); + int8x16_t a_2rows = vcombine_s8(a0_8, a1_8); + + acc_01 = vmmlaq_s32(acc_01, a_2rows, b01); + acc_23 = vmmlaq_s32(acc_23, a_2rows, b23); + } + + const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; + float32x4_t scales = vcvt_f32_f16(vld1_f16(scale_ptr)); + + float32x2_t r0_01 = vcvt_f32_s32(vget_low_s32(acc_01)); + float32x2_t r0_23 = vcvt_f32_s32(vget_low_s32(acc_23)); + float32x4_t group_row0 = vcombine_f32(r0_01, r0_23); + sum_row0 = vmlaq_f32(sum_row0, group_row0, scales); + + float32x2_t r1_01 = vcvt_f32_s32(vget_high_s32(acc_01)); + float32x2_t r1_23 = vcvt_f32_s32(vget_high_s32(acc_23)); + float32x4_t group_row1 = vcombine_f32(r1_01, r1_23); + sum_row1 = vmlaq_f32(sum_row1, group_row1, scales); + } + + { + float32x4_t result = vmulq_n_f32(sum_row0, A_scales[m0]); + float16x4_t result_f16 = vcvt_f16_f32(result); + if (actual_n == 4) { + vst1_f16(C + m0 * N + n_start, result_f16); + } else { + for (size_t ni = 0; ni < actual_n; ni++) { + C[m0 * N + n_start + ni] = vget_lane_f16(result_f16, 0); + result_f16 = vext_f16(result_f16, result_f16, 1); + } + } + } + + if (m0 + 1 < M) { + float32x4_t result = vmulq_n_f32(sum_row1, A_scales[m1]); + float16x4_t result_f16 = vcvt_f16_f32(result); + if (actual_n == 4) { + vst1_f16(C + m1 * N + n_start, result_f16); + } else { + for (size_t ni = 0; ni < actual_n; ni++) { + C[m1 * N + n_start + ni] = vget_lane_f16(result_f16, 0); + result_f16 = vext_f16(result_f16, result_f16, 1); + } + } + } + } + }); +} diff --git a/cactus-transpile/kernel/kernel_lstm.cpp b/cactus-transpile/kernel/kernel_lstm.cpp new file mode 100644 index 000000000..84d0077a1 --- /dev/null +++ b/cactus-transpile/kernel/kernel_lstm.cpp @@ -0,0 +1,117 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include + +void cactus_lstm_cell_f16( + const __fp16* x_input, + const __fp16* h_prev, + const __fp16* c_prev, + const __fp16* weight_ih, + const __fp16* weight_hh, + const __fp16* bias_ih, + const __fp16* bias_hh, + __fp16* h_new, + __fp16* c_new, + size_t batch_size, + size_t input_size, + size_t hidden_size +) { + constexpr size_t SIMD_WIDTH = 8; + const size_t gate_size = 4 * hidden_size; + + std::vector<__fp16> gates_ih(batch_size * gate_size); + std::vector<__fp16> gates_hh(batch_size * gate_size); + + cactus_matmul_f16(x_input, weight_ih, gates_ih.data(), batch_size, input_size, gate_size); + cactus_matmul_f16(h_prev, weight_hh, gates_hh.data(), batch_size, hidden_size, gate_size); + + const size_t simd_end = (hidden_size / SIMD_WIDTH) * SIMD_WIDTH; + + CactusThreading::parallel_for(batch_size, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t batch_start, size_t batch_end) { + const float32x4_t one = vdupq_n_f32(1.0f); + + for (size_t b = batch_start; b < batch_end; ++b) { + const size_t gate_offset = b * gate_size; + const size_t hidden_offset = b * hidden_size; + + for (size_t h = 0; h < simd_end; h += SIMD_WIDTH) { + float16x8_t i_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + h]), + vld1q_f16(&gates_hh[gate_offset + h])), + vaddq_f16(vld1q_f16(&bias_ih[h]), + vld1q_f16(&bias_hh[h]))); + + float16x8_t f_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + hidden_size + h]), + vld1q_f16(&gates_hh[gate_offset + hidden_size + h])), + vaddq_f16(vld1q_f16(&bias_ih[hidden_size + h]), + vld1q_f16(&bias_hh[hidden_size + h]))); + + float16x8_t g_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + 2 * hidden_size + h]), + vld1q_f16(&gates_hh[gate_offset + 2 * hidden_size + h])), + vaddq_f16(vld1q_f16(&bias_ih[2 * hidden_size + h]), + vld1q_f16(&bias_hh[2 * hidden_size + h]))); + + float16x8_t o_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + 3 * hidden_size + h]), + vld1q_f16(&gates_hh[gate_offset + 3 * hidden_size + h])), + vaddq_f16(vld1q_f16(&bias_ih[3 * hidden_size + h]), + vld1q_f16(&bias_hh[3 * hidden_size + h]))); + + float32x4_t i_low = vcvt_f32_f16(vget_low_f16(i_gate)); + float32x4_t i_high = vcvt_f32_f16(vget_high_f16(i_gate)); + float16x8_t i_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_low))))), + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_high)))))); + + float32x4_t f_low = vcvt_f32_f16(vget_low_f16(f_gate)); + float32x4_t f_high = vcvt_f32_f16(vget_high_f16(f_gate)); + float16x8_t f_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_low))))), + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_high)))))); + + float32x4_t g_low = vcvt_f32_f16(vget_low_f16(g_gate)); + float32x4_t g_high = vcvt_f32_f16(vget_high_f16(g_gate)); + float16x8_t g_act = vcombine_f16(vcvt_f16_f32(fast_tanh_f32x4(g_low)), + vcvt_f16_f32(fast_tanh_f32x4(g_high))); + + float32x4_t o_low = vcvt_f32_f16(vget_low_f16(o_gate)); + float32x4_t o_high = vcvt_f32_f16(vget_high_f16(o_gate)); + float16x8_t o_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_low))))), + vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_high)))))); + + float16x8_t c_prev_vec = vld1q_f16(&c_prev[hidden_offset + h]); + float16x8_t c_update = vfmaq_f16(vmulq_f16(f_act, c_prev_vec), i_act, g_act); + vst1q_f16(&c_new[hidden_offset + h], c_update); + + float32x4_t c_low = vcvt_f32_f16(vget_low_f16(c_update)); + float32x4_t c_high = vcvt_f32_f16(vget_high_f16(c_update)); + float16x8_t c_tanh = vcombine_f16(vcvt_f16_f32(fast_tanh_f32x4(c_low)), + vcvt_f16_f32(fast_tanh_f32x4(c_high))); + vst1q_f16(&h_new[hidden_offset + h], vmulq_f16(o_act, c_tanh)); + } + + for (size_t h = simd_end; h < hidden_size; ++h) { + float i_gate_val = static_cast(gates_ih[gate_offset + h] + + gates_hh[gate_offset + h] + + bias_ih[h] + bias_hh[h]); + float f_gate_val = static_cast(gates_ih[gate_offset + hidden_size + h] + + gates_hh[gate_offset + hidden_size + h] + + bias_ih[hidden_size + h] + bias_hh[hidden_size + h]); + float g_gate_val = static_cast(gates_ih[gate_offset + 2 * hidden_size + h] + + gates_hh[gate_offset + 2 * hidden_size + h] + + bias_ih[2 * hidden_size + h] + bias_hh[2 * hidden_size + h]); + float o_gate_val = static_cast(gates_ih[gate_offset + 3 * hidden_size + h] + + gates_hh[gate_offset + 3 * hidden_size + h] + + bias_ih[3 * hidden_size + h] + bias_hh[3 * hidden_size + h]); + + float i_act = 1.0f / (1.0f + expf(-i_gate_val)); + float f_act = 1.0f / (1.0f + expf(-f_gate_val)); + float g_act = tanhf(g_gate_val); + float o_act = 1.0f / (1.0f + expf(-o_gate_val)); + + float c_val = f_act * static_cast(c_prev[hidden_offset + h]) + i_act * g_act; + c_new[hidden_offset + h] = static_cast<__fp16>(c_val); + h_new[hidden_offset + h] = static_cast<__fp16>(o_act * tanhf(c_val)); + } + } + }); +} diff --git a/cactus-transpile/kernel/kernel_matmul.cpp b/cactus-transpile/kernel/kernel_matmul.cpp new file mode 100644 index 000000000..75ada0d99 --- /dev/null +++ b/cactus-transpile/kernel/kernel_matmul.cpp @@ -0,0 +1,787 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include "../graph/graph.h" +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +constexpr size_t ACCELERATE_M_THRESHOLD = 4; +constexpr size_t ACCELERATE_K_THRESHOLD = 256; +#endif + +// Do NOT Remove: Uncomment for testing on various paths +// ----- +// TEMPORARY: Force fallback path for testing on DOTPROD devices +// #undef __ARM_FEATURE_DOTPROD + +#if defined(__ARM_FEATURE_DOTPROD) + #define CACTUS_DOTQ_LANE(acc, b, a, lane) vdotq_laneq_s32(acc, b, a, lane) +#else + static inline int32x4_t cactus_dotq_with_pattern(int32x4_t acc, int8x16_t b, int8x8_t a_pattern) { + int8x8_t b_lo = vget_low_s8(b); + int8x8_t b_hi = vget_high_s8(b); + + int16x8_t prod_lo = vmull_s8(b_lo, a_pattern); + int16x8_t prod_hi = vmull_s8(b_hi, a_pattern); + + int32x4_t sum_lo = vpaddlq_s16(prod_lo); + int32x4_t sum_hi = vpaddlq_s16(prod_hi); + + int32x2_t final_lo = vpadd_s32(vget_low_s32(sum_lo), vget_high_s32(sum_lo)); + int32x2_t final_hi = vpadd_s32(vget_low_s32(sum_hi), vget_high_s32(sum_hi)); + + return vaddq_s32(acc, vcombine_s32(final_lo, final_hi)); + } + + static inline int32x4_t cactus_dotq_lane0(int32x4_t acc, int8x16_t b, int8x16_t a) { + int8x8_t a_lo = vget_low_s8(a); + int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_lo), 0)); + return cactus_dotq_with_pattern(acc, b, a_pattern); + } + + static inline int32x4_t cactus_dotq_lane1(int32x4_t acc, int8x16_t b, int8x16_t a) { + int8x8_t a_lo = vget_low_s8(a); + int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_lo), 1)); + return cactus_dotq_with_pattern(acc, b, a_pattern); + } + + static inline int32x4_t cactus_dotq_lane2(int32x4_t acc, int8x16_t b, int8x16_t a) { + int8x8_t a_hi = vget_high_s8(a); + int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_hi), 0)); + return cactus_dotq_with_pattern(acc, b, a_pattern); + } + + static inline int32x4_t cactus_dotq_lane3(int32x4_t acc, int8x16_t b, int8x16_t a) { + int8x8_t a_hi = vget_high_s8(a); + int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_hi), 1)); + return cactus_dotq_with_pattern(acc, b, a_pattern); + } + + #define CACTUS_DOTQ_LANE(acc, b, a, lane) cactus_dotq_lane##lane(acc, b, a) +#endif + +static inline __fp16 hsum_f16x8(float16x8_t v) { + float16x4_t lo = vget_low_f16(v); + float16x4_t hi = vget_high_f16(v); + float16x4_t sum4 = vadd_f16(lo, hi); + float16x4_t sum2 = vadd_f16(sum4, vext_f16(sum4, sum4, 2)); + float16x4_t sum1 = vadd_f16(sum2, vext_f16(sum2, sum2, 1)); + return vget_lane_f16(sum1, 0); +} + +static void cactus_matmul_f16_worker( + const __fp16* a, + const __fp16* b_transposed, + __fp16* c, + size_t /*M*/, + size_t K, + size_t N, + size_t start_row, + size_t end_row +) { + constexpr size_t TILE_M = 4; + constexpr size_t TILE_N = 4; + const size_t K16 = (K / 16) * 16; + const size_t K8 = (K / 8) * 8; + + for (size_t row_block = start_row; row_block < end_row; row_block += TILE_M) { + const size_t m_end = std::min(row_block + TILE_M, end_row); + + for (size_t col_block = 0; col_block < N; col_block += TILE_N) { + const size_t n_end = std::min(col_block + TILE_N, N); + + float16x8_t acc[TILE_M][TILE_N]; + for (size_t m = 0; m < TILE_M; ++m) + for (size_t n = 0; n < TILE_N; ++n) + acc[m][n] = vdupq_n_f16(0); + + for (size_t k = 0; k < K16; k += 16) { + float16x8_t a0_lo = (row_block < m_end) ? vld1q_f16(a + row_block * K + k) : vdupq_n_f16(0); + float16x8_t a0_hi = (row_block < m_end) ? vld1q_f16(a + row_block * K + k + 8) : vdupq_n_f16(0); + float16x8_t a1_lo = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k) : vdupq_n_f16(0); + float16x8_t a1_hi = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k + 8) : vdupq_n_f16(0); + float16x8_t a2_lo = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k) : vdupq_n_f16(0); + float16x8_t a2_hi = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k + 8) : vdupq_n_f16(0); + float16x8_t a3_lo = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k) : vdupq_n_f16(0); + float16x8_t a3_hi = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k + 8) : vdupq_n_f16(0); + + for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { + float16x8_t b_lo = vld1q_f16(b_transposed + (col_block + ni) * K + k); + float16x8_t b_hi = vld1q_f16(b_transposed + (col_block + ni) * K + k + 8); + + acc[0][ni] = vfmaq_f16(acc[0][ni], a0_lo, b_lo); + acc[0][ni] = vfmaq_f16(acc[0][ni], a0_hi, b_hi); + acc[1][ni] = vfmaq_f16(acc[1][ni], a1_lo, b_lo); + acc[1][ni] = vfmaq_f16(acc[1][ni], a1_hi, b_hi); + acc[2][ni] = vfmaq_f16(acc[2][ni], a2_lo, b_lo); + acc[2][ni] = vfmaq_f16(acc[2][ni], a2_hi, b_hi); + acc[3][ni] = vfmaq_f16(acc[3][ni], a3_lo, b_lo); + acc[3][ni] = vfmaq_f16(acc[3][ni], a3_hi, b_hi); + } + } + + for (size_t k = K16; k < K8; k += 8) { + float16x8_t a0_v = (row_block < m_end) ? vld1q_f16(a + row_block * K + k) : vdupq_n_f16(0); + float16x8_t a1_v = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k) : vdupq_n_f16(0); + float16x8_t a2_v = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k) : vdupq_n_f16(0); + float16x8_t a3_v = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k) : vdupq_n_f16(0); + + for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { + float16x8_t b_v = vld1q_f16(b_transposed + (col_block + ni) * K + k); + acc[0][ni] = vfmaq_f16(acc[0][ni], a0_v, b_v); + acc[1][ni] = vfmaq_f16(acc[1][ni], a1_v, b_v); + acc[2][ni] = vfmaq_f16(acc[2][ni], a2_v, b_v); + acc[3][ni] = vfmaq_f16(acc[3][ni], a3_v, b_v); + } + } + + for (size_t k = K8; k < K; ++k) { + for (size_t mi = 0; mi < TILE_M && row_block + mi < m_end; ++mi) { + __fp16 av = a[(row_block + mi) * K + k]; + for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { + __fp16 bv = b_transposed[(col_block + ni) * K + k]; + acc[mi][ni] = vsetq_lane_f16(vgetq_lane_f16(acc[mi][ni], 0) + av * bv, acc[mi][ni], 0); + } + } + } + + for (size_t mi = 0; mi < TILE_M && row_block + mi < m_end; ++mi) { + for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { + c[(row_block + mi) * N + col_block + ni] = hsum_f16x8(acc[mi][ni]); + } + } + } + } +} + +#if defined(CACTUS_COMPILE_SME2) +constexpr size_t SME2_M_THRESHOLD = 4; + +void cactus_matmul_f16_sme2_caller( + const __fp16* a, + const __fp16* b_transposed, + __fp16* c, + size_t M, + size_t K, + size_t N +); +#endif + +void cactus_matmul_f16( + const __fp16* a, + const __fp16* b_transposed, + __fp16* c, + size_t M, + size_t K, + size_t N +) { + +#if defined(CACTUS_COMPILE_SME2) + if (cpu_has_sme2() && M >= SME2_M_THRESHOLD) { + cactus_matmul_f16_sme2_caller( + a, b_transposed, c, + M, K, N + ); + return; + } +#endif + +#ifdef __APPLE__ + if (K >= ACCELERATE_K_THRESHOLD && M >= ACCELERATE_M_THRESHOLD) { + const size_t a_len = M * K; + const size_t b_len = N * K; + const size_t c_len = M * N; + + std::vector A_f32(a_len); + std::vector BT_f32(b_len); + std::vector C_f32(c_len); + + for (size_t i = 0; i < a_len; i++) A_f32[i] = (float)a[i]; + for (size_t i = 0; i < b_len; i++) BT_f32[i] = (float)b_transposed[i]; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + (int)M, (int)N, (int)K, + 1.0f, A_f32.data(), (int)K, + BT_f32.data(), (int)K, + 0.0f, C_f32.data(), (int)N); + + for (size_t i = 0; i < c_len; i++) { + float v = C_f32[i]; + if (v > 65504.f) v = 65504.f; + else if (v < -65504.f) v = -65504.f; + c[i] = (__fp16)v; + } + return; + } +#endif + + constexpr size_t TILE_M = 4; + const size_t num_row_blocks = (M + TILE_M - 1) / TILE_M; + + CactusThreading::parallel_for(num_row_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [=](size_t start_block, size_t end_block) { + for (size_t block_idx = start_block; block_idx < end_block; ++block_idx) { + size_t start_row = block_idx * TILE_M; + size_t end_row = std::min(start_row + TILE_M, M); + + cactus_matmul_f16_worker( + a, b_transposed, c, + M, K, N, + start_row, end_row + ); + + } + }); +} + + +void cactus_gemv_int8( + const int8_t* A, + const float A_scale, + const int8_t* B, + const __fp16* B_scales, + __fp16* C, + size_t K, size_t N, + size_t group_size +) { + if (K == 0 || N == 0) return; + + const size_t num_groups = K / group_size; + const size_t N_blocks = (N + 3) / 4; + + auto process_blocks = [=](size_t block_start, size_t block_end) { + for (size_t n_block = block_start; n_block < block_end; ++n_block) { + const size_t n_start = n_block * 4; + const size_t actual_n = std::min(size_t(4), N - n_start); + + float32x4_t running_sum = vdupq_n_f32(0.0f); + + size_t g = 0; + for (; g + 1 < num_groups; g += 2) { + const size_t k_base0 = g * group_size; + const size_t k_base1 = (g + 1) * group_size; + + const int8_t* a_ptr0 = A + k_base0; + const int8_t* a_ptr1 = A + k_base1; + const int8_t* b_base0 = B + (n_block * K + k_base0) * 4; + const int8_t* b_base1 = B + (n_block * K + k_base1) * 4; + + __builtin_prefetch(b_base0 + group_size * 8, 0, 3); + + int32x4_t acc0 = vdupq_n_s32(0); + int32x4_t acc1 = vdupq_n_s32(0); + + { + int8x16_t a_vec = vld1q_s8(a_ptr0); + int8x16_t b0 = vld1q_s8(b_base0); + int8x16_t b1 = vld1q_s8(b_base0 + 16); + int8x16_t b2 = vld1q_s8(b_base0 + 32); + int8x16_t b3 = vld1q_s8(b_base0 + 48); + + acc0 = CACTUS_DOTQ_LANE(acc0, b0, a_vec, 0); + acc0 = CACTUS_DOTQ_LANE(acc0, b1, a_vec, 1); + acc0 = CACTUS_DOTQ_LANE(acc0, b2, a_vec, 2); + acc0 = CACTUS_DOTQ_LANE(acc0, b3, a_vec, 3); + + a_vec = vld1q_s8(a_ptr0 + 16); + b0 = vld1q_s8(b_base0 + 64); + b1 = vld1q_s8(b_base0 + 80); + b2 = vld1q_s8(b_base0 + 96); + b3 = vld1q_s8(b_base0 + 112); + + acc0 = CACTUS_DOTQ_LANE(acc0, b0, a_vec, 0); + acc0 = CACTUS_DOTQ_LANE(acc0, b1, a_vec, 1); + acc0 = CACTUS_DOTQ_LANE(acc0, b2, a_vec, 2); + acc0 = CACTUS_DOTQ_LANE(acc0, b3, a_vec, 3); + } + + { + int8x16_t a_vec = vld1q_s8(a_ptr1); + int8x16_t b0 = vld1q_s8(b_base1); + int8x16_t b1 = vld1q_s8(b_base1 + 16); + int8x16_t b2 = vld1q_s8(b_base1 + 32); + int8x16_t b3 = vld1q_s8(b_base1 + 48); + + acc1 = CACTUS_DOTQ_LANE(acc1, b0, a_vec, 0); + acc1 = CACTUS_DOTQ_LANE(acc1, b1, a_vec, 1); + acc1 = CACTUS_DOTQ_LANE(acc1, b2, a_vec, 2); + acc1 = CACTUS_DOTQ_LANE(acc1, b3, a_vec, 3); + + a_vec = vld1q_s8(a_ptr1 + 16); + b0 = vld1q_s8(b_base1 + 64); + b1 = vld1q_s8(b_base1 + 80); + b2 = vld1q_s8(b_base1 + 96); + b3 = vld1q_s8(b_base1 + 112); + + acc1 = CACTUS_DOTQ_LANE(acc1, b0, a_vec, 0); + acc1 = CACTUS_DOTQ_LANE(acc1, b1, a_vec, 1); + acc1 = CACTUS_DOTQ_LANE(acc1, b2, a_vec, 2); + acc1 = CACTUS_DOTQ_LANE(acc1, b3, a_vec, 3); + } + + const __fp16* scale_ptr0 = B_scales + (n_block * num_groups + g) * 4; + const __fp16* scale_ptr1 = B_scales + (n_block * num_groups + g + 1) * 4; + + float16x4_t scales0_f16 = vld1_f16(scale_ptr0); + float16x4_t scales1_f16 = vld1_f16(scale_ptr1); + float32x4_t scales0 = vcvt_f32_f16(scales0_f16); + float32x4_t scales1 = vcvt_f32_f16(scales1_f16); + + running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc0), scales0); + running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc1), scales1); + } + + for (; g < num_groups; g++) { + const size_t k_base = g * group_size; + const int8_t* a_ptr = A + k_base; + const int8_t* b_base = B + (n_block * K + k_base) * 4; + + int32x4_t acc = vdupq_n_s32(0); + + int8x16_t a_vec = vld1q_s8(a_ptr); + int8x16_t b0 = vld1q_s8(b_base); + int8x16_t b1 = vld1q_s8(b_base + 16); + int8x16_t b2 = vld1q_s8(b_base + 32); + int8x16_t b3 = vld1q_s8(b_base + 48); + + acc = CACTUS_DOTQ_LANE(acc, b0, a_vec, 0); + acc = CACTUS_DOTQ_LANE(acc, b1, a_vec, 1); + acc = CACTUS_DOTQ_LANE(acc, b2, a_vec, 2); + acc = CACTUS_DOTQ_LANE(acc, b3, a_vec, 3); + + a_vec = vld1q_s8(a_ptr + 16); + b0 = vld1q_s8(b_base + 64); + b1 = vld1q_s8(b_base + 80); + b2 = vld1q_s8(b_base + 96); + b3 = vld1q_s8(b_base + 112); + + acc = CACTUS_DOTQ_LANE(acc, b0, a_vec, 0); + acc = CACTUS_DOTQ_LANE(acc, b1, a_vec, 1); + acc = CACTUS_DOTQ_LANE(acc, b2, a_vec, 2); + acc = CACTUS_DOTQ_LANE(acc, b3, a_vec, 3); + + const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; + float16x4_t scales_f16 = vld1_f16(scale_ptr); + float32x4_t scales = vcvt_f32_f16(scales_f16); + + running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc), scales); + } + + float32x4_t result = vmulq_n_f32(running_sum, A_scale); + float16x4_t result_f16 = vcvt_f16_f32(result); + + if (actual_n == 4) { + vst1_f16(C + n_start, result_f16); + } else { + for (size_t ni = 0; ni < actual_n; ni++) { + C[n_start + ni] = vget_lane_f16(result_f16, 0); + result_f16 = vext_f16(result_f16, result_f16, 1); + } + } + } + }; + + auto& pool = CactusThreading::get_thread_pool(); + size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); + num_threads = std::min(num_threads, N_blocks); + + if (num_threads <= 1) { + process_blocks(0, N_blocks); + } else { + pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); + pool.wait_all(); + } +} + +void cactus_gemm_int8( + const int8_t* A, + const float* A_scales, + const int8_t* B, + const __fp16* B_scales, + __fp16* C, + size_t M, size_t K, size_t N, + size_t group_size +) { + if (M == 0 || K == 0 || N == 0) return; + + constexpr size_t TILE_M = 8; + constexpr size_t TILE_N = 4; + + const size_t num_groups = K / group_size; + const size_t N_blocks = (N + TILE_N - 1) / TILE_N; + const size_t num_row_tiles = (M + TILE_M - 1) / TILE_M; + const size_t total_tiles = num_row_tiles * N_blocks; + + CactusThreading::parallel_gemm_tiles(M, total_tiles, + [=](size_t tile_start, size_t tile_end) { + for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { + const size_t tile_row = tile_idx / N_blocks; + const size_t n_block = tile_idx % N_blocks; + const size_t m_start = tile_row * TILE_M; + const size_t m_end = std::min(m_start + TILE_M, M); + const size_t n_start = n_block * TILE_N; + const size_t n_end = std::min(n_start + TILE_N, N); + const size_t actual_m = m_end - m_start; + const size_t actual_n = n_end - n_start; + + const int8_t* a_rows[TILE_M]; + for (size_t mi = 0; mi < TILE_M; mi++) { + size_t row = m_start + (mi < actual_m ? mi : actual_m - 1); + a_rows[mi] = A + row * K; + } + + float32x4_t running_sum[TILE_M]; + for (size_t mi = 0; mi < TILE_M; mi++) { + running_sum[mi] = vdupq_n_f32(0.0f); + } + + for (size_t g = 0; g < num_groups; g++) { + const size_t k_base = g * group_size; + const int8_t* b_base = B + (n_block * K + k_base) * 4; + + __builtin_prefetch(b_base + group_size * 4, 0, 3); + + int8x16_t b00 = vld1q_s8(b_base); + int8x16_t b01 = vld1q_s8(b_base + 16); + int8x16_t b02 = vld1q_s8(b_base + 32); + int8x16_t b03 = vld1q_s8(b_base + 48); + + int8x16_t b10 = vld1q_s8(b_base + 64); + int8x16_t b11 = vld1q_s8(b_base + 80); + int8x16_t b12 = vld1q_s8(b_base + 96); + int8x16_t b13 = vld1q_s8(b_base + 112); + + const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; + float16x4_t scales_f16 = vld1_f16(scale_ptr); + float32x4_t scales = vcvt_f32_f16(scales_f16); + + #define CACTUS_GEMM_ROW(ROW) do { \ + const int8_t* a_ptr_##ROW = a_rows[ROW] + k_base; \ + int32x4_t acc_##ROW = vdupq_n_s32(0); \ + int8x16_t a_lo_##ROW = vld1q_s8(a_ptr_##ROW); \ + acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b00, a_lo_##ROW, 0); \ + acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b01, a_lo_##ROW, 1); \ + acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b02, a_lo_##ROW, 2); \ + acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b03, a_lo_##ROW, 3); \ + int8x16_t a_hi_##ROW = vld1q_s8(a_ptr_##ROW + 16); \ + acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b10, a_hi_##ROW, 0); \ + acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b11, a_hi_##ROW, 1); \ + acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b12, a_hi_##ROW, 2); \ + acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b13, a_hi_##ROW, 3); \ + running_sum[ROW] = vmlaq_f32(running_sum[ROW], vcvtq_f32_s32(acc_##ROW), scales); \ + } while(0) + + CACTUS_GEMM_ROW(0); + CACTUS_GEMM_ROW(1); + CACTUS_GEMM_ROW(2); + CACTUS_GEMM_ROW(3); + CACTUS_GEMM_ROW(4); + CACTUS_GEMM_ROW(5); + CACTUS_GEMM_ROW(6); + CACTUS_GEMM_ROW(7); + #undef CACTUS_GEMM_ROW + } + + for (size_t mi = 0; mi < actual_m; mi++) { + const float a_scale = A_scales[m_start + mi]; + float32x4_t result = vmulq_n_f32(running_sum[mi], a_scale); + float16x4_t result_f16 = vcvt_f16_f32(result); + + if (actual_n == 4) { + vst1_f16(C + (m_start + mi) * N + n_start, result_f16); + } else { + for (size_t ni = 0; ni < actual_n; ni++) { + C[(m_start + mi) * N + n_start + ni] = vget_lane_f16(result_f16, 0); + result_f16 = vext_f16(result_f16, result_f16, 1); + } + } + } + } + }); +} + +void cactus_matmul_int8( + const int8_t* A, + const float* A_scales, + const int8_t* B, + const __fp16* B_scales, + __fp16* C, + size_t M, size_t K, size_t N, + size_t group_size +) { + if (M == 0 || K == 0 || N == 0) return; + +#if defined(CACTUS_COMPILE_I8MM) + if (cpu_has_i8mm()) { + if (M == 1) { + cactus_gemv_int8_i8mm(A, A_scales[0], B, B_scales, C, K, N, group_size); + } else { + cactus_gemm_int8_i8mm(A, A_scales, B, B_scales, C, M, K, N, group_size); + } + return; + } +#endif + + if (M == 1) { + cactus_gemv_int8(A, A_scales[0], B, B_scales, C, K, N, group_size); + } else { + cactus_gemm_int8(A, A_scales, B, B_scales, C, M, K, N, group_size); + } +} + +void cactus_gemv_int4( + const int8_t* A, + const float A_scale, + const int8_t* B_packed_raw, + const __fp16* B_scales, + __fp16* C, + size_t K, size_t N, + size_t group_size +) { + const uint8_t* B_packed = reinterpret_cast(B_packed_raw); + if (K == 0 || N == 0) return; + + const size_t num_groups = K / group_size; + const size_t N_blocks = (N + 3) / 4; + + auto process_blocks = [=](size_t block_start, size_t block_end) { + size_t n_block = block_start; + + for (; n_block + 1 < block_end; n_block += 2) { + float32x4_t sum_a = vdupq_n_f32(0.0f); + float32x4_t sum_b = vdupq_n_f32(0.0f); + + for (size_t g = 0; g < num_groups; g++) { + const size_t k_base = g * group_size; + const int8_t* a_ptr = A + k_base; + const uint8_t* ba = B_packed + (n_block * K + k_base) * 2; + const uint8_t* bb = B_packed + ((n_block + 1) * K + k_base) * 2; + + int32x4_t acc_a = vdupq_n_s32(0); + int32x4_t acc_b = vdupq_n_s32(0); + + int8x16_t a_lo = vld1q_s8(a_ptr); + int8x16_t a_hi = vld1q_s8(a_ptr + 16); + + { + int8x16_t b0, b1, b2, b3; + unpack_int4_as_int8x16x2(ba, b1, b0); + unpack_int4_as_int8x16x2(ba + 16, b3, b2); + acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_lo, 0); + acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_lo, 1); + acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_lo, 2); + acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_lo, 3); + unpack_int4_as_int8x16x2(ba + 32, b1, b0); + unpack_int4_as_int8x16x2(ba + 48, b3, b2); + acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_hi, 0); + acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_hi, 1); + acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_hi, 2); + acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_hi, 3); + } + { + int8x16_t b0, b1, b2, b3; + unpack_int4_as_int8x16x2(bb, b1, b0); + unpack_int4_as_int8x16x2(bb + 16, b3, b2); + acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_lo, 0); + acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_lo, 1); + acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_lo, 2); + acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_lo, 3); + unpack_int4_as_int8x16x2(bb + 32, b1, b0); + unpack_int4_as_int8x16x2(bb + 48, b3, b2); + acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_hi, 0); + acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_hi, 1); + acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_hi, 2); + acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_hi, 3); + } + + const __fp16* spa = B_scales + (n_block * num_groups + g) * 4; + const __fp16* spb = B_scales + ((n_block + 1) * num_groups + g) * 4; + float32x4_t sa = vcvt_f32_f16(vld1_f16(spa)); + float32x4_t sb = vcvt_f32_f16(vld1_f16(spb)); + sum_a = vmlaq_f32(sum_a, vcvtq_f32_s32(acc_a), sa); + sum_b = vmlaq_f32(sum_b, vcvtq_f32_s32(acc_b), sb); + } + + vst1_f16(C + n_block * 4, vcvt_f16_f32(vmulq_n_f32(sum_a, A_scale))); + vst1_f16(C + (n_block + 1) * 4, vcvt_f16_f32(vmulq_n_f32(sum_b, A_scale))); + } + + for (; n_block < block_end; ++n_block) { + const size_t n_start = n_block * 4; + const size_t actual_n = std::min(size_t(4), N - n_start); + float32x4_t running_sum = vdupq_n_f32(0.0f); + + for (size_t g = 0; g < num_groups; g++) { + const size_t k_base = g * group_size; + const int8_t* a_ptr = A + k_base; + const uint8_t* b_base = B_packed + (n_block * K + k_base) * 2; + + int32x4_t acc = vdupq_n_s32(0); + int8x16_t a_lo = vld1q_s8(a_ptr); + int8x16_t a_hi = vld1q_s8(a_ptr + 16); + + int8x16_t b0, b1, b2, b3; + unpack_int4_as_int8x16x2(b_base, b1, b0); + unpack_int4_as_int8x16x2(b_base + 16, b3, b2); + acc = CACTUS_DOTQ_LANE(acc, b0, a_lo, 0); + acc = CACTUS_DOTQ_LANE(acc, b1, a_lo, 1); + acc = CACTUS_DOTQ_LANE(acc, b2, a_lo, 2); + acc = CACTUS_DOTQ_LANE(acc, b3, a_lo, 3); + unpack_int4_as_int8x16x2(b_base + 32, b1, b0); + unpack_int4_as_int8x16x2(b_base + 48, b3, b2); + acc = CACTUS_DOTQ_LANE(acc, b0, a_hi, 0); + acc = CACTUS_DOTQ_LANE(acc, b1, a_hi, 1); + acc = CACTUS_DOTQ_LANE(acc, b2, a_hi, 2); + acc = CACTUS_DOTQ_LANE(acc, b3, a_hi, 3); + + float32x4_t scales = vcvt_f32_f16(vld1_f16(B_scales + (n_block * num_groups + g) * 4)); + running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc), scales); + } + + float32x4_t result = vmulq_n_f32(running_sum, A_scale); + float16x4_t result_f16 = vcvt_f16_f32(result); + if (actual_n == 4) { + vst1_f16(C + n_start, result_f16); + } else { + for (size_t ni = 0; ni < actual_n; ni++) { + C[n_start + ni] = vget_lane_f16(result_f16, 0); + result_f16 = vext_f16(result_f16, result_f16, 1); + } + } + } + }; + + auto& pool = CactusThreading::get_thread_pool(); + size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); + num_threads = std::min(num_threads, N_blocks); + + if (num_threads <= 1) { + process_blocks(0, N_blocks); + } else { + pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); + pool.wait_all(); + } +} + + +void cactus_gemm_int4( + const int8_t* A, + const float* A_scales, + const int8_t* B_packed_raw, + const __fp16* B_scales, + __fp16* C, + size_t M, size_t K, size_t N, + size_t group_size +) { + if (M == 0 || K == 0 || N == 0) return; + + const uint8_t* B_packed = reinterpret_cast(B_packed_raw); + + constexpr size_t TILE_M = 4; + constexpr size_t TILE_N = 4; + + const size_t num_groups = K / group_size; + const size_t N_blocks = (N + TILE_N - 1) / TILE_N; + const size_t num_row_tiles = (M + TILE_M - 1) / TILE_M; + const size_t total_tiles = num_row_tiles * N_blocks; + + CactusThreading::parallel_gemm_tiles(M, total_tiles, + [=](size_t tile_start, size_t tile_end) { + for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { + const size_t tile_row = tile_idx / N_blocks; + const size_t n_block = tile_idx % N_blocks; + const size_t m_start = tile_row * TILE_M; + const size_t m_end = std::min(m_start + TILE_M, M); + const size_t n_start = n_block * TILE_N; + const size_t n_end = std::min(n_start + TILE_N, N); + const size_t actual_m = m_end - m_start; + const size_t actual_n = n_end - n_start; + + float32x4_t running_sum[TILE_M] = { + vdupq_n_f32(0.0f), vdupq_n_f32(0.0f), + vdupq_n_f32(0.0f), vdupq_n_f32(0.0f) + }; + + for (size_t g = 0; g < num_groups; g++) { + const size_t k_base = g * group_size; + const uint8_t* b_base = B_packed + (n_block * K + k_base) * 2; + + __builtin_prefetch(b_base + group_size * 2, 0, 3); + + int8x16_t b00, b01, b02, b03; + int8x16_t b10, b11, b12, b13; + + unpack_int4_as_int8x16x2(b_base, b01, b00); + unpack_int4_as_int8x16x2(b_base + 16, b03, b02); + unpack_int4_as_int8x16x2(b_base + 32, b11, b10); + unpack_int4_as_int8x16x2(b_base + 48, b13, b12); + + const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; + float16x4_t scales_f16 = vld1_f16(scale_ptr); + float32x4_t scales = vcvt_f32_f16(scales_f16); + + for (size_t mi = 0; mi < actual_m; mi++) { + const int8_t* a_ptr = A + (m_start + mi) * K + k_base; + + int32x4_t acc = vdupq_n_s32(0); + + int8x16_t a_vec = vld1q_s8(a_ptr); + acc = CACTUS_DOTQ_LANE(acc, b00, a_vec, 0); + acc = CACTUS_DOTQ_LANE(acc, b01, a_vec, 1); + acc = CACTUS_DOTQ_LANE(acc, b02, a_vec, 2); + acc = CACTUS_DOTQ_LANE(acc, b03, a_vec, 3); + + a_vec = vld1q_s8(a_ptr + 16); + acc = CACTUS_DOTQ_LANE(acc, b10, a_vec, 0); + acc = CACTUS_DOTQ_LANE(acc, b11, a_vec, 1); + acc = CACTUS_DOTQ_LANE(acc, b12, a_vec, 2); + acc = CACTUS_DOTQ_LANE(acc, b13, a_vec, 3); + + running_sum[mi] = vmlaq_f32(running_sum[mi], vcvtq_f32_s32(acc), scales); + } + } + + for (size_t mi = 0; mi < actual_m; mi++) { + const float a_scale = A_scales[m_start + mi]; + float32x4_t result = vmulq_n_f32(running_sum[mi], a_scale); + float16x4_t result_f16 = vcvt_f16_f32(result); + + if (actual_n == 4) { + vst1_f16(C + (m_start + mi) * N + n_start, result_f16); + } else { + for (size_t ni = 0; ni < actual_n; ni++) { + C[(m_start + mi) * N + n_start + ni] = vget_lane_f16(result_f16, 0); + result_f16 = vext_f16(result_f16, result_f16, 1); + } + } + } + } + }); +} + +void cactus_matmul_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size) { + if (M == 0 || K == 0 || N == 0) return; + + if (M == 1) { + cactus_gemv_int4(A, A_scales[0], B_packed, B_scales, C, K, N, group_size); + } else { + cactus_gemm_int4(A, A_scales, B_packed, B_scales, C, M, K, N, group_size); + } +} + +void cactus_matmul_integer(Precision precision, + const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size) { + if (precision == Precision::INT4) { + cactus_matmul_int4(A, A_scales, B, B_scales, C, M, K, N, group_size); + } else { + cactus_matmul_int8(A, A_scales, B, B_scales, C, M, K, N, group_size); + } +} diff --git a/cactus-transpile/kernel/kernel_nn.cpp b/cactus-transpile/kernel/kernel_nn.cpp new file mode 100644 index 000000000..7dd96ddc0 --- /dev/null +++ b/cactus-transpile/kernel/kernel_nn.cpp @@ -0,0 +1,874 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include +#include +#include +#include +#include + +void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements) { + for (size_t i = 0; i < num_elements; ++i) { + __fp16 x = input[i]; + output[i] = x > static_cast<__fp16>(0) ? x : static_cast<__fp16>(0); + } +} + +void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope) { + for (size_t i = 0; i < num_elements; ++i) { + __fp16 x = input[i]; + output[i] = x > static_cast<__fp16>(0) ? x : static_cast<__fp16>(static_cast(x) * negative_slope); + } +} + +void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements) { + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_idx, size_t end_idx) { + for (size_t i = start_idx; i < end_idx; ++i) { + float x_f32 = static_cast(input[i]); + float sigmoid = 1.0f / (1.0f + expf(-x_f32)); + output[i] = static_cast<__fp16>(x_f32 * sigmoid); + } + }); +} + +void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements) { + const float sqrt_2_over_pi = 0.7978845608028654f; + const float coeff = 0.044715f; + + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_idx, size_t end_idx) { + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; + + const float32x4_t half = vdupq_n_f32(0.5f); + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t sqrt_2_pi_vec = vdupq_n_f32(sqrt_2_over_pi); + const float32x4_t coeff_vec = vdupq_n_f32(coeff); + + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t x_f16 = vld1q_f16(&input[i]); + + float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_f16)); + float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_f16)); + + float32x4_t x_cubed_low = vmulq_f32(vmulq_f32(x_low, x_low), x_low); + float32x4_t x_cubed_high = vmulq_f32(vmulq_f32(x_high, x_high), x_high); + + float32x4_t inner_low = vfmaq_f32(x_low, coeff_vec, x_cubed_low); + float32x4_t inner_high = vfmaq_f32(x_high, coeff_vec, x_cubed_high); + inner_low = vmulq_f32(sqrt_2_pi_vec, inner_low); + inner_high = vmulq_f32(sqrt_2_pi_vec, inner_high); + + float32x4_t tanh_low = fast_tanh_f32x4(inner_low); + float32x4_t tanh_high = fast_tanh_f32x4(inner_high); + + float32x4_t gelu_low = vmulq_f32(vmulq_f32(half, x_low), vaddq_f32(one, tanh_low)); + float32x4_t gelu_high = vmulq_f32(vmulq_f32(half, x_high), vaddq_f32(one, tanh_high)); + + float16x8_t gelu_f16 = vcombine_f16(vcvt_f16_f32(gelu_low), vcvt_f16_f32(gelu_high)); + vst1q_f16(&output[i], gelu_f16); + } + + for (size_t i = vectorized_end; i < end_idx; ++i) { + float x = static_cast(input[i]); + float inner = sqrt_2_over_pi * (x + coeff * x * x * x); + float gelu = 0.5f * x * (1.0f + tanhf(inner)); + output[i] = static_cast<__fp16>(gelu); + } + }); +} + +void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements) +{ + const float inv_sqrt2 = 0.70710678118654752440f; + + CactusThreading::parallel_for( + num_elements, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_idx, size_t end_idx) { + + constexpr size_t SIMD = 8; + size_t vec_end = start_idx + ((end_idx - start_idx) / SIMD) * SIMD; + + for (size_t i = start_idx; i < vec_end; i += SIMD) { + + float16x8_t xh = vld1q_f16(&input[i]); + + float32x4_t x0 = vcvt_f32_f16(vget_low_f16(xh)); + float32x4_t x1 = vcvt_f32_f16(vget_high_f16(xh)); + + float32x4_t arg0 = vmulq_n_f32(x0, inv_sqrt2); + float32x4_t arg1 = vmulq_n_f32(x1, inv_sqrt2); + + float arg0_s[4], arg1_s[4]; + float erf0_s[4], erf1_s[4]; + vst1q_f32(arg0_s, arg0); + vst1q_f32(arg1_s, arg1); + + for (int j = 0; j < 4; j++) { + erf0_s[j] = erff(arg0_s[j]); + erf1_s[j] = erff(arg1_s[j]); + } + + float32x4_t erf0 = vld1q_f32(erf0_s); + float32x4_t erf1 = vld1q_f32(erf1_s); + + float32x4_t t0 = vaddq_f32(vdupq_n_f32(1.0f), erf0); + float32x4_t t1 = vaddq_f32(vdupq_n_f32(1.0f), erf1); + + float32x4_t gelu0 = vmulq_f32(vmulq_n_f32(x0, 0.5f), t0); + float32x4_t gelu1 = vmulq_f32(vmulq_n_f32(x1, 0.5f), t1); + + float16x8_t gelu_h = vcombine_f16( + vcvt_f16_f32(gelu0), + vcvt_f16_f32(gelu1) + ); + + vst1q_f16(&output[i], gelu_h); + } + + for (size_t i = vec_end; i < end_idx; i++) { + float x = (float)input[i]; + float arg = x * inv_sqrt2; + float res = 0.5f * x * (1.0f + erff(arg)); + output[i] = (__fp16)res; + } + } + ); +} + +void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements) { + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_idx, size_t end_idx) { + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; + const float32x4_t one_f32 = vdupq_n_f32(1.0f); + + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t x = vld1q_f16(&input[i]); + + float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x)); + float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x)); + + float32x4_t exp_low = fast_exp_f32x4(vnegq_f32(x_low)); + float32x4_t exp_high = fast_exp_f32x4(vnegq_f32(x_high)); + + float32x4_t sigmoid_low = vdivq_f32(one_f32, vaddq_f32(one_f32, exp_low)); + float32x4_t sigmoid_high = vdivq_f32(one_f32, vaddq_f32(one_f32, exp_high)); + + float16x8_t sigmoid = vcombine_f16(vcvt_f16_f32(sigmoid_low), vcvt_f16_f32(sigmoid_high)); + vst1q_f16(&output[i], sigmoid); + } + + for (size_t i = vectorized_end; i < end_idx; ++i) { + float x_f32 = static_cast(input[i]); + float sigmoid = 1.0f / (1.0f + expf(-x_f32)); + output[i] = static_cast<__fp16>(sigmoid); + } + }); +} + +void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements) { + CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_idx, size_t end_idx) { + constexpr size_t SIMD_WIDTH = 8; + const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; + + for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { + float16x8_t x_f16 = vld1q_f16(&input[i]); + + float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_f16)); + float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_f16)); + + float32x4_t tanh_low = fast_tanh_f32x4(x_low); + float32x4_t tanh_high = fast_tanh_f32x4(x_high); + + float16x8_t tanh_f16 = vcombine_f16(vcvt_f16_f32(tanh_low), vcvt_f16_f32(tanh_high)); + vst1q_f16(&output[i], tanh_f16); + } + + for (size_t i = vectorized_end; i < end_idx; ++i) { + float x = static_cast(input[i]); + output[i] = static_cast<__fp16>(std::tanh(x)); + } + }); +} + +void cactus_glu_f16( + const __fp16* input, + __fp16* output, + size_t outer_size, + size_t split_size, + size_t inner_size +) { + if (outer_size == 0 || split_size == 0 || inner_size == 0) return; + + const size_t axis_size = split_size * 2; + const size_t rows = outer_size * split_size; + + CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t row_start, size_t row_end) { + for (size_t row = row_start; row < row_end; ++row) { + const size_t outer_idx = row / split_size; + const size_t split_idx = row - outer_idx * split_size; + const size_t in_base = outer_idx * axis_size * inner_size; + const size_t out_base = outer_idx * split_size * inner_size; + + const __fp16* a = input + in_base + split_idx * inner_size; + const __fp16* b = input + in_base + (split_idx + split_size) * inner_size; + __fp16* y = output + out_base + split_idx * inner_size; + + for (size_t i = 0; i < inner_size; ++i) { + const float gate_input = static_cast(b[i]); + const float gate = gate_input >= 0.0f + ? 1.0f / (1.0f + expf(-gate_input)) + : expf(gate_input) / (1.0f + expf(gate_input)); + y[i] = static_cast<__fp16>(static_cast(a[i]) * gate); + } + } + }); +} + +void cactus_glu_f32( + const float* input, + float* output, + size_t outer_size, + size_t split_size, + size_t inner_size +) { + if (outer_size == 0 || split_size == 0 || inner_size == 0) return; + + const size_t axis_size = split_size * 2; + const size_t rows = outer_size * split_size; + + CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t row_start, size_t row_end) { + for (size_t row = row_start; row < row_end; ++row) { + const size_t outer_idx = row / split_size; + const size_t split_idx = row - outer_idx * split_size; + const size_t in_base = outer_idx * axis_size * inner_size; + const size_t out_base = outer_idx * split_size * inner_size; + + const float* a = input + in_base + split_idx * inner_size; + const float* b = input + in_base + (split_idx + split_size) * inner_size; + float* y = output + out_base + split_idx * inner_size; + + for (size_t i = 0; i < inner_size; ++i) { + const float gate_input = b[i]; + const float gate = gate_input >= 0.0f + ? 1.0f / (1.0f + expf(-gate_input)) + : expf(gate_input) / (1.0f + expf(gate_input)); + y[i] = a[i] * gate; + } + } + }); +} + +void cactus_batchnorm_f16( + const __fp16* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + __fp16* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +) { + if (outer_size == 0 || channels == 0 || inner_size == 0) return; + + std::vector scale(channels); + std::vector shift(channels); + for (size_t c = 0; c < channels; ++c) { + const float inv_std = 1.0f / std::sqrt(running_var[c] + epsilon); + scale[c] = weight[c] * inv_std; + shift[c] = bias[c] - running_mean[c] * scale[c]; + } + + const size_t rows = outer_size * channels; + CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t row_start, size_t row_end) { + for (size_t row = row_start; row < row_end; ++row) { + const size_t c = row % channels; + const float32x4_t scv = vdupq_n_f32(scale[c]); + const float32x4_t shv = vdupq_n_f32(shift[c]); + + const __fp16* x = input + row * inner_size; + __fp16* y = output + row * inner_size; + + size_t i = 0; + for (; i + 8 <= inner_size; i += 8) { + const float16x8_t xv = vld1q_f16(x + i); + float32x4_t lo = vcvt_f32_f16(vget_low_f16(xv)); + float32x4_t hi = vcvt_f32_f16(vget_high_f16(xv)); + lo = vfmaq_f32(shv, lo, scv); + hi = vfmaq_f32(shv, hi, scv); + const float16x8_t yv = vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); + vst1q_f16(y + i, yv); + } + + for (; i < inner_size; ++i) { + y[i] = static_cast<__fp16>(static_cast(x[i]) * scale[c] + shift[c]); + } + } + }); +} + +void cactus_batchnorm_f32( + const float* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + float* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +) { + if (outer_size == 0 || channels == 0 || inner_size == 0) return; + + std::vector scale(channels); + std::vector shift(channels); + for (size_t c = 0; c < channels; ++c) { + const float inv_std = 1.0f / std::sqrt(running_var[c] + epsilon); + scale[c] = weight[c] * inv_std; + shift[c] = bias[c] - running_mean[c] * scale[c]; + } + + const size_t rows = outer_size * channels; + CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, + [&](size_t row_start, size_t row_end) { + for (size_t row = row_start; row < row_end; ++row) { + const size_t c = row % channels; + const float32x4_t scv = vdupq_n_f32(scale[c]); + const float32x4_t shv = vdupq_n_f32(shift[c]); + + const float* x = input + row * inner_size; + float* y = output + row * inner_size; + + size_t i = 0; + for (; i + 4 <= inner_size; i += 4) { + const float32x4_t xv = vld1q_f32(x + i); + vst1q_f32(y + i, vfmaq_f32(shv, xv, scv)); + } + + for (; i < inner_size; ++i) { + y[i] = x[i] * scale[c] + shift[c]; + } + } + }); +} + +void kernel_softmax_f16_single(const __fp16* input, __fp16* output, size_t vocab_size) { + + constexpr size_t SIMD_WIDTH = 8; + constexpr size_t UNROLL_FACTOR = 4; + constexpr size_t VECTORIZED_WIDTH = SIMD_WIDTH * UNROLL_FACTOR; + const size_t vocab_vectorized = (vocab_size / VECTORIZED_WIDTH) * VECTORIZED_WIDTH; + + float32x4_t max_vec[UNROLL_FACTOR * 2]; + for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { + max_vec[u] = vdupq_n_f32(-std::numeric_limits::infinity()); + } + + for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { + for (size_t u = 0; u < UNROLL_FACTOR; u++) { + float16x8_t x_vec_f16 = vld1q_f16(&input[i + u * SIMD_WIDTH]); + float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_vec_f16)); + float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_vec_f16)); + max_vec[u * 2] = vmaxq_f32(max_vec[u * 2], x_low); + max_vec[u * 2 + 1] = vmaxq_f32(max_vec[u * 2 + 1], x_high); + } + } + + float32x4_t final_max = max_vec[0]; + for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { + final_max = vmaxq_f32(final_max, max_vec[u]); + } + + float max_val = vmaxvq_f32(final_max); + for (size_t i = vocab_vectorized; i < vocab_size; ++i) { + max_val = std::max(max_val, static_cast(input[i])); + } + + const float32x4_t max_broadcast = vdupq_n_f32(max_val); + + float32x4_t sum_vec[UNROLL_FACTOR * 2]; + for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { + sum_vec[u] = vdupq_n_f32(0.0f); + } + + for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { + for (size_t u = 0; u < UNROLL_FACTOR; u++) { + float16x8_t x_vec_f16 = vld1q_f16(&input[i + u * SIMD_WIDTH]); + + float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_vec_f16)); + float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_vec_f16)); + + float32x4_t exp_low = fast_exp_f32x4(vsubq_f32(x_low, max_broadcast)); + float32x4_t exp_high = fast_exp_f32x4(vsubq_f32(x_high, max_broadcast)); + + float16x8_t exp_f16 = vcombine_f16(vcvt_f16_f32(exp_low), vcvt_f16_f32(exp_high)); + vst1q_f16(&output[i + u * SIMD_WIDTH], exp_f16); + + sum_vec[u * 2] = vaddq_f32(sum_vec[u * 2], exp_low); + sum_vec[u * 2 + 1] = vaddq_f32(sum_vec[u * 2 + 1], exp_high); + } + } + + float32x4_t final_sum = sum_vec[0]; + for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { + final_sum = vaddq_f32(final_sum, sum_vec[u]); + } + + float sum = vaddvq_f32(final_sum); + for (size_t i = vocab_vectorized; i < vocab_size; ++i) { + float exp_val = expf(static_cast(input[i]) - max_val); + output[i] = static_cast<__fp16>(exp_val); + sum += exp_val; + } + + const float inv_sum = 1.0f / sum; + const float16x8_t inv_sum_vec_f16 = vdupq_n_f16(static_cast<__fp16>(inv_sum)); + + for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { + for (size_t u = 0; u < UNROLL_FACTOR; u++) { + float16x8_t exp_vec = vld1q_f16(&output[i + u * SIMD_WIDTH]); + float16x8_t result = vmulq_f16(exp_vec, inv_sum_vec_f16); + vst1q_f16(&output[i + u * SIMD_WIDTH], result); + } + } + + for (size_t i = vocab_vectorized; i < vocab_size; ++i) { + output[i] = static_cast<__fp16>(static_cast(output[i]) * inv_sum); + } +} + +void cactus_softmax_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t vocab_size +) { + CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start_idx, size_t end_idx) { + for (size_t idx = start_idx; idx < end_idx; ++idx) { + const size_t offset = idx * vocab_size; + kernel_softmax_f16_single(input + offset, output + offset, vocab_size); + } + }); +} + +void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values, const uint32_t* bias_indices, + size_t bias_count) { + cactus_sample_f32_ex(logits, output, vocab_size, + temperature, top_p, 0.15f, 1.1f, + top_k, random_seed, + bias_values, bias_indices, bias_count); +} + +void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values, const uint32_t* bias_indices, + size_t bias_count) { + if (vocab_size == 0) { + output[0] = 0; + return; + } + + const bool has_bias = bias_values && bias_indices && bias_count > 0; + + std::vector filtered_logits(vocab_size); + std::memcpy(filtered_logits.data(), logits, vocab_size * sizeof(float)); + + if (has_bias) { + for (size_t i = 0; i < bias_count; ++i) { + uint32_t idx = bias_indices[i]; + if (idx < vocab_size) { + filtered_logits[idx] += bias_values[i]; + } + } + } + + if (temperature == 0.0f) { + auto it = std::max_element(filtered_logits.begin(), filtered_logits.end()); + output[0] = static_cast(std::distance(filtered_logits.begin(), it)); + return; + } + + if (temperature > 0) { + for (size_t i = 0; i < vocab_size; ++i) { + filtered_logits[i] /= temperature; + } + } + + (void)repetition_penalty; + + if (top_k > 0) { + std::vector> logit_pairs; + logit_pairs.reserve(vocab_size); + for (size_t i = 0; i < vocab_size; ++i) { + logit_pairs.emplace_back(filtered_logits[i], i); + } + std::sort(logit_pairs.begin(), logit_pairs.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + + if (top_k < vocab_size) { + float kth_value = logit_pairs[top_k - 1].first; + for (size_t i = 0; i < vocab_size; ++i) { + if (filtered_logits[i] < kth_value) { + filtered_logits[i] = -std::numeric_limits::infinity(); + } + } + } + } + + if (min_p > 0.0f) { + float max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); + if (!std::isinf(max_logit)) { + std::vector temp_probs(vocab_size); + float sum = 0.0f; + for (size_t i = 0; i < vocab_size; ++i) { + if (!std::isinf(filtered_logits[i])) { + temp_probs[i] = std::exp(filtered_logits[i] - max_logit); + sum += temp_probs[i]; + } else { + temp_probs[i] = 0.0f; + } + } + + if (sum > 0.0f) { + for (size_t i = 0; i < vocab_size; ++i) { + temp_probs[i] /= sum; + } + + float max_prob = *std::max_element(temp_probs.begin(), temp_probs.end()); + float threshold = max_prob * min_p; + + for (size_t i = 0; i < vocab_size; ++i) { + if (temp_probs[i] < threshold) { + filtered_logits[i] = -std::numeric_limits::infinity(); + } + } + } + } + } + + if (top_p > 0.0f && top_p < 1.0f) { + std::vector> sorted_logits; + sorted_logits.reserve(vocab_size); + for (size_t i = 0; i < vocab_size; ++i) { + if (!std::isinf(filtered_logits[i])) { + sorted_logits.emplace_back(filtered_logits[i], i); + } + } + std::sort(sorted_logits.begin(), sorted_logits.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + + float max_logit = sorted_logits.empty() ? 0.0f : sorted_logits[0].first; + std::vector temp_probs; + temp_probs.reserve(sorted_logits.size()); + float sum = 0.0f; + for (const auto& pair : sorted_logits) { + float prob = std::exp(pair.first - max_logit); + temp_probs.push_back(prob); + sum += prob; + } + + for (float& prob : temp_probs) { + prob /= sum; + } + + float cumulative_prob = 0.0f; + std::vector indices_to_remove(sorted_logits.size(), false); + for (size_t i = 0; i < sorted_logits.size(); ++i) { + cumulative_prob += temp_probs[i]; + if (cumulative_prob > top_p) { + indices_to_remove[i] = true; + } + } + + if (!indices_to_remove.empty()) { + for (size_t i = 1; i < indices_to_remove.size(); ++i) { + indices_to_remove[i] = indices_to_remove[i-1] || indices_to_remove[i]; + } + indices_to_remove[0] = false; + } + + for (size_t i = 0; i < sorted_logits.size(); ++i) { + if (indices_to_remove[i]) { + filtered_logits[sorted_logits[i].second] = -std::numeric_limits::infinity(); + } + } + } + + float max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); + if (std::isinf(max_logit)) { + output[0] = 0; + return; + } + + std::vector probs(vocab_size); + float sum = 0.0f; + for (size_t i = 0; i < vocab_size; ++i) { + if (std::isinf(filtered_logits[i])) { + probs[i] = 0.0f; + } else { + probs[i] = std::exp(filtered_logits[i] - max_logit); + sum += probs[i]; + } + } + + if (sum == 0.0f) { + output[0] = 0; + return; + } + + for (size_t i = 0; i < vocab_size; ++i) { + probs[i] /= sum; + } + + uint32_t actual_seed = (random_seed == 0) ? std::random_device{}() : random_seed; + std::mt19937 gen(actual_seed); + std::uniform_real_distribution dist(0.0f, 1.0f); + float sample = dist(gen); + + float cumulative = 0.0f; + for (size_t i = 0; i < vocab_size; ++i) { + cumulative += probs[i]; + if (cumulative >= sample) { + output[0] = static_cast(i); + return; + } + } + + for (size_t i = vocab_size; i > 0; --i) { + if (probs[i-1] > 0.0f) { + output[0] = static_cast(i-1); + return; + } + } + + output[0] = 0; +} + +void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values, const uint32_t* bias_indices, + size_t bias_count) { + cactus_sample_f16_ex(logits, output, vocab_size, + temperature, top_p, 0.15f, 1.1f, + top_k, random_seed, + bias_values, bias_indices, bias_count); +} + +void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values, const uint32_t* bias_indices, + size_t bias_count) { + if (vocab_size == 0) { + output[0] = 0; + return; + } + + const bool has_bias = bias_values && bias_indices && bias_count > 0; + + std::vector<__fp16> filtered_logits(vocab_size); + std::memcpy(filtered_logits.data(), logits, vocab_size * sizeof(__fp16)); + + if (has_bias) { + for (size_t i = 0; i < bias_count; ++i) { + uint32_t idx = bias_indices[i]; + if (idx < vocab_size) { + filtered_logits[idx] = static_cast<__fp16>(static_cast(filtered_logits[idx]) + bias_values[i]); + } + } + } + + if (temperature == 0.0f) { + auto it = std::max_element(filtered_logits.begin(), filtered_logits.end()); + output[0] = static_cast(std::distance(filtered_logits.begin(), it)); + return; + } + + if (temperature > 0) { + __fp16 inv_temp = static_cast<__fp16>(1.0f / temperature); + float16x8_t inv_temp_vec = vdupq_n_f16(inv_temp); + size_t i = 0; + for (; i + 8 <= vocab_size; i += 8) { + float16x8_t logits_vec = vld1q_f16(&filtered_logits[i]); + float16x8_t scaled = vmulq_f16(logits_vec, inv_temp_vec); + vst1q_f16(&filtered_logits[i], scaled); + } + for (; i < vocab_size; ++i) { + filtered_logits[i] = filtered_logits[i] * inv_temp; + } + } + + (void)repetition_penalty; + + if (top_k > 0) { + std::vector> logit_pairs; + logit_pairs.reserve(vocab_size); + for (size_t i = 0; i < vocab_size; ++i) { + logit_pairs.emplace_back(filtered_logits[i], i); + } + std::partial_sort(logit_pairs.begin(), + logit_pairs.begin() + std::min(top_k, vocab_size), + logit_pairs.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + + if (top_k < vocab_size) { + __fp16 kth_value = logit_pairs[top_k - 1].first; + __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); + for (size_t i = 0; i < vocab_size; ++i) { + if (filtered_logits[i] < kth_value) { + filtered_logits[i] = neg_inf; + } + } + } + } + + if (min_p > 0.0f) { + float max_logit = -std::numeric_limits::infinity(); + for (size_t i = 0; i < vocab_size; ++i) { + max_logit = std::max(max_logit, static_cast(filtered_logits[i])); + } + if (!std::isinf(max_logit)) { + std::vector temp_probs(vocab_size); + float sum = 0.0f; + for (size_t i = 0; i < vocab_size; ++i) { + float v = static_cast(filtered_logits[i]); + if (!std::isinf(v)) { + temp_probs[i] = std::exp(v - max_logit); + sum += temp_probs[i]; + } else { + temp_probs[i] = 0.0f; + } + } + if (sum > 0.0f) { + for (float& p : temp_probs) p /= sum; + float max_prob = *std::max_element(temp_probs.begin(), temp_probs.end()); + float threshold = max_prob * min_p; + __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); + for (size_t i = 0; i < vocab_size; ++i) { + if (temp_probs[i] < threshold) { + filtered_logits[i] = neg_inf; + } + } + } + } + } + + if (top_p > 0.0f && top_p < 1.0f) { + std::vector> sorted_logits; + sorted_logits.reserve(vocab_size); + __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); + for (size_t i = 0; i < vocab_size; ++i) { + if (filtered_logits[i] != neg_inf) { + sorted_logits.emplace_back(filtered_logits[i], i); + } + } + std::sort(sorted_logits.begin(), sorted_logits.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + + __fp16 max_logit = sorted_logits.empty() ? static_cast<__fp16>(0.0f) : sorted_logits[0].first; + std::vector temp_probs; + temp_probs.reserve(sorted_logits.size()); + float sum = 0.0f; + for (const auto& pair : sorted_logits) { + float prob = std::exp(static_cast(pair.first - max_logit)); + temp_probs.push_back(prob); + sum += prob; + } + + for (float& prob : temp_probs) { + prob /= sum; + } + + float cumulative_prob = 0.0f; + std::vector indices_to_remove(sorted_logits.size(), false); + bool threshold_reached = false; + for (size_t i = 0; i < sorted_logits.size(); ++i) { + cumulative_prob += temp_probs[i]; + if (cumulative_prob > top_p && i > 0) { + threshold_reached = true; + } + if (threshold_reached) { + indices_to_remove[i] = true; + } + } + + if (!indices_to_remove.empty()) { + for (size_t i = 1; i < indices_to_remove.size(); ++i) { + indices_to_remove[i] = indices_to_remove[i-1] || indices_to_remove[i]; + } + indices_to_remove[0] = false; + } + + for (size_t i = 0; i < sorted_logits.size(); ++i) { + if (indices_to_remove[i]) { + filtered_logits[sorted_logits[i].second] = neg_inf; + } + } + } + + __fp16 max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); + __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); + if (max_logit == neg_inf) { + output[0] = 0; + return; + } + + std::vector probs(vocab_size); + float sum = 0.0f; + for (size_t i = 0; i < vocab_size; ++i) { + if (filtered_logits[i] == neg_inf) { + probs[i] = 0.0f; + } else { + probs[i] = std::exp(static_cast(filtered_logits[i] - max_logit)); + sum += probs[i]; + } + } + + if (sum == 0.0f) { + output[0] = 0; + return; + } + + for (size_t i = 0; i < vocab_size; ++i) { + probs[i] /= sum; + } + + uint32_t actual_seed = (random_seed == 0) ? std::random_device{}() : random_seed; + std::mt19937 gen(actual_seed); + std::uniform_real_distribution dist(0.0f, 1.0f); + float sample = dist(gen); + + float cumulative = 0.0f; + for (size_t i = 0; i < vocab_size; ++i) { + cumulative += probs[i]; + if (cumulative >= sample) { + output[0] = static_cast(i); + return; + } + } + + for (size_t i = vocab_size; i > 0; --i) { + if (probs[i-1] > 0.0f) { + output[0] = static_cast(i-1); + return; + } + } + + output[0] = 0; +} diff --git a/cactus-transpile/kernel/kernel_quants.cpp b/cactus-transpile/kernel/kernel_quants.cpp new file mode 100644 index 000000000..d3553603b --- /dev/null +++ b/cactus-transpile/kernel/kernel_quants.cpp @@ -0,0 +1,312 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include +#include + +void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale) { + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst, scale](size_t start, size_t end) { + const size_t simd_end = start + ((end - start) / 16) * 16; + float32x4_t scale_vec = vdupq_n_f32(scale); + + for (size_t i = start; i < simd_end; i += 16) { + int8x16_t input = vld1q_s8(&src[i]); + + int16x8_t low = vmovl_s8(vget_low_s8(input)); + int16x8_t high = vmovl_s8(vget_high_s8(input)); + + int32x4_t low_low = vmovl_s16(vget_low_s16(low)); + int32x4_t low_high = vmovl_s16(vget_high_s16(low)); + int32x4_t high_low = vmovl_s16(vget_low_s16(high)); + int32x4_t high_high = vmovl_s16(vget_high_s16(high)); + + float32x4_t f_low_low = vcvtq_f32_s32(low_low); + float32x4_t f_low_high = vcvtq_f32_s32(low_high); + float32x4_t f_high_low = vcvtq_f32_s32(high_low); + float32x4_t f_high_high = vcvtq_f32_s32(high_high); + + vst1q_f32(&dst[i], vmulq_f32(f_low_low, scale_vec)); + vst1q_f32(&dst[i + 4], vmulq_f32(f_low_high, scale_vec)); + vst1q_f32(&dst[i + 8], vmulq_f32(f_high_low, scale_vec)); + vst1q_f32(&dst[i + 12], vmulq_f32(f_high_high, scale_vec)); + } + + for (size_t i = simd_end; i < end; ++i) { + dst[i] = static_cast(src[i]) * scale; + } + }); +} + +void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale) { + const float inv_scale = 1.0f / scale; + + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst, inv_scale](size_t start, size_t end) { + const size_t simd_end = start + ((end - start) / 16) * 16; + float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); + float32x4_t min_vec = vdupq_n_f32(-128.0f); + float32x4_t max_vec = vdupq_n_f32(127.0f); + + for (size_t i = start; i < simd_end; i += 16) { + float32x4_t input_0 = vld1q_f32(&src[i]); + float32x4_t input_1 = vld1q_f32(&src[i + 4]); + float32x4_t input_2 = vld1q_f32(&src[i + 8]); + float32x4_t input_3 = vld1q_f32(&src[i + 12]); + + float32x4_t scaled_0 = vmulq_f32(input_0, inv_scale_vec); + float32x4_t scaled_1 = vmulq_f32(input_1, inv_scale_vec); + float32x4_t scaled_2 = vmulq_f32(input_2, inv_scale_vec); + float32x4_t scaled_3 = vmulq_f32(input_3, inv_scale_vec); + + scaled_0 = vmaxq_f32(vminq_f32(scaled_0, max_vec), min_vec); + scaled_1 = vmaxq_f32(vminq_f32(scaled_1, max_vec), min_vec); + scaled_2 = vmaxq_f32(vminq_f32(scaled_2, max_vec), min_vec); + scaled_3 = vmaxq_f32(vminq_f32(scaled_3, max_vec), min_vec); + + int32x4_t int_0 = vcvtnq_s32_f32(scaled_0); + int32x4_t int_1 = vcvtnq_s32_f32(scaled_1); + int32x4_t int_2 = vcvtnq_s32_f32(scaled_2); + int32x4_t int_3 = vcvtnq_s32_f32(scaled_3); + + int16x8_t int16_low = vcombine_s16(vqmovn_s32(int_0), vqmovn_s32(int_1)); + int16x8_t int16_high = vcombine_s16(vqmovn_s32(int_2), vqmovn_s32(int_3)); + + int8x16_t result = vcombine_s8(vqmovn_s16(int16_low), vqmovn_s16(int16_high)); + vst1q_s8(&dst[i], result); + } + + for (size_t i = simd_end; i < end; ++i) { + float quantized = src[i] * inv_scale; + dst[i] = static_cast(std::round(std::max(-128.0f, std::min(127.0f, quantized)))); + } + }); +} + +void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count) { + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst](size_t start, size_t end) { + const size_t simd_end = start + ((end - start) / 8) * 8; + + for (size_t i = start; i < simd_end; i += 8) { + float16x8_t input = vld1q_f16(&src[i]); + + float32x4_t output_low = vcvt_f32_f16(vget_low_f16(input)); + float32x4_t output_high = vcvt_f32_f16(vget_high_f16(input)); + + vst1q_f32(&dst[i], output_low); + vst1q_f32(&dst[i + 4], output_high); + } + + for (size_t i = simd_end; i < end; ++i) { + dst[i] = static_cast(src[i]); + } + }); +} + +void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count) { + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst](size_t start, size_t end) { + const size_t simd_end = start + ((end - start) / 8) * 8; + + for (size_t i = start; i < simd_end; i += 8) { + float32x4_t input_low = vld1q_f32(&src[i]); + float32x4_t input_high = vld1q_f32(&src[i + 4]); + + float16x4_t output_low = vcvt_f16_f32(input_low); + float16x4_t output_high = vcvt_f16_f32(input_high); + + float16x8_t output = vcombine_f16(output_low, output_high); + vst1q_f16(&dst[i], output); + } + + for (size_t i = simd_end; i < end; ++i) { + dst[i] = static_cast<__fp16>(src[i]); + } + }); +} + +void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale) { + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst, scale](size_t start, size_t end) { + const size_t simd_end = start + ((end - start) / 8) * 8; + float32x4_t scale_vec = vdupq_n_f32(scale); + + for (size_t i = start; i < simd_end; i += 8) { + int8x8_t input = vld1_s8(&src[i]); + + int16x8_t int16 = vmovl_s8(input); + int32x4_t int32_low = vmovl_s16(vget_low_s16(int16)); + int32x4_t int32_high = vmovl_s16(vget_high_s16(int16)); + + float32x4_t float_low = vcvtq_f32_s32(int32_low); + float32x4_t float_high = vcvtq_f32_s32(int32_high); + + float_low = vmulq_f32(float_low, scale_vec); + float_high = vmulq_f32(float_high, scale_vec); + + float16x8_t output = vcombine_f16(vcvt_f16_f32(float_low), vcvt_f16_f32(float_high)); + vst1q_f16(&dst[i], output); + } + + for (size_t i = simd_end; i < end; ++i) { + dst[i] = static_cast<__fp16>(static_cast(src[i]) * scale); + } + }); +} + +void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale) { + const float inv_scale = 1.0f / scale; + + CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst, inv_scale](size_t start, size_t end) { + const size_t simd_end = start + ((end - start) / 8) * 8; + float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); + float32x4_t min_vec = vdupq_n_f32(-128.0f); + float32x4_t max_vec = vdupq_n_f32(127.0f); + + for (size_t i = start; i < simd_end; i += 8) { + float16x8_t input = vld1q_f16(&src[i]); + + float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input)); + float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input)); + + float32x4_t scaled_low = vmulq_f32(input_low, inv_scale_vec); + float32x4_t scaled_high = vmulq_f32(input_high, inv_scale_vec); + + scaled_low = vmaxq_f32(vminq_f32(scaled_low, max_vec), min_vec); + scaled_high = vmaxq_f32(vminq_f32(scaled_high, max_vec), min_vec); + + int32x4_t int_low = vcvtnq_s32_f32(scaled_low); + int32x4_t int_high = vcvtnq_s32_f32(scaled_high); + + int16x8_t int16_combined = vcombine_s16(vqmovn_s32(int_low), vqmovn_s32(int_high)); + int8x8_t result = vqmovn_s16(int16_combined); + + vst1_s8(&dst[i], result); + } + + for (size_t i = simd_end; i < end; ++i) { + float quantized = static_cast(src[i]) * inv_scale; + dst[i] = static_cast(std::round(std::max(-128.0f, std::min(127.0f, quantized)))); + } + }); +} + +float cactus_fp16_max_abs(const __fp16* src, size_t count) { + float32x4_t abs_max_vec = vdupq_n_f32(0.0f); + const size_t simd_end = (count / 8) * 8; + + for (size_t i = 0; i < simd_end; i += 8) { + float16x8_t input = vld1q_f16(&src[i]); + + float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input)); + float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input)); + + float32x4_t abs_low = vabsq_f32(input_low); + float32x4_t abs_high = vabsq_f32(input_high); + + abs_max_vec = vmaxq_f32(abs_max_vec, abs_low); + abs_max_vec = vmaxq_f32(abs_max_vec, abs_high); + } + + float max_abs = vmaxvq_f32(abs_max_vec); + + for (size_t i = simd_end; i < count; ++i) { + float abs_val = std::abs(static_cast(src[i])); + max_abs = std::max(max_abs, abs_val); + } + + return max_abs; +} + +static inline float quantize_group_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count) { + + float32x4_t max_vec = vdupq_n_f32(0.0f); + size_t k = 0; + + for (; k + 8 <= count; k += 8) { + float16x8_t vals = vld1q_f16(src + k); + max_vec = vmaxq_f32(max_vec, vabsq_f32(vcvt_f32_f16(vget_low_f16(vals)))); + max_vec = vmaxq_f32(max_vec, vabsq_f32(vcvt_f32_f16(vget_high_f16(vals)))); + } + + float max_abs = vmaxvq_f32(max_vec); + for (; k < count; k++) { + float val = std::abs(static_cast(src[k])); + if (val > max_abs) max_abs = val; + } + + float scale = max_abs / 127.0f; + if (scale < 1e-10f) scale = 1e-10f; + float inv_scale = 1.0f / scale; + float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); + + k = 0; + for (; k + 8 <= count; k += 8) { + float16x8_t vals = vld1q_f16(src + k); + int32x4_t i0 = vcvtnq_s32_f32(vmulq_f32(vcvt_f32_f16(vget_low_f16(vals)), inv_scale_vec)); + int32x4_t i1 = vcvtnq_s32_f32(vmulq_f32(vcvt_f32_f16(vget_high_f16(vals)), inv_scale_vec)); + int16x4_t s0 = vqmovn_s32(i0); + int16x4_t s1 = vqmovn_s32(i1); + int8x8_t result = vqmovn_s16(vcombine_s16(s0, s1)); + vst1_s8(dst + k, result); + } + + for (; k < count; k++) { + float val = static_cast(src[k]) * inv_scale; + int32_t q = static_cast(roundf(val)); + q = std::max(-128, std::min(127, q)); + dst[k] = static_cast(q); + } + + return scale; +} + +void cactus_quantize_kv_fp16_to_int8( + const __fp16* src, + int8_t* dst, + float* scales, + size_t seq_len, size_t kv_heads, size_t head_dim, + size_t group_size +) { + const size_t num_groups = (head_dim + group_size - 1) / group_size; + + CactusThreading::parallel_for(seq_len * kv_heads, CactusThreading::Thresholds::ELEMENT_WISE, + [src, dst, scales, head_dim, group_size, num_groups](size_t start, size_t end) { + for (size_t idx = start; idx < end; idx++) { + const __fp16* head_src = src + idx * head_dim; + int8_t* head_dst = dst + idx * head_dim; + float* head_scales = scales + idx * num_groups; + + for (size_t g = 0; g < num_groups; g++) { + size_t group_start = g * group_size; + size_t group_count = std::min(group_size, head_dim - group_start); + head_scales[g] = quantize_group_fp16_to_int8( + head_src + group_start, + head_dst + group_start, + group_count + ); + } + } + }); +} + +void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count) { + assert (unpacked_count % 32 == 0 && "Unpacked count must be a multiple of 32 for int4 unpacking"); + const size_t num_groups = unpacked_count / 32; + + CactusThreading::parallel_for(num_groups, CactusThreading::Thresholds::ELEMENT_WISE, + [packed, unpacked](size_t start, size_t end) { + for (size_t group = start; group < end; ++group) { + const uint8_t* src = &packed[group * 16]; + int8_t* dst = &unpacked[group * 32]; + + int8x16_t high_signed, low_signed; + unpack_int4_as_int8x16x2(src, high_signed, low_signed); + vst1q_s8(dst, low_signed); + vst1q_s8(dst + 16, high_signed); + } + }); +} diff --git a/cactus-transpile/kernel/kernel_reduce.cpp b/cactus-transpile/kernel/kernel_reduce.cpp new file mode 100644 index 000000000..5cea15cbf --- /dev/null +++ b/cactus-transpile/kernel/kernel_reduce.cpp @@ -0,0 +1,430 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include + +template +static void axis_reduce_f32_impl(const __fp16* input, __fp16* output, + size_t outer_size, size_t axis_size, size_t inner_size, + FinalizeFn finalize) { + CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t outer, size_t inner) { + constexpr size_t W = SIMD_F16_WIDTH; + const size_t vec_axis = simd_align(axis_size); + + float32x4_t sum_lo = vdupq_n_f32(0.0f); + float32x4_t sum_hi = vdupq_n_f32(0.0f); + + for (size_t a = 0; a < vec_axis; a += W) { + __fp16 values[W]; + for (size_t j = 0; j < W; j++) { + values[j] = input[outer * axis_size * inner_size + (a + j) * inner_size + inner]; + } + float32x4_t lo, hi; + f16x8_split_f32(vld1q_f16(values), lo, hi); + sum_lo = vaddq_f32(sum_lo, lo); + sum_hi = vaddq_f32(sum_hi, hi); + } + + float total = vaddvq_f32(vaddq_f32(sum_lo, sum_hi)); + + for (size_t a = vec_axis; a < axis_size; a++) { + total += static_cast(input[outer * axis_size * inner_size + a * inner_size + inner]); + } + + output[outer * inner_size + inner] = static_cast<__fp16>(finalize(total, axis_size)); + }); +} + +template +static void axis_reduce_f32_impl(const float* input, float* output, + size_t outer_size, size_t axis_size, size_t inner_size, + FinalizeFn finalize) { + CactusThreading::parallel_for_2d( + outer_size, + inner_size, + CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t outer, size_t inner) { + float total = 0.0f; + for (size_t a = 0; a < axis_size; ++a) { + total += input[outer * axis_size * inner_size + a * inner_size + inner]; + } + output[outer * inner_size + inner] = finalize(total, axis_size); + } + ); +} + +template +static void axis_reduce_f16_impl(const __fp16* input, __fp16* output, + size_t outer_size, size_t axis_size, size_t inner_size, + float16x8_t init_vec, __fp16 init_scalar, + SimdReduceOp simd_reduce, ScalarReduceOp scalar_reduce) { + CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t outer, size_t inner) { + constexpr size_t W = SIMD_F16_WIDTH; + const size_t vec_axis = simd_align(axis_size); + float16x8_t acc = init_vec; + + for (size_t a = 0; a < vec_axis; a += W) { + __fp16 values[W]; + for (size_t j = 0; j < W; j++) { + values[j] = input[outer * axis_size * inner_size + (a + j) * inner_size + inner]; + } + acc = simd_reduce(acc, vld1q_f16(values)); + } + + __fp16 result = init_scalar; + __fp16 arr[8]; + vst1q_f16(arr, acc); + for (int j = 0; j < 8; j++) { + result = scalar_reduce(result, arr[j]); + } + + for (size_t a = vec_axis; a < axis_size; a++) { + result = scalar_reduce(result, input[outer * axis_size * inner_size + a * inner_size + inner]); + } + + output[outer * inner_size + inner] = result; + }); +} + +template +static void axis_reduce_f32_impl(const float* input, float* output, + size_t outer_size, size_t axis_size, size_t inner_size, + float init_scalar, + ScalarReduceOp scalar_reduce) { + CactusThreading::parallel_for_2d( + outer_size, + inner_size, + CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t outer, size_t inner) { + float result = init_scalar; + for (size_t a = 0; a < axis_size; ++a) { + result = scalar_reduce(result, input[outer * axis_size * inner_size + a * inner_size + inner]); + } + output[outer * inner_size + inner] = result; + } + ); +} + +double cactus_sum_all_f16(const __fp16* data, size_t num_elements) { + return CactusThreading::parallel_reduce( + num_elements, CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) -> double { + const size_t vec_end = start + simd_align(end - start); + float32x4_t sum_lo = vdupq_n_f32(0.0f); + float32x4_t sum_hi = vdupq_n_f32(0.0f); + + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + float32x4_t lo, hi; + f16x8_split_f32(vld1q_f16(&data[i]), lo, hi); + sum_lo = vaddq_f32(sum_lo, lo); + sum_hi = vaddq_f32(sum_hi, hi); + } + + double s = static_cast(vaddvq_f32(vaddq_f32(sum_lo, sum_hi))); + for (size_t i = vec_end; i < end; ++i) s += static_cast(data[i]); + return s; + }, + 0.0, [](double a, double b) { return a + b; } + ); +} + +void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { + axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, + [](float total, size_t) { return total; }); +} + +double cactus_mean_all_f16(const __fp16* data, size_t num_elements) { + return cactus_sum_all_f16(data, num_elements) / static_cast(num_elements); +} + +void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { + axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, + [](float total, size_t axis_size) { return total / static_cast(axis_size); }); +} + +double cactus_sum_all_f32(const float* data, size_t num_elements) { + return CactusThreading::parallel_reduce( + num_elements, + CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) -> double { + double sum = 0.0; + for (size_t i = start; i < end; ++i) { + sum += static_cast(data[i]); + } + return sum; + }, + 0.0, + [](double a, double b) { return a + b; } + ); +} + +void cactus_sum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { + axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, + [](float total, size_t) { return total; }); +} + +double cactus_mean_all_f32(const float* data, size_t num_elements) { + return cactus_sum_all_f32(data, num_elements) / static_cast(num_elements); +} + +void cactus_mean_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { + axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, + [](float total, size_t axis_size) { return total / static_cast(axis_size); }); +} + +struct VarianceState { + double sum; + double sum_sq; + VarianceState() : sum(0.0), sum_sq(0.0) {} + VarianceState(double s, double sq) : sum(s), sum_sq(sq) {} +}; + +double cactus_variance_all_f16(const __fp16* data, size_t num_elements) { + VarianceState result = CactusThreading::parallel_reduce( + num_elements, CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) -> VarianceState { + const size_t vec_end = start + simd_align(end - start); + + float32x4_t sum_lo = vdupq_n_f32(0.0f), sum_hi = vdupq_n_f32(0.0f); + float32x4_t sq_lo = vdupq_n_f32(0.0f), sq_hi = vdupq_n_f32(0.0f); + + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + float32x4_t lo, hi; + f16x8_split_f32(vld1q_f16(&data[i]), lo, hi); + sum_lo = vaddq_f32(sum_lo, lo); + sum_hi = vaddq_f32(sum_hi, hi); + sq_lo = vfmaq_f32(sq_lo, lo, lo); + sq_hi = vfmaq_f32(sq_hi, hi, hi); + } + + double sum = static_cast(vaddvq_f32(vaddq_f32(sum_lo, sum_hi))); + double sum_sq = static_cast(vaddvq_f32(vaddq_f32(sq_lo, sq_hi))); + + for (size_t i = vec_end; i < end; ++i) { + double x = static_cast(data[i]); + sum += x; + sum_sq += x * x; + } + return VarianceState(sum, sum_sq); + }, + VarianceState(), + [](const VarianceState& a, const VarianceState& b) { + return VarianceState(a.sum + b.sum, a.sum_sq + b.sum_sq); + } + ); + + double mean = result.sum / static_cast(num_elements); + double mean_sq = result.sum_sq / static_cast(num_elements); + return mean_sq - mean * mean; +} + +void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { + CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t outer, size_t inner) { + float sum = 0.0f, sum_sq = 0.0f; + for (size_t a = 0; a < axis_size; a++) { + float x = static_cast(input[outer * axis_size * inner_size + a * inner_size + inner]); + sum += x; + sum_sq += x * x; + } + float mean = sum / static_cast(axis_size); + float mean_sq = sum_sq / static_cast(axis_size); + output[outer * inner_size + inner] = static_cast<__fp16>(mean_sq - mean * mean); + }); +} + +double cactus_variance_all_f32(const float* data, size_t num_elements) { + VarianceState result = CactusThreading::parallel_reduce( + num_elements, + CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) -> VarianceState { + double sum = 0.0; + double sum_sq = 0.0; + for (size_t i = start; i < end; ++i) { + double x = static_cast(data[i]); + sum += x; + sum_sq += x * x; + } + return VarianceState(sum, sum_sq); + }, + VarianceState(), + [](const VarianceState& a, const VarianceState& b) { + return VarianceState(a.sum + b.sum, a.sum_sq + b.sum_sq); + } + ); + + double mean = result.sum / static_cast(num_elements); + double mean_sq = result.sum_sq / static_cast(num_elements); + return mean_sq - mean * mean; +} + +void cactus_variance_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { + CactusThreading::parallel_for_2d( + outer_size, + inner_size, + CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t outer, size_t inner) { + float sum = 0.0f; + float sum_sq = 0.0f; + for (size_t a = 0; a < axis_size; ++a) { + float x = input[outer * axis_size * inner_size + a * inner_size + inner]; + sum += x; + sum_sq += x * x; + } + float mean = sum / static_cast(axis_size); + float mean_sq = sum_sq / static_cast(axis_size); + output[outer * inner_size + inner] = mean_sq - mean * mean; + } + ); +} + +__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements) { + return CactusThreading::parallel_reduce( + num_elements, CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) -> __fp16 { + const size_t vec_end = start + simd_align(end - start); + float16x8_t acc = vdupq_n_f16(static_cast<__fp16>(65504.0f)); + + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + acc = vminq_f16(acc, vld1q_f16(&data[i])); + } + + __fp16 result = static_cast<__fp16>(65504.0f); + __fp16 arr[8]; + vst1q_f16(arr, acc); + for (int j = 0; j < 8; j++) result = std::min(result, arr[j]); + for (size_t i = vec_end; i < end; ++i) result = std::min(result, data[i]); + return result; + }, + static_cast<__fp16>(65504.0f), + [](__fp16 a, __fp16 b) { return std::min(a, b); } + ); +} + +void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { + axis_reduce_f16_impl(input, output, outer_size, axis_size, inner_size, + vdupq_n_f16(static_cast<__fp16>(65504.0f)), static_cast<__fp16>(65504.0f), + [](float16x8_t a, float16x8_t b) { return vminq_f16(a, b); }, + [](__fp16 a, __fp16 b) { return std::min(a, b); }); +} + +float cactus_min_all_f32(const float* data, size_t num_elements) { + return CactusThreading::parallel_reduce( + num_elements, + CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) -> float { + float result = std::numeric_limits::infinity(); + for (size_t i = start; i < end; ++i) { + result = std::min(result, data[i]); + } + return result; + }, + std::numeric_limits::infinity(), + [](float a, float b) { return std::min(a, b); } + ); +} + +void cactus_min_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { + axis_reduce_f32_impl( + input, + output, + outer_size, + axis_size, + inner_size, + std::numeric_limits::infinity(), + [](float a, float b) { return std::min(a, b); } + ); +} + +__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements) { + return CactusThreading::parallel_reduce( + num_elements, CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) -> __fp16 { + const size_t vec_end = start + simd_align(end - start); + float16x8_t acc = vdupq_n_f16(static_cast<__fp16>(-65504.0f)); + + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + acc = vmaxq_f16(acc, vld1q_f16(&data[i])); + } + + __fp16 result = static_cast<__fp16>(-65504.0f); + __fp16 arr[8]; + vst1q_f16(arr, acc); + for (int j = 0; j < 8; j++) result = std::max(result, arr[j]); + for (size_t i = vec_end; i < end; ++i) result = std::max(result, data[i]); + return result; + }, + static_cast<__fp16>(-65504.0f), + [](__fp16 a, __fp16 b) { return std::max(a, b); } + ); +} + +void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { + axis_reduce_f16_impl(input, output, outer_size, axis_size, inner_size, + vdupq_n_f16(static_cast<__fp16>(-65504.0f)), static_cast<__fp16>(-65504.0f), + [](float16x8_t a, float16x8_t b) { return vmaxq_f16(a, b); }, + [](__fp16 a, __fp16 b) { return std::max(a, b); }); +} + +float cactus_max_all_f32(const float* data, size_t num_elements) { + return CactusThreading::parallel_reduce( + num_elements, + CactusThreading::Thresholds::ALL_REDUCE, + [&](size_t start, size_t end) -> float { + float result = -std::numeric_limits::infinity(); + for (size_t i = start; i < end; ++i) { + result = std::max(result, data[i]); + } + return result; + }, + -std::numeric_limits::infinity(), + [](float a, float b) { return std::max(a, b); } + ); +} + +void cactus_max_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { + axis_reduce_f32_impl( + input, + output, + outer_size, + axis_size, + inner_size, + -std::numeric_limits::infinity(), + [](float a, float b) { return std::max(a, b); } + ); +} + +void cactus_cumsum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { + CactusThreading::parallel_for_2d( + outer_size, + inner_size, + CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t outer, size_t inner) { + float running = 0.0f; + for (size_t a = 0; a < axis_size; ++a) { + const size_t offset = outer * axis_size * inner_size + a * inner_size + inner; + running += static_cast(input[offset]); + output[offset] = static_cast<__fp16>(running); + } + } + ); +} + +void cactus_cumsum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { + CactusThreading::parallel_for_2d( + outer_size, + inner_size, + CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t outer, size_t inner) { + float running = 0.0f; + for (size_t a = 0; a < axis_size; ++a) { + const size_t offset = outer * axis_size * inner_size + a * inner_size + inner; + running += input[offset]; + output[offset] = running; + } + } + ); +} diff --git a/cactus-transpile/kernel/kernel_scalar.cpp b/cactus-transpile/kernel/kernel_scalar.cpp new file mode 100644 index 000000000..15e9f6dea --- /dev/null +++ b/cactus-transpile/kernel/kernel_scalar.cpp @@ -0,0 +1,119 @@ +#include "kernel.h" +#include "kernel_utils.h" +#include +#include + +void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type) { + const __fp16 scalar_f16 = static_cast<__fp16>(scalar_value); + const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; + + switch (op_type) { + case ScalarOpType::ADD: { + const float16x8_t sv = vdupq_n_f16(scalar_f16); + elementwise_op_f16(input, output, num_elements, use_streaming, + CactusThreading::Thresholds::SCALAR_BASIC, + [sv](float16x8_t v) { return vaddq_f16(v, sv); }, + [scalar_f16](__fp16 v) { return static_cast<__fp16>(v + scalar_f16); }); + break; + } + case ScalarOpType::SUBTRACT: { + const float16x8_t sv = vdupq_n_f16(scalar_f16); + elementwise_op_f16(input, output, num_elements, use_streaming, + CactusThreading::Thresholds::SCALAR_BASIC, + [sv](float16x8_t v) { return vsubq_f16(v, sv); }, + [scalar_f16](__fp16 v) { return static_cast<__fp16>(v - scalar_f16); }); + break; + } + case ScalarOpType::MULTIPLY: { + const float16x8_t sv = vdupq_n_f16(scalar_f16); + elementwise_op_f16(input, output, num_elements, use_streaming, + CactusThreading::Thresholds::SCALAR_BASIC, + [sv](float16x8_t v) { return vmulq_f16(v, sv); }, + [scalar_f16](__fp16 v) { return static_cast<__fp16>(v * scalar_f16); }); + break; + } + case ScalarOpType::DIVIDE: { + const float16x8_t sv = vdupq_n_f16(scalar_f16); + elementwise_op_f16(input, output, num_elements, use_streaming, + CactusThreading::Thresholds::SCALAR_BASIC, + [sv](float16x8_t v) { return vdivq_f16(v, sv); }, + [scalar_f16](__fp16 v) { return static_cast<__fp16>(v / scalar_f16); }); + break; + } + case ScalarOpType::NOT_EQUAL: + CactusThreading::parallel_for(num_elements, + CactusThreading::Thresholds::SCALAR_BASIC, + [&](size_t start, size_t end) { + for (size_t i = start; i < end; ++i) { + output[i] = static_cast<__fp16>( + static_cast(input[i]) != scalar_value ? 1.0f : 0.0f); + } + }); + break; + case ScalarOpType::ABS: + elementwise_op_f16(input, output, num_elements, use_streaming, + CactusThreading::Thresholds::SCALAR_BASIC, + [](float16x8_t v) { return vabsq_f16(v); }, + [](__fp16 v) { return static_cast<__fp16>(std::abs(static_cast(v))); }); + break; + + case ScalarOpType::EXP: + elementwise_op_f16(input, output, num_elements, use_streaming, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [](float16x8_t v) { return apply_f32_op_on_f16x8(v, fast_exp_f32x4); }, + [](__fp16 v) { return static_cast<__fp16>(std::exp(static_cast(v))); }, 2); + break; + + case ScalarOpType::SQRT: + elementwise_op_f16(input, output, num_elements, use_streaming, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [](float16x8_t v) { return apply_f32_op_on_f16x8(v, vsqrtq_f32); }, + [](__fp16 v) { return static_cast<__fp16>(std::sqrt(static_cast(v))); }, 2); + break; + + case ScalarOpType::POW: + CactusThreading::parallel_for(num_elements, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start, size_t end) { + for (size_t i = start; i < end; ++i) { + output[i] = static_cast<__fp16>( + std::pow(static_cast(input[i]), static_cast(scalar_f16))); + } + }); + break; + + case ScalarOpType::COS: + case ScalarOpType::SIN: + CactusThreading::parallel_for(num_elements, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start, size_t end) { + for (size_t i = start; i < end; ++i) { + float val = static_cast(input[i]); + float result = (op_type == ScalarOpType::COS) ? std::cos(val) : std::sin(val); + output[i] = static_cast<__fp16>(result); + } + }); + break; + + case ScalarOpType::LOG: + CactusThreading::parallel_for(num_elements, + CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [&](size_t start, size_t end) { + for (size_t i = start; i < end; ++i) { + output[i] = static_cast<__fp16>(std::log(static_cast(input[i]))); + } + }); + break; + + case ScalarOpType::FLOOR_DIVIDE: + CactusThreading::parallel_for(num_elements, + CactusThreading::Thresholds::SCALAR_BASIC, + [&](size_t start, size_t end) { + for (size_t i = start; i < end; ++i) { + output[i] = static_cast<__fp16>( + std::floor(static_cast(input[i]) / scalar_value)); + } + }); + break; + } +} diff --git a/cactus-transpile/kernel/kernel_sme2.cpp b/cactus-transpile/kernel/kernel_sme2.cpp new file mode 100644 index 000000000..9ce5b410b --- /dev/null +++ b/cactus-transpile/kernel/kernel_sme2.cpp @@ -0,0 +1,616 @@ +#include +#include +#ifndef __ARM_FEATURE_SME2 +#error "kernel_sme2.cpp must be compiled with SME2 enabled (e.g. -march=armv9.2-a+sme2)" +#endif + +#include "kernel.h" +#include "kernel_utils.h" +#include +#include +#include +#include + +#if defined(__clang__) +#define CACTUS_UNROLL4 _Pragma("clang loop unroll_count(4)") +#else +#define CACTUS_UNROLL4 +#endif + +static inline void cactus_pack_a_f16_row_block( + const __fp16* __restrict a, + __fp16* __restrict a_packed, + size_t K, + size_t rb, + size_t row0, + size_t active_r, + size_t tile_rows, + size_t tile_pairs, + size_t k_pairs +) { + const size_t block_stride = k_pairs * tile_pairs; + const bool even_k = ((K & 1u) == 0); + for (size_t kp = 0; kp < k_pairs; ++kp) { + const size_t k0 = kp * 2; + const size_t k1 = k0 + 1; + __fp16* dst = a_packed + rb * block_stride + kp * tile_pairs; + const __fp16* src_col = a + row0 * K + k0; + + if (even_k) { + CACTUS_UNROLL4 + for (size_t r = 0; r < active_r; ++r) { + dst[2 * r] = src_col[0]; + dst[2 * r + 1] = src_col[1]; + src_col += K; + } + } else { + const bool has_k1 = (k1 < K); + CACTUS_UNROLL4 + for (size_t r = 0; r < active_r; ++r) { + dst[2 * r] = src_col[0]; + dst[2 * r + 1] = has_k1 ? src_col[1] : static_cast<__fp16>(0); + src_col += K; + } + } + CACTUS_UNROLL4 + for (size_t r = active_r; r < tile_rows; ++r) { + dst[2 * r] = static_cast<__fp16>(0); + dst[2 * r + 1] = static_cast<__fp16>(0); + } + } +} + +static void cactus_pack_b_f16_from_bt( + const __fp16* __restrict b_transposed, + __fp16* __restrict b_packed, + size_t K, + size_t N, + size_t tile_cols, + size_t tile_pairs +) { + const size_t k_pairs = (K + 1) / 2; + const size_t col_blocks = (N + tile_cols - 1) / tile_cols; + const size_t full_col_blocks = N / tile_cols; + const size_t cb4_tiles = (full_col_blocks / 4) * 4; + const size_t cb2_tiles = ((full_col_blocks - cb4_tiles) / 2) * 2; + const size_t cb1_tiles = col_blocks - cb4_tiles - cb2_tiles; + const size_t cb4_groups = cb4_tiles / 4; + const size_t cb2_groups = cb2_tiles / 2; + + const size_t off_cb4 = 0; + const size_t off_cb2 = off_cb4 + cb4_groups * k_pairs * 4 * tile_pairs; + const size_t off_cb1 = off_cb2 + cb2_groups * k_pairs * 2 * tile_pairs; + + const bool even_k = ((K & 1u) == 0); + + CactusThreading::parallel_for(cb4_groups * k_pairs, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [=](size_t start, size_t end) { + for (size_t idx = start; idx < end; ++idx) { + const size_t g4 = idx / k_pairs; + const size_t kp = idx % k_pairs; + const size_t col0 = g4 * 4 * tile_cols; + + const size_t k0 = kp * 2; + const size_t k1 = k0 + 1; + __fp16* dst = b_packed + off_cb4 + (g4 * k_pairs + kp) * (4 * tile_pairs); + + for (size_t t = 0; t < 4; ++t) { + __fp16* dst_t = dst + t * tile_pairs; + const size_t col_t = col0 + t * tile_cols; + const __fp16* src_col = b_transposed + col_t * K + k0; + if (even_k) { + CACTUS_UNROLL4 + for (size_t c = 0; c < tile_cols; ++c) { + dst_t[2 * c] = src_col[0]; + dst_t[2 * c + 1] = src_col[1]; + src_col += K; + } + } else { + const bool has_k1 = (k1 < K); + CACTUS_UNROLL4 + for (size_t c = 0; c < tile_cols; ++c) { + dst_t[2 * c] = src_col[0]; + dst_t[2 * c + 1] = has_k1 ? src_col[1] : static_cast<__fp16>(0); + src_col += K; + } + } + } + } + }); + + CactusThreading::parallel_for(cb2_groups * k_pairs, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [=](size_t start, size_t end) { + for (size_t idx = start; idx < end; ++idx) { + const size_t g2 = idx / k_pairs; + const size_t kp = idx % k_pairs; + const size_t col0 = cb4_tiles * tile_cols + g2 * 2 * tile_cols; + + const size_t k0 = kp * 2; + const size_t k1 = k0 + 1; + __fp16* dst = b_packed + off_cb2 + (g2 * k_pairs + kp) * (2 * tile_pairs); + + for (size_t t = 0; t < 2; ++t) { + __fp16* dst_t = dst + t * tile_pairs; + const size_t col_t = col0 + t * tile_cols; + const __fp16* src_col = b_transposed + col_t * K + k0; + if (even_k) { + CACTUS_UNROLL4 + for (size_t c = 0; c < tile_cols; ++c) { + dst_t[2 * c] = src_col[0]; + dst_t[2 * c + 1] = src_col[1]; + src_col += K; + } + } else { + const bool has_k1 = (k1 < K); + CACTUS_UNROLL4 + for (size_t c = 0; c < tile_cols; ++c) { + dst_t[2 * c] = src_col[0]; + dst_t[2 * c + 1] = has_k1 ? src_col[1] : static_cast<__fp16>(0); + src_col += K; + } + } + } + } + }); + + CactusThreading::parallel_for(cb1_tiles * k_pairs, CactusThreading::Thresholds::SCALAR_EXPENSIVE, + [=](size_t start, size_t end) { + for (size_t idx = start; idx < end; ++idx) { + const size_t g1 = idx / k_pairs; + const size_t kp = idx % k_pairs; + const size_t cb = cb4_tiles + cb2_tiles + g1; + const size_t col0 = cb * tile_cols; + const size_t active_c = (col0 < N) ? std::min(tile_cols, N - col0) : 0; + + const size_t k0 = kp * 2; + const size_t k1 = k0 + 1; + __fp16* dst = b_packed + off_cb1 + (g1 * k_pairs + kp) * tile_pairs; + const __fp16* src_col = b_transposed + col0 * K + k0; + if (even_k) { + CACTUS_UNROLL4 + for (size_t c = 0; c < active_c; ++c) { + dst[2 * c] = src_col[0]; + dst[2 * c + 1] = src_col[1]; + src_col += K; + } + } else { + const bool has_k1 = (k1 < K); + CACTUS_UNROLL4 + for (size_t c = 0; c < active_c; ++c) { + dst[2 * c] = src_col[0]; + dst[2 * c + 1] = has_k1 ? src_col[1] : static_cast<__fp16>(0); + src_col += K; + } + } + CACTUS_UNROLL4 + for (size_t c = active_c; c < tile_cols; ++c) { + dst[2 * c] = static_cast<__fp16>(0); + dst[2 * c + 1] = static_cast<__fp16>(0); + } + } + }); +} + +static void cactus_matmul_f16_sme2_worker( + const __fp16* a_packed, + const __fp16* b_packed, + __fp16* c, + size_t M, + size_t K, + size_t N, + size_t start_row, + size_t end_row, + size_t tile_rows, + size_t tile_pairs +) __arm_streaming __arm_inout("za") { + if (start_row >= end_row) return; + (void)M; + + const size_t k_pairs = (K + 1) / 2; + const size_t col_blocks = (N + tile_rows - 1) / tile_rows; + const size_t full_col_blocks = N / tile_rows; + const size_t cb4_tiles = (full_col_blocks / 4) * 4; + const size_t cb2_tiles = ((full_col_blocks - cb4_tiles) / 2) * 2; + const size_t cb4_groups = cb4_tiles / 4; + const size_t cb2_groups = cb2_tiles / 2; + const size_t cb1_off = cb4_groups * k_pairs * 4 * tile_pairs + cb2_groups * k_pairs * 2 * tile_pairs; + const size_t a_row_block_stride = k_pairs * tile_pairs; + + const svcount_t pNh_full_c = svptrue_c16(); + const svbool_t pNh_full = svptrue_b16(); + const svbool_t pN32_full = svwhilelt_b32(static_cast(0), static_cast(tile_rows)); + const svcount_t pOut16_full_c = svwhilelt_c16(static_cast(0), static_cast(tile_rows), 2); + const svfloat32_t z0_f32 = svdup_n_f32(0.0f); +#define CACTUS_STORE_FULL_TILE_ROW(dst_idx, out32_vec) \ + do { \ + svfloat16_t out16_local = svcvt_f16_f32_z(pN32_full, (out32_vec)); \ + out16_local = svuzp1_f16(out16_local, out16_local); \ + svst1_f16_x2(pOut16_full_c, &c[(dst_idx)], svcreate2(out16_local, out16_local)); \ + } while (0) + + for (size_t row = start_row; row < end_row; row += tile_rows) { + const size_t rb = row / tile_rows; + const size_t active_r = std::min(tile_rows, end_row - row); + const bool full_r = (active_r == tile_rows); + const svbool_t pMh = full_r + ? svptrue_b16() + : svwhilelt_b16(static_cast(0), static_cast(active_r * 2)); + + size_t cb = 0; + + for (; cb < cb4_tiles; cb += 4) { + svzero_za(); + const size_t g4 = cb / 4; + const __fp16* b_g4_base = b_packed + g4 * k_pairs * 4 * tile_pairs; + + size_t kp = 0; + for (; kp + 3 < k_pairs; kp += 4) { + const __fp16* a_ptr0 = a_packed + rb * a_row_block_stride + kp * tile_pairs; + const __fp16* b_ptr0 = b_g4_base + kp * (4 * tile_pairs); + const __fp16* a_ptr1 = a_ptr0 + tile_pairs; + const __fp16* b_ptr1 = b_ptr0 + 4 * tile_pairs; + const __fp16* a_ptr2 = a_ptr1 + tile_pairs; + const __fp16* b_ptr2 = b_ptr1 + 4 * tile_pairs; + const __fp16* a_ptr3 = a_ptr2 + tile_pairs; + const __fp16* b_ptr3 = b_ptr2 + 4 * tile_pairs; + + const svfloat16_t zA0 = svld1(pMh, a_ptr0); + const svfloat16x4_t zB40 = svld1_f16_x4(pNh_full_c, b_ptr0); + svmopa_za32_f16_m(0, pMh, pNh_full, zA0, svget4(zB40, 0)); + svmopa_za32_f16_m(1, pMh, pNh_full, zA0, svget4(zB40, 1)); + svmopa_za32_f16_m(2, pMh, pNh_full, zA0, svget4(zB40, 2)); + svmopa_za32_f16_m(3, pMh, pNh_full, zA0, svget4(zB40, 3)); + + const svfloat16_t zA1 = svld1(pMh, a_ptr1); + const svfloat16x4_t zB41 = svld1_f16_x4(pNh_full_c, b_ptr1); + svmopa_za32_f16_m(0, pMh, pNh_full, zA1, svget4(zB41, 0)); + svmopa_za32_f16_m(1, pMh, pNh_full, zA1, svget4(zB41, 1)); + svmopa_za32_f16_m(2, pMh, pNh_full, zA1, svget4(zB41, 2)); + svmopa_za32_f16_m(3, pMh, pNh_full, zA1, svget4(zB41, 3)); + + const svfloat16_t zA2 = svld1(pMh, a_ptr2); + const svfloat16x4_t zB42 = svld1_f16_x4(pNh_full_c, b_ptr2); + svmopa_za32_f16_m(0, pMh, pNh_full, zA2, svget4(zB42, 0)); + svmopa_za32_f16_m(1, pMh, pNh_full, zA2, svget4(zB42, 1)); + svmopa_za32_f16_m(2, pMh, pNh_full, zA2, svget4(zB42, 2)); + svmopa_za32_f16_m(3, pMh, pNh_full, zA2, svget4(zB42, 3)); + + const svfloat16_t zA3 = svld1(pMh, a_ptr3); + const svfloat16x4_t zB43 = svld1_f16_x4(pNh_full_c, b_ptr3); + svmopa_za32_f16_m(0, pMh, pNh_full, zA3, svget4(zB43, 0)); + svmopa_za32_f16_m(1, pMh, pNh_full, zA3, svget4(zB43, 1)); + svmopa_za32_f16_m(2, pMh, pNh_full, zA3, svget4(zB43, 2)); + svmopa_za32_f16_m(3, pMh, pNh_full, zA3, svget4(zB43, 3)); + } + + for (; kp < k_pairs; ++kp) { + const __fp16* a_ptr = a_packed + rb * a_row_block_stride + kp * tile_pairs; + const __fp16* b_ptr = b_g4_base + kp * (4 * tile_pairs); + + const svfloat16_t zA = svld1(pMh, a_ptr); + const svfloat16x4_t zB4 = svld1_f16_x4(pNh_full_c, b_ptr); + svmopa_za32_f16_m(0, pMh, pNh_full, zA, svget4(zB4, 0)); + svmopa_za32_f16_m(1, pMh, pNh_full, zA, svget4(zB4, 1)); + svmopa_za32_f16_m(2, pMh, pNh_full, zA, svget4(zB4, 2)); + svmopa_za32_f16_m(3, pMh, pNh_full, zA, svget4(zB4, 3)); + } + + const size_t col = cb * tile_rows; + if (full_r) { + for (size_t trow = 0; trow < tile_rows; trow += 4) { + const svfloat32x4_t zT0 = svread_hor_za32_f32_vg4(0, static_cast(trow)); + const svfloat32x4_t zT1 = svread_hor_za32_f32_vg4(1, static_cast(trow)); + const svfloat32x4_t zT2 = svread_hor_za32_f32_vg4(2, static_cast(trow)); + const svfloat32x4_t zT3 = svread_hor_za32_f32_vg4(3, static_cast(trow)); + const size_t dst0 = (row + trow + 0) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst0, svget4(zT0, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget4(zT1, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + 2 * tile_rows, svget4(zT2, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + 3 * tile_rows, svget4(zT3, 0)); + const size_t dst1 = (row + trow + 1) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst1, svget4(zT0, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget4(zT1, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + 2 * tile_rows, svget4(zT2, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + 3 * tile_rows, svget4(zT3, 1)); + const size_t dst2 = (row + trow + 2) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst2, svget4(zT0, 2)); + CACTUS_STORE_FULL_TILE_ROW(dst2 + tile_rows, svget4(zT1, 2)); + CACTUS_STORE_FULL_TILE_ROW(dst2 + 2 * tile_rows, svget4(zT2, 2)); + CACTUS_STORE_FULL_TILE_ROW(dst2 + 3 * tile_rows, svget4(zT3, 2)); + const size_t dst3 = (row + trow + 3) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst3, svget4(zT0, 3)); + CACTUS_STORE_FULL_TILE_ROW(dst3 + tile_rows, svget4(zT1, 3)); + CACTUS_STORE_FULL_TILE_ROW(dst3 + 2 * tile_rows, svget4(zT2, 3)); + CACTUS_STORE_FULL_TILE_ROW(dst3 + 3 * tile_rows, svget4(zT3, 3)); + } + } else { + size_t trow = 0; + for (; trow + 3 < active_r; trow += 4) { + const svfloat32x4_t zT0 = svread_hor_za32_f32_vg4(0, static_cast(trow)); + const svfloat32x4_t zT1 = svread_hor_za32_f32_vg4(1, static_cast(trow)); + const svfloat32x4_t zT2 = svread_hor_za32_f32_vg4(2, static_cast(trow)); + const svfloat32x4_t zT3 = svread_hor_za32_f32_vg4(3, static_cast(trow)); + const size_t dst0 = (row + trow + 0) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst0, svget4(zT0, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget4(zT1, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + 2 * tile_rows, svget4(zT2, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + 3 * tile_rows, svget4(zT3, 0)); + const size_t dst1 = (row + trow + 1) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst1, svget4(zT0, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget4(zT1, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + 2 * tile_rows, svget4(zT2, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + 3 * tile_rows, svget4(zT3, 1)); + const size_t dst2 = (row + trow + 2) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst2, svget4(zT0, 2)); + CACTUS_STORE_FULL_TILE_ROW(dst2 + tile_rows, svget4(zT1, 2)); + CACTUS_STORE_FULL_TILE_ROW(dst2 + 2 * tile_rows, svget4(zT2, 2)); + CACTUS_STORE_FULL_TILE_ROW(dst2 + 3 * tile_rows, svget4(zT3, 2)); + const size_t dst3 = (row + trow + 3) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst3, svget4(zT0, 3)); + CACTUS_STORE_FULL_TILE_ROW(dst3 + tile_rows, svget4(zT1, 3)); + CACTUS_STORE_FULL_TILE_ROW(dst3 + 2 * tile_rows, svget4(zT2, 3)); + CACTUS_STORE_FULL_TILE_ROW(dst3 + 3 * tile_rows, svget4(zT3, 3)); + } + for (; trow + 1 < active_r; trow += 2) { + const svfloat32x2_t zT0 = svread_hor_za32_f32_vg2(0, static_cast(trow)); + const svfloat32x2_t zT1 = svread_hor_za32_f32_vg2(1, static_cast(trow)); + const svfloat32x2_t zT2 = svread_hor_za32_f32_vg2(2, static_cast(trow)); + const svfloat32x2_t zT3 = svread_hor_za32_f32_vg2(3, static_cast(trow)); + const size_t dst0 = (row + trow + 0) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst0, svget2(zT0, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget2(zT1, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + 2 * tile_rows, svget2(zT2, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + 3 * tile_rows, svget2(zT3, 0)); + const size_t dst1 = (row + trow + 1) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst1, svget2(zT0, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget2(zT1, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + 2 * tile_rows, svget2(zT2, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + 3 * tile_rows, svget2(zT3, 1)); + } + for (; trow < active_r; ++trow) { + const size_t dst = (row + trow) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst, svread_hor_za32_f32_m(z0_f32, pN32_full, 0, static_cast(trow))); + CACTUS_STORE_FULL_TILE_ROW(dst + tile_rows, svread_hor_za32_f32_m(z0_f32, pN32_full, 1, static_cast(trow))); + CACTUS_STORE_FULL_TILE_ROW(dst + 2 * tile_rows, svread_hor_za32_f32_m(z0_f32, pN32_full, 2, static_cast(trow))); + CACTUS_STORE_FULL_TILE_ROW(dst + 3 * tile_rows, svread_hor_za32_f32_m(z0_f32, pN32_full, 3, static_cast(trow))); + } + } + } + + for (; cb < cb4_tiles + cb2_tiles; cb += 2) { + svzero_za(); + const size_t g2 = (cb - cb4_tiles) / 2; + const __fp16* b_g2_base = b_packed + cb4_groups * k_pairs * 4 * tile_pairs + g2 * k_pairs * 2 * tile_pairs; + + size_t kp = 0; + for (; kp < k_pairs; ++kp) { + const __fp16* a_ptr = a_packed + rb * a_row_block_stride + kp * tile_pairs; + const __fp16* b_ptr = b_g2_base + kp * (2 * tile_pairs); + + const svfloat16_t zA = svld1(pMh, a_ptr); + const svfloat16x2_t zB2 = svld1_f16_x2(pNh_full_c, b_ptr); + svmopa_za32_f16_m(0, pMh, pNh_full, zA, svget2(zB2, 0)); + svmopa_za32_f16_m(1, pMh, pNh_full, zA, svget2(zB2, 1)); + } + + const size_t col = cb * tile_rows; + if (full_r) { + for (size_t trow = 0; trow < tile_rows; trow += 4) { + const svfloat32x4_t zT0 = svread_hor_za32_f32_vg4(0, static_cast(trow)); + const svfloat32x4_t zT1 = svread_hor_za32_f32_vg4(1, static_cast(trow)); + const size_t dst0 = (row + trow + 0) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst0, svget4(zT0, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget4(zT1, 0)); + const size_t dst1 = (row + trow + 1) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst1, svget4(zT0, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget4(zT1, 1)); + const size_t dst2 = (row + trow + 2) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst2, svget4(zT0, 2)); + CACTUS_STORE_FULL_TILE_ROW(dst2 + tile_rows, svget4(zT1, 2)); + const size_t dst3 = (row + trow + 3) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst3, svget4(zT0, 3)); + CACTUS_STORE_FULL_TILE_ROW(dst3 + tile_rows, svget4(zT1, 3)); + } + } else { + size_t trow = 0; + for (; trow + 3 < active_r; trow += 4) { + const svfloat32x4_t zT0 = svread_hor_za32_f32_vg4(0, static_cast(trow)); + const svfloat32x4_t zT1 = svread_hor_za32_f32_vg4(1, static_cast(trow)); + const size_t dst0 = (row + trow + 0) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst0, svget4(zT0, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget4(zT1, 0)); + const size_t dst1 = (row + trow + 1) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst1, svget4(zT0, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget4(zT1, 1)); + const size_t dst2 = (row + trow + 2) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst2, svget4(zT0, 2)); + CACTUS_STORE_FULL_TILE_ROW(dst2 + tile_rows, svget4(zT1, 2)); + const size_t dst3 = (row + trow + 3) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst3, svget4(zT0, 3)); + CACTUS_STORE_FULL_TILE_ROW(dst3 + tile_rows, svget4(zT1, 3)); + } + for (; trow + 1 < active_r; trow += 2) { + const svfloat32x2_t zT0 = svread_hor_za32_f32_vg2(0, static_cast(trow)); + const svfloat32x2_t zT1 = svread_hor_za32_f32_vg2(1, static_cast(trow)); + const size_t dst0 = (row + trow + 0) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst0, svget2(zT0, 0)); + CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget2(zT1, 0)); + const size_t dst1 = (row + trow + 1) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst1, svget2(zT0, 1)); + CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget2(zT1, 1)); + } + for (; trow < active_r; ++trow) { + const size_t dst = (row + trow) * N + col; + CACTUS_STORE_FULL_TILE_ROW(dst, svread_hor_za32_f32_m(z0_f32, pN32_full, 0, static_cast(trow))); + CACTUS_STORE_FULL_TILE_ROW(dst + tile_rows, svread_hor_za32_f32_m(z0_f32, pN32_full, 1, static_cast(trow))); + } + } + } + + for (; cb < col_blocks; ++cb) { + const size_t col = cb * tile_rows; + const size_t active_c = std::min(tile_rows, N - col); + const svbool_t pNh = svwhilelt_b16(static_cast(0), static_cast(active_c * 2)); + const svbool_t pN32 = svwhilelt_b32(static_cast(0), static_cast(active_c)); + const svbool_t pOut16 = svwhilelt_b16(static_cast(0), static_cast(active_c)); + const size_t g1 = cb - cb4_tiles - cb2_tiles; + const __fp16* b_g1_base = b_packed + cb1_off + g1 * k_pairs * tile_pairs; + + svzero_za(); + + size_t kp = 0; + for (; kp + 1 < k_pairs; kp += 2) { + const __fp16* a_ptr0 = a_packed + rb * a_row_block_stride + kp * tile_pairs; + const __fp16* b_ptr0 = b_g1_base + kp * tile_pairs; + const __fp16* a_ptr1 = a_ptr0 + tile_pairs; + const __fp16* b_ptr1 = b_ptr0 + tile_pairs; + + const svfloat16_t zA0 = svld1(pMh, a_ptr0); + const svfloat16_t zB0 = svld1(pNh, b_ptr0); + svmopa_za32_f16_m(0, pMh, pNh, zA0, zB0); + + const svfloat16_t zA1 = svld1(pMh, a_ptr1); + const svfloat16_t zB1 = svld1(pNh, b_ptr1); + svmopa_za32_f16_m(0, pMh, pNh, zA1, zB1); + } + + for (; kp < k_pairs; ++kp) { + const __fp16* a_ptr = a_packed + rb * a_row_block_stride + kp * tile_pairs; + const __fp16* b_ptr = b_g1_base + kp * tile_pairs; + + const svfloat16_t zA = svld1(pMh, a_ptr); + const svfloat16_t zB = svld1(pNh, b_ptr); + svmopa_za32_f16_m(0, pMh, pNh, zA, zB); + } + + for (size_t trow = 0; trow < active_r; ++trow) { + svfloat32_t out32 = svread_hor_za32_f32_m(z0_f32, pN32, 0, static_cast(trow)); + svfloat16_t out16 = svcvt_f16_f32_z(pN32, out32); + out16 = svuzp1_f16(out16, out16); + svst1(pOut16, &c[(row + trow) * N + col], out16); + } + } + } +#undef CACTUS_STORE_FULL_TILE_ROW +} + +__arm_new("za") __arm_locally_streaming +static void cactus_matmul_f16_sme2_thread_entry( + const __fp16* a, + __fp16* a_packed, + const __fp16* b_packed, + __fp16* c, + size_t M, + size_t K, + size_t N, + size_t row_block_size, + size_t start_block, + size_t end_block +) { + const size_t tile_rows = svcntsw(); + const size_t tile_pairs = svcnth(); + const size_t k_pairs = (K + 1) / 2; + + for (size_t block_idx = start_block; block_idx < end_block; ++block_idx) { + const size_t start_row = block_idx * row_block_size; + const size_t end_row = std::min(start_row + row_block_size, M); + + for (size_t row = start_row; row < end_row; row += tile_rows) { + const size_t rb = row / tile_rows; + const size_t active_r = std::min(tile_rows, end_row - row); + cactus_pack_a_f16_row_block( + a, + a_packed, + K, + rb, + row, + active_r, + tile_rows, + tile_pairs, + k_pairs + ); + } + + cactus_matmul_f16_sme2_worker( + a_packed, + b_packed, + c, + M, + K, + N, + start_row, + end_row, + tile_rows, + tile_pairs + ); + } +} + +__arm_new("za") __arm_locally_streaming +void cactus_matmul_f16_sme2_caller( + const __fp16* a, + const __fp16* b_transposed, + __fp16* c, + size_t M, + size_t K, + size_t N +) { + const size_t tile_rows = svcntsw(); + const size_t tile_pairs = svcnth(); + constexpr size_t SME2_TILES_PER_THREAD = 3; + + const size_t row_blocks = (M + tile_rows - 1) / tile_rows; + const size_t k_pairs = (K + 1) / 2; + const size_t col_blocks = (N + tile_rows - 1) / tile_rows; + + std::unique_ptr<__fp16[]> a_packed(new __fp16[row_blocks * k_pairs * tile_pairs]); + std::unique_ptr<__fp16[]> b_packed(new __fp16[k_pairs * col_blocks * tile_pairs]); + __fp16* a_packed_ptr = a_packed.get(); + const __fp16* b_packed_ptr = b_packed.get(); + + cactus_pack_b_f16_from_bt(b_transposed, b_packed.get(), K, N, tile_rows, tile_pairs); + + const size_t row_block_size = SME2_TILES_PER_THREAD * tile_rows; + const size_t num_row_blocks = (M + row_block_size - 1) / row_block_size; + + auto& pool = CactusThreading::get_thread_pool(); + const size_t num_workers = std::min(pool.num_workers(), num_row_blocks); + if (num_workers <= 1) { + cactus_matmul_f16_sme2_thread_entry( + a, + a_packed_ptr, + b_packed_ptr, + c, + M, + K, + N, + row_block_size, + 0, + num_row_blocks + ); + return; + } + + std::atomic next_block{0}; + pool.enqueue_n_threads(num_workers, num_workers, [&](size_t, size_t) { + while (true) { + const size_t block_idx = next_block.fetch_add(1, std::memory_order_relaxed); + if (block_idx >= num_row_blocks) break; + cactus_matmul_f16_sme2_thread_entry( + a, + a_packed_ptr, + b_packed_ptr, + c, + M, + K, + N, + row_block_size, + block_idx, + block_idx + 1 + ); + } + }); + pool.wait_all(); +} + +#undef CACTUS_UNROLL4 diff --git a/cactus-transpile/kernel/kernel_utils.h b/cactus-transpile/kernel/kernel_utils.h new file mode 100644 index 000000000..916d9bf46 --- /dev/null +++ b/cactus-transpile/kernel/kernel_utils.h @@ -0,0 +1,724 @@ +#ifndef KERNEL_UTILS_H +#define KERNEL_UTILS_H + +#include +#if defined(__APPLE__) +#include +#include +#endif +#if defined(__ANDROID__) +#include +#include +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr size_t NEON_VECTOR_SIZE = 16; +constexpr size_t STREAMING_STORE_THRESHOLD = 32768; + +inline void stream_store_f16x8(__fp16* dst, float16x8_t val) { +#if defined(__aarch64__) + float16x4_t lo = vget_low_f16(val); + float16x4_t hi = vget_high_f16(val); + __asm__ __volatile__( + "stnp %d0, %d1, [%2]" + : + : "w"(lo), "w"(hi), "r"(dst) + : "memory" + ); +#else + vst1q_f16(dst, val); +#endif +} + +inline bool cpu_has_i8mm() { +#if defined(__aarch64__) + static std::once_flag once; + static bool has = false; + + std::call_once(once, []() { +#if defined(__APPLE__) + int ret = 0; + size_t size = sizeof(ret); + if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &ret, &size, nullptr, 0) == 0) { + has = (ret == 1); + } +#elif defined(__ANDROID__) + unsigned long hwcap2 = getauxval(AT_HWCAP2); + #ifndef HWCAP2_I8MM + #define HWCAP2_I8MM (1 << 13) + #endif + has = (hwcap2 & HWCAP2_I8MM) != 0; +#endif + }); + + return has; +#else + return false; +#endif +} + +inline bool cpu_has_sme2() { +#if defined(__aarch64__) + static std::once_flag once; + static bool has = false; + + std::call_once(once, []() { + +#if defined(__APPLE__) + int ret = 0; + size_t size = sizeof(ret); + if (sysctlbyname("hw.optional.arm.FEAT_SME2", &ret, &size, nullptr, 0) == 0) { + has = ret == 1; + } + +#elif defined(__ANDROID__) + unsigned long hwcap2 = getauxval(AT_HWCAP2); +#ifdef HWCAP2_SME2 + has = (hwcap2 & HWCAP2_SME2) != 0; +#endif + +#endif + }); + + return has; +#else + return false; +#endif +} + +inline float32x4_t fast_exp_f32x4(float32x4_t x) { + const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); + const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); + + const float32x4_t c0 = vdupq_n_f32(1.0f); + const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); + const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f); + const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f); + const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); + + x = vmaxq_f32(x, vdupq_n_f32(-87.0f)); + x = vminq_f32(x, vdupq_n_f32(87.0f)); + + float32x4_t z = vmulq_f32(x, log2e); + + int32x4_t zi = vcvtq_s32_f32(z); + float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi)); + + uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f)); + zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1))); + zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f))))); + + float32x4_t zf_ln2 = vmulq_f32(zf, ln2); + float32x4_t p = c4; + p = vfmaq_f32(c3, p, zf_ln2); + p = vfmaq_f32(c2, p, zf_ln2); + p = vfmaq_f32(c1, p, zf_ln2); + p = vfmaq_f32(c0, p, zf_ln2); + + int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23); + float32x4_t scale = vreinterpretq_f32_s32(exp_bits); + + return vmulq_f32(p, scale); +} + +// Cephes-style 13/6 rational tanh approximation (same coefficients as Eigen). +// Constants are stored as static splatted arrays so the compiler emits a single +// pc-relative `ldr q` per load. +alignas(16) inline constexpr float kFastTanhAlpha[7][4] = { + { 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f }, + { 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f }, + { 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f }, + { 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f }, + {-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f }, + { 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f }, + {-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f }, +}; +alignas(16) inline constexpr float kFastTanhBeta[4][4] = { + { 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f }, + { 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f }, + { 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f }, + { 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f }, +}; +alignas(16) inline constexpr float kFastTanhClampHi[4] = { 9.0f, 9.0f, 9.0f, 9.0f }; +alignas(16) inline constexpr float kFastTanhClampLo[4] = {-9.0f,-9.0f,-9.0f,-9.0f }; + +inline float32x4_t fast_tanh_f32x4(float32x4_t x) { + x = vmaxq_f32(vld1q_f32(kFastTanhClampLo), vminq_f32(vld1q_f32(kFastTanhClampHi), x)); + float32x4_t x2 = vmulq_f32(x, x); + float32x4_t p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[5]), vld1q_f32(kFastTanhAlpha[6]), x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[4]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[3]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[2]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[1]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[0]), p, x2); + p = vmulq_f32(p, x); + float32x4_t q = vfmaq_f32(vld1q_f32(kFastTanhBeta[2]), vld1q_f32(kFastTanhBeta[3]), x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[1]), q, x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[0]), q, x2); + return vdivq_f32(p, q); +} + +constexpr size_t SIMD_F16_WIDTH = 8; + +inline size_t simd_align(size_t count, size_t width = SIMD_F16_WIDTH) { + return (count / width) * width; +} + +inline void f16x8_split_f32(float16x8_t v, float32x4_t& lo, float32x4_t& hi) { + lo = vcvt_f32_f16(vget_low_f16(v)); + hi = vcvt_f32_f16(vget_high_f16(v)); +} + +inline float16x8_t f32_merge_f16(float32x4_t lo, float32x4_t hi) { + return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); +} + +inline float32x4_t fast_sigmoid_f32x4(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + return vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(x)))); +} + +template +inline float16x8_t apply_f32_op_on_f16x8(float16x8_t v, F32x4Op op) { + float32x4_t lo, hi; + f16x8_split_f32(v, lo, hi); + return f32_merge_f16(op(lo), op(hi)); +} + +inline void unpack_int4_as_int8x16x2(const uint8_t* ptr, int8x16_t& high_decoded, int8x16_t& low_decoded) { + int8x16_t packed = vreinterpretq_s8_u8(vld1q_u8(ptr)); + high_decoded = vshrq_n_s8(packed, 4); + low_decoded = vshrq_n_s8(vshlq_n_s8(packed, 4), 4); +} + +namespace CactusThreading { + +#if defined(__ANDROID__) + struct CoreTopology { + std::vector performance_cores; + std::vector all_cores; + + static CoreTopology& get() { + static CoreTopology topo = detect(); + return topo; + } + + private: + static int read_sysfs_int(const char* path) { + std::ifstream f(path); + if (!f.is_open()) return -1; + int val = -1; + f >> val; + return val; + } + + static CoreTopology detect() { + CoreTopology topo; + constexpr int MAX_CPUS = 16; + std::vector> core_caps; + + for (int i = 0; i < MAX_CPUS; ++i) { + char path[128]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpu_capacity", i); + int cap = read_sysfs_int(path); + if (cap > 0) { + core_caps.push_back({i, cap}); + topo.all_cores.push_back(i); + continue; + } + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); + int freq = read_sysfs_int(path); + if (freq > 0) { + core_caps.push_back({i, freq}); + topo.all_cores.push_back(i); + } + } + + if (core_caps.empty()) return topo; + + int max_cap = 0; + for (auto& [id, cap] : core_caps) { + max_cap = std::max(max_cap, cap); + } + + int threshold = static_cast(max_cap * 0.70); + for (auto& [id, cap] : core_caps) { + if (cap >= threshold) { + topo.performance_cores.push_back(id); + } + } + + return topo; + } + }; + + inline bool pin_current_thread_to_cores(const std::vector& cores) { + if (cores.empty()) return false; + cpu_set_t mask; + CPU_ZERO(&mask); + for (int core : cores) { + CPU_SET(core, &mask); + } + return sched_setaffinity(0, sizeof(mask), &mask) == 0; + } +#endif + + class ThreadPool { + private: + static constexpr size_t MAX_WORKERS = 16; + + std::vector workers; + std::deque> tasks; + + std::mutex mutex; + std::condition_variable work_available; + std::condition_variable work_done; + + bool stop{false}; + std::atomic pending_tasks{0}; + size_t num_workers_; + + void worker_thread() { + while (true) { + std::function task; + { + std::unique_lock lock(mutex); + work_available.wait(lock, [this] { + return stop || !tasks.empty(); + }); + + if (stop && tasks.empty()) { + return; + } + + task = std::move(tasks.front()); + tasks.pop_front(); + } + + task(); + + if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { + std::lock_guard lock(mutex); + work_done.notify_one(); + } + } + } + + public: + explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) + : stop(false), pending_tasks(0) { + num_workers_ = std::min(num_threads, MAX_WORKERS); + if (num_workers_ == 0) num_workers_ = 1; + +#if defined(__ANDROID__) + auto& topo = CoreTopology::get(); + if (!topo.performance_cores.empty()) { + num_workers_ = std::min(num_workers_, topo.performance_cores.size()); + } +#endif + + workers.reserve(num_workers_); + for (size_t i = 0; i < num_workers_; ++i) { + workers.emplace_back([this]() { +#if defined(__ANDROID__) + auto& perf = CoreTopology::get().performance_cores; + if (!perf.empty()) { + pin_current_thread_to_cores(perf); + } +#endif + worker_thread(); + }); + } + } + + ~ThreadPool() { + { + std::lock_guard lock(mutex); + stop = true; + } + work_available.notify_all(); + for (auto& worker : workers) { + if (worker.joinable()) { + worker.join(); + } + } + } + + template + auto enqueue(F&& f) -> std::future { + using return_type = decltype(f()); + + auto task = std::make_shared>( + std::forward(f) + ); + + std::future res = task->get_future(); + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(1, std::memory_order_relaxed); + tasks.emplace_back([task](){ (*task)(); }); + } + work_available.notify_one(); + + return res; + } + + template + void enqueue_batch(size_t total_work, F task_func) { + if (total_work == 0) return; + + const size_t num_tasks = std::min(num_workers_, total_work); + const size_t per_worker = total_work / num_tasks; + const size_t remainder = total_work % num_tasks; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed); + + for (size_t w = 0; w < num_tasks; ++w) { + size_t start = w * per_worker + std::min(w, remainder); + size_t end = start + per_worker + (w < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + void wait_all() { + std::unique_lock lock(mutex); + work_done.wait(lock, [this] { + return pending_tasks.load(std::memory_order_acquire) == 0; + }); + } + + template + void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) { + if (total_work == 0 || num_threads == 0) return; + + num_threads = std::min(num_threads, std::min(num_workers_, total_work)); + const size_t per_thread = total_work / num_threads; + const size_t remainder = total_work % num_threads; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_threads, std::memory_order_relaxed); + + for (size_t t = 0; t < num_threads; ++t) { + size_t start = t * per_thread + std::min(t, remainder); + size_t end = start + per_thread + (t < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + size_t num_workers() const { return num_workers_; } + }; + + inline size_t get_configured_thread_count() { + const char* raw = std::getenv("CACTUS_NUM_THREADS"); + if (!raw || !*raw) { + return std::thread::hardware_concurrency(); + } + char* end = nullptr; + long parsed = std::strtol(raw, &end, 10); + if (end == raw || parsed <= 0) { + return std::thread::hardware_concurrency(); + } + return static_cast(parsed); + } + + inline ThreadPool& get_thread_pool() { + static ThreadPool pool(get_configured_thread_count()); + return pool; + } + + struct ParallelConfig { + size_t min_work_gate; + size_t work_per_thread; + + constexpr ParallelConfig(size_t gate, size_t per_thread) + : min_work_gate(gate), work_per_thread(per_thread) {} + }; + + inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) { + if (total_work < config.min_work_gate) return 1; + + size_t pool_size = get_thread_pool().num_workers(); + size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread; + return std::min(pool_size, std::max(static_cast(1), num_threads)); + } + + struct Thresholds { + #if defined(__ANDROID__) + static constexpr ParallelConfig ATTENTION{64, 32}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{30000, 15000}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000}; + #else // Apple + static constexpr ParallelConfig ATTENTION{32, 16}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{5000, 2500}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250}; + #endif + }; + + struct GemmThreading { + #if defined(__ANDROID__) + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return 1; + return pool_size; + } + static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) { + return 1; + } + #elif defined(__APPLE__) && TARGET_OS_IPHONE + static constexpr size_t GEMV_MIN_N_BLOCKS = 512; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(2)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + return std::min(pool_size, static_cast(3)); + } + #else + static constexpr size_t GEMV_MIN_N_BLOCKS = 256; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(4)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + if (N_blocks < 512) return std::min(pool_size, static_cast(2)); + return std::min(pool_size, static_cast(5)); + } + #endif + }; + + inline size_t& get_gemm_thread_override() { + static size_t override_threads = 0; + return override_threads; + } + + inline void set_gemm_threads(size_t num_threads) { + get_gemm_thread_override() = num_threads; + } + + inline void reset_gemm_threads() { + get_gemm_thread_override() = 0; + } + + class TaskHandle { + private: + std::vector> futures_; + bool auto_wait_; + + public: + TaskHandle(bool auto_wait = true) : auto_wait_(auto_wait) {} + + ~TaskHandle() { + if (auto_wait_) { + wait(); + } + } + + TaskHandle(TaskHandle&&) = default; + TaskHandle& operator=(TaskHandle&&) = default; + TaskHandle(const TaskHandle&) = delete; + TaskHandle& operator=(const TaskHandle&) = delete; + + void add_future(std::future&& f) { + futures_.push_back(std::move(f)); + } + + void wait() { + for (auto& f : futures_) { + if (f.valid()) { + f.wait(); + } + } + futures_.clear(); + } + + bool is_ready() const { + for (const auto& f : futures_) { + if (f.valid() && f.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { + return false; + } + } + return true; + } + + size_t task_count() const { return futures_.size(); } + }; + + template + TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + TaskHandle handle(!wait); + + if (num_threads == 1) { + if (wait) { + work_func(0, total_work); + return handle; + } + auto& pool = get_thread_pool(); + handle.add_future(pool.enqueue([work_func, total_work]() { + work_func(0, total_work); + })); + return handle; + } + + auto& pool = get_thread_pool(); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + work_func(start_idx, end_idx); + })); + } + + if (wait) { + handle.wait(); + } + return handle; + } + + template + void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) { + const size_t total_work = outer_size * inner_size; + parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) { + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t outer = work_idx / inner_size; + const size_t inner = work_idx % inner_size; + work_func(outer, inner); + } + }); + } + + template + ResultType parallel_reduce(size_t total_work, ParallelConfig config, + WorkFunc work_func, ResultType init_value, CombineFunc combine_func) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + + if (num_threads == 1) { + return work_func(0, total_work); + } + + auto& pool = get_thread_pool(); + std::vector> futures; + std::vector partial_results(num_threads, init_value); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + futures.push_back(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() -> ResultType { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + return work_func(start_idx, end_idx); + })); + } + + ResultType result = init_value; + for (auto& future : futures) { + result = combine_func(result, future.get()); + } + return result; + } + + template + void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) { + auto& pool = get_thread_pool(); + + size_t override = get_gemm_thread_override(); + size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers()); + num_threads = std::min(num_threads, total_tiles); + + if (num_threads <= 1) { + work_func(0, total_tiles); + return; + } + + pool.enqueue_n_threads(total_tiles, num_threads, work_func); + pool.wait_all(); + } + +} + +template +void elementwise_op_f16(const __fp16* input, __fp16* output, size_t num_elements, + bool use_streaming, CactusThreading::ParallelConfig config, + SimdOp simd_op, ScalarOp scalar_op, size_t unroll = 4) { + CactusThreading::parallel_for(num_elements, config, + [&](size_t start, size_t end) { + const size_t n = end - start; + const size_t vec_end = start + simd_align(n); + + if (use_streaming && unroll >= 4) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 4); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 4) { + __builtin_prefetch(&input[i + 256], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + float16x8_t v2 = simd_op(vld1q_f16(&input[i + 16])); + float16x8_t v3 = simd_op(vld1q_f16(&input[i + 24])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + stream_store_f16x8(&output[i + 16], v2); + stream_store_f16x8(&output[i + 24], v3); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else if (use_streaming && unroll >= 2) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 2); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 2) { + __builtin_prefetch(&input[i + 128], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else { + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + vst1q_f16(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } + for (size_t i = vec_end; i < end; ++i) { + output[i] = scalar_op(input[i]); + } + }); +} + +#endif // KERNEL_UTILS_H diff --git a/cactus-transpile/models/gemma4/model_gemma4.cpp b/cactus-transpile/models/gemma4/model_gemma4.cpp new file mode 100644 index 000000000..c3f9c5b70 --- /dev/null +++ b/cactus-transpile/models/gemma4/model_gemma4.cpp @@ -0,0 +1,594 @@ +#include "model_gemma4.h" +#include "../../graph/graph.h" +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +Gemma4Model::Gemma4Model() : Model() {} + +Gemma4Model::Gemma4Model(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); +} + +bool Gemma4Model::is_global_layer(uint32_t idx) const { + return config_.layer_types[idx] == "global"; +} + +std::vector Gemma4Model::get_kv_layer_dims() const { + uint32_t n = config_.num_layers; + + std::vector dims(n); + for (uint32_t i = 0; i < n; i++) { + if (i >= first_shared_layer_) { + dims[i] = 0; + } else if (is_global_layer(i)) { + dims[i] = config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2; + } else { + dims[i] = config_.attention_head_dim; + } + } + return dims; +} + +std::vector Gemma4Model::get_kv_layer_heads() const { + uint32_t n = config_.num_layers; + + std::vector heads(n); + for (uint32_t i = 0; i < n; i++) { + if (i >= first_shared_layer_) { + heads[i] = 0; + } else if (is_global_layer(i) && config_.num_global_kv_heads > 0) { + heads[i] = config_.num_global_kv_heads; + } else { + heads[i] = config_.attention_kv_heads; + } + } + return heads; +} + +void Gemma4Model::compact_kv_cache() { + uint32_t n = config_.num_layers; + + std::vector target_windows(n, 0); + for (uint32_t i = 0; i < n; i++) { + if (i >= first_shared_layer_) continue; + if (!is_global_layer(i)) + target_windows[i] = config_.sliding_window; + } + kv_cache_.compact_to_windows(target_windows); +} + +void Gemma4Model::post_init() { + uint32_t n = config_.num_layers; + + kv_cache_.set_window_size(0, 0); + + kv_share_map_.resize(n, -1); + shared_k_nodes_.resize(n, 0); + shared_v_nodes_.resize(n, 0); + + for (uint32_t i = first_shared_layer_; i < n; i++) { + bool is_global = is_global_layer(i); + for (int j = static_cast(first_shared_layer_) - 1; j >= 0; j--) { + if (is_global_layer(j) == is_global) { + kv_share_map_[i] = j; + break; + } + } + } +} + +void Gemma4Model::load_weights_to_graph(CactusGraph* gb) { + uint32_t n = config_.num_layers; + uint32_t num_shared = config_.num_kv_shared_layers; + first_shared_layer_ = (n > num_shared) ? n - num_shared : n; + + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + } + + bool has_pli = config_.hidden_size_per_layer_input > 0; + if (has_pli) { + weight_nodes_.embed_tokens_per_layer = gb->mmap_embeddings(model_folder_path_ + "/embed_tokens_per_layer.weights"); + weight_nodes_.per_layer_model_proj = gb->mmap_weights(model_folder_path_ + "/per_layer_model_proj.weights"); + weight_nodes_.per_layer_proj_norm = gb->mmap_weights(model_folder_path_ + "/per_layer_proj_norm.weights"); + } + + bool has_moe = config_.enable_moe_block; + + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer = weight_nodes_.layers[i]; + std::string prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + bool is_shared = (i >= first_shared_layer_); + + layer.attn_q_weight = gb->mmap_weights(prefix + "attn_q.weights"); + layer.attn_k_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k.weights"); + bool k_eq_v = config_.attention_k_eq_v && is_global_layer(i); + layer.attn_v_weight = is_shared ? 0 : (k_eq_v ? layer.attn_k_weight : gb->mmap_weights(prefix + "attn_v.weights")); + layer.attn_output_weight = gb->mmap_weights(prefix + "attn_output.weights"); + layer.input_layernorm_weight = gb->mmap_weights(prefix + "input_norm.weights"); + layer.attn_q_norm_weight = gb->mmap_weights(prefix + "attn_q_norm.weights"); + layer.attn_k_norm_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k_norm.weights"); + layer.ffn_gate_weight = gb->mmap_weights(prefix + "ffn_gate.weights"); + layer.ffn_up_weight = gb->mmap_weights(prefix + "ffn_up.weights"); + layer.ffn_down_weight = gb->mmap_weights(prefix + "ffn_down.weights"); + layer.post_attention_layernorm_weight = gb->mmap_weights(prefix + "post_attn_norm.weights"); + layer.pre_feedforward_layernorm_weight = gb->mmap_weights(prefix + "pre_ffn_norm.weights"); + layer.post_feedforward_layernorm_weight = gb->mmap_weights(prefix + "post_ffn_norm.weights"); + if (has_pli) { + layer.per_layer_gate = gb->mmap_weights(prefix + "per_layer_gate.weights"); + layer.per_layer_proj = gb->mmap_weights(prefix + "per_layer_proj.weights"); + layer.post_per_layer_norm = gb->mmap_weights(prefix + "post_per_layer_norm.weights"); + } + layer.layer_scalar = std::filesystem::exists(prefix + "layer_scalar.weights") ? gb->mmap_weights(prefix + "layer_scalar.weights") : 0; + + if (has_moe) { + uint32_t num_experts = config_.num_experts; + uint32_t hidden = config_.hidden_dim; + uint32_t expert_dim = config_.expert_intermediate_size > 0 ? config_.expert_intermediate_size : config_.ffn_intermediate_dim; + layer.moe_w1_experts.resize(num_experts); + layer.moe_w3_experts.resize(num_experts); + layer.moe_w2_experts.resize(num_experts); + + auto w1_packed = gb->mmap_weights(prefix + "moe_gate_proj.weights"); + auto w3_packed = gb->mmap_weights(prefix + "moe_up_proj.weights"); + auto w2_packed = gb->mmap_weights(prefix + "moe_down_proj.weights"); + + const auto& w1_buf = gb->get_output_buffer(w1_packed); + const auto& w3_buf = gb->get_output_buffer(w3_packed); + const auto& w2_buf = gb->get_output_buffer(w2_packed); + + auto setup_experts = [&](const BufferDesc& buf, + std::vector& expert_nodes, + size_t out_dim, size_t in_dim) { + auto* base = static_cast(const_cast(buf.get_data())); + Precision prec = buf.precision; + bool is_quantized = PrecisionTraits::is_integer(prec) && buf.group_size > 0; + size_t K = in_dim; + if (is_quantized && K % buf.group_size != 0) + K = ((K + buf.group_size - 1) / buf.group_size) * buf.group_size; + size_t expert_data_bytes = PrecisionTraits::packed_size_of(prec, out_dim * K); + size_t groups_per_expert = is_quantized ? (out_dim * K + buf.group_size - 1) / buf.group_size : 0; + size_t expert_scales_bytes = groups_per_expert * sizeof(__fp16); + + char* scales_base = is_quantized ? static_cast(const_cast(buf.scales_data)) : nullptr; + + for (uint32_t e = 0; e < num_experts; e++) { + expert_nodes[e] = gb->input({out_dim, K}, prec); + gb->set_external_input(expert_nodes[e], base + e * expert_data_bytes, prec); + if (is_quantized) { + gb->set_grouped_scales(expert_nodes[e], buf.group_size, groups_per_expert, + scales_base + e * expert_scales_bytes); + if (buf.is_interleaved) + gb->set_interleaved(expert_nodes[e], true, out_dim); + } + } + }; + + setup_experts(w1_buf, layer.moe_w1_experts, expert_dim, hidden); + setup_experts(w3_buf, layer.moe_w3_experts, expert_dim, hidden); + setup_experts(w2_buf, layer.moe_w2_experts, hidden, expert_dim); + + layer.router_proj = gb->mmap_weights(prefix + "router_proj.weights"); + layer.router_scale = gb->mmap_weights(prefix + "router_scale.weights"); + layer.moe_per_expert_scale = gb->mmap_weights(prefix + "moe_per_expert_scale.weights"); + layer.post_ffn_norm_1 = gb->mmap_weights(prefix + "post_ffn_norm_1.weights"); + layer.pre_ffn_norm_2 = gb->mmap_weights(prefix + "pre_ffn_norm_2.weights"); + layer.post_ffn_norm_2 = gb->mmap_weights(prefix + "post_ffn_norm_2.weights"); + } + } + + size_t sliding_head_dim = config_.attention_head_dim; + size_t global_head_dim = config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2; + v_norm_ones_weight_.assign(std::max(sliding_head_dim, global_head_dim), static_cast<__fp16>(1.0f)); + v_norm_ones_node_ = gb->input({sliding_head_dim}, Precision::FP16); + gb->set_external_input(v_norm_ones_node_, v_norm_ones_weight_.data(), Precision::FP16); + v_norm_ones_global_node_ = gb->input({global_head_dim}, Precision::FP16); + gb->set_external_input(v_norm_ones_global_node_, v_norm_ones_weight_.data(), Precision::FP16); + + if (npu::is_npu_available()) { + std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; + if (std::filesystem::exists(npu_prefill_path)) { + if (!load_npu_prefill(npu_prefill_path) || !has_npu_prefill()) { + CACTUS_LOG_WARN("npu", "[gemma4] found model.mlpackage but failed to enable NPU prefill; using CPU prefill"); + } + } else { + CACTUS_LOG_WARN("npu", "[gemma4] model.mlpackage not found; using CPU prefill"); + } + } else { + CACTUS_LOG_WARN("npu", "[gemma4] NPU backend unavailable on this device; using CPU prefill"); + } +} + +size_t Gemma4Model::build_per_layer_input(CactusGraph* gb, size_t hidden, size_t pli_combined, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + uint32_t pli_dim = config_.hidden_size_per_layer_input; + + auto gate = gb->gelu(gb->matmul(hidden, layer.per_layer_gate, true, backend)); + auto pli_slice = gb->slice(pli_combined, 1, layer_idx * pli_dim, pli_dim); + auto gated = gb->multiply(gate, pli_slice); + auto pli_proj = gb->matmul(gated, layer.per_layer_proj, true, backend); + auto pli_normed = gb->rms_norm(pli_proj, layer.post_per_layer_norm, config_.layer_norm_eps); + + return gb->add(hidden, pli_normed); +} + +size_t Gemma4Model::apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, + float rope_freq, size_t position_offset) { + if (rot_dim < head_dim) { + float adjusted_theta = std::pow(rope_freq, static_cast(rot_dim) / static_cast(head_dim)); + auto rot_part = gb->slice(tensor, 3, 0, rot_dim); + auto pass_part = gb->slice(tensor, 3, rot_dim, head_dim - rot_dim); + return gb->concat(gb->rope(rot_part, adjusted_theta, position_offset), pass_part, 3); + } + return gb->rope(tensor, rope_freq, position_offset); +} + +size_t Gemma4Model::build_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + size_t seq_len = gb->get_output_buffer(input).shape[0]; + int share_src = (layer_idx < kv_share_map_.size()) ? kv_share_map_[layer_idx] : -1; + + bool is_global = is_global_layer(layer_idx); + + size_t head_dim = is_global ? (config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2) : config_.attention_head_dim; + size_t num_heads = config_.attention_heads; + size_t kv_heads = is_global && config_.num_global_kv_heads > 0 ? config_.num_global_kv_heads : config_.attention_kv_heads; + float rope_freq = is_global ? config_.rope_theta : config_.rope_local_base_freq; + size_t window = is_global ? 0 : config_.sliding_window; + size_t rot_dim = static_cast(head_dim * (is_global ? config_.global_partial_rotary_factor : 1.0f)); + + auto q = gb->matmul(input, layer.attn_q_weight, true, backend); + q = gb->reshape(q, {seq_len * num_heads, head_dim}); + q = gb->rms_norm(q, layer.attn_q_norm_weight, config_.layer_norm_eps); + q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); + + size_t q4 = apply_partial_rope(gb, q, head_dim, rot_dim, rope_freq, position_offset); + + size_t k4, v4; + if (share_src >= 0 && shared_k_nodes_[share_src] != 0) { + k4 = shared_k_nodes_[share_src]; + v4 = shared_v_nodes_[share_src]; + } else { + auto k = gb->matmul(input, layer.attn_k_weight, true, backend); + k = gb->reshape(k, {seq_len * kv_heads, head_dim}); + k = gb->rms_norm(k, layer.attn_k_norm_weight, config_.layer_norm_eps); + k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); + + k4 = apply_partial_rope(gb, k, head_dim, rot_dim, rope_freq, position_offset); + + auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); + size_t v_ones = is_global ? v_norm_ones_global_node_ : v_norm_ones_node_; + auto v = gb->rms_norm(gb->reshape(v_proj, {seq_len * kv_heads, head_dim}), v_ones, config_.layer_norm_eps); + v4 = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); + + shared_k_nodes_[layer_idx] = k4; + shared_v_nodes_[layer_idx] = v4; + } + + if (use_cache && share_src < 0) { + cache_k_output_nodes_[layer_idx] = k4; + cache_v_output_nodes_[layer_idx] = v4; + } + + size_t cache_src = (share_src >= 0) ? static_cast(share_src) : layer_idx; + size_t attn; + if (use_cache && !kv_cache_.is_empty()) { + attn = gb->attention_int8_hybrid(q4, k4, v4, attention_scale_, position_offset, + kv_cache_.get_keys_int8(cache_src), kv_cache_.get_values_int8(cache_src), + kv_cache_.get_key_scales(cache_src), kv_cache_.get_value_scales(cache_src), + kv_cache_.current_seq_len, kv_heads, head_dim, window); + } else { + attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); + } + + return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), layer.attn_output_weight, true, backend); +} + +size_t Gemma4Model::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + auto gate = gb->gelu(gb->matmul(input, layer.ffn_gate_weight, true, backend)); + auto up = gb->matmul(input, layer.ffn_up_weight, true, backend); + return gb->matmul(gb->multiply(gate, up), layer.ffn_down_weight, true, backend); +} + +size_t Gemma4Model::build_moe(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto router_logits = gb->matmul(input, layer.router_proj, true, backend); + auto topk_result = gb->topk(router_logits, config_.num_experts_per_tok); + auto topk_indices = gb->index(topk_result, 0, 0); + auto routing_probs = gb->softmax(router_logits); + + return gb->moe_layer(input, routing_probs, topk_indices, + layer.moe_w1_experts, layer.moe_w3_experts, layer.moe_w2_experts, + config_.num_experts, config_.num_experts_per_tok, + true, 1e-6f, 1.0f, Activation::GELU, + layer.moe_per_expert_scale); +} + +size_t Gemma4Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto normed = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); + auto attn_raw = build_attention(gb, normed, layer_idx, backend, use_cache, position_offset); + auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm_weight, config_.layer_norm_eps); + auto residual = gb->add(hidden, attn); + + if (config_.enable_moe_block) { + auto h1 = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); + h1 = build_mlp(gb, h1, layer_idx, backend); + h1 = gb->rms_norm(h1, layer.post_ffn_norm_1, config_.layer_norm_eps); + + auto h2 = gb->multiply(residual, layer.router_scale); + h2 = gb->rms_norm(h2, layer.pre_ffn_norm_2, config_.layer_norm_eps); + h2 = build_moe(gb, h2, layer_idx, backend); + h2 = gb->rms_norm(h2, layer.post_ffn_norm_2, config_.layer_norm_eps); + + auto combined = gb->rms_norm(gb->add(h1, h2), layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); + return gb->add(residual, combined); + } + + auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); + auto mlp_raw = build_mlp(gb, pre_mlp, layer_idx, backend); + auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); + + return gb->add(residual, mlp); +} + +size_t Gemma4Model::apply_transformer_layer(CactusGraph* gb, size_t hidden, size_t pli, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t pos_offset) { + hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, pos_offset); + if (config_.hidden_size_per_layer_input > 0) + hidden = build_per_layer_input(gb, hidden, pli, layer_idx, backend); + if (weight_nodes_.layers[layer_idx].layer_scalar != 0) + hidden = gb->multiply(hidden, weight_nodes_.layers[layer_idx].layer_scalar); + return hidden; +} + +size_t Gemma4Model::build_pli_combined(CactusGraph* gb, size_t hidden, size_t pli_embed, + size_t seq_len, ComputeBackend backend) { + uint32_t num_layers = config_.num_layers; + uint32_t pli_dim = config_.hidden_size_per_layer_input; + + auto pli_proj = gb->scalar_multiply(gb->matmul(hidden, weight_nodes_.per_layer_model_proj, true, backend), + 1.0f / std::sqrt(static_cast(config_.hidden_dim))); + pli_proj = gb->reshape(pli_proj, {seq_len * num_layers, pli_dim}); + pli_proj = gb->rms_norm(pli_proj, weight_nodes_.per_layer_proj_norm, config_.layer_norm_eps); + pli_proj = gb->reshape(pli_proj, {seq_len, num_layers * pli_dim}); + return gb->scalar_multiply(gb->add(pli_proj, pli_embed), 1.0f / std::sqrt(2.0f)); +} + +std::pair Gemma4Model::build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, + size_t& token_input, size_t& pli_input) { + uint32_t pli_dim = config_.hidden_size_per_layer_input; + + token_input = gb->input({seq_len}, Precision::FP32); + auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), + std::sqrt(static_cast(config_.hidden_dim))); + + pli_input = gb->input({seq_len}, Precision::FP32); + auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), + std::sqrt(static_cast(pli_dim))); + auto pli_combined = build_pli_combined(gb, hidden, pli_embed, seq_len, backend); + + return {hidden, pli_combined}; +} + +void Gemma4Model::set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, + const std::vector& tokens) { + std::vector input_data(tokens.size()); + for (size_t i = 0; i < tokens.size(); i++) + input_data[i] = static_cast(tokens[i]); + gb->set_input(token_input, input_data.data(), Precision::FP32); + gb->set_input(pli_input, input_data.data(), Precision::FP32); +} + +size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, const std::vector& pli_tokens, + size_t seq_len, ComputeBackend backend, bool use_cache) { + return forward_from_embeddings(gb, hidden, hidden, pli_tokens, seq_len, backend, use_cache); +} + +size_t Gemma4Model::build_pli_combined_from_tokens(CactusGraph* gb, size_t hidden, + const std::vector& pli_tokens, + size_t seq_len, ComputeBackend backend) { + if (config_.hidden_size_per_layer_input == 0) + return 0; + + uint32_t pli_dim = config_.hidden_size_per_layer_input; + + auto pli_input = gb->input({seq_len}, Precision::FP32); + auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), + std::sqrt(static_cast(pli_dim))); + auto pli_combined = build_pli_combined(gb, hidden, pli_embed, seq_len, backend); + + std::vector pli_data(pli_tokens.size()); + for (size_t i = 0; i < pli_tokens.size(); i++) + pli_data[i] = static_cast(pli_tokens[i]); + gb->set_input(pli_input, pli_data.data(), Precision::FP32); + + return pli_combined; +} + +size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, size_t pli_hidden_source, + const std::vector& pli_tokens, size_t seq_len, + ComputeBackend backend, bool use_cache) { + size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + + std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); + std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); + + if (config_.hidden_size_per_layer_input == 0) { + for (uint32_t i = 0; i < config_.num_layers; i++) + hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); + return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + } + + auto pli_combined = build_pli_combined_from_tokens(gb, pli_hidden_source, pli_tokens, seq_len, backend); + + for (uint32_t i = 0; i < config_.num_layers; i++) + hidden = apply_transformer_layer(gb, hidden, pli_combined, i, backend, use_cache, pos_offset); + + return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); +} + +size_t Gemma4Model::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); + std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); + + size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + if (config_.hidden_size_per_layer_input == 0) { + auto token_input = gb->input({tokens.size()}, Precision::FP32); + auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), + std::sqrt(static_cast(config_.hidden_dim))); + + for (uint32_t i = 0; i < config_.num_layers; i++) + hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); + + std::vector input_data(tokens.size()); + for (size_t i = 0; i < tokens.size(); i++) + input_data[i] = static_cast(tokens[i]); + gb->set_input(token_input, input_data.data(), Precision::FP32); + return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + } + + size_t token_input, pli_input; + auto hidden_pli = build_preamble_and_embed(gb, tokens.size(), backend, token_input, pli_input); + size_t hidden = hidden_pli.first, pli = hidden_pli.second; + + for (uint32_t i = 0; i < config_.num_layers; i++) + hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); + + set_token_inputs(gb, token_input, pli_input, tokens); + return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); +} + +size_t Gemma4Model::forward_split(const std::vector& tokens, bool use_cache) { + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); + std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); + + size_t seq_len = tokens.size(); + size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + if (config_.hidden_size_per_layer_input == 0) { + auto token_input = gb->input({seq_len}, Precision::FP32); + auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), + std::sqrt(static_cast(config_.hidden_dim))); + + for (uint32_t i = 0; i < first_shared_layer_; i++) + hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); + + hidden = gb->index(hidden, seq_len - 1, 0); + hidden = gb->reshape(hidden, {1, config_.hidden_dim}); + + size_t shared_pos_offset = pos_offset + seq_len - 1; + for (uint32_t i = first_shared_layer_; i < config_.num_layers; i++) + hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, shared_pos_offset); + + std::vector input_data(tokens.size()); + for (size_t i = 0; i < tokens.size(); i++) + input_data[i] = static_cast(tokens[i]); + gb->set_input(token_input, input_data.data(), Precision::FP32); + return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + } + + size_t token_input, pli_input; + auto hidden_pli = build_preamble_and_embed(gb, seq_len, backend, token_input, pli_input); + size_t hidden = hidden_pli.first, pli = hidden_pli.second; + + for (uint32_t i = 0; i < first_shared_layer_; i++) + hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); + + hidden = gb->index(hidden, seq_len - 1, 0); + hidden = gb->reshape(hidden, {1, config_.hidden_dim}); + auto pli_last = gb->index(pli, seq_len - 1, 0); + pli_last = gb->reshape(pli_last, {1, config_.num_layers * config_.hidden_size_per_layer_input}); + + size_t shared_pos_offset = pos_offset + seq_len - 1; + for (uint32_t i = first_shared_layer_; i < config_.num_layers; i++) + hidden = apply_transformer_layer(gb, hidden, pli_last, i, backend, use_cache, shared_pos_offset); + + set_token_inputs(gb, token_input, pli_input, tokens); + return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); +} + + +void Gemma4Model::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { + if (tokens.empty()) + return; + + if (has_npu_prefill()) { + size_t npu_chunk_size = static_cast(npu_prefill_->get_chunk_size()); + if (tokens.size() > npu_chunk_size) { + Model::prefill(tokens, chunk_size, profile_file); + return; + } + } + + static constexpr size_t SPLIT_PREFILL_MIN_TOKENS = 32; + bool use_split = config_.num_kv_shared_layers > 0 + && tokens.size() >= SPLIT_PREFILL_MIN_TOKENS + && !std::getenv("CACTUS_DISABLE_SPLIT_PREFILL"); + + if (!use_split) { + Model::prefill(tokens, chunk_size, profile_file); + return; + } + + auto* gb = static_cast(graph_handle_); + auto process_chunk = [&](const std::vector& chunk) { + forward_split(chunk, true); + gb->execute(profile_file); + update_kv_cache(gb, chunk.size()); + }; + + if (tokens.size() <= chunk_size) { + process_chunk(tokens); + return; + } + + size_t num_full_chunks = (tokens.size() - 1) / chunk_size; + for (size_t i = 0; i < num_full_chunks; ++i) { + size_t start = i * chunk_size; + std::vector chunk(tokens.begin() + start, tokens.begin() + start + chunk_size); + if (i == 1) + gb->set_prefill_mode(true); + process_chunk(chunk); + } + + gb->set_prefill_mode(false); + size_t final_start = num_full_chunks * chunk_size; + process_chunk(std::vector(tokens.begin() + final_start, tokens.end())); +} + +} +} diff --git a/cactus-transpile/models/gemma4/model_gemma4.h b/cactus-transpile/models/gemma4/model_gemma4.h new file mode 100644 index 000000000..2e061917e --- /dev/null +++ b/cactus-transpile/models/gemma4/model_gemma4.h @@ -0,0 +1,404 @@ +#pragma once + +#include "../model.h" + +bool test_gemma4_vision(bool expect_npu); +bool test_gemma4_audio(bool expect_npu); + +namespace cactus { +namespace engine { + +class Gemma4Model : public Model { + friend class Gemma4MmModel; +public: + Gemma4Model(); + explicit Gemma4Model(const Config& config); + ~Gemma4Model() override = default; + +protected: + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_moe(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend) const; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; + void load_weights_to_graph(CactusGraph* gb) override; + void post_init() override; + std::vector get_kv_layer_dims() const override; + std::vector get_kv_layer_heads() const override; + void compact_kv_cache() override; + + size_t forward_from_embeddings(CactusGraph* gb, size_t hidden, const std::vector& pli_tokens, + size_t seq_len, ComputeBackend backend, bool use_cache); + size_t forward_from_embeddings(CactusGraph* gb, size_t hidden, size_t pli_hidden_source, + const std::vector& pli_tokens, size_t seq_len, + ComputeBackend backend, bool use_cache); + size_t build_pli_combined_from_tokens(CactusGraph* gb, size_t hidden, + const std::vector& pli_tokens, + size_t seq_len, ComputeBackend backend); + +private: + size_t forward_split(const std::vector& tokens, bool use_cache); + + std::pair build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, + size_t& token_input, size_t& pli_input); + + void set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, + const std::vector& tokens); + + size_t build_pli_combined(CactusGraph* gb, size_t hidden, size_t pli_embed, + size_t seq_len, ComputeBackend backend); + + size_t build_per_layer_input(CactusGraph* gb, size_t hidden, size_t pli_combined, uint32_t layer_idx, + ComputeBackend backend) const; + + bool is_global_layer(uint32_t idx) const; + size_t apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, + float rope_freq, size_t position_offset); + size_t apply_transformer_layer(CactusGraph* gb, size_t hidden, size_t pli, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t pos_offset); + + struct WeightNodeIDs { + size_t output_weight; + size_t output_norm_weight; + + size_t embed_tokens_per_layer; + size_t per_layer_model_proj; + size_t per_layer_proj_norm; + + struct LayerWeights { + size_t attn_q_weight; + size_t attn_k_weight; + size_t attn_v_weight; + size_t attn_output_weight; + size_t input_layernorm_weight; + size_t attn_q_norm_weight; + size_t attn_k_norm_weight; + size_t pre_feedforward_layernorm_weight; + size_t post_feedforward_layernorm_weight; + size_t ffn_gate_weight; + size_t ffn_up_weight; + size_t ffn_down_weight; + size_t post_attention_layernorm_weight; + size_t per_layer_gate; + size_t per_layer_proj; + size_t post_per_layer_norm; + size_t layer_scalar; + + std::vector moe_w1_experts; + std::vector moe_w3_experts; + std::vector moe_w2_experts; + size_t moe_per_expert_scale = 0; + size_t router_proj = 0; + size_t router_scale = 0; + size_t post_ffn_norm_1 = 0; + size_t pre_ffn_norm_2 = 0; + size_t post_ffn_norm_2 = 0; + }; + + std::vector layers; + } weight_nodes_; + + uint32_t first_shared_layer_ = 0; + std::vector kv_share_map_; + std::vector shared_k_nodes_; + std::vector shared_v_nodes_; + + std::vector<__fp16> v_norm_ones_weight_; + size_t v_norm_ones_node_ = 0; + size_t v_norm_ones_global_node_ = 0; +}; + +class Gemma4VisionModel : public Model { + friend class Gemma4MmModel; +public: + struct PreprocessedImage { + std::vector pixel_values; + size_t height; + size_t width; + size_t patch_height; + size_t patch_width; + size_t num_patches; + }; + + Gemma4VisionModel(); + explicit Gemma4VisionModel(const Config& config); + ~Gemma4VisionModel() override = default; + + PreprocessedImage preprocess_image(const std::string& image_path); + size_t forward_vision(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend); + size_t build_vision_projector(CactusGraph* gb, size_t vision_features, ComputeBackend backend); + +protected: + size_t forward(const std::vector&, bool) override { + throw std::runtime_error("Gemma4VisionModel: use forward_vision() instead"); + } + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Gemma4VisionModel: build_attention unused"); + } + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("Gemma4VisionModel: build_mlp unused"); + } + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Gemma4VisionModel: build_transformer_block unused"); + } + void load_weights_to_graph(CactusGraph* gb) override; + +private: + size_t build_vision_patch_embedding(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend); + size_t build_vision_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, + size_t cos_node, size_t sin_node, + size_t attn_mask_node, ComputeBackend backend); + size_t build_vision_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend); + size_t build_vision_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + size_t cos_node, size_t sin_node, + size_t attn_mask_node, ComputeBackend backend); + size_t build_vision_pooler(CactusGraph* gb, size_t hidden, const PreprocessedImage& img, ComputeBackend backend); + std::pair build_2d_rope_nodes(CactusGraph* gb, const PreprocessedImage& img, size_t max_patches); + size_t build_padding_mask(CactusGraph* gb, size_t num_real, size_t max_patches); + + struct VisionWeightNodes { + size_t patch_input_proj = 0; + size_t position_table = 0; + + struct LayerWeights { + size_t attn_q_weight = 0; + size_t attn_k_weight = 0; + size_t attn_v_weight = 0; + size_t attn_output_weight = 0; + size_t attn_q_norm = 0; + size_t attn_k_norm = 0; + size_t input_layernorm = 0; + size_t post_attention_layernorm = 0; + size_t pre_feedforward_layernorm = 0; + size_t post_feedforward_layernorm = 0; + size_t mlp_gate_proj = 0; + size_t mlp_up_proj = 0; + size_t mlp_down_proj = 0; + size_t layer_scalar = 0; + }; + + std::vector layers; + size_t embed_vision_proj = 0; + size_t post_proj_norm = 0; + } vision_weights_; + + std::vector<__fp16> vision_v_norm_ones_; + size_t vision_v_norm_ones_node_ = 0; + + std::unique_ptr npu_encoder_; + bool use_npu_encoder_ = false; + bool disable_npu_ = false; + + friend bool ::test_gemma4_vision(bool); +}; + +class Gemma4AudioModel : public Model { + friend class Gemma4MmModel; +public: + Gemma4AudioModel(); + explicit Gemma4AudioModel(const Config& config); + ~Gemma4AudioModel() override = default; + + struct ConformerContext { + size_t timing_fp16 = 0; + size_t front_pad = 0; + size_t back_pad = 0; + size_t seq_len = 0; + }; + + size_t forward_audio(CactusGraph* gb, const std::vector& mel_features, + size_t num_frames, ComputeBackend backend); + + size_t build_audio_projector(CactusGraph* gb, size_t audio_features, ComputeBackend backend); + + size_t build_sscp(CactusGraph* gb, const std::vector& mel_features, + size_t num_frames, ComputeBackend backend); + + ConformerContext build_conformer_context(CactusGraph* gb, size_t sscp_output); + + size_t build_conformer_ffw(CactusGraph* gb, size_t input, uint32_t layer_idx, + bool is_end, ComputeBackend backend); + size_t build_conformer_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, + const ConformerContext& ctx, ComputeBackend backend); + size_t build_conformer_lconv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend); + size_t build_conformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + const ConformerContext& ctx, ComputeBackend backend); + + struct AudioWeightNodes { + size_t sscp_conv0_weight = 0; + size_t sscp_conv0_norm = 0; + size_t sscp_conv1_weight = 0; + size_t sscp_conv1_norm = 0; + size_t sscp_input_proj = 0; + + struct ClipBounds { + float in_min = -1e10f, in_max = 1e10f; + float out_min = -1e10f, out_max = 1e10f; + }; + + struct ConformerLayerWeights { + size_t ffw_start_1 = 0, ffw_start_2 = 0; + ClipBounds ffw_start_1_clip, ffw_start_2_clip; + size_t ffw_start_pre_norm = 0, ffw_start_post_norm = 0; + size_t attn_q = 0, attn_k = 0, attn_v = 0; + ClipBounds attn_q_clip, attn_k_clip, attn_v_clip; + size_t attn_per_dim_scale = 0; + size_t attn_rel_pos_proj = 0; + size_t attn_post = 0; + ClipBounds attn_post_clip; + size_t attn_pre_norm = 0, attn_post_norm = 0; + size_t lconv_start = 0, lconv_depthwise = 0, lconv_end = 0; + ClipBounds lconv_start_clip, lconv_end_clip; + size_t lconv_pre_norm = 0, lconv_conv_norm = 0; + size_t ffw_end_1 = 0, ffw_end_2 = 0; + ClipBounds ffw_end_1_clip, ffw_end_2_clip; + size_t ffw_end_pre_norm = 0, ffw_end_post_norm = 0; + size_t block_norm = 0; + }; + std::vector layers; + + size_t output_proj = 0; + size_t output_proj_bias = 0; + + size_t embed_audio_proj = 0; + } audio_weights_; + + std::vector<__fp16> audio_proj_norm_ones_; + size_t audio_proj_norm_ones_node_ = 0; + + std::unique_ptr npu_encoder_; + bool use_npu_encoder_ = false; + bool disable_npu_ = false; + std::vector<__fp16> npu_audio_input_scratch_; + std::vector<__fp16> npu_audio_output_scratch_; + std::vector<__fp16> npu_audio_reorder_scratch_; + + friend bool ::test_gemma4_audio(bool); + +protected: + size_t forward(const std::vector&, bool) override { + throw std::runtime_error("Gemma4AudioModel: use forward_audio() instead"); + } + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Gemma4AudioModel: build_attention unused"); + } + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("Gemma4AudioModel: build_mlp unused"); + } + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Gemma4AudioModel: build_transformer_block unused"); + } + void load_weights_to_graph(CactusGraph* gb) override; +}; + +class Gemma4MmModel : public Model { +public: + Gemma4MmModel(); + explicit Gemma4MmModel(const Config& config); + ~Gemma4MmModel() override = default; + + bool init(const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + + uint32_t decode(const std::vector& tokens, + float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, + const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f) override; + + void prefill(const std::vector& tokens, size_t chunk_size = 256, + const std::string& profile_file = "") override; + + void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file = "") override; + + uint32_t decode_with_images( + const std::vector& tokens, + const std::vector& image_paths, + float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, + const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f) override; + + uint32_t decode_with_audio( + const std::vector& tokens, + const std::vector& audio_features, + float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, + const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; + + void reset_cache() override; + std::vector get_image_embeddings(const std::string& image_path) override; + std::vector get_audio_embeddings(const std::vector& audio_features) override; + void compact_kv_cache() override; + void remove_thinking_tokens(const std::vector>& ranges) override; + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override; + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; + void load_weights_to_graph(CactusGraph* gb) override; + +private: + struct ForwardResult { + size_t final_hidden_node; + size_t seq_len; + }; + + ForwardResult forward_multimodal(CactusGraph* gb, const std::vector& tokens, + const std::vector& image_paths, + const std::vector* audio_features, + size_t audio_num_frames, + ComputeBackend backend, bool use_cache); + + uint32_t decode_multimodal(const std::vector& tokens, + const std::vector& image_paths, + const std::vector* audio_features, + size_t audio_num_frames, + float temperature, float top_p, size_t top_k, + const std::string& profile_file, float* out_entropy, + float min_p, float repetition_penalty); + +public: + struct MultimodalInputs { + size_t hidden_node = 0; + size_t pli_hidden_source_node = 0; + std::vector pli_tokens; + size_t seq_len = 0; + }; + + const Gemma4VisionModel& vision_encoder() const { return vision_encoder_; } + Gemma4VisionModel& vision_encoder() { return vision_encoder_; } + const Gemma4AudioModel& audio_encoder() const { return audio_encoder_; } + Gemma4AudioModel& audio_encoder() { return audio_encoder_; } + const Gemma4Model& language_model() const { return language_model_; } + Gemma4Model& language_model() { return language_model_; } + MultimodalInputs build_multimodal_inputs( + CactusGraph* gb, const std::vector& tokens, + const std::vector& image_paths, + const std::vector* audio_features, + size_t audio_num_frames, + ComputeBackend backend); + +private: + Gemma4VisionModel vision_encoder_; + Gemma4AudioModel audio_encoder_; + Gemma4Model language_model_; + + bool prefill_completed_ = false; + size_t last_token_count_ = 0; +}; + +} +} diff --git a/cactus-transpile/models/gemma4/model_gemma4_audio.cpp b/cactus-transpile/models/gemma4/model_gemma4_audio.cpp new file mode 100644 index 000000000..c1829558c --- /dev/null +++ b/cactus-transpile/models/gemma4/model_gemma4_audio.cpp @@ -0,0 +1,777 @@ +#include "model_gemma4.h" +#include "../../graph/graph.h" +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +static const float INV_LN2 = 1.0f / std::log(2.0f); +static const float GEMMA4_AUDIO_K_SCALE = std::log(1.0f + std::exp(1.0f)) / std::log(2.0f); + +static size_t shape_elements(const std::vector& shape) { + if (shape.empty()) return 0; + size_t total = 1; + for (int d : shape) { + if (d <= 0) return 0; + total *= static_cast(d); + } + return total; +} + +static size_t infer_npu_audio_max_frames(const std::vector& shape, size_t mel_bins) { + if (shape.size() == 2) { + size_t s0 = static_cast(shape[0]); + size_t s1 = static_cast(shape[1]); + if (s1 == mel_bins) return s0; + if (s0 == mel_bins) return s1; + } else if (shape.size() == 3) { + size_t s0 = static_cast(shape[0]); + size_t s1 = static_cast(shape[1]); + size_t s2 = static_cast(shape[2]); + if (s0 == 1 && s2 == mel_bins) return s1; // [1, T, F] + if (s0 == 1 && s1 == mel_bins) return s2; // [1, F, T] + if (s1 == mel_bins && s2 == 1) return s0; // [T, F, 1] + if (s0 == mel_bins && s2 == 1) return s1; // [F, T, 1] + } else if (shape.size() == 4) { + size_t s0 = static_cast(shape[0]); + size_t s1 = static_cast(shape[1]); + size_t s2 = static_cast(shape[2]); + size_t s3 = static_cast(shape[3]); + if (s0 == 1 && s1 == 1 && s3 == mel_bins) return s2; // [1, 1, T, F] + if (s0 == 1 && s1 == 1 && s2 == mel_bins) return s3; // [1, 1, F, T] + if (s0 == 1 && s1 == mel_bins && s3 == 1) return s2; // [1, F, T, 1] + if (s0 == 1 && s2 == mel_bins && s3 == 1) return s1; // [1, T, F, 1] + } + return 0; +} + +static bool pack_gemma4_audio_for_npu(const std::vector& mel_features, + size_t frames, + size_t mel_bins, + const std::vector& input_shape, + std::vector<__fp16>& packed) { + if (input_shape.empty()) return false; + const size_t total = shape_elements(input_shape); + if (total == 0) return false; + packed.resize(total); + std::fill(packed.begin(), packed.end(), static_cast<__fp16>(0.0f)); + + auto tm = [&](size_t t, size_t m) -> __fp16 { + return static_cast<__fp16>(mel_features[t * mel_bins + m]); + }; + + if (input_shape.size() == 4) { + const size_t s0 = static_cast(input_shape[0]); + const size_t s1 = static_cast(input_shape[1]); + const size_t s2 = static_cast(input_shape[2]); + const size_t s3 = static_cast(input_shape[3]); + if (s0 != 1) return false; + + if (s1 == 1 && s2 >= frames && s3 == mel_bins) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[(t * s3) + m] = tm(t, m); + return true; + } + if (s1 == 1 && s2 == mel_bins && s3 >= frames) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[(m * s3) + t] = tm(t, m); + return true; + } + if (s1 >= frames && s2 == mel_bins && s3 == 1) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[((t * s2 + m) * s3)] = tm(t, m); + return true; + } + if (s1 == mel_bins && s2 >= frames && s3 == 1) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[((m * s2 + t) * s3)] = tm(t, m); + return true; + } + return false; + } + + if (input_shape.size() == 3) { + const size_t s0 = static_cast(input_shape[0]); + const size_t s1 = static_cast(input_shape[1]); + const size_t s2 = static_cast(input_shape[2]); + + if (s0 == 1 && s1 >= frames && s2 == mel_bins) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[t * s2 + m] = tm(t, m); + return true; + } + if (s0 == 1 && s1 == mel_bins && s2 >= frames) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[m * s2 + t] = tm(t, m); + return true; + } + if (s0 >= frames && s1 == mel_bins && s2 == 1) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[(t * s1 + m) * s2] = tm(t, m); + return true; + } + if (s0 == mel_bins && s1 >= frames && s2 == 1) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[(m * s1 + t) * s2] = tm(t, m); + return true; + } + return false; + } + + if (input_shape.size() == 2) { + const size_t s0 = static_cast(input_shape[0]); + const size_t s1 = static_cast(input_shape[1]); + + if (s0 >= frames && s1 == mel_bins) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[t * s1 + m] = tm(t, m); + return true; + } + if (s0 == mel_bins && s1 >= frames) { + for (size_t t = 0; t < frames; ++t) + for (size_t m = 0; m < mel_bins; ++m) + packed[m * s1 + t] = tm(t, m); + return true; + } + return false; + } + + return false; +} + +struct NPUAudioOutputLayout { + std::vector dims; + size_t hidden_axis = SIZE_MAX; + size_t hidden_dim = 0; + size_t time_steps = 0; +}; + +static bool infer_npu_audio_output_layout(const std::vector& output_shape, + size_t elements_written, + size_t expected_hidden_dim, + NPUAudioOutputLayout& layout) { + if (elements_written == 0) return false; + + layout = NPUAudioOutputLayout{}; + for (int d : output_shape) { + if (d > 0) layout.dims.push_back(static_cast(d)); + } + + if (layout.dims.empty()) { + if (expected_hidden_dim == 0 || (elements_written % expected_hidden_dim) != 0) { + return false; + } + layout.dims = {elements_written / expected_hidden_dim, expected_hidden_dim}; + } + + const size_t rank = layout.dims.size(); + size_t hidden_axis = SIZE_MAX; + if (expected_hidden_dim > 0) { + if (layout.dims[rank - 1] == expected_hidden_dim) { + hidden_axis = rank - 1; + } else if (rank >= 2 && layout.dims[rank - 2] == expected_hidden_dim) { + hidden_axis = rank - 2; + } else { + for (size_t i = 0; i < rank; ++i) { + if (layout.dims[i] == expected_hidden_dim) { + hidden_axis = i; + break; + } + } + } + } + + if (hidden_axis == SIZE_MAX) { + hidden_axis = rank - 1; + } + + const size_t hidden_dim = layout.dims[hidden_axis]; + if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) { + return false; + } + + layout.hidden_axis = hidden_axis; + layout.hidden_dim = hidden_dim; + layout.time_steps = elements_written / hidden_dim; + return layout.time_steps > 0; +} + +static bool materialize_npu_audio_time_major(const __fp16* src, + const NPUAudioOutputLayout& layout, + size_t time_steps, + std::vector<__fp16>& dst) { + if (!src || layout.dims.empty() || layout.hidden_dim == 0 || layout.time_steps == 0) { + return false; + } + if (layout.hidden_axis >= layout.dims.size() || time_steps == 0 || time_steps > layout.time_steps) { + return false; + } + + const size_t rank = layout.dims.size(); + if (layout.hidden_axis == rank - 1) { + // Already [time, hidden] in row-major flattening. + return false; + } + + std::vector strides(rank, 1); + for (size_t i = rank; i-- > 1;) { + strides[i - 1] = strides[i] * layout.dims[i]; + } + + std::vector non_hidden_axes; + non_hidden_axes.reserve(rank - 1); + for (size_t i = 0; i < rank; ++i) { + if (i != layout.hidden_axis) non_hidden_axes.push_back(i); + } + + dst.assign(time_steps * layout.hidden_dim, static_cast<__fp16>(0.0f)); + + for (size_t t = 0; t < time_steps; ++t) { + size_t rem = t; + size_t base_offset = 0; + for (size_t i = non_hidden_axes.size(); i-- > 0;) { + size_t axis = non_hidden_axes[i]; + size_t dim = layout.dims[axis]; + size_t coord = rem % dim; + rem /= dim; + base_offset += coord * strides[axis]; + } + + for (size_t h = 0; h < layout.hidden_dim; ++h) { + size_t src_index = base_offset + h * strides[layout.hidden_axis]; + dst[t * layout.hidden_dim + h] = src[src_index]; + } + } + + return true; +} + +static size_t graph_clamp(CactusGraph* gb, size_t x, float lo, float hi) { + if (lo >= hi || std::isinf(lo) || std::isinf(hi)) return x; + x = gb->scalar_add(x, -lo); + x = gb->relu(x); + x = gb->scalar_add(x, lo); + x = gb->scalar_multiply(gb->scalar_add(x, -hi), -1.0f); + x = gb->relu(x); + x = gb->scalar_add(gb->scalar_multiply(x, -1.0f), hi); + return x; +} + +static size_t clipped_matmul(CactusGraph* gb, size_t input, size_t weight, + float in_min, float in_max, float out_min, float out_max, + ComputeBackend backend) { + size_t clamped_in = graph_clamp(gb, input, in_min, in_max); + size_t result = gb->matmul(clamped_in, weight, true, backend); + return graph_clamp(gb, result, out_min, out_max); +} + +static float read_scalar_weight(CactusGraph* gb, const std::string& path) { + size_t node = gb->mmap_weights(path); + const auto& buf = gb->get_output_buffer(node); + if (buf.precision == Precision::FP16) + return static_cast(*buf.data_as<__fp16>()); + if (buf.precision == Precision::FP32) + return *buf.data_as(); + return 0.0f; +} + +static Gemma4AudioModel::AudioWeightNodes::ClipBounds load_clip_bounds(CactusGraph* gb, const std::string& prefix) { + Gemma4AudioModel::AudioWeightNodes::ClipBounds cb{0, 0, 0, 0}; + std::string in_min_path = prefix + "_input_min.weights"; + if (!std::filesystem::exists(in_min_path)) + return cb; + cb.in_min = read_scalar_weight(gb, in_min_path); + cb.in_max = read_scalar_weight(gb, prefix + "_input_max.weights"); + cb.out_min = read_scalar_weight(gb, prefix + "_output_min.weights"); + cb.out_max = read_scalar_weight(gb, prefix + "_output_max.weights"); + return cb; +} + +static void get_conv2d_output_hw(const CactusGraph* gb, size_t node, size_t& h, size_t& w) { + const auto& buf = gb->get_output_buffer(node); + h = buf.shape[2]; + w = buf.shape[3]; +} + +Gemma4AudioModel::Gemma4AudioModel() : Model() {} +Gemma4AudioModel::Gemma4AudioModel(const Config& config) : Model(config) {} + +void Gemma4AudioModel::load_weights_to_graph(CactusGraph* gb) { + auto resolve = [&](const std::string& name) -> std::string { + return model_folder_path_ + "/" + name; + }; + auto resolve_existing = [&](const std::string& preferred, + const std::string& legacy = std::string()) -> std::string { + std::string preferred_path = resolve(preferred); + if (std::filesystem::exists(preferred_path)) { + return preferred_path; + } + if (!legacy.empty()) { + std::string legacy_path = resolve(legacy); + if (std::filesystem::exists(legacy_path)) { + return legacy_path; + } + } + return preferred_path; + }; + auto load_weight = [&](const std::string& preferred, + const std::string& legacy = std::string()) -> size_t { + return gb->mmap_weights(resolve_existing(preferred, legacy)); + }; + auto load_clip_from_weight = [&](const std::string& preferred, + const std::string& legacy = std::string()) + -> Gemma4AudioModel::AudioWeightNodes::ClipBounds { + std::string weight_path = resolve_existing(preferred, legacy); + static const std::string suffix = ".weights"; + if (weight_path.size() > suffix.size() && + weight_path.compare(weight_path.size() - suffix.size(), suffix.size(), suffix) == 0) { + return load_clip_bounds(gb, weight_path.substr(0, weight_path.size() - suffix.size())); + } + return Gemma4AudioModel::AudioWeightNodes::ClipBounds{0, 0, 0, 0}; + }; + + if (!disable_npu_ && npu::is_npu_available()) { + std::string npu_path = model_folder_path_ + "/audio_encoder.mlpackage"; + if (std::filesystem::exists(npu_path)) { + npu_encoder_ = npu::create_encoder(); + if (npu_encoder_ && npu_encoder_->load(npu_path)) { + use_npu_encoder_ = true; + std::vector input_shape = npu_encoder_->get_input_shape(); + npu_encoder_->preallocate(input_shape, "mel", ""); + } else { + use_npu_encoder_ = false; + npu_encoder_.reset(); + CACTUS_LOG_WARN("npu", "[gemma4-audio] found audio_encoder.mlpackage but failed to enable NPU audio encoder; using CPU"); + } + } else { + CACTUS_LOG_WARN("npu", "[gemma4-audio] audio_encoder.mlpackage not found; using CPU audio encoder"); + } + } else if (!disable_npu_) { + CACTUS_LOG_WARN("npu", "[gemma4-audio] NPU backend unavailable on this device; using CPU audio encoder"); + } + + if (!use_npu_encoder_) { + audio_weights_.sscp_conv0_weight = load_weight( + "audio_subsample_conv_projection_conv_0_conv.weights", + "audio_subsample_conv_projection_layer0_conv.weights"); + audio_weights_.sscp_conv0_norm = load_weight( + "audio_subsample_conv_projection_conv_0_norm.weights", + "audio_subsample_conv_projection_layer0_norm.weights"); + audio_weights_.sscp_conv1_weight = load_weight( + "audio_subsample_conv_projection_conv_1_conv.weights", + "audio_subsample_conv_projection_layer1_conv.weights"); + audio_weights_.sscp_conv1_norm = load_weight( + "audio_subsample_conv_projection_conv_1_norm.weights", + "audio_subsample_conv_projection_layer1_norm.weights"); + audio_weights_.sscp_input_proj = load_weight( + "audio_subsample_conv_projection_input_proj.weights", + "audio_subsample_conv_projection_input_proj_linear.weights"); + + audio_weights_.layers.resize(config_.audio_num_layers); + for (uint32_t i = 0; i < config_.audio_num_layers; i++) { + auto& layer = audio_weights_.layers[i]; + + auto layer_weight = [&](const std::string& preferred_suffix, + const std::string& legacy_suffix = std::string()) -> size_t { + std::string legacy = legacy_suffix.empty() ? preferred_suffix : legacy_suffix; + return load_weight( + "audio_conformer_" + std::to_string(i) + "_" + preferred_suffix, + "audio_layers_" + std::to_string(i) + "_" + legacy); + }; + auto layer_clip = [&](const std::string& preferred_suffix, + const std::string& legacy_suffix = std::string()) + -> Gemma4AudioModel::AudioWeightNodes::ClipBounds { + std::string legacy = legacy_suffix.empty() ? preferred_suffix : legacy_suffix; + return load_clip_from_weight( + "audio_conformer_" + std::to_string(i) + "_" + preferred_suffix, + "audio_layers_" + std::to_string(i) + "_" + legacy); + }; + + layer.ffw_start_1 = layer_weight("ffw_layer_start_ffw_layer_1.weights", "feed_forward1_ffw_layer_1.weights"); + layer.ffw_start_2 = layer_weight("ffw_layer_start_ffw_layer_2.weights", "feed_forward1_ffw_layer_2.weights"); + layer.ffw_start_1_clip = layer_clip("ffw_layer_start_ffw_layer_1.weights", "feed_forward1_ffw_layer_1.weights"); + layer.ffw_start_2_clip = layer_clip("ffw_layer_start_ffw_layer_2.weights", "feed_forward1_ffw_layer_2.weights"); + layer.ffw_start_pre_norm = layer_weight("ffw_layer_start_pre_layer_norm.weights", "feed_forward1_pre_layer_norm.weights"); + layer.ffw_start_post_norm = layer_weight("ffw_layer_start_post_layer_norm.weights", "feed_forward1_post_layer_norm.weights"); + + layer.attn_q = layer_weight("attention_attn_q_proj.weights", "self_attn_q_proj.weights"); + layer.attn_k = layer_weight("attention_attn_k_proj.weights", "self_attn_k_proj.weights"); + layer.attn_v = layer_weight("attention_attn_v_proj.weights", "self_attn_v_proj.weights"); + layer.attn_q_clip = layer_clip("attention_attn_q_proj.weights", "self_attn_q_proj.weights"); + layer.attn_k_clip = layer_clip("attention_attn_k_proj.weights", "self_attn_k_proj.weights"); + layer.attn_v_clip = layer_clip("attention_attn_v_proj.weights", "self_attn_v_proj.weights"); + layer.attn_per_dim_scale = layer_weight("attention_attn_per_dim_scale.weights", "self_attn_per_dim_scale.weights"); + layer.attn_rel_pos_proj = layer_weight( + "attention_attn_relative_position_embedding_pos_proj.weights", + "self_attn_relative_k_proj.weights"); + layer.attn_post = layer_weight("attention_post.weights", "self_attn_post.weights"); + layer.attn_post_clip = layer_clip("attention_post.weights", "self_attn_post.weights"); + layer.attn_pre_norm = layer_weight("attention_pre_attn_norm.weights", "norm_pre_attn.weights"); + layer.attn_post_norm = layer_weight("attention_post_norm.weights", "norm_post_attn.weights"); + + layer.lconv_start = layer_weight("lconv1d_linear_start.weights"); + layer.lconv_depthwise = layer_weight("lconv1d_depthwise_conv1d.weights"); + layer.lconv_end = layer_weight("lconv1d_linear_end.weights"); + layer.lconv_start_clip = layer_clip("lconv1d_linear_start.weights"); + layer.lconv_end_clip = layer_clip("lconv1d_linear_end.weights"); + layer.lconv_pre_norm = layer_weight("lconv1d_pre_layer_norm.weights"); + layer.lconv_conv_norm = layer_weight("lconv1d_conv_norm.weights"); + + layer.ffw_end_1 = layer_weight("ffw_layer_end_ffw_layer_1.weights", "feed_forward2_ffw_layer_1.weights"); + layer.ffw_end_2 = layer_weight("ffw_layer_end_ffw_layer_2.weights", "feed_forward2_ffw_layer_2.weights"); + layer.ffw_end_1_clip = layer_clip("ffw_layer_end_ffw_layer_1.weights", "feed_forward2_ffw_layer_1.weights"); + layer.ffw_end_2_clip = layer_clip("ffw_layer_end_ffw_layer_2.weights", "feed_forward2_ffw_layer_2.weights"); + layer.ffw_end_pre_norm = layer_weight("ffw_layer_end_pre_layer_norm.weights", "feed_forward2_pre_layer_norm.weights"); + layer.ffw_end_post_norm = layer_weight("ffw_layer_end_post_layer_norm.weights", "feed_forward2_post_layer_norm.weights"); + + layer.block_norm = layer_weight("norm.weights", "norm_out.weights"); + } + + audio_weights_.output_proj = load_weight("audio_output_proj.weights"); + audio_weights_.output_proj_bias = load_weight("audio_output_proj.bias"); + } + audio_weights_.embed_audio_proj = load_weight("embed_audio_proj.weights"); + + size_t proj_out_dim = config_.audio_output_proj_dims > 0 + ? static_cast(config_.audio_output_proj_dims) + : static_cast(config_.audio_hidden_dim); + audio_proj_norm_ones_.assign(proj_out_dim, static_cast<__fp16>(1.0f)); + audio_proj_norm_ones_node_ = gb->input({proj_out_dim}, Precision::FP16); + gb->set_external_input(audio_proj_norm_ones_node_, audio_proj_norm_ones_.data(), Precision::FP16); + + output_weight_node_id_ = 0; +} + +size_t Gemma4AudioModel::build_sscp(CactusGraph* gb, const std::vector& mel_features, + size_t num_frames, ComputeBackend backend) { + size_t mel_bins = config_.audio_input_feat_size; + size_t conv0_ch = config_.audio_sscp_conv0_channels; + size_t conv1_ch = config_.audio_sscp_conv1_channels; + + size_t mel_input = gb->input({num_frames, mel_bins}, Precision::FP32); + gb->set_input(mel_input, mel_features.data(), Precision::FP32); + size_t mel_fp16 = gb->precision_cast(mel_input, Precision::FP16); + + size_t x = gb->reshape(mel_fp16, {1, 1, num_frames, mel_bins}); + + x = gb->conv2d_k3s2p1(x, audio_weights_.sscp_conv0_weight); + size_t h1, w1; + get_conv2d_output_hw(gb, x, h1, w1); + + x = gb->reshape(x, {conv0_ch, h1, w1}); + x = gb->transposeN(x, {1, 2, 0}); + x = gb->reshape(x, {h1 * w1, conv0_ch}); + x = gb->layernorm(x, audio_weights_.sscp_conv0_norm, config_.audio_sscp_conv_eps); + x = gb->reshape(x, {h1, w1, conv0_ch}); + x = gb->transposeN(x, {2, 0, 1}); + x = gb->relu(x); + x = gb->reshape(x, {1, conv0_ch, h1, w1}); + + x = gb->conv2d_k3s2p1(x, audio_weights_.sscp_conv1_weight); + size_t h2, w2; + get_conv2d_output_hw(gb, x, h2, w2); + + x = gb->reshape(x, {conv1_ch, h2, w2}); + x = gb->transposeN(x, {1, 2, 0}); + x = gb->reshape(x, {h2 * w2, conv1_ch}); + x = gb->layernorm(x, audio_weights_.sscp_conv1_norm, config_.audio_sscp_conv_eps); + x = gb->reshape(x, {h2, w2, conv1_ch}); + x = gb->transposeN(x, {2, 0, 1}); + x = gb->relu(x); + + x = gb->transposeN(x, {1, 2, 0}); + x = gb->reshape(x, {h2, w2 * conv1_ch}); + + return gb->matmul(x, audio_weights_.sscp_input_proj, true, backend); +} + +static std::vector compute_timing_signal(size_t num_positions, size_t hidden_dim, size_t max_past) { + size_t num_timescales = hidden_dim / 2; + float log_ts_inc = std::log(1.0e4f) / std::max(static_cast(num_timescales) - 1, 1); + + std::vector signal(num_positions * hidden_dim); + for (size_t p = 0; p < num_positions; p++) { + float pos = static_cast(max_past) - static_cast(p); + for (size_t i = 0; i < num_timescales; i++) { + float scaled = pos * std::exp(-static_cast(i) * log_ts_inc); + signal[p * hidden_dim + i] = std::sin(scaled); + signal[p * hidden_dim + num_timescales + i] = std::cos(scaled); + } + } + return signal; +} + +static size_t create_zero_pad_node(CactusGraph* gb, size_t time_len, size_t num_heads, size_t head_dim) { + if (time_len == 0) return 0; + std::vector<__fp16> zeros(time_len * num_heads * head_dim, static_cast<__fp16>(0.0f)); + size_t node = gb->input({time_len, num_heads, head_dim}, Precision::FP16); + gb->set_input(node, zeros.data(), Precision::FP16); + return node; +} + +Gemma4AudioModel::ConformerContext Gemma4AudioModel::build_conformer_context( + CactusGraph* gb, size_t sscp_output) { + + size_t num_heads = config_.audio_num_heads; + size_t head_dim = config_.audio_head_dim; + size_t hidden_dim = num_heads * head_dim; + size_t max_past = config_.audio_context_left > 0 ? config_.audio_context_left - 1 : 0; + size_t num_positions = max_past + config_.audio_context_right + 1; + size_t seq_len = gb->get_output_buffer(sscp_output).shape[0]; + + auto timing_signal = compute_timing_signal(num_positions, hidden_dim, max_past); + size_t timing_input = gb->input({num_positions, hidden_dim}, Precision::FP32); + gb->set_input(timing_input, timing_signal.data(), Precision::FP32); + size_t timing_fp16 = gb->precision_cast(timing_input, Precision::FP16); + + size_t front_pad_len = (seq_len >= num_positions) ? seq_len - num_positions : 0; + size_t back_pad_len = seq_len > 0 ? seq_len - 1 : 0; + + return { + timing_fp16, + create_zero_pad_node(gb, front_pad_len, num_heads, head_dim), + create_zero_pad_node(gb, back_pad_len, num_heads, head_dim), + seq_len + }; +} + +size_t Gemma4AudioModel::build_conformer_ffw(CactusGraph* gb, size_t input, uint32_t layer_idx, + bool is_end, ComputeBackend backend) { + const auto& layer = audio_weights_.layers[layer_idx]; + float residual_weight = config_.audio_residual_weight; + float eps = config_.audio_rms_norm_eps; + + size_t residual = input; + + size_t pre_norm = is_end ? layer.ffw_end_pre_norm : layer.ffw_start_pre_norm; + size_t post_norm = is_end ? layer.ffw_end_post_norm : layer.ffw_start_post_norm; + size_t w1 = is_end ? layer.ffw_end_1 : layer.ffw_start_1; + size_t w2 = is_end ? layer.ffw_end_2 : layer.ffw_start_2; + + auto& c1 = is_end ? layer.ffw_end_1_clip : layer.ffw_start_1_clip; + auto& c2 = is_end ? layer.ffw_end_2_clip : layer.ffw_start_2_clip; + + size_t x = gb->rms_norm(input, pre_norm, eps); + + x = clipped_matmul(gb, x, w1, c1.in_min, c1.in_max, c1.out_min, c1.out_max, backend); + x = gb->silu(x); + x = clipped_matmul(gb, x, w2, c2.in_min, c2.in_max, c2.out_min, c2.out_max, backend); + + x = gb->rms_norm(x, post_norm, eps); + + x = gb->scalar_multiply(x, residual_weight); + return gb->add(residual, x); +} + +size_t Gemma4AudioModel::build_conformer_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, + const ConformerContext& ctx, ComputeBackend backend) { + const auto& layer = audio_weights_.layers[layer_idx]; + float eps = config_.audio_rms_norm_eps; + size_t num_heads = config_.audio_num_heads; + size_t head_dim = config_.audio_head_dim; + size_t hidden_dim = num_heads * head_dim; + float logit_cap = config_.audio_logit_cap; + size_t max_past = config_.audio_context_left > 0 ? config_.audio_context_left - 1 : 0; + size_t num_positions = max_past + config_.audio_context_right + 1; + size_t seq_len = ctx.seq_len; + + size_t residual = input; + + size_t x = gb->rms_norm(input, layer.attn_pre_norm, eps); + + size_t q = clipped_matmul(gb, x, layer.attn_q, layer.attn_q_clip.in_min, layer.attn_q_clip.in_max, layer.attn_q_clip.out_min, layer.attn_q_clip.out_max, backend); + size_t k = clipped_matmul(gb, x, layer.attn_k, layer.attn_k_clip.in_min, layer.attn_k_clip.in_max, layer.attn_k_clip.out_min, layer.attn_k_clip.out_max, backend); + size_t v = clipped_matmul(gb, x, layer.attn_v, layer.attn_v_clip.in_min, layer.attn_v_clip.in_max, layer.attn_v_clip.out_min, layer.attn_v_clip.out_max, backend); + + float q_scale = (1.0f / std::sqrt(static_cast(head_dim))) * INV_LN2; + size_t q_dim_scale_fp16 = gb->precision_cast(layer.attn_per_dim_scale, Precision::FP16); + size_t q_dim_scale = gb->scalar_exp(q_dim_scale_fp16); + q_dim_scale = gb->scalar_add(q_dim_scale, 1.0f); + q_dim_scale = gb->scalar_log(q_dim_scale); + q_dim_scale = gb->scalar_multiply(q_dim_scale, q_scale); + + size_t q_flat = gb->reshape(q, {seq_len * num_heads, head_dim}); + q_flat = gb->multiply(q_flat, q_dim_scale); + q = gb->reshape(q_flat, {seq_len, hidden_dim}); + + k = gb->scalar_multiply(k, GEMMA4_AUDIO_K_SCALE); + + size_t sin_emb = gb->matmul(ctx.timing_fp16, layer.attn_rel_pos_proj, true, backend); + size_t sin_emb_3d = gb->reshape(sin_emb, {num_positions, num_heads, head_dim}); + size_t R = 2 * seq_len - 1; + + size_t rel_key; + if (seq_len >= num_positions) { + rel_key = (ctx.front_pad != 0) ? gb->concat(ctx.front_pad, sin_emb_3d, 0) : sin_emb_3d; + if (ctx.back_pad != 0) + rel_key = gb->concat(rel_key, ctx.back_pad, 0); + } else { + size_t start_idx = num_positions - seq_len; + size_t sin_emb_sliced = gb->slice(sin_emb_3d, 0, start_idx, seq_len); + rel_key = (ctx.back_pad != 0) ? gb->concat(sin_emb_sliced, ctx.back_pad, 0) : sin_emb_sliced; + } + + size_t rel_key_4d = gb->reshape(rel_key, {1, R, num_heads, head_dim}); + size_t q4 = gb->reshape(q, {1, seq_len, num_heads, head_dim}); + size_t rel_bias = gb->rel_pos_bias(q4, rel_key_4d, 1.0f); + + size_t k4 = gb->reshape(k, {1, seq_len, num_heads, head_dim}); + size_t v4 = gb->reshape(v, {1, seq_len, num_heads, head_dim}); + + size_t attn = gb->attention_masked(q4, k4, v4, rel_bias, 1.0f, + true, backend, true, 0, max_past, logit_cap); + + size_t attn_out = gb->reshape(attn, {seq_len, hidden_dim}); + + size_t out = clipped_matmul(gb, attn_out, layer.attn_post, layer.attn_post_clip.in_min, layer.attn_post_clip.in_max, layer.attn_post_clip.out_min, layer.attn_post_clip.out_max, backend); + out = gb->rms_norm(out, layer.attn_post_norm, eps); + + return gb->add(residual, out); +} + +size_t Gemma4AudioModel::build_conformer_lconv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend) { + const auto& layer = audio_weights_.layers[layer_idx]; + float eps = config_.audio_rms_norm_eps; + size_t kernel_size = config_.audio_conf_conv_kernel_size; + size_t hidden_dim = config_.audio_hidden_dim; + + size_t residual = input; + + size_t x = gb->rms_norm(input, layer.lconv_pre_norm, eps); + + x = clipped_matmul(gb, x, layer.lconv_start, layer.lconv_start_clip.in_min, layer.lconv_start_clip.in_max, layer.lconv_start_clip.out_min, layer.lconv_start_clip.out_max, backend); + + x = gb->glu(x, -1); + + const auto& x_buf = gb->get_output_buffer(x); + size_t seq_len = x_buf.shape[0]; + x = gb->reshape(x, {1, seq_len, hidden_dim}); + + x = gb->conv1d_causal(x, layer.lconv_depthwise, kernel_size); + + x = gb->reshape(x, {seq_len, hidden_dim}); + + x = gb->rms_norm(x, layer.lconv_conv_norm, eps); + x = gb->silu(x); + + x = clipped_matmul(gb, x, layer.lconv_end, layer.lconv_end_clip.in_min, layer.lconv_end_clip.in_max, layer.lconv_end_clip.out_min, layer.lconv_end_clip.out_max, backend); + + return gb->add(x, residual); +} + +size_t Gemma4AudioModel::build_conformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + const ConformerContext& ctx, ComputeBackend backend) { + hidden = build_conformer_ffw(gb, hidden, layer_idx, false, backend); + hidden = build_conformer_attention(gb, hidden, layer_idx, ctx, backend); + hidden = build_conformer_lconv1d(gb, hidden, layer_idx, backend); + hidden = build_conformer_ffw(gb, hidden, layer_idx, true, backend); + return gb->rms_norm(hidden, audio_weights_.layers[layer_idx].block_norm, config_.audio_rms_norm_eps); +} + +size_t Gemma4AudioModel::forward_audio(CactusGraph* gb, const std::vector& mel_features, + size_t num_frames, ComputeBackend backend) { + if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { + std::vector npu_input_shape = npu_encoder_->get_input_shape(); + size_t mel_bins = static_cast(config_.audio_input_feat_size); + size_t max_npu_frames = infer_npu_audio_max_frames(npu_input_shape, mel_bins); + size_t copy_frames = max_npu_frames > 0 ? std::min(num_frames, max_npu_frames) : num_frames; + size_t input_values = copy_frames * mel_bins; + + if (mel_features.size() < input_values) { + CACTUS_LOG_WARN("npu", "[gemma4-audio] insufficient mel input values; falling back to CPU audio encoder"); + } else if (!pack_gemma4_audio_for_npu(mel_features, copy_frames, mel_bins, npu_input_shape, + npu_audio_input_scratch_)) { + CACTUS_LOG_WARN("npu", "[gemma4-audio] unsupported NPU input shape; falling back to CPU audio encoder"); + } else { + size_t t1 = (copy_frames + 1) / 2; + size_t t2 = (t1 + 1) / 2; + size_t expected_out_dim = config_.audio_output_proj_dims > 0 + ? static_cast(config_.audio_output_proj_dims) + : static_cast(config_.audio_hidden_dim); + + std::vector out_shape = npu_encoder_->get_output_shape(); + size_t out_elements = npu_encoder_->get_output_buffer_size(); + if (out_elements == 0) out_elements = shape_elements(out_shape); + if (out_elements == 0) out_elements = std::max(1, t2 * expected_out_dim); + + if (npu_audio_output_scratch_.size() < out_elements) + npu_audio_output_scratch_.resize(out_elements); + + size_t written = npu_encoder_->encode( + npu_audio_input_scratch_.data(), npu_audio_output_scratch_.data(), npu_input_shape, "mel", ""); + + NPUAudioOutputLayout out_layout; + if (written > 0 && + infer_npu_audio_output_layout(out_shape, written, expected_out_dim, out_layout) && + out_layout.hidden_dim == expected_out_dim) { + const __fp16* src = npu_audio_output_scratch_.data(); + __fp16* cached_output = npu_encoder_->get_output_buffer(); + size_t cached_count = npu_encoder_->get_output_buffer_size(); + size_t required = out_layout.time_steps * out_layout.hidden_dim; + if (cached_output != nullptr && cached_count >= required) { + src = cached_output; + } + + size_t actual_time = std::min(t2, out_layout.time_steps); + if (actual_time > 0) { + const __fp16* final_src = src; + if (out_layout.hidden_axis != (out_layout.dims.size() - 1)) { + if (!materialize_npu_audio_time_major(src, out_layout, actual_time, + npu_audio_reorder_scratch_)) { + CACTUS_LOG_WARN("npu", "[gemma4-audio] failed to reorder NPU output layout; falling back to CPU audio encoder"); + } else { + final_src = npu_audio_reorder_scratch_.data(); + } + } + + if (final_src != nullptr) { + size_t hidden = gb->input({actual_time, out_layout.hidden_dim}, Precision::FP16); + gb->set_input(hidden, final_src, Precision::FP16); + return hidden; + } + } + } + + if (written > 0 && out_layout.hidden_dim != expected_out_dim) { + CACTUS_LOG_WARN("npu", "[gemma4-audio] NPU output dim mismatch; falling back to CPU audio encoder"); + } else { + CACTUS_LOG_WARN("npu", "audio encode failed, falling back to CPU"); + } + } + } + + size_t hidden = build_sscp(gb, mel_features, num_frames, backend); + ConformerContext ctx = build_conformer_context(gb, hidden); + for (uint32_t i = 0; i < config_.audio_num_layers; i++) + hidden = build_conformer_block(gb, hidden, i, ctx, backend); + + if (config_.audio_output_proj_dims > 0) { + hidden = gb->matmul(hidden, audio_weights_.output_proj, true, backend); + hidden = gb->add(hidden, audio_weights_.output_proj_bias); + } + + return hidden; +} + +size_t Gemma4AudioModel::build_audio_projector(CactusGraph* gb, size_t audio_features, ComputeBackend backend) { + size_t normed = gb->rms_norm(audio_features, audio_proj_norm_ones_node_, config_.audio_rms_norm_eps); + size_t projected = gb->matmul(normed, audio_weights_.embed_audio_proj, true, backend); + return gb->scalar_multiply(projected, 1.0f / 16.0f); +} + +} +} diff --git a/cactus-transpile/models/gemma4/model_gemma4_mm.cpp b/cactus-transpile/models/gemma4/model_gemma4_mm.cpp new file mode 100644 index 000000000..55be7c98d --- /dev/null +++ b/cactus-transpile/models/gemma4/model_gemma4_mm.cpp @@ -0,0 +1,459 @@ +#include "model_gemma4.h" +#include "../../graph/graph.h" +#include +#include +#include + +namespace cactus { +namespace engine { + +Gemma4MmModel::Gemma4MmModel() : Model() { + config_.model_type = Config::ModelType::GEMMA4; +} + +Gemma4MmModel::Gemma4MmModel(const Config& config) + : Model(config), vision_encoder_(config), audio_encoder_(config), language_model_(config) {} + +bool Gemma4MmModel::init(const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup) { + if (!Model::init(model_folder, context_size, system_prompt, false)) + return false; + + auto* shared_graph = static_cast(graph_handle_); + if (!shared_graph) + throw std::runtime_error("Shared graph was not initialized for Gemma4MmModel"); + + bool has_vision = config_.vision_num_layers > 0 || config_.vision_embed_dim > 0; + bool has_audio = config_.audio_num_layers > 0 || config_.audio_hidden_dim > 0; + + if (has_vision) { + if (!vision_encoder_.init(shared_graph, model_folder, context_size, "", false)) + throw std::runtime_error("Failed to initialize vision encoder"); + } + + if (has_audio) { + if (!audio_encoder_.init(shared_graph, model_folder, context_size, "", false)) + throw std::runtime_error("Failed to initialize audio encoder"); + } + + if (!language_model_.init(shared_graph, model_folder, context_size, system_prompt, false)) + throw std::runtime_error("Failed to initialize language model"); + + output_weight_node_id_ = language_model_.output_weight_node_id_; + + if (do_warmup) { + std::vector warmup_tokens = {2}; + language_model_.forward(warmup_tokens); + auto* gb2 = static_cast(language_model_.graph_handle_); + gb2->execute(); + language_model_.reset_cache(); + } + + return true; +} + +void Gemma4MmModel::reset_cache() { + Model::reset_cache(); + language_model_.reset_cache(); + prefill_completed_ = false; + last_token_count_ = 0; +} + +void Gemma4MmModel::compact_kv_cache() { + language_model_.compact_kv_cache(); +} + +void Gemma4MmModel::remove_thinking_tokens(const std::vector>& ranges) { + language_model_.remove_thinking_tokens(ranges); +} + +void Gemma4MmModel::load_weights_to_graph(CactusGraph*) { + output_weight_node_id_ = 0; +} + +Gemma4MmModel::ForwardResult Gemma4MmModel::forward_multimodal( + CactusGraph* gb, const std::vector& tokens, + const std::vector& image_paths, + const std::vector* audio_features, + size_t audio_num_frames, + ComputeBackend backend, bool use_cache) { + + auto inputs = build_multimodal_inputs( + gb, tokens, image_paths, audio_features, audio_num_frames, backend); + + size_t final_hidden = language_model_.forward_from_embeddings( + gb, + inputs.hidden_node, + inputs.pli_hidden_source_node, + inputs.pli_tokens, + inputs.seq_len, + backend, + use_cache); + + return ForwardResult{final_hidden, inputs.seq_len}; +} + +Gemma4MmModel::MultimodalInputs Gemma4MmModel::build_multimodal_inputs( + CactusGraph* gb, const std::vector& tokens, + const std::vector& image_paths, + const std::vector* audio_features, + size_t audio_num_frames, + ComputeBackend backend) { + + size_t vision_soft_node = 0; + size_t num_vision_soft_tokens = 0; + size_t audio_soft_node = 0; + size_t num_audio_soft_tokens = 0; + + if (!image_paths.empty()) { + auto preprocessed = vision_encoder_.preprocess_image(image_paths[0]); + size_t vision_output = vision_encoder_.forward_vision(gb, preprocessed, backend); + vision_soft_node = vision_encoder_.build_vision_projector(gb, vision_output, backend); + uint32_t k = config_.vision_pooling_kernel_size; + num_vision_soft_tokens = (preprocessed.patch_width / k) * (preprocessed.patch_height / k); + } + + if (audio_features && !audio_features->empty()) { + size_t audio_output = audio_encoder_.forward_audio(gb, *audio_features, audio_num_frames, backend); + audio_soft_node = audio_encoder_.build_audio_projector(gb, audio_output, backend); + const auto& audio_buf = gb->get_output_buffer(audio_soft_node); + num_audio_soft_tokens = audio_buf.shape[0]; + } + + uint32_t image_token_id = config_.image_token_id; + uint32_t audio_token_id = config_.audio_token_id; + uint32_t pad_token_id = config_.pad_token_id; + + std::vector sequence_nodes; + std::vector current_text; + std::vector pli_tokens; + size_t total_seq_len = 0; + size_t vision_offset = 0; + size_t audio_offset = 0; + + auto flush_text = [&]() { + if (current_text.empty()) return; + size_t seg_len = current_text.size(); + size_t input_node = gb->input({seg_len}, Precision::FP32); + + auto hidden = gb->scalar_multiply( + gb->embedding(language_model_.embedding_node_id_, input_node), + std::sqrt(static_cast(config_.hidden_dim))); + + std::vector input_data(seg_len); + for (size_t i = 0; i < seg_len; i++) + input_data[i] = static_cast(current_text[i]); + gb->set_input(input_node, input_data.data(), Precision::FP32); + + sequence_nodes.push_back(hidden); + for (auto t : current_text) + pli_tokens.push_back(t); + total_seq_len += seg_len; + current_text.clear(); + }; + + auto append_soft_region = [&](size_t soft_node, size_t& soft_offset, size_t total_soft_tokens, + size_t placeholder_count) { + size_t to_insert = std::min(placeholder_count, total_soft_tokens - soft_offset); + if (to_insert > 0) { + sequence_nodes.push_back(gb->slice(soft_node, 0, soft_offset, to_insert)); + for (size_t j = 0; j < to_insert; j++) + pli_tokens.push_back(pad_token_id); + total_seq_len += to_insert; + soft_offset += to_insert; + } + }; + + auto flush_vision_region = [&](size_t placeholder_count) { + append_soft_region(vision_soft_node, vision_offset, num_vision_soft_tokens, placeholder_count); + }; + + auto flush_audio_region = [&](size_t placeholder_count) { + append_soft_region(audio_soft_node, audio_offset, num_audio_soft_tokens, placeholder_count); + }; + + bool in_image_region = false; + bool in_audio_region = false; + size_t region_count = 0; + + for (size_t i = 0; i < tokens.size(); i++) { + uint32_t tok = tokens[i]; + bool is_vision_token = (tok == image_token_id && image_token_id != 0); + bool is_audio_token = (tok == audio_token_id && audio_token_id != 0); + + if (is_vision_token) { + if (in_audio_region) { + flush_audio_region(region_count); + in_audio_region = false; + } + if (!in_image_region) { + flush_text(); + in_image_region = true; + region_count = 0; + } + region_count++; + } else if (is_audio_token) { + if (in_image_region) { + flush_vision_region(region_count); + in_image_region = false; + } + if (!in_audio_region) { + flush_text(); + in_audio_region = true; + region_count = 0; + } + region_count++; + } else { + if (in_image_region) { + flush_vision_region(region_count); + in_image_region = false; + } + if (in_audio_region) { + flush_audio_region(region_count); + in_audio_region = false; + } + current_text.push_back(tok); + } + } + + if (in_image_region) + flush_vision_region(region_count); + if (in_audio_region) + flush_audio_region(region_count); + flush_text(); + + if (sequence_nodes.empty()) + throw std::runtime_error("No embedding nodes built"); + + size_t merged = sequence_nodes[0]; + for (size_t i = 1; i < sequence_nodes.size(); i++) + merged = gb->concat(merged, sequence_nodes[i], 0); + + return MultimodalInputs{ + .hidden_node = merged, + .pli_hidden_source_node = merged, + .pli_tokens = std::move(pli_tokens), + .seq_len = total_seq_len, + }; +} + +uint32_t Gemma4MmModel::decode_multimodal( + const std::vector& tokens, + const std::vector& image_paths, + const std::vector* audio_features, + size_t audio_num_frames, + float temperature, float top_p, size_t top_k, + const std::string& profile_file, float* out_entropy, + float min_p, float repetition_penalty) { + + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + + bool has_media = !image_paths.empty() || (audio_features && !audio_features->empty()); + + if (!has_media) { + prefill_completed_ = false; + last_token_count_ = tokens.size(); + return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); + } + + if (temperature < 0) temperature = config_.default_temperature; + if (top_p < 0) top_p = config_.default_top_p; + if (top_k == 0) top_k = config_.default_top_k; + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + bool cache_empty = language_model_.kv_cache_.is_empty(); + bool need_prefill = cache_empty || !prefill_completed_; + + if (!need_prefill && tokens.size() <= last_token_count_) { + reset_cache(); + need_prefill = true; + } + + size_t seq_len_for_updates = 0; + size_t final_hidden_node = 0; + + if (need_prefill) { + auto result = forward_multimodal(gb, tokens, image_paths, audio_features, + audio_num_frames, backend, true); + final_hidden_node = result.final_hidden_node; + seq_len_for_updates = result.seq_len; + prefill_completed_ = true; + last_token_count_ = tokens.size(); + } else { + size_t delta = tokens.size() - last_token_count_; + if (delta == 0) delta = 1; + std::vector incremental_tokens(tokens.end() - delta, tokens.end()); + final_hidden_node = language_model_.forward(incremental_tokens, true); + seq_len_for_updates = incremental_tokens.size(); + last_token_count_ = tokens.size(); + } + + auto last_hidden = gb->index(final_hidden_node, seq_len_for_updates - 1, 0); + const auto& last_buf = gb->get_output_buffer(last_hidden); + last_hidden = gb->reshape(last_hidden, {1, last_buf.shape[0]}); + + auto logits_node = gb->matmul(last_hidden, language_model_.output_weight_node_id_, true, backend); + + if (config_.final_logit_softcapping > 0.0f) { + float inv_cap = 1.0f / config_.final_logit_softcapping; + logits_node = gb->scalar_multiply(logits_node, inv_cap); + logits_node = gb->tanh(logits_node); + logits_node = gb->scalar_multiply(logits_node, config_.final_logit_softcapping); + } + + size_t sampled_token = + language_model_.sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty, nullptr); + + if (!profile_file.empty()) + gb->execute(profile_file); + else + gb->execute(); + + compute_entropy(gb, logits_node, out_entropy); + + language_model_.post_execute_updates(gb, seq_len_for_updates); + language_model_.update_kv_cache(gb, seq_len_for_updates); + + auto* output_ptr = gb->get_output(sampled_token); + uint32_t result_token = *static_cast(output_ptr); + language_model_.record_sampled_token(result_token); + return result_token; +} + +size_t Gemma4MmModel::forward(const std::vector& tokens, bool use_cache) { + return language_model_.forward(tokens, use_cache); +} + +uint32_t Gemma4MmModel::decode(const std::vector& tokens, + float temperature, float top_p, size_t top_k, + const std::string& profile_file, float* out_entropy, + float min_p, float repetition_penalty) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + prefill_completed_ = false; + last_token_count_ = tokens.size(); + return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); +} + +void Gemma4MmModel::prefill(const std::vector& tokens, size_t chunk_size, + const std::string& profile_file) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + prefill_completed_ = false; + last_token_count_ = tokens.size(); + language_model_.prefill(tokens, chunk_size, profile_file); +} + +void Gemma4MmModel::prefill_with_images(const std::vector& tokens, + const std::vector& image_paths, + const std::string& profile_file) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + + if (image_paths.empty()) { + prefill(tokens, get_prefill_chunk_size(), profile_file); + return; + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + auto result = forward_multimodal(gb, tokens, image_paths, nullptr, 0, backend, true); + + if (!profile_file.empty()) + gb->execute(profile_file); + else + gb->execute(); + + language_model_.post_execute_updates(gb, result.seq_len); + language_model_.update_kv_cache(gb, result.seq_len); + + prefill_completed_ = true; + last_token_count_ = tokens.size(); +} + +uint32_t Gemma4MmModel::decode_with_images( + const std::vector& tokens, const std::vector& image_paths, + float temperature, float top_p, size_t top_k, + const std::string& profile_file, float* out_entropy, + float min_p, float repetition_penalty) { + return decode_multimodal(tokens, image_paths, nullptr, 0, + temperature, top_p, top_k, profile_file, out_entropy, + min_p, repetition_penalty); +} + +uint32_t Gemma4MmModel::decode_with_audio( + const std::vector& tokens, const std::vector& audio_features, + float temperature, float top_p, size_t top_k, + const std::string& profile_file, float* out_entropy, + float min_p, float repetition_penalty, + float* /*out_token_time_start*/, float* /*out_token_time_end*/) { + size_t num_frames = audio_features.size() / config_.audio_input_feat_size; + std::vector empty_images; + return decode_multimodal(tokens, empty_images, &audio_features, num_frames, + temperature, top_p, top_k, profile_file, out_entropy, + min_p, repetition_penalty); +} + +std::vector Gemma4MmModel::get_image_embeddings(const std::string& image_path) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + auto preprocessed = vision_encoder_.preprocess_image(image_path); + size_t vision_output = vision_encoder_.forward_vision(gb, preprocessed, backend); + size_t projected = vision_encoder_.build_vision_projector(gb, vision_output, backend); + + gb->execute(); + + const auto& buf = gb->get_output_buffer(projected); + size_t total = buf.total_size; + std::vector embedding(total); + const __fp16* fp16_data = buf.data_as<__fp16>(); + for (size_t i = 0; i < total; i++) + embedding[i] = static_cast(fp16_data[i]); + return embedding; +} + +std::vector Gemma4MmModel::get_audio_embeddings(const std::vector& audio_features) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + size_t num_frames = audio_features.size() / config_.audio_input_feat_size; + size_t audio_output = audio_encoder_.forward_audio(gb, audio_features, num_frames, backend); + size_t projected = audio_encoder_.build_audio_projector(gb, audio_output, backend); + + gb->execute(); + + const auto& buf = gb->get_output_buffer(projected); + size_t total = buf.total_size; + std::vector embedding(total); + const __fp16* fp16_data = buf.data_as<__fp16>(); + for (size_t i = 0; i < total; i++) + embedding[i] = static_cast(fp16_data[i]); + return embedding; +} + +size_t Gemma4MmModel::build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { + throw std::runtime_error("build_attention should not be called directly on Gemma4MmModel"); +} + +size_t Gemma4MmModel::build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const { + throw std::runtime_error("build_mlp should not be called directly on Gemma4MmModel"); +} + +size_t Gemma4MmModel::build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { + throw std::runtime_error("build_transformer_block should not be called directly on Gemma4MmModel"); +} + +} +} diff --git a/cactus-transpile/models/gemma4/model_gemma4_vision.cpp b/cactus-transpile/models/gemma4/model_gemma4_vision.cpp new file mode 100644 index 000000000..6d495bd4c --- /dev/null +++ b/cactus-transpile/models/gemma4/model_gemma4_vision.cpp @@ -0,0 +1,540 @@ +#include "model_gemma4.h" +#include "../../graph/graph.h" +#include +#include +#include +#include + +extern "C" { + #include "../../../libs/stb/stb_image.h" + #include "../../../libs/stb/stb_image_resize2.h" +} + +namespace cactus { +namespace engine { + +static std::pair, std::vector> compute_2d_rope_tables( + const Gemma4VisionModel::PreprocessedImage& img, size_t max_patches, size_t head_dim, float theta) { + + size_t half_dim = head_dim / 2; + size_t freq_per_dim = half_dim / 2; + + std::vector cos_table(max_patches * head_dim, 1.0f); + std::vector sin_table(max_patches * head_dim, 0.0f); + + for (size_t py = 0; py < img.patch_height; py++) { + for (size_t px = 0; px < img.patch_width; px++) { + size_t patch_idx = py * img.patch_width + px; + + for (size_t i = 0; i < freq_per_dim; i++) { + float inv_freq = 1.0f / std::pow(theta, 2.0f * i / static_cast(half_dim)); + + float x_angle = static_cast(px) * inv_freq; + cos_table[patch_idx * head_dim + i] = std::cos(x_angle); + cos_table[patch_idx * head_dim + freq_per_dim + i] = std::cos(x_angle); + sin_table[patch_idx * head_dim + i] = std::sin(x_angle); + sin_table[patch_idx * head_dim + freq_per_dim + i] = std::sin(x_angle); + + float y_angle = static_cast(py) * inv_freq; + cos_table[patch_idx * head_dim + half_dim + i] = std::cos(y_angle); + cos_table[patch_idx * head_dim + half_dim + freq_per_dim + i] = std::cos(y_angle); + sin_table[patch_idx * head_dim + half_dim + i] = std::sin(y_angle); + sin_table[patch_idx * head_dim + half_dim + freq_per_dim + i] = std::sin(y_angle); + } + } + } + + return {cos_table, sin_table}; +} + +static std::pair, std::vector<__fp16>> compute_npu_rope_tables( + const Gemma4VisionModel::PreprocessedImage& img, size_t max_patches, size_t head_dim, float theta) { + + auto [cos_f32, sin_f32] = compute_2d_rope_tables(img, max_patches, head_dim, theta); + + std::vector<__fp16> cos_full(max_patches * head_dim); + std::vector<__fp16> sin_signed(max_patches * head_dim); + + for (size_t p = 0; p < max_patches; p++) { + size_t base = p * head_dim; + for (size_t d = 0; d < head_dim; d++) { + cos_full[base + d] = static_cast<__fp16>(cos_f32[base + d]); + sin_signed[base + d] = static_cast<__fp16>(sin_f32[base + d]); + } + } + + return {cos_full, sin_signed}; +} + +Gemma4VisionModel::Gemma4VisionModel() : Model() {} +Gemma4VisionModel::Gemma4VisionModel(const Config& config) : Model(config) {} + +void Gemma4VisionModel::load_weights_to_graph(CactusGraph* gb) { + auto resolve = [&](const std::string& name) -> std::string { + return model_folder_path_ + "/" + name; + }; + + if (!disable_npu_ && npu::is_npu_available()) { + std::string npu_path = model_folder_path_ + "/vision_encoder.mlpackage"; + if (std::filesystem::exists(npu_path)) { + npu_encoder_ = npu::create_encoder(); + if (npu_encoder_ && npu_encoder_->load(npu_path)) { + use_npu_encoder_ = true; + std::vector input_shape = npu_encoder_->get_input_shape(); + if (input_shape.empty()) { + input_shape = {static_cast(config_.vision_default_output_length * + config_.vision_pooling_kernel_size * config_.vision_pooling_kernel_size), + static_cast(config_.vision_embed_dim)}; + } + npu_encoder_->preallocate(input_shape, "hidden_states", "output"); + } else { + use_npu_encoder_ = false; + npu_encoder_.reset(); + CACTUS_LOG_WARN("npu", "[gemma4-vision] found vision_encoder.mlpackage but failed to enable NPU vision encoder; using CPU"); + } + } else { + CACTUS_LOG_WARN("npu", "[gemma4-vision] vision_encoder.mlpackage not found; using CPU vision encoder"); + } + } else if (!disable_npu_) { + CACTUS_LOG_WARN("npu", "[gemma4-vision] NPU backend unavailable on this device; using CPU vision encoder"); + } + + vision_weights_.patch_input_proj = gb->mmap_weights(resolve("vision_patch_embedder_input_proj.weights")); + vision_weights_.position_table = gb->mmap_weights(resolve("vision_patch_embedder_position_embedding_table.weights")); + + if (!use_npu_encoder_) { + vision_weights_.layers.resize(config_.vision_num_layers); + for (uint32_t i = 0; i < config_.vision_num_layers; i++) { + auto& layer = vision_weights_.layers[i]; + std::string prefix = resolve("vision_encoder_layers_" + std::to_string(i) + "_"); + + layer.attn_q_weight = gb->mmap_weights(prefix + "self_attn_q_proj.weights"); + layer.attn_k_weight = gb->mmap_weights(prefix + "self_attn_k_proj.weights"); + layer.attn_v_weight = gb->mmap_weights(prefix + "self_attn_v_proj.weights"); + layer.attn_output_weight = gb->mmap_weights(prefix + "self_attn_o_proj.weights"); + layer.attn_q_norm = gb->mmap_weights(prefix + "self_attn_q_norm.weights"); + layer.attn_k_norm = gb->mmap_weights(prefix + "self_attn_k_norm.weights"); + layer.input_layernorm = gb->mmap_weights(prefix + "input_layernorm.weights"); + layer.post_attention_layernorm = gb->mmap_weights(prefix + "post_attention_layernorm.weights"); + layer.pre_feedforward_layernorm = gb->mmap_weights(prefix + "pre_feedforward_layernorm.weights"); + layer.post_feedforward_layernorm = gb->mmap_weights(prefix + "post_feedforward_layernorm.weights"); + layer.mlp_gate_proj = gb->mmap_weights(prefix + "mlp_gate_proj.weights"); + layer.mlp_up_proj = gb->mmap_weights(prefix + "mlp_up_proj.weights"); + layer.mlp_down_proj = gb->mmap_weights(prefix + "mlp_down_proj.weights"); + layer.layer_scalar = std::filesystem::exists(prefix + "layer_scalar.weights") ? gb->mmap_weights(prefix + "layer_scalar.weights") : 0; + } + } + + vision_weights_.embed_vision_proj = gb->mmap_weights(resolve("embed_vision_proj.weights")); + + size_t vision_head_dim = config_.vision_head_dim; + vision_v_norm_ones_.assign(vision_head_dim, static_cast<__fp16>(1.0f)); + vision_v_norm_ones_node_ = gb->input({vision_head_dim}, Precision::FP16); + gb->set_external_input(vision_v_norm_ones_node_, vision_v_norm_ones_.data(), Precision::FP16); + + vision_weights_.post_proj_norm = gb->mmap_weights(resolve("embed_vision_post_proj_norm.weights")); + + output_weight_node_id_ = 0; +} + +Gemma4VisionModel::PreprocessedImage Gemma4VisionModel::preprocess_image(const std::string& image_path) { + int w, h, channels; + unsigned char* data = stbi_load(image_path.c_str(), &w, &h, &channels, 3); + if (!data) + throw std::runtime_error("Failed to load image: " + image_path); + + uint32_t patch_size = config_.vision_patch_size; + uint32_t pooling_k = config_.vision_pooling_kernel_size; + uint32_t max_patches = config_.vision_default_output_length * pooling_k * pooling_k; + uint32_t side_mult = pooling_k * patch_size; + + float total_px = static_cast(h) * static_cast(w); + float target_px = static_cast(max_patches) * static_cast(patch_size * patch_size); + float factor = std::sqrt(target_px / total_px); + + int target_h = static_cast(std::floor(factor * h / side_mult)) * side_mult; + int target_w = static_cast(std::floor(factor * w / side_mult)) * side_mult; + + if (target_h == 0) target_h = side_mult; + if (target_w == 0) target_w = side_mult; + + std::vector resized_data; + unsigned char* src = data; + if (target_h != h || target_w != w) { + resized_data.resize(target_h * target_w * 3); + stbir_resize_uint8_linear(data, w, h, 0, resized_data.data(), target_w, target_h, 0, + static_cast(3)); + src = resized_data.data(); + } + + size_t ph = target_h / patch_size; + size_t pw = target_w / patch_size; + size_t num_patches = ph * pw; + + std::vector pixels(3 * target_h * target_w); + for (int y = 0; y < target_h; y++) { + for (int x = 0; x < target_w; x++) { + size_t src_idx = (y * target_w + x) * 3; + for (int c = 0; c < 3; c++) { + pixels[c * target_h * target_w + y * target_w + x] = + static_cast(src[src_idx + c]) * config_.rescale_factor; + } + } + } + + stbi_image_free(data); + + return PreprocessedImage{ + std::move(pixels), + static_cast(target_h), + static_cast(target_w), + ph, pw, num_patches + }; +} + +size_t Gemma4VisionModel::build_vision_patch_embedding(CactusGraph* gb, const PreprocessedImage& img, + ComputeBackend backend) { + uint32_t patch_size = config_.vision_patch_size; + size_t patch_dim = 3 * patch_size * patch_size; + size_t num_patches = img.num_patches; + size_t hidden_size = config_.vision_embed_dim; + size_t pos_embed_size = config_.vision_position_embedding_size; + + size_t pixel_input = gb->input({3, img.height, img.width}, Precision::FP32); + gb->set_input(pixel_input, img.pixel_values.data(), Precision::FP32); + + size_t pixel_fp16 = gb->precision_cast(pixel_input, Precision::FP16); + size_t reshaped = gb->reshape(pixel_fp16, {3, img.patch_height, patch_size, img.patch_width, patch_size}); + size_t permuted = gb->transposeN(reshaped, {1, 3, 2, 4, 0}); + size_t patches = gb->reshape(permuted, {num_patches, patch_dim}); + + float mean = config_.image_mean; + float norm_std = config_.image_std; + patches = gb->scalar_multiply(gb->scalar_add(patches, -mean), 1.0f / norm_std); + + size_t projected = gb->matmul(patches, vision_weights_.patch_input_proj, true, backend); + + size_t pos_table = gb->reshape(vision_weights_.position_table, {2, pos_embed_size, hidden_size}); + + std::vector x_positions(num_patches); + std::vector y_positions(num_patches); + for (size_t py = 0; py < img.patch_height; py++) { + for (size_t px = 0; px < img.patch_width; px++) { + size_t idx = py * img.patch_width + px; + x_positions[idx] = static_cast(px); + y_positions[idx] = static_cast(py); + } + } + + size_t x_pos_input = gb->input({num_patches}, Precision::FP32); + gb->set_input(x_pos_input, x_positions.data(), Precision::FP32); + size_t y_pos_input = gb->input({num_patches}, Precision::FP32); + gb->set_input(y_pos_input, y_positions.data(), Precision::FP32); + + size_t x_table = gb->slice(pos_table, 0, 0, 1); + x_table = gb->reshape(x_table, {pos_embed_size, hidden_size}); + size_t y_table = gb->slice(pos_table, 0, 1, 1); + y_table = gb->reshape(y_table, {pos_embed_size, hidden_size}); + + size_t x_embed = gb->embedding(x_table, x_pos_input); + size_t y_embed = gb->embedding(y_table, y_pos_input); + size_t pos_embed = gb->add(x_embed, y_embed); + + size_t pos_embed_fp16 = gb->precision_cast(pos_embed, Precision::FP16); + + return gb->add(projected, pos_embed_fp16); +} + +std::pair Gemma4VisionModel::build_2d_rope_nodes( + CactusGraph* gb, const PreprocessedImage& img, size_t max_patches) { + + size_t head_dim = config_.vision_head_dim; + auto rope_tables = compute_2d_rope_tables(img, max_patches, head_dim, config_.vision_rope_theta); + auto& cos_table = rope_tables.first, sin_table = rope_tables.second; + + size_t cos_input = gb->input({max_patches, head_dim}, Precision::FP32); + gb->set_input(cos_input, cos_table.data(), Precision::FP32); + size_t cos_node = gb->precision_cast(cos_input, Precision::FP16); + cos_node = gb->reshape(cos_node, {1, max_patches, 1, head_dim}); + + size_t sin_input = gb->input({max_patches, head_dim}, Precision::FP32); + gb->set_input(sin_input, sin_table.data(), Precision::FP32); + size_t sin_node = gb->precision_cast(sin_input, Precision::FP16); + sin_node = gb->reshape(sin_node, {1, max_patches, 1, head_dim}); + + return {cos_node, sin_node}; +} + +size_t Gemma4VisionModel::build_padding_mask(CactusGraph* gb, size_t num_real, size_t max_patches) { + if (num_real >= max_patches) + return 0; + + std::vector mask(max_patches * max_patches, 0.0f); + for (size_t q = 0; q < num_real; q++) + for (size_t k = 0; k < num_real; k++) + mask[q * max_patches + k] = 1.0f; + size_t mask_input = gb->input({max_patches, max_patches}, Precision::FP32); + gb->set_input(mask_input, mask.data(), Precision::FP32); + size_t mask_node = gb->precision_cast(mask_input, Precision::FP16); + return gb->reshape(mask_node, {1, max_patches, max_patches}); +} + +size_t Gemma4VisionModel::build_vision_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, + size_t cos_node, size_t sin_node, + size_t attn_mask_node, ComputeBackend backend) { + const auto& layer = vision_weights_.layers[layer_idx]; + size_t head_dim = config_.vision_head_dim; + size_t num_heads = config_.vision_attention_heads; + size_t kv_heads = config_.vision_kv_heads; + + const auto& input_buf = gb->get_output_buffer(input); + size_t seq_len = input_buf.shape[0]; + + auto q = gb->matmul(input, layer.attn_q_weight, true, backend); + q = gb->reshape(q, {seq_len * num_heads, head_dim}); + q = gb->rms_norm(q, layer.attn_q_norm, config_.layer_norm_eps); + q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); + + auto k = gb->matmul(input, layer.attn_k_weight, true, backend); + k = gb->reshape(k, {seq_len * kv_heads, head_dim}); + k = gb->rms_norm(k, layer.attn_k_norm, config_.layer_norm_eps); + k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); + + auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); + auto v = gb->rms_norm(gb->reshape(v_proj, {seq_len * kv_heads, head_dim}), + vision_v_norm_ones_node_, config_.layer_norm_eps); + v = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); + + size_t half_dim = head_dim / 2; + + auto apply_2d_rope = [&](size_t tensor) -> size_t { + auto first_half = gb->slice(tensor, 3, 0, half_dim); + auto second_half = gb->slice(tensor, 3, half_dim, half_dim); + + auto rot_first_a = gb->slice(first_half, 3, 0, half_dim / 2); + auto rot_first_b = gb->slice(first_half, 3, half_dim / 2, half_dim / 2); + auto neg_b = gb->scalar_multiply(rot_first_b, -1.0f); + + size_t cos_x = gb->slice(cos_node, 3, 0, half_dim); + size_t sin_x = gb->slice(sin_node, 3, 0, half_dim); + + auto first_rotated = gb->add( + gb->multiply(first_half, cos_x), + gb->multiply(gb->concat(neg_b, rot_first_a, 3), sin_x)); + + auto rot_second_a = gb->slice(second_half, 3, 0, half_dim / 2); + auto rot_second_b = gb->slice(second_half, 3, half_dim / 2, half_dim / 2); + auto neg_sb = gb->scalar_multiply(rot_second_b, -1.0f); + + size_t cos_y = gb->slice(cos_node, 3, half_dim, half_dim); + size_t sin_y = gb->slice(sin_node, 3, half_dim, half_dim); + + auto second_rotated = gb->add( + gb->multiply(second_half, cos_y), + gb->multiply(gb->concat(neg_sb, rot_second_a, 3), sin_y)); + + return gb->concat(first_rotated, second_rotated, 3); + }; + + auto q_rot = apply_2d_rope(q); + auto k_rot = apply_2d_rope(k); + + size_t attn; + if (attn_mask_node != 0) + attn = gb->attention_masked(q_rot, k_rot, v, attn_mask_node, 1.f, false, backend); + else + attn = gb->attention(q_rot, k_rot, v, 1.f, false, backend); + + return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), + layer.attn_output_weight, true, backend); +} + +size_t Gemma4VisionModel::build_vision_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend) { + const auto& layer = vision_weights_.layers[layer_idx]; + auto gate = gb->gelu(gb->matmul(input, layer.mlp_gate_proj, true, backend)); + auto up = gb->matmul(input, layer.mlp_up_proj, true, backend); + return gb->matmul(gb->multiply(gate, up), layer.mlp_down_proj, true, backend); +} + +size_t Gemma4VisionModel::build_vision_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + size_t cos_node, size_t sin_node, + size_t attn_mask_node, ComputeBackend backend) { + const auto& layer = vision_weights_.layers[layer_idx]; + + auto normed = gb->rms_norm(hidden, layer.input_layernorm, config_.layer_norm_eps); + auto attn_raw = build_vision_attention(gb, normed, layer_idx, cos_node, sin_node, attn_mask_node, backend); + auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm, config_.layer_norm_eps); + auto residual = gb->add(hidden, attn); + + auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm, config_.layer_norm_eps); + auto mlp_raw = build_vision_mlp(gb, pre_mlp, layer_idx, backend); + auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm, config_.layer_norm_eps); + + auto out = gb->add(residual, mlp); + + if (layer.layer_scalar != 0) + out = gb->multiply(out, layer.layer_scalar); + + return out; +} + +size_t Gemma4VisionModel::build_vision_pooler(CactusGraph* gb, size_t hidden, const PreprocessedImage& img, + ComputeBackend backend) { + size_t k = config_.vision_pooling_kernel_size; + size_t output_length = config_.vision_default_output_length; + size_t num_patches = img.num_patches; + size_t k_squared = k * k; + + if (num_patches == output_length) + return hidden; + + size_t max_x = img.patch_width; + size_t max_y = img.patch_height; + + std::vector pool_weights(num_patches * output_length, 0.0f); + std::vector valid_bins(output_length, false); + for (size_t py = 0; py < max_y; py++) { + for (size_t px = 0; px < max_x; px++) { + size_t patch_idx = py * max_x + px; + size_t kx = px / k; + size_t ky = py / k; + size_t kernel_idx = kx + (max_x / k) * ky; + if (kernel_idx < output_length) { + pool_weights[kernel_idx * num_patches + patch_idx] = 1.0f / static_cast(k_squared); + valid_bins[kernel_idx] = true; + } + } + } + + size_t weights_node = gb->input({output_length, num_patches}, Precision::FP32); + gb->set_input(weights_node, pool_weights.data(), Precision::FP32); + size_t weights_fp16 = gb->precision_cast(weights_node, Precision::FP16); + + size_t pooled = gb->matmul(weights_fp16, hidden, false, backend); + + size_t valid_count = 0; + for (bool v : valid_bins) if (v) valid_count++; + + if (valid_count == output_length) + return pooled; + + std::vector valid_slices; + for (size_t i = 0; i < output_length; i++) { + if (valid_bins[i]) { + auto slice = gb->slice(pooled, 0, i, 1); + valid_slices.push_back(slice); + } + } + + size_t stripped = valid_slices[0]; + for (size_t i = 1; i < valid_slices.size(); i++) + stripped = gb->concat(stripped, valid_slices[i], 0); + + return stripped; +} + +size_t Gemma4VisionModel::forward_vision(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend) { + size_t hidden = build_vision_patch_embedding(gb, img, backend); + + size_t num_real = img.num_patches; + size_t hidden_size = config_.vision_embed_dim; + uint32_t pooling_k = config_.vision_pooling_kernel_size; + size_t max_patches = config_.vision_default_output_length * pooling_k * pooling_k; + bool can_use_npu_path = use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available(); + if (can_use_npu_path) { + std::vector npu_input_shape = npu_encoder_->get_input_shape(); + if (npu_input_shape.size() >= 2) { + size_t a = static_cast(npu_input_shape[0]); + size_t b = static_cast(npu_input_shape[1]); + if (b == hidden_size) { + max_patches = a; + } else if (a == hidden_size) { + max_patches = b; + } else if (a > 0) { + max_patches = a; + } + } + } + if (num_real > max_patches) { + if (can_use_npu_path) { + CACTUS_LOG_WARN("npu", "[gemma4-vision] image has more patches than NPU encoder supports; falling back to CPU vision encoder"); + can_use_npu_path = false; + max_patches = config_.vision_default_output_length * pooling_k * pooling_k; + } + } + size_t num_padding = max_patches - num_real; + + if (num_padding > 0) { + std::vector<__fp16> zeros(num_padding * hidden_size, static_cast<__fp16>(0.0f)); + size_t pad_node = gb->input({num_padding, hidden_size}, Precision::FP16); + gb->set_input(pad_node, zeros.data(), Precision::FP16); + hidden = gb->concat(hidden, pad_node, 0); + } + + auto [cos_node, sin_node] = build_2d_rope_nodes(gb, img, max_patches); + size_t attn_mask_node = build_padding_mask(gb, num_real, max_patches); + + if (can_use_npu_path) { + gb->execute(); + + const auto& h_buf = gb->get_output_buffer(hidden); + if (h_buf.precision != Precision::FP16) + throw std::runtime_error("[gemma4-vision] expected FP16 hidden output for NPU path"); + std::vector<__fp16> hidden_fp16(max_patches * hidden_size); + memcpy(hidden_fp16.data(), h_buf.data_as<__fp16>(), max_patches * hidden_size * sizeof(__fp16)); + + size_t head_dim = config_.vision_head_dim; + auto [cos_fp16, sin_fp16] = compute_npu_rope_tables(img, max_patches, head_dim, config_.vision_rope_theta); + + std::vector<__fp16> additive_mask(max_patches * max_patches, static_cast<__fp16>(0.0f)); + if (num_padding > 0) { + for (size_t q = 0; q < max_patches; q++) + for (size_t k = 0; k < max_patches; k++) + if (q >= num_real || k >= num_real) + additive_mask[q * max_patches + k] = static_cast<__fp16>(-65504.0f); + } + + std::vector inputs = { + {"hidden_states", hidden_fp16.data(), {static_cast(max_patches), static_cast(hidden_size)}}, + {"cos_full", cos_fp16.data(), {static_cast(max_patches), 1, static_cast(head_dim)}}, + {"sin_signed", sin_fp16.data(), {static_cast(max_patches), 1, static_cast(head_dim)}}, + {"attention_mask", additive_mask.data(), {1, static_cast(max_patches), static_cast(max_patches)}}, + }; + + std::vector out_shape = npu_encoder_->get_output_shape(); + size_t out_elements = 1; + for (int d : out_shape) out_elements *= d; + size_t required = max_patches * hidden_size; + if (out_elements < required) + out_elements = required; + + std::vector<__fp16> npu_output(out_elements); + size_t written = npu_encoder_->encode_multimodal_input(inputs, npu_output.data(), "output"); + + if (written >= required) { + hidden = gb->input({max_patches, hidden_size}, Precision::FP16); + gb->set_input(hidden, npu_output.data(), Precision::FP16); + } else { + CACTUS_LOG_WARN("npu", "[gemma4-vision] encode_multimodal_input returned insufficient output, falling back to CPU"); + for (uint32_t i = 0; i < config_.vision_num_layers; i++) + hidden = build_vision_transformer_block(gb, hidden, i, cos_node, sin_node, attn_mask_node, backend); + } + } else { + for (uint32_t i = 0; i < config_.vision_num_layers; i++) + hidden = build_vision_transformer_block(gb, hidden, i, cos_node, sin_node, attn_mask_node, backend); + } + + if (num_padding > 0) + hidden = gb->slice(hidden, 0, 0, num_real); + + return build_vision_pooler(gb, hidden, img, backend); +} + +size_t Gemma4VisionModel::build_vision_projector(CactusGraph* gb, size_t vision_features, ComputeBackend backend) { + size_t projected = gb->matmul(vision_features, vision_weights_.embed_vision_proj, true, backend); + return gb->rms_norm(projected, vision_weights_.post_proj_norm, config_.layer_norm_eps); +} + +} +} diff --git a/cactus-transpile/models/model.h b/cactus-transpile/models/model.h new file mode 100644 index 000000000..13382cd96 --- /dev/null +++ b/cactus-transpile/models/model.h @@ -0,0 +1,1773 @@ +#pragma once + +#include "../engine/engine.h" +#include "../npu/npu.h" + +namespace cactus { +namespace engine { + + + +class QwenModel : public Model { +public: + QwenModel(); + explicit QwenModel(const Config& config); + ~QwenModel() override = default; + +protected: + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + void load_weights_to_graph(CactusGraph* gb) override; + +private: + struct WeightNodeIDs { + size_t output_weight; + size_t output_norm_weight; + + struct LayerWeights { + size_t attn_q_weight; + size_t attn_k_weight; + size_t attn_v_weight; + size_t attn_output_weight; + size_t input_layernorm_weight; + size_t attn_q_norm_weight; + size_t attn_k_norm_weight; + size_t pre_feedforward_layernorm_weight; + size_t post_feedforward_layernorm_weight; + size_t ffn_gate_weight; + size_t ffn_up_weight; + size_t ffn_down_weight; + size_t post_attention_layernorm_weight; + }; + + std::vector layers; + } weight_nodes_; +}; + +class Qwen3p5Model : public Model { +public: + Qwen3p5Model(); + explicit Qwen3p5Model(const Config& config); + ~Qwen3p5Model() override = default; + +protected: + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + void load_weights_to_graph(CactusGraph* gb) override; + void post_init() override; + void post_execute_updates(CactusGraph* gb, size_t seq_len) override; + void reset_cache() override; + +private: + size_t build_gated_deltanet(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset); + + struct WeightNodeIDs { + size_t output_weight = 0; + size_t output_norm_weight = 0; + + enum class LayerType : uint8_t { ATTENTION, DELTANET }; + + struct LayerWeights { + size_t attn_q_weight = 0; + size_t attn_k_weight = 0; + size_t attn_v_weight = 0; + size_t attn_output_weight = 0; + size_t input_layernorm_weight = 0; + size_t attn_q_norm_weight = 0; + size_t attn_k_norm_weight = 0; + size_t deltanet_qkv_weight = 0; + size_t deltanet_gate_weight = 0; + size_t deltanet_beta_weight = 0; + size_t deltanet_gate_bias = 0; + size_t deltanet_beta_bias = 0; + size_t deltanet_z_weight = 0; + size_t deltanet_conv_weight = 0; + size_t ffn_gate_weight = 0; + size_t ffn_up_weight = 0; + size_t ffn_down_weight = 0; + size_t post_attention_layernorm_weight = 0; + }; + + struct LayerEntry { + LayerType type = LayerType::ATTENTION; + LayerWeights weights; + }; + + std::vector layers; + } weight_nodes_; + + ConvCache conv_cache_; + std::vector conv_cache_state_nodes_; + bool last_forward_used_cache_ = false; + size_t deltanet_total_seq_len_ = 0; + size_t deltanet_heads_ = 0; + size_t deltanet_key_dim_ = 0; + size_t deltanet_value_dim_ = 0; + size_t deltanet_state_flat_dim_ = 0; + size_t deltanet_mixed_dim_ = 0; + size_t deltanet_conv_history_len_ = 0; + size_t deltanet_conv_flat_dim_ = 0; + size_t deltanet_cache_row_dim_ = 0; +}; + + + +class GemmaModel : public Model { +public: + GemmaModel(); + explicit GemmaModel(const Config& config); + ~GemmaModel() override = default; + +protected: + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + void load_weights_to_graph(CactusGraph* gb) override; + void post_init() override; + +private: + struct WeightNodeIDs { + size_t output_weight; + size_t output_norm_weight; + + struct LayerWeights { + size_t attn_q_weight; + size_t attn_k_weight; + size_t attn_v_weight; + size_t attn_output_weight; + size_t input_layernorm_weight; + size_t attn_q_norm_weight; + size_t attn_k_norm_weight; + size_t pre_feedforward_layernorm_weight; + size_t post_feedforward_layernorm_weight; + size_t ffn_gate_weight; + size_t ffn_up_weight; + size_t ffn_down_weight; + size_t post_attention_layernorm_weight; + }; + + std::vector layers; + } weight_nodes_; +}; + +class GemmaModel3n : public Model { +public: + GemmaModel3n(); + explicit GemmaModel3n(const Config& config); + ~GemmaModel3n() override = default; + +protected: + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; + void load_weights_to_graph(CactusGraph* gb) override; + void post_init() override; + std::vector get_kv_layer_dims() const override; + +private: + size_t forward_split(const std::vector& tokens, bool use_cache); + + size_t build_preamble(CactusGraph* gb, size_t seq_len, ComputeBackend backend, + size_t& token_input, size_t& pli_input, size_t* streams); + void build_layer(CactusGraph* gb, uint32_t layer_idx, ComputeBackend backend, + bool use_cache, size_t pos_offset, size_t pli, size_t* streams); + size_t build_output_head(CactusGraph* gb, size_t* streams, ComputeBackend backend); + void set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, + const std::vector& tokens); + + size_t build_laurel(CactusGraph* gb, size_t normed_input, uint32_t layer_idx, ComputeBackend backend) const; + size_t build_gaussian_topk(CactusGraph* gb, size_t input, float ppf) const; + size_t build_rms_norm_no_weight(CactusGraph* gb, size_t input, size_t num_rows, size_t row_dim) const; + size_t build_magnitude_normalize(CactusGraph* gb, size_t reference, size_t target) const; + + size_t build_altup_router_modalities(CactusGraph* gb, size_t stream0, uint32_t layer_idx, ComputeBackend backend) const; + void build_altup_predict(CactusGraph* gb, size_t modalities, uint32_t layer_idx, + const size_t* streams, size_t* predictions) const; + void build_altup_correct(CactusGraph* gb, size_t activated, size_t modalities, uint32_t layer_idx, + ComputeBackend backend, const size_t* predictions, size_t* corrected) const; + void build_per_layer_input(CactusGraph* gb, size_t pli_combined, uint32_t layer_idx, + ComputeBackend backend, size_t* streams) const; + + struct WeightNodeIDs { + size_t output_weight; + size_t output_norm_weight; + + size_t altup_proj_weights[3]; + size_t altup_unembed_proj_weights[3]; + size_t embed_tokens_per_layer; + size_t per_layer_model_proj; + size_t per_layer_proj_norm; + + struct LayerWeights { + size_t attn_q_weight; + size_t attn_k_weight; + size_t attn_v_weight; + size_t attn_output_weight; + size_t input_layernorm_weight; + size_t attn_q_norm_weight; + size_t attn_k_norm_weight; + size_t pre_feedforward_layernorm_weight; + size_t post_feedforward_layernorm_weight; + size_t ffn_gate_weight; + size_t ffn_up_weight; + size_t ffn_down_weight; + size_t post_attention_layernorm_weight; + + size_t altup_router_norm; + size_t altup_prediction_coefs; + size_t altup_correction_coefs; + size_t altup_correct_output_scale; + size_t altup_modality_router; + size_t laurel_left; + size_t laurel_right; + size_t laurel_norm; + size_t per_layer_gate; + size_t per_layer_proj; + size_t post_per_layer_norm; + }; + + std::vector layers; + } weight_nodes_; + + std::vector kv_share_map_; + std::vector shared_k_nodes_; + std::vector shared_v_nodes_; + + std::vector<__fp16> v_norm_ones_weight_; + size_t v_norm_ones_node_ = 0; +}; + +class Siglip2VisionModel : public Model { + friend class Lfm2VlModel; + +public: + struct VisionEmbeddingResult { + size_t combined_embeddings; + std::vector tile_embeddings; + }; + + Siglip2VisionModel(); + explicit Siglip2VisionModel(const Config& cfg); + ~Siglip2VisionModel() override = default; + virtual size_t forward_vision(const Siglip2Preprocessor::PreprocessedImage& preprocessed_image); + virtual size_t forward_vision(CactusGraph* gb, + const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, + ComputeBackend backend); + std::vector get_image_embedding(const std::string& image_path); + Siglip2Preprocessor& get_preprocessor() { return preprocessor_; } + const Siglip2Preprocessor& get_preprocessor() const { return preprocessor_; } + +protected: + VisionEmbeddingResult build_vision_embeddings(CactusGraph* gb, + const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, + ComputeBackend backend); + + size_t build_vision_transformer_layer(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, + ComputeBackend backend); + + size_t build_vision_attention(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, + ComputeBackend backend); + + size_t build_vision_mlp(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, + ComputeBackend backend); + + void load_weights_to_graph(CactusGraph* gb) override; + size_t forward(const std::vector& tokens, bool use_cache = false) override; + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + +protected: + struct VisionWeightNodeIDs { + size_t patch_embedding_weight; + size_t patch_embedding_bias; + size_t position_embedding; + size_t post_layernorm_weight; + size_t post_layernorm_bias; + + struct VisionLayerWeights { + size_t attn_q_weight; + size_t attn_k_weight; + size_t attn_v_weight; + size_t attn_output_weight; + size_t attn_q_bias; + size_t attn_k_bias; + size_t attn_v_bias; + size_t attn_output_bias; + size_t layer_norm1_weight; + size_t layer_norm1_bias; + size_t layer_norm2_weight; + size_t layer_norm2_bias; + size_t mlp_fc1_weight; + size_t mlp_fc1_bias; + size_t mlp_fc2_weight; + size_t mlp_fc2_bias; + }; + + std::vector vision_layers; + } vision_weight_nodes_; + + Siglip2Preprocessor preprocessor_; + + std::unique_ptr npu_encoder_; + bool use_npu_encoder_ = false; +}; + + +class LFM2Model : public Model { + friend class Lfm2VlModel; + +public: + LFM2Model(); + explicit LFM2Model(const Config& config); + ~LFM2Model() override = default; + + bool is_cache_empty() const; + + bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; + bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true) override; + +protected: + using Model::forward; + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache); + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + size_t forward(CactusGraph* gb, const std::vector& tokens, ComputeBackend backend, bool use_cache = false); + size_t forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, ComputeBackend backend, bool use_cache = false); + void post_init() override; + void post_execute_updates(CactusGraph* gb, size_t seq_len) override; + void reset_cache() override; + void load_weights_to_graph(CactusGraph* gb) override; + +private: + + struct WeightNodeIDs { + size_t output_weight; + size_t output_norm_weight; + + struct LayerWeights { + size_t attn_q_weight; + size_t attn_k_weight; + size_t attn_v_weight; + size_t attn_output_weight; + size_t attn_q_norm_weight; + size_t attn_k_norm_weight; + + size_t conv_depthwise_weight; + size_t conv_in_proj_weight; + size_t conv_out_proj_weight; + + size_t input_layernorm_weight; + size_t post_attention_layernorm_weight; + size_t ffn_gate_weight; + size_t ffn_up_weight; + size_t ffn_down_weight; + }; + + enum class LayerType : uint8_t { ATTENTION, CONV }; + + struct LayerEntry { + LayerType type; + LayerWeights weights; + }; + + std::vector layers; + } weight_nodes_; + + ConvCache conv_cache_; + std::vector conv_cache_bx_nodes_; + bool last_forward_used_cache_ = false; +}; + + +class LFM2MoEModel : public Model { +public: + LFM2MoEModel(); + explicit LFM2MoEModel(const Config& config); + ~LFM2MoEModel() override = default; + + bool is_cache_empty() const; + + bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; + bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true) override; + +protected: + using Model::forward; + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache); + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + size_t forward(CactusGraph* gb, const std::vector& tokens, ComputeBackend backend, bool use_cache = false); + size_t forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, ComputeBackend backend, bool use_cache = false); + void post_init() override; + void post_execute_updates(CactusGraph* gb, size_t seq_len) override; + void reset_cache() override; + void load_weights_to_graph(CactusGraph* gb) override; + +private: + bool is_dense_layer(uint32_t layer_idx) const; + + struct WeightNodeIDs { + size_t output_weight; + size_t output_norm_weight; + + struct ExpertWeights { + size_t w1_weight = 0; + size_t w3_weight = 0; + size_t w2_weight = 0; + }; + + struct LayerWeights { + size_t attn_q_weight = 0; + size_t attn_k_weight = 0; + size_t attn_v_weight = 0; + size_t attn_output_weight = 0; + size_t attn_q_norm_weight = 0; + size_t attn_k_norm_weight = 0; + + size_t conv_depthwise_weight = 0; + size_t conv_in_proj_weight = 0; + size_t conv_out_proj_weight = 0; + + size_t input_layernorm_weight = 0; + size_t post_attention_layernorm_weight = 0; + size_t ffn_gate_weight = 0; + size_t ffn_up_weight = 0; + size_t ffn_down_weight = 0; + + size_t moe_router_weight = 0; + size_t moe_expert_bias = 0; + std::vector moe_experts; + }; + + enum class LayerType : uint8_t { ATTENTION, CONV }; + + struct LayerEntry { + LayerType type; + LayerWeights weights; + }; + + std::vector layers; + } weight_nodes_; + + ConvCache conv_cache_; + std::vector conv_cache_bx_nodes_; + bool last_forward_used_cache_ = false; +}; + + +class NomicModel : public Model { +public: + NomicModel(); + explicit NomicModel(const Config& config); + ~NomicModel() override = default; + +protected: + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + + void load_weights_to_graph(CactusGraph* gb) override; + +private: + size_t build_standard_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const; + + size_t build_moe_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const; + + struct WeightNodeIDs { + size_t embedding_layernorm_weight; + size_t embedding_layernorm_bias; + + struct LayerWeights { + size_t attn_q_weight; + size_t attn_k_weight; + size_t attn_v_weight; + size_t attn_q_bias; + size_t attn_k_bias; + size_t attn_v_bias; + size_t attn_output_weight; + size_t attn_output_bias; + size_t ffn_up_weight; + size_t ffn_up_bias; + size_t ffn_norm_1_weight; + size_t ffn_norm_1_bias; + size_t ffn_down_weight; + size_t ffn_down_bias; + size_t ffn_norm_2_weight; + size_t ffn_norm_2_bias; + size_t mlp_router_layer_weight; + size_t mlp_experts_bias; + std::vector mlp_experts_mlp1_weight; + std::vector mlp_experts_mlp2_weight; + }; + + std::vector layers; + } weight_nodes_; +}; + +class WhisperModel : public Model { +public: + WhisperModel(); + explicit WhisperModel(const Config& config); + ~WhisperModel() override = default; + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t,ComputeBackend, bool, size_t) override { + throw std::runtime_error("Whisper: build_attention unused"); + } + + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("Whisper: build_mlp unused"); + } + + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Whisper: build_transformer_block unused"); + } + + size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { + throw std::runtime_error("Whisper requires mel+token forward()."); + } + + size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; + + void run_encoder(const std::vector& audio_features); + void reset_graph_side_cache_nodes(); + + size_t run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only); + + void load_weights_to_graph(CactusGraph* gb) override; + + size_t build_encoder_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_encoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend); + + size_t build_decoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const; + + size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_conv1d(CactusGraph* gb, size_t input); + + uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, + float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; + + std::vector get_audio_embeddings(const std::vector& audio_features) override; + + void reset_cache() override; + +private: + struct WeightNodeIDs { + size_t output_weight; + size_t output_norm_weight; + + size_t decoder_norm_weight; + size_t decoder_norm_bias; + size_t decoder_position_embeddings_weight; + + size_t encoder_position_embeddings; + size_t encoder_conv1_weight; + size_t encoder_conv1_bias; + size_t encoder_conv2_weight; + size_t encoder_conv2_bias; + size_t encoder_conv3_weight; + size_t encoder_conv3_bias; + size_t encoder_norm_weight; + size_t encoder_norm_bias; + + size_t encoder_output; + + struct LayerWeights { + size_t decoder_output_norm_bias; + size_t decoder_output_norm_weight; + size_t decoder_position_embeddings_weight; + size_t decoder_token_embeddings_weight; + + size_t decoder_encoder_attn_q_weight; + size_t decoder_encoder_attn_k_weight; + size_t decoder_encoder_attn_v_weight; + size_t decoder_encoder_attn_q_bias; + size_t decoder_encoder_attn_v_bias; + size_t decoder_encoder_attn_output_weight; + size_t decoder_encoder_attn_output_bias; + + size_t decoder_post_encoder_layernorm_weight; + size_t decoder_post_encoder_layernorm_bias; + + size_t decoder_ffn1_weight; + size_t decoder_ffn1_bias; + size_t decoder_ffn2_weight; + size_t decoder_ffn2_bias; + + size_t decoder_post_ffn_layernorm_weight; + size_t decoder_post_ffn_layernorm_bias; + + size_t decoder_self_attn_q_weight; + size_t decoder_self_attn_k_weight; + size_t decoder_self_attn_v_weight; + size_t decoder_self_attn_q_bias; + size_t decoder_self_attn_v_bias; + size_t decoder_self_attn_output_weight; + size_t decoder_self_attn_output_bias; + + size_t decoder_post_attn_layernorm_weight; + size_t decoder_post_attn_layernorm_bias; + + //Encoder layers + size_t encoder_ffn1_weight; + size_t encoder_ffn1_bias; + size_t encoder_ffn2_weight; + size_t encoder_ffn2_bias; + + size_t encoder_post_ffn_layernorm_weight; + size_t encoder_post_ffn_layernorm_bias; + + size_t encoder_self_attn_q_weight; + size_t encoder_self_attn_k_weight; + size_t encoder_self_attn_v_weight; + size_t encoder_self_attn_q_bias; + size_t encoder_self_attn_v_bias; + size_t encoder_self_attn_output_weight; + size_t encoder_self_attn_output_bias; + + size_t encoder_post_attn_layernorm_weight; + size_t encoder_post_attn_layernorm_bias; + }; + + std::vector layers; + } weight_nodes_; + + bool encoder_ready_ = false; + size_t last_new_tokens_; + size_t last_conv1_node_ = 0; + size_t last_conv2_node_ = 0; + size_t last_encoder_post_norm_node_ = 0; + size_t last_enc_plus_pos_node_ = 0; + size_t encoder_transformer_block_0 = 0; + size_t encoder_pre_gelu = 0; + size_t encoder_post_gelu = 0; + + std::vector encoder_block_out_nodes_; + + size_t encoder_output_persistent_ = 0; + + std::vector suppress_tokens_ = { + 1, + 2, + 7, + 8, + 9, + 10, + 14, + 25, + 26, + 27, + 28, + 29, + 31, + 58, + 59, + 60, + 61, + 62, + 63, + 90, + 91, + 92, + 93, + 359, + 503, + 522, + 542, + 873, + 893, + 902, + 918, + 922, + 931, + 1350, + 1853, + 1982, + 2460, + 2627, + 3246, + 3253, + 3268, + 3536, + 3846, + 3961, + 4183, + 4667, + 6585, + 6647, + 7273, + 9061, + 9383, + 10428, + 10929, + 11938, + 12033, + 12331, + 12562, + 13793, + 14157, + 14635, + 15265, + 15618, + 16553, + 16604, + 18362, + 18956, + 20075, + 21675, + 22520, + 26130, + 26161, + 26435, + 28279, + 29464, + 31650, + 32302, + 32470, + 36865, + 42863, + 47425, + 49870, + 50254, + 50258, + 50358, + 50359, + 50360, + 50361, + 50362 + }; + + std::vector begin_suppress_tokens_ = { + 220, + 50257 + }; + + bool first_decode_step_ = true; + + std::unordered_map suppress_bias_; + std::unordered_map suppress_bias_first_step_; + + std::vector encoder_k_persistent_; + std::vector encoder_v_persistent_; + + std::unique_ptr npu_encoder_; + bool use_npu_encoder_ = false; + + uint32_t enc_layers_ = 0; + uint32_t dec_layers_ = 0; +}; + + +class MoonshineModel : public Model { +public: + MoonshineModel(); + explicit MoonshineModel(const Config& config); + ~MoonshineModel() override = default; + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t,ComputeBackend, bool, size_t) override { + throw std::runtime_error("Moonshine: build_attention unused"); + } + + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("Moonshine: build_mlp unused"); + } + + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Moonshine: build_transformer_block unused"); + } + + size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { + throw std::runtime_error("Moonshine requires mel+token forward()."); + } + + size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; + + size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); + + void reset_graph_side_cache_nodes(); + + size_t build_decoder(const std::vector& tokens, bool use_cache, bool last_token_only); + + void load_weights_to_graph(CactusGraph* gb) override; + + size_t build_decoder_cross_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_encoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend); + + size_t build_decoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const; + + size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + + size_t build_conv1d(CactusGraph* gb, size_t input); + size_t build_audio_preprocessor(CactusGraph* gb, size_t input); + + uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, + float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; + + std::vector get_audio_embeddings(const std::vector& audio_features) override; + + void reset_cache() override; + +private: + struct WeightNodeIDs { + size_t output_weight; + + size_t decoder_norm_weight; + + size_t encoder_conv1_weight; + size_t encoder_conv2_weight; + size_t encoder_conv2_bias; + size_t encoder_conv3_weight; + size_t encoder_conv3_bias; + size_t encoder_norm_weight; + size_t encoder_norm_bias; + size_t encoder_layer_norm_weight; + + + struct LayerWeights { + //Decoder layers + + size_t decoder_encoder_attn_q_weight; + size_t decoder_encoder_attn_k_weight; + size_t decoder_encoder_attn_v_weight; + size_t decoder_encoder_attn_output_weight; + + size_t decoder_post_encoder_layernorm_weight; + + size_t decoder_ffn1_weight; + size_t decoder_ffn1_bias; + size_t decoder_ffn_gate_weight; + size_t decoder_ffn_gate_bias; + size_t decoder_ffn_up_weight; + size_t decoder_ffn_up_bias; + size_t decoder_ffn2_weight; + size_t decoder_ffn2_bias; + + size_t decoder_post_ffn_layernorm_weight; + + size_t decoder_self_attn_q_weight; + size_t decoder_self_attn_k_weight; + size_t decoder_self_attn_v_weight; + size_t decoder_self_attn_output_weight; + + size_t decoder_post_attn_layernorm_weight; + + //Encoder layers + size_t encoder_ffn1_weight; + size_t encoder_ffn1_bias; + size_t encoder_ffn_gate_weight; + size_t encoder_ffn_gate_bias; + size_t encoder_ffn_up_weight; + size_t encoder_ffn_up_bias; + size_t encoder_ffn2_weight; + size_t encoder_ffn2_bias; + + size_t encoder_post_ffn_layernorm_weight; + + size_t encoder_self_attn_q_weight; + size_t encoder_self_attn_k_weight; + size_t encoder_self_attn_v_weight; + size_t encoder_self_attn_output_weight; + + size_t encoder_post_attn_layernorm_weight; + }; + + std::vector encoder_layers; + std::vector decoder_layers; + } weight_nodes_; + + bool encoder_ready_ = false; + size_t last_new_tokens_; + std::vector encoder_output_host_; + std::vector encoder_output_shape_; + size_t last_conv1_node_ = 0; + size_t last_conv2_node_ = 0; + size_t last_encoder_post_norm_node_ = 0; + size_t last_enc_plus_pos_node_ = 0; + size_t encoder_transformer_block_0 = 0; + size_t encoder_pre_gelu = 0; + size_t encoder_post_gelu = 0; + + std::vector encoder_block_out_nodes_; + std::vector encoder_output_bytes_; + Precision encoder_output_precision_ = Precision::FP32; + + bool first_decode_step_ = true; + + std::vector encoder_k_persistent_; + std::vector encoder_v_persistent_; + + Precision encoder_kv_precision_ = Precision::FP32; + bool encoder_kv_ready_ = false; + + std::unique_ptr npu_encoder_; + bool use_npu_encoder_ = false; + +}; + +class NeedleModel : public Model { +public: + NeedleModel(); + explicit NeedleModel(const Config& config); + ~NeedleModel() override = default; + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Needle: build_attention unused"); + } + + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("Needle: build_mlp unused"); + } + + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Needle: build_transformer_block unused"); + } + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; + uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f) override; + void load_weights_to_graph(CactusGraph* gb) override; + void reset_cache() override; + + void run_encoder(const std::vector& encoder_tokens); + void reset_graph_side_cache_nodes(); + size_t run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only); + + size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + size_t build_decoder_cross_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); + +private: + struct WeightNodeIDs { + size_t output_weight = 0; + size_t encoder_norm_weight = 0; + size_t decoder_norm_weight = 0; + + struct EncoderLayerWeights { + size_t attn_gate_weight = 0; + size_t attn_q_weight = 0; + size_t attn_k_weight = 0; + size_t attn_v_weight = 0; + size_t attn_output_weight = 0; + size_t input_norm_weight = 0; + size_t attn_q_norm_weight = 0; + size_t attn_k_norm_weight = 0; + }; + + struct DecoderLayerWeights { + size_t self_attn_gate_weight = 0; + size_t cross_attn_gate_weight = 0; + size_t self_attn_q_weight = 0; + size_t self_attn_k_weight = 0; + size_t self_attn_v_weight = 0; + size_t self_attn_output_weight = 0; + size_t self_attn_q_norm_weight = 0; + size_t self_attn_k_norm_weight = 0; + + size_t encoder_attn_q_weight = 0; + size_t encoder_attn_k_weight = 0; + size_t encoder_attn_v_weight = 0; + size_t encoder_attn_output_weight = 0; + size_t encoder_attn_q_norm_weight = 0; + size_t encoder_attn_k_norm_weight = 0; + + size_t input_norm_weight = 0; + size_t post_attn_norm_weight = 0; + }; + + std::vector encoder_layers; + std::vector decoder_layers; + } weight_nodes_; + + bool encoder_ready_ = false; + size_t last_encoder_post_norm_node_ = 0; + std::vector encoder_k_persistent_; + std::vector encoder_v_persistent_; +}; + +class ParakeetModel : public Model { +public: + ParakeetModel(); + explicit ParakeetModel(const Config& config); + ~ParakeetModel() override = default; + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Parakeet: build_attention unused"); + } + + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("Parakeet: build_mlp unused"); + } + + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("Parakeet: build_transformer_block unused"); + } + + size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { + throw std::runtime_error("Parakeet requires audio feature forward()."); + } + + size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; + void load_weights_to_graph(CactusGraph* gb) override; + uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, + float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, + const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; + std::vector get_audio_embeddings(const std::vector& audio_features) override; + void reset_cache() override; + +private: + size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); + size_t build_ctc_logits(CactusGraph* gb, size_t hidden_states); + size_t build_subsampling(CactusGraph* gb, const std::vector& audio_features); + size_t build_relative_position_embeddings(CactusGraph* gb, size_t seq_len); + size_t build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); + size_t build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend); + size_t build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend); + size_t build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); + std::vector greedy_decode_tokens(CactusGraph* gb, size_t logits_node) const; + + struct WeightNodeIDs { + size_t ctc_head_weight = 0; + size_t ctc_head_bias = 0; + + size_t subsampling_conv0_weight = 0; + size_t subsampling_conv0_bias = 0; + size_t subsampling_depthwise1_weight = 0; + size_t subsampling_depthwise1_bias = 0; + size_t subsampling_pointwise1_weight = 0; + size_t subsampling_pointwise1_bias = 0; + size_t subsampling_depthwise2_weight = 0; + size_t subsampling_depthwise2_bias = 0; + size_t subsampling_pointwise2_weight = 0; + size_t subsampling_pointwise2_bias = 0; + size_t subsampling_linear_weight = 0; + size_t subsampling_linear_bias = 0; + + struct LayerWeights { + size_t ff1_linear1_weight = 0; + size_t ff1_linear1_bias = 0; + size_t ff1_linear2_weight = 0; + size_t ff1_linear2_bias = 0; + + size_t ff2_linear1_weight = 0; + size_t ff2_linear1_bias = 0; + size_t ff2_linear2_weight = 0; + size_t ff2_linear2_bias = 0; + + size_t self_attn_q_weight = 0; + size_t self_attn_q_bias = 0; + size_t self_attn_k_weight = 0; + size_t self_attn_k_bias = 0; + size_t self_attn_v_weight = 0; + size_t self_attn_v_bias = 0; + size_t self_attn_output_weight = 0; + size_t self_attn_output_bias = 0; + size_t self_attn_relative_k_weight = 0; + size_t self_attn_bias_u = 0; + size_t self_attn_bias_v = 0; + + size_t norm_ff1_weight = 0; + size_t norm_ff1_bias = 0; + size_t norm_self_attn_weight = 0; + size_t norm_self_attn_bias = 0; + size_t norm_conv_weight = 0; + size_t norm_conv_bias = 0; + size_t norm_ff2_weight = 0; + size_t norm_ff2_bias = 0; + size_t norm_out_weight = 0; + size_t norm_out_bias = 0; + + size_t conv_pointwise1_weight = 0; + size_t conv_pointwise1_bias = 0; + size_t conv_depthwise_weight = 0; + size_t conv_depthwise_bias = 0; + size_t conv_pointwise2_weight = 0; + size_t conv_pointwise2_bias = 0; + size_t conv_batchnorm_weight = 0; + size_t conv_batchnorm_bias = 0; + size_t conv_batchnorm_running_mean = 0; + size_t conv_batchnorm_running_var = 0; + }; + + std::vector layers; + } weight_nodes_; + + bool ctc_tokens_ready_ = false; + size_t ctc_emit_index_ = 0; + std::vector ctc_tokens_; + size_t last_input_token_count_ = 0; + + std::unique_ptr npu_encoder_; + bool use_npu_encoder_ = false; + bool has_cpu_encoder_weights_ = false; +}; + +class ParakeetTDTModel : public Model { +public: + ParakeetTDTModel(); + explicit ParakeetTDTModel(const Config& config); + ~ParakeetTDTModel() override = default; + + struct TDTToken { uint32_t id; float time_start; float time_end; }; + + struct ChunkStreamState { + bool initialized = false; + uint32_t last_token = 0; + std::vector> h; + std::vector> c; + }; + + struct ChunkStreamResult { + std::string text; + std::string confirmed_text; + std::string pending_text; + size_t token_count = 0; + size_t confirmed_token_count = 0; + double raw_decoder_tps = 0.0; + double raw_decoder_time_ms = 0.0; + float start_sec = 0.0f; + float confirmed_end_sec = 0.0f; + float resume_end_sec = 0.0f; + float end_sec = 0.0f; + }; + + ChunkStreamResult decode_chunk_stream( + const std::vector& audio_features, + size_t replay_start_frame, + size_t start_frame, + size_t end_frame, + ChunkStreamState& state); + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("ParakeetTDT: build_attention unused"); + } + + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("ParakeetTDT: build_mlp unused"); + } + + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("ParakeetTDT: build_transformer_block unused"); + } + + size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { + throw std::runtime_error("ParakeetTDT requires audio feature forward()."); + } + + size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; + void load_weights_to_graph(CactusGraph* gb) override; + uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, + float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, + const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; + std::vector get_audio_embeddings(const std::vector& audio_features) override; + void reset_cache() override; + +private: + size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); + size_t build_subsampling(CactusGraph* gb, const std::vector& audio_features); + size_t build_relative_position_embeddings(CactusGraph* gb, size_t seq_len); + size_t build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); + size_t build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend); + size_t build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend); + size_t build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); + std::vector decode_tdt_tokens_with_state( + CactusGraph* gb, + size_t encoder_hidden_node, + size_t replay_start_frame, + size_t start_frame, + size_t end_frame, + ChunkStreamState* stream_state, + size_t* out_confirmed_count = nullptr, + double* out_raw_decoder_time_ms = nullptr) const; + std::vector greedy_decode_tdt_tokens(CactusGraph* gb, size_t encoder_hidden_node) const; + + struct WeightNodeIDs { + size_t subsampling_conv0_weight = 0; + size_t subsampling_conv0_bias = 0; + size_t subsampling_depthwise1_weight = 0; + size_t subsampling_depthwise1_bias = 0; + size_t subsampling_pointwise1_weight = 0; + size_t subsampling_pointwise1_bias = 0; + size_t subsampling_depthwise2_weight = 0; + size_t subsampling_depthwise2_bias = 0; + size_t subsampling_pointwise2_weight = 0; + size_t subsampling_pointwise2_bias = 0; + size_t subsampling_linear_weight = 0; + size_t subsampling_linear_bias = 0; + + struct LayerWeights { + size_t ff1_linear1_weight = 0; + size_t ff1_linear1_bias = 0; + size_t ff1_linear2_weight = 0; + size_t ff1_linear2_bias = 0; + + size_t ff2_linear1_weight = 0; + size_t ff2_linear1_bias = 0; + size_t ff2_linear2_weight = 0; + size_t ff2_linear2_bias = 0; + + size_t self_attn_q_weight = 0; + size_t self_attn_q_bias = 0; + size_t self_attn_k_weight = 0; + size_t self_attn_k_bias = 0; + size_t self_attn_v_weight = 0; + size_t self_attn_v_bias = 0; + size_t self_attn_output_weight = 0; + size_t self_attn_output_bias = 0; + size_t self_attn_relative_k_weight = 0; + size_t self_attn_bias_u = 0; + size_t self_attn_bias_v = 0; + + size_t norm_ff1_weight = 0; + size_t norm_ff1_bias = 0; + size_t norm_self_attn_weight = 0; + size_t norm_self_attn_bias = 0; + size_t norm_conv_weight = 0; + size_t norm_conv_bias = 0; + size_t norm_ff2_weight = 0; + size_t norm_ff2_bias = 0; + size_t norm_out_weight = 0; + size_t norm_out_bias = 0; + + size_t conv_pointwise1_weight = 0; + size_t conv_pointwise1_bias = 0; + size_t conv_depthwise_weight = 0; + size_t conv_depthwise_bias = 0; + size_t conv_pointwise2_weight = 0; + size_t conv_pointwise2_bias = 0; + size_t conv_batchnorm_weight = 0; + size_t conv_batchnorm_bias = 0; + size_t conv_batchnorm_running_mean = 0; + size_t conv_batchnorm_running_var = 0; + }; + + struct PredictorLayerWeights { + size_t weight_ih = 0; + size_t weight_hh = 0; + size_t bias = 0; + }; + + size_t predictor_embed = 0; + std::vector predictor_layers; + size_t joint_enc_weight = 0; + size_t joint_enc_bias = 0; + size_t joint_pred_weight = 0; + size_t joint_pred_bias = 0; + size_t joint_out_weight = 0; + size_t joint_out_bias = 0; + + std::vector layers; + } weight_nodes_; + + bool tdt_tokens_ready_ = false; + size_t tdt_emit_index_ = 0; + std::vector tdt_tokens_; + size_t last_input_token_count_ = 0; + + std::unique_ptr npu_encoder_; + bool use_npu_encoder_ = false; + bool has_cpu_encoder_weights_ = false; +}; + + +class Lfm2VlModel : public Model { +public: + Lfm2VlModel(); + explicit Lfm2VlModel(const Config& config); + ~Lfm2VlModel() override = default; + + bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; + size_t forward(const std::vector& tokens, bool use_cache = false) override; + + uint32_t decode(const std::vector& tokens, + float temperature = -1.0f, + float top_p = -1.0f, + size_t top_k = 0, + const std::string& profile_file = "", + float* out_entropy = nullptr, + float min_p = 0.15f, + float repetition_penalty = 1.1f) override; + + void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; + + void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file = "") override; + + uint32_t decode_with_images( + const std::vector& tokens, + const std::vector& image_paths, + float temperature = -1.0f, + float top_p = -1.0f, + size_t top_k = 0, + const std::string& profile_file = "", + float* out_entropy = nullptr, + float min_p = 0.15f, + float repetition_penalty = 1.1f) override; + + void reset_cache() override; + std::vector get_image_embeddings(const std::string& image_path) override; + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override; + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; + + void load_weights_to_graph(CactusGraph* gb) override; + +private: + struct ProjectedTileFeature { + size_t node_id; + size_t token_count; + }; + + struct TextEmbeddingInput { + size_t input_node; + std::vector tokens; + }; + + struct MergedEmbeddingResult { + size_t node_id; + size_t seq_len; + }; + + struct ForwardImageResult { + size_t final_hidden_node; + size_t seq_len; + }; + std::vector get_image_features( + CactusGraph* gb, + const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, + ComputeBackend backend); + + ForwardImageResult forward_images( + CactusGraph* gb, + const std::vector& tokens, + const std::vector& image_paths, + ComputeBackend backend, + bool use_cache); + size_t build_multimodal_projector( + CactusGraph* gb, + size_t image_features, + size_t tile_h, + size_t tile_w, + ComputeBackend backend); + size_t pixel_unshuffle(CactusGraph* gb, size_t hidden_states, size_t height, size_t width, size_t channels); + MergedEmbeddingResult merge_image_text_embeddings( + CactusGraph* gb, + const std::vector& tokens, + const std::vector>& image_embedding_nodes, + std::vector& text_embedding_inputs); + Siglip2VisionModel vision_tower_; + LFM2Model language_model_; + Siglip2Preprocessor preprocessor_; + struct ProjectorWeights { + size_t layer_norm_weight; + size_t layer_norm_bias; + size_t linear_1_weight; + size_t linear_1_bias; + size_t linear_2_weight; + size_t linear_2_bias; + } projector_weights_; + + bool vision_weights_loaded_ = false; + bool language_weights_loaded_ = false; + + bool image_prefill_completed_ = false; + size_t last_token_count_ = 0; +}; + +class SileroVADModel : public Model { +public: + static constexpr size_t CONTEXT_SIZE = 64; + static constexpr size_t CHUNK_SIZE = 512; + static constexpr size_t REFLECT_PAD_SIZE = 64; + static constexpr size_t HIDDEN_SIZE = 128; + static constexpr size_t GATE_SIZE = 512; + + SileroVADModel(); + explicit SileroVADModel(const Config& config); + ~SileroVADModel() override; + + bool init(const std::string& model_folder, size_t context_size = 0, + const std::string& system_prompt = "", bool do_warmup = false) override; + + struct SpeechTimestamp { + size_t start; + size_t end; + }; + + struct SpeechTimestampsOptions { + float threshold = 0.5f; + float neg_threshold = 0.0f; + int min_speech_duration_ms = 250; + float max_speech_duration_s = std::numeric_limits::infinity(); + int min_silence_duration_ms = 100; + int speech_pad_ms = 30; + int window_size_samples = 512; + int min_silence_at_max_speech = 98; + bool use_max_poss_sil_at_max_speech = true; + int sampling_rate = 16000; + }; + + float process_chunk(const std::vector& audio_chunk); + void reset_states(); + std::vector get_speech_timestamps(const std::vector& audio, const SpeechTimestampsOptions& options); + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("SileroVAD: build_attention unused"); + } + + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("SileroVAD: build_mlp unused"); + } + + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("SileroVAD: build_transformer_block unused"); + } + + size_t forward(const std::vector&, bool = false) override { + throw std::runtime_error("SileroVAD: use process_chunk() instead"); + } + + size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; + + void load_weights_to_graph(CactusGraph* gb) override; + +private: + void build_graph(); + + struct VADGraphNodes { + size_t input; + size_t h_prev; + size_t c_prev; + size_t h_new; + size_t c_new; + size_t output; + size_t lstm_output; + size_t encoder_output; + } graph_nodes_; + + struct VADWeightNodes { + size_t stft_basis; + struct EncoderBlock { + size_t conv_weight; + size_t conv_bias; + }; + std::vector encoder_blocks; + size_t lstm_weight_ih; + size_t lstm_weight_hh; + size_t lstm_bias_ih; + size_t lstm_bias_hh; + size_t output_conv_weight; + size_t output_conv_bias; + } weight_nodes_; + + struct VADState { + std::vector<__fp16> h; + std::vector<__fp16> c; + std::vector context; + std::vector input_buf; + std::vector<__fp16> input_fp16; + } state_; + + CactusGraph graph_; + std::string weights_path_; +}; + +class YoutuModel : public Model { +public: + YoutuModel(); + explicit YoutuModel(const Config& config); + ~YoutuModel() override = default; + + void reset_cache() override; + +protected: + size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const override; + + size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; + + size_t forward(const std::vector& tokens, bool use_cache = false) override; + void load_weights_to_graph(CactusGraph* gb) override; + std::vector get_kv_layer_dims() const override; + void post_init() override; + void post_execute_updates(CactusGraph* gb, size_t seq_len) override; + +private: + KVCache v_cache_; + std::vector cache_v_nodes_; + + struct WeightNodeIDs { + size_t output_weight = 0; + size_t output_norm_weight = 0; + + struct LayerWeights { + size_t attn_q_weight = 0; + size_t attn_q_a_weight = 0; + size_t attn_q_a_norm_weight = 0; + size_t attn_q_b_weight = 0; + size_t attn_kv_a_weight = 0; + size_t attn_kv_a_norm_weight = 0; + size_t attn_kv_b_weight = 0; + size_t attn_output_weight = 0; + size_t input_layernorm_weight = 0; + size_t post_attention_layernorm_weight = 0; + size_t ffn_gate_weight = 0; + size_t ffn_up_weight = 0; + size_t ffn_down_weight = 0; + size_t attn_q_a_bias = 0; + size_t attn_kv_a_bias = 0; + size_t attn_output_bias = 0; + }; + + std::vector layers; + } weight_nodes_; +}; + +class PyAnnoteModel : public Model { +public: + PyAnnoteModel(); + explicit PyAnnoteModel(const Config& config); + ~PyAnnoteModel() override = default; + + bool init(const std::string& model_folder, size_t context_size = 0, + const std::string& system_prompt = "", bool do_warmup = false) override; + + std::vector diarize(const float* pcm_f32, size_t num_samples, size_t step_samples = 16000, bool raw_powerset = false); + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("PyAnnote: build_attention unused"); + } + + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("PyAnnote: build_mlp unused"); + } + + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("PyAnnote: build_transformer_block unused"); + } + + size_t forward(const std::vector&, bool = false) override { + throw std::runtime_error("PyAnnote: use diarize() instead"); + } + + void load_weights_to_graph(CactusGraph* gb) override; + +private: + void build_graph(); + + struct LSTMLayerWeights { + size_t w_ih_fwd, w_hh_fwd, b_ih_fwd, b_hh_fwd; + size_t w_ih_bwd, w_hh_bwd, b_ih_bwd, b_hh_bwd; + }; + + struct WeightNodeIDs { + size_t sinc_filters; + size_t wav_norm_weight, wav_norm_bias; + size_t norm0_weight, norm0_bias; + size_t conv1_weight, conv1_bias; + size_t norm1_weight, norm1_bias; + size_t conv2_weight, conv2_bias; + size_t norm2_weight, norm2_bias; + LSTMLayerWeights lstm_layers[4]; + size_t linear0_weight, linear0_bias; + size_t linear1_weight, linear1_bias; + size_t classifier_weight, classifier_bias; + } weight_nodes_; + + CactusGraph graph_; + size_t audio_input_ = 0; + size_t output_node_ = 0; + std::vector hamming_; + std::vector<__fp16> chunk_buf_; +}; + +class WeSpeakerModel : public Model { +public: + WeSpeakerModel(); + explicit WeSpeakerModel(const Config& config); + ~WeSpeakerModel() override = default; + + bool init(const std::string& model_folder, size_t context_size = 0, + const std::string& system_prompt = "", bool do_warmup = false) override; + + std::vector embed(const float* fbank_features, size_t num_features); + std::vector embed(const float* fbank_features, size_t num_features, + const float* mask_weights, size_t mask_num_frames); + +protected: + size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("WeSpeaker: build_attention unused"); + } + + size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { + throw std::runtime_error("WeSpeaker: build_mlp unused"); + } + + size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { + throw std::runtime_error("WeSpeaker: build_transformer_block unused"); + } + + size_t forward(const std::vector&, bool = false) override { + throw std::runtime_error("WeSpeaker: use embed() instead"); + } + + void load_weights_to_graph(CactusGraph* gb) override; + +private: + void build_graph(size_t num_frames); + void build_graph_masked(size_t num_frames); + + struct ResBlockWeights { + size_t conv1_w, conv1_b; + size_t conv2_w, conv2_b; + size_t shortcut_conv_w, shortcut_conv_b; + bool has_shortcut = false; + }; + + ResBlockWeights load_resblock(CactusGraph* gb, const std::string& prefix, bool has_shortcut); + size_t build_resblock(CactusGraph* gb, size_t x, const ResBlockWeights& rb, bool stride2); + size_t build_resnet_body(CactusGraph* gb, size_t x); + + struct WeightNodeIDs { + size_t conv1_w, conv1_b; + std::vector layer1, layer2, layer3, layer4; + size_t seg1_w, seg1_b; + } weight_nodes_; + + CactusGraph graph_; + size_t audio_input_ = 0; + size_t mask_input_ = 0; + size_t output_node_ = 0; + size_t current_num_frames_ = 0; + bool current_masked_ = false; + std::vector<__fp16> input_buf_; + std::vector mask_buf_; +}; + +} +} + +#include "gemma4/model_gemma4.h" diff --git a/cactus-transpile/models/model_gemma.cpp b/cactus-transpile/models/model_gemma.cpp new file mode 100644 index 000000000..a128c8905 --- /dev/null +++ b/cactus-transpile/models/model_gemma.cpp @@ -0,0 +1,192 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../npu/npu.h" +#include +#include + + +namespace cactus { +namespace engine { + +GemmaModel::GemmaModel() : Model() {} + +GemmaModel::GemmaModel(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); +} + +void GemmaModel::post_init() { + kv_cache_.set_window_size(0, 0); +} + +void GemmaModel::load_weights_to_graph(CactusGraph* gb) { + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + } + + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer = weight_nodes_.layers[i]; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); + layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); + layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); + layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); + layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); + layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); + layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); + layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); + layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); + layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); + layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); + layer.pre_feedforward_layernorm_weight = gb->mmap_weights(layer_prefix + "pre_ffn_norm.weights"); + layer.post_feedforward_layernorm_weight = gb->mmap_weights(layer_prefix + "post_ffn_norm.weights"); + } + + if (npu::is_npu_available()) { + std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; + load_npu_prefill(npu_prefill_path); + } +} + +size_t GemmaModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); + auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); + auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); + + const auto& q_shape = gb->get_output_buffer(q_proj).shape; + size_t batch_seq = q_shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); + q_proj = gb->rms_norm(q_proj, layer.attn_q_norm_weight, config_.layer_norm_eps); + q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); + + size_t num_kv_heads = config_.attention_kv_heads; + k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); + k_proj = gb->rms_norm(k_proj, layer.attn_k_norm_weight, config_.layer_norm_eps); + k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); + + size_t seq_len = batch_seq; + + auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); + auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + + bool is_global_attention = ((layer_idx + 1) % 6) == 0; + + if (config_.rope_theta > 0) { + float rope_freq = is_global_attention ? 1000000.0f : 10000.0f; + + q_proj_4d = gb->rope(q_proj_4d, rope_freq, position_offset); + k_proj_4d = gb->rope(k_proj_4d, rope_freq, position_offset); + } + + if (use_cache) { + cache_k_output_nodes_[layer_idx] = k_proj_4d; + cache_v_output_nodes_[layer_idx] = v_proj_4d; + } + + size_t attn_output_4d; + size_t window_size = is_global_attention ? 0 : 512; + + if (use_cache && !kv_cache_.is_empty()) { + attn_output_4d = gb->attention_int8_hybrid( + q_proj_4d, k_proj_4d, v_proj_4d, + attention_scale_, position_offset, + kv_cache_.get_keys_int8(layer_idx), + kv_cache_.get_values_int8(layer_idx), + kv_cache_.get_key_scales(layer_idx), + kv_cache_.get_value_scales(layer_idx), + kv_cache_.current_seq_len, num_kv_heads, head_dim, window_size + ); + } else { + attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset, window_size); + } + auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); + return gb->matmul(attn_output, layer.attn_output_weight, true, backend); +} + + +size_t GemmaModel::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t gate_output = gb->matmul(input, layer.ffn_gate_weight, true, backend); + size_t up_output = gb->matmul(input, layer.ffn_up_weight, true, backend); + size_t gate_gelu = gb->gelu(gate_output); + size_t gated = gb->multiply(gate_gelu, up_output); + return gb->matmul(gated, layer.ffn_down_weight, true, backend); +} + +size_t GemmaModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); + auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); + + auto normalized_attn = gb->rms_norm(attn_output, layer.post_attention_layernorm_weight, config_.layer_norm_eps); + + auto after_attention = gb->add_clipped(hidden, normalized_attn); + + auto pre_mlp_norm = gb->rms_norm(after_attention, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); + auto mlp_output = build_mlp(gb, pre_mlp_norm, layer_idx, backend); + + auto normalized_mlp = gb->rms_norm(mlp_output, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); + + return gb->add_clipped(after_attention, normalized_mlp); +} + + +size_t GemmaModel::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + if (tokens.empty()) { + throw std::runtime_error("Token sequence cannot be empty"); + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + auto seq_len = static_cast(tokens.size()); + + size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + auto input_node_id = gb->input({seq_len}, Precision::FP32); + auto hidden = gb->embedding(embedding_node_id_, input_node_id); + + float embed_scale = std::sqrt(static_cast(config_.hidden_dim)); + hidden = gb->scalar_multiply(hidden, embed_scale); + + for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { + hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); + } + + auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + + std::vector input_data(seq_len); + for (size_t i = 0; i < seq_len; i++) { + input_data[i] = static_cast(tokens[i]); + } + gb->set_input(input_node_id, input_data.data(), Precision::FP32); + + return final_hidden; +} + +} +} \ No newline at end of file diff --git a/cactus-transpile/models/model_gemma3n.cpp b/cactus-transpile/models/model_gemma3n.cpp new file mode 100644 index 000000000..65a3b8173 --- /dev/null +++ b/cactus-transpile/models/model_gemma3n.cpp @@ -0,0 +1,475 @@ +#include "model.h" +#include "../graph/graph.h" +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +static const float RSQRT2 = 1.0f / std::sqrt(2.0f); + +GemmaModel3n::GemmaModel3n() : Model() {} + +GemmaModel3n::GemmaModel3n(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); +} + +std::vector GemmaModel3n::get_kv_layer_dims() const { + uint32_t n = config_.num_layers; + uint32_t num_shared = config_.num_kv_shared_layers; + uint32_t first_shared = (n > num_shared) ? n - num_shared : n; + + std::vector dims(n); + for (uint32_t i = 0; i < n; i++) { + dims[i] = (i >= first_shared) ? 0 : config_.attention_head_dim; + } + return dims; +} + +void GemmaModel3n::post_init() { + kv_cache_.set_window_size(0, 0); + + uint32_t n = config_.num_layers; + uint32_t num_shared = config_.num_kv_shared_layers; + uint32_t first_shared = (n > num_shared) ? n - num_shared : n; + + kv_share_map_.resize(n, -1); + shared_k_nodes_.resize(n, 0); + shared_v_nodes_.resize(n, 0); + + auto is_global_layer = [&](uint32_t idx) -> bool { + if (!config_.layer_types.empty() && idx < config_.layer_types.size()) { + const auto& lt = config_.layer_types[idx]; + return (lt == "global" || lt == "full" || lt == "full_attention"); + } + return (idx % 5) == 4; + }; + + for (uint32_t i = first_shared; i < n; i++) { + bool is_global = is_global_layer(i); + for (int j = static_cast(first_shared) - 1; j >= 0; j--) { + if (is_global_layer(j) == is_global) { + kv_share_map_[i] = j; + break; + } + } + } +} + +void GemmaModel3n::load_weights_to_graph(CactusGraph* gb) { + assert(config_.altup_num_inputs == 4 && "WeightNodeIDs altup arrays assume exactly 4 AltUp streams"); + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + } + + for (int i = 0; i < 3; i++) { + auto idx = std::to_string(i); + weight_nodes_.altup_proj_weights[i] = gb->mmap_weights(model_folder_path_ + "/altup_proj_" + idx + ".weights"); + weight_nodes_.altup_unembed_proj_weights[i] = gb->mmap_weights(model_folder_path_ + "/altup_unembed_proj_" + idx + ".weights"); + } + + weight_nodes_.embed_tokens_per_layer = gb->mmap_embeddings(model_folder_path_ + "/embed_tokens_per_layer.weights"); + weight_nodes_.per_layer_model_proj = gb->mmap_weights(model_folder_path_ + "/per_layer_model_proj.weights"); + weight_nodes_.per_layer_proj_norm = gb->mmap_weights(model_folder_path_ + "/per_layer_proj_norm.weights"); + + uint32_t num_shared = config_.num_kv_shared_layers; + uint32_t first_shared = (config_.num_layers > num_shared) ? config_.num_layers - num_shared : config_.num_layers; + + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer = weight_nodes_.layers[i]; + std::string prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + bool is_shared = (i >= first_shared); + + layer.attn_q_weight = gb->mmap_weights(prefix + "attn_q.weights"); + layer.attn_k_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k.weights"); + layer.attn_v_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_v.weights"); + layer.attn_output_weight = gb->mmap_weights(prefix + "attn_output.weights"); + layer.input_layernorm_weight = gb->mmap_weights(prefix + "input_norm.weights"); + layer.attn_q_norm_weight = gb->mmap_weights(prefix + "attn_q_norm.weights"); + layer.attn_k_norm_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k_norm.weights"); + layer.ffn_gate_weight = gb->mmap_weights(prefix + "ffn_gate.weights"); + layer.ffn_up_weight = gb->mmap_weights(prefix + "ffn_up.weights"); + layer.ffn_down_weight = gb->mmap_weights(prefix + "ffn_down.weights"); + layer.post_attention_layernorm_weight = gb->mmap_weights(prefix + "post_attn_norm.weights"); + layer.pre_feedforward_layernorm_weight = gb->mmap_weights(prefix + "pre_ffn_norm.weights"); + layer.post_feedforward_layernorm_weight = gb->mmap_weights(prefix + "post_ffn_norm.weights"); + + layer.altup_router_norm = gb->mmap_weights(prefix + "altup_router_norm.weights"); + layer.altup_prediction_coefs = gb->mmap_weights(prefix + "altup_prediction_coefs.weights"); + layer.altup_correction_coefs = gb->mmap_weights(prefix + "altup_correction_coefs.weights"); + layer.altup_correct_output_scale = gb->mmap_weights(prefix + "altup_correct_output_scale.weights"); + layer.altup_modality_router = gb->mmap_weights(prefix + "altup_modality_router.weights"); + layer.laurel_left = gb->mmap_weights(prefix + "laurel_left.weights"); + layer.laurel_right = gb->mmap_weights(prefix + "laurel_right.weights"); + layer.laurel_norm = gb->mmap_weights(prefix + "laurel_norm.weights"); + layer.per_layer_gate = gb->mmap_weights(prefix + "per_layer_gate.weights"); + layer.per_layer_proj = gb->mmap_weights(prefix + "per_layer_proj.weights"); + layer.post_per_layer_norm = gb->mmap_weights(prefix + "post_per_layer_norm.weights"); + } + + size_t head_dim = config_.attention_head_dim; + v_norm_ones_weight_.assign(head_dim, static_cast<__fp16>(1.0f)); + v_norm_ones_node_ = gb->input({head_dim}, Precision::FP16); + gb->set_external_input(v_norm_ones_node_, v_norm_ones_weight_.data(), Precision::FP16); +} + + +size_t GemmaModel3n::build_rms_norm_no_weight(CactusGraph* gb, size_t input, size_t num_rows, size_t row_dim) const { + auto flat = gb->reshape(input, {num_rows, row_dim}); + return gb->rms_norm(flat, v_norm_ones_node_, config_.layer_norm_eps); +} + +size_t GemmaModel3n::build_magnitude_normalize(CactusGraph* gb, size_t reference, size_t target) const { + size_t rows = gb->get_output_buffer(reference).shape[0]; + auto ref_sq = gb->reshape(gb->mean(gb->multiply(reference, reference), 1), {rows, 1}); + auto tgt_sq = gb->reshape(gb->mean(gb->multiply(target, target), 1), {rows, 1}); + auto ref_mag = gb->scalar_sqrt(ref_sq); + auto tgt_mag = gb->scalar_sqrt(gb->scalar_add(tgt_sq, 1e-5f)); + auto ratio = gb->divide(ref_mag, tgt_mag); + return gb->multiply(target, ratio); +} + +size_t GemmaModel3n::build_gaussian_topk(CactusGraph* gb, size_t input, float ppf) const { + return gb->gaussian_topk(input, ppf); +} + +size_t GemmaModel3n::build_laurel(CactusGraph* gb, size_t normed_input, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + auto x = gb->matmul(normed_input, layer.laurel_left, true, backend); + x = gb->matmul(x, layer.laurel_right, true, backend); + return gb->add(normed_input, gb->rms_norm(x, layer.laurel_norm, config_.layer_norm_eps)); +} + +size_t GemmaModel3n::build_altup_router_modalities(CactusGraph* gb, size_t stream0, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + auto x = gb->rms_norm(stream0, layer.altup_router_norm, config_.layer_norm_eps); + x = gb->scalar_multiply(x, 1.0f / static_cast(config_.hidden_dim)); + return gb->tanh(gb->matmul(x, layer.altup_modality_router, true, backend)); +} + +void GemmaModel3n::build_altup_predict(CactusGraph* gb, size_t modalities, uint32_t layer_idx, + const size_t* streams, size_t* predictions) const { + uint32_t n = config_.altup_num_inputs; + auto coefs = gb->matmul(modalities, weight_nodes_.layers[layer_idx].altup_prediction_coefs, true, ComputeBackend::CPU); + + size_t seq_len = gb->get_output_buffer(streams[0]).shape[0]; + auto fused = gb->altup_predict(coefs, streams, n); + for (uint32_t i = 0; i < n; i++) { + predictions[i] = gb->slice(fused, 0, i * seq_len, seq_len); + } +} + +void GemmaModel3n::build_altup_correct(CactusGraph* gb, size_t activated, size_t modalities, uint32_t layer_idx, + ComputeBackend backend, const size_t* predictions, size_t* corrected) const { + uint32_t n = config_.altup_num_inputs; + auto coefs = gb->scalar_add(gb->matmul(modalities, weight_nodes_.layers[layer_idx].altup_correction_coefs, true, backend), 1.0f); + auto innovation = gb->subtract(activated, predictions[0]); + + size_t seq_len = gb->get_output_buffer(predictions[0]).shape[0]; + auto fused = gb->altup_correct(coefs, innovation, predictions, n); + for (uint32_t i = 0; i < n; i++) { + corrected[i] = gb->slice(fused, 0, i * seq_len, seq_len); + } +} + +void GemmaModel3n::build_per_layer_input(CactusGraph* gb, size_t pli_combined, uint32_t layer_idx, + ComputeBackend backend, size_t* streams) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + uint32_t pli_dim = config_.hidden_size_per_layer_input; + + auto gated = gb->multiply(streams[0], layer.altup_correct_output_scale); + gated = gb->gelu(gb->matmul(gated, layer.per_layer_gate, true, backend)); + + auto pli = gb->multiply(gated, gb->slice(pli_combined, 1, layer_idx * pli_dim, pli_dim)); + pli = gb->rms_norm(gb->matmul(pli, layer.per_layer_proj, true, backend), layer.post_per_layer_norm, config_.layer_norm_eps); + + for (uint32_t i = 1; i < config_.altup_num_inputs; i++) + streams[i] = gb->add(streams[i], pli); +} + + +size_t GemmaModel3n::build_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + size_t seq_len = gb->get_output_buffer(input).shape[0]; + size_t num_heads = config_.attention_heads; + size_t kv_heads = config_.attention_kv_heads; + size_t head_dim = config_.attention_head_dim; + int share_src = (layer_idx < kv_share_map_.size()) ? kv_share_map_[layer_idx] : -1; + + bool is_global = false; + if (!config_.layer_types.empty() && layer_idx < config_.layer_types.size()) { + const auto& lt = config_.layer_types[layer_idx]; + is_global = (lt == "global" || lt == "full" || lt == "full_attention"); + } else { + is_global = (layer_idx % 5) == 4; + } + float rope_freq = is_global ? config_.rope_theta : config_.rope_local_base_freq; + size_t window = is_global ? 0 : config_.sliding_window; + + auto q = gb->matmul(input, layer.attn_q_weight, true, backend); + q = gb->reshape(q, {seq_len * num_heads, head_dim}); + q = gb->rms_norm(q, layer.attn_q_norm_weight, config_.layer_norm_eps); + q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); + auto q4 = gb->rope(q, rope_freq, position_offset); + + size_t k4, v4; + if (share_src >= 0 && shared_k_nodes_[share_src] != 0) { + k4 = shared_k_nodes_[share_src]; + v4 = shared_v_nodes_[share_src]; + } else { + auto k = gb->matmul(input, layer.attn_k_weight, true, backend); + k = gb->reshape(k, {seq_len * kv_heads, head_dim}); + k = gb->rms_norm(k, layer.attn_k_norm_weight, config_.layer_norm_eps); + k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); + k4 = gb->rope(k, rope_freq, position_offset); + + auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); + auto v = build_rms_norm_no_weight(gb, v_proj, seq_len * kv_heads, head_dim); + v4 = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); + + shared_k_nodes_[layer_idx] = k4; + shared_v_nodes_[layer_idx] = v4; + } + + if (use_cache && share_src < 0) { + cache_k_output_nodes_[layer_idx] = k4; + cache_v_output_nodes_[layer_idx] = v4; + } + + size_t cache_src = (share_src >= 0) ? static_cast(share_src) : layer_idx; + size_t attn; + if (use_cache && !kv_cache_.is_empty()) { + attn = gb->attention_int8_hybrid( + q4, k4, v4, attention_scale_, position_offset, + kv_cache_.get_keys_int8(cache_src), kv_cache_.get_values_int8(cache_src), + kv_cache_.get_key_scales(cache_src), kv_cache_.get_value_scales(cache_src), + kv_cache_.current_seq_len, kv_heads, head_dim, window); + } else { + attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); + } + + auto o_proj = gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), layer.attn_output_weight, true, backend); + return o_proj; +} + + +size_t GemmaModel3n::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto gate = gb->matmul(input, layer.ffn_gate_weight, true, backend); + auto up = gb->matmul(input, layer.ffn_up_weight, true, backend); + + if (layer_idx < config_.activation_sparsity_ppf.size() && config_.activation_sparsity_ppf[layer_idx] > 0.0f) + gate = build_gaussian_topk(gb, gate, config_.activation_sparsity_ppf[layer_idx]); + + auto gate_activated = gb->gelu(gate); + auto activated = gb->multiply(gate_activated, up); + return gb->matmul(activated, layer.ffn_down_weight, true, backend); +} + + +size_t GemmaModel3n::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + auto normed = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); + + auto laurel = build_laurel(gb, normed, layer_idx, backend); + auto attn_raw = build_attention(gb, normed, layer_idx, backend, use_cache, position_offset); + auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm_weight, config_.layer_norm_eps); + auto combined = gb->add(gb->add(hidden, attn), laurel); + auto residual = gb->scalar_multiply(combined, RSQRT2); + + auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); + auto mlp_raw = build_mlp(gb, pre_mlp, layer_idx, backend); + auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); + + auto block_out = gb->add(residual, mlp); + return block_out; +} + + +size_t GemmaModel3n::build_preamble(CactusGraph* gb, size_t seq_len, ComputeBackend backend, + size_t& token_input, size_t& pli_input, size_t* streams) { + uint32_t num_layers = config_.num_layers; + uint32_t pli_dim = config_.hidden_size_per_layer_input; + uint32_t num_altup = config_.altup_num_inputs; + + token_input = gb->input({seq_len}, Precision::FP32); + auto x = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), + std::sqrt(static_cast(config_.hidden_dim))); + + pli_input = gb->input({seq_len}, Precision::FP32); + auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), + std::sqrt(static_cast(pli_dim))); + auto pli_proj = gb->scalar_multiply(gb->matmul(x, weight_nodes_.per_layer_model_proj, true, backend), + 1.0f / std::sqrt(static_cast(config_.hidden_dim))); + pli_proj = gb->reshape(pli_proj, {seq_len * num_layers, pli_dim}); + pli_proj = gb->rms_norm(pli_proj, weight_nodes_.per_layer_proj_norm, config_.layer_norm_eps); + pli_proj = gb->reshape(pli_proj, {seq_len, num_layers * pli_dim}); + auto pli_combined = gb->scalar_multiply(gb->add(pli_proj, pli_embed), RSQRT2); + + streams[0] = x; + for (uint32_t i = 1; i < num_altup; i++) { + streams[i] = build_magnitude_normalize(gb, x, gb->matmul(x, weight_nodes_.altup_proj_weights[i - 1], true, backend)); + } + + return pli_combined; +} + +void GemmaModel3n::build_layer(CactusGraph* gb, uint32_t layer_idx, ComputeBackend backend, + bool use_cache, size_t pos_offset, size_t pli, size_t* streams) { + auto modalities = build_altup_router_modalities(gb, streams[0], layer_idx, backend); + + size_t predictions[4]; + build_altup_predict(gb, modalities, layer_idx, streams, predictions); + + auto activated = build_transformer_block(gb, predictions[0], layer_idx, backend, use_cache, pos_offset); + + auto modalities_post = build_altup_router_modalities(gb, activated, layer_idx, backend); + + build_altup_correct(gb, activated, modalities_post, layer_idx, backend, predictions, streams); + + if (config_.hidden_size_per_layer_input > 0) { + build_per_layer_input(gb, pli, layer_idx, backend, streams); + } +} + +size_t GemmaModel3n::build_output_head(CactusGraph* gb, size_t* streams, ComputeBackend backend) { + uint32_t num_altup = config_.altup_num_inputs; + + for (uint32_t i = 1; i < num_altup; i++) { + streams[i] = gb->matmul(streams[i], weight_nodes_.altup_unembed_proj_weights[i - 1], true, backend); + streams[i] = build_magnitude_normalize(gb, streams[0], streams[i]); + } + + auto hidden = streams[0]; + for (uint32_t i = 1; i < num_altup; i++) + hidden = gb->add(hidden, streams[i]); + hidden = gb->scalar_multiply(hidden, 1.0f / static_cast(num_altup)); + + auto final_normed = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + return final_normed; +} + +void GemmaModel3n::set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, + const std::vector& tokens) { + std::vector input_data(tokens.size()); + for (size_t i = 0; i < tokens.size(); i++) + input_data[i] = static_cast(tokens[i]); + gb->set_input(token_input, input_data.data(), Precision::FP32); + gb->set_input(pli_input, input_data.data(), Precision::FP32); +} + + +size_t GemmaModel3n::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (tokens.empty()) + throw std::runtime_error("Token sequence cannot be empty"); + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + + size_t token_input, pli_input, streams[4]; + auto pli = build_preamble(gb, tokens.size(), backend, token_input, pli_input, streams); + + for (uint32_t i = 0; i < config_.num_layers; i++) + build_layer(gb, i, backend, use_cache, pos_offset, pli, streams); + + set_token_inputs(gb, token_input, pli_input, tokens); + return build_output_head(gb, streams, backend); +} + + +size_t GemmaModel3n::forward_split(const std::vector& tokens, bool use_cache) { + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + size_t seq_len = tokens.size(); + size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + uint32_t num_layers = config_.num_layers; + uint32_t first_shared = num_layers - config_.num_kv_shared_layers; + + size_t token_input, pli_input, streams[4]; + auto pli = build_preamble(gb, seq_len, backend, token_input, pli_input, streams); + + for (uint32_t i = 0; i < first_shared; i++) + build_layer(gb, i, backend, use_cache, pos_offset, pli, streams); + + size_t hidden_dim = config_.hidden_dim; + for (uint32_t i = 0; i < config_.altup_num_inputs; i++) { + streams[i] = gb->index(streams[i], seq_len - 1, 0); + streams[i] = gb->reshape(streams[i], {1, hidden_dim}); + } + + auto pli_last = gb->index(pli, seq_len - 1, 0); + pli_last = gb->reshape(pli_last, {1, num_layers * config_.hidden_size_per_layer_input}); + + for (uint32_t i = first_shared; i < num_layers; i++) + build_layer(gb, i, backend, use_cache, pos_offset, pli_last, streams); + + set_token_inputs(gb, token_input, pli_input, tokens); + return build_output_head(gb, streams, backend); +} + + +void GemmaModel3n::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { + if (tokens.empty()) + return; + + static constexpr size_t SPLIT_PREFILL_MIN_TOKENS = 32; + bool use_split = config_.num_kv_shared_layers > 0 + && tokens.size() >= SPLIT_PREFILL_MIN_TOKENS + && !std::getenv("CACTUS_DISABLE_SPLIT_PREFILL"); + + if (!use_split) { + Model::prefill(tokens, chunk_size, profile_file); + return; + } + + auto* gb = static_cast(graph_handle_); + + auto process_chunk = [&](const std::vector& chunk) { + forward_split(chunk, true); + gb->execute(profile_file); + update_kv_cache(gb, chunk.size()); + }; + + if (tokens.size() <= chunk_size) { + process_chunk(tokens); + return; + } + + size_t num_full_chunks = (tokens.size() - 1) / chunk_size; + for (size_t i = 0; i < num_full_chunks; ++i) { + size_t start = i * chunk_size; + std::vector chunk(tokens.begin() + start, tokens.begin() + start + chunk_size); + if (i == 1) + gb->set_prefill_mode(true); + process_chunk(chunk); + } + + gb->set_prefill_mode(false); + size_t final_start = num_full_chunks * chunk_size; + std::vector final_chunk(tokens.begin() + final_start, tokens.end()); + process_chunk(final_chunk); +} + +} +} diff --git a/cactus-transpile/models/model_lfm2.cpp b/cactus-transpile/models/model_lfm2.cpp new file mode 100644 index 000000000..12d10cad9 --- /dev/null +++ b/cactus-transpile/models/model_lfm2.cpp @@ -0,0 +1,389 @@ +#include "model.h" +#include "../graph/graph.h" +#include +#include +#include +#include +#include +#include +#include +namespace cactus { +namespace engine { +LFM2Model::LFM2Model() : Model() { + weight_nodes_.layers.resize(config_.num_layers); + conv_cache_bx_nodes_.assign(config_.num_layers, 0); +} +LFM2Model::LFM2Model(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config_.num_layers); + conv_cache_bx_nodes_.assign(config_.num_layers, 0); +} +void LFM2Model::post_init() { + if (config_.conv_L_cache > 0) { + Precision cache_precision = Precision::FP16; + size_t conv_window = config_.conv_L_cache > 0 ? config_.conv_L_cache - 1 : 0; + conv_cache_.init(config_.num_layers, config_.hidden_dim, conv_window, cache_precision); + + } + last_forward_used_cache_ = false; + +} +void LFM2Model::reset_cache() { + Model::reset_cache(); + + if (conv_cache_.window_size > 0) { + conv_cache_.reset(); + + } +} +bool LFM2Model::is_cache_empty() const { + return kv_cache_.is_empty(); +} +bool LFM2Model::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { + if (!Model::init(model_folder, context_size, system_prompt, do_warmup)) { + return false; + } + + if (weight_nodes_.layers.size() != config_.num_layers) { + weight_nodes_.layers.resize(config_.num_layers); + + } + conv_cache_bx_nodes_.assign(config_.num_layers, 0); + + return true; +} +bool LFM2Model::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup) { + if (!Model::init(external_graph, model_folder, context_size, system_prompt, do_warmup)) { + return false; + } + + if (weight_nodes_.layers.size() != config_.num_layers) { + weight_nodes_.layers.resize(config_.num_layers); + + } + conv_cache_bx_nodes_.assign(config_.num_layers, 0); + + return true; +} +void LFM2Model::load_weights_to_graph(CactusGraph* gb) { + if (weight_nodes_.layers.size() != config_.num_layers) { + weight_nodes_.layers.resize(config_.num_layers); + } + if (conv_cache_bx_nodes_.size() != config_.num_layers) { + conv_cache_bx_nodes_.assign(config_.num_layers, 0); + } + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + + } + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer_entry = weight_nodes_.layers[i]; + auto& layer = layer_entry.weights; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + + bool is_conv_layer = false; + if (i < config_.layer_types.size()) { + std::string layer_type = config_.layer_types[i]; + is_conv_layer = (layer_type == "conv" || layer_type == "CONV"); + + } + if (is_conv_layer) { + layer_entry.type = WeightNodeIDs::LayerType::CONV; + layer.conv_in_proj_weight = gb->mmap_weights(layer_prefix + "conv_in_proj.weights"); + layer.conv_out_proj_weight = gb->mmap_weights(layer_prefix + "conv_out_proj.weights"); + layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); + + } else { + layer_entry.type = WeightNodeIDs::LayerType::ATTENTION; + layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); + layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); + layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); + layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); + layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); + layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); + + } + layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); + layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); + layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); + layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); + layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); + + } + +} +size_t LFM2Model::build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache) { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + size_t in_proj = gb->matmul(input, layer.conv_in_proj_weight, true, backend); + const auto& in_proj_buf = gb->get_output_buffer(in_proj); + if (in_proj_buf.shape.size() != 2 || (in_proj_buf.shape[1] % 3) != 0) { + throw std::runtime_error("Conv in_proj output must be [L, 3*C]"); + } + + const size_t L = in_proj_buf.shape[0]; + const size_t C = in_proj_buf.shape[1] / 3; + + size_t triplet = gb->reshape(in_proj, {L, static_cast(3), C}); + size_t B = gb->slice(triplet, 1, 0, 1); + size_t Cg = gb->slice(triplet, 1, 1, 1); + size_t X = gb->slice(triplet, 1, 2, 1); + + B = gb->reshape(B, {L, C}); + + Cg = gb->reshape(Cg, {L, C}); + + X = gb->reshape(X, {L, C}); + size_t Bx = gb->multiply(B, X); + if (use_cache) { + conv_cache_bx_nodes_[layer_idx] = Bx; + + } else { + conv_cache_bx_nodes_[layer_idx] = 0; + + } + const auto& wbuf = gb->get_output_buffer(layer.conv_depthwise_weight); + size_t K = wbuf.shape.back(); + + size_t conv_w = layer.conv_depthwise_weight; + if (wbuf.shape.size() == 2) { + K = wbuf.shape[1]; + conv_w = gb->reshape(conv_w, {wbuf.shape[0], static_cast(1), K}); + } else if (wbuf.shape.size() != 3){ + throw std::runtime_error("Unexpected depthwise weight rank"); + } + K = wbuf.shape.back(); + + size_t conv_input_lc = Bx; + if (use_cache && conv_cache_.window_size > 0) { + auto view = conv_cache_.get_window(layer_idx); + std::vector segments; + if (view.len2 > 0) { + size_t left_node = gb->input({view.len2, C}, conv_cache_.precision); + gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); + segments.push_back(left_node); + + } + if (view.len1 > 0) { + size_t right_node = gb->input({view.len1, C}, conv_cache_.precision); + gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); + segments.push_back(right_node); + + } + if (!segments.empty()) { + conv_input_lc = segments[0]; + for (size_t idx = 1; idx < segments.size(); ++idx) { + conv_input_lc = gb->concat(conv_input_lc, segments[idx], 0); + + } + conv_input_lc = gb->concat(conv_input_lc, Bx, 0); + + } + } + + const auto& conv_input_buf = gb->get_output_buffer(conv_input_lc); + size_t total_len = conv_input_buf.shape[0]; + + size_t x_nlc = gb->reshape(conv_input_lc, {static_cast(1), total_len, C}); + const size_t dilation = 1; + + size_t y_nlc = gb->conv1d_causal(x_nlc, conv_w, K, dilation); + size_t start = total_len > L ? total_len - L : 0; + size_t y_slice = gb->slice(y_nlc, 1, start, L); + + size_t y_lc = gb->reshape(y_slice, {L, C}); + size_t gated = gb->multiply(Cg, y_lc); + size_t projected = gb->matmul(gated, layer.conv_out_proj_weight, true, backend); + return projected; +} + +size_t LFM2Model::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + auto q_proj_linear = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); + auto k_proj_linear = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); + auto v_proj_linear = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); + const auto& q_shape = gb->get_output_buffer(q_proj_linear).shape; + size_t batch_seq = q_shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + + auto q_proj_reshaped = gb->reshape(q_proj_linear, {batch_seq * num_heads, head_dim}); + + auto q_proj_norm = gb->rms_norm(q_proj_reshaped, layer.attn_q_norm_weight, config_.layer_norm_eps); + + auto q_proj = gb->reshape(q_proj_norm, {batch_seq, num_heads * head_dim}); + size_t num_kv_heads = config_.attention_kv_heads; + auto k_proj_reshaped = gb->reshape(k_proj_linear, {batch_seq * num_kv_heads, head_dim}); + + auto k_proj_norm = gb->rms_norm(k_proj_reshaped, layer.attn_k_norm_weight, config_.layer_norm_eps); + + auto k_proj = gb->reshape(k_proj_norm, {batch_seq, num_kv_heads * head_dim}); + size_t seq_len = batch_seq; + + auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); + auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + auto v_proj_4d = gb->reshape(v_proj_linear, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + + if (config_.rope_theta > 0) { + q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); + k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); + } + if (use_cache) { + cache_k_output_nodes_[layer_idx] = k_proj_4d; + cache_v_output_nodes_[layer_idx] = v_proj_4d; + } + + size_t attn_output_4d; + + if (use_cache && !kv_cache_.is_empty()) { + attn_output_4d = gb->attention_int8_hybrid( + q_proj_4d, k_proj_4d, v_proj_4d, + attention_scale_, position_offset, + kv_cache_.get_keys_int8(layer_idx), + kv_cache_.get_values_int8(layer_idx), + kv_cache_.get_key_scales(layer_idx), + kv_cache_.get_value_scales(layer_idx), + kv_cache_.current_seq_len, num_kv_heads, head_dim + ); + } else { + attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); + } + auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); + auto projected = gb->matmul(attn_output, layer.attn_output_weight, true, backend); + return projected; +} + +size_t LFM2Model::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, ComputeBackend backend) const { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + auto gate = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); + auto up = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); + auto activated = gb->multiply(gb->silu(gate), up); + + auto down = gb->matmul(activated, layer.ffn_down_weight, true, backend); + + return down; +} + +size_t LFM2Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); + size_t block_output; + if (layer_entry.type == WeightNodeIDs::LayerType::CONV) { + block_output = build_conv1d(gb, normalized_input, layer_idx, backend, use_cache); + } else { + if (layer_idx < conv_cache_bx_nodes_.size()) { + conv_cache_bx_nodes_[layer_idx] = 0; + } + block_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); + } + + auto after_block = gb->add(hidden, block_output); + auto normalized_after_block = gb->rms_norm(after_block, layer.post_attention_layernorm_weight, config_.layer_norm_eps); + auto mlp_output = build_mlp(gb, normalized_after_block, layer_idx, backend); + auto block_result = gb->add(after_block, mlp_output); + return block_result; +} + +size_t LFM2Model::forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, + ComputeBackend backend, bool use_cache) { + if (seq_len == 0) { + throw std::runtime_error("Sequence length must be greater than zero"); + } + + if (conv_cache_bx_nodes_.size() != config_.num_layers) { + conv_cache_bx_nodes_.assign(config_.num_layers, 0); + + } + std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); + + last_forward_used_cache_ = use_cache; + + if (!use_cache && conv_cache_.window_size > 0) { + conv_cache_.reset(); + + } + size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + + size_t hidden = input_embeddings; + + for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { + hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); + + } + + auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + + return final_hidden; +} +size_t LFM2Model::forward(CactusGraph* gb, const std::vector& tokens, + ComputeBackend backend, bool use_cache) { + if (tokens.empty()) { + throw std::runtime_error("Token sequence cannot be empty"); + } + auto seq_len = static_cast(tokens.size()); + auto input_node_id = gb->input({seq_len}, Precision::FP32); + auto hidden = gb->embedding(embedding_node_id_, input_node_id); + auto final_hidden = forward(gb, hidden, seq_len, backend, use_cache); + + std::vector input_data(seq_len); + for (size_t i = 0; i < seq_len; i++) { + input_data[i] = static_cast(tokens[i]); + } + gb->set_input(input_node_id, input_data.data(), Precision::FP32); + + return final_hidden; +} +size_t LFM2Model::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + return forward(gb, tokens, backend, use_cache); +} +void LFM2Model::post_execute_updates(CactusGraph* gb, size_t /*seq_len*/) { + if (conv_cache_bx_nodes_.empty()) { + return; + } + + if (!last_forward_used_cache_ || conv_cache_.window_size == 0) { + std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); + + return; + } + size_t layer_count = std::min(conv_cache_bx_nodes_.size(), weight_nodes_.layers.size()); + for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { + if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::CONV) { + conv_cache_bx_nodes_[layer_idx] = 0; + continue; + } + size_t bx_node = conv_cache_bx_nodes_[layer_idx]; + if (bx_node != 0) { + conv_cache_.update(gb, layer_idx, bx_node); + } + conv_cache_bx_nodes_[layer_idx] = 0; + } + last_forward_used_cache_ = false; + +} +} +} \ No newline at end of file diff --git a/cactus-transpile/models/model_lfm2moe.cpp b/cactus-transpile/models/model_lfm2moe.cpp new file mode 100644 index 000000000..e867531bb --- /dev/null +++ b/cactus-transpile/models/model_lfm2moe.cpp @@ -0,0 +1,456 @@ +#include "model.h" +#include "../graph/graph.h" +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +LFM2MoEModel::LFM2MoEModel() : Model() { + weight_nodes_.layers.resize(config_.num_layers); + conv_cache_bx_nodes_.assign(config_.num_layers, 0); +} +LFM2MoEModel::LFM2MoEModel(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config_.num_layers); + conv_cache_bx_nodes_.assign(config_.num_layers, 0); +} +void LFM2MoEModel::post_init() { + if (config_.num_dense_layers > config_.num_layers) { + throw std::runtime_error("num_dense_layers cannot be greater than num_layers for LFM2MoEModel"); + } + if (config_.num_experts == 0 || config_.num_experts_per_tok == 0 || config_.moe_intermediate_dim == 0) { + throw std::runtime_error("LFM2MoEModel requires num_experts, num_experts_per_tok, and moe_intermediate_dim to be non-zero"); + } + if (!config_.use_expert_bias || !config_.norm_topk_prob) { + throw std::runtime_error("LFM2MoEModel currently requires use_expert_bias=true and norm_topk_prob=true"); + } + if (config_.num_experts_per_tok > config_.num_experts) { + throw std::runtime_error("num_experts_per_tok cannot be greater than num_experts"); + } + + if (config_.conv_L_cache > 0) { + Precision cache_precision = Precision::FP16; + size_t conv_window = config_.conv_L_cache > 0 ? config_.conv_L_cache - 1 : 0; + conv_cache_.init(config_.num_layers, config_.hidden_dim, conv_window, cache_precision); + + } + last_forward_used_cache_ = false; + +} + +bool LFM2MoEModel::is_dense_layer(uint32_t layer_idx) const { + return layer_idx < config_.num_dense_layers; +} +void LFM2MoEModel::reset_cache() { + Model::reset_cache(); + + if (conv_cache_.window_size > 0) { + conv_cache_.reset(); + + } +} +bool LFM2MoEModel::is_cache_empty() const { + return kv_cache_.is_empty(); +} +bool LFM2MoEModel::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { + if (!Model::init(model_folder, context_size, system_prompt, do_warmup)) { + return false; + } + + if (weight_nodes_.layers.size() != config_.num_layers) { + weight_nodes_.layers.resize(config_.num_layers); + + } + conv_cache_bx_nodes_.assign(config_.num_layers, 0); + + return true; +} +bool LFM2MoEModel::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup) { + if (!Model::init(external_graph, model_folder, context_size, system_prompt, do_warmup)) { + return false; + } + + if (weight_nodes_.layers.size() != config_.num_layers) { + weight_nodes_.layers.resize(config_.num_layers); + + } + conv_cache_bx_nodes_.assign(config_.num_layers, 0); + + return true; +} +void LFM2MoEModel::load_weights_to_graph(CactusGraph* gb) { + if (weight_nodes_.layers.size() != config_.num_layers) { + weight_nodes_.layers.resize(config_.num_layers); + } + if (conv_cache_bx_nodes_.size() != config_.num_layers) { + conv_cache_bx_nodes_.assign(config_.num_layers, 0); + } + auto mmap_or_throw = [&](const std::string& path, bool embedding) -> size_t { + size_t node_id = embedding ? gb->mmap_embeddings(path) : gb->mmap_weights(path); + return node_id; + }; + + embedding_node_id_ = mmap_or_throw(embedding_file_path_, true); + weight_nodes_.output_norm_weight = mmap_or_throw(model_folder_path_ + "/output_norm.weights", false); + + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + + } else { + weight_nodes_.output_weight = mmap_or_throw(model_folder_path_ + "/output_weight.weights", false); + output_weight_node_id_ = weight_nodes_.output_weight; + + } + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer_entry = weight_nodes_.layers[i]; + auto& layer = layer_entry.weights; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + + bool is_conv_layer = false; + if (i < config_.layer_types.size()) { + std::string layer_type = config_.layer_types[i]; + is_conv_layer = (layer_type == "conv" || layer_type == "CONV"); + + } + if (is_conv_layer) { + layer_entry.type = WeightNodeIDs::LayerType::CONV; + layer.conv_in_proj_weight = mmap_or_throw(layer_prefix + "conv_in_proj.weights", false); + layer.conv_out_proj_weight = mmap_or_throw(layer_prefix + "conv_out_proj.weights", false); + layer.conv_depthwise_weight = mmap_or_throw(layer_prefix + "conv_depthwise.weights", false); + + } else { + layer_entry.type = WeightNodeIDs::LayerType::ATTENTION; + layer.attn_q_weight = mmap_or_throw(layer_prefix + "attn_q.weights", false); + layer.attn_k_weight = mmap_or_throw(layer_prefix + "attn_k.weights", false); + layer.attn_v_weight = mmap_or_throw(layer_prefix + "attn_v.weights", false); + layer.attn_output_weight = mmap_or_throw(layer_prefix + "attn_output.weights", false); + layer.attn_q_norm_weight = mmap_or_throw(layer_prefix + "attn_q_norm.weights", false); + layer.attn_k_norm_weight = mmap_or_throw(layer_prefix + "attn_k_norm.weights", false); + + } + layer.input_layernorm_weight = mmap_or_throw(layer_prefix + "input_norm.weights", false); + layer.post_attention_layernorm_weight = mmap_or_throw(layer_prefix + "post_attn_norm.weights", false); + if (is_dense_layer(i)) { + layer.ffn_gate_weight = mmap_or_throw(layer_prefix + "ffn_gate.weights", false); + layer.ffn_up_weight = mmap_or_throw(layer_prefix + "ffn_up.weights", false); + layer.ffn_down_weight = mmap_or_throw(layer_prefix + "ffn_down.weights", false); + } else { + layer.moe_router_weight = mmap_or_throw(layer_prefix + "moe_router.weights", false); + layer.moe_expert_bias = mmap_or_throw(layer_prefix + "moe_expert_bias.weights", false); + layer.moe_experts.resize(config_.num_experts); + for (uint32_t expert_idx = 0; expert_idx < config_.num_experts; ++expert_idx) { + auto& expert = layer.moe_experts[expert_idx]; + std::string expert_prefix = layer_prefix + "moe_expert_" + std::to_string(expert_idx) + "_"; + expert.w1_weight = mmap_or_throw(expert_prefix + "w1.weights", false); + expert.w3_weight = mmap_or_throw(expert_prefix + "w3.weights", false); + expert.w2_weight = mmap_or_throw(expert_prefix + "w2.weights", false); + } + } + + } + +} +size_t LFM2MoEModel::build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache) { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + size_t in_proj = gb->matmul(input, layer.conv_in_proj_weight, true, backend); + const auto& in_proj_buf = gb->get_output_buffer(in_proj); + if (in_proj_buf.shape.size() != 2 || (in_proj_buf.shape[1] % 3) != 0) { + throw std::runtime_error("Conv in_proj output must be [L, 3*C]"); + } + + const size_t L = in_proj_buf.shape[0]; + const size_t C = in_proj_buf.shape[1] / 3; + + size_t triplet = gb->reshape(in_proj, {L, static_cast(3), C}); + size_t B = gb->slice(triplet, 1, 0, 1); + size_t Cg = gb->slice(triplet, 1, 1, 1); + size_t X = gb->slice(triplet, 1, 2, 1); + + B = gb->reshape(B, {L, C}); + + Cg = gb->reshape(Cg, {L, C}); + + X = gb->reshape(X, {L, C}); + size_t Bx = gb->multiply(B, X); + if (use_cache) { + conv_cache_bx_nodes_[layer_idx] = Bx; + + } else { + conv_cache_bx_nodes_[layer_idx] = 0; + + } + const auto& wbuf = gb->get_output_buffer(layer.conv_depthwise_weight); + size_t K = wbuf.shape.back(); + + size_t conv_w = layer.conv_depthwise_weight; + if (wbuf.shape.size() == 2) { + K = wbuf.shape[1]; + conv_w = gb->reshape(conv_w, {wbuf.shape[0], static_cast(1), K}); + } else if (wbuf.shape.size() != 3){ + throw std::runtime_error("Unexpected depthwise weight rank"); + } + K = wbuf.shape.back(); + + size_t conv_input_lc = Bx; + if (use_cache && conv_cache_.window_size > 0) { + auto view = conv_cache_.get_window(layer_idx); + std::vector segments; + if (view.len2 > 0) { + size_t left_node = gb->input({view.len2, C}, conv_cache_.precision); + gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); + segments.push_back(left_node); + + } + if (view.len1 > 0) { + size_t right_node = gb->input({view.len1, C}, conv_cache_.precision); + gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); + segments.push_back(right_node); + + } + if (!segments.empty()) { + conv_input_lc = segments[0]; + for (size_t idx = 1; idx < segments.size(); ++idx) { + conv_input_lc = gb->concat(conv_input_lc, segments[idx], 0); + + } + conv_input_lc = gb->concat(conv_input_lc, Bx, 0); + + } + } + + const auto& conv_input_buf = gb->get_output_buffer(conv_input_lc); + size_t total_len = conv_input_buf.shape[0]; + + size_t x_nlc = gb->reshape(conv_input_lc, {static_cast(1), total_len, C}); + const size_t dilation = 1; + + size_t y_nlc = gb->conv1d_causal(x_nlc, conv_w, K, dilation); + size_t start = total_len > L ? total_len - L : 0; + size_t y_slice = gb->slice(y_nlc, 1, start, L); + + size_t y_lc = gb->reshape(y_slice, {L, C}); + size_t gated = gb->multiply(Cg, y_lc); + size_t projected = gb->matmul(gated, layer.conv_out_proj_weight, true, backend); + return projected; +} + +size_t LFM2MoEModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + auto q_proj_linear = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); + auto k_proj_linear = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); + auto v_proj_linear = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); + const auto& q_shape = gb->get_output_buffer(q_proj_linear).shape; + size_t batch_seq = q_shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + + auto q_proj_reshaped = gb->reshape(q_proj_linear, {batch_seq * num_heads, head_dim}); + + auto q_proj_norm = gb->rms_norm(q_proj_reshaped, layer.attn_q_norm_weight, config_.layer_norm_eps); + + auto q_proj = gb->reshape(q_proj_norm, {batch_seq, num_heads * head_dim}); + size_t num_kv_heads = config_.attention_kv_heads; + auto k_proj_reshaped = gb->reshape(k_proj_linear, {batch_seq * num_kv_heads, head_dim}); + + auto k_proj_norm = gb->rms_norm(k_proj_reshaped, layer.attn_k_norm_weight, config_.layer_norm_eps); + + auto k_proj = gb->reshape(k_proj_norm, {batch_seq, num_kv_heads * head_dim}); + size_t seq_len = batch_seq; + + auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); + auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + auto v_proj_4d = gb->reshape(v_proj_linear, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + + if (config_.rope_theta > 0) { + q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); + k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); + } + if (use_cache) { + cache_k_output_nodes_[layer_idx] = k_proj_4d; + cache_v_output_nodes_[layer_idx] = v_proj_4d; + } + + size_t attn_output_4d; + + if (use_cache && !kv_cache_.is_empty()) { + attn_output_4d = gb->attention_int8_hybrid( + q_proj_4d, k_proj_4d, v_proj_4d, + attention_scale_, position_offset, + kv_cache_.get_keys_int8(layer_idx), + kv_cache_.get_values_int8(layer_idx), + kv_cache_.get_key_scales(layer_idx), + kv_cache_.get_value_scales(layer_idx), + kv_cache_.current_seq_len, num_kv_heads, head_dim + ); + } else { + attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); + } + auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); + auto projected = gb->matmul(attn_output, layer.attn_output_weight, true, backend); + return projected; +} + +size_t LFM2MoEModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, ComputeBackend backend) const { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + if (is_dense_layer(layer_idx)) { + auto gate = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); + auto up = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); + auto activated = gb->multiply(gb->silu(gate), up); + auto down = gb->matmul(activated, layer.ffn_down_weight, true, backend); + return down; + } + + auto router_logits = gb->matmul(normalized_h, layer.moe_router_weight, true, backend); + auto routing_probs = gb->sigmoid(router_logits); + auto scores_for_routing = gb->add(routing_probs, layer.moe_expert_bias); + auto topk_indices_and_values = gb->topk(scores_for_routing, config_.num_experts_per_tok); + auto topk_indices = gb->index(topk_indices_and_values, 0, 0); + + std::vector w1_weights, w3_weights, w2_weights; + w1_weights.reserve(config_.num_experts); + w3_weights.reserve(config_.num_experts); + w2_weights.reserve(config_.num_experts); + for (uint32_t expert_idx = 0; expert_idx < config_.num_experts; ++expert_idx) { + const auto& expert = layer.moe_experts[expert_idx]; + w1_weights.push_back(expert.w1_weight); + w3_weights.push_back(expert.w3_weight); + w2_weights.push_back(expert.w2_weight); + } + + return gb->moe_layer( + normalized_h, + routing_probs, + topk_indices, + w1_weights, + w3_weights, + w2_weights, + config_.num_experts, + config_.num_experts_per_tok, + config_.norm_topk_prob, + 1e-6f, + config_.routed_scaling_factor + ); +} + +size_t LFM2MoEModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); + size_t block_output; + if (layer_entry.type == WeightNodeIDs::LayerType::CONV) { + block_output = build_conv1d(gb, normalized_input, layer_idx, backend, use_cache); + } else { + if (layer_idx < conv_cache_bx_nodes_.size()) { + conv_cache_bx_nodes_[layer_idx] = 0; + } + block_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); + } + + auto after_block = gb->add(hidden, block_output); + auto normalized_after_block = gb->rms_norm(after_block, layer.post_attention_layernorm_weight, config_.layer_norm_eps); + auto mlp_output = build_mlp(gb, normalized_after_block, layer_idx, backend); + auto block_result = gb->add(after_block, mlp_output); + return block_result; +} + +size_t LFM2MoEModel::forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, + ComputeBackend backend, bool use_cache) { + if (seq_len == 0) { + throw std::runtime_error("Sequence length must be greater than zero"); + } + + if (conv_cache_bx_nodes_.size() != config_.num_layers) { + conv_cache_bx_nodes_.assign(config_.num_layers, 0); + + } + std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); + + last_forward_used_cache_ = use_cache; + + if (!use_cache && conv_cache_.window_size > 0) { + conv_cache_.reset(); + + } + size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + + size_t hidden = input_embeddings; + + for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { + hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); + + } + + auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + + return final_hidden; +} +size_t LFM2MoEModel::forward(CactusGraph* gb, const std::vector& tokens, + ComputeBackend backend, bool use_cache) { + if (tokens.empty()) { + throw std::runtime_error("Token sequence cannot be empty"); + } + auto seq_len = static_cast(tokens.size()); + auto input_node_id = gb->input({seq_len}, Precision::FP32); + auto hidden = gb->embedding(embedding_node_id_, input_node_id); + auto final_hidden = forward(gb, hidden, seq_len, backend, use_cache); + + std::vector input_data(seq_len); + for (size_t i = 0; i < seq_len; i++) { + input_data[i] = static_cast(tokens[i]); + } + gb->set_input(input_node_id, input_data.data(), Precision::FP32); + + return final_hidden; +} +size_t LFM2MoEModel::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + return forward(gb, tokens, backend, use_cache); +} +void LFM2MoEModel::post_execute_updates(CactusGraph* gb, size_t /*seq_len*/) { + if (conv_cache_bx_nodes_.empty()) { + return; + } + + if (!last_forward_used_cache_ || conv_cache_.window_size == 0) { + std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); + + return; + } + size_t layer_count = std::min(conv_cache_bx_nodes_.size(), weight_nodes_.layers.size()); + for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { + if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::CONV) { + conv_cache_bx_nodes_[layer_idx] = 0; + continue; + } + size_t bx_node = conv_cache_bx_nodes_[layer_idx]; + if (bx_node != 0) { + conv_cache_.update(gb, layer_idx, bx_node); + } + conv_cache_bx_nodes_[layer_idx] = 0; + } + last_forward_used_cache_ = false; + +} +} +} diff --git a/cactus-transpile/models/model_lfm2vl.cpp b/cactus-transpile/models/model_lfm2vl.cpp new file mode 100644 index 000000000..c4357ad0b --- /dev/null +++ b/cactus-transpile/models/model_lfm2vl.cpp @@ -0,0 +1,607 @@ +#include "model.h" +#include "../graph/graph.h" +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +Lfm2VlModel::Lfm2VlModel() : Model() { + config_.model_type = Config::ModelType::LFM2; +} + +Lfm2VlModel::Lfm2VlModel(const Config& config) + : Model(config), + vision_tower_(config), + language_model_(config) { + Siglip2Preprocessor::Config preprocessor_config; + preprocessor_config.patch_size = static_cast(config.vision_patch_size); + preprocessor_config.downsample_factor = static_cast(config.downsample_factor); + preprocessor_config.min_tiles = static_cast(config.min_tiles); + preprocessor_config.max_tiles = static_cast(config.max_tiles); + preprocessor_config.use_thumbnail = config.use_thumbnail; + preprocessor_config.min_image_tokens = static_cast(config.min_image_tokens); + preprocessor_config.max_image_tokens = static_cast(config.max_image_tokens); + preprocessor_config.max_num_patches = static_cast(config.max_num_patches); + preprocessor_config.tile_size = static_cast(config.tile_size); + preprocessor_config.max_pixels_tolerance = config.max_pixels_tolerance; + preprocessor_config.do_resize = true; + preprocessor_config.do_rescale = true; + preprocessor_config.do_normalize = true; + preprocessor_config.do_convert_rgb = true; + preprocessor_config.do_image_splitting = config.do_image_splitting; + preprocessor_config.rescale_factor = config.rescale_factor; + preprocessor_config.image_mean[0] = config.image_mean; + preprocessor_config.image_mean[1] = config.image_mean; + preprocessor_config.image_mean[2] = config.image_mean; + preprocessor_config.image_std[0] = config.image_std; + preprocessor_config.image_std[1] = config.image_std; + preprocessor_config.image_std[2] = config.image_std; + + preprocessor_ = Siglip2Preprocessor(preprocessor_config); +} + +bool Lfm2VlModel::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { + + if (!Model::init(model_folder, context_size, system_prompt, false)) { + return false; + } + auto* shared_graph = static_cast(graph_handle_); + if (!shared_graph) { + throw std::runtime_error("Shared graph was not initialized for Lfm2VlModel"); + } + std::string vision_folder = model_folder; + if (!vision_tower_.init(shared_graph, vision_folder, context_size, "", false)) { + throw std::runtime_error("Failed to initialize vision tower"); + } + + vision_weights_loaded_ = true; + if (!language_model_.init(shared_graph, model_folder, context_size, system_prompt, false)) { + throw std::runtime_error("Failed to initialize language model"); + } + + language_model_.output_weight_node_id_ = output_weight_node_id_; + + language_weights_loaded_ = true; + + if (do_warmup) { + std::string warmup_text = system_prompt.empty() ? "Hello" : system_prompt; + auto warmup_tokens = tokenizer_->encode(warmup_text); + language_model_.decode(warmup_tokens, config_.default_temperature, config_.default_top_p, config_.default_top_k, ""); + language_model_.reset_cache(); + } + + return true; +} + +void Lfm2VlModel::reset_cache() { + Model::reset_cache(); + language_model_.reset_cache(); + image_prefill_completed_ = false; + last_token_count_ = 0; +} + +void Lfm2VlModel::load_weights_to_graph(CactusGraph* gb) { + namespace fs = std::filesystem; + fs::path base(model_folder_path_); + + auto resolve_weight = [&](const std::string& primary, const std::string& fallback = "") -> std::string { + fs::path primary_path = base / primary; + if (fs::exists(primary_path)) { + return primary_path.string(); + } + if (!fallback.empty()) { + fs::path fallback_path = base / fallback; + if (fs::exists(fallback_path)) { + return fallback_path.string(); + } + } + return primary_path.string(); + }; + + try { + projector_weights_.layer_norm_weight = gb->mmap_weights(resolve_weight("projector_layer_norm.weights")); + projector_weights_.layer_norm_bias = gb->mmap_weights(resolve_weight("projector_layer_norm.bias.weights")); + } catch (const std::exception&) { + projector_weights_.layer_norm_weight = 0; + projector_weights_.layer_norm_bias = 0; + } + + projector_weights_.linear_1_weight = gb->mmap_weights(resolve_weight("projector_linear_1.weights", "projector_linear1.weights")); + projector_weights_.linear_1_bias = gb->mmap_weights(resolve_weight("projector_linear_1.bias.weights", "projector_linear1.bias.weights")); + projector_weights_.linear_2_weight = gb->mmap_weights(resolve_weight("projector_linear_2.weights", "projector_linear2.weights")); + projector_weights_.linear_2_bias = gb->mmap_weights(resolve_weight("projector_linear_2.bias.weights", "projector_linear2.bias.weights")); + output_weight_node_id_ = gb->mmap_weights(resolve_weight("token_embeddings.weights")); +} + +size_t Lfm2VlModel::pixel_unshuffle(CactusGraph* gb, size_t hidden_states, + size_t height, size_t width, size_t channels) { + + const size_t factor = config_.downsample_factor; + const size_t new_height = height / factor; + const size_t new_width = width / factor; + size_t step1 = gb->reshape(hidden_states, {1, height, new_width, channels * factor}); + step1 = gb->transposeN(step1, {0, 2, 1, 3}); + size_t step2 = gb->reshape(step1, {1, new_width, new_height, channels * factor * factor}); + size_t result = gb->transposeN(step2, {0, 2, 1, 3}); + return result; +} + +size_t Lfm2VlModel::build_multimodal_projector(CactusGraph* gb, size_t image_features, + size_t tile_h, size_t tile_w, ComputeBackend backend) { + const size_t vision_hidden = config_.vision_embed_dim; + + const auto& input_buf = gb->get_output_buffer(image_features); + size_t image_features_fp16 = (input_buf.precision == Precision::FP16) + ? image_features + : gb->precision_cast(image_features, Precision::FP16); + + size_t unshuffled = pixel_unshuffle(gb, image_features_fp16, tile_h, tile_w, vision_hidden); + const size_t factor = config_.downsample_factor; + const size_t new_h = tile_h / factor; + const size_t new_w = tile_w / factor; + const size_t in_channels = vision_hidden * factor * factor; + const size_t seq_len = new_h * new_w; + size_t flattened = gb->reshape(unshuffled, {seq_len, in_channels}); + size_t normalized = flattened; + if (projector_weights_.layer_norm_weight != 0 && projector_weights_.layer_norm_bias != 0) { + normalized = gb->layernorm(flattened, projector_weights_.layer_norm_weight, + projector_weights_.layer_norm_bias, config_.layer_norm_eps); + } + size_t hidden = gb->matmul(normalized, projector_weights_.linear_1_weight, true, backend); + hidden = gb->add(hidden, projector_weights_.linear_1_bias); + hidden = gb->gelu(hidden); + size_t output = gb->matmul(hidden, projector_weights_.linear_2_weight, true, backend); + output = gb->add(output, projector_weights_.linear_2_bias); + return output; +} + +std::vector Lfm2VlModel::get_image_embeddings(const std::string& image_path) { + return vision_tower_.get_image_embedding(image_path); +} + +std::vector Lfm2VlModel::get_image_features( + CactusGraph* gb, + const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, + ComputeBackend backend) { + + size_t vision_output = vision_tower_.forward_vision(gb, preprocessed_image, backend); + std::vector projected_features; + projected_features.reserve(preprocessed_image.spatial_shapes.size()); + + size_t offset = 0; + for (size_t tile_idx = 0; tile_idx < preprocessed_image.spatial_shapes.size(); ++tile_idx) { + const auto& shape = preprocessed_image.spatial_shapes[tile_idx]; + const size_t tile_h = shape.first; + const size_t tile_w = shape.second; + const size_t tile_tokens = tile_h * tile_w; + const size_t factor = config_.downsample_factor; + if (factor == 0) { + throw std::runtime_error("Downsample factor must be greater than zero"); + } + if (tile_h % factor != 0 || tile_w % factor != 0) { + throw std::runtime_error("Tile dimensions must be divisible by downsample factor"); + } + const size_t new_h = tile_h / factor; + const size_t new_w = tile_w / factor; + const size_t projected_tokens = new_h * new_w; + + size_t tile_features = gb->slice(vision_output, 0, offset, tile_tokens); + offset += tile_tokens; + + size_t reshaped = gb->reshape(tile_features, {1, tile_h, tile_w, config_.vision_embed_dim}); + size_t projected = build_multimodal_projector(gb, reshaped, tile_h, tile_w, backend); + ProjectedTileFeature feature{}; + feature.node_id = projected; + feature.token_count = projected_tokens; + projected_features.push_back(feature); + + } + + return projected_features; +} + +Lfm2VlModel::MergedEmbeddingResult Lfm2VlModel::merge_image_text_embeddings( + CactusGraph* gb, + const std::vector& tokens, + const std::vector>& image_embedding_nodes, + std::vector& text_embedding_inputs) { + + text_embedding_inputs.clear(); + + Tokenizer* tokenizer = language_model_.get_tokenizer(); + if (!tokenizer) { + throw std::runtime_error("Tokenizer must be initialized before merging embeddings"); + } + + const uint32_t image_token_id = tokenizer->get_image_token_id(); + + auto get_token_id = [tokenizer](const std::string& token_text) -> uint32_t { + auto encoded = tokenizer->encode(token_text); + if (encoded.size() != 1) { + + throw std::runtime_error("Expected single token encoding for " + token_text); + } + return encoded[0]; + }; + + const uint32_t image_start_id = get_token_id("<|image_start|>"); + const uint32_t image_end_id = get_token_id("<|image_end|>"); + + std::vector sequence_nodes; + sequence_nodes.reserve(tokens.size() + image_embedding_nodes.size()); + + std::vector current_segment; + current_segment.reserve(tokens.size()); + + size_t total_seq_len = 0; + + auto flush_segment = [&](void) { + if (current_segment.empty()) { + + return; + } + + const size_t segment_len = current_segment.size(); + TextEmbeddingInput segment; + segment.tokens.swap(current_segment); + segment.input_node = gb->input({segment.tokens.size()}, Precision::FP32); + + const auto& embedding_buffer = gb->get_output_buffer(language_model_.embedding_node_id_); + + for (size_t i = 0; i < embedding_buffer.shape.size(); ++i) { + (void)i; + } + + size_t embedding_node = gb->embedding(language_model_.embedding_node_id_, segment.input_node); + + text_embedding_inputs.push_back(std::move(segment)); + + sequence_nodes.push_back(embedding_node); + + total_seq_len += segment_len; + + current_segment.clear(); + + }; + + size_t token_index = 0; + size_t image_index = 0; + + while (token_index < tokens.size()) { + uint32_t token_id = tokens[token_index]; + + if (token_id == image_start_id) { + flush_segment(); + + if (image_index >= image_embedding_nodes.size()) { + + throw std::runtime_error("Encountered <|image_start|> without corresponding image features"); + } + + current_segment.push_back(token_id); + ++token_index; + + const auto& tiles = image_embedding_nodes[image_index]; + size_t tile_index = 0; + while (token_index < tokens.size()) { + uint32_t inner_token = tokens[token_index]; + if (inner_token == image_token_id) { + flush_segment(); + + if (tile_index >= tiles.size()) { + + throw std::runtime_error("More placeholders than projected tile features"); + } + + const auto& tile = tiles[tile_index++]; + sequence_nodes.push_back(tile.node_id); + + total_seq_len += tile.token_count; + + for (size_t count = 0; count < tile.token_count; ++count) { + if (token_index >= tokens.size()) { + throw std::runtime_error("Insufficient tokens for projected features"); + } + if (tokens[token_index] != image_token_id) { + throw std::runtime_error("Unexpected token encountered within image feature span"); + } + ++token_index; + + } + continue; + } + + current_segment.push_back(inner_token); + + ++token_index; + + if (inner_token == image_end_id) { + flush_segment(); + break; + } + } + + if (tile_index != tiles.size()) { + if (tile_index < tiles.size()) { + for (size_t remaining = tile_index; remaining < tiles.size(); ++remaining) { + (void)tiles[remaining]; + } + } + throw std::runtime_error("Unused projected tile features remain after processing image block"); + } + + ++image_index; + + } else { + current_segment.push_back(token_id); + + ++token_index; + + } + } + + flush_segment(); + if (image_index != image_embedding_nodes.size()) { + throw std::runtime_error("Not all image features were consumed while merging embeddings"); + } + + if (sequence_nodes.empty()) { + throw std::runtime_error("Failed to build embedding sequence from provided tokens"); + } + + size_t merged = sequence_nodes[0]; + for (size_t idx = 1; idx < sequence_nodes.size(); ++idx) { + merged = gb->concat(merged, sequence_nodes[idx], 0); + + } + + return MergedEmbeddingResult{merged, total_seq_len}; +} + +size_t Lfm2VlModel::build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { + throw std::runtime_error("build_attention should not be called directly on Lfm2VlModel"); +} + +size_t Lfm2VlModel::build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const { + throw std::runtime_error("build_mlp should not be called directly on Lfm2VlModel"); +} + +size_t Lfm2VlModel::build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { + throw std::runtime_error("build_transformer_block should not be called directly on Lfm2VlModel"); +} + +size_t Lfm2VlModel::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + size_t final_hidden = language_model_.forward(gb, tokens, backend, use_cache); + + return final_hidden; +} + +uint32_t Lfm2VlModel::decode(const std::vector& tokens, + float temperature, + float top_p, + size_t top_k, + const std::string& profile_file, + float* out_entropy, + float min_p, + float repetition_penalty) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + if (temperature < 0) { + temperature = config_.default_temperature; + } + if (top_p < 0) { + top_p = config_.default_top_p; + } + if (top_k == 0) { + top_k = config_.default_top_k; + } + + image_prefill_completed_ = false; + last_token_count_ = tokens.size(); + + return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); +} + +void Lfm2VlModel::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + image_prefill_completed_ = false; + last_token_count_ = tokens.size(); + + language_model_.prefill(tokens, chunk_size, profile_file); +} + +void Lfm2VlModel::prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + if (tokens.empty()) { + return; + } + if (image_paths.empty()) { + prefill(tokens, get_prefill_chunk_size(), profile_file); + return; + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + auto forward_result = forward_images(gb, tokens, image_paths, backend, true); + gb->execute(profile_file); + + language_model_.post_execute_updates(gb, forward_result.seq_len); + language_model_.update_kv_cache(gb, forward_result.seq_len); + + image_prefill_completed_ = true; + last_token_count_ = tokens.size(); +} + +Lfm2VlModel::ForwardImageResult Lfm2VlModel::forward_images( + CactusGraph* gb, + const std::vector& tokens, + const std::vector& image_paths, + ComputeBackend backend, + bool use_cache) { + if (!gb) { + throw std::runtime_error("Graph must be initialized before forwarding"); + } + if (tokens.empty()) { + throw std::runtime_error("Token sequence cannot be empty"); + } + + std::vector> all_image_embeddings; + all_image_embeddings.reserve(image_paths.size()); + for (const auto& image_path : image_paths) { + auto preprocessed = preprocessor_.preprocess_from_file(image_path); + + auto image_features = get_image_features(gb, preprocessed, backend); + + all_image_embeddings.push_back(std::move(image_features)); + } + + std::vector text_embedding_inputs; + text_embedding_inputs.reserve(tokens.size() / 4 + 1); + + auto merged_embeddings = merge_image_text_embeddings(gb, tokens, all_image_embeddings, text_embedding_inputs); + if (merged_embeddings.seq_len == 0) { + throw std::runtime_error("Merged embedding sequence length cannot be zero"); + } + + for (const auto& embedding_input : text_embedding_inputs) { + if (embedding_input.tokens.empty()) { + continue; + } + + std::vector segment_data(embedding_input.tokens.size()); + for (size_t i = 0; i < embedding_input.tokens.size(); ++i) { + segment_data[i] = static_cast(embedding_input.tokens[i]); + } + gb->set_input(embedding_input.input_node, segment_data.data(), Precision::FP32); + } + size_t final_hidden = language_model_.forward(gb, merged_embeddings.node_id, merged_embeddings.seq_len, backend, use_cache); + return ForwardImageResult{final_hidden, merged_embeddings.seq_len}; +} + +uint32_t Lfm2VlModel::decode_with_images( + const std::vector& tokens, + const std::vector& image_paths, + float temperature, + float top_p, + size_t top_k, + const std::string& profile_file, + float* out_entropy, + float min_p, + float repetition_penalty) { + + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + if (image_paths.empty()) { + + image_prefill_completed_ = false; + last_token_count_ = tokens.size(); + return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); + } + + if (temperature < 0) { + temperature = config_.default_temperature; + } + if (top_p < 0) { + top_p = config_.default_top_p; + } + if (top_k == 0) { + top_k = config_.default_top_k; + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + bool cache_empty = language_model_.is_cache_empty(); + bool need_prefill = cache_empty || !image_prefill_completed_; + + if (!need_prefill && tokens.size() <= last_token_count_) { + + reset_cache(); + need_prefill = true; + } + + size_t seq_len_for_updates = 0; + size_t final_hidden_node = 0; + + if (need_prefill) { + auto forward_result = forward_images(gb, tokens, image_paths, backend, true); + + final_hidden_node = forward_result.final_hidden_node; + seq_len_for_updates = forward_result.seq_len; + image_prefill_completed_ = true; + last_token_count_ = tokens.size(); + } else { + size_t delta = tokens.size() - last_token_count_; + if (delta > tokens.size()) { + delta = tokens.size(); + } + if (delta == 0) { + if (tokens.empty()) { + throw std::runtime_error("Token sequence cannot be empty for cached decode step"); + } + delta = 1; + + } + std::vector incremental_tokens(tokens.end() - delta, tokens.end()); + + final_hidden_node = language_model_.forward(gb, incremental_tokens, backend, true); + + seq_len_for_updates = incremental_tokens.size(); + last_token_count_ = tokens.size(); + } + + auto logits_node_id = gb->matmul(final_hidden_node, language_model_.output_weight_node_id_, true, backend); + size_t sampled_token_id = + language_model_.sample_token(gb, logits_node_id, temperature, top_p, top_k, min_p, repetition_penalty, nullptr); + if (!profile_file.empty()) { + gb->execute(profile_file); + + } else { + gb->execute(); + + } + + compute_entropy(gb, logits_node_id, out_entropy); + + language_model_.post_execute_updates(gb, seq_len_for_updates); + language_model_.update_kv_cache(gb, seq_len_for_updates); + + auto* output_ptr = gb->get_output(sampled_token_id); + uint32_t result = *static_cast(output_ptr); + language_model_.record_sampled_token(result); + return result; +} + +} +} diff --git a/cactus-transpile/models/model_moonshine.cpp b/cactus-transpile/models/model_moonshine.cpp new file mode 100644 index 000000000..fe03f5983 --- /dev/null +++ b/cactus-transpile/models/model_moonshine.cpp @@ -0,0 +1,596 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../npu/npu.h" +#include "../kernel/kernel.h" +#include +#include +#include +#include +#include +#include +namespace cactus { +namespace engine { + +static size_t moonshine_downsampled_len(size_t audio_len) { + if (audio_len < 127) { + return 0; + } + size_t l1 = (audio_len - 127) / 64 + 1; + if (l1 < 7) { + return 0; + } + size_t l2 = (l1 - 7) / 3 + 1; + if (l2 < 3) { + return 0; + } + size_t l3 = (l2 - 1) / 2 + 1; + return l3; +} + +MoonshineModel::MoonshineModel() : Model() {} +MoonshineModel::MoonshineModel(const Config& config) : Model(config) { + weight_nodes_.encoder_layers.resize(config.num_encoder_layers); + weight_nodes_.decoder_layers.resize(config.num_decoder_layers); + float hd = static_cast(config.attention_head_dim); + if (hd <= 0.0f) { + hd = 64.0f; + } + attention_scale_ = 1.0f / std::sqrt(hd); + encoder_block_out_nodes_.resize(config.num_encoder_layers, 0); + encoder_k_persistent_.assign(config.num_decoder_layers, 0); + encoder_v_persistent_.assign(config.num_decoder_layers, 0); +} +void MoonshineModel::load_weights_to_graph(CactusGraph* gb) { + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + output_weight_node_id_ = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + if (output_weight_node_id_ == 0) { + output_weight_node_id_ = gb->mmap_weights(model_folder_path_ + "/output_layer.weights"); + } + weight_nodes_.output_weight = output_weight_node_id_; + if (npu::is_npu_available()) { + std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; + npu_encoder_ = npu::create_encoder(); + if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { + use_npu_encoder_ = true; + std::vector input_shape = npu_encoder_->get_input_shape(); + if (!input_shape.empty()) { + npu_encoder_->preallocate(input_shape, "x", ""); + } + } + else { + use_npu_encoder_ = false; + npu_encoder_.reset(); + } + } + if (!use_npu_encoder_) { + weight_nodes_.encoder_conv1_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_weight.weights"); + weight_nodes_.encoder_conv2_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_weight.weights"); + weight_nodes_.encoder_conv2_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_bias.bias"); + weight_nodes_.encoder_conv3_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv3_weight.weights"); + weight_nodes_.encoder_conv3_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv3_bias.bias"); + weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_norm_weight.weights"); + weight_nodes_.encoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/encoder_norm_bias.bias"); + weight_nodes_.encoder_layer_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_layer_norm_weight.weights"); + } + for (uint32_t i = 0; i < config_.num_decoder_layers; i++) { + auto& layer = weight_nodes_.decoder_layers[i]; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + layer.decoder_encoder_attn_k_weight = gb->mmap_weights(layer_prefix + "encoder_attn_k.weights"); + layer.decoder_encoder_attn_q_weight = gb->mmap_weights(layer_prefix + "encoder_attn_q.weights"); + layer.decoder_encoder_attn_v_weight = gb->mmap_weights(layer_prefix + "encoder_attn_v.weights"); + layer.decoder_encoder_attn_output_weight = gb->mmap_weights(layer_prefix + "encoder_attn_output.weights"); + layer.decoder_post_encoder_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); + if (config_.decoder_act_gelu) { + layer.decoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); + layer.decoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); + } + else { + layer.decoder_ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); + layer.decoder_ffn_gate_bias = gb->mmap_weights(layer_prefix + "ffn_gate.bias"); + layer.decoder_ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); + layer.decoder_ffn_up_bias = gb->mmap_weights(layer_prefix + "ffn_up.bias"); + } + layer.decoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); + layer.decoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); + layer.decoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); + layer.decoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); + layer.decoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); + layer.decoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); + layer.decoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); + layer.decoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); + } + if (!use_npu_encoder_) { + for (uint32_t i = 0; i < config_.num_encoder_layers; i++) { + auto& layer = weight_nodes_.encoder_layers[i]; + std::string layer_prefix = model_folder_path_ + "/encoder_layer_" + std::to_string(i) + "_"; + if (config_.encoder_act_gelu) { + layer.encoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); + layer.encoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); + } + else { + layer.encoder_ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); + layer.encoder_ffn_gate_bias = gb->mmap_weights(layer_prefix + "ffn_gate.bias"); + layer.encoder_ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); + layer.encoder_ffn_up_bias = gb->mmap_weights(layer_prefix + "ffn_up.bias"); + } + layer.encoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); + layer.encoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); + layer.encoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); + layer.encoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); + layer.encoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); + layer.encoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); + layer.encoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); + layer.encoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); + } + } +} +static size_t build_encoder_mlp_gelu(CactusGraph* gb, size_t input, size_t w1, size_t b1, size_t w2, size_t b2, ComputeBackend backend, uint32_t /*layer_idx*/) { + auto ffn1_weight = gb->matmul(input, w1, true, backend); + auto ffn1_bias = gb->add(ffn1_weight, b1); + auto ffn1_act = gb->gelu(ffn1_bias); + auto ffn2_weight = gb->matmul(ffn1_act, w2, true, backend); + auto ffn2_bias = gb->add(ffn2_weight, b2); + return ffn2_bias; +} +static size_t build_decoder_mlp_silu(CactusGraph* gb, size_t input, size_t w_gate, size_t b_gate, size_t w_up, size_t b_up, size_t w2, size_t b2, ComputeBackend backend, uint32_t /*layer_idx*/) { + auto gate_weight = gb->matmul(input, w_gate, true, backend); + auto gate_bias = gb->add(gate_weight, b_gate); + auto gate_act = gb->silu(gate_bias); + auto up_weight = gb->matmul(input, w_up, true, backend); + auto up_bias = gb->add(up_weight, b_up); + auto intermediate = gb->multiply(gate_act, up_bias); + auto output_weight = gb->matmul(intermediate, w2, true, backend); + auto output_bias = gb->add(output_weight, b2); + return output_bias; +} +size_t MoonshineModel::build_decoder_cross_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool /*use_cache*/, size_t /*position_offset*/){ + const auto& layer = weight_nodes_.decoder_layers[layer_idx]; + size_t q = gb->matmul(input, layer.decoder_encoder_attn_q_weight, true, backend); + const auto& q_buf = gb->get_output_buffer(q); + if (q_buf.shape.size() != 2) { + throw std::runtime_error("encoder cross-attn: q must be [T_dec, D]"); + } + size_t T_dec = q_buf.shape[0]; + size_t q_heads = config_.attention_heads; + size_t kv_heads = config_.attention_kv_heads; + size_t head_dim = config_.attention_head_dim; + q = gb->reshape(q, {1, T_dec, q_heads, head_dim}); + size_t k_4d = 0; + size_t v_4d = 0; + bool is_pop = (encoder_k_persistent_[layer_idx] != 0 && gb->is_populated(encoder_k_persistent_[layer_idx])); + std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + ".encoder_attn."; + if (is_pop) { + k_4d = encoder_k_persistent_[layer_idx]; + v_4d = encoder_v_persistent_[layer_idx]; + } + else { + size_t enc_norm = last_encoder_post_norm_node_; + size_t k = gb->matmul(enc_norm, layer.decoder_encoder_attn_k_weight, true, backend); + size_t v = gb->matmul(enc_norm, layer.decoder_encoder_attn_v_weight, true, backend); + const auto& k_buf = gb->get_output_buffer(k); + if (k_buf.shape.size() != 2) { + throw std::runtime_error("encoder cross-attn: k must be [T_enc, D]"); + } + size_t T_enc = k_buf.shape[0]; + k_4d = gb->reshape(k, {1, T_enc, kv_heads, head_dim}); + v_4d = gb->reshape(v, {1, T_enc, kv_heads, head_dim}); + if (encoder_k_persistent_[layer_idx] == 0) { + encoder_k_persistent_[layer_idx] = gb->persistent(k_4d); + encoder_v_persistent_[layer_idx] = gb->persistent(v_4d); + } + k_4d = encoder_k_persistent_[layer_idx]; + v_4d = encoder_v_persistent_[layer_idx]; + } + size_t attn = gb->attention(q, k_4d, v_4d, attention_scale_, false); + attn = gb->reshape(attn, {T_dec, q_heads * head_dim}); + size_t output = gb->matmul(attn, layer.decoder_encoder_attn_output_weight, true, backend); + return output; +} +void MoonshineModel::reset_graph_side_cache_nodes() { + cache_k_output_nodes_.assign(config_.num_decoder_layers, 0); + cache_v_output_nodes_.assign(config_.num_decoder_layers, 0); +} +void MoonshineModel::reset_cache() { + Model::reset_cache(); + encoder_ready_ = false; + encoder_kv_ready_ = false; + first_decode_step_ = true; + encoder_output_host_.clear(); + auto* gb = static_cast(graph_handle_); + if (gb) { + if (last_encoder_post_norm_node_ != 0) { + gb->invalidate_persistent(last_encoder_post_norm_node_); + last_encoder_post_norm_node_ = 0; + } + for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { + if (encoder_k_persistent_[i] != 0) { + gb->invalidate_persistent(encoder_k_persistent_[i]); + encoder_k_persistent_[i] = 0; + } + if (encoder_v_persistent_[i] != 0) { + gb->invalidate_persistent(encoder_v_persistent_[i]); + encoder_v_persistent_[i] = 0; + } + } + } +} +size_t MoonshineModel::build_decoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ + const auto& layer = weight_nodes_.decoder_layers[layer_idx]; + size_t q_proj = gb->matmul(input, layer.decoder_self_attn_q_weight, true, backend); + size_t k_proj = gb->matmul(input, layer.decoder_self_attn_k_weight, true, backend); + size_t v_proj = gb->matmul(input, layer.decoder_self_attn_v_weight, true, backend); + std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + ".self_attn."; + const auto& q_shape = gb->get_output_buffer(q_proj).shape; + if (q_shape.size() != 2) { + throw std::runtime_error("decoder self-attn: q must be [T_new, D]"); + } + size_t seq_new = q_shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + size_t num_kv_heads = config_.attention_kv_heads; + size_t q_4d = gb->reshape(q_proj, {1, seq_new, num_heads, head_dim}); + size_t k_4d = gb->reshape(k_proj, {1, seq_new, num_kv_heads, head_dim}); + size_t v_4d = gb->reshape(v_proj, {1, seq_new, num_kv_heads, head_dim}); + size_t rot_dim = std::max(head_dim / 2, (size_t)32); + if (config_.rope_theta > 0) { + q_4d = gb->rope_gptj(q_4d, config_.rope_theta, position_offset, rot_dim); + k_4d = gb->rope_gptj(k_4d, config_.rope_theta, position_offset, rot_dim); + } + size_t final_k = k_4d; + size_t final_v = v_4d; + if (use_cache && !kv_cache_.is_empty()) { + auto k_view = kv_cache_.get_key_view(layer_idx); + auto v_view = kv_cache_.get_value_view(layer_idx); + if (!k_view.ptr1 || !v_view.ptr1) { + throw std::runtime_error("KV cache view is empty but kv_cache_.is_empty()==false"); + } + size_t cache_len = kv_cache_.current_seq_len; + size_t cache_k_node = gb->input( + {1, cache_len, num_kv_heads, head_dim} +, + kv_cache_.precision + ); + size_t cache_v_node = gb->input( + {1, cache_len, num_kv_heads, head_dim} +, + kv_cache_.precision + ); + if (k_view.ptr2 == nullptr && v_view.ptr2 == nullptr) { + gb->set_external_input(cache_k_node, const_cast(k_view.ptr1), kv_cache_.precision); + gb->set_external_input(cache_v_node, const_cast(v_view.ptr1), kv_cache_.precision); + } + else { + gb->set_external_input(cache_k_node, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); + gb->set_external_input(cache_v_node, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); + } + final_k = gb->concat(cache_k_node, k_4d, 1); + final_v = gb->concat(cache_v_node, v_4d, 1); + } + if (use_cache) { + cache_k_output_nodes_[layer_idx] = final_k; + cache_v_output_nodes_[layer_idx] = final_v; + } + else { + cache_k_output_nodes_[layer_idx] = k_4d; + cache_v_output_nodes_[layer_idx] = v_4d; + } + auto attn_out_4d = gb->attention(q_4d, final_k, final_v, attention_scale_, position_offset); + auto attn_out = gb->reshape(attn_out_4d, {seq_new, num_heads * head_dim}); + auto output = gb->matmul(attn_out, layer.decoder_self_attn_output_weight, true, backend); + return output; +} +size_t MoonshineModel::build_encoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t /*position_offset*/){ + const auto& layer = weight_nodes_.encoder_layers[layer_idx]; + if(use_cache) + throw std::runtime_error("The encoder attention layers are not auto-regressive, and thus don't use KV caching!"); + auto q_proj = gb->matmul(input, layer.encoder_self_attn_q_weight, true, backend); + auto v_proj = gb->matmul(input, layer.encoder_self_attn_v_weight, true, backend); + auto k_proj = gb->matmul(input, layer.encoder_self_attn_k_weight, true, backend); + std::string prefix = "model.encoder.layers." + std::to_string(layer_idx) + ".self_attn."; + size_t seq_len = gb->get_output_buffer(q_proj).shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + auto q = gb->reshape(q_proj, {1, seq_len, num_heads, head_dim}); + auto k = gb->reshape(k_proj, {1, seq_len, num_heads, head_dim}); + auto v = gb->reshape(v_proj, {1, seq_len, num_heads, head_dim}); + size_t rot_dim = static_cast(head_dim * config_.partial_rotary_factor); + if (rot_dim % 2 != 0) throw std::runtime_error("rot_dim must be even"); + if (config_.rope_theta > 0) { + q = gb->rope_gptj(q, config_.rope_theta, 0, rot_dim); + k = gb->rope_gptj(k, config_.rope_theta, 0, rot_dim); + } + auto attn = gb->attention(q, k, v, attention_scale_, false); + attn = gb->reshape(attn, {seq_len, num_heads * head_dim}); + auto output = gb->matmul(attn, layer.encoder_self_attn_output_weight, true, backend); + return output; +} +size_t MoonshineModel::build_audio_preprocessor(CactusGraph* gb, size_t input) +{ + size_t conv_input = input; + const auto& xbuf = gb->get_output_buffer(input); + if (xbuf.precision == Precision::INT8) { + conv_input = gb->precision_cast(input, Precision::FP16); + } + size_t conv1 = gb->conv1d(conv_input, weight_nodes_.encoder_conv1_weight, 64); + last_conv1_node_ = conv1; + size_t conv1_act = gb->tanh(conv1); + size_t group_norm = gb->groupnorm(conv1_act, weight_nodes_.encoder_norm_weight, weight_nodes_.encoder_norm_bias, 1); + + size_t conv2_with_bias = gb->conv1d_k7s3(group_norm, weight_nodes_.encoder_conv2_weight, weight_nodes_.encoder_conv2_bias); + size_t conv2_act = gb->gelu(conv2_with_bias); + size_t conv3 = gb->conv1d_k3(conv2_act, weight_nodes_.encoder_conv3_weight, 2); + auto bias3_shape = gb->get_output_buffer(weight_nodes_.encoder_conv3_bias).shape; + size_t C3 = bias3_shape[0]; + size_t bias3 = gb->reshape(weight_nodes_.encoder_conv3_bias, {1, C3, 1}); + size_t conv3_with_bias = gb->add(conv3, bias3); + size_t conv3_act = gb->gelu(conv3_with_bias); + const auto& buf = gb->get_output_buffer(conv3_act); + size_t conv_out_transposed; + if (buf.precision == Precision::FP16) { + conv_out_transposed = gb->transpose(conv3_act, ComputeBackend::CPU); + } + else { + size_t conv3_f16 = gb->precision_cast(conv3_act, Precision::FP16); + conv_out_transposed = gb->transpose(conv3_f16, ComputeBackend::CPU); + } + return conv_out_transposed; +} +size_t MoonshineModel::build_encoder_transformer_block( + CactusGraph* gb, + size_t hidden, + uint32_t layer_idx, + ComputeBackend backend, + bool use_cache, + size_t position_offset) +{ + const auto& layer = weight_nodes_.encoder_layers[layer_idx]; + size_t input_layernorm = gb->layernorm( + hidden, + layer.encoder_post_attn_layernorm_weight + ); + size_t self_attn_out = build_encoder_self_attention( + gb, input_layernorm, layer_idx, backend, use_cache, position_offset + ); + size_t post_attention_layernorm_input = gb->add(hidden, self_attn_out); + size_t post_attention_layernorm = gb->layernorm( + post_attention_layernorm_input, + layer.encoder_post_ffn_layernorm_weight + ); + std::string prefix = "model.encoder.layers." + std::to_string(layer_idx) + "."; + size_t mlp_out = build_encoder_mlp_gelu( + gb, post_attention_layernorm, layer.encoder_ffn1_weight, layer.encoder_ffn1_bias, + layer.encoder_ffn2_weight, layer.encoder_ffn2_bias, backend, layer_idx + ); + size_t out = gb->add(post_attention_layernorm_input, mlp_out); + if (layer_idx < encoder_block_out_nodes_.size()) { + encoder_block_out_nodes_[layer_idx] = out; + } + return out; +} +size_t MoonshineModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ + const auto& layer = weight_nodes_.decoder_layers[layer_idx]; + size_t input_layernorm = gb->layernorm(hidden, layer.decoder_post_attn_layernorm_weight); + size_t sa = build_decoder_self_attention(gb, input_layernorm, layer_idx, backend, use_cache, position_offset); + size_t x_post_sa = gb->add(hidden, sa); + size_t post_attention_layernorm = gb->layernorm(x_post_sa, layer.decoder_post_encoder_layernorm_weight); + size_t ca = build_decoder_cross_attention(gb, post_attention_layernorm, layer_idx, backend, use_cache, position_offset); + size_t x_post_ca = gb->add(x_post_sa, ca); + size_t final_layernorm = gb->layernorm(x_post_ca, layer.decoder_post_ffn_layernorm_weight); + size_t mlp_out = build_decoder_mlp_silu( + gb, final_layernorm, layer.decoder_ffn_gate_weight, layer.decoder_ffn_gate_bias, + layer.decoder_ffn_up_weight, layer.decoder_ffn_up_bias, + layer.decoder_ffn2_weight, layer.decoder_ffn2_bias, backend, layer_idx + ); + size_t x_post_ffn = gb->add(x_post_ca, mlp_out); + std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + "."; + return x_post_ffn; +} +size_t MoonshineModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) +{ + if (!audio_features.empty()) { + float min_val = audio_features[0]; + float max_val = audio_features[0]; + double sum_val = 0.0; + for (float val : audio_features) { + if (val < min_val) min_val = val; + if (val > max_val) max_val = val; + sum_val += val; + } + (void)sum_val; + } + if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { + std::vector input_shape = npu_encoder_->get_input_shape(); + if (input_shape.size() != 3) { + input_shape = {1, 1, static_cast(audio_features.size())}; + } + if (input_shape[1] != 1) { + throw std::runtime_error("Moonshine NPU encoder expects mono audio input"); + } + size_t expected_samples = static_cast(input_shape[2]); + std::vector<__fp16> audio_f16(expected_samples, 0); + size_t copy_samples = std::min(audio_features.size(), expected_samples); + if (copy_samples > 0) { + cactus_fp32_to_fp16(audio_features.data(), audio_f16.data(), copy_samples); + } + size_t T_enc = moonshine_downsampled_len(expected_samples); + size_t D_enc = static_cast(config_.hidden_dim); + if (T_enc == 0 || D_enc == 0) { + std::cout << "NPU encoder output has unexpected shape for input size " + << expected_samples << std::endl; + std::cout << "Falling back to CPU encoder path." << std::endl; + goto encoder_cpu_fallback; + } + + size_t total_elements = T_enc * D_enc; + std::vector<__fp16> npu_output(total_elements); + size_t elements_written = npu_encoder_->encode( + audio_f16.data(), + npu_output.data(), + input_shape, + "x", + "" + ); + if (elements_written > 0) { + size_t enc_output_node = gb->input({T_enc, D_enc}, Precision::FP16); + gb->set_input(enc_output_node, npu_output.data(), Precision::FP16); + size_t enc_output_persistent = gb->persistent(enc_output_node); + last_encoder_post_norm_node_ = enc_output_persistent; + return enc_output_persistent; + } + } +encoder_cpu_fallback: + auto backend = + (config_.default_backend == Config::Backend::CPU) + ? ComputeBackend::CPU + : ComputeBackend::NPU; + size_t audio_input = 0; + const std::vector* cpu_audio = &audio_features; + std::vector<__fp16> audio_f16(cpu_audio->size()); + if (!cpu_audio->empty()) { + cactus_fp32_to_fp16(cpu_audio->data(), audio_f16.data(), cpu_audio->size()); + } + size_t audio_length = cpu_audio->size(); + audio_input = gb->input({1, 1, audio_length}, Precision::FP16); + gb->set_input(audio_input, audio_f16.data(), Precision::FP16); + size_t conv2_transposed = build_audio_preprocessor(gb, audio_input); + const auto& conv_shape = gb->get_output_buffer(conv2_transposed).shape; + if (conv_shape.size() != 3 || conv_shape[0] != 1) + throw std::runtime_error("Conv2 transpose should be [1, T_enc, D]."); + size_t T_enc = conv_shape[1]; + size_t D_enc = conv_shape[2]; + size_t h = gb->reshape(conv2_transposed, {T_enc, D_enc}); + for (uint32_t i = 0; i < config_.num_encoder_layers; ++i){ + h = build_encoder_transformer_block(gb, h, i, backend, false, 0); + } + size_t h_norm = gb->layernorm( + h, + weight_nodes_.encoder_layer_norm_weight + ); + size_t h_norm_persistent = gb->persistent(h_norm); + last_encoder_post_norm_node_ = h_norm_persistent; + return h_norm_persistent; +} +size_t MoonshineModel::build_decoder(const std::vector& tokens, bool use_cache, bool last_token_only) { + auto* gb = static_cast(graph_handle_); + const size_t full_len = tokens.size(); + if (full_len == 0) { + throw std::runtime_error("Decoder token list cannot be empty."); + } + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + size_t start_idx = (use_cache && kv_cache_.current_seq_len > 0) ? full_len - 1 : 0; + size_t new_tokens = full_len - start_idx; + size_t position_offset = use_cache ? kv_cache_.current_seq_len : 0; + size_t tok_input = gb->input({new_tokens}, Precision::FP32); + std::vector tok_f(new_tokens); + for (size_t i = 0; i < new_tokens; i++) { + tok_f[i] = static_cast(tokens[start_idx + i]); + } + gb->set_input(tok_input, tok_f.data(), Precision::FP32); + size_t dec_hidden = gb->embedding(embedding_node_id_, tok_input); + for (uint32_t layer_idx = 0; layer_idx < config_.num_decoder_layers; ++layer_idx) { + dec_hidden = build_decoder_transformer_block( + gb, + dec_hidden, + layer_idx, + backend, + use_cache, + position_offset + ); + } + size_t dec_norm = gb->layernorm( + dec_hidden, + weight_nodes_.decoder_norm_weight + ); + size_t logits_input = dec_norm; + if (last_token_only) { + size_t row_index = new_tokens - 1; + logits_input = gb->slice(logits_input, 0, row_index, 1); + } + auto w_shape = gb->get_output_buffer(output_weight_node_id_).shape; + size_t logits = gb->matmul(logits_input, output_weight_node_id_, true, backend); + last_new_tokens_ = new_tokens; + return logits; +} +size_t MoonshineModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) +{ + auto* gb = static_cast(graph_handle_); + gb->clear_debug_nodes(); + build_encoder(gb, audio_features); + size_t logits = build_decoder(tokens, use_cache, true); + return logits; +} +std::vector MoonshineModel::get_audio_embeddings(const std::vector& audio_features) { + auto* gb = static_cast(graph_handle_); + build_encoder(gb, audio_features); + size_t pooled = gb->mean(last_encoder_post_norm_node_, 0); + gb->execute(); + const auto& output_buf = gb->get_output_buffer(pooled); + size_t hidden_dim = output_buf.total_size; + std::vector embedding(hidden_dim); + void* output_data = gb->get_output(pooled); + const float* output_ptr = static_cast(output_data); + std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); + reset_cache(); + return embedding; +} +uint32_t MoonshineModel::decode_with_audio( + const std::vector& tokens, + const std::vector& audio_features, + float temperature, + float top_p, + size_t top_k, + const std::string& profile_file, + float* out_entropy, + float min_p, + float repetition_penalty, + float* /*out_token_time_start*/, + float* /*out_token_time_end*/) +{ + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (audio_features.empty()) + throw std::runtime_error("Audio features cannot be empty in Moonshine decode_with_audio"); + auto* gb = static_cast(graph_handle_); + gb->clear_debug_nodes(); + bool cold_start = !encoder_ready_; + size_t logits_node = 0; + uint32_t bos = static_cast(get_tokenizer()->get_bos_token()); + std::vector full_tokens; + full_tokens.reserve(tokens.size() + 1); + full_tokens.push_back(bos); + full_tokens.insert(full_tokens.end(), tokens.begin(), tokens.end()); + if (cold_start) + { + gb->soft_reset(); + kv_cache_.reset(); + kv_cache_.current_seq_len = 0; + reset_graph_side_cache_nodes(); + encoder_kv_ready_ = false; + first_decode_step_ = true; + build_encoder(gb, audio_features); + logits_node = build_decoder(full_tokens, false, false); + encoder_ready_ = true; + } + else + { + gb->soft_reset(); + reset_graph_side_cache_nodes(); + std::vector last_token_vec = { tokens.back() }; + logits_node = build_decoder(last_token_vec, true, true); + } + size_t sampled_token_id = sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty); + if (!profile_file.empty()) gb->execute(profile_file); + else gb->execute(); + compute_entropy(gb, logits_node, out_entropy); + post_execute_updates(gb, full_tokens.size()); + update_kv_cache(gb, last_new_tokens_); + auto* out_ptr = gb->get_output(sampled_token_id); + uint32_t sampled = *reinterpret_cast(out_ptr); + record_sampled_token(sampled); + return sampled; +} +} +} diff --git a/cactus-transpile/models/model_needle.cpp b/cactus-transpile/models/model_needle.cpp new file mode 100644 index 000000000..e7220a97f --- /dev/null +++ b/cactus-transpile/models/model_needle.cpp @@ -0,0 +1,337 @@ +#include "model.h" +#include "../graph/graph.h" +#include +#include + +namespace cactus { +namespace engine { + +namespace { + +size_t delta_rms_norm(CactusGraph* gb, size_t input, size_t weight, float epsilon) { + return gb->rms_norm(input, gb->scalar_add(weight, 1.0f), epsilon); +} + +size_t normalize_qk_proj(CactusGraph* gb, size_t proj, size_t norm_weight, + size_t seq_len, size_t num_heads, size_t head_dim, float epsilon) { + proj = gb->reshape(proj, {seq_len * num_heads, head_dim}); + proj = delta_rms_norm(gb, proj, norm_weight, epsilon); + return gb->reshape(proj, {seq_len, num_heads * head_dim}); +} + +size_t apply_residual_gate(CactusGraph* gb, size_t residual, size_t update, size_t gate_weight) { + return gb->add_clipped(residual, gb->multiply(update, gb->sigmoid(gate_weight))); +} + +size_t embed_tokens(CactusGraph* gb, size_t embedding_id, const std::vector& tokens, size_t hidden_dim) { + size_t n = tokens.size(); + size_t input_node = gb->input({n}, Precision::FP32); + std::vector data(n); + for (size_t i = 0; i < n; ++i) data[i] = static_cast(tokens[i]); + gb->set_input(input_node, data.data(), Precision::FP32); + return gb->scalar_multiply(gb->embedding(embedding_id, input_node), std::sqrt(static_cast(hidden_dim))); +} + +} // namespace + +NeedleModel::NeedleModel() : Model() {} + +NeedleModel::NeedleModel(const Config& config) : Model(config) { + weight_nodes_.encoder_layers.resize(config.num_encoder_layers); + weight_nodes_.decoder_layers.resize(config.num_decoder_layers); + float hd = static_cast(config.attention_head_dim); + attention_scale_ = 1.0f / std::sqrt(hd > 0.0f ? hd : 64.0f); + encoder_k_persistent_.assign(config.num_decoder_layers, 0); + encoder_v_persistent_.assign(config.num_decoder_layers, 0); +} + +void NeedleModel::load_weights_to_graph(CactusGraph* gb) { + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_layer_norm_weight.weights"); + weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + } + + for (uint32_t i = 0; i < config_.num_encoder_layers; ++i) { + auto& l = weight_nodes_.encoder_layers[i]; + std::string p = model_folder_path_ + "/encoder_layer_" + std::to_string(i) + "_"; + l.attn_gate_weight = gb->mmap_weights(p + "attn_gate.weights"); + l.input_norm_weight = gb->mmap_weights(p + "input_norm.weights"); + l.attn_q_weight = gb->mmap_weights(p + "attn_q.weights"); + l.attn_k_weight = gb->mmap_weights(p + "attn_k.weights"); + l.attn_v_weight = gb->mmap_weights(p + "attn_v.weights"); + l.attn_output_weight = gb->mmap_weights(p + "attn_output.weights"); + l.attn_q_norm_weight = gb->mmap_weights(p + "attn_q_norm.weights"); + l.attn_k_norm_weight = gb->mmap_weights(p + "attn_k_norm.weights"); + } + + for (uint32_t i = 0; i < config_.num_decoder_layers; ++i) { + auto& l = weight_nodes_.decoder_layers[i]; + std::string p = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + l.self_attn_gate_weight = gb->mmap_weights(p + "self_attn_gate.weights"); + l.cross_attn_gate_weight = gb->mmap_weights(p + "cross_attn_gate.weights"); + l.input_norm_weight = gb->mmap_weights(p + "input_norm.weights"); + l.post_attn_norm_weight = gb->mmap_weights(p + "post_attn_norm.weights"); + l.self_attn_q_weight = gb->mmap_weights(p + "attn_q.weights"); + l.self_attn_k_weight = gb->mmap_weights(p + "attn_k.weights"); + l.self_attn_v_weight = gb->mmap_weights(p + "attn_v.weights"); + l.self_attn_output_weight = gb->mmap_weights(p + "attn_output.weights"); + l.self_attn_q_norm_weight = gb->mmap_weights(p + "attn_q_norm.weights"); + l.self_attn_k_norm_weight = gb->mmap_weights(p + "attn_k_norm.weights"); + l.encoder_attn_q_weight = gb->mmap_weights(p + "encoder_attn_q.weights"); + l.encoder_attn_k_weight = gb->mmap_weights(p + "encoder_attn_k.weights"); + l.encoder_attn_v_weight = gb->mmap_weights(p + "encoder_attn_v.weights"); + l.encoder_attn_output_weight = gb->mmap_weights(p + "encoder_attn_output.weights"); + l.encoder_attn_q_norm_weight = gb->mmap_weights(p + "encoder_attn_q_norm.weights"); + l.encoder_attn_k_norm_weight = gb->mmap_weights(p + "encoder_attn_k_norm.weights"); + } +} + +void NeedleModel::reset_graph_side_cache_nodes() { + cache_k_output_nodes_.assign(config_.num_decoder_layers, 0); + cache_v_output_nodes_.assign(config_.num_decoder_layers, 0); +} + +void NeedleModel::reset_cache() { + Model::reset_cache(); + encoder_ready_ = false; + auto* gb = static_cast(graph_handle_); + if (!gb) { reset_graph_side_cache_nodes(); return; } + + if (last_encoder_post_norm_node_ != 0) { + gb->invalidate_persistent(last_encoder_post_norm_node_); + last_encoder_post_norm_node_ = 0; + } + for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { + if (encoder_k_persistent_[i] != 0) { gb->invalidate_persistent(encoder_k_persistent_[i]); encoder_k_persistent_[i] = 0; } + if (encoder_v_persistent_[i] != 0) { gb->invalidate_persistent(encoder_v_persistent_[i]); encoder_v_persistent_[i] = 0; } + } + reset_graph_side_cache_nodes(); +} + +size_t NeedleModel::build_encoder_self_attention(CactusGraph* gb, size_t input, + uint32_t layer_idx, ComputeBackend backend, + bool /*use_cache*/, size_t /*position_offset*/) { + const auto& l = weight_nodes_.encoder_layers[layer_idx]; + auto q = gb->matmul(input, l.attn_q_weight, true, backend); + auto k = gb->matmul(input, l.attn_k_weight, true, backend); + auto v = gb->matmul(input, l.attn_v_weight, true, backend); + + size_t seq = gb->get_output_buffer(q).shape[0]; + size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; + + q = normalize_qk_proj(gb, q, l.attn_q_norm_weight, seq, nh, hd, config_.layer_norm_eps); + k = normalize_qk_proj(gb, k, l.attn_k_norm_weight, seq, nkv, hd, config_.layer_norm_eps); + + auto q4 = gb->reshape(q, {1, seq, nh, hd}); + auto k4 = gb->reshape(k, {1, seq, nkv, hd}); + auto v4 = gb->reshape(v, {1, seq, nkv, hd}); + if (config_.rope_theta > 0) { q4 = gb->rope(q4, config_.rope_theta, 0); k4 = gb->rope(k4, config_.rope_theta, 0); } + + auto attn = gb->attention(q4, k4, v4, attention_scale_, false); + return gb->matmul(gb->reshape(attn, {seq, nh * hd}), l.attn_output_weight, true, backend); +} + +size_t NeedleModel::build_decoder_self_attention(CactusGraph* gb, size_t input, + uint32_t layer_idx, ComputeBackend backend, + bool use_cache, size_t position_offset) { + const auto& l = weight_nodes_.decoder_layers[layer_idx]; + auto q = gb->matmul(input, l.self_attn_q_weight, true, backend); + auto k = gb->matmul(input, l.self_attn_k_weight, true, backend); + auto v = gb->matmul(input, l.self_attn_v_weight, true, backend); + + size_t seq = gb->get_output_buffer(q).shape[0]; + size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; + + q = normalize_qk_proj(gb, q, l.self_attn_q_norm_weight, seq, nh, hd, config_.layer_norm_eps); + k = normalize_qk_proj(gb, k, l.self_attn_k_norm_weight, seq, nkv, hd, config_.layer_norm_eps); + + auto q4 = gb->reshape(q, {1, seq, nh, hd}); + auto k4 = gb->reshape(k, {1, seq, nkv, hd}); + auto v4 = gb->reshape(v, {1, seq, nkv, hd}); + if (config_.rope_theta > 0) { q4 = gb->rope(q4, config_.rope_theta, position_offset); k4 = gb->rope(k4, config_.rope_theta, position_offset); } + + size_t final_k = k4, final_v = v4; + if (use_cache && !kv_cache_.is_empty()) { + size_t cache_len = kv_cache_.current_seq_len; + size_t ck = gb->input({1, cache_len, nkv, hd}, kv_cache_.precision); + size_t cv = gb->input({1, cache_len, nkv, hd}, kv_cache_.precision); + auto kv = kv_cache_.get_key_view(layer_idx); + auto vv = kv_cache_.get_value_view(layer_idx); + if (!kv.ptr2 && !vv.ptr2) { + gb->set_external_input(ck, const_cast(kv.ptr1), kv_cache_.precision); + gb->set_external_input(cv, const_cast(vv.ptr1), kv_cache_.precision); + } else { + gb->set_external_input(ck, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); + gb->set_external_input(cv, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); + } + final_k = gb->concat(ck, k4, 1); + final_v = gb->concat(cv, v4, 1); + } + + cache_k_output_nodes_[layer_idx] = use_cache ? final_k : k4; + cache_v_output_nodes_[layer_idx] = use_cache ? final_v : v4; + + auto attn = gb->attention(q4, final_k, final_v, attention_scale_, position_offset); + return gb->matmul(gb->reshape(attn, {seq, nh * hd}), l.self_attn_output_weight, true, backend); +} + +size_t NeedleModel::build_decoder_cross_attention(CactusGraph* gb, size_t input, + uint32_t layer_idx, ComputeBackend backend, + bool /*use_cache*/, size_t /*position_offset*/) { + const auto& l = weight_nodes_.decoder_layers[layer_idx]; + size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; + + auto q = gb->matmul(input, l.encoder_attn_q_weight, true, backend); + size_t seq_dec = gb->get_output_buffer(q).shape[0]; + q = normalize_qk_proj(gb, q, l.encoder_attn_q_norm_weight, seq_dec, nh, hd, config_.layer_norm_eps); + auto q4 = gb->reshape(q, {1, seq_dec, nh, hd}); + + size_t k4 = 0, v4 = 0; + bool cached = encoder_k_persistent_[layer_idx] != 0 && gb->is_populated(encoder_k_persistent_[layer_idx]); + if (cached) { + k4 = encoder_k_persistent_[layer_idx]; + v4 = encoder_v_persistent_[layer_idx]; + } else { + auto kp = gb->matmul(last_encoder_post_norm_node_, l.encoder_attn_k_weight, true, backend); + auto vp = gb->matmul(last_encoder_post_norm_node_, l.encoder_attn_v_weight, true, backend); + size_t seq_enc = gb->get_output_buffer(kp).shape[0]; + kp = normalize_qk_proj(gb, kp, l.encoder_attn_k_norm_weight, seq_enc, nkv, hd, config_.layer_norm_eps); + k4 = gb->reshape(kp, {1, seq_enc, nkv, hd}); + v4 = gb->reshape(vp, {1, seq_enc, nkv, hd}); + if (encoder_k_persistent_[layer_idx] == 0) { + encoder_k_persistent_[layer_idx] = gb->persistent(k4); + encoder_v_persistent_[layer_idx] = gb->persistent(v4); + } + k4 = encoder_k_persistent_[layer_idx]; + v4 = encoder_v_persistent_[layer_idx]; + } + + auto attn = gb->attention(q4, k4, v4, attention_scale_, false); + return gb->matmul(gb->reshape(attn, {seq_dec, nh * hd}), l.encoder_attn_output_weight, true, backend); +} + +size_t NeedleModel::build_encoder_transformer_block(CactusGraph* gb, size_t hidden, + uint32_t layer_idx, ComputeBackend backend, + bool use_cache, size_t position_offset) { + const auto& l = weight_nodes_.encoder_layers[layer_idx]; + auto normed = delta_rms_norm(gb, hidden, l.input_norm_weight, config_.layer_norm_eps); + auto attn = build_encoder_self_attention(gb, normed, layer_idx, backend, use_cache, position_offset); + return apply_residual_gate(gb, hidden, attn, l.attn_gate_weight); +} + +size_t NeedleModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, + uint32_t layer_idx, ComputeBackend backend, + bool use_cache, size_t position_offset) { + const auto& l = weight_nodes_.decoder_layers[layer_idx]; + auto normed = delta_rms_norm(gb, hidden, l.input_norm_weight, config_.layer_norm_eps); + auto self_out = build_decoder_self_attention(gb, normed, layer_idx, backend, use_cache, position_offset); + auto after_self = apply_residual_gate(gb, hidden, self_out, l.self_attn_gate_weight); + auto cross_in = delta_rms_norm(gb, after_self, l.post_attn_norm_weight, config_.layer_norm_eps); + auto cross_out = build_decoder_cross_attention(gb, cross_in, layer_idx, backend, use_cache, position_offset); + return apply_residual_gate(gb, after_self, cross_out, l.cross_attn_gate_weight); +} + +void NeedleModel::run_encoder(const std::vector& encoder_tokens) { + auto* gb = static_cast(graph_handle_); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + auto hidden = embed_tokens(gb, embedding_node_id_, encoder_tokens, config_.hidden_dim); + + for (uint32_t i = 0; i < config_.num_encoder_layers; ++i) + hidden = build_encoder_transformer_block(gb, hidden, i, backend, false, 0); + + last_encoder_post_norm_node_ = gb->persistent( + delta_rms_norm(gb, hidden, weight_nodes_.encoder_norm_weight, config_.layer_norm_eps)); +} + +size_t NeedleModel::run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only) { + auto* gb = static_cast(graph_handle_); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + size_t pos_offset = use_cache ? kv_cache_.current_seq_len : 0; + + auto hidden = embed_tokens(gb, embedding_node_id_, tokens, config_.hidden_dim); + for (uint32_t i = 0; i < config_.num_decoder_layers; ++i) + hidden = build_decoder_transformer_block(gb, hidden, i, backend, use_cache, pos_offset); + + auto norm = delta_rms_norm(gb, hidden, weight_nodes_.decoder_norm_weight, config_.layer_norm_eps); + if (last_token_only) norm = gb->slice(norm, 0, tokens.size() - 1, 1); + return gb->matmul(norm, output_weight_node_id_, true, backend); +} + +size_t NeedleModel::forward(const std::vector& tokens, bool use_cache) { + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + reset_graph_side_cache_nodes(); + + if (!encoder_ready_ || !use_cache) { + std::vector enc(tokens.begin(), tokens.end() - 1); + std::vector seed(tokens.end() - 1, tokens.end()); + if (!use_cache) { reset_cache(); gb->soft_reset(); } + run_encoder(enc); + encoder_ready_ = true; + return run_decoder_step(seed, false, false); + } + return run_decoder_step(tokens, true, tokens.size() == 1); +} + +void NeedleModel::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& profile_file) { + if (tokens.empty()) return; + reset_cache(); + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + run_encoder(tokens); + if (!profile_file.empty()) gb->execute(profile_file); else gb->execute(); + encoder_ready_ = true; +} + +uint32_t NeedleModel::decode(const std::vector& tokens, + float temperature, float top_p, size_t top_k, + const std::string& profile_file, float* out_entropy, + float /*min_p*/, float /*repetition_penalty*/) { + if (temperature < 0) temperature = config_.default_temperature; + if (top_p < 0) top_p = config_.default_top_p; + if (top_k == 0) top_k = config_.default_top_k; + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + reset_graph_side_cache_nodes(); + + bool cold_start = kv_cache_.is_empty(); + std::vector cold_tokens; + const std::vector* dec_tokens = &tokens; + if (!encoder_ready_) { + reset_cache(); gb->soft_reset(); + std::vector enc(tokens.begin(), tokens.end() - 1); + cold_tokens.assign(tokens.end() - 1, tokens.end()); + run_encoder(enc); + encoder_ready_ = true; + dec_tokens = &cold_tokens; + cold_start = true; + } + + size_t logits = cold_start + ? run_decoder_step(*dec_tokens, false, false) + : run_decoder_step(tokens, true, tokens.size() == 1); + + if (config_.final_logit_softcapping > 0.0f) { + float inv = 1.0f / config_.final_logit_softcapping; + logits = gb->scalar_multiply(gb->tanh(gb->scalar_multiply(logits, inv)), config_.final_logit_softcapping); + } + + auto sampled = sample_token(gb, logits, temperature, top_p, top_k, 0.0f, 1.0f); + if (!profile_file.empty()) gb->execute(profile_file); else gb->execute(); + + compute_entropy(gb, logits, out_entropy); + post_execute_updates(gb, tokens.size()); + update_kv_cache(gb, dec_tokens->size()); + return *static_cast(gb->get_output(sampled)); +} + +} // namespace engine +} // namespace cactus diff --git a/cactus-transpile/models/model_nomic.cpp b/cactus-transpile/models/model_nomic.cpp new file mode 100644 index 000000000..a9477a865 --- /dev/null +++ b/cactus-transpile/models/model_nomic.cpp @@ -0,0 +1,201 @@ +#include "model.h" +#include "../graph/graph.h" +#include + + +namespace cactus { +namespace engine { + +NomicModel::NomicModel() : Model() {} + +NomicModel::NomicModel(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); +} + +void NomicModel::load_weights_to_graph(CactusGraph* gb) { + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.embedding_layernorm_weight = gb->mmap_weights(model_folder_path_ + "/embedding_layernorm.weight"); + weight_nodes_.embedding_layernorm_bias = gb->mmap_weights(model_folder_path_ + "/embedding_layernorm.bias"); + + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer = weight_nodes_.layers[i]; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); + layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); + layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); + layer.attn_q_bias = gb->mmap_weights(layer_prefix + "attn_q.bias"); + layer.attn_k_bias = gb->mmap_weights(layer_prefix + "attn_k.bias"); + layer.attn_v_bias = gb->mmap_weights(layer_prefix + "attn_v.bias"); + + layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); + layer.attn_output_bias = gb->mmap_weights(layer_prefix + "attn_output.bias"); + layer.ffn_norm_1_weight = gb->mmap_weights(layer_prefix + "norm1.weights"); + layer.ffn_norm_1_bias = gb->mmap_weights(layer_prefix + "norm1.bias"); + layer.ffn_norm_2_weight = gb->mmap_weights(layer_prefix + "norm2.weights"); + layer.ffn_norm_2_bias = gb->mmap_weights(layer_prefix + "norm2.bias"); + + if ((i + 1) % config_.moe_every_n_layers != 0) { + layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); + layer.ffn_up_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); + layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); + layer.ffn_down_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); + + } else { + layer.mlp_router_layer_weight = gb->mmap_weights(layer_prefix + "mlp_router.layer.weights"); + layer.mlp_experts_bias = gb->mmap_weights(layer_prefix + "mlp_experts.bias"); + + for (uint32_t j = 0; j < config_.num_experts; j++) { + layer.mlp_experts_mlp1_weight.push_back(gb->mmap_weights(layer_prefix + "mlp_expert_" + std::to_string(j) + ".mlp1.weights")); + layer.mlp_experts_mlp2_weight.push_back(gb->mmap_weights(layer_prefix + "mlp_expert_" + std::to_string(j) + ".mlp2.weights")); + } + } + } +} + +size_t NomicModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + if (use_cache) { + throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); + } + (void)position_offset; + + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); + q_proj = gb->add(q_proj, layer.attn_q_bias); + + auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); + k_proj = gb->add(k_proj, layer.attn_k_bias); + + auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); + v_proj = gb->add(v_proj, layer.attn_v_bias); + + const auto& q_shape = gb->get_output_buffer(q_proj).shape; + const size_t seq_len = q_shape[0]; + const size_t num_heads = config_.attention_heads; + const size_t head_dim = config_.attention_head_dim; + + if (num_heads == 0 || head_dim == 0) { + throw std::runtime_error("Invalid attention head configuration for Nomic model"); + } + + auto reshape_to_heads = [&](size_t tensor) { + return gb->reshape(tensor, {1, seq_len, num_heads, head_dim}); + }; + + auto q_proj_4d = reshape_to_heads(q_proj); + auto k_proj_4d = reshape_to_heads(k_proj); + auto v_proj_4d = reshape_to_heads(v_proj); + + if (config_.rope_theta > 0) { + q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, 0); + k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, 0); + } + + auto attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, false); + auto attn_output = gb->reshape(attn_output_4d, {seq_len, num_heads * head_dim}); + + auto output = gb->matmul(attn_output, layer.attn_output_weight, true, backend); + output = gb->add(output, layer.attn_output_bias); + return output; +} + +size_t NomicModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const { + if ((layer_idx + 1) % config_.moe_every_n_layers != 0) { + return build_standard_mlp(gb, normalized_h, layer_idx, backend); + } else { + return build_moe_mlp(gb, normalized_h, layer_idx, backend); + } +} + +size_t NomicModel::build_standard_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + auto hidden = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); + hidden = gb->add(hidden, layer.ffn_up_bias); + hidden = gb->gelu(hidden); + hidden = gb->matmul(hidden, layer.ffn_down_weight, true, backend); + hidden = gb->add(hidden, layer.ffn_down_bias); + return hidden; +} + +size_t NomicModel::build_moe_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + const size_t num_experts = config_.num_experts != 0 + ? config_.num_experts + : gb->get_output_buffer(layer.mlp_router_layer_weight).shape[0]; + + auto gate_weights = gb->matmul(normalized_h, layer.mlp_router_layer_weight, true, backend); + auto gate_probs = gb->softmax(gate_weights); + auto topk_result = gb->topk(gate_probs, config_.num_top_experts); + auto topk_idx = gb->index(topk_result, 0, 0); + + std::vector w1_weights, w2_weights; + for (size_t e = 0; e < num_experts; ++e) { + w1_weights.push_back(layer.mlp_experts_mlp1_weight[e]); + w2_weights.push_back(layer.mlp_experts_mlp2_weight[e]); + } + + auto moe_out = gb->moe_layer( + normalized_h, gate_probs, topk_idx, + w1_weights, w2_weights, + num_experts, config_.num_top_experts, + false, + 1e-6f, + 1.0f, + Activation::GELU + ); + + return gb->add(moe_out, layer.mlp_experts_bias); +} + +size_t NomicModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + if (use_cache) { + throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); + } + (void)position_offset; + const auto& layer = weight_nodes_.layers[layer_idx]; + auto attn_output = build_attention(gb, hidden, layer_idx, backend); + auto residual = gb->add(hidden, attn_output); + auto normalized_residual = gb->layernorm(residual, layer.ffn_norm_1_weight, layer.ffn_norm_1_bias, config_.layer_norm_eps); + auto mlp_output = build_mlp(gb, normalized_residual, layer_idx, backend); + auto final_residual = gb->add(normalized_residual, mlp_output); + auto normalized_final_residual = gb->layernorm(final_residual, layer.ffn_norm_2_weight, layer.ffn_norm_2_bias, config_.layer_norm_eps); + return normalized_final_residual; +} + +size_t NomicModel::forward(const std::vector& tokens, bool use_cache) { + if (use_cache) { + throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); + } + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + size_t seq_len = static_cast(tokens.size()); + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + size_t input_node_id = gb->input({seq_len}, Precision::FP32); + std::vector input_data(seq_len); + for (size_t i = 0; i < seq_len; i++) { + input_data[i] = static_cast(tokens[i]); + } + gb->set_input(input_node_id, input_data.data(), Precision::FP32); + + size_t hidden = gb->embedding(embedding_node_id_, input_node_id); + + hidden = gb->layernorm(hidden, weight_nodes_.embedding_layernorm_weight, weight_nodes_.embedding_layernorm_bias, config_.layer_norm_eps); + + for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { + hidden = build_transformer_block(gb, hidden, layer_idx, backend); + } + + return hidden; +} + +} +} diff --git a/cactus-transpile/models/model_parakeet.cpp b/cactus-transpile/models/model_parakeet.cpp new file mode 100644 index 000000000..7aee878b3 --- /dev/null +++ b/cactus-transpile/models/model_parakeet.cpp @@ -0,0 +1,860 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../npu/npu.h" +#include "../kernel/kernel.h" +#include +#include +#include +#include +#include +#include +#include + +size_t shape_elements(const std::vector& shape) { + if (shape.empty()) return 0; + size_t total = 1; + for (int d : shape) { + if (d <= 0) return 0; + total *= static_cast(d); + } + return total; +} + +bool pack_parakeet_features_for_npu( + const std::vector<__fp16>& time_major_f16, + size_t frames, + size_t num_mels, + const std::vector& input_shape, + std::vector<__fp16>& packed) +{ + if (input_shape.empty()) return false; + const size_t total = shape_elements(input_shape); + if (total == 0) return false; + packed.assign(total, static_cast<__fp16>(0.0f)); + + auto tm = [&](size_t t, size_t m) -> __fp16 { + return time_major_f16[t * num_mels + m]; + }; + + if (input_shape.size() == 4) { + const size_t s0 = static_cast(input_shape[0]); + const size_t s1 = static_cast(input_shape[1]); + const size_t s2 = static_cast(input_shape[2]); + const size_t s3 = static_cast(input_shape[3]); + if (s0 != 1) return false; + + if (s1 == 1 && s2 >= frames && s3 == num_mels) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[(t * s3) + m] = tm(t, m); + } + } + return true; + } + if (s1 >= frames && s2 == num_mels && s3 == 1) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[((t * s2 + m) * s3)] = tm(t, m); + } + } + return true; + } + if (s1 == num_mels && s2 >= frames && s3 == 1) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[((m * s2 + t) * s3)] = tm(t, m); + } + } + return true; + } + if (s1 == 1 && s2 == num_mels && s3 >= frames) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[(m * s3) + t] = tm(t, m); + } + } + return true; + } + return false; + } + + if (input_shape.size() == 3) { + const size_t s0 = static_cast(input_shape[0]); + const size_t s1 = static_cast(input_shape[1]); + const size_t s2 = static_cast(input_shape[2]); + + if (s0 == 1 && s1 >= frames && s2 == num_mels) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[t * s2 + m] = tm(t, m); + } + } + return true; + } + if (s0 == 1 && s1 == num_mels && s2 >= frames) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[m * s2 + t] = tm(t, m); + } + } + return true; + } + if (s0 >= frames && s1 == num_mels && s2 == 1) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[(t * s1 + m) * s2] = tm(t, m); + } + } + return true; + } + if (s0 == num_mels && s1 >= frames && s2 == 1) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[(m * s1 + t) * s2] = tm(t, m); + } + } + return true; + } + return false; + } + + if (input_shape.size() == 2) { + const size_t s0 = static_cast(input_shape[0]); + const size_t s1 = static_cast(input_shape[1]); + + if (s0 >= frames && s1 == num_mels) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[t * s1 + m] = tm(t, m); + } + } + return true; + } + if (s0 == num_mels && s1 >= frames) { + for (size_t t = 0; t < frames; ++t) { + for (size_t m = 0; m < num_mels; ++m) { + packed[m * s1 + t] = tm(t, m); + } + } + return true; + } + return false; + } + + return false; +} + +bool infer_npu_encoder_output_shape( + const std::vector& output_shape, + size_t elements_written, + size_t fallback_hidden_dim, + size_t& time_steps, + size_t& hidden_dim) +{ + if (elements_written == 0) return false; + + std::vector dims; + dims.reserve(output_shape.size()); + for (int d : output_shape) { + if (d > 0) dims.push_back(static_cast(d)); + } + + if (dims.size() >= 2) { + time_steps = dims[dims.size() - 2]; + hidden_dim = dims[dims.size() - 1]; + } else if (dims.size() == 1) { + hidden_dim = dims[0]; + if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; + time_steps = elements_written / hidden_dim; + } else { + hidden_dim = fallback_hidden_dim; + if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; + time_steps = elements_written / hidden_dim; + } + + if (time_steps == 0 || hidden_dim == 0) return false; + if (time_steps * hidden_dim > elements_written) return false; + return true; +} + +namespace cactus { +namespace engine { + +ParakeetModel::ParakeetModel() : Model() {} + +ParakeetModel::ParakeetModel(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); + + float hd = static_cast(config.attention_head_dim); + if (hd <= 0.0f) { + hd = 64.0f; + } + attention_scale_ = 1.0f / std::sqrt(hd); + + // Legacy converted Parakeet configs inherited generic rope_theta defaults. + // Keep RoPE disabled unless explicitly set to a non-default value. + if (std::fabs(config_.rope_theta - 10000.0f) < 1e-3f || + std::fabs(config_.rope_theta - 1000000.0f) < 1e-3f) { + config_.rope_theta = 0.0f; + } +} + +void ParakeetModel::load_weights_to_graph(CactusGraph* gb) { + weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); + weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); + weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); + weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); + weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); + weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); + weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); + weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); + weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); + weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); + weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); + weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); + + weight_nodes_.ctc_head_weight = gb->mmap_weights(model_folder_path_ + "/ctc_head_weight.weights"); + weight_nodes_.ctc_head_bias = gb->mmap_weights(model_folder_path_ + "/ctc_head_bias.bias"); + + if (npu::is_npu_available()) { + std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; + npu_encoder_ = npu::create_encoder(); + if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { + use_npu_encoder_ = true; + std::vector input_shape = npu_encoder_->get_input_shape(); + if (!input_shape.empty()) { + npu_encoder_->preallocate(input_shape, "x", ""); + } + } else { + use_npu_encoder_ = false; + npu_encoder_.reset(); + } + } + + const std::filesystem::path model_path(model_folder_path_); + has_cpu_encoder_weights_ = + std::filesystem::exists(model_path / "subsampling_conv0_weight.weights") && + std::filesystem::exists(model_path / "subsampling_linear_weight.weights"); + + if (has_cpu_encoder_weights_) { + weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); + weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); + weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); + weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); + weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); + weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); + weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); + weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); + weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); + weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); + weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); + weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); + } + + for (uint32_t i = 0; i < config_.num_layers; ++i) { + auto& layer = weight_nodes_.layers[i]; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + + layer.ff1_linear1_weight = gb->mmap_weights(layer_prefix + "ff1_linear1.weights"); + layer.ff1_linear1_bias = gb->mmap_weights(layer_prefix + "ff1_linear1.bias"); + layer.ff1_linear2_weight = gb->mmap_weights(layer_prefix + "ff1_linear2.weights"); + layer.ff1_linear2_bias = gb->mmap_weights(layer_prefix + "ff1_linear2.bias"); + + layer.ff2_linear1_weight = gb->mmap_weights(layer_prefix + "ff2_linear1.weights"); + layer.ff2_linear1_bias = gb->mmap_weights(layer_prefix + "ff2_linear1.bias"); + layer.ff2_linear2_weight = gb->mmap_weights(layer_prefix + "ff2_linear2.weights"); + layer.ff2_linear2_bias = gb->mmap_weights(layer_prefix + "ff2_linear2.bias"); + + layer.self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); + layer.self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); + layer.self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); + layer.self_attn_k_bias = gb->mmap_weights(layer_prefix + "self_attn_k.bias"); + layer.self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); + layer.self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); + layer.self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); + layer.self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); + layer.self_attn_relative_k_weight = gb->mmap_weights(layer_prefix + "self_attn_relative_k.weights"); + layer.self_attn_bias_u = gb->mmap_weights(layer_prefix + "self_attn_bias_u.weights"); + layer.self_attn_bias_v = gb->mmap_weights(layer_prefix + "self_attn_bias_v.weights"); + + layer.norm_ff1_weight = gb->mmap_weights(layer_prefix + "norm_ff1.weights"); + layer.norm_ff1_bias = gb->mmap_weights(layer_prefix + "norm_ff1.bias"); + layer.norm_self_attn_weight = gb->mmap_weights(layer_prefix + "norm_self_attn.weights"); + layer.norm_self_attn_bias = gb->mmap_weights(layer_prefix + "norm_self_attn.bias"); + layer.norm_conv_weight = gb->mmap_weights(layer_prefix + "norm_conv.weights"); + layer.norm_conv_bias = gb->mmap_weights(layer_prefix + "norm_conv.bias"); + layer.norm_ff2_weight = gb->mmap_weights(layer_prefix + "norm_ff2.weights"); + layer.norm_ff2_bias = gb->mmap_weights(layer_prefix + "norm_ff2.bias"); + layer.norm_out_weight = gb->mmap_weights(layer_prefix + "norm_out.weights"); + layer.norm_out_bias = gb->mmap_weights(layer_prefix + "norm_out.bias"); + + layer.conv_pointwise1_weight = gb->mmap_weights(layer_prefix + "conv_pointwise1.weights"); + layer.conv_pointwise1_bias = gb->mmap_weights(layer_prefix + "conv_pointwise1.bias"); + layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); + layer.conv_depthwise_bias = gb->mmap_weights(layer_prefix + "conv_depthwise.bias"); + layer.conv_pointwise2_weight = gb->mmap_weights(layer_prefix + "conv_pointwise2.weights"); + layer.conv_pointwise2_bias = gb->mmap_weights(layer_prefix + "conv_pointwise2.bias"); + layer.conv_batchnorm_weight = gb->mmap_weights(layer_prefix + "conv_batchnorm_weight.weights"); + layer.conv_batchnorm_bias = gb->mmap_weights(layer_prefix + "conv_batchnorm_bias.bias"); + layer.conv_batchnorm_running_mean = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_mean.weights"); + layer.conv_batchnorm_running_var = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_var.weights"); + } +} + +size_t ParakeetModel::build_subsampling(CactusGraph* gb, const std::vector& audio_features) { + const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); + if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { + throw std::runtime_error("Parakeet expects audio_features with shape [num_mels, num_frames]"); + } + + const size_t frames = audio_features.size() / num_mels; + std::vector time_major(frames * num_mels); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &audio_features[m * frames]; + for (size_t t = 0; t < frames; ++t) { + time_major[t * num_mels + m] = src[t]; + } + } + + std::vector<__fp16> features_f16(time_major.size()); + cactus_fp32_to_fp16(time_major.data(), features_f16.data(), time_major.size()); + + size_t x = gb->input({1, 1, frames, num_mels}, Precision::FP16); + gb->set_input(x, features_f16.data(), Precision::FP16); + + x = gb->conv2d_k3s2p1(x, weight_nodes_.subsampling_conv0_weight, weight_nodes_.subsampling_conv0_bias); + x = gb->relu(x); + + x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise1_weight, weight_nodes_.subsampling_depthwise1_bias); + x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise1_weight, weight_nodes_.subsampling_pointwise1_bias); + x = gb->relu(x); + + x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise2_weight, weight_nodes_.subsampling_depthwise2_bias); + x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise2_weight, weight_nodes_.subsampling_pointwise2_bias); + x = gb->relu(x); + + const auto& conv_shape = gb->get_output_buffer(x).shape; + if (conv_shape.size() != 4 || conv_shape[0] != 1) { + throw std::runtime_error("Parakeet subsampling produced invalid shape"); + } + + const size_t C = conv_shape[1]; + const size_t T = conv_shape[2]; + const size_t W = conv_shape[3]; + + size_t t_major = gb->transposeN(x, {0, 2, 1, 3}, ComputeBackend::CPU); + size_t flattened = gb->reshape(t_major, {T, C * W}); + size_t projected = gb->matmul(flattened, weight_nodes_.subsampling_linear_weight, true, ComputeBackend::CPU); + projected = gb->add(projected, weight_nodes_.subsampling_linear_bias); + return projected; +} + +size_t ParakeetModel::build_relative_position_embeddings(CactusGraph* gb, size_t seq_len) { + const size_t hidden_dim = std::max(1, static_cast(config_.hidden_dim)); + const size_t half_dim = hidden_dim / 2; + const size_t rel_len = 2 * seq_len - 1; + + std::vector pos_embed(rel_len * hidden_dim, 0.0f); + for (size_t p = 0; p < rel_len; ++p) { + const int rel_pos = static_cast(seq_len - 1) - static_cast(p); + for (size_t i = 0; i < half_dim; ++i) { + const float exponent = static_cast(2 * i) / static_cast(hidden_dim); + const float inv_freq = 1.0f / std::pow(10000.0f, exponent); + const float angle = static_cast(rel_pos) * inv_freq; + pos_embed[p * hidden_dim + 2 * i] = std::sin(angle); + if (2 * i + 1 < hidden_dim) { + pos_embed[p * hidden_dim + 2 * i + 1] = std::cos(angle); + } + } + } + + std::vector<__fp16> pos_embed_f16(pos_embed.size()); + cactus_fp32_to_fp16(pos_embed.data(), pos_embed_f16.data(), pos_embed.size()); + + size_t pos_node = gb->input({rel_len, hidden_dim}, Precision::FP16); + gb->set_input(pos_node, pos_embed_f16.data(), Precision::FP16); + return pos_node; +} + +size_t ParakeetModel::build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, + uint32_t layer_idx, ComputeBackend backend) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t q = gb->matmul(hidden, layer.self_attn_q_weight, true, backend); + q = gb->add(q, layer.self_attn_q_bias); + size_t k = gb->matmul(hidden, layer.self_attn_k_weight, true, backend); + k = gb->add(k, layer.self_attn_k_bias); + size_t v = gb->matmul(hidden, layer.self_attn_v_weight, true, backend); + v = gb->add(v, layer.self_attn_v_bias); + + const auto& q_shape = gb->get_output_buffer(q).shape; + if (q_shape.size() != 2) { + throw std::runtime_error("Parakeet self-attention expects [T, D]"); + } + + const size_t T = q_shape[0]; + const size_t q_heads = std::max(1, static_cast(config_.attention_heads)); + const size_t kv_heads = std::max(1, static_cast(config_.attention_kv_heads)); + const size_t head_dim = std::max(1, static_cast(config_.attention_head_dim)); + + size_t q4 = gb->reshape(q, {1, T, q_heads, head_dim}); + size_t k4 = gb->reshape(k, {1, T, kv_heads, head_dim}); + size_t v4 = gb->reshape(v, {1, T, kv_heads, head_dim}); + + size_t bias_u = gb->reshape(layer.self_attn_bias_u, {1, static_cast(1), q_heads, head_dim}); + size_t bias_v = gb->reshape(layer.self_attn_bias_v, {1, static_cast(1), q_heads, head_dim}); + size_t q_u4 = gb->add(q4, bias_u); + size_t q_v4 = gb->add(q4, bias_v); + + size_t rel_k_flat = gb->matmul(position_embeddings, layer.self_attn_relative_k_weight, true, backend); + size_t rel_k4 = gb->reshape(rel_k_flat, {1, 2 * T - 1, q_heads, head_dim}); + size_t rel_bias = gb->rel_pos_bias(q_v4, rel_k4, attention_scale_); + + size_t attn = gb->attention_masked(q_u4, k4, v4, rel_bias, attention_scale_, false, backend, true); + attn = gb->reshape(attn, {T, q_heads * head_dim}); + + size_t out = gb->matmul(attn, layer.self_attn_output_weight, true, backend); + out = gb->add(out, layer.self_attn_output_bias); + return out; +} + +size_t ParakeetModel::build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t w1 = second_ff ? layer.ff2_linear1_weight : layer.ff1_linear1_weight; + size_t b1 = second_ff ? layer.ff2_linear1_bias : layer.ff1_linear1_bias; + size_t w2 = second_ff ? layer.ff2_linear2_weight : layer.ff1_linear2_weight; + size_t b2 = second_ff ? layer.ff2_linear2_bias : layer.ff1_linear2_bias; + + size_t x = gb->matmul(hidden, w1, true, backend); + x = gb->add(x, b1); + + std::string act = config_.encoder_hidden_act; + std::transform(act.begin(), act.end(), act.begin(), ::tolower); + if (act.find("gelu") != std::string::npos) { + x = gb->gelu(x); + } else if (act == "relu") { + x = gb->relu(x); + } else { + x = gb->silu(x); + } + + x = gb->matmul(x, w2, true, backend); + x = gb->add(x, b2); + return x; +} + +size_t ParakeetModel::build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend) { + (void)backend; + const auto& layer = weight_nodes_.layers[layer_idx]; + const auto& hidden_shape = gb->get_output_buffer(hidden).shape; + if (hidden_shape.size() != 2) { + throw std::runtime_error("Parakeet convolution module expects [T, D]"); + } + + const size_t T = hidden_shape[0]; + const size_t D = hidden_shape[1]; + + size_t x = gb->reshape(hidden, {1, T, D}); + x = gb->conv1d_pointwise(x, layer.conv_pointwise1_weight, layer.conv_pointwise1_bias); + x = gb->glu(x, -1); + + x = gb->conv1d_same_depthwise_k9(x, layer.conv_depthwise_weight, layer.conv_depthwise_bias); + + x = gb->batchnorm( + x, + layer.conv_batchnorm_weight, + layer.conv_batchnorm_bias, + layer.conv_batchnorm_running_mean, + layer.conv_batchnorm_running_var, + 2, + 1e-5f + ); + + std::string act = config_.encoder_hidden_act; + std::transform(act.begin(), act.end(), act.begin(), ::tolower); + if (act.find("gelu") != std::string::npos) { + x = gb->gelu(x); + } else if (act == "relu") { + x = gb->relu(x); + } else { + x = gb->silu(x); + } + + x = gb->conv1d_pointwise(x, layer.conv_pointwise2_weight, layer.conv_pointwise2_bias); + + x = gb->reshape(x, {T, D}); + return x; +} + +size_t ParakeetModel::build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, + uint32_t layer_idx, ComputeBackend backend) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t ff1_in = gb->layernorm(hidden, layer.norm_ff1_weight, layer.norm_ff1_bias); + size_t ff1 = build_feed_forward(gb, ff1_in, layer_idx, false, backend); + ff1 = gb->scalar_multiply(ff1, 0.5f); + size_t x = gb->add(hidden, ff1); + + size_t attn_in = gb->layernorm(x, layer.norm_self_attn_weight, layer.norm_self_attn_bias); + size_t attn = build_self_attention(gb, attn_in, position_embeddings, layer_idx, backend); + x = gb->add(x, attn); + + size_t conv_in = gb->layernorm(x, layer.norm_conv_weight, layer.norm_conv_bias); + size_t conv = build_convolution_module(gb, conv_in, layer_idx, backend); + x = gb->add(x, conv); + + size_t ff2_in = gb->layernorm(x, layer.norm_ff2_weight, layer.norm_ff2_bias); + size_t ff2 = build_feed_forward(gb, ff2_in, layer_idx, true, backend); + ff2 = gb->scalar_multiply(ff2, 0.5f); + x = gb->add(x, ff2); + + x = gb->layernorm(x, layer.norm_out_weight, layer.norm_out_bias); + return x; +} + +size_t ParakeetModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) { + const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); + if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { + throw std::runtime_error("Parakeet expects audio_features with shape [num_mels, num_frames]"); + } + const size_t frames = audio_features.size() / num_mels; + size_t expected_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); + size_t expected_vocab_size = std::max(1, static_cast(config_.vocab_size)); + { + const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; + if (ctc_w_shape.size() == 2) { + expected_vocab_size = ctc_w_shape[0]; + expected_hidden_dim = ctc_w_shape[1]; + } else if (ctc_w_shape.size() == 3) { + expected_vocab_size = ctc_w_shape[0]; + expected_hidden_dim = ctc_w_shape[1]; + } + } + + if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { + std::vector time_major(frames * num_mels); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &audio_features[m * frames]; + for (size_t t = 0; t < frames; ++t) { + time_major[t * num_mels + m] = src[t]; + } + } + + std::vector<__fp16> time_major_f16(time_major.size()); + cactus_fp32_to_fp16(time_major.data(), time_major_f16.data(), time_major.size()); + + std::vector input_shape = npu_encoder_->get_input_shape(); + std::vector<__fp16> npu_input; + if (pack_parakeet_features_for_npu(time_major_f16, frames, num_mels, input_shape, npu_input)) { + npu_encoder_->preallocate(input_shape, "x", ""); + + std::vector output_shape = npu_encoder_->get_output_shape(); + size_t output_capacity = npu_encoder_->get_output_buffer_size(); + if (output_capacity == 0) { + output_capacity = shape_elements(output_shape); + } + if (output_capacity == 0) { + output_capacity = std::max( + 1, frames * std::max(1, static_cast(config_.hidden_dim))); + } + + std::vector<__fp16> npu_output(output_capacity); + const size_t elements_written = npu_encoder_->encode( + npu_input.data(), + npu_output.data(), + input_shape, + "x", + "" + ); + + size_t T_enc = 0; + size_t D_enc = 0; + if (elements_written > 0 && + infer_npu_encoder_output_shape(output_shape, elements_written, + static_cast(config_.hidden_dim), T_enc, D_enc)) { + const __fp16* src = npu_output.data(); + __fp16* cached_output = npu_encoder_->get_output_buffer(); + const size_t cached_count = npu_encoder_->get_output_buffer_size(); + const size_t required = T_enc * D_enc; + if (cached_output != nullptr && cached_count >= required) { + src = cached_output; + } + + // Accept encoder hidden output [T, hidden] or direct logits [T, vocab]. + if (D_enc == expected_hidden_dim || D_enc == expected_vocab_size) { + size_t enc_output = gb->input({T_enc, D_enc}, Precision::FP16); + gb->set_input(enc_output, src, Precision::FP16); + return enc_output; + } + } + } + } + + if (!has_cpu_encoder_weights_) { + throw std::runtime_error( + "Parakeet requires either CPU encoder weights or model.mlpackage encoder output."); + } + + ComputeBackend backend = ComputeBackend::CPU; + size_t hidden = build_subsampling(gb, audio_features); + const auto& hidden_shape = gb->get_output_buffer(hidden).shape; + if (hidden_shape.size() != 2) { + throw std::runtime_error("Parakeet encoder expects subsampling output [T, D]"); + } + size_t position_embeddings = build_relative_position_embeddings(gb, hidden_shape[0]); + for (uint32_t i = 0; i < config_.num_layers; ++i) { + hidden = build_encoder_block(gb, hidden, position_embeddings, i, backend); + } + return hidden; +} + +size_t ParakeetModel::build_ctc_logits(CactusGraph* gb, size_t hidden_states) { + const auto& hidden_shape = gb->get_output_buffer(hidden_states).shape; + if (hidden_shape.size() != 2) { + throw std::runtime_error("Parakeet CTC head expects hidden states [T, D]"); + } + const size_t T = hidden_shape[0]; + const size_t D = hidden_shape[1]; + + size_t hidden_nlc = gb->reshape(hidden_states, {1, T, D}); + const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; + size_t vocab_size = ctc_w_shape.empty() ? config_.vocab_size : ctc_w_shape[0]; + size_t logits_nlc = gb->conv1d_pointwise(hidden_nlc, weight_nodes_.ctc_head_weight, weight_nodes_.ctc_head_bias); + + return gb->reshape(logits_nlc, {T, vocab_size}); +} + +size_t ParakeetModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) { + (void)tokens; + (void)use_cache; + auto* gb = static_cast(graph_handle_); + gb->clear_debug_nodes(); + size_t encoder_out = build_encoder(gb, audio_features); + + const auto& enc_shape = gb->get_output_buffer(encoder_out).shape; + if (enc_shape.size() != 2) { + throw std::runtime_error("Parakeet encoder output must be rank-2 [T, D]"); + } + + size_t ctc_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); + size_t ctc_vocab_size = std::max(1, static_cast(config_.vocab_size)); + const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; + if (ctc_w_shape.size() == 2) { + ctc_vocab_size = ctc_w_shape[0]; + ctc_hidden_dim = ctc_w_shape[1]; + } else if (ctc_w_shape.size() == 3) { + ctc_vocab_size = ctc_w_shape[0]; + ctc_hidden_dim = ctc_w_shape[1]; + } + + // If the NPU graph already emits [T, vocab] logits, skip CPU CTC head. + if (enc_shape[1] == ctc_vocab_size && ctc_vocab_size != ctc_hidden_dim) { + return encoder_out; + } + + if (enc_shape[1] != ctc_hidden_dim) { + throw std::runtime_error( + "Parakeet encoder output dim mismatch: expected hidden dim " + + std::to_string(ctc_hidden_dim) + ", got " + std::to_string(enc_shape[1])); + } + + return build_ctc_logits(gb, encoder_out); +} + +std::vector ParakeetModel::greedy_decode_tokens(CactusGraph* gb, size_t logits_node) const { + const auto& logits_buf = gb->get_output_buffer(logits_node); + if (logits_buf.shape.size() != 2) { + throw std::runtime_error("Parakeet logits must be rank-2 [T, vocab]"); + } + + const size_t T = logits_buf.shape[0]; + const size_t vocab_size = logits_buf.shape[1]; + std::vector frame_ids(T, 0); + + if (logits_buf.precision == Precision::FP32) { + const float* src = logits_buf.data_as(); + for (size_t t = 0; t < T; ++t) { + const float* row = src + t * vocab_size; + size_t best_idx = 0; + float best_val = row[0]; + for (size_t v = 1; v < vocab_size; ++v) { + if (row[v] > best_val) { + best_val = row[v]; + best_idx = v; + } + } + frame_ids[t] = static_cast(best_idx); + } + } else if (logits_buf.precision == Precision::FP16) { + const __fp16* src = logits_buf.data_as<__fp16>(); + for (size_t t = 0; t < T; ++t) { + const __fp16* row = src + t * vocab_size; + size_t best_idx = 0; + float best_val = static_cast(row[0]); + for (size_t v = 1; v < vocab_size; ++v) { + const float val = static_cast(row[v]); + if (val > best_val) { + best_val = val; + best_idx = v; + } + } + frame_ids[t] = static_cast(best_idx); + } + } else { + const int8_t* src = logits_buf.data_as(); + for (size_t t = 0; t < T; ++t) { + const int8_t* row = src + t * vocab_size; + size_t best_idx = 0; + int best_val = static_cast(row[0]); + for (size_t v = 1; v < vocab_size; ++v) { + const int val = static_cast(row[v]); + if (val > best_val) { + best_val = val; + best_idx = v; + } + } + frame_ids[t] = static_cast(best_idx); + } + } + + const uint32_t blank_id = config_.pad_token_id > 0 ? config_.pad_token_id : static_cast(vocab_size - 1); + std::vector decoded; + decoded.reserve(frame_ids.size()); + + uint32_t prev = blank_id; + for (uint32_t id : frame_ids) { + if (id == blank_id) { + prev = blank_id; + continue; + } + if (id == prev) { + continue; + } + decoded.push_back(id); + prev = id; + } + + return decoded; +} + +uint32_t ParakeetModel::decode_with_audio( + const std::vector& tokens, + const std::vector& audio_features, + float temperature, + float top_p, + size_t top_k, + const std::string& profile_file, + float* out_entropy, + float min_p, + float repetition_penalty, + float* /*out_token_time_start*/, + float* /*out_token_time_end*/) +{ + (void)temperature; + (void)top_p; + (void)top_k; + (void)min_p; + (void)repetition_penalty; + + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + if (audio_features.empty()) { + throw std::runtime_error("Audio features cannot be empty in Parakeet decode_with_audio"); + } + + auto* gb = static_cast(graph_handle_); + const bool new_request = !ctc_tokens_ready_ || tokens.empty() || tokens.size() < last_input_token_count_; + if (new_request) { + gb->soft_reset(); + size_t logits_node = forward(audio_features, tokens, false); + + if (!profile_file.empty()) { + gb->execute(profile_file); + } else { + gb->execute(); + } + + ctc_tokens_ = greedy_decode_tokens(gb, logits_node); + ctc_emit_index_ = 0; + ctc_tokens_ready_ = true; + } + + last_input_token_count_ = tokens.size(); + if (out_entropy) { + *out_entropy = 0.0f; + } + + if (ctc_emit_index_ < ctc_tokens_.size()) { + return ctc_tokens_[ctc_emit_index_++]; + } + return get_tokenizer()->get_eos_token(); +} + +std::vector ParakeetModel::get_audio_embeddings(const std::vector& audio_features) { + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + size_t hidden = build_encoder(gb, audio_features); + + const auto& hidden_shape = gb->get_output_buffer(hidden).shape; + size_t ctc_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); + size_t ctc_vocab_size = std::max(1, static_cast(config_.vocab_size)); + const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; + if (ctc_w_shape.size() == 2) { + ctc_vocab_size = ctc_w_shape[0]; + ctc_hidden_dim = ctc_w_shape[1]; + } else if (ctc_w_shape.size() == 3) { + ctc_vocab_size = ctc_w_shape[0]; + ctc_hidden_dim = ctc_w_shape[1]; + } + + // Embeddings require hidden states; if NPU returned logits, rerun CPU encoder. + if (hidden_shape.size() == 2 && + hidden_shape[1] == ctc_vocab_size && + ctc_vocab_size != ctc_hidden_dim && + use_npu_encoder_) { + if (!has_cpu_encoder_weights_) { + throw std::runtime_error( + "Parakeet audio embeddings require hidden-state encoder output; " + "CPU encoder fallback weights are not available."); + } + const bool prev_use_npu = use_npu_encoder_; + use_npu_encoder_ = false; + gb->soft_reset(); + hidden = build_encoder(gb, audio_features); + use_npu_encoder_ = prev_use_npu; + } + + size_t pooled = gb->mean(hidden, 0); + gb->execute(); + + const auto& output_buf = gb->get_output_buffer(pooled); + const size_t hidden_dim = output_buf.total_size; + std::vector embedding(hidden_dim); + + if (output_buf.precision == Precision::FP32) { + const float* src = output_buf.data_as(); + std::copy(src, src + hidden_dim, embedding.begin()); + } else if (output_buf.precision == Precision::FP16) { + const __fp16* src = output_buf.data_as<__fp16>(); + Quantization::fp16_to_fp32(src, embedding.data(), hidden_dim); + } else { + const int8_t* src = output_buf.data_as(); + Quantization::int8_to_fp32(src, embedding.data(), hidden_dim, 1.0f); + } + + reset_cache(); + return embedding; +} + +void ParakeetModel::reset_cache() { + Model::reset_cache(); + ctc_tokens_ready_ = false; + ctc_emit_index_ = 0; + ctc_tokens_.clear(); + last_input_token_count_ = 0; +} + +} // namespace engine +} // namespace cactus \ No newline at end of file diff --git a/cactus-transpile/models/model_parakeet_tdt.cpp b/cactus-transpile/models/model_parakeet_tdt.cpp new file mode 100644 index 000000000..da0e827d4 --- /dev/null +++ b/cactus-transpile/models/model_parakeet_tdt.cpp @@ -0,0 +1,1107 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../npu/npu.h" +#include "../kernel/kernel.h" +#include "../telemetry/telemetry.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +size_t shape_elements(const std::vector& shape); +bool pack_parakeet_features_for_npu( + const std::vector<__fp16>& time_major_f16, + size_t frames, + size_t num_mels, + const std::vector& input_shape, + std::vector<__fp16>& packed); + +namespace { + +constexpr uint32_t kMaxStreamDurationSkipFrames = 2; + +bool infer_npu_encoder_output_shape( + const std::vector& output_shape, + size_t elements_written, + size_t fallback_hidden_dim, + size_t& time_steps, + size_t& hidden_dim) +{ + if (elements_written == 0) return false; + + std::vector dims; + dims.reserve(output_shape.size()); + for (int d : output_shape) { + if (d > 0) dims.push_back(static_cast(d)); + } + + if (dims.size() >= 2) { + time_steps = dims[dims.size() - 2]; + hidden_dim = dims[dims.size() - 1]; + } else if (dims.size() == 1) { + hidden_dim = dims[0]; + if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; + time_steps = elements_written / hidden_dim; + } else { + hidden_dim = fallback_hidden_dim; + if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; + time_steps = elements_written / hidden_dim; + } + + if (time_steps == 0 || hidden_dim == 0) return false; + if (time_steps * hidden_dim > elements_written) return false; + return true; +} + +std::vector<__fp16> copy_buffer_to_fp16(const BufferDesc& buffer) { + std::vector<__fp16> out(buffer.total_size, static_cast<__fp16>(0.0f)); + if (buffer.total_size == 0) { + return out; + } + + if (buffer.precision == Precision::FP16) { + const __fp16* src = buffer.data_as<__fp16>(); + std::copy(src, src + buffer.total_size, out.begin()); + return out; + } + + if (buffer.precision == Precision::FP32) { + const float* src = buffer.data_as(); + cactus_fp32_to_fp16(src, out.data(), buffer.total_size); + return out; + } + + if (buffer.precision == Precision::INT8 && !buffer.is_interleaved) { + const int8_t* src = buffer.data_as(); + Quantization::int8_to_fp16(src, out.data(), buffer.total_size, 1.0f); + return out; + } + + throw std::runtime_error("Unsupported precision for FP16 conversion in ParakeetTDT"); +} + +size_t argmax_range(const BufferDesc& buffer, size_t offset, size_t length) { + if (length == 0 || (offset + length) > buffer.total_size) { + throw std::runtime_error("Invalid argmax range"); + } + + size_t best_idx = 0; + float best_val = -std::numeric_limits::infinity(); + + if (buffer.precision == Precision::FP16) { + const __fp16* src = buffer.data_as<__fp16>() + offset; + best_val = static_cast(src[0]); + for (size_t i = 1; i < length; ++i) { + float v = static_cast(src[i]); + if (v > best_val) { + best_val = v; + best_idx = i; + } + } + return best_idx; + } + + if (buffer.precision == Precision::FP32) { + const float* src = buffer.data_as() + offset; + best_val = src[0]; + for (size_t i = 1; i < length; ++i) { + float v = src[i]; + if (v > best_val) { + best_val = v; + best_idx = i; + } + } + return best_idx; + } + + if (buffer.precision == Precision::INT8) { + const int8_t* src = buffer.data_as() + offset; + best_val = static_cast(src[0]); + for (size_t i = 1; i < length; ++i) { + float v = static_cast(src[i]); + if (v > best_val) { + best_val = v; + best_idx = i; + } + } + return best_idx; + } + + throw std::runtime_error("Unsupported logits precision in argmax"); +} + +size_t argmax_range_with_bias(const BufferDesc& buffer, size_t offset, size_t length, + const std::unordered_map& bias) { + if (bias.empty()) { + return argmax_range(buffer, offset, length); + } + if (length == 0 || (offset + length) > buffer.total_size) { + throw std::runtime_error("Invalid argmax range"); + } + + size_t best_idx = 0; + float best_val = -std::numeric_limits::infinity(); + + if (buffer.precision == Precision::FP16) { + const __fp16* src = buffer.data_as<__fp16>() + offset; + for (size_t i = 0; i < length; ++i) { + float v = static_cast(src[i]); + auto it = bias.find(static_cast(i)); + if (it != bias.end()) v += it->second; + if (v > best_val) { best_val = v; best_idx = i; } + } + return best_idx; + } + + if (buffer.precision == Precision::FP32) { + const float* src = buffer.data_as() + offset; + for (size_t i = 0; i < length; ++i) { + float v = src[i]; + auto it = bias.find(static_cast(i)); + if (it != bias.end()) v += it->second; + if (v > best_val) { best_val = v; best_idx = i; } + } + return best_idx; + } + + if (buffer.precision == Precision::INT8) { + const int8_t* src = buffer.data_as() + offset; + for (size_t i = 0; i < length; ++i) { + float v = static_cast(src[i]); + auto it = bias.find(static_cast(i)); + if (it != bias.end()) v += it->second; + if (v > best_val) { best_val = v; best_idx = i; } + } + return best_idx; + } + + throw std::runtime_error("Unsupported logits precision in biased argmax"); +} + +} // namespace + +namespace cactus { +namespace engine { + +ParakeetTDTModel::ParakeetTDTModel() : Model() {} + +ParakeetTDTModel::ParakeetTDTModel(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); + + float hd = static_cast(config.attention_head_dim); + if (hd <= 0.0f) { + hd = 64.0f; + } + attention_scale_ = 1.0f / std::sqrt(hd); + + if (std::fabs(config_.rope_theta - 10000.0f) < 1e-3f || + std::fabs(config_.rope_theta - 1000000.0f) < 1e-3f) { + config_.rope_theta = 0.0f; + } +} + +void ParakeetTDTModel::load_weights_to_graph(CactusGraph* gb) { + weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); + weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); + weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); + weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); + weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); + weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); + weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); + weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); + weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); + weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); + weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); + weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); + + weight_nodes_.predictor_embed = gb->mmap_weights(model_folder_path_ + "/tdt_predictor_embed.weights"); + + size_t predictor_layers = static_cast(config_.predictor_num_layers); + if (predictor_layers == 0) { + while (true) { + const std::string prefix = model_folder_path_ + "/tdt_predictor_lstm_" + std::to_string(predictor_layers); + if (!std::filesystem::exists(prefix + "_weight_ih.weights") || + !std::filesystem::exists(prefix + "_weight_hh.weights") || + !std::filesystem::exists(prefix + "_bias.weights")) { + break; + } + ++predictor_layers; + } + } + if (predictor_layers == 0) { + predictor_layers = 1; + } + + weight_nodes_.predictor_layers.resize(predictor_layers); + for (size_t i = 0; i < predictor_layers; ++i) { + auto& layer = weight_nodes_.predictor_layers[i]; + const std::string prefix = model_folder_path_ + "/tdt_predictor_lstm_" + std::to_string(i); + layer.weight_ih = gb->mmap_weights(prefix + "_weight_ih.weights"); + layer.weight_hh = gb->mmap_weights(prefix + "_weight_hh.weights"); + layer.bias = gb->mmap_weights(prefix + "_bias.weights"); + } + + weight_nodes_.joint_enc_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_enc.weights"); + weight_nodes_.joint_enc_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_enc.bias"); + weight_nodes_.joint_pred_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_pred.weights"); + weight_nodes_.joint_pred_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_pred.bias"); + weight_nodes_.joint_out_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_out.weights"); + weight_nodes_.joint_out_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_out.bias"); + + if (npu::is_npu_available()) { + std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; + npu_encoder_ = npu::create_encoder(); + if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { + use_npu_encoder_ = true; + std::vector input_shape = npu_encoder_->get_input_shape(); + if (!input_shape.empty()) { + npu_encoder_->preallocate(input_shape, "x", ""); + } + } else { + use_npu_encoder_ = false; + npu_encoder_.reset(); + } + } + + const std::filesystem::path model_path(model_folder_path_); + has_cpu_encoder_weights_ = + std::filesystem::exists(model_path / "subsampling_conv0_weight.weights") && + std::filesystem::exists(model_path / "subsampling_linear_weight.weights"); + + if (has_cpu_encoder_weights_) { + weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); + weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); + weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); + weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); + weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); + weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); + weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); + weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); + weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); + weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); + weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); + weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); + } + + for (uint32_t i = 0; i < config_.num_layers; ++i) { + auto& layer = weight_nodes_.layers[i]; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + + layer.ff1_linear1_weight = gb->mmap_weights(layer_prefix + "ff1_linear1.weights"); + layer.ff1_linear1_bias = gb->mmap_weights(layer_prefix + "ff1_linear1.bias"); + layer.ff1_linear2_weight = gb->mmap_weights(layer_prefix + "ff1_linear2.weights"); + layer.ff1_linear2_bias = gb->mmap_weights(layer_prefix + "ff1_linear2.bias"); + + layer.ff2_linear1_weight = gb->mmap_weights(layer_prefix + "ff2_linear1.weights"); + layer.ff2_linear1_bias = gb->mmap_weights(layer_prefix + "ff2_linear1.bias"); + layer.ff2_linear2_weight = gb->mmap_weights(layer_prefix + "ff2_linear2.weights"); + layer.ff2_linear2_bias = gb->mmap_weights(layer_prefix + "ff2_linear2.bias"); + + layer.self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); + layer.self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); + layer.self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); + layer.self_attn_k_bias = gb->mmap_weights(layer_prefix + "self_attn_k.bias"); + layer.self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); + layer.self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); + layer.self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); + layer.self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); + layer.self_attn_relative_k_weight = gb->mmap_weights(layer_prefix + "self_attn_relative_k.weights"); + layer.self_attn_bias_u = gb->mmap_weights(layer_prefix + "self_attn_bias_u.weights"); + layer.self_attn_bias_v = gb->mmap_weights(layer_prefix + "self_attn_bias_v.weights"); + + layer.norm_ff1_weight = gb->mmap_weights(layer_prefix + "norm_ff1.weights"); + layer.norm_ff1_bias = gb->mmap_weights(layer_prefix + "norm_ff1.bias"); + layer.norm_self_attn_weight = gb->mmap_weights(layer_prefix + "norm_self_attn.weights"); + layer.norm_self_attn_bias = gb->mmap_weights(layer_prefix + "norm_self_attn.bias"); + layer.norm_conv_weight = gb->mmap_weights(layer_prefix + "norm_conv.weights"); + layer.norm_conv_bias = gb->mmap_weights(layer_prefix + "norm_conv.bias"); + layer.norm_ff2_weight = gb->mmap_weights(layer_prefix + "norm_ff2.weights"); + layer.norm_ff2_bias = gb->mmap_weights(layer_prefix + "norm_ff2.bias"); + layer.norm_out_weight = gb->mmap_weights(layer_prefix + "norm_out.weights"); + layer.norm_out_bias = gb->mmap_weights(layer_prefix + "norm_out.bias"); + + layer.conv_pointwise1_weight = gb->mmap_weights(layer_prefix + "conv_pointwise1.weights"); + layer.conv_pointwise1_bias = gb->mmap_weights(layer_prefix + "conv_pointwise1.bias"); + layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); + layer.conv_depthwise_bias = gb->mmap_weights(layer_prefix + "conv_depthwise.bias"); + layer.conv_pointwise2_weight = gb->mmap_weights(layer_prefix + "conv_pointwise2.weights"); + layer.conv_pointwise2_bias = gb->mmap_weights(layer_prefix + "conv_pointwise2.bias"); + layer.conv_batchnorm_weight = gb->mmap_weights(layer_prefix + "conv_batchnorm_weight.weights"); + layer.conv_batchnorm_bias = gb->mmap_weights(layer_prefix + "conv_batchnorm_bias.bias"); + layer.conv_batchnorm_running_mean = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_mean.weights"); + layer.conv_batchnorm_running_var = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_var.weights"); + } +} + +size_t ParakeetTDTModel::build_subsampling(CactusGraph* gb, const std::vector& audio_features) { + const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); + if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { + throw std::runtime_error("ParakeetTDT expects audio_features with shape [num_mels, num_frames]"); + } + + const size_t frames = audio_features.size() / num_mels; + std::vector time_major(frames * num_mels); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &audio_features[m * frames]; + for (size_t t = 0; t < frames; ++t) { + time_major[t * num_mels + m] = src[t]; + } + } + + std::vector<__fp16> features_f16(time_major.size()); + cactus_fp32_to_fp16(time_major.data(), features_f16.data(), time_major.size()); + + size_t x = gb->input({1, 1, frames, num_mels}, Precision::FP16); + gb->set_input(x, features_f16.data(), Precision::FP16); + + x = gb->conv2d_k3s2p1(x, weight_nodes_.subsampling_conv0_weight, weight_nodes_.subsampling_conv0_bias); + x = gb->relu(x); + + x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise1_weight, weight_nodes_.subsampling_depthwise1_bias); + x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise1_weight, weight_nodes_.subsampling_pointwise1_bias); + x = gb->relu(x); + + x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise2_weight, weight_nodes_.subsampling_depthwise2_bias); + x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise2_weight, weight_nodes_.subsampling_pointwise2_bias); + x = gb->relu(x); + + const auto& conv_shape = gb->get_output_buffer(x).shape; + if (conv_shape.size() != 4 || conv_shape[0] != 1) { + throw std::runtime_error("ParakeetTDT subsampling produced invalid shape"); + } + + const size_t C = conv_shape[1]; + const size_t T = conv_shape[2]; + const size_t W = conv_shape[3]; + + size_t t_major = gb->transposeN(x, {0, 2, 1, 3}, ComputeBackend::CPU); + size_t flattened = gb->reshape(t_major, {T, C * W}); + size_t projected = gb->matmul(flattened, weight_nodes_.subsampling_linear_weight, true, ComputeBackend::CPU); + projected = gb->add(projected, weight_nodes_.subsampling_linear_bias); + return projected; +} + +size_t ParakeetTDTModel::build_relative_position_embeddings(CactusGraph* gb, size_t seq_len) { + const size_t hidden_dim = std::max(1, static_cast(config_.hidden_dim)); + const size_t half_dim = hidden_dim / 2; + const size_t rel_len = 2 * seq_len - 1; + + std::vector pos_embed(rel_len * hidden_dim, 0.0f); + for (size_t p = 0; p < rel_len; ++p) { + const int rel_pos = static_cast(seq_len - 1) - static_cast(p); + for (size_t i = 0; i < half_dim; ++i) { + const float exponent = static_cast(2 * i) / static_cast(hidden_dim); + const float inv_freq = 1.0f / std::pow(10000.0f, exponent); + const float angle = static_cast(rel_pos) * inv_freq; + pos_embed[p * hidden_dim + 2 * i] = std::sin(angle); + if (2 * i + 1 < hidden_dim) { + pos_embed[p * hidden_dim + 2 * i + 1] = std::cos(angle); + } + } + } + + std::vector<__fp16> pos_embed_f16(pos_embed.size()); + cactus_fp32_to_fp16(pos_embed.data(), pos_embed_f16.data(), pos_embed.size()); + + size_t pos_node = gb->input({rel_len, hidden_dim}, Precision::FP16); + gb->set_input(pos_node, pos_embed_f16.data(), Precision::FP16); + return pos_node; +} + +size_t ParakeetTDTModel::build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, + uint32_t layer_idx, ComputeBackend backend) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t q = gb->matmul(hidden, layer.self_attn_q_weight, true, backend); + q = gb->add(q, layer.self_attn_q_bias); + size_t k = gb->matmul(hidden, layer.self_attn_k_weight, true, backend); + k = gb->add(k, layer.self_attn_k_bias); + size_t v = gb->matmul(hidden, layer.self_attn_v_weight, true, backend); + v = gb->add(v, layer.self_attn_v_bias); + + const auto& q_shape = gb->get_output_buffer(q).shape; + if (q_shape.size() != 2) { + throw std::runtime_error("ParakeetTDT self-attention expects [T, D]"); + } + + const size_t T = q_shape[0]; + const size_t q_heads = std::max(1, static_cast(config_.attention_heads)); + const size_t kv_heads = std::max(1, static_cast(config_.attention_kv_heads)); + const size_t head_dim = std::max(1, static_cast(config_.attention_head_dim)); + + size_t q4 = gb->reshape(q, {1, T, q_heads, head_dim}); + size_t k4 = gb->reshape(k, {1, T, kv_heads, head_dim}); + size_t v4 = gb->reshape(v, {1, T, kv_heads, head_dim}); + + size_t bias_u = gb->reshape(layer.self_attn_bias_u, {1, static_cast(1), q_heads, head_dim}); + size_t bias_v = gb->reshape(layer.self_attn_bias_v, {1, static_cast(1), q_heads, head_dim}); + size_t q_u4 = gb->add(q4, bias_u); + size_t q_v4 = gb->add(q4, bias_v); + + size_t rel_k_flat = gb->matmul(position_embeddings, layer.self_attn_relative_k_weight, true, backend); + size_t rel_k4 = gb->reshape(rel_k_flat, {1, 2 * T - 1, q_heads, head_dim}); + size_t rel_bias = gb->rel_pos_bias(q_v4, rel_k4, attention_scale_); + + size_t attn = gb->attention_masked(q_u4, k4, v4, rel_bias, attention_scale_, false, backend, true); + attn = gb->reshape(attn, {T, q_heads * head_dim}); + + size_t out = gb->matmul(attn, layer.self_attn_output_weight, true, backend); + out = gb->add(out, layer.self_attn_output_bias); + return out; +} + +size_t ParakeetTDTModel::build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + bool second_ff, ComputeBackend backend) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t w1 = second_ff ? layer.ff2_linear1_weight : layer.ff1_linear1_weight; + size_t b1 = second_ff ? layer.ff2_linear1_bias : layer.ff1_linear1_bias; + size_t w2 = second_ff ? layer.ff2_linear2_weight : layer.ff1_linear2_weight; + size_t b2 = second_ff ? layer.ff2_linear2_bias : layer.ff1_linear2_bias; + + size_t x = gb->matmul(hidden, w1, true, backend); + x = gb->add(x, b1); + + std::string act = config_.encoder_hidden_act; + std::transform(act.begin(), act.end(), act.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (act.find("gelu") != std::string::npos) { + x = gb->gelu(x); + } else if (act == "relu") { + x = gb->relu(x); + } else { + x = gb->silu(x); + } + + x = gb->matmul(x, w2, true, backend); + x = gb->add(x, b2); + return x; +} + +size_t ParakeetTDTModel::build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend) { + (void)backend; + const auto& layer = weight_nodes_.layers[layer_idx]; + const auto& hidden_shape = gb->get_output_buffer(hidden).shape; + if (hidden_shape.size() != 2) { + throw std::runtime_error("ParakeetTDT convolution module expects [T, D]"); + } + + const size_t T = hidden_shape[0]; + const size_t D = hidden_shape[1]; + + size_t x = gb->reshape(hidden, {1, T, D}); + x = gb->conv1d_pointwise(x, layer.conv_pointwise1_weight, layer.conv_pointwise1_bias); + x = gb->glu(x, -1); + + x = gb->conv1d_same_depthwise_k9(x, layer.conv_depthwise_weight, layer.conv_depthwise_bias); + + x = gb->batchnorm( + x, + layer.conv_batchnorm_weight, + layer.conv_batchnorm_bias, + layer.conv_batchnorm_running_mean, + layer.conv_batchnorm_running_var, + 2, + 1e-5f + ); + + std::string act = config_.encoder_hidden_act; + std::transform(act.begin(), act.end(), act.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (act.find("gelu") != std::string::npos) { + x = gb->gelu(x); + } else if (act == "relu") { + x = gb->relu(x); + } else { + x = gb->silu(x); + } + + x = gb->conv1d_pointwise(x, layer.conv_pointwise2_weight, layer.conv_pointwise2_bias); + + x = gb->reshape(x, {T, D}); + return x; +} + +size_t ParakeetTDTModel::build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, + uint32_t layer_idx, ComputeBackend backend) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t ff1_in = gb->layernorm(hidden, layer.norm_ff1_weight, layer.norm_ff1_bias); + size_t ff1 = build_feed_forward(gb, ff1_in, layer_idx, false, backend); + ff1 = gb->scalar_multiply(ff1, 0.5f); + size_t x = gb->add(hidden, ff1); + + size_t attn_in = gb->layernorm(x, layer.norm_self_attn_weight, layer.norm_self_attn_bias); + size_t attn = build_self_attention(gb, attn_in, position_embeddings, layer_idx, backend); + x = gb->add(x, attn); + + size_t conv_in = gb->layernorm(x, layer.norm_conv_weight, layer.norm_conv_bias); + size_t conv = build_convolution_module(gb, conv_in, layer_idx, backend); + x = gb->add(x, conv); + + size_t ff2_in = gb->layernorm(x, layer.norm_ff2_weight, layer.norm_ff2_bias); + size_t ff2 = build_feed_forward(gb, ff2_in, layer_idx, true, backend); + ff2 = gb->scalar_multiply(ff2, 0.5f); + x = gb->add(x, ff2); + + x = gb->layernorm(x, layer.norm_out_weight, layer.norm_out_bias); + return x; +} + +size_t ParakeetTDTModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) { + const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); + if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { + throw std::runtime_error("ParakeetTDT expects audio_features with shape [num_mels, num_frames]"); + } + + const size_t frames = audio_features.size() / num_mels; + + size_t expected_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); + { + const auto& joint_enc_shape = gb->get_output_buffer(weight_nodes_.joint_enc_weight).shape; + if (joint_enc_shape.size() == 2) { + expected_hidden_dim = joint_enc_shape[1]; + } + } + + if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { + std::vector time_major(frames * num_mels); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &audio_features[m * frames]; + for (size_t t = 0; t < frames; ++t) { + time_major[t * num_mels + m] = src[t]; + } + } + + std::vector<__fp16> time_major_f16(time_major.size()); + cactus_fp32_to_fp16(time_major.data(), time_major_f16.data(), time_major.size()); + + std::vector input_shape = npu_encoder_->get_input_shape(); + std::vector<__fp16> npu_input; + if (pack_parakeet_features_for_npu(time_major_f16, frames, num_mels, input_shape, npu_input)) { + npu_encoder_->preallocate(input_shape, "x", ""); + + std::vector output_shape = npu_encoder_->get_output_shape(); + size_t output_capacity = npu_encoder_->get_output_buffer_size(); + if (output_capacity == 0) { + output_capacity = shape_elements(output_shape); + } + if (output_capacity == 0) { + output_capacity = std::max(1, frames * expected_hidden_dim); + } + + std::vector<__fp16> npu_output(output_capacity); + const size_t elements_written = npu_encoder_->encode( + npu_input.data(), + npu_output.data(), + input_shape, + "x", + "" + ); + + size_t T_enc = 0; + size_t D_enc = 0; + if (elements_written > 0 && + infer_npu_encoder_output_shape(output_shape, elements_written, expected_hidden_dim, T_enc, D_enc)) { + const __fp16* src = npu_output.data(); + __fp16* cached_output = npu_encoder_->get_output_buffer(); + const size_t cached_count = npu_encoder_->get_output_buffer_size(); + const size_t required = T_enc * D_enc; + if (cached_output != nullptr && cached_count >= required) { + src = cached_output; + } + + if (D_enc == expected_hidden_dim) { + size_t enc_output = gb->input({T_enc, D_enc}, Precision::FP16); + gb->set_input(enc_output, src, Precision::FP16); + return enc_output; + } + } + } + } + + if (!has_cpu_encoder_weights_) { + throw std::runtime_error( + "Parakeet-TDT requires either CPU encoder weights or model.mlpackage encoder output."); + } + + ComputeBackend backend = ComputeBackend::CPU; + size_t hidden = build_subsampling(gb, audio_features); + const auto& hidden_shape = gb->get_output_buffer(hidden).shape; + if (hidden_shape.size() != 2) { + throw std::runtime_error("ParakeetTDT encoder expects subsampling output [T, D]"); + } + + size_t position_embeddings = build_relative_position_embeddings(gb, hidden_shape[0]); + for (uint32_t i = 0; i < config_.num_layers; ++i) { + hidden = build_encoder_block(gb, hidden, position_embeddings, i, backend); + } + + return hidden; +} + +size_t ParakeetTDTModel::forward(const std::vector& audio_features, + const std::vector& tokens, + bool use_cache) { + (void)tokens; + (void)use_cache; + + auto* gb = static_cast(graph_handle_); + gb->clear_debug_nodes(); + return build_encoder(gb, audio_features); +} + +std::vector ParakeetTDTModel::decode_tdt_tokens_with_state( + CactusGraph* gb, + size_t encoder_hidden_node, + size_t replay_start_frame, + size_t start_frame, + size_t end_frame, + ChunkStreamState* stream_state, + size_t* out_confirmed_count, + double* out_raw_decoder_time_ms) const { + const auto& enc_buf = gb->get_output_buffer(encoder_hidden_node); + if (enc_buf.shape.size() != 2) { + throw std::runtime_error("ParakeetTDT encoder output must be rank-2 [T, D]"); + } + + const size_t T = enc_buf.shape[0]; + const size_t D = enc_buf.shape[1]; + if (T == 0 || D == 0) { + return {}; + } + + std::vector<__fp16> encoder_fp16 = copy_buffer_to_fp16(enc_buf); + + gb->soft_reset(); + + size_t frame_in = gb->input({1, D}, Precision::FP16); + size_t token_idx = gb->input({1}, Precision::FP32); + + size_t pred = gb->embedding(weight_nodes_.predictor_embed, token_idx); + + const size_t predictor_layers = weight_nodes_.predictor_layers.size(); + std::vector h_prev_nodes; + std::vector c_prev_nodes; + std::vector h_new_nodes; + std::vector c_new_nodes; + std::vector bias_hh_zero_nodes; + std::vector hidden_sizes; + + h_prev_nodes.reserve(predictor_layers); + c_prev_nodes.reserve(predictor_layers); + h_new_nodes.reserve(predictor_layers); + c_new_nodes.reserve(predictor_layers); + bias_hh_zero_nodes.reserve(predictor_layers); + hidden_sizes.reserve(predictor_layers); + + for (size_t i = 0; i < predictor_layers; ++i) { + const auto w_ih = weight_nodes_.predictor_layers[i].weight_ih; + const auto w_hh = weight_nodes_.predictor_layers[i].weight_hh; + const auto b_ih = weight_nodes_.predictor_layers[i].bias; + + if (gb->get_output_buffer(w_ih).precision != Precision::FP16 || + gb->get_output_buffer(w_hh).precision != Precision::FP16 || + gb->get_output_buffer(b_ih).precision != Precision::FP16) { + throw std::runtime_error( + "ParakeetTDT predictor LSTM weights must be FP16. " + "Re-convert the model with updated converter."); + } + + const auto& w_ih_shape = gb->get_output_buffer(w_ih).shape; + if (w_ih_shape.size() != 2 || (w_ih_shape[0] % 4) != 0) { + throw std::runtime_error("ParakeetTDT predictor LSTM weight_ih must be [4*hidden, input]"); + } + const size_t hidden_size = w_ih_shape[0] / 4; + hidden_sizes.push_back(hidden_size); + + size_t h_prev = gb->input({1, hidden_size}, Precision::FP16); + size_t c_prev = gb->input({1, hidden_size}, Precision::FP16); + size_t b_hh_zero = gb->input({4 * hidden_size}, Precision::FP16); + + size_t lstm_out = gb->lstm_cell(pred, h_prev, c_prev, w_ih, w_hh, b_ih, b_hh_zero); + size_t h_new = gb->slice(lstm_out, 2, 0, 1); + size_t c_new = gb->slice(lstm_out, 2, 1, 1); + + h_new = gb->reshape(h_new, {1, hidden_size}); + c_new = gb->reshape(c_new, {1, hidden_size}); + + pred = h_new; + + h_prev_nodes.push_back(h_prev); + c_prev_nodes.push_back(c_prev); + h_new_nodes.push_back(h_new); + c_new_nodes.push_back(c_new); + bias_hh_zero_nodes.push_back(b_hh_zero); + } + + size_t enc_proj = gb->matmul(frame_in, weight_nodes_.joint_enc_weight, true, ComputeBackend::CPU); + enc_proj = gb->add(enc_proj, weight_nodes_.joint_enc_bias); + + size_t pred_proj = gb->matmul(pred, weight_nodes_.joint_pred_weight, true, ComputeBackend::CPU); + pred_proj = gb->add(pred_proj, weight_nodes_.joint_pred_bias); + + size_t joint = gb->add(enc_proj, pred_proj); + joint = gb->relu(joint); + + size_t logits = gb->matmul(joint, weight_nodes_.joint_out_weight, true, ComputeBackend::CPU); + logits = gb->add(logits, weight_nodes_.joint_out_bias); + + const auto& logits_shape = gb->get_output_buffer(logits).shape; + if (logits_shape.size() != 2 || logits_shape[0] != 1) { + throw std::runtime_error("ParakeetTDT joint output must be [1, classes]"); + } + + const size_t total_classes = logits_shape[1]; + std::vector durations = config_.tdt_durations; + size_t duration_classes = durations.size(); + if (duration_classes == 0) { + const size_t fallback = static_cast(config_.tdt_num_durations); + if (fallback > 0) { + durations.resize(fallback); + for (size_t i = 0; i < fallback; ++i) { + durations[i] = static_cast(i); + } + duration_classes = fallback; + } + } + + if (duration_classes == 0 || duration_classes >= total_classes) { + throw std::runtime_error("ParakeetTDT duration classes are invalid"); + } + + const size_t token_classes = total_classes - duration_classes; + + uint32_t blank_id = config_.tdt_blank_id; + if (blank_id >= token_classes) { + const auto& emb_shape = gb->get_output_buffer(weight_nodes_.predictor_embed).shape; + if (emb_shape.size() == 2 && emb_shape[0] > 0) { + const size_t inferred_blank = emb_shape[0] - 1; + blank_id = static_cast(std::min(inferred_blank, token_classes - 1)); + } else { + blank_id = static_cast(token_classes - 1); + } + } + + const size_t time_limit = std::min(end_frame, T); + const size_t emit_begin = std::min(start_frame, time_limit); + const size_t time_begin = std::min(replay_start_frame, emit_begin); + + std::vector> h_state(predictor_layers); + std::vector> c_state(predictor_layers); + std::vector> bias_hh_zero_state(predictor_layers); + const bool can_resume_stream_state = + stream_state && + stream_state->initialized && + stream_state->h.size() == predictor_layers && + stream_state->c.size() == predictor_layers; + + for (size_t i = 0; i < predictor_layers; ++i) { + const bool can_resume_layer = + can_resume_stream_state && + stream_state->h[i].size() == hidden_sizes[i] && + stream_state->c[i].size() == hidden_sizes[i]; + if (can_resume_layer) { + h_state[i] = stream_state->h[i]; + c_state[i] = stream_state->c[i]; + } else { + h_state[i].assign(hidden_sizes[i], static_cast<__fp16>(0.0f)); + c_state[i].assign(hidden_sizes[i], static_cast<__fp16>(0.0f)); + } + bias_hh_zero_state[i].assign(4 * hidden_sizes[i], static_cast<__fp16>(0.0f)); + gb->set_input(bias_hh_zero_nodes[i], bias_hh_zero_state[i].data(), Precision::FP16); + } + + constexpr float kHopSec = 160.0f / 16000.0f; + const float frame_sec = kHopSec * static_cast(config_.subsampling_factor); + auto* tokenizer = get_tokenizer(); + + std::vector output_tokens; + output_tokens.reserve(T * 2); + + uint32_t last_token = blank_id; + if (can_resume_stream_state && + stream_state->last_token < token_classes) { + last_token = stream_state->last_token; + } + constexpr size_t kMaxSymbolsPerStep = 10; + size_t time_idx = time_begin; + const bool is_stream_mode = cactus::telemetry::isStreamMode(); + std::vector> snap_h = h_state; + std::vector> snap_c = c_state; + uint32_t snap_last_token = last_token; + size_t confirmed_count = 0; + double raw_decoder_time_ms = 0.0; + + while (time_idx < time_limit) { + bool advanced = false; + size_t symbols_added = 0; + + while (symbols_added < kMaxSymbolsPerStep) { + const __fp16* frame_ptr = encoder_fp16.data() + time_idx * D; + gb->set_input(frame_in, frame_ptr, Precision::FP16); + + float token_value = static_cast(last_token); + gb->set_input(token_idx, &token_value, Precision::FP32); + + for (size_t i = 0; i < predictor_layers; ++i) { + gb->set_input(h_prev_nodes[i], h_state[i].data(), Precision::FP16); + gb->set_input(c_prev_nodes[i], c_state[i].data(), Precision::FP16); + } + + const auto decoder_step_start = std::chrono::steady_clock::now(); + gb->execute(); + const auto decoder_step_end = std::chrono::steady_clock::now(); + raw_decoder_time_ms += + std::chrono::duration_cast( + decoder_step_end - decoder_step_start).count() / 1000.0; + + const auto& logits_buf = gb->get_output_buffer(logits); + const auto& bias = get_vocab_bias(); + const size_t best_token = argmax_range_with_bias(logits_buf, 0, token_classes, bias); + const size_t best_duration_idx = argmax_range(logits_buf, token_classes, duration_classes); + const uint32_t predicted_skip = durations[best_duration_idx]; + uint32_t skip = predicted_skip; + if (is_stream_mode && skip > kMaxStreamDurationSkipFrames) { + skip = kMaxStreamDurationSkipFrames; + } + + if (best_token != blank_id) { + if (stream_state && time_idx >= emit_begin && tokenizer) { + std::string piece = tokenizer->decode({static_cast(best_token)}); + if (!piece.empty() && piece[0] == ' ' && !output_tokens.empty()) { + snap_h = h_state; + snap_c = c_state; + snap_last_token = last_token; + confirmed_count = output_tokens.size(); + } + } + if (time_idx >= emit_begin) { + output_tokens.push_back( + {static_cast(best_token), time_idx * frame_sec, (time_idx + skip) * frame_sec}); + } + last_token = static_cast(best_token); + + for (size_t i = 0; i < predictor_layers; ++i) { + const auto& h_buf = gb->get_output_buffer(h_new_nodes[i]); + const auto& c_buf = gb->get_output_buffer(c_new_nodes[i]); + const __fp16* h_ptr = h_buf.data_as<__fp16>(); + const __fp16* c_ptr = c_buf.data_as<__fp16>(); + std::copy(h_ptr, h_ptr + hidden_sizes[i], h_state[i].begin()); + std::copy(c_ptr, c_ptr + hidden_sizes[i], c_state[i].begin()); + } + } + + ++symbols_added; + + if (skip > 0) { + time_idx += skip; + advanced = true; + break; + } + + if (best_token == blank_id) { + ++time_idx; + advanced = true; + break; + } + } + + if (!advanced) { + ++time_idx; + } + } + + if (out_confirmed_count) { + *out_confirmed_count = confirmed_count; + } + if (out_raw_decoder_time_ms) { + *out_raw_decoder_time_ms = raw_decoder_time_ms; + } + + if (stream_state) { + stream_state->initialized = true; + stream_state->last_token = snap_last_token; + stream_state->h = std::move(snap_h); + stream_state->c = std::move(snap_c); + } + + return output_tokens; +} + +std::vector ParakeetTDTModel::greedy_decode_tdt_tokens( + CactusGraph* gb, + size_t encoder_hidden_node) const { + return decode_tdt_tokens_with_state( + gb, + encoder_hidden_node, + 0, + 0, + std::numeric_limits::max(), + nullptr); +} + +uint32_t ParakeetTDTModel::decode_with_audio( + const std::vector& tokens, + const std::vector& audio_features, + float temperature, + float top_p, + size_t top_k, + const std::string& profile_file, + float* out_entropy, + float min_p, + float repetition_penalty, + float* out_token_time_start, + float* out_token_time_end) +{ + (void)temperature; + (void)top_p; + (void)top_k; + (void)min_p; + (void)repetition_penalty; + + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + if (audio_features.empty()) { + throw std::runtime_error("Audio features cannot be empty in ParakeetTDT decode_with_audio"); + } + + auto* gb = static_cast(graph_handle_); + + const bool new_request = !tdt_tokens_ready_ || tokens.empty() || tokens.size() < last_input_token_count_; + if (new_request) { + gb->soft_reset(); + size_t encoder_out = forward(audio_features, tokens, false); + + if (!profile_file.empty()) { + gb->execute(profile_file); + } else { + gb->execute(); + } + + tdt_tokens_ = greedy_decode_tdt_tokens(gb, encoder_out); + tdt_emit_index_ = 0; + tdt_tokens_ready_ = true; + } + + last_input_token_count_ = tokens.size(); + if (out_entropy) { + *out_entropy = 0.0f; + } + + if (tdt_emit_index_ < tdt_tokens_.size()) { + const auto& tok = tdt_tokens_[tdt_emit_index_++]; + if (out_token_time_start) *out_token_time_start = tok.time_start; + if (out_token_time_end) *out_token_time_end = tok.time_end; + return tok.id; + } + + return get_tokenizer()->get_eos_token(); +} + +ParakeetTDTModel::ChunkStreamResult ParakeetTDTModel::decode_chunk_stream( + const std::vector& audio_features, + size_t replay_start_frame, + size_t start_frame, + size_t end_frame, + ChunkStreamState& state) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + if (audio_features.empty()) { + throw std::runtime_error("Audio features cannot be empty in ParakeetTDT decode_chunk_stream"); + } + + auto* gb = static_cast(graph_handle_); + + gb->soft_reset(); + size_t encoder_out = forward(audio_features, {}, false); + gb->execute(); + + ChunkStreamResult result; + double raw_decoder_time_ms = 0.0; + std::vector tokens = decode_tdt_tokens_with_state( + gb, encoder_out, replay_start_frame, start_frame, end_frame, &state, + &result.confirmed_token_count, &raw_decoder_time_ms); + + result.token_count = tokens.size(); + result.raw_decoder_time_ms = raw_decoder_time_ms; + result.raw_decoder_tps = + (result.token_count > 0 && raw_decoder_time_ms > 0.0) + ? (static_cast(result.token_count) * 1000.0) / raw_decoder_time_ms + : 0.0; + constexpr float kHopSec = 160.0f / 16000.0f; + const float frame_sec = kHopSec * static_cast(config_.subsampling_factor); + result.start_sec = start_frame * frame_sec; + result.confirmed_end_sec = start_frame * frame_sec; + result.resume_end_sec = replay_start_frame * frame_sec; + result.end_sec = start_frame * frame_sec; + if (!tokens.empty()) { + result.start_sec = tokens.front().time_start; + result.end_sec = tokens.back().time_end; + if (result.confirmed_token_count > 0 && + result.confirmed_token_count <= tokens.size()) { + result.confirmed_end_sec = tokens[result.confirmed_token_count - 1].time_end; + result.resume_end_sec = result.confirmed_end_sec; + } + } + + auto* tokenizer = get_tokenizer(); + if (!tokenizer) { + throw std::runtime_error("Tokenizer unavailable in ParakeetTDT decode_chunk_stream"); + } + for (size_t i = 0; i < tokens.size(); ++i) { + const std::string piece = tokenizer->decode({tokens[i].id}); + result.text += piece; + if (i < result.confirmed_token_count) { + result.confirmed_text += piece; + } else { + result.pending_text += piece; + } + } + return result; +} + +std::vector ParakeetTDTModel::get_audio_embeddings(const std::vector& audio_features) { + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + size_t hidden = build_encoder(gb, audio_features); + size_t pooled = gb->mean(hidden, 0); + gb->execute(); + + const auto& output_buf = gb->get_output_buffer(pooled); + const size_t hidden_dim = output_buf.total_size; + std::vector embedding(hidden_dim); + + if (output_buf.precision == Precision::FP32) { + const float* src = output_buf.data_as(); + std::copy(src, src + hidden_dim, embedding.begin()); + } else if (output_buf.precision == Precision::FP16) { + const __fp16* src = output_buf.data_as<__fp16>(); + Quantization::fp16_to_fp32(src, embedding.data(), hidden_dim); + } else { + const int8_t* src = output_buf.data_as(); + Quantization::int8_to_fp32(src, embedding.data(), hidden_dim, 1.0f); + } + + reset_cache(); + return embedding; +} + +void ParakeetTDTModel::reset_cache() { + Model::reset_cache(); + tdt_tokens_ready_ = false; + tdt_emit_index_ = 0; + tdt_tokens_.clear(); + last_input_token_count_ = 0; +} + +} // namespace engine +} // namespace cactus diff --git a/cactus-transpile/models/model_pyannote.cpp b/cactus-transpile/models/model_pyannote.cpp new file mode 100644 index 000000000..44ac4c8c3 --- /dev/null +++ b/cactus-transpile/models/model_pyannote.cpp @@ -0,0 +1,201 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../kernel/kernel.h" +#include +#include +#include + +namespace cactus { +namespace engine { + +static constexpr size_t WINDOW_SAMPLES = 160000; +static constexpr size_t FRAMES_PER_WINDOW = 589; +static constexpr int NUM_CLASSES = 7; +static constexpr int NUM_SPEAKERS = 3; + +PyAnnoteModel::PyAnnoteModel() : Model() {} +PyAnnoteModel::PyAnnoteModel(const Config& config) : Model(config) {} + +void PyAnnoteModel::load_weights_to_graph(CactusGraph* gb) { + const std::string& p = model_folder_path_; + + weight_nodes_.sinc_filters = gb->mmap_weights(p + "/sincnet_sinc_filters.weights"); + weight_nodes_.wav_norm_weight = gb->mmap_weights(p + "/sincnet_wav_norm_weight.weights"); + weight_nodes_.wav_norm_bias = gb->mmap_weights(p + "/sincnet_wav_norm_bias.weights"); + weight_nodes_.norm0_weight = gb->mmap_weights(p + "/sincnet_norm0_weight.weights"); + weight_nodes_.norm0_bias = gb->mmap_weights(p + "/sincnet_norm0_bias.weights"); + weight_nodes_.conv1_weight = gb->mmap_weights(p + "/sincnet_conv1_weight.weights"); + weight_nodes_.conv1_bias = gb->mmap_weights(p + "/sincnet_conv1_bias.weights"); + weight_nodes_.norm1_weight = gb->mmap_weights(p + "/sincnet_norm1_weight.weights"); + weight_nodes_.norm1_bias = gb->mmap_weights(p + "/sincnet_norm1_bias.weights"); + weight_nodes_.conv2_weight = gb->mmap_weights(p + "/sincnet_conv2_weight.weights"); + weight_nodes_.conv2_bias = gb->mmap_weights(p + "/sincnet_conv2_bias.weights"); + weight_nodes_.norm2_weight = gb->mmap_weights(p + "/sincnet_norm2_weight.weights"); + weight_nodes_.norm2_bias = gb->mmap_weights(p + "/sincnet_norm2_bias.weights"); + + for (int layer = 0; layer < 4; ++layer) { + auto& lw = weight_nodes_.lstm_layers[layer]; + std::string fwd = p + "/lstm_fwd_" + std::to_string(layer); + std::string bwd = p + "/lstm_bwd_" + std::to_string(layer); + lw.w_ih_fwd = gb->mmap_weights(fwd + "_weight_ih.weights"); + lw.w_hh_fwd = gb->mmap_weights(fwd + "_weight_hh.weights"); + lw.b_ih_fwd = gb->mmap_weights(fwd + "_bias_ih.weights"); + lw.b_hh_fwd = gb->mmap_weights(fwd + "_bias_hh.weights"); + lw.w_ih_bwd = gb->mmap_weights(bwd + "_weight_ih.weights"); + lw.w_hh_bwd = gb->mmap_weights(bwd + "_weight_hh.weights"); + lw.b_ih_bwd = gb->mmap_weights(bwd + "_bias_ih.weights"); + lw.b_hh_bwd = gb->mmap_weights(bwd + "_bias_hh.weights"); + } + + weight_nodes_.linear0_weight = gb->mmap_weights(p + "/linear_0_weight.weights"); + weight_nodes_.linear0_bias = gb->mmap_weights(p + "/linear_0_bias.weights"); + weight_nodes_.linear1_weight = gb->mmap_weights(p + "/linear_1_weight.weights"); + weight_nodes_.linear1_bias = gb->mmap_weights(p + "/linear_1_bias.weights"); + weight_nodes_.classifier_weight = gb->mmap_weights(p + "/classifier_weight.weights"); + weight_nodes_.classifier_bias = gb->mmap_weights(p + "/classifier_bias.weights"); +} + +void PyAnnoteModel::build_graph() { + audio_input_ = graph_.input({1, 1, 160000}, Precision::FP16); + + size_t x = graph_.groupnorm(audio_input_, weight_nodes_.wav_norm_weight, weight_nodes_.wav_norm_bias, 1, 1e-5f); + x = graph_.conv1d(x, weight_nodes_.sinc_filters, 10); + x = graph_.abs(x); + x = graph_.maxpool1d(x, 3, 3); + x = graph_.groupnorm(x, weight_nodes_.norm0_weight, weight_nodes_.norm0_bias, 80, 1e-5f); + x = graph_.leaky_relu(x, 0.01f); + + x = graph_.conv1d(x, weight_nodes_.conv1_weight, weight_nodes_.conv1_bias, 1); + x = graph_.maxpool1d(x, 3, 3); + x = graph_.groupnorm(x, weight_nodes_.norm1_weight, weight_nodes_.norm1_bias, 60, 1e-5f); + x = graph_.leaky_relu(x, 0.01f); + + x = graph_.conv1d(x, weight_nodes_.conv2_weight, weight_nodes_.conv2_bias, 1); + x = graph_.maxpool1d(x, 3, 3); + x = graph_.groupnorm(x, weight_nodes_.norm2_weight, weight_nodes_.norm2_bias, 60, 1e-5f); + x = graph_.leaky_relu(x, 0.01f); + + x = graph_.transposeN(x, {0, 2, 1}); + + for (int layer = 0; layer < 4; ++layer) { + auto& lw = weight_nodes_.lstm_layers[layer]; + x = graph_.bilstm_sequence(x, + lw.w_ih_fwd, lw.w_hh_fwd, lw.b_ih_fwd, lw.b_hh_fwd, + lw.w_ih_bwd, lw.w_hh_bwd, lw.b_ih_bwd, lw.b_hh_bwd); + } + + const auto& bilstm_shape = graph_.get_output_buffer(x).shape; + size_t T = bilstm_shape[1]; + x = graph_.reshape(x, {T, bilstm_shape[2]}); + + x = graph_.add(graph_.matmul(x, weight_nodes_.linear0_weight, true), weight_nodes_.linear0_bias); + x = graph_.leaky_relu(x, 0.01f); + x = graph_.add(graph_.matmul(x, weight_nodes_.linear1_weight, true), weight_nodes_.linear1_bias); + x = graph_.leaky_relu(x, 0.01f); + + x = graph_.add(graph_.matmul(x, weight_nodes_.classifier_weight, true), weight_nodes_.classifier_bias); + x = graph_.softmax(x, -1); + x = graph_.reshape(x, {1, T, 7}); + + output_node_ = x; +} + +bool PyAnnoteModel::init(const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup) { + (void)context_size; + (void)system_prompt; + (void)do_warmup; + + if (initialized_) { + return true; + } + + model_folder_path_ = model_folder; + + try { + load_weights_to_graph(&graph_); + build_graph(); + + chunk_buf_.resize(WINDOW_SAMPLES); + + hamming_.resize(FRAMES_PER_WINDOW); + for (size_t i = 0; i < FRAMES_PER_WINDOW; ++i) + hamming_[i] = 0.54f - 0.46f * std::cos(2.0f * M_PI * i / (FRAMES_PER_WINDOW - 1)); + + initialized_ = true; + return true; + } catch (const std::exception& e) { + initialized_ = false; + return false; + } +} + +std::vector PyAnnoteModel::diarize(const float* pcm_f32, size_t num_samples, size_t step_samples, bool raw_powerset) { + if (!initialized_) throw std::runtime_error("PyAnnote model not initialized"); + + const size_t total_frames = std::max( + FRAMES_PER_WINDOW, + static_cast(std::round((double)num_samples * FRAMES_PER_WINDOW / WINDOW_SAMPLES)) + ); + + std::vector aggregated(total_frames * NUM_CLASSES, 0.0f); + std::vector weight_sum(total_frames, 0.0f); + + auto process_chunk = [&](size_t chunk_start) { + const size_t copy_len = std::min(WINDOW_SAMPLES, num_samples - chunk_start); + for (size_t i = 0; i < copy_len; ++i) + chunk_buf_[i] = static_cast<__fp16>(pcm_f32[chunk_start + i]); + std::fill(chunk_buf_.begin() + copy_len, chunk_buf_.end(), static_cast<__fp16>(0.0f)); + + graph_.set_input(audio_input_, chunk_buf_.data(), Precision::FP16); + graph_.execute(); + + const __fp16* chunk_scores = graph_.get_output_buffer(output_node_).data_as<__fp16>(); + const size_t frame_offset = static_cast(std::round((double)chunk_start * FRAMES_PER_WINDOW / WINDOW_SAMPLES)); + + for (size_t f = 0; f < FRAMES_PER_WINDOW; ++f) { + const size_t out_f = frame_offset + f; + if (out_f >= total_frames) break; + const float w = hamming_[f]; + weight_sum[out_f] += w; + for (int c = 0; c < NUM_CLASSES; ++c) + aggregated[out_f * NUM_CLASSES + c] += w * static_cast(chunk_scores[f * NUM_CLASSES + c]); + } + }; + + const size_t last_start = num_samples > WINDOW_SAMPLES ? num_samples - WINDOW_SAMPLES : 0; + bool last_processed = false; + for (size_t s = 0; s + WINDOW_SAMPLES <= num_samples; s += step_samples) { + if (s >= last_start) last_processed = true; + process_chunk(s); + } + if (!last_processed) + process_chunk(last_start); + + for (size_t f = 0; f < total_frames; ++f) { + if (weight_sum[f] > 0.0f) { + const float inv_w = 1.0f / weight_sum[f]; + float* agg = aggregated.data() + f * NUM_CLASSES; + for (int c = 0; c < NUM_CLASSES; ++c) + agg[c] *= inv_w; + } + } + + if (raw_powerset) { + return aggregated; + } + + std::vector speaker_probs(total_frames * NUM_SPEAKERS, 0.0f); + for (size_t f = 0; f < total_frames; ++f) { + const float* agg = aggregated.data() + f * NUM_CLASSES; + float* dst = speaker_probs.data() + f * NUM_SPEAKERS; + dst[0] = agg[1] + agg[4] + agg[5]; + dst[1] = agg[2] + agg[4] + agg[6]; + dst[2] = agg[3] + agg[5] + agg[6]; + } + + return speaker_probs; +} + +} +} diff --git a/cactus-transpile/models/model_qwen.cpp b/cactus-transpile/models/model_qwen.cpp new file mode 100644 index 000000000..601c05334 --- /dev/null +++ b/cactus-transpile/models/model_qwen.cpp @@ -0,0 +1,171 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../npu/npu.h" +#include +#include + + +namespace cactus { +namespace engine { + +QwenModel::QwenModel() : Model() {} + +QwenModel::QwenModel(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); +} + +void QwenModel::load_weights_to_graph(CactusGraph* gb) { + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + } + + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer = weight_nodes_.layers[i]; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); + layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); + layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); + layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); + layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); + layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); + layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); + layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); + layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); + layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); + layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); + } + + if (npu::is_npu_available()) { + std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; + load_npu_prefill(npu_prefill_path); + } +} + +size_t QwenModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); + auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); + auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); + + const auto& q_shape = gb->get_output_buffer(q_proj).shape; + size_t batch_seq = q_shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); + q_proj = gb->rms_norm(q_proj, layer.attn_q_norm_weight, config_.layer_norm_eps); + q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); + + size_t num_kv_heads = config_.attention_kv_heads; + k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); + k_proj = gb->rms_norm(k_proj, layer.attn_k_norm_weight, config_.layer_norm_eps); + k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); + + size_t seq_len = batch_seq; + + auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); + auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + + if (config_.rope_theta > 0) { + q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); + k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); + } + + size_t attn_output_4d; + + if (use_cache) { + cache_k_output_nodes_[layer_idx] = k_proj_4d; + cache_v_output_nodes_[layer_idx] = v_proj_4d; + } + + if (use_cache && !kv_cache_.is_empty()) { + attn_output_4d = gb->attention_int8_hybrid( + q_proj_4d, k_proj_4d, v_proj_4d, + attention_scale_, position_offset, + kv_cache_.get_keys_int8(layer_idx), + kv_cache_.get_values_int8(layer_idx), + kv_cache_.get_key_scales(layer_idx), + kv_cache_.get_value_scales(layer_idx), + kv_cache_.current_seq_len, num_kv_heads, head_dim + ); + } else { + attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); + } + + auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); + return gb->matmul(attn_output, layer.attn_output_weight, true, backend); +} + + +size_t QwenModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); + size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); + size_t gate_silu = gb->silu(gate_output); + size_t gated = gb->multiply(gate_silu, up_output); + return gb->matmul(gated, layer.ffn_down_weight, true, backend); +} + + +size_t QwenModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); + auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); + auto after_attention = gb->add(hidden, attn_output); + auto normalized_after_attention = gb->rms_norm(after_attention, layer.post_attention_layernorm_weight, config_.layer_norm_eps); + auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); + return gb->add(after_attention, mlp_output); +} + + +size_t QwenModel::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + if (tokens.empty()) { + throw std::runtime_error("Token sequence cannot be empty"); + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + auto seq_len = static_cast(tokens.size()); + + size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + auto input_node_id = gb->input({seq_len}, Precision::FP32); + auto hidden = gb->embedding(embedding_node_id_, input_node_id); + + std::vector input_data(seq_len); + for (size_t i = 0; i < seq_len; i++) { + input_data[i] = static_cast(tokens[i]); + } + gb->set_input(input_node_id, input_data.data(), Precision::FP32); + + for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { + hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); + } + + auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); + + return final_hidden; +} + +} +} \ No newline at end of file diff --git a/cactus-transpile/models/model_qwen3p5.cpp b/cactus-transpile/models/model_qwen3p5.cpp new file mode 100644 index 000000000..0f438e1c0 --- /dev/null +++ b/cactus-transpile/models/model_qwen3p5.cpp @@ -0,0 +1,771 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../npu/npu.h" +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +Qwen3p5Model::Qwen3p5Model() : Model() { + weight_nodes_.layers.resize(config_.num_layers); + conv_cache_state_nodes_.assign(config_.num_layers, 0); +} + +Qwen3p5Model::Qwen3p5Model(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); + conv_cache_state_nodes_.assign(config.num_layers, 0); +} + +void Qwen3p5Model::post_init() { + bool has_deltanet_layer = false; + const WeightNodeIDs::LayerWeights* first_linear = nullptr; + for (const auto& layer : weight_nodes_.layers) { + if (layer.type == WeightNodeIDs::LayerType::DELTANET) { + has_deltanet_layer = true; + if (!first_linear) { + first_linear = &layer.weights; + } + break; + } + } + + deltanet_heads_ = static_cast(config_.attention_heads); + deltanet_key_dim_ = static_cast(config_.attention_head_dim); + deltanet_value_dim_ = static_cast(config_.attention_head_dim); + + if (has_deltanet_layer && + config_.linear_num_key_heads > 0 && + config_.linear_key_head_dim > 0) { + const size_t num_k_heads = static_cast(config_.linear_num_key_heads); + const size_t key_head_dim = static_cast(config_.linear_key_head_dim); + size_t num_v_heads = static_cast(config_.linear_num_value_heads > 0 + ? config_.linear_num_value_heads + : config_.linear_num_key_heads); + size_t value_head_dim = static_cast(config_.linear_value_head_dim); + + if (num_v_heads == 0) { + throw std::runtime_error("Qwen3p5Model invalid linear-attention value head count in config"); + } + if (value_head_dim == 0) { + if (config_.linear_v_proj_dim > 0 && (config_.linear_v_proj_dim % num_v_heads == 0)) { + value_head_dim = static_cast(config_.linear_v_proj_dim / num_v_heads); + } else { + throw std::runtime_error("Qwen3p5Model cannot derive linear-attention value head dim from config"); + } + } + if (config_.linear_q_proj_dim > 0 && + static_cast(config_.linear_q_proj_dim) != num_k_heads * key_head_dim) { + throw std::runtime_error("Qwen3p5Model linear-attention q projection dim mismatch with key-head topology"); + } + if (config_.linear_v_proj_dim > 0 && + static_cast(config_.linear_v_proj_dim) != num_v_heads * value_head_dim) { + throw std::runtime_error("Qwen3p5Model linear-attention v projection dim mismatch with value-head topology"); + } + + deltanet_heads_ = num_v_heads; + deltanet_key_dim_ = key_head_dim; + deltanet_value_dim_ = value_head_dim; + } else if (has_deltanet_layer) { + auto* gb = static_cast(graph_handle_); + if (!gb) { + throw std::runtime_error("Qwen3p5Model cannot infer linear-attention dims: graph unavailable"); + } + if (!first_linear) { + throw std::runtime_error("Qwen3p5Model expected deltanet layer weights for dim inference"); + } + + const auto& q_buf = gb->get_output_buffer(first_linear->attn_q_weight); + const auto& v_buf = gb->get_output_buffer(first_linear->attn_v_weight); + const auto& n_buf = gb->get_output_buffer(first_linear->attn_q_norm_weight); + if (q_buf.shape.size() < 2 || v_buf.shape.size() < 2 || n_buf.shape.empty()) { + throw std::runtime_error("Qwen3p5Model invalid linear-attention weight shapes"); + } + + const size_t q_proj_dim = q_buf.shape[0]; + const size_t v_proj_dim = v_buf.shape[0]; + const size_t inferred_key_dim = n_buf.shape[0]; + if (inferred_key_dim == 0 || q_proj_dim == 0 || v_proj_dim == 0) { + throw std::runtime_error("Qwen3p5Model failed to infer non-zero linear-attention dims"); + } + if (q_proj_dim % inferred_key_dim != 0) { + throw std::runtime_error("Qwen3p5Model q projection dim is not divisible by inferred key dim"); + } + + const size_t inferred_k_heads = q_proj_dim / inferred_key_dim; + const size_t inferred_v_heads = v_proj_dim / inferred_key_dim; + if (inferred_k_heads == 0 || inferred_v_heads == 0 || inferred_v_heads % inferred_k_heads != 0) { + throw std::runtime_error("Qwen3p5Model failed to infer compatible linear-attention head topology"); + } + + deltanet_heads_ = inferred_v_heads; + deltanet_key_dim_ = inferred_key_dim; + deltanet_value_dim_ = inferred_key_dim; + } + + if (deltanet_heads_ == 0 || deltanet_key_dim_ == 0 || deltanet_value_dim_ == 0) { + throw std::runtime_error("Qwen3p5Model requires non-zero deltanet heads/key/value dims"); + } + + deltanet_state_flat_dim_ = deltanet_heads_ * deltanet_key_dim_ * deltanet_value_dim_; + + deltanet_mixed_dim_ = 0; + if (config_.linear_q_proj_dim > 0 && config_.linear_v_proj_dim > 0) { + deltanet_mixed_dim_ = static_cast(2 * config_.linear_q_proj_dim + config_.linear_v_proj_dim); + } else if (has_deltanet_layer) { + auto* gb = static_cast(graph_handle_); + if (!gb || !first_linear) { + throw std::runtime_error("Qwen3p5Model cannot infer linear-attention mixed projection dim"); + } + const auto& q_buf = gb->get_output_buffer(first_linear->attn_q_weight); + const auto& v_buf = gb->get_output_buffer(first_linear->attn_v_weight); + if (q_buf.shape.size() < 2 || v_buf.shape.size() < 2) { + throw std::runtime_error("Qwen3p5Model invalid q/v projection shapes for mixed dim inference"); + } + deltanet_mixed_dim_ = (2 * q_buf.shape[0]) + v_buf.shape[0]; + } + + deltanet_conv_history_len_ = 0; + if (has_deltanet_layer) { + auto* gb = static_cast(graph_handle_); + if (!gb) { + throw std::runtime_error("Qwen3p5Model cannot infer linear-attention conv history: graph unavailable"); + } + for (const auto& layer : weight_nodes_.layers) { + if (layer.type != WeightNodeIDs::LayerType::DELTANET || layer.weights.deltanet_conv_weight == 0) { + continue; + } + const auto& conv_buf = gb->get_output_buffer(layer.weights.deltanet_conv_weight); + if (conv_buf.shape.empty()) { + continue; + } + const size_t kernel = conv_buf.shape.back(); + if (kernel > 1) { + deltanet_conv_history_len_ = std::max(deltanet_conv_history_len_, kernel - 1); + } + } + } + + if (deltanet_conv_history_len_ > 0 && deltanet_mixed_dim_ == 0) { + throw std::runtime_error("Qwen3p5Model linear-attention conv history requires non-zero mixed dim"); + } + deltanet_conv_flat_dim_ = deltanet_conv_history_len_ * deltanet_mixed_dim_; + deltanet_cache_row_dim_ = deltanet_state_flat_dim_ + deltanet_conv_flat_dim_; + if (deltanet_cache_row_dim_ == 0) { + throw std::runtime_error("Qwen3p5Model computed zero cache row dim"); + } + + conv_cache_.init(config_.num_layers, deltanet_cache_row_dim_, 1, Precision::FP32); + last_forward_used_cache_ = false; + deltanet_total_seq_len_ = 0; +} + +void Qwen3p5Model::reset_cache() { + Model::reset_cache(); + if (conv_cache_.window_size > 0) { + conv_cache_.reset(); + } + std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); + deltanet_total_seq_len_ = 0; + last_forward_used_cache_ = false; +} + +void Qwen3p5Model::post_execute_updates(CactusGraph* gb, size_t seq_len) { + if (!last_forward_used_cache_) { + std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); + return; + } + + const size_t layer_count = std::min(conv_cache_state_nodes_.size(), weight_nodes_.layers.size()); + for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { + if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::DELTANET) { + conv_cache_state_nodes_[layer_idx] = 0; + continue; + } + + const size_t state_node = conv_cache_state_nodes_[layer_idx]; + if (state_node != 0) { + conv_cache_.update(gb, layer_idx, state_node); + } + conv_cache_state_nodes_[layer_idx] = 0; + } + + deltanet_total_seq_len_ += seq_len; + last_forward_used_cache_ = false; +} + +void Qwen3p5Model::load_weights_to_graph(CactusGraph* gb) { + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + } + + auto choose_existing_weight = [&](const std::vector& candidates) -> std::string { + for (const auto& p : candidates) { + if (std::filesystem::exists(p)) { + return p; + } + } + return std::string(); + }; + + auto mmap_required = [&](const std::vector& candidates, const std::string& weight_name) -> size_t { + const std::string path = choose_existing_weight(candidates); + if (path.empty()) { + throw std::runtime_error("Qwen3p5Model missing required weight: " + weight_name); + } + return gb->mmap_weights(path); + }; + + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer_entry = weight_nodes_.layers[i]; + auto& layer = layer_entry.weights; + + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + bool is_deltanet = false; + if (i < config_.layer_types.size()) { + std::string lt = config_.layer_types[i]; + std::transform(lt.begin(), lt.end(), lt.begin(), ::tolower); + is_deltanet = (lt.find("deltanet") != std::string::npos || + lt.find("linear_attention") != std::string::npos || + lt.find("linear_attn") != std::string::npos); + } + layer_entry.type = is_deltanet ? WeightNodeIDs::LayerType::DELTANET : WeightNodeIDs::LayerType::ATTENTION; + + layer.attn_q_weight = mmap_required( + {layer_prefix + "attn_q.weights", layer_prefix + "linear_attn_q.weights"}, + "layer_" + std::to_string(i) + "_attn_q/linear_attn_q"); + layer.attn_k_weight = mmap_required( + {layer_prefix + "attn_k.weights", layer_prefix + "linear_attn_k.weights"}, + "layer_" + std::to_string(i) + "_attn_k/linear_attn_k"); + layer.attn_v_weight = mmap_required( + {layer_prefix + "attn_v.weights", layer_prefix + "linear_attn_v.weights"}, + "layer_" + std::to_string(i) + "_attn_v/linear_attn_v"); + layer.attn_output_weight = mmap_required( + {layer_prefix + "attn_output.weights", layer_prefix + "linear_attn_output.weights"}, + "layer_" + std::to_string(i) + "_attn_output/linear_attn_output"); + layer.input_layernorm_weight = mmap_required( + {layer_prefix + "input_norm.weights"}, + "layer_" + std::to_string(i) + "_input_norm"); + layer.attn_q_norm_weight = mmap_required( + {layer_prefix + "attn_q_norm.weights", layer_prefix + "linear_attn_norm.weights"}, + "layer_" + std::to_string(i) + "_attn_q_norm/linear_attn_norm"); + layer.attn_k_norm_weight = mmap_required( + {layer_prefix + "attn_k_norm.weights", layer_prefix + "linear_attn_norm.weights"}, + "layer_" + std::to_string(i) + "_attn_k_norm/linear_attn_norm"); + + if (is_deltanet) { + const std::string qkv_path = choose_existing_weight({ + layer_prefix + "linear_attn_qkv.weights" + }); + layer.deltanet_qkv_weight = qkv_path.empty() ? 0 : gb->mmap_weights(qkv_path); + + const std::string gate_path = choose_existing_weight({ + layer_prefix + "deltanet_gate.weights", + layer_prefix + "attn_gate.weights", + layer_prefix + "attn_f_gate.weights", + layer_prefix + "linear_attn_a.weights" + }); + const std::string beta_path = choose_existing_weight({ + layer_prefix + "deltanet_beta.weights", + layer_prefix + "attn_beta.weights", + layer_prefix + "attn_f_beta.weights", + layer_prefix + "linear_attn_b.weights" + }); + if (gate_path.empty() || beta_path.empty()) { + throw std::runtime_error( + "Qwen3p5Model deltanet layer missing gate/beta weights at layer " + std::to_string(i)); + } + layer.deltanet_gate_weight = gb->mmap_weights(gate_path); + layer.deltanet_beta_weight = gb->mmap_weights(beta_path); + + const std::string gate_bias_path = choose_existing_weight({ + layer_prefix + "linear_attn_A_log.weights" + }); + const std::string beta_bias_path = choose_existing_weight({ + layer_prefix + "linear_attn_dt_bias.weights" + }); + layer.deltanet_gate_bias = gate_bias_path.empty() ? 0 : gb->mmap_weights(gate_bias_path); + layer.deltanet_beta_bias = beta_bias_path.empty() ? 0 : gb->mmap_weights(beta_bias_path); + const std::string z_path = choose_existing_weight({ + layer_prefix + "linear_attn_z.weights" + }); + const std::string conv_path = choose_existing_weight({ + layer_prefix + "linear_attn_conv1d.weights" + }); + layer.deltanet_z_weight = z_path.empty() ? 0 : gb->mmap_weights(z_path); + layer.deltanet_conv_weight = conv_path.empty() ? 0 : gb->mmap_weights(conv_path); + } + + layer.ffn_gate_weight = mmap_required( + {layer_prefix + "ffn_gate.weights"}, + "layer_" + std::to_string(i) + "_ffn_gate"); + layer.ffn_up_weight = mmap_required( + {layer_prefix + "ffn_up.weights"}, + "layer_" + std::to_string(i) + "_ffn_up"); + layer.ffn_down_weight = mmap_required( + {layer_prefix + "ffn_down.weights"}, + "layer_" + std::to_string(i) + "_ffn_down"); + layer.post_attention_layernorm_weight = mmap_required( + {layer_prefix + "post_attn_norm.weights"}, + "layer_" + std::to_string(i) + "_post_attn_norm"); + } + + if (npu::is_npu_available()) { + std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; + load_npu_prefill(npu_prefill_path); + } +} + +size_t Qwen3p5Model::build_gated_deltanet(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + (void)position_offset; + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + + const auto& in_shape = gb->get_output_buffer(normalized_input).shape; + if (in_shape.size() < 2) { + throw std::runtime_error("Qwen3p5Model linear-attention expects rank-2 normalized input"); + } + const size_t seq_len = in_shape[0]; + + size_t q_proj = 0; + size_t k_proj = 0; + size_t v_proj = 0; + size_t mixed_qkv = 0; + size_t z_proj = layer.deltanet_z_weight ? gb->matmul(normalized_input, layer.deltanet_z_weight, true, backend) : 0; + + const size_t num_heads = deltanet_heads_; + const size_t key_dim = deltanet_key_dim_; + const size_t value_dim = deltanet_value_dim_; + const size_t v_proj_dim = num_heads * value_dim; + size_t qk_proj_dim = 0; + + if (layer.deltanet_qkv_weight != 0) { + mixed_qkv = gb->matmul(normalized_input, layer.deltanet_qkv_weight, true, backend); + const auto& mixed_shape = gb->get_output_buffer(mixed_qkv).shape; + if (mixed_shape.size() < 2 || mixed_shape[0] != seq_len) { + throw std::runtime_error("Qwen3p5Model invalid linear-attention joint qkv projection shape"); + } + if (mixed_shape[1] <= v_proj_dim || ((mixed_shape[1] - v_proj_dim) % 2) != 0) { + throw std::runtime_error("Qwen3p5Model invalid linear-attention joint qkv projection width"); + } + qk_proj_dim = (mixed_shape[1] - v_proj_dim) / 2; + } else { + q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); + k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); + v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); + const auto& q_shape = gb->get_output_buffer(q_proj).shape; + if (q_shape.size() < 2 || q_shape[0] != seq_len) { + throw std::runtime_error("Qwen3p5Model invalid linear-attention q projection shape"); + } + qk_proj_dim = q_shape[1]; + size_t mixed_qk = gb->concat(q_proj, k_proj, 1); + mixed_qkv = gb->concat(mixed_qk, v_proj, 1); + } + + if (qk_proj_dim == 0 || qk_proj_dim % key_dim != 0 || v_proj_dim == 0) { + throw std::runtime_error("Qwen3p5Model invalid linear-attention projection dimensions"); + } + const size_t num_k_heads = qk_proj_dim / key_dim; + if (num_heads % num_k_heads != 0) { + throw std::runtime_error("Qwen3p5Model requires num_v_heads divisible by num_k_heads"); + } + + const size_t mixed_proj_dim = qk_proj_dim + qk_proj_dim + v_proj_dim; + if (deltanet_mixed_dim_ != 0 && mixed_proj_dim != deltanet_mixed_dim_) { + throw std::runtime_error("Qwen3p5Model linear-attention mixed projection dim mismatch"); + } + const size_t state_flat_dim = deltanet_state_flat_dim_; + size_t prev_state_flat = 0; + size_t prev_conv_flat = 0; + if (use_cache && conv_cache_.window_size > 0) { + const auto view = conv_cache_.get_window(layer_idx); + if (conv_cache_.window_size == 1 && view.total_len == 1) { + const uint8_t* cache_row = nullptr; + if (view.len2 == 1 && view.ptr2 != nullptr) { + cache_row = static_cast(view.ptr2); + } else if (view.len1 == 1 && view.ptr1 != nullptr) { + cache_row = static_cast(view.ptr1); + } + if (cache_row != nullptr) { + prev_state_flat = gb->input({1, state_flat_dim}, conv_cache_.precision); + gb->set_external_input(prev_state_flat, const_cast(static_cast(cache_row)), + conv_cache_.precision); + if (deltanet_conv_flat_dim_ > 0) { + const size_t state_bytes = state_flat_dim * conv_cache_.element_size; + prev_conv_flat = gb->input({1, deltanet_conv_flat_dim_}, conv_cache_.precision); + gb->set_external_input( + prev_conv_flat, + const_cast(static_cast(cache_row + state_bytes)), + conv_cache_.precision); + } + } + } + + if (prev_state_flat == 0) { + std::vector segments; + if (view.len2 > 0) { + size_t left_node = gb->input({view.len2, deltanet_cache_row_dim_}, conv_cache_.precision); + gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); + segments.push_back(left_node); + } + if (view.len1 > 0) { + size_t right_node = gb->input({view.len1, deltanet_cache_row_dim_}, conv_cache_.precision); + gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); + segments.push_back(right_node); + } + + if (!segments.empty()) { + size_t stacked = segments[0]; + for (size_t i = 1; i < segments.size(); ++i) { + stacked = gb->concat(stacked, segments[i], 0); + } + if (view.total_len > 1) { + stacked = gb->slice(stacked, 0, view.total_len - 1, 1); + } + prev_state_flat = gb->slice(stacked, 1, 0, state_flat_dim); + if (deltanet_conv_flat_dim_ > 0) { + prev_conv_flat = gb->slice(stacked, 1, state_flat_dim, deltanet_conv_flat_dim_); + } + } + } + } + + size_t conv_source_with_history = mixed_qkv; + if (layer.deltanet_conv_weight) { + const auto& conv_wbuf = gb->get_output_buffer(layer.deltanet_conv_weight); + const size_t kernel = conv_wbuf.shape.back(); + if (kernel > 1 && use_cache && deltanet_conv_history_len_ > 0) { + size_t history_2d = 0; + if (prev_conv_flat != 0) { + history_2d = gb->reshape(prev_conv_flat, {deltanet_conv_history_len_, mixed_proj_dim}); + history_2d = gb->precision_cast(history_2d, Precision::FP16); + } else { + history_2d = gb->input({deltanet_conv_history_len_, mixed_proj_dim}, Precision::FP16); + std::vector<__fp16> zeros(deltanet_conv_flat_dim_, static_cast<__fp16>(0.0f)); + gb->set_input(history_2d, zeros.data(), Precision::FP16); + } + conv_source_with_history = gb->concat(history_2d, mixed_qkv, 0); + } + + const size_t total_conv_len = gb->get_output_buffer(conv_source_with_history).shape[0]; + size_t conv_input = gb->reshape(conv_source_with_history, {1, total_conv_len, mixed_proj_dim}); + size_t conv_out = gb->conv1d_causal(conv_input, layer.deltanet_conv_weight, kernel, 1); + const size_t conv_start = total_conv_len > seq_len ? total_conv_len - seq_len : 0; + conv_out = gb->slice(conv_out, 1, conv_start, seq_len); + mixed_qkv = gb->reshape(gb->silu(conv_out), {seq_len, mixed_proj_dim}); + } + + q_proj = gb->slice(mixed_qkv, 1, 0, qk_proj_dim); + k_proj = gb->slice(mixed_qkv, 1, qk_proj_dim, qk_proj_dim); + v_proj = gb->slice(mixed_qkv, 1, qk_proj_dim + qk_proj_dim, v_proj_dim); + + size_t q_4d = gb->reshape(q_proj, {1, seq_len, num_k_heads, key_dim}); + size_t k_4d = gb->reshape(k_proj, {1, seq_len, num_k_heads, key_dim}); + size_t v_4d = gb->reshape(v_proj, {1, seq_len, num_heads, value_dim}); + + size_t q_norm = gb->sum(gb->multiply(q_4d, q_4d), 3); + q_norm = gb->scalar_sqrt(gb->scalar_add(q_norm, 1e-6f)); + q_norm = gb->reshape(q_norm, {1, seq_len, num_k_heads, 1}); + q_4d = gb->divide(q_4d, q_norm); + + size_t k_norm = gb->sum(gb->multiply(k_4d, k_4d), 3); + k_norm = gb->scalar_sqrt(gb->scalar_add(k_norm, 1e-6f)); + k_norm = gb->reshape(k_norm, {1, seq_len, num_k_heads, 1}); + k_4d = gb->divide(k_4d, k_norm); + + size_t a_logits = gb->matmul(normalized_input, layer.deltanet_gate_weight, true, backend); + size_t b_logits = gb->matmul(normalized_input, layer.deltanet_beta_weight, true, backend); + a_logits = gb->precision_cast(a_logits, Precision::FP16); + b_logits = gb->precision_cast(b_logits, Precision::FP16); + if (layer.deltanet_beta_bias) { + size_t dt_bias_vec = layer.deltanet_beta_bias; + const auto& dt_buf = gb->get_output_buffer(dt_bias_vec); + if (!dt_buf.shape.empty() && dt_buf.shape[0] > num_heads) { + dt_bias_vec = gb->slice(dt_bias_vec, 0, 0, num_heads); + } + size_t dt_bias_2d = gb->reshape(dt_bias_vec, {1, num_heads}); + dt_bias_2d = gb->precision_cast(dt_bias_2d, Precision::FP16); + a_logits = gb->add(a_logits, dt_bias_2d); + } + size_t a_softplus = gb->scalar_log(gb->scalar_add(gb->scalar_exp(a_logits), 1.0f)); + + size_t gate_log; + if (layer.deltanet_gate_bias) { + size_t a_log_vec = layer.deltanet_gate_bias; + const auto& a_log_buf = gb->get_output_buffer(a_log_vec); + if (!a_log_buf.shape.empty() && a_log_buf.shape[0] > num_heads) { + a_log_vec = gb->slice(a_log_vec, 0, 0, num_heads); + } + size_t a_log_2d = gb->reshape(a_log_vec, {1, num_heads}); + a_log_2d = gb->precision_cast(a_log_2d, Precision::FP16); + size_t neg_exp_a = gb->scalar_multiply(gb->scalar_exp(a_log_2d), -1.0f); + gate_log = gb->multiply(neg_exp_a, a_softplus); + } else { + gate_log = gb->scalar_multiply(a_softplus, -1.0f); + } + size_t beta = gb->sigmoid(b_logits); + + size_t gate_3d = gb->reshape(gate_log, {1, seq_len, num_heads}); + size_t beta_3d = gb->reshape(beta, {1, seq_len, num_heads}); + + size_t initial_state; + + if (prev_state_flat != 0) { + initial_state = gb->reshape(prev_state_flat, {1, key_dim, num_heads, value_dim}); + } else { + initial_state = gb->input({1, key_dim, num_heads, value_dim}, Precision::FP16); + std::vector<__fp16> zeros(state_flat_dim, static_cast<__fp16>(0.0f)); + gb->set_input(initial_state, zeros.data(), Precision::FP16); + } + + size_t deltanet_out; + if (use_cache && seq_len == 1) { + deltanet_out = gb->gated_deltanet_decode(q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, 0.0f); + } else { + const size_t chunk_for_op = std::min(64, std::max(1, seq_len)); + deltanet_out = gb->gated_deltanet_prefill( + q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, chunk_for_op, 0.0f); + } + + size_t y_4d = gb->slice(deltanet_out, 1, 0, seq_len); + size_t state_tail = gb->slice(deltanet_out, 1, seq_len, key_dim); + + if (use_cache) { + size_t packed_cache = gb->reshape(state_tail, {1, state_flat_dim}); + if (deltanet_conv_flat_dim_ > 0) { + size_t history_2d = 0; + if (layer.deltanet_conv_weight) { + const size_t total_history_len = gb->get_output_buffer(conv_source_with_history).shape[0]; + if (total_history_len >= deltanet_conv_history_len_) { + const size_t hist_start = total_history_len - deltanet_conv_history_len_; + history_2d = gb->slice(conv_source_with_history, 0, hist_start, deltanet_conv_history_len_); + } else { + size_t pad_rows = deltanet_conv_history_len_ - total_history_len; + size_t pad = gb->input({pad_rows, mixed_proj_dim}, Precision::FP16); + std::vector<__fp16> zeros(pad_rows * mixed_proj_dim, static_cast<__fp16>(0.0f)); + gb->set_input(pad, zeros.data(), Precision::FP16); + history_2d = gb->concat(pad, conv_source_with_history, 0); + } + } else if (prev_conv_flat != 0) { + history_2d = gb->reshape(prev_conv_flat, {deltanet_conv_history_len_, mixed_proj_dim}); + history_2d = gb->precision_cast(history_2d, Precision::FP16); + } else { + history_2d = gb->input({deltanet_conv_history_len_, mixed_proj_dim}, Precision::FP16); + std::vector<__fp16> zeros(deltanet_conv_flat_dim_, static_cast<__fp16>(0.0f)); + gb->set_input(history_2d, zeros.data(), Precision::FP16); + } + + size_t history_flat = gb->reshape(history_2d, {1, deltanet_conv_flat_dim_}); + packed_cache = gb->concat(packed_cache, history_flat, 1); + } + packed_cache = gb->precision_cast(packed_cache, conv_cache_.precision); + conv_cache_state_nodes_[layer_idx] = packed_cache; + cache_k_output_nodes_[layer_idx] = 0; + cache_v_output_nodes_[layer_idx] = 0; + } else { + conv_cache_state_nodes_[layer_idx] = 0; + } + + size_t y_2d = gb->reshape(y_4d, {seq_len * num_heads, value_dim}); + if (z_proj != 0) { + size_t z_3d = gb->reshape(z_proj, {seq_len, num_heads, value_dim}); + size_t z_2d = gb->reshape(z_3d, {seq_len * num_heads, value_dim}); + y_2d = gb->multiply(gb->rms_norm(y_2d, layer.attn_q_norm_weight, config_.layer_norm_eps), gb->silu(z_2d)); + } + y_2d = gb->reshape(y_2d, {seq_len, num_heads * value_dim}); + return gb->matmul(y_2d, layer.attn_output_weight, true, backend); +} + +size_t Qwen3p5Model::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer_entry = weight_nodes_.layers[layer_idx]; + const auto& layer = layer_entry.weights; + + if (layer_entry.type == WeightNodeIDs::LayerType::DELTANET) { + return build_gated_deltanet(gb, normalized_input, layer_idx, backend, use_cache, position_offset); + } + + auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); + auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); + auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); + + const auto& q_shape = gb->get_output_buffer(q_proj).shape; + size_t batch_seq = q_shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + const size_t q_target_dim = num_heads * head_dim; + + size_t q_gate = 0; + const size_t q_proj_dim = q_shape.size() > 1 ? q_shape[1] : 0; + if (q_proj_dim == 2 * q_target_dim) { + size_t q_packed = gb->reshape(q_proj, {batch_seq, num_heads, 2 * head_dim}); + q_gate = gb->slice(q_packed, 2, head_dim, head_dim); + q_proj = gb->slice(q_packed, 2, 0, head_dim); + q_gate = gb->reshape(q_gate, {batch_seq, q_target_dim}); + q_proj = gb->reshape(q_proj, {batch_seq, q_target_dim}); + } + + const size_t q_norm_weight = gb->scalar_add(layer.attn_q_norm_weight, 1.0f); + q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); + q_proj = gb->rms_norm(q_proj, q_norm_weight, config_.layer_norm_eps); + q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); + + size_t num_kv_heads = config_.attention_kv_heads; + const size_t k_norm_weight = gb->scalar_add(layer.attn_k_norm_weight, 1.0f); + k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); + k_proj = gb->rms_norm(k_proj, k_norm_weight, config_.layer_norm_eps); + k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); + + size_t seq_len = batch_seq; + + auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); + auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); + + if (config_.rope_theta > 0) { + float rotary_factor = config_.partial_rotary_factor; + if (rotary_factor <= 0.0f) { + rotary_factor = 0.25f; + } + size_t rot_dim = static_cast(static_cast(head_dim) * rotary_factor); + rot_dim = std::min(rot_dim, head_dim); + if ((rot_dim & 1u) != 0) { + rot_dim -= 1; + } + + auto apply_partial_rope = [&](size_t x_4d) -> size_t { + if (rot_dim == 0) { + return x_4d; + } + if (rot_dim >= head_dim) { + return gb->rope(x_4d, config_.rope_theta, position_offset); + } + size_t x_rot = gb->slice(x_4d, 3, 0, rot_dim); + size_t x_tail = gb->slice(x_4d, 3, rot_dim, head_dim - rot_dim); + x_rot = gb->rope(x_rot, config_.rope_theta, position_offset); + return gb->concat(x_rot, x_tail, 3); + }; + + q_proj_4d = apply_partial_rope(q_proj_4d); + k_proj_4d = apply_partial_rope(k_proj_4d); + } + + size_t attn_output_4d; + + if (use_cache) { + cache_k_output_nodes_[layer_idx] = k_proj_4d; + cache_v_output_nodes_[layer_idx] = v_proj_4d; + conv_cache_state_nodes_[layer_idx] = 0; + } + + if (use_cache && !kv_cache_.is_empty()) { + attn_output_4d = gb->attention_int8_hybrid( + q_proj_4d, k_proj_4d, v_proj_4d, + attention_scale_, position_offset, + kv_cache_.get_keys_int8(layer_idx), + kv_cache_.get_values_int8(layer_idx), + kv_cache_.get_key_scales(layer_idx), + kv_cache_.get_value_scales(layer_idx), + kv_cache_.current_seq_len, num_kv_heads, head_dim + ); + } else { + attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); + } + + auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); + if (q_gate != 0) { + attn_output = gb->multiply(attn_output, gb->sigmoid(q_gate)); + } + return gb->matmul(attn_output, layer.attn_output_weight, true, backend); +} + +size_t Qwen3p5Model::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx].weights; + size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); + size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); + size_t gate_silu = gb->silu(gate_output); + size_t gated = gb->multiply(gate_silu, up_output); + return gb->matmul(gated, layer.ffn_down_weight, true, backend); +} + +size_t Qwen3p5Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx].weights; + const size_t input_norm_weight = gb->scalar_add(layer.input_layernorm_weight, 1.0f); + auto normalized_input = gb->rms_norm(hidden, input_norm_weight, config_.layer_norm_eps); + auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); + auto after_attention = gb->add(hidden, attn_output); + const size_t post_attn_norm_weight = gb->scalar_add(layer.post_attention_layernorm_weight, 1.0f); + auto normalized_after_attention = gb->rms_norm(after_attention, post_attn_norm_weight, config_.layer_norm_eps); + auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); + return gb->add(after_attention, mlp_output); +} + +size_t Qwen3p5Model::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + + if (tokens.empty()) { + throw std::runtime_error("Token sequence cannot be empty"); + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + if (conv_cache_state_nodes_.size() != config_.num_layers) { + conv_cache_state_nodes_.assign(config_.num_layers, 0); + } + std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); + last_forward_used_cache_ = use_cache; + if (!use_cache && conv_cache_.window_size > 0) { + conv_cache_.reset(); + deltanet_total_seq_len_ = 0; + } + + auto seq_len = static_cast(tokens.size()); + const size_t kv_pos = use_cache ? kv_cache_.get_total_seq_len() : 0; + const size_t position_offset = use_cache ? std::max(kv_pos, deltanet_total_seq_len_) : 0; + + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + auto input_node_id = gb->input({seq_len}, Precision::FP32); + auto hidden = gb->embedding(embedding_node_id_, input_node_id); + + std::vector input_data(seq_len); + for (size_t i = 0; i < seq_len; i++) { + input_data[i] = static_cast(tokens[i]); + } + gb->set_input(input_node_id, input_data.data(), Precision::FP32); + + static std::set skip_layers = {}; + for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { + if (skip_layers.count(layer_idx)) { + continue; + } + hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); + } + + const size_t final_norm_weight = gb->scalar_add(weight_nodes_.output_norm_weight, 1.0f); + auto final_hidden = gb->rms_norm(hidden, final_norm_weight, config_.layer_norm_eps); + return final_hidden; +} + +} +} diff --git a/cactus-transpile/models/model_siglip2.cpp b/cactus-transpile/models/model_siglip2.cpp new file mode 100644 index 000000000..49c3c24b0 --- /dev/null +++ b/cactus-transpile/models/model_siglip2.cpp @@ -0,0 +1,387 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../kernel/kernel.h" +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +Siglip2VisionModel::Siglip2VisionModel() : Model() { + config_.model_type = Config::ModelType::SIGLIP2; +} + +Siglip2VisionModel::Siglip2VisionModel(const Config& cfg) : Model(cfg) { + Siglip2Preprocessor::Config preprocessor_config; + preprocessor_config.patch_size = static_cast(config_.vision_patch_size); + preprocessor_config.downsample_factor = static_cast(config_.downsample_factor); + preprocessor_config.min_tiles = static_cast(config_.min_tiles); + preprocessor_config.max_tiles = static_cast(config_.max_tiles); + preprocessor_config.use_thumbnail = config_.use_thumbnail; + preprocessor_config.min_image_tokens = static_cast(config_.min_image_tokens); + preprocessor_config.max_image_tokens = static_cast(config_.max_image_tokens); + preprocessor_config.max_num_patches = static_cast(config_.max_num_patches); + preprocessor_config.tile_size = static_cast(config_.tile_size); + preprocessor_config.max_pixels_tolerance = config_.max_pixels_tolerance; + preprocessor_config.do_resize = true; + preprocessor_config.do_rescale = true; + preprocessor_config.do_normalize = true; + preprocessor_config.do_convert_rgb = true; + preprocessor_config.do_image_splitting = config_.do_image_splitting; + preprocessor_config.rescale_factor = config_.rescale_factor; + preprocessor_config.image_mean[0] = config_.image_mean; + preprocessor_config.image_mean[1] = config_.image_mean; + preprocessor_config.image_mean[2] = config_.image_mean; + preprocessor_config.image_std[0] = config_.image_std; + preprocessor_config.image_std[1] = config_.image_std; + preprocessor_config.image_std[2] = config_.image_std; + + preprocessor_ = Siglip2Preprocessor(preprocessor_config); +} + +void Siglip2VisionModel::load_weights_to_graph(CactusGraph* gb) { + std::string base = model_folder_path_ + "/"; + + if (npu::is_npu_available()) { + std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; + npu_encoder_ = npu::create_encoder(); + if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { + use_npu_encoder_ = true; + + std::vector typical_input_shape = { + static_cast(config_.max_num_patches), + static_cast(config_.vision_embed_dim) + }; + npu_encoder_->preallocate(typical_input_shape, "x", ""); + } else { + use_npu_encoder_ = false; + npu_encoder_.reset(); + } + } + + // Always load patch embedding and position embedding weights (needed for both CPU and NPU paths) + vision_weight_nodes_.patch_embedding_weight = gb->mmap_weights(base + "vision_patch_embedding.weights"); + vision_weight_nodes_.patch_embedding_bias = gb->mmap_weights(base + "vision_patch_embedding.bias.weights"); + vision_weight_nodes_.position_embedding = gb->mmap_weights(base + "vision_position_embedding.weights"); + + if (!use_npu_encoder_) { + vision_weight_nodes_.vision_layers.resize(config_.vision_num_layers); + + vision_weight_nodes_.post_layernorm_weight = gb->mmap_weights(base + "vision_post_layernorm.weights"); + vision_weight_nodes_.post_layernorm_bias = gb->mmap_weights(base + "vision_post_layernorm.bias.weights"); + + for (uint32_t i = 0; i < vision_weight_nodes_.vision_layers.size(); ++i) { + auto& layer = vision_weight_nodes_.vision_layers[i]; + std::string prefix = base + "vision_layer_" + std::to_string(i) + "_"; + + layer.attn_q_weight = gb->mmap_weights(prefix + "self_attn_q.weights"); + layer.attn_q_bias = gb->mmap_weights(prefix + "self_attn_q.bias.weights"); + layer.attn_k_weight = gb->mmap_weights(prefix + "self_attn_k.weights"); + layer.attn_k_bias = gb->mmap_weights(prefix + "self_attn_k.bias.weights"); + layer.attn_v_weight = gb->mmap_weights(prefix + "self_attn_v.weights"); + layer.attn_v_bias = gb->mmap_weights(prefix + "self_attn_v.bias.weights"); + layer.attn_output_weight = gb->mmap_weights(prefix + "self_attn_out.weights"); + layer.attn_output_bias = gb->mmap_weights(prefix + "self_attn_out.bias.weights"); + + layer.layer_norm1_weight = gb->mmap_weights(prefix + "layer_norm1.weights"); + layer.layer_norm1_bias = gb->mmap_weights(prefix + "layer_norm1.bias.weights"); + layer.layer_norm2_weight = gb->mmap_weights(prefix + "layer_norm2.weights"); + layer.layer_norm2_bias = gb->mmap_weights(prefix + "layer_norm2.bias.weights"); + + layer.mlp_fc1_weight = gb->mmap_weights(prefix + "ffn_fc1.weights"); + layer.mlp_fc1_bias = gb->mmap_weights(prefix + "ffn_fc1.bias.weights"); + layer.mlp_fc2_weight = gb->mmap_weights(prefix + "ffn_fc2.weights"); + layer.mlp_fc2_bias = gb->mmap_weights(prefix + "ffn_fc2.bias.weights"); + } + } +} + +Siglip2VisionModel::VisionEmbeddingResult Siglip2VisionModel::build_vision_embeddings( + CactusGraph* gb, + const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, + ComputeBackend backend) { + const int num_tiles = preprocessed_image.num_tiles; + const int max_patches = preprocessed_image.max_patches_per_tile; + const int patch_dim = preprocessed_image.patch_dim; + + const size_t expected_size = static_cast(num_tiles) * static_cast(max_patches) * + static_cast(patch_dim); + if (preprocessed_image.pixel_values.size() != expected_size) { + throw std::runtime_error( + "Pixel values size mismatch: expected " + std::to_string(expected_size) + + " (tiles=" + std::to_string(num_tiles) + " * max_patches=" + std::to_string(max_patches) + + " * patch_dim=" + std::to_string(patch_dim) + ") but got " + + std::to_string(preprocessed_image.pixel_values.size())); + } + for (size_t i = 0; i < std::min(100, preprocessed_image.pixel_values.size()); ++i) { + float val = preprocessed_image.pixel_values[i]; + if (std::isnan(val) || std::isinf(val)) { + throw std::runtime_error( + "Invalid value in pixel_values at index " + std::to_string(i) + ": " + std::to_string(val)); + } + } + + size_t reshaped_weight = gb->reshape( + vision_weight_nodes_.patch_embedding_weight, + {static_cast(config_.vision_embed_dim), static_cast(patch_dim)}); + + size_t patch_bias = vision_weight_nodes_.patch_embedding_bias; + std::vector tile_embeddings; + tile_embeddings.reserve(static_cast(num_tiles)); + + for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { + const auto& shape = preprocessed_image.spatial_shapes[tile_idx]; + const int tile_h = shape.first; + const int tile_w = shape.second; + const int actual_patches = tile_h * tile_w; + if (actual_patches <= 0) { + continue; + } + + const float* tile_data = preprocessed_image.pixel_values.data() + + static_cast(tile_idx) * static_cast(max_patches) * + static_cast(patch_dim); + + size_t tile_input_fp32 = gb->input( + {static_cast(actual_patches), static_cast(patch_dim)}, Precision::FP32); + gb->set_input(tile_input_fp32, tile_data, Precision::FP32); + size_t tile_input = gb->precision_cast(tile_input_fp32, Precision::FP16); + size_t tile_patch = gb->matmul(tile_input, reshaped_weight, true, backend); + size_t tile_bias = gb->add(tile_patch, patch_bias); + size_t tile_pos = gb->bilinear_interpolation( + vision_weight_nodes_.position_embedding, + static_cast(tile_h), + static_cast(tile_w), + /*align_corners=*/false); + size_t tile_pos_cast = gb->precision_cast(tile_pos, Precision::FP16); + size_t tile_embed = gb->add(tile_bias, tile_pos_cast); + tile_embeddings.push_back(tile_embed); + } + + if (tile_embeddings.empty()) { + throw std::runtime_error("No valid tiles produced embeddings in build_vision_embeddings"); + } + auto concat_nodes = [&](const std::vector& nodes) { + if (nodes.empty()) { + throw std::runtime_error("Attempted to concatenate an empty node list"); + } + size_t combined = nodes.front(); + for (size_t i = 1; i < nodes.size(); ++i) { + combined = gb->concat(combined, nodes[i], /*axis=*/0); + } + return combined; + }; + + size_t embeddings = concat_nodes(tile_embeddings); + return VisionEmbeddingResult{embeddings, std::move(tile_embeddings)}; +} + +size_t Siglip2VisionModel::build_vision_attention(CactusGraph* gb, size_t hidden_states, + uint32_t layer_idx, ComputeBackend backend) { + const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; + + size_t q = gb->matmul(hidden_states, layer.attn_q_weight, true, backend); + q = gb->add(q, layer.attn_q_bias); + size_t k = gb->matmul(hidden_states, layer.attn_k_weight, true, backend); + k = gb->add(k, layer.attn_k_bias); + size_t v = gb->matmul(hidden_states, layer.attn_v_weight, true, backend); + v = gb->add(v, layer.attn_v_bias); + const size_t num_heads = static_cast(config_.vision_attention_heads); + const size_t head_dim = static_cast(config_.vision_embed_dim / config_.vision_attention_heads); + const auto& q_buf = gb->get_output_buffer(q); + size_t seq_len = q_buf.shape[0]; + + size_t q_4d = gb->reshape(q, {1, seq_len, num_heads, head_dim}); + size_t k_4d = gb->reshape(k, {1, seq_len, num_heads, head_dim}); + size_t v_4d = gb->reshape(v, {1, seq_len, num_heads, head_dim}); + + float scale = 1.0f / std::sqrt(static_cast(head_dim)); + size_t attn_output = gb->attention(q_4d, k_4d, v_4d, scale, false, backend); + size_t attn_2d = gb->reshape(attn_output, {seq_len, num_heads * head_dim}); + size_t output = gb->matmul(attn_2d, layer.attn_output_weight, true, backend); + output = gb->add(output, layer.attn_output_bias); + return output; +} + +size_t Siglip2VisionModel::build_vision_mlp(CactusGraph* gb, size_t hidden_states, + uint32_t layer_idx, ComputeBackend backend) { + const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; + + size_t fc1_output = gb->matmul(hidden_states, layer.mlp_fc1_weight, true, backend); + fc1_output = gb->add(fc1_output, layer.mlp_fc1_bias); + size_t activated = gb->gelu(fc1_output); + size_t fc2_output = gb->matmul(activated, layer.mlp_fc2_weight, true, backend); + fc2_output = gb->add(fc2_output, layer.mlp_fc2_bias); + return fc2_output; +} + +size_t Siglip2VisionModel::build_vision_transformer_layer(CactusGraph* gb, size_t hidden_states, + uint32_t layer_idx, ComputeBackend backend) { + const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; + + size_t residual = hidden_states; + size_t normalized = gb->layernorm(hidden_states, layer.layer_norm1_weight, + layer.layer_norm1_bias, config_.layer_norm_eps); + size_t attn_output = build_vision_attention(gb, normalized, layer_idx, backend); + hidden_states = gb->add(residual, attn_output); + residual = hidden_states; + normalized = gb->layernorm(hidden_states, layer.layer_norm2_weight, + layer.layer_norm2_bias, config_.layer_norm_eps); + size_t mlp_output = build_vision_mlp(gb, normalized, layer_idx, backend); + hidden_states = gb->add(residual, mlp_output); + return hidden_states; +} + +size_t Siglip2VisionModel::forward_vision( + CactusGraph* gb, + const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, + ComputeBackend backend) { + + if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { + // NPU path: build patch embeddings + position embeddings on CPU, then run transformer on NPU + auto embedding_result = build_vision_embeddings(gb, preprocessed_image, backend); + + size_t total_patches = 0; + for (const auto& shape : preprocessed_image.spatial_shapes) { + total_patches += shape.first * shape.second; + } + + gb->execute(); + + const auto& embed_buffer = gb->get_output_buffer(embedding_result.combined_embeddings); + void* embed_ptr = gb->get_output(embedding_result.combined_embeddings); + + std::vector<__fp16> embed_f16; + const __fp16* input_ptr; + if (embed_buffer.precision == Precision::FP16) { + input_ptr = static_cast(embed_ptr); + } else { + embed_f16.resize(total_patches * config_.vision_embed_dim); + cactus_fp32_to_fp16(static_cast(embed_ptr), + embed_f16.data(), embed_f16.size()); + input_ptr = embed_f16.data(); + } + + std::vector input_shape = { + static_cast(total_patches), + static_cast(config_.vision_embed_dim) + }; + + __fp16* output_buffer = npu_encoder_->get_output_buffer(); + if (output_buffer) { + size_t elements = npu_encoder_->encode( + input_ptr, output_buffer, input_shape, "x", ""); + + if (elements > 0) { + gb->soft_reset(); + size_t vision_output = gb->input({total_patches, config_.vision_embed_dim}, + Precision::FP16); + gb->set_external_input(vision_output, output_buffer, Precision::FP16); + return vision_output; + } + } else { + std::vector<__fp16> npu_output(total_patches * config_.vision_embed_dim); + size_t elements = npu_encoder_->encode( + input_ptr, npu_output.data(), input_shape, "x", ""); + + if (elements > 0) { + gb->soft_reset(); + size_t vision_output = gb->input({total_patches, config_.vision_embed_dim}, + Precision::FP16); + gb->set_input(vision_output, npu_output.data(), Precision::FP16); + return vision_output; + } + } + + throw std::runtime_error("NPU encoder failed"); + } + + // CPU path: full forward pass through transformer layers + auto embedding_result = build_vision_embeddings(gb, preprocessed_image, backend); + + auto concat_nodes = [&](const std::vector& nodes) { + if (nodes.empty()) { + throw std::runtime_error("Attempted to concatenate an empty node list in forward_vision"); + } + size_t combined = nodes.front(); + for (size_t i = 1; i < nodes.size(); ++i) { + combined = gb->concat(combined, nodes[i], /*axis=*/0); + } + return combined; + }; + + std::vector tile_outputs; + tile_outputs.reserve(embedding_result.tile_embeddings.size()); + + for (size_t tile_idx = 0; tile_idx < embedding_result.tile_embeddings.size(); ++tile_idx) { + size_t hidden_states = embedding_result.tile_embeddings[tile_idx]; + + for (uint32_t layer_idx = 0; layer_idx < config_.vision_num_layers; ++layer_idx) { + hidden_states = build_vision_transformer_layer(gb, hidden_states, layer_idx, backend); + } + + hidden_states = gb->layernorm(hidden_states, + vision_weight_nodes_.post_layernorm_weight, + vision_weight_nodes_.post_layernorm_bias, + config_.layer_norm_eps); + tile_outputs.push_back(hidden_states); + } + + if (tile_outputs.empty()) { + throw std::runtime_error("No tile outputs generated in forward_vision"); + } + + size_t combined_output = concat_nodes(tile_outputs); + + return combined_output; +} + +size_t Siglip2VisionModel::forward_vision(const Siglip2Preprocessor::PreprocessedImage& preprocessed_image) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; + return forward_vision(gb, preprocessed_image, backend); +} + +std::vector Siglip2VisionModel::get_image_embedding(const std::string& image_path) { + auto preprocessed = preprocessor_.preprocess_from_file(image_path); + size_t last_hidden_state = forward_vision(preprocessed); + + auto* gb = static_cast(graph_handle_); + size_t pooled = gb->mean(last_hidden_state, 0); + gb->execute(); + + const auto& output_buf = gb->get_output_buffer(pooled); + size_t hidden_dim = output_buf.total_size; + + std::vector embedding(hidden_dim); + void* output_data = gb->get_output(pooled); + const float* output_ptr = static_cast(output_data); + std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); + return embedding; +} + +size_t Siglip2VisionModel::forward(const std::vector&, bool) {return 0;} + +size_t Siglip2VisionModel::build_attention(CactusGraph*, size_t, uint32_t, + ComputeBackend, bool, size_t) { + return 0; +} + +size_t Siglip2VisionModel::build_mlp(CactusGraph*, size_t, uint32_t, + ComputeBackend) const { + return 0; +} + +size_t Siglip2VisionModel::build_transformer_block(CactusGraph*, size_t, uint32_t, + ComputeBackend, bool, size_t) { + return 0; +} + +} +} diff --git a/cactus-transpile/models/model_silero_vad.cpp b/cactus-transpile/models/model_silero_vad.cpp new file mode 100644 index 000000000..7573ac2de --- /dev/null +++ b/cactus-transpile/models/model_silero_vad.cpp @@ -0,0 +1,338 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../kernel/kernel.h" +#include +#include + +namespace cactus { +namespace engine { + +SileroVADModel::SileroVADModel() : Model() { + weight_nodes_.encoder_blocks.resize(4); +} + +SileroVADModel::SileroVADModel(const Config& config) : Model(config) { + weight_nodes_.encoder_blocks.resize(4); +} + +SileroVADModel::~SileroVADModel() = default; + +void SileroVADModel::load_weights_to_graph(CactusGraph* gb) { + weight_nodes_.stft_basis = gb->mmap_weights(model_folder_path_ + "/stft_basis.weights"); + + for (uint32_t i = 0; i < 4; i++) { + std::string block_prefix = model_folder_path_ + "/encoder_block_" + std::to_string(i) + "_"; + weight_nodes_.encoder_blocks[i].conv_weight = gb->mmap_weights(block_prefix + "conv_weight.weights"); + weight_nodes_.encoder_blocks[i].conv_bias = gb->mmap_weights(block_prefix + "conv_bias.weights"); + } + + weight_nodes_.lstm_weight_ih = gb->mmap_weights(model_folder_path_ + "/lstm_weight_ih.weights"); + weight_nodes_.lstm_weight_hh = gb->mmap_weights(model_folder_path_ + "/lstm_weight_hh.weights"); + weight_nodes_.lstm_bias_ih = gb->mmap_weights(model_folder_path_ + "/lstm_bias_ih.weights"); + weight_nodes_.lstm_bias_hh = gb->mmap_weights(model_folder_path_ + "/lstm_bias_hh.weights"); + + weight_nodes_.output_conv_weight = gb->mmap_weights(model_folder_path_ + "/output_conv_weight.weights"); + weight_nodes_.output_conv_bias = gb->mmap_weights(model_folder_path_ + "/output_conv_bias.weights"); +} + +void SileroVADModel::build_graph() { + const size_t input_size = CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE; + + graph_nodes_.input = graph_.input({1, 1, input_size}, Precision::FP16); + graph_nodes_.h_prev = graph_.input({1, HIDDEN_SIZE}, Precision::FP16); + graph_nodes_.c_prev = graph_.input({1, HIDDEN_SIZE}, Precision::FP16); + + auto cplx = graph_.stft(graph_nodes_.input, weight_nodes_.stft_basis, 128, 129); + auto re = graph_.slice(cplx, 1, 0, 129); + auto im = graph_.slice(cplx, 1, 129, 129); + auto re_sq = graph_.multiply(re, re); + auto im_sq = graph_.multiply(im, im); + auto mag = graph_.scalar_sqrt(graph_.add(re_sq, im_sq)); + + const size_t strides[4] = {1, 2, 2, 1}; + auto x = mag; + for (uint32_t i = 0; i < 4; i++) { + auto conv = graph_.conv1d_k3(x, weight_nodes_.encoder_blocks[i].conv_weight, strides[i]); + auto bias_reshaped = graph_.reshape(weight_nodes_.encoder_blocks[i].conv_bias, + {1, graph_.get_output_buffer(weight_nodes_.encoder_blocks[i].conv_bias).shape[0], 1}); + auto with_bias = graph_.add(conv, bias_reshaped); + x = graph_.relu(with_bias); + } + + auto x_squeezed = graph_.reshape(x, {1, HIDDEN_SIZE}); + + graph_nodes_.lstm_output = graph_.lstm_cell( + x_squeezed, + graph_nodes_.h_prev, + graph_nodes_.c_prev, + weight_nodes_.lstm_weight_ih, + weight_nodes_.lstm_weight_hh, + weight_nodes_.lstm_bias_ih, + weight_nodes_.lstm_bias_hh + ); + + auto h_new = graph_.slice(graph_nodes_.lstm_output, 2, 0, 1); + auto c_new = graph_.slice(graph_nodes_.lstm_output, 2, 1, 1); + graph_nodes_.h_new = graph_.reshape(h_new, {1, HIDDEN_SIZE}); + graph_nodes_.c_new = graph_.reshape(c_new, {1, HIDDEN_SIZE}); + + auto h_relu = graph_.relu(graph_nodes_.h_new); + auto h_unsqueezed = graph_.reshape(h_relu, {1, HIDDEN_SIZE, 1}); + auto logits = graph_.conv1d(h_unsqueezed, weight_nodes_.output_conv_weight, weight_nodes_.output_conv_bias, 1); + graph_nodes_.output = graph_.sigmoid(logits); +} + +bool SileroVADModel::init(const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup) { + (void)context_size; + (void)system_prompt; + (void)do_warmup; + + if (initialized_) { + return true; + } + + state_.h.resize(HIDDEN_SIZE, (__fp16)0.0f); + state_.c.resize(HIDDEN_SIZE, (__fp16)0.0f); + state_.context.resize(CONTEXT_SIZE, 0.0f); + state_.input_buf.resize(CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE, 0.0f); + state_.input_fp16.resize(CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE); + + model_folder_path_ = model_folder; + + try { + load_weights_to_graph(&graph_); + build_graph(); + initialized_ = true; + return true; + } catch (const std::exception& e) { + initialized_ = false; + return false; + } +} + +float SileroVADModel::process_chunk(const std::vector& audio_chunk) { + const float* audio = audio_chunk.data(); + size_t samples = audio_chunk.size(); + + if (samples != CHUNK_SIZE) { + throw std::runtime_error("SileroVAD: Expected 512 samples"); + } + if (!initialized_) { + throw std::runtime_error("SileroVAD: Not initialized"); + } + + const size_t input_size = CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE; + + float* input_buf = state_.input_buf.data(); + __fp16* input_fp16 = state_.input_fp16.data(); + + std::memcpy(input_buf, state_.context.data(), CONTEXT_SIZE * sizeof(float)); + std::memcpy(input_buf + CONTEXT_SIZE, audio, CHUNK_SIZE * sizeof(float)); + + for (size_t i = 0; i < REFLECT_PAD_SIZE; i++) { + input_buf[CONTEXT_SIZE + CHUNK_SIZE + i] = + input_buf[CONTEXT_SIZE + CHUNK_SIZE - 2 - i]; + } + + cactus_fp32_to_fp16(input_buf, input_fp16, input_size); + + graph_.set_input(graph_nodes_.input, input_fp16, Precision::FP16); + graph_.set_input(graph_nodes_.h_prev, state_.h.data(), Precision::FP16); + graph_.set_input(graph_nodes_.c_prev, state_.c.data(), Precision::FP16); + graph_.execute(); + + void* h_out = graph_.get_output(graph_nodes_.h_new); + void* c_out = graph_.get_output(graph_nodes_.c_new); + void* vad_out = graph_.get_output(graph_nodes_.output); + + const __fp16* h_new = static_cast(h_out); + const __fp16* c_new = static_cast(c_out); + const __fp16* vad_score = static_cast(vad_out); + + std::memcpy(state_.h.data(), h_new, HIDDEN_SIZE * sizeof(__fp16)); + std::memcpy(state_.c.data(), c_new, HIDDEN_SIZE * sizeof(__fp16)); + + float vad_score_f32 = static_cast(vad_score[0]); + + std::memcpy(state_.context.data(), input_buf + CHUNK_SIZE, CONTEXT_SIZE * sizeof(float)); + + return vad_score_f32; +} + +void SileroVADModel::reset_states() { + state_.h.assign(HIDDEN_SIZE, (__fp16)0.0f); + state_.c.assign(HIDDEN_SIZE, (__fp16)0.0f); + state_.context.assign(CONTEXT_SIZE, 0.0f); +} + +std::vector SileroVADModel::get_speech_timestamps(const std::vector& audio, const SpeechTimestampsOptions& options) { + const size_t audio_length_samples = audio.size(); + const size_t window_size_samples = options.window_size_samples; + const float min_speech_samples = options.sampling_rate * options.min_speech_duration_ms / 1000.0f; + const float speech_pad_samples = options.sampling_rate * options.speech_pad_ms / 1000.0f; + const float max_speech_samples = options.sampling_rate * options.max_speech_duration_s - window_size_samples - 2 * speech_pad_samples; + const float min_silence_samples = options.sampling_rate * options.min_silence_duration_ms / 1000.0f; + const float neg_threshold = (options.neg_threshold == 0.0f) + ? std::max(options.threshold - 0.15f, 0.01f) + : options.neg_threshold; + reset_states(); + + std::vector speech_probs; + speech_probs.reserve(audio_length_samples / window_size_samples + 1); + + std::vector chunk(window_size_samples, 0.0f); + + for (size_t current_start = 0; current_start < audio_length_samples; current_start += window_size_samples) { + size_t remaining = audio_length_samples - current_start; + size_t copy_len = std::min(remaining, window_size_samples); + + std::memcpy(chunk.data(), audio.data() + current_start, copy_len * sizeof(float)); + + if (copy_len < window_size_samples) { + std::memset(chunk.data() + copy_len, 0, (window_size_samples - copy_len) * sizeof(float)); + } + + float speech_prob = process_chunk(chunk); + speech_probs.push_back(speech_prob); + } + + bool triggered = false; + std::vector speeches; + SpeechTimestamp current_speech = {0, 0}; + size_t temp_end = 0; + size_t prev_end = 0; + size_t next_start = 0; + std::vector> possible_ends; + + const float min_silence_samples_at_max_speech = options.sampling_rate * options.min_silence_at_max_speech / 1000.0f; + + for (size_t i = 0; i < speech_probs.size(); ++i) { + float speech_prob = speech_probs[i]; + size_t cur_sample = window_size_samples * i; + + if (speech_prob >= options.threshold && temp_end) { + size_t sil_dur = cur_sample - temp_end; + if (sil_dur > min_silence_samples_at_max_speech) { + possible_ends.push_back({temp_end, sil_dur}); + } + temp_end = 0; + if (next_start < prev_end) { + next_start = cur_sample; + } + } + + if (speech_prob >= options.threshold && !triggered) { + triggered = true; + current_speech.start = cur_sample; + continue; + } + + if (triggered && (cur_sample - current_speech.start > max_speech_samples)) { + if (options.use_max_poss_sil_at_max_speech && !possible_ends.empty()) { + auto max_silence = std::max_element(possible_ends.begin(), possible_ends.end(), + [](const std::pair& a, const std::pair& b) { + return a.second < b.second; + }); + prev_end = max_silence->first; + size_t dur = max_silence->second; + current_speech.end = prev_end; + speeches.push_back(current_speech); + current_speech = {0, 0}; + next_start = prev_end + dur; + + if (next_start < prev_end + cur_sample) { + current_speech.start = next_start; + } else { + triggered = false; + } + prev_end = next_start = temp_end = 0; + possible_ends.clear(); + } else { + if (prev_end) { + current_speech.end = prev_end; + speeches.push_back(current_speech); + current_speech = {0, 0}; + if (next_start < prev_end) { + triggered = false; + } else { + current_speech.start = next_start; + } + prev_end = next_start = temp_end = 0; + possible_ends.clear(); + } else { + current_speech.end = cur_sample; + speeches.push_back(current_speech); + current_speech = {0, 0}; + prev_end = next_start = temp_end = 0; + triggered = false; + possible_ends.clear(); + continue; + } + } + } + + if (speech_prob < neg_threshold && triggered) { + if (!temp_end) { + temp_end = cur_sample; + } + size_t sil_dur_now = cur_sample - temp_end; + + if (!options.use_max_poss_sil_at_max_speech && sil_dur_now > min_silence_samples_at_max_speech) { + prev_end = temp_end; + } + + if (sil_dur_now < min_silence_samples) { + continue; + } else { + current_speech.end = temp_end; + if ((current_speech.end - current_speech.start) > min_speech_samples) { + speeches.push_back(current_speech); + } + current_speech = {0, 0}; + prev_end = next_start = temp_end = 0; + triggered = false; + possible_ends.clear(); + continue; + } + } + } + + if (triggered && (audio_length_samples - current_speech.start) > min_speech_samples) { + current_speech.end = audio_length_samples; + speeches.push_back(current_speech); + } + + for (size_t i = 0; i < speeches.size(); ++i) { + if (i == 0) { + speeches[i].start = speeches[i].start > static_cast(speech_pad_samples) + ? speeches[i].start - static_cast(speech_pad_samples) + : 0; + } + if (i != speeches.size() - 1) { + size_t silence_duration = speeches[i + 1].start - speeches[i].end; + if (silence_duration < 2 * static_cast(speech_pad_samples)) { + speeches[i].end += silence_duration / 2; + speeches[i + 1].start = speeches[i + 1].start > silence_duration / 2 + ? speeches[i + 1].start - silence_duration / 2 + : 0; + } else { + speeches[i].end = std::min(audio_length_samples, speeches[i].end + static_cast(speech_pad_samples)); + speeches[i + 1].start = speeches[i + 1].start > static_cast(speech_pad_samples) + ? speeches[i + 1].start - static_cast(speech_pad_samples) + : 0; + } + } else { + speeches[i].end = std::min(audio_length_samples, speeches[i].end + static_cast(speech_pad_samples)); + } + } + + return speeches; +} + +size_t SileroVADModel::forward(const std::vector&, const std::vector&, bool) { + return 0; +} + +} +} diff --git a/cactus-transpile/models/model_wespeaker.cpp b/cactus-transpile/models/model_wespeaker.cpp new file mode 100644 index 000000000..62d4642c8 --- /dev/null +++ b/cactus-transpile/models/model_wespeaker.cpp @@ -0,0 +1,204 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../kernel/kernel.h" +#include + +namespace cactus { +namespace engine { + +WeSpeakerModel::WeSpeakerModel() : Model() {} +WeSpeakerModel::WeSpeakerModel(const Config& config) : Model(config) {} + +WeSpeakerModel::ResBlockWeights WeSpeakerModel::load_resblock(CactusGraph* gb, const std::string& prefix, bool has_shortcut) { + ResBlockWeights rb; + rb.has_shortcut = has_shortcut; + rb.conv1_w = gb->mmap_weights(prefix + "_conv1_weight.weights"); + rb.conv1_b = gb->mmap_weights(prefix + "_conv1_bias.weights"); + rb.conv2_w = gb->mmap_weights(prefix + "_conv2_weight.weights"); + rb.conv2_b = gb->mmap_weights(prefix + "_conv2_bias.weights"); + if (has_shortcut) { + rb.shortcut_conv_w = gb->mmap_weights(prefix + "_shortcut_0_weight.weights"); + rb.shortcut_conv_b = gb->mmap_weights(prefix + "_shortcut_0_bias.weights"); + } + return rb; +} + +size_t WeSpeakerModel::build_resblock(CactusGraph* gb, size_t x, const ResBlockWeights& rb, bool stride2) { + size_t identity = x; + + size_t out = stride2 ? gb->conv2d_k3s2p1(x, rb.conv1_w, rb.conv1_b) + : gb->conv2d_k3s1p1(x, rb.conv1_w, rb.conv1_b); + out = gb->relu(out); + out = gb->conv2d_k3s1p1(out, rb.conv2_w, rb.conv2_b); + + if (rb.has_shortcut) { + identity = gb->conv2d_k3s2p1(x, rb.shortcut_conv_w, rb.shortcut_conv_b); + } + + out = gb->add(out, identity); + out = gb->relu(out); + return out; +} + +void WeSpeakerModel::load_weights_to_graph(CactusGraph* gb) { + const std::string& p = model_folder_path_; + + weight_nodes_.conv1_w = gb->mmap_weights(p + "/resnet_conv1_weight.weights"); + weight_nodes_.conv1_b = gb->mmap_weights(p + "/resnet_conv1_bias.weights"); + + auto load_layer = [&](const std::string& layer_name, int num_blocks, bool first_has_shortcut) { + std::vector blocks; + for (int i = 0; i < num_blocks; ++i) { + std::string prefix = p + "/resnet_" + layer_name + "_" + std::to_string(i); + blocks.push_back(load_resblock(gb, prefix, i == 0 && first_has_shortcut)); + } + return blocks; + }; + + weight_nodes_.layer1 = load_layer("layer1", 3, false); + weight_nodes_.layer2 = load_layer("layer2", 4, true); + weight_nodes_.layer3 = load_layer("layer3", 6, true); + weight_nodes_.layer4 = load_layer("layer4", 3, true); + + weight_nodes_.seg1_w = gb->mmap_weights(p + "/resnet_seg_1_weight.weights"); + weight_nodes_.seg1_b = gb->mmap_weights(p + "/resnet_seg_1_bias.weights"); +} + +size_t WeSpeakerModel::build_resnet_body(CactusGraph* gb, size_t x) { + x = gb->conv2d_k3s1p1(x, weight_nodes_.conv1_w, weight_nodes_.conv1_b); + x = gb->relu(x); + + for (auto& rb : weight_nodes_.layer1) x = build_resblock(gb, x, rb, false); + for (size_t i = 0; i < weight_nodes_.layer2.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer2[i], i == 0); + for (size_t i = 0; i < weight_nodes_.layer3.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer3[i], i == 0); + for (size_t i = 0; i < weight_nodes_.layer4.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer4[i], i == 0); + return x; +} + +void WeSpeakerModel::build_graph(size_t num_frames) { + audio_input_ = graph_.input({1, 1, 80, num_frames}, Precision::FP16); + + size_t x = build_resnet_body(&graph_, audio_input_); + + x = graph_.stats_pool(x); + x = graph_.add(graph_.matmul(x, weight_nodes_.seg1_w, true), weight_nodes_.seg1_b); + + output_node_ = x; +} + +void WeSpeakerModel::build_graph_masked(size_t num_frames) { + audio_input_ = graph_.input({1, 1, 80, num_frames}, Precision::FP16); + + size_t x = build_resnet_body(&graph_, audio_input_); + + const auto& post_resnet = graph_.get_output_buffer(x); + size_t T_post = post_resnet.shape.back(); + + mask_input_ = graph_.input({1, T_post}, Precision::FP32); + + x = graph_.weighted_stats_pool(x, mask_input_); + x = graph_.add(graph_.matmul(x, weight_nodes_.seg1_w, true), weight_nodes_.seg1_b); + + output_node_ = x; +} + +bool WeSpeakerModel::init(const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup) { + (void)context_size; + (void)system_prompt; + (void)do_warmup; + + if (initialized_) { + return true; + } + + model_folder_path_ = model_folder; + + try { + load_weights_to_graph(&graph_); + initialized_ = true; + return true; + } catch (const std::exception& e) { + initialized_ = false; + return false; + } +} + +std::vector WeSpeakerModel::embed(const float* fbank_features, size_t num_features) { + if (!initialized_) throw std::runtime_error("WeSpeaker model not initialized"); + + static constexpr size_t NUM_MEL = 80; + const size_t num_frames = num_features / NUM_MEL; + if (num_frames == 0) throw std::runtime_error("WeSpeaker: empty fbank input"); + + if (num_frames != current_num_frames_ || current_masked_) { + graph_.hard_reset(); + load_weights_to_graph(&graph_); + build_graph(num_frames); + current_num_frames_ = num_frames; + current_masked_ = false; + } + + input_buf_.resize(num_features); + for (size_t i = 0; i < num_features; ++i) + input_buf_[i] = static_cast<__fp16>(fbank_features[i]); + + graph_.set_input(audio_input_, input_buf_.data(), Precision::FP16); + graph_.execute(); + + const auto& out_buf = graph_.get_output_buffer(output_node_); + const __fp16* out_data = out_buf.data_as<__fp16>(); + size_t total = out_buf.total_size; + + std::vector result(total); + for (size_t i = 0; i < total; ++i) + result[i] = static_cast(out_data[i]); + return result; +} + +std::vector WeSpeakerModel::embed(const float* fbank_features, size_t num_features, + const float* mask_weights, size_t mask_num_frames) { + if (!initialized_) throw std::runtime_error("WeSpeaker model not initialized"); + + static constexpr size_t NUM_MEL = 80; + const size_t num_frames = num_features / NUM_MEL; + if (num_frames == 0) throw std::runtime_error("WeSpeaker: empty fbank input"); + + if (num_frames != current_num_frames_ || !current_masked_) { + graph_.hard_reset(); + load_weights_to_graph(&graph_); + build_graph_masked(num_frames); + current_num_frames_ = num_frames; + current_masked_ = true; + } + + input_buf_.resize(num_features); + for (size_t i = 0; i < num_features; ++i) + input_buf_[i] = static_cast<__fp16>(fbank_features[i]); + graph_.set_input(audio_input_, input_buf_.data(), Precision::FP16); + + const auto& mask_shape = graph_.get_output_buffer(mask_input_); + size_t T_post = mask_shape.shape.back(); + + mask_buf_.resize(T_post); + for (size_t t = 0; t < T_post; ++t) { + size_t src_t = (t * mask_num_frames) / T_post; + if (src_t >= mask_num_frames) src_t = mask_num_frames - 1; + mask_buf_[t] = mask_weights[src_t]; + } + graph_.set_input(mask_input_, mask_buf_.data(), Precision::FP32); + + graph_.execute(); + + const auto& out_buf = graph_.get_output_buffer(output_node_); + const __fp16* out_data = out_buf.data_as<__fp16>(); + size_t total = out_buf.total_size; + + std::vector result(total); + for (size_t i = 0; i < total; ++i) + result[i] = static_cast(out_data[i]); + return result; +} + +} +} diff --git a/cactus-transpile/models/model_whisper.cpp b/cactus-transpile/models/model_whisper.cpp new file mode 100644 index 000000000..70ea31c92 --- /dev/null +++ b/cactus-transpile/models/model_whisper.cpp @@ -0,0 +1,778 @@ +#include "model.h" +#include "../graph/graph.h" +#include "../npu/npu.h" +#include "../kernel/kernel.h" +#include +#include +#include +#include +#include +#include + +namespace { + +bool has_whisper_cpu_encoder_weights(const std::string& model_folder_path) { + const std::filesystem::path model_path(model_folder_path); + return std::filesystem::exists(model_path / "encoder_conv1_weight.weights") && + std::filesystem::exists(model_path / "encoder_position_embeddings.weights"); +} + +} // namespace + +namespace cactus { +namespace engine { + +WhisperModel::WhisperModel() : Model() {} + +WhisperModel::WhisperModel(const Config& config) : Model(config) { + enc_layers_ = config.num_encoder_layers > 0 ? config.num_encoder_layers : config.num_layers; + dec_layers_ = config.num_decoder_layers > 0 ? config.num_decoder_layers : config.num_layers; + + weight_nodes_.layers.resize(std::max(enc_layers_, dec_layers_)); + + float hd = static_cast(config.attention_head_dim); + if (hd <= 0.0f) { + hd = 64.0f; + } + + attention_scale_ = 1.0f / std::sqrt(hd); + + encoder_block_out_nodes_.resize(enc_layers_, 0); + encoder_k_persistent_.assign(enc_layers_, 0); + encoder_v_persistent_.assign(enc_layers_, 0); + + const float neg_inf = -std::numeric_limits::infinity(); + for (size_t tok : suppress_tokens_) { + suppress_bias_[static_cast(tok)] = neg_inf; + } + suppress_bias_first_step_ = suppress_bias_; + for (size_t tok : begin_suppress_tokens_) { + suppress_bias_first_step_[static_cast(tok)] = neg_inf; + } +} + +void WhisperModel::load_weights_to_graph(CactusGraph* gb) { + + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + + weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/decoder_norm.weights"); + weight_nodes_.decoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/decoder_norm.bias"); + weight_nodes_.decoder_position_embeddings_weight = gb->mmap_weights(model_folder_path_ + "/decoder_position_embeddings.weights"); + + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + } + + if (npu::is_npu_available()) { + std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; + npu_encoder_ = npu::create_encoder(); + if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { + use_npu_encoder_ = true; + + std::vector typical_input_shape = {1, static_cast(config_.num_mel_bins), 3000}; + npu_encoder_->preallocate(typical_input_shape, "x", ""); + } else { + use_npu_encoder_ = false; + npu_encoder_.reset(); + } + } + + const bool has_cpu_encoder_weights = has_whisper_cpu_encoder_weights(model_folder_path_); + if (has_cpu_encoder_weights) { + weight_nodes_.encoder_position_embeddings = gb->mmap_weights(model_folder_path_ + "/encoder_position_embeddings.weights"); + weight_nodes_.encoder_conv1_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_weight.weights"); + weight_nodes_.encoder_conv1_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_bias.bias"); + weight_nodes_.encoder_conv2_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_weight.weights"); + weight_nodes_.encoder_conv2_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_bias.bias"); + weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_norm_weight.weights"); + weight_nodes_.encoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/encoder_norm_bias.bias"); + } + + const uint32_t max_layers = std::max(enc_layers_, dec_layers_); + for (uint32_t i = 0; i < max_layers; i++) { + auto& layer = weight_nodes_.layers[i]; + + if (i < dec_layers_) { + // Decoder Layers + std::string layer_prefix = model_folder_path_ + "/decoder.layer_" + std::to_string(i) + "_"; + + layer.decoder_encoder_attn_k_weight = gb->mmap_weights(layer_prefix + "encoder_attn_k.weights"); + layer.decoder_encoder_attn_q_weight = gb->mmap_weights(layer_prefix + "encoder_attn_q.weights"); + layer.decoder_encoder_attn_v_weight = gb->mmap_weights(layer_prefix + "encoder_attn_v.weights"); + layer.decoder_encoder_attn_output_weight = gb->mmap_weights(layer_prefix + "encoder_attn_output.weights"); + layer.decoder_encoder_attn_q_bias = gb->mmap_weights(layer_prefix + "encoder_attn_q.bias"); + layer.decoder_encoder_attn_v_bias = gb->mmap_weights(layer_prefix + "encoder_attn_v.bias"); + layer.decoder_encoder_attn_output_bias = gb->mmap_weights(layer_prefix + "encoder_attn_output.bias"); + + layer.decoder_post_encoder_layernorm_weight = gb->mmap_weights(layer_prefix + "encoder_attn_norm.weights"); + layer.decoder_post_encoder_layernorm_bias = gb->mmap_weights(layer_prefix + "encoder_attn_norm.bias"); + + layer.decoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); + layer.decoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); + layer.decoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); + layer.decoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); + + layer.decoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); + layer.decoder_post_ffn_layernorm_bias = gb->mmap_weights(layer_prefix + "final_norm.bias"); + + layer.decoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); + layer.decoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); + layer.decoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); + layer.decoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); + layer.decoder_self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); + layer.decoder_self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); + layer.decoder_self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); + + layer.decoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "self_attn_norm.weights"); + layer.decoder_post_attn_layernorm_bias = gb->mmap_weights(layer_prefix + "self_attn_norm.bias"); + } + + if (has_cpu_encoder_weights && i < enc_layers_) { + std::string layer_prefix = model_folder_path_ + "/encoder.layer_" + std::to_string(i) + "_"; + + layer.encoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); + layer.encoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); + layer.encoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); + layer.encoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); + + layer.encoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); + layer.encoder_post_ffn_layernorm_bias = gb->mmap_weights(layer_prefix + "final_norm.bias"); + + layer.encoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); + layer.encoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); + layer.encoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); + layer.encoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); + layer.encoder_self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); + layer.encoder_self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); + layer.encoder_self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); + + layer.encoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "self_attn_norm.weights"); + layer.encoder_post_attn_layernorm_bias = gb->mmap_weights(layer_prefix + "self_attn_norm.bias"); + } + } +} + +size_t WhisperModel::build_encoder_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto ffn1_weight = gb->matmul(input, layer.encoder_ffn1_weight, true, backend); + auto ffn1_bias = gb->add(ffn1_weight, layer.encoder_ffn1_bias); + + encoder_pre_gelu = ffn1_bias; + + auto ffn1_act = gb->gelu_erf(ffn1_bias); + + encoder_post_gelu = ffn1_act; + + auto ffn2_weight = gb->matmul(ffn1_act, layer.encoder_ffn2_weight, true, backend); + auto ffn2_bias = gb->add(ffn2_weight, layer.encoder_ffn2_bias); + return ffn2_bias; +} + +size_t WhisperModel::build_decoder_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + auto ffn1_weight = gb->matmul(input, layer.decoder_ffn1_weight, true, backend); + auto ffn1_bias = gb->add(ffn1_weight, layer.decoder_ffn1_bias); + auto ffn1_act = gb->gelu_erf(ffn1_bias); + auto ffn2_weight = gb->matmul(ffn1_act, layer.decoder_ffn2_weight, true, backend); + auto ffn2_bias = gb->add(ffn2_weight, layer.decoder_ffn2_bias); + return ffn2_bias; +} + + + +size_t WhisperModel::build_encoder_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool /*use_cache*/, size_t /*position_offset*/){ + + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t q = gb->matmul(input, layer.decoder_encoder_attn_q_weight, true, backend); + q = gb->add(q, layer.decoder_encoder_attn_q_bias); + + const auto& q_buf = gb->get_output_buffer(q); + if (q_buf.shape.size() != 2) { + throw std::runtime_error("encoder cross-attn: q must be [T_dec, D]"); + } + + size_t T_dec = q_buf.shape[0]; + size_t q_heads = config_.attention_heads; + size_t kv_heads = config_.attention_kv_heads; + size_t head_dim = config_.attention_head_dim; + + q = gb->reshape(q, {1, T_dec, q_heads, head_dim}); + + size_t k_4d = 0; + size_t v_4d = 0; + + bool is_populated = (encoder_k_persistent_[layer_idx] != 0 && + gb->is_populated(encoder_k_persistent_[layer_idx])); + + if (is_populated) { + k_4d = encoder_k_persistent_[layer_idx]; + v_4d = encoder_v_persistent_[layer_idx]; + } else { + size_t enc_norm = last_encoder_post_norm_node_; + + size_t k = gb->matmul(enc_norm, layer.decoder_encoder_attn_k_weight, true, backend); + size_t v = gb->matmul(enc_norm, layer.decoder_encoder_attn_v_weight, true, backend); + v = gb->add(v, layer.decoder_encoder_attn_v_bias); + + const auto& k_buf = gb->get_output_buffer(k); + if (k_buf.shape.size() != 2) { + throw std::runtime_error("encoder cross-attn: k must be [T_enc, D]"); + } + size_t T_enc = k_buf.shape[0]; + + k_4d = gb->reshape(k, {1, T_enc, kv_heads, head_dim}); + v_4d = gb->reshape(v, {1, T_enc, kv_heads, head_dim}); + + if (encoder_k_persistent_[layer_idx] == 0) { + encoder_k_persistent_[layer_idx] = gb->persistent(k_4d); + encoder_v_persistent_[layer_idx] = gb->persistent(v_4d); + } + k_4d = encoder_k_persistent_[layer_idx]; + v_4d = encoder_v_persistent_[layer_idx]; + } + + size_t attn = gb->attention(q, k_4d, v_4d, attention_scale_, false); + + attn = gb->reshape(attn, {T_dec, q_heads * head_dim}); + size_t out = gb->matmul(attn, layer.decoder_encoder_attn_output_weight, true, backend); + out = gb->add(out, layer.decoder_encoder_attn_output_bias); + + return out; +} + +void WhisperModel::reset_graph_side_cache_nodes() { + cache_k_output_nodes_.assign(dec_layers_, 0); + cache_v_output_nodes_.assign(dec_layers_, 0); +} + +void WhisperModel::reset_cache() { + Model::reset_cache(); + encoder_ready_ = false; + first_decode_step_ = true; + + auto* gb = static_cast(graph_handle_); + if (gb) { + if (encoder_output_persistent_ != 0) { + gb->invalidate_persistent(encoder_output_persistent_); + encoder_output_persistent_ = 0; + } + for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { + if (encoder_k_persistent_[i] != 0) { + gb->invalidate_persistent(encoder_k_persistent_[i]); + encoder_k_persistent_[i] = 0; + } + if (encoder_v_persistent_[i] != 0) { + gb->invalidate_persistent(encoder_v_persistent_[i]); + encoder_v_persistent_[i] = 0; + } + } + } +} + +size_t WhisperModel::build_decoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ + const auto& layer = weight_nodes_.layers[layer_idx]; + + auto q = gb->matmul(input, layer.decoder_self_attn_q_weight, true, backend); + q = gb->add(q, layer.decoder_self_attn_q_bias); + + auto k = gb->matmul(input, layer.decoder_self_attn_k_weight, true, backend); + auto v = gb->matmul(input, layer.decoder_self_attn_v_weight, true, backend); + v = gb->add(v, layer.decoder_self_attn_v_bias); + + const auto& q_shape = gb->get_output_buffer(q).shape; + if (q_shape.size() != 2) { + throw std::runtime_error("decoder self-attn: q must be [T_new, D]"); + } + + size_t seq_new = q_shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + size_t num_kv_heads = config_.attention_kv_heads; + + auto q_4d = gb->reshape(q, {1, seq_new, num_heads, head_dim}); + auto k_4d = gb->reshape(k, {1, seq_new, num_kv_heads, head_dim}); + auto v_4d = gb->reshape(v, {1, seq_new, num_kv_heads, head_dim}); + + size_t final_k = k_4d; + size_t final_v = v_4d; + + if (use_cache && !kv_cache_.is_empty()) { + auto k_view = kv_cache_.get_key_view(layer_idx); + auto v_view = kv_cache_.get_value_view(layer_idx); + + if (!k_view.ptr1 || !v_view.ptr1) { + throw std::runtime_error("KV cache view is empty but kv_cache_.is_empty()==false"); + } + + size_t cache_len = kv_cache_.current_seq_len; + + size_t cache_k_node = gb->input( + {1, cache_len, num_kv_heads, head_dim}, + kv_cache_.precision + ); + size_t cache_v_node = gb->input( + {1, cache_len, num_kv_heads, head_dim}, + kv_cache_.precision + ); + + if (k_view.ptr2 == nullptr && v_view.ptr2 == nullptr) { + gb->set_external_input(cache_k_node, const_cast(k_view.ptr1), kv_cache_.precision); + gb->set_external_input(cache_v_node, const_cast(v_view.ptr1), kv_cache_.precision); + } else { + gb->set_external_input(cache_k_node, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); + gb->set_external_input(cache_v_node, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); + } + + final_k = gb->concat(cache_k_node, k_4d, 1); + final_v = gb->concat(cache_v_node, v_4d, 1); + } + + if (use_cache) { + cache_k_output_nodes_[layer_idx] = final_k; + cache_v_output_nodes_[layer_idx] = final_v; + } else { + cache_k_output_nodes_[layer_idx] = k_4d; + cache_v_output_nodes_[layer_idx] = v_4d; + } + + auto attn_out_4d = gb->attention(q_4d, final_k, final_v, attention_scale_, position_offset); + auto attn_out = gb->reshape(attn_out_4d, {seq_new, num_heads * head_dim}); + + auto output = gb->matmul(attn_out, layer.decoder_self_attn_output_weight, true, backend); + output = gb->add(output, layer.decoder_self_attn_output_bias); + return output; +} + +size_t WhisperModel::build_encoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t /*position_offset*/){ + const auto& layer = weight_nodes_.layers[layer_idx]; + + if(use_cache) + throw std::runtime_error("The encoder attention layers are not auto-regressive, and thus don't use KV caching!"); + + auto q = gb->matmul(input, layer.encoder_self_attn_q_weight, true, backend); + q = gb->add(q, layer.encoder_self_attn_q_bias); + auto v = gb->matmul(input, layer.encoder_self_attn_v_weight, true, backend); + v = gb->add(v, layer.encoder_self_attn_v_bias); + auto k = gb->matmul(input, layer.encoder_self_attn_k_weight, true, backend); + + size_t seq_len = gb->get_output_buffer(q).shape[0]; + size_t num_heads = config_.attention_heads; + size_t head_dim = config_.attention_head_dim; + + q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); + k = gb->reshape(k, {1, seq_len, num_heads, head_dim}); + v = gb->reshape(v, {1, seq_len, num_heads, head_dim}); + + auto attn = gb->attention(q, k, v, attention_scale_, false); + + attn = gb->reshape(attn, {seq_len, num_heads * head_dim}); + + auto output = gb->matmul(attn, layer.encoder_self_attn_output_weight, true, backend); + output = gb->add(output, layer.encoder_self_attn_output_bias); + + return output; +} + +size_t WhisperModel::build_conv1d(CactusGraph* gb, size_t input) +{ + size_t conv_input = input; + const auto& xbuf = gb->get_output_buffer(input); + + if (xbuf.precision == Precision::INT8) { + conv_input = gb->precision_cast(input, Precision::FP16); + } + + size_t conv1 = gb->conv1d_k3(conv_input, weight_nodes_.encoder_conv1_weight, 1); + + auto bias1_shape = gb->get_output_buffer(weight_nodes_.encoder_conv1_bias).shape; + size_t C1 = bias1_shape[0]; + size_t bias1 = gb->reshape(weight_nodes_.encoder_conv1_bias, {1, C1, 1}); + conv1 = gb->add(conv1, bias1); + + last_conv1_node_ = conv1; + + conv1 = gb->gelu_erf(conv1); + + size_t conv2 = gb->conv1d_k3(conv1, weight_nodes_.encoder_conv2_weight, 2); + auto bias2_shape = gb->get_output_buffer(weight_nodes_.encoder_conv2_bias).shape; + size_t C2 = bias2_shape[0]; + size_t bias2 = gb->reshape(weight_nodes_.encoder_conv2_bias, {1, C2, 1}); + conv2 = gb->add(conv2, bias2); + + last_conv2_node_ = conv2; + + conv2 = gb->gelu_erf(conv2); + + const auto& buf = gb->get_output_buffer(conv2); + + size_t conv2_transposed; + if (buf.precision == Precision::FP16) { + conv2_transposed = gb->transpose(conv2, ComputeBackend::CPU); + } else { + size_t conv2_f16 = gb->precision_cast(conv2, Precision::FP16); + conv2_transposed = gb->transpose(conv2_f16, ComputeBackend::CPU); + } + + return conv2_transposed; +} + +size_t WhisperModel::build_encoder_transformer_block( + CactusGraph* gb, + size_t hidden, + uint32_t layer_idx, + ComputeBackend backend, + bool use_cache, + size_t position_offset) +{ + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t ln1 = gb->layernorm( + hidden, + layer.encoder_post_attn_layernorm_weight, + layer.encoder_post_attn_layernorm_bias + ); + + size_t sa = build_encoder_self_attention( + gb, ln1, layer_idx, backend, use_cache, position_offset + ); + + size_t x_post_sa = gb->add(hidden, sa); + + size_t ln2 = gb->layernorm( + x_post_sa, + layer.encoder_post_ffn_layernorm_weight, + layer.encoder_post_ffn_layernorm_bias + ); + + size_t ffn_out = build_encoder_mlp( + gb, ln2, layer_idx, backend + ); + + size_t out = gb->add(x_post_sa, ffn_out); + + if (layer_idx < encoder_block_out_nodes_.size()) { + encoder_block_out_nodes_[layer_idx] = out; + } + + return out; +} + +size_t WhisperModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ + const auto& layer = weight_nodes_.layers[layer_idx]; + + size_t ln1 = gb->layernorm(hidden, layer.decoder_post_attn_layernorm_weight, layer.decoder_post_attn_layernorm_bias); + size_t sa = build_decoder_self_attention(gb, ln1, layer_idx, backend, use_cache, position_offset); + size_t x_post_sa = gb->add(hidden, sa); + + size_t ln2 = gb->layernorm(x_post_sa, layer.decoder_post_encoder_layernorm_weight, layer.decoder_post_encoder_layernorm_bias); + size_t ca = build_encoder_attention(gb, ln2, layer_idx, backend, use_cache, position_offset); + size_t x_post_ca = gb->add(x_post_sa, ca); + + size_t ln3 = gb->layernorm(x_post_ca,layer.decoder_post_ffn_layernorm_weight,layer.decoder_post_ffn_layernorm_bias); + size_t ffn_out = build_decoder_mlp(gb, ln3, layer_idx, backend); + size_t x_post_ffn = gb->add(x_post_ca, ffn_out); + + return x_post_ffn; + +} + +void WhisperModel::run_encoder(const std::vector& audio_features) +{ + const size_t mel_bins = config_.num_mel_bins; + if (audio_features.size() % mel_bins != 0) + throw std::runtime_error("Mel bins length must be divisible by " + std::to_string(mel_bins) + "."); + + size_t T_mel = audio_features.size() / mel_bins; + if (T_mel == 0) + throw std::runtime_error("Mel bins has zero frames."); + + auto* gb = static_cast(graph_handle_); + if (!gb) + throw std::runtime_error("Graph handle is null in run_encoder."); + + if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { + std::vector out_shape = npu_encoder_->get_output_shape(); + size_t T_enc, D_enc; + if (out_shape.size() == 3) { + T_enc = static_cast(out_shape[1]); + D_enc = static_cast(out_shape[2]); + } else if (out_shape.size() == 2) { + T_enc = static_cast(out_shape[0]); + D_enc = static_cast(out_shape[1]); + } else { + throw std::runtime_error("NPU encoder output has unexpected shape"); + } + + std::vector<__fp16> audio_features_f16(audio_features.size()); + cactus_fp32_to_fp16(audio_features.data(), audio_features_f16.data(), audio_features.size()); + + std::vector input_shape = {1, static_cast(mel_bins), static_cast(T_mel)}; + size_t total_elements = T_enc * D_enc; + std::vector<__fp16> npu_output(total_elements); + size_t elements_written = npu_encoder_->encode( + audio_features_f16.data(), + npu_output.data(), + input_shape, + "x", + "" + ); + + if (elements_written > 0) { + size_t enc_output_node = gb->input({T_enc, D_enc}, Precision::FP16); + gb->set_input(enc_output_node, npu_output.data(), Precision::FP16); + + size_t enc_output_persistent = gb->persistent(enc_output_node); + encoder_output_persistent_ = enc_output_persistent; + last_encoder_post_norm_node_ = enc_output_persistent; + return; + } + } + + if (!has_whisper_cpu_encoder_weights(model_folder_path_)) { + throw std::runtime_error( + "Whisper requires either CPU encoder weights or model.mlpackage encoder output."); + } + + auto backend = + (config_.default_backend == Config::Backend::CPU) + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + size_t mel_input = 0; + std::vector<__fp16> audio_features_f16(audio_features.size()); + cactus_fp32_to_fp16(audio_features.data(), audio_features_f16.data(), audio_features.size()); + + mel_input = gb->input({1, mel_bins, T_mel}, Precision::FP16); + gb->set_input(mel_input, audio_features_f16.data(), Precision::FP16); + + size_t conv2_transposed = build_conv1d(gb, mel_input); + + const auto& conv_shape = gb->get_output_buffer(conv2_transposed).shape; + if (conv_shape.size() != 3 || conv_shape[0] != 1) + throw std::runtime_error("Conv2 transpose should be [1, T_enc, D]."); + + size_t T_enc = conv_shape[1]; + size_t D_enc = conv_shape[2]; + + size_t pos_slice = gb->slice(weight_nodes_.encoder_position_embeddings, 0, 0, T_enc); + + size_t h2d = gb->reshape(conv2_transposed, {T_enc, D_enc}); + + auto& h2d_buf = gb->get_output_buffer(h2d); + auto& pos_buf = gb->get_output_buffer(pos_slice); + + if (pos_buf.precision != h2d_buf.precision) { + pos_slice = gb->precision_cast(pos_slice, h2d_buf.precision); + } + + size_t h_pos = gb->add(h2d, pos_slice); + last_enc_plus_pos_node_ = h_pos; + + size_t h = h_pos; + for (uint32_t i = 0; i < enc_layers_; ++i){ + h = build_encoder_transformer_block(gb, h, i, backend, false, 0); + if (i == 0) { + encoder_transformer_block_0 = h; + } + } + + size_t h_norm = gb->layernorm( + h, + weight_nodes_.encoder_norm_weight, + weight_nodes_.encoder_norm_bias + ); + + size_t h_norm_persistent = gb->persistent(h_norm); + encoder_output_persistent_ = h_norm_persistent; + last_encoder_post_norm_node_ = h_norm_persistent; +} + + + +size_t WhisperModel::run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only) { + auto* gb = static_cast(graph_handle_); + + const size_t full_len = tokens.size(); + if (full_len == 0) { + throw std::runtime_error("Decoder token list cannot be empty."); + } + + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + size_t start_idx = (use_cache && kv_cache_.current_seq_len > 0) ? full_len - 1 : 0; + size_t new_tokens = full_len - start_idx; + + size_t tok_input = gb->input({new_tokens}, Precision::FP32); + std::vector tok_f(new_tokens); + for (size_t i = 0; i < new_tokens; i++) { + tok_f[i] = static_cast(tokens[start_idx + i]); + } + gb->set_input(tok_input, tok_f.data(), Precision::FP32); + + size_t dec_hidden = gb->embedding(embedding_node_id_, tok_input); + + size_t position_offset = kv_cache_.current_seq_len; + size_t dec_pos = gb->slice(weight_nodes_.decoder_position_embeddings_weight,0,position_offset,new_tokens); + + { + const auto& h_buf = gb->get_output_buffer(dec_hidden); + const auto& pos_buf = gb->get_output_buffer(dec_pos); + + size_t pos_node_for_add = dec_pos; + if (pos_buf.precision != h_buf.precision) { + pos_node_for_add = gb->precision_cast(dec_pos, h_buf.precision); + } + + dec_hidden = gb->add(dec_hidden, pos_node_for_add); + } + + for (uint32_t layer_idx = 0; layer_idx < dec_layers_; ++layer_idx) { + dec_hidden = build_decoder_transformer_block( + gb, + dec_hidden, + layer_idx, + backend, + use_cache, + position_offset + ); + } + + size_t dec_norm = gb->layernorm( + dec_hidden, + weight_nodes_.decoder_norm_weight, + weight_nodes_.decoder_norm_bias + ); + + size_t logits_input = dec_norm; + if (last_token_only) { + size_t row_index = new_tokens - 1; + logits_input = gb->slice(logits_input, 0, row_index, 1); + } + + auto w_shape = gb->get_output_buffer(output_weight_node_id_).shape; + + size_t logits = gb->matmul(logits_input, output_weight_node_id_, true, backend); + + last_new_tokens_ = new_tokens; + return logits; +} + + +size_t WhisperModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) +{ + + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized"); + } + + if (!use_cache) { + kv_cache_.reset(); + kv_cache_.current_seq_len = 0; + reset_graph_side_cache_nodes(); + run_encoder(audio_features); + } + + return run_decoder_step(tokens, use_cache, false); +} + +std::vector WhisperModel::get_audio_embeddings(const std::vector& audio_features) { + run_encoder(audio_features); + + auto* gb = static_cast(graph_handle_); + + size_t pooled = gb->mean(encoder_output_persistent_, 0); + gb->execute(); + + const auto& output_buf = gb->get_output_buffer(pooled); + size_t hidden_dim = output_buf.total_size; + + std::vector embedding(hidden_dim); + void* output_data = gb->get_output(pooled); + const float* output_ptr = static_cast(output_data); + std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); + + reset_cache(); + return embedding; +} + +uint32_t WhisperModel::decode_with_audio( + const std::vector& tokens, + const std::vector& audio_features, + float temperature, + float top_p, + size_t top_k, + const std::string& profile_file, + float* out_entropy, + float min_p, + float repetition_penalty, + float* /*out_token_time_start*/, + float* /*out_token_time_end*/) +{ + if (!initialized_ || !graph_handle_) + throw std::runtime_error("Model not initialized - call init() first"); + if (audio_features.empty()) + throw std::runtime_error("Mel bins cannot be empty in Whisper decode_with_audio"); + + auto* gb = static_cast(graph_handle_); + + bool cold_start = !encoder_ready_; + size_t logits_node = 0; + + std::vector full_tokens; + if (tokens.empty()) { + full_tokens.push_back(static_cast(get_tokenizer()->get_bos_token())); + } else { + full_tokens = tokens; + } + + if (cold_start) + { + gb->soft_reset(); + kv_cache_.reset(); + kv_cache_.current_seq_len = 0; + reset_graph_side_cache_nodes(); + + first_decode_step_ = true; + run_encoder(audio_features); + logits_node = run_decoder_step(full_tokens, false, false); + encoder_ready_ = true; + } + else + { + gb->soft_reset(); + reset_graph_side_cache_nodes(); + + std::vector last_token_vec = { tokens.back() }; + logits_node = run_decoder_step(last_token_vec, true, true); + } + + const auto& suppress_bias = first_decode_step_ ? suppress_bias_first_step_ : suppress_bias_; + if (first_decode_step_) first_decode_step_ = false; + size_t sampled_token_id = sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty, &suppress_bias); + if (!profile_file.empty()) gb->execute(profile_file); + else gb->execute(); + + + compute_entropy(gb, logits_node, out_entropy); + + post_execute_updates(gb, full_tokens.size()); + update_kv_cache(gb, last_new_tokens_); + + auto* out_ptr = gb->get_output(sampled_token_id); + uint32_t sampled = *reinterpret_cast(out_ptr); + record_sampled_token(sampled); + + return sampled; +} + + +} +} diff --git a/cactus-transpile/models/model_youtu.cpp b/cactus-transpile/models/model_youtu.cpp new file mode 100644 index 000000000..cfd1028ab --- /dev/null +++ b/cactus-transpile/models/model_youtu.cpp @@ -0,0 +1,246 @@ +#include "model.h" +#include "../graph/graph.h" +#include +#include + +namespace cactus { +namespace engine { + +YoutuModel::YoutuModel() : Model() {} + +YoutuModel::YoutuModel(const Config& config) : Model(config) { + weight_nodes_.layers.resize(config.num_layers); +} + +std::vector YoutuModel::get_kv_layer_dims() const { + return std::vector(config_.num_layers, static_cast(config_.qk_head_dim)); +} + +void YoutuModel::post_init() { + std::vector v_dims(config_.num_layers, static_cast(config_.v_head_dim)); + std::vector v_kv_heads(config_.num_layers, static_cast(config_.attention_kv_heads)); + v_cache_.init(config_.num_layers, kv_cache_.max_seq_len, + v_dims, v_kv_heads, Precision::INT8); + v_cache_.set_window_size(kv_cache_.window_size, kv_cache_.sink_size); + cache_v_nodes_.resize(config_.num_layers, 0); +} + +void YoutuModel::post_execute_updates(CactusGraph* gb, size_t seq_len) { + v_cache_.update_from_graph(gb, cache_v_nodes_, cache_v_nodes_, seq_len, + config_.num_layers); +} + +void YoutuModel::reset_cache() { + Model::reset_cache(); + v_cache_.reset(); +} + +void YoutuModel::load_weights_to_graph(CactusGraph* gb) { + embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); + weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); + + if (config_.tie_word_embeddings) { + weight_nodes_.output_weight = embedding_node_id_; + output_weight_node_id_ = embedding_node_id_; + } else { + weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); + output_weight_node_id_ = weight_nodes_.output_weight; + } + + for (uint32_t i = 0; i < config_.num_layers; i++) { + auto& layer = weight_nodes_.layers[i]; + std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; + if (config_.q_lora_rank == 0) { + layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); + } else { + layer.attn_q_a_weight = gb->mmap_weights(layer_prefix + "attn_q_a.weights"); + layer.attn_q_a_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_a_norm.weights"); + layer.attn_q_b_weight = gb->mmap_weights(layer_prefix + "attn_q_b.weights"); + } + layer.attn_kv_a_weight = gb->mmap_weights(layer_prefix + "attn_kv_a.weights"); + layer.attn_kv_a_norm_weight = gb->mmap_weights(layer_prefix + "attn_kv_a_norm.weights"); + layer.attn_kv_b_weight = gb->mmap_weights(layer_prefix + "attn_kv_b.weights"); + layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); + if (config_.attention_bias) { + layer.attn_q_a_bias = gb->mmap_weights(layer_prefix + "attn_q_a_bias.weights"); + layer.attn_kv_a_bias = gb->mmap_weights(layer_prefix + "attn_kv_a_bias.weights"); + layer.attn_output_bias = gb->mmap_weights(layer_prefix + "attn_output_bias.weights"); + } + layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); + layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); + layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); + layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); + layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); + } +} + +static float yarn_get_mscale(float scale, float mscale) { + if (scale <= 1.0f) return 1.0f; + return 0.1f * mscale * logf(scale) + 1.0f; +} + +size_t YoutuModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + + const size_t seq_len = gb->get_output_buffer(normalized_input).shape[0]; + const size_t num_heads = config_.attention_heads; + const size_t num_kv_heads = config_.attention_kv_heads; + const size_t kv_lora = config_.kv_lora_rank; + const size_t qk_head = config_.qk_head_dim; + const size_t qk_nope = config_.qk_nope_head_dim; + const size_t qk_rope = config_.qk_rope_head_dim; + const size_t v_dim = config_.v_head_dim; + const float eps = config_.layer_norm_eps; + + size_t q_full; + if (config_.q_lora_rank == 0) { + q_full = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); + } else { + auto q_latent = gb->matmul(normalized_input, layer.attn_q_a_weight, true, backend); + if (config_.attention_bias) { + q_latent = gb->add(q_latent, layer.attn_q_a_bias); + } + q_latent = gb->rms_norm(q_latent, layer.attn_q_a_norm_weight, eps); + q_full = gb->matmul(q_latent, layer.attn_q_b_weight, true, backend); + } + + q_full = gb->reshape(q_full, {seq_len * num_heads, qk_head}); + auto q_nope = gb->slice(q_full, 1, 0, qk_nope); + auto q_rope_raw = gb->slice(q_full, 1, qk_nope, qk_rope); + + q_rope_raw = gb->reshape(q_rope_raw, {1, seq_len, num_heads, qk_rope}); + size_t q_rope_rotated; + if (config_.rope_interleave) { + q_rope_rotated = gb->rope_gptj(q_rope_raw, config_.rope_theta, position_offset, qk_rope); + } else { + q_rope_rotated = gb->rope(q_rope_raw, config_.rope_theta, position_offset); + } + q_rope_rotated = gb->reshape(q_rope_rotated, {seq_len * num_heads, qk_rope}); + + auto q_combined = gb->concat(q_nope, q_rope_rotated, -1); + auto q_4d = gb->reshape(q_combined, {1, seq_len, num_heads, qk_head}); + + auto kv_combined = gb->matmul(normalized_input, layer.attn_kv_a_weight, true, backend); + if (config_.attention_bias) { + kv_combined = gb->add(kv_combined, layer.attn_kv_a_bias); + } + auto kv_latent = gb->slice(kv_combined, 1, 0, kv_lora); + auto k_rope_raw = gb->slice(kv_combined, 1, kv_lora, qk_rope); + + kv_latent = gb->rms_norm(kv_latent, layer.attn_kv_a_norm_weight, eps); + + auto kv_decoded = gb->matmul(kv_latent, layer.attn_kv_b_weight, true, backend); + kv_decoded = gb->reshape(kv_decoded, {seq_len * num_kv_heads, qk_nope + v_dim}); + auto k_nope = gb->slice(kv_decoded, 1, 0, qk_nope); + auto v_flat = gb->slice(kv_decoded, 1, qk_nope, v_dim); + + k_rope_raw = gb->reshape(k_rope_raw, {1, seq_len, 1, qk_rope}); + size_t k_rope_rotated; + if (config_.rope_interleave) { + k_rope_rotated = gb->rope_gptj(k_rope_raw, config_.rope_theta, position_offset, qk_rope); + } else { + k_rope_rotated = gb->rope(k_rope_raw, config_.rope_theta, position_offset); + } + std::vector k_rope_copies(num_kv_heads, k_rope_rotated); + auto k_rope_4d = gb->cat(k_rope_copies, 2); + auto k_rope_flat = gb->reshape(k_rope_4d, {seq_len * num_kv_heads, qk_rope}); + + auto k_combined = gb->concat(k_nope, k_rope_flat, -1); + auto k_4d = gb->reshape(k_combined, {1, seq_len, num_kv_heads, qk_head}); + auto v_4d = gb->reshape(v_flat, {1, seq_len, num_kv_heads, v_dim}); + + if (use_cache && layer_idx < cache_k_output_nodes_.size()) { + cache_k_output_nodes_[layer_idx] = gb->reshape(k_4d, {seq_len * num_kv_heads, qk_head}); + cache_v_output_nodes_[layer_idx] = cache_k_output_nodes_[layer_idx]; + cache_v_nodes_[layer_idx] = gb->reshape(v_4d, {seq_len * num_kv_heads, v_dim}); + } + + float scale = 1.0f / sqrtf(static_cast(qk_head)); + if (config_.rope_mscale_all_dim != 0.0f && config_.rope_scaling_factor > 1.0f) { + float mscale = yarn_get_mscale(config_.rope_scaling_factor, config_.rope_mscale_all_dim); + scale *= mscale * mscale; + } + + size_t attn_4d; + if (use_cache && !kv_cache_.is_empty()) { + attn_4d = gb->attention_int8_hybrid( + q_4d, k_4d, v_4d, scale, position_offset, + kv_cache_.get_keys_int8(layer_idx), + v_cache_.get_values_int8(layer_idx), + kv_cache_.get_key_scales(layer_idx), + v_cache_.get_value_scales(layer_idx), + kv_cache_.current_seq_len, num_kv_heads, qk_head, + kv_cache_.window_size, v_dim); + } else { + attn_4d = gb->attention(q_4d, k_4d, v_4d, scale, position_offset); + } + + auto attn_out = gb->reshape(attn_4d, {seq_len, num_heads * v_dim}); + size_t out = gb->matmul(attn_out, layer.attn_output_weight, true, backend); + if (config_.attention_bias) { + out = gb->add(out, layer.attn_output_bias); + } + return out; +} + + +size_t YoutuModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const { + const auto& layer = weight_nodes_.layers[layer_idx]; + size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); + size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); + size_t gate_silu = gb->silu(gate_output); + size_t gated = gb->multiply(gate_silu, up_output); + return gb->matmul(gated, layer.ffn_down_weight, true, backend); +} + + +size_t YoutuModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache, size_t position_offset) { + const auto& layer = weight_nodes_.layers[layer_idx]; + auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); + auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); + auto after_attention = gb->add_clipped(hidden, attn_output); + auto normalized_after_attention = gb->rms_norm(after_attention, layer.post_attention_layernorm_weight, config_.layer_norm_eps); + auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); + return gb->add_clipped(after_attention, mlp_output); +} + + +size_t YoutuModel::forward(const std::vector& tokens, bool use_cache) { + if (!initialized_ || !graph_handle_) { + throw std::runtime_error("Model not initialized - call init() first"); + } + if (tokens.empty()) { + throw std::runtime_error("Token sequence cannot be empty"); + } + + auto* gb = static_cast(graph_handle_); + gb->soft_reset(); + + const size_t seq_len = tokens.size(); + const size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; + + auto backend = config_.default_backend == Config::Backend::CPU + ? ComputeBackend::CPU + : ComputeBackend::NPU; + + auto input_node_id = gb->input({seq_len}, Precision::FP32); + auto hidden = gb->embedding(embedding_node_id_, input_node_id); + + std::vector input_data(seq_len); + for (size_t i = 0; i < seq_len; i++) { + input_data[i] = static_cast(tokens[i]); + } + gb->set_input(input_node_id, input_data.data(), Precision::FP32); + + for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { + hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); + } + + return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); +} + +} +} diff --git a/cactus-transpile/npu/npu.cpp b/cactus-transpile/npu/npu.cpp new file mode 100644 index 000000000..69a9f80c7 --- /dev/null +++ b/cactus-transpile/npu/npu.cpp @@ -0,0 +1,22 @@ +// NPU stub implementation for non-Apple platforms +// On Apple platforms, npu_ane.mm provides the real implementation + +#include "npu.h" + +namespace cactus { +namespace npu { + +std::unique_ptr create_encoder() { + return nullptr; +} + +std::unique_ptr create_prefill() { + return nullptr; +} + +bool is_npu_available() { + return false; +} + +} // namespace npu +} // namespace cactus diff --git a/cactus-transpile/npu/npu.h b/cactus-transpile/npu/npu.h new file mode 100644 index 000000000..5474c872f --- /dev/null +++ b/cactus-transpile/npu/npu.h @@ -0,0 +1,89 @@ +#ifndef CACTUS_NPU_H +#define CACTUS_NPU_H + +#include +#include +#include +#include + +namespace cactus { +namespace npu { + + +struct NPUNamedInput { + std::string name; + const __fp16* data; + std::vector shape; +}; + +class NPUEncoder { +public: + virtual ~NPUEncoder() = default; + + virtual bool load(const std::string& model_path) = 0; + + virtual bool preallocate(const std::vector& input_shape, + const std::string& input_name = "x", + const std::string& output_name = "") = 0; + + virtual size_t encode(const __fp16* input, + __fp16* output, + const std::vector& shape, + const std::string& input_name = "x", + const std::string& output_name = "") = 0; + + virtual bool is_available() const = 0; + + virtual std::vector get_input_shape() const = 0; + + virtual std::vector get_output_shape() const = 0; + + virtual __fp16* get_output_buffer() = 0; + + virtual size_t get_output_buffer_size() const = 0; + + virtual size_t encode_multimodal_input( + const std::vector& inputs, + __fp16* output, + const std::string& output_name = "") = 0; +}; + +std::unique_ptr create_encoder(); + +bool is_npu_available(); + +struct NPUBufferRef { + const __fp16* data; + size_t count; +}; + +struct NPUPrefillDirectResult { + NPUBufferRef hidden; + std::vector k_caches; + std::vector v_caches; + bool valid; +}; + +class NPUPrefill { +public: + virtual ~NPUPrefill() = default; + virtual bool load(const std::string& model_path) = 0; + virtual bool is_available() const = 0; + virtual int get_chunk_size() const = 0; + virtual int get_hidden_dim() const = 0; + virtual int get_num_layers() const = 0; + virtual int get_num_kv_heads() const = 0; + virtual int get_head_dim() const = 0; + + virtual NPUPrefillDirectResult prefill_chunk_direct( + const std::vector<__fp16>& embeddings, + int position_offset = 0, + const std::string& input_name = "x") = 0; +}; + +std::unique_ptr create_prefill(); + +} // namespace npu +} // namespace cactus + +#endif // CACTUS_NPU_H \ No newline at end of file diff --git a/cactus-transpile/npu/npu_ane.h b/cactus-transpile/npu/npu_ane.h new file mode 100644 index 000000000..c590fb0c4 --- /dev/null +++ b/cactus-transpile/npu/npu_ane.h @@ -0,0 +1,154 @@ +#ifndef CACTUS_NPU_ANE_H +#define CACTUS_NPU_ANE_H + +#include "npu.h" + +#if defined(__APPLE__) +#define CACTUS_HAS_ANE 1 +#else +#define CACTUS_HAS_ANE 0 +#endif + +namespace cactus { +namespace npu { + +#if CACTUS_HAS_ANE + + +class ANEEncoder : public NPUEncoder { +public: + ANEEncoder(); + ~ANEEncoder() override; + + ANEEncoder(const ANEEncoder&) = delete; + ANEEncoder& operator=(const ANEEncoder&) = delete; + + ANEEncoder(ANEEncoder&& other) noexcept; + ANEEncoder& operator=(ANEEncoder&& other) noexcept; + + bool load(const std::string& model_path) override; + + bool preallocate(const std::vector& input_shape, + const std::string& input_name = "x", + const std::string& output_name = "") override; + + size_t encode(const __fp16* input, + __fp16* output, + const std::vector& shape, + const std::string& input_name = "x", + const std::string& output_name = "") override; + + bool is_available() const override; + + std::vector get_input_shape() const override; + + std::vector get_output_shape() const override; + + __fp16* get_output_buffer() override; + + size_t get_output_buffer_size() const override; + + size_t encode_multimodal_input( + const std::vector& inputs, + __fp16* output, + const std::string& output_name = "") override; + +private: + void* impl_; +}; + +class ANEPrefill : public NPUPrefill { +public: + ANEPrefill(); + ~ANEPrefill() override; + + ANEPrefill(const ANEPrefill&) = delete; + ANEPrefill& operator=(const ANEPrefill&) = delete; + + ANEPrefill(ANEPrefill&& other) noexcept; + ANEPrefill& operator=(ANEPrefill&& other) noexcept; + + bool load(const std::string& model_path) override; + bool is_available() const override; + + int get_chunk_size() const override; + int get_hidden_dim() const override; + int get_num_layers() const override; + int get_num_kv_heads() const override; + int get_head_dim() const override; + + NPUPrefillDirectResult prefill_chunk_direct( + const std::vector<__fp16>& embeddings, + int position_offset = 0, + const std::string& input_name = "x") override; + +private: + void* impl_; + int chunk_size_ = 256; + int hidden_dim_ = 0; + int num_layers_ = 0; + int num_kv_heads_ = 0; + int head_dim_ = 0; +}; + +#else + +class ANEEncoder : public NPUEncoder { +public: + ANEEncoder() = default; + ~ANEEncoder() override = default; + + bool load(const std::string&) override { return false; } + + bool preallocate(const std::vector&, + const std::string& = "x", + const std::string& = "") override { return false; } + + size_t encode(const __fp16*, + __fp16*, + const std::vector&, + const std::string& = "x", + const std::string& = "") override { return 0; } + + bool is_available() const override { return false; } + + std::vector get_input_shape() const override { return {}; } + + std::vector get_output_shape() const override { return {}; } + + __fp16* get_output_buffer() override { return nullptr; } + + size_t get_output_buffer_size() const override { return 0; } + + size_t encode_multimodal_input( + const std::vector&, + __fp16*, + const std::string& = "") override { return 0; } +}; + +class ANEPrefill : public NPUPrefill { +public: + ANEPrefill() = default; + ~ANEPrefill() override = default; + + bool load(const std::string&) override { return false; } + bool is_available() const override { return false; } + + int get_chunk_size() const override { return 0; } + int get_hidden_dim() const override { return 0; } + int get_num_layers() const override { return 0; } + int get_num_kv_heads() const override { return 0; } + int get_head_dim() const override { return 0; } + + NPUPrefillDirectResult prefill_chunk_direct( + const std::vector<__fp16>&, + int = 0, + const std::string& = "x") override { return {}; } +}; + +#endif // CACTUS_HAS_ANE + +} // namespace npu +} // namespace cactus + +#endif // CACTUS_NPU_ANE_H \ No newline at end of file diff --git a/cactus-transpile/npu/npu_ane.mm b/cactus-transpile/npu/npu_ane.mm new file mode 100644 index 000000000..e6c379486 --- /dev/null +++ b/cactus-transpile/npu/npu_ane.mm @@ -0,0 +1,1305 @@ +#include "npu_ane.h" + +#if CACTUS_HAS_ANE + +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include "../graph/graph.h" + +namespace { + +static std::string to_lower_ascii(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return value; +} + +static bool parse_compute_units(const char* raw_value, MLComputeUnits& out_units) { + if (!raw_value || raw_value[0] == '\0') { + return false; + } + + std::string value = to_lower_ascii(raw_value); + if (value == "all" || value == "default") { + out_units = MLComputeUnitsAll; + return true; + } + if (value == "cpu_and_ne" || value == "cpu-ne" || value == "cpu_ne" || + value == "ne" || value == "ane" || value == "cpuandneuralengine") { + out_units = MLComputeUnitsCPUAndNeuralEngine; + return true; + } + if (value == "cpu_and_gpu" || value == "cpu_gpu" || value == "cpu-gpu" || + value == "cpuandgpu") { + out_units = MLComputeUnitsCPUAndGPU; + return true; + } + if (value == "cpu_only" || value == "cpu-only" || value == "cpu") { + out_units = MLComputeUnitsCPUOnly; + return true; + } + + return false; +} + +static const char* compute_units_to_string(MLComputeUnits units) { + switch (units) { + case MLComputeUnitsAll: + return "ALL"; + case MLComputeUnitsCPUAndGPU: + return "CPU_AND_GPU"; + case MLComputeUnitsCPUOnly: + return "CPU_ONLY"; + case MLComputeUnitsCPUAndNeuralEngine: + return "CPU_AND_NE"; + } + return "UNKNOWN"; +} + +static bool model_path_looks_like_audio_encoder(NSString* model_path) { + if (!model_path) return false; + NSString* lower = [[model_path lastPathComponent] lowercaseString]; + return [lower containsString:@"audio_encoder"]; +} + +static bool model_path_looks_like_vision_encoder(NSString* model_path) { + if (!model_path) return false; + NSString* lower = [[model_path lastPathComponent] lowercaseString]; + return [lower containsString:@"vision_encoder"]; +} + +static void maybe_apply_compute_units_env(const char* env_name, MLComputeUnits& target) { + const char* raw = std::getenv(env_name); + if (!raw || raw[0] == '\0') return; + + MLComputeUnits parsed; + if (parse_compute_units(raw, parsed)) { + target = parsed; + return; + } + + CACTUS_LOG_WARN("npu", "Ignoring invalid " << env_name << "=" << raw); +} + +static MLComputeUnits resolve_compute_units_for_model(NSString* model_path, bool is_prefill) { + MLComputeUnits units = MLComputeUnitsCPUAndNeuralEngine; + + // Gemma4 multimodal encoders show better fidelity on CPU+GPU. + if (!is_prefill && model_path_looks_like_audio_encoder(model_path)) { + units = MLComputeUnitsCPUAndGPU; + } + if (!is_prefill && model_path_looks_like_vision_encoder(model_path)) { + units = MLComputeUnitsCPUAndGPU; + } + + maybe_apply_compute_units_env("CACTUS_ANE_COMPUTE_UNITS", units); + if (is_prefill) { + maybe_apply_compute_units_env("CACTUS_ANE_PREFILL_COMPUTE_UNITS", units); + } else { + maybe_apply_compute_units_env("CACTUS_ANE_ENCODER_COMPUTE_UNITS", units); + if (model_path_looks_like_audio_encoder(model_path)) { + maybe_apply_compute_units_env("CACTUS_ANE_AUDIO_COMPUTE_UNITS", units); + } + if (model_path_looks_like_vision_encoder(model_path)) { + maybe_apply_compute_units_env("CACTUS_ANE_VISION_COMPUTE_UNITS", units); + } + } + + return units; +} + +static bool should_recompile_mlpackage(NSString* mlpackage_path, NSString* mlmodelc_path) { + NSFileManager* fm = [NSFileManager defaultManager]; + if (![fm fileExistsAtPath:mlmodelc_path]) return true; + + const char* force = std::getenv("CACTUS_ANE_FORCE_RECOMPILE"); + if (force && force[0] != '\0' && std::strcmp(force, "0") != 0) { + return true; + } + + NSError* pkg_err = nil; + NSError* cache_err = nil; + NSDictionary* pkg_attr = [fm attributesOfItemAtPath:mlpackage_path error:&pkg_err]; + NSDictionary* cache_attr = [fm attributesOfItemAtPath:mlmodelc_path error:&cache_err]; + if (!pkg_attr || !cache_attr || pkg_err || cache_err) { + return false; + } + + NSDate* pkg_mtime = pkg_attr[NSFileModificationDate]; + NSDate* cache_mtime = cache_attr[NSFileModificationDate]; + if (!pkg_mtime || !cache_mtime) { + return false; + } + return ([pkg_mtime compare:cache_mtime] == NSOrderedDescending); +} + +static NSURL* resolve_or_compile_model_url(NSString* path, NSError** error) { + NSURL* modelURL = [NSURL fileURLWithPath:path]; + if (![path hasSuffix:@".mlpackage"]) { + return modelURL; + } + + BOOL isDir = NO; + if (![[NSFileManager defaultManager] fileExistsAtPath:path isDirectory:&isDir] || !isDir) { + CACTUS_LOG_ERROR("npu", "ANE mlpackage path is not a valid directory: " << [path UTF8String]); + return modelURL; + } + + NSString* cachedPath = [[path stringByDeletingPathExtension] stringByAppendingPathExtension:@"mlmodelc"]; + NSURL* cachedURL = [NSURL fileURLWithPath:cachedPath]; + NSFileManager* fm = [NSFileManager defaultManager]; + + if (!should_recompile_mlpackage(path, cachedPath)) { + return cachedURL; + } + + if ([fm fileExistsAtPath:cachedPath]) { + NSError* rmErr = nil; + [fm removeItemAtPath:cachedPath error:&rmErr]; + if (rmErr) { + CACTUS_LOG_WARN("npu", "Failed to remove stale mlmodelc: " << [[rmErr localizedDescription] UTF8String]); + } else { + CACTUS_LOG_INFO("npu", "Removed stale mlmodelc cache, recompiling: " << [cachedPath UTF8String]); + } + } + + NSURL* compiledURL = [MLModel compileModelAtURL:modelURL error:error]; + if (*error || !compiledURL) { + CACTUS_LOG_ERROR("npu", "ANE model compilation failed: " << [[*error localizedDescription] UTF8String]); + return modelURL; + } + + NSError* moveError = nil; + [fm moveItemAtURL:compiledURL toURL:cachedURL error:&moveError]; + if (moveError) { + CACTUS_LOG_WARN("npu", "Could not persist compiled mlmodelc cache: " << [[moveError localizedDescription] UTF8String]); + return compiledURL; + } + return cachedURL; +} + +} // namespace + +@interface CactusANEImpl : NSObject + +@property (nonatomic, strong) MLModel* model; +@property (nonatomic, strong) MLModelDescription* modelDescription; +@property (nonatomic, strong) MLMultiArray* cachedInputArray; +@property (nonatomic, strong) MLMultiArray* cachedOutputArray; +@property (nonatomic, strong) NSArray* cachedShape; +@property (nonatomic, strong) NSArray* cachedOutputShape; +@property (nonatomic, strong) NSString* cachedInputName; +@property (nonatomic, strong) NSString* cachedOutputName; +@property (nonatomic, assign) NSUInteger cachedOutputSize; +@property (nonatomic, strong) MLPredictionOptions* predictionOptions; +@property (nonatomic, strong) NSMutableDictionary* cachedMultiInputArrays; +@property (nonatomic, strong) MLMultiArray* cachedMultiOutputArray; +@property (nonatomic, strong) MLPredictionOptions* multiPredictionOptions; +@property (nonatomic, assign) BOOL multiInputPreallocated; +@property (nonatomic, assign) BOOL hasOutputBackings; +@property (nonatomic, assign) BOOL hasMultiOutputBackings; + +- (instancetype)initWithModelPath:(NSString*)path; +- (NSArray*)getInputShape; +- (NSArray*)getOutputShape; +- (BOOL)preallocateBuffersWithInput:(NSString*)inputName + shape:(NSArray*)shape + outputName:(NSString*)outputName; +- (BOOL)canUseCachedBufferWithInput:(NSString*)inputName + shape:(NSArray*)shape + outputName:(NSString*)outputName; +- (MLMultiArray*)predictWithInput:(NSString*)inputName + data:(const __fp16*)data + shape:(NSArray*)shape + outputName:(NSString*)outputName; +- (BOOL)preallocateMultiInputBuffersWithOutputName:(NSString*)outputName; +- (size_t)predictMultiInputWithDict:(NSMutableDictionary*)inputDict + outputData:(__fp16*)output + outputName:(NSString*)outputName; + +@end + +@implementation CactusANEImpl + +- (instancetype)initWithModelPath:(NSString*)path { + self = [super init]; + if (self) { + NSError* error = nil; + NSURL* modelURL = resolve_or_compile_model_url(path, &error); + + MLModelConfiguration* config = [[MLModelConfiguration alloc] init]; + config.computeUnits = resolve_compute_units_for_model(path, false); + CACTUS_LOG_INFO("npu", + "ANEEncoder compute units: " + << compute_units_to_string(config.computeUnits) + << " for " << [path UTF8String]); + + _model = [MLModel modelWithContentsOfURL:modelURL configuration:config error:&error]; + if (_model) { + _modelDescription = _model.modelDescription; + } + if (error) { + CACTUS_LOG_ERROR("npu", "ANE model load failed: " << [[error localizedDescription] UTF8String]); + } + } + return self; +} + +- (NSArray*)getInputShape { + if (!_modelDescription) return @[]; + + NSString* inputName = _cachedInputName; + if (!inputName) { + inputName = _modelDescription.inputDescriptionsByName.allKeys.firstObject; + } + + MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; + if (inputDesc && inputDesc.type == MLFeatureTypeMultiArray) { + return inputDesc.multiArrayConstraint.shape; + } + + return @[]; +} + +- (NSArray*)getOutputShape { + if (!_modelDescription) return @[]; + + NSString* outputName = _cachedOutputName; + if (!outputName) { + outputName = _modelDescription.outputDescriptionsByName.allKeys.firstObject; + } + + MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; + if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { + return outputDesc.multiArrayConstraint.shape; + } + + return @[]; +} + +- (BOOL)preallocateBuffersWithInput:(NSString*)inputName + shape:(NSArray*)shape + outputName:(NSString*)outputName { + if (!_model) return NO; + + NSError* error = nil; + + MLMultiArrayDataType inputDataType = MLMultiArrayDataTypeFloat16; + MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; + if (inputDesc && inputDesc.multiArrayConstraint) { + inputDataType = inputDesc.multiArrayConstraint.dataType; + } + + _cachedInputArray = [[MLMultiArray alloc] + initWithShape:shape + dataType:inputDataType + error:&error]; + + if (error) { + CACTUS_LOG_ERROR("npu", "ANE preallocate input array failed: " << [[error localizedDescription] UTF8String]); + return NO; + } + + _cachedShape = [shape copy]; + _cachedInputName = [inputName copy]; + _cachedOutputName = outputName ? [outputName copy] + : _modelDescription.outputDescriptionsByName.allKeys.firstObject; + + CACTUS_LOG_DEBUG("npu", + "[CactusANE] prealloc input name=" + << [_cachedInputName UTF8String] + << " shape=" << [[_cachedInputArray.shape description] UTF8String] + << " strides=" << [[_cachedInputArray.strides description] UTF8String] + << " dtype=" << (long)_cachedInputArray.dataType); + + MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[_cachedOutputName]; + if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { + NSArray* outputShape = outputDesc.multiArrayConstraint.shape; + MLMultiArrayDataType outputDataType = outputDesc.multiArrayConstraint.dataType; + + _cachedOutputArray = [[MLMultiArray alloc] + initWithShape:outputShape + dataType:outputDataType + error:&error]; + + if (error) { + CACTUS_LOG_ERROR("npu", "ANE preallocate output array failed: " << [[error localizedDescription] UTF8String]); + return NO; + } + + _cachedOutputShape = [outputShape copy]; + _cachedOutputSize = 1; + for (NSNumber* dim in outputShape) { + _cachedOutputSize *= [dim unsignedIntegerValue]; + } + + CACTUS_LOG_DEBUG("npu", + "[CactusANE] prealloc output name=" + << [_cachedOutputName UTF8String] + << " shape=" << [[_cachedOutputArray.shape description] UTF8String] + << " strides=" << [[_cachedOutputArray.strides description] UTF8String] + << " dtype=" << (long)_cachedOutputArray.dataType); + + _predictionOptions = [[MLPredictionOptions alloc] init]; + _hasOutputBackings = NO; + if (_predictionOptions && _cachedOutputArray && _cachedOutputName) { + _predictionOptions.outputBackings = + @{ _cachedOutputName: _cachedOutputArray }; + _hasOutputBackings = YES; + } + } + + return YES; +} + +- (BOOL)canUseCachedBufferWithInput:(NSString*)inputName + shape:(NSArray*)shape + outputName:(NSString*)outputName { + if (!_cachedInputArray || !_cachedShape) return NO; + if (![_cachedInputName isEqualToString:inputName]) return NO; + if (_cachedShape.count != shape.count) return NO; + + for (NSUInteger i = 0; i < shape.count; i++) { + if (![_cachedShape[i] isEqualToNumber:shape[i]]) return NO; + } + + if (outputName && outputName.length > 0 && ![_cachedOutputName isEqualToString:outputName]) { + return NO; + } + + return YES; +} + +- (MLMultiArray*)predictWithInput:(NSString*)inputName + data:(const __fp16*)data + shape:(NSArray*)shape + outputName:(NSString*)outputName { + if (!_model) return nil; + + NSError* error = nil; + MLMultiArray* inputArray = nil; + + MLMultiArrayDataType expectedDataType = MLMultiArrayDataTypeFloat16; + MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; + if (inputDesc && inputDesc.multiArrayConstraint) { + expectedDataType = inputDesc.multiArrayConstraint.dataType; + } + + BOOL useCached = (expectedDataType == MLMultiArrayDataTypeFloat16) && + [self canUseCachedBufferWithInput:inputName shape:shape outputName:outputName]; + + if (useCached) { + inputArray = _cachedInputArray; + } else { + inputArray = [[MLMultiArray alloc] + initWithShape:shape + dataType:expectedDataType + error:&error]; + + if (error) { + CACTUS_LOG_ERROR("npu", "ANE create input array failed: " << [[error localizedDescription] UTF8String]); + return nil; + } + } + + NSUInteger totalElements = 1; + for (NSNumber* dim in shape) { + totalElements *= [dim unsignedIntegerValue]; + } + + if (expectedDataType == MLMultiArrayDataTypeFloat16) { + __fp16* inputPtr = (__fp16*)inputArray.dataPointer; + memcpy(inputPtr, data, totalElements * sizeof(__fp16)); + } else { + float* inputPtr = (float*)inputArray.dataPointer; + for (NSUInteger i = 0; i < totalElements; i++) { + inputPtr[i] = (float)data[i]; + } + } + + MLFeatureValue* inputFeature = [MLFeatureValue featureValueWithMultiArray:inputArray]; + NSDictionary* inputDict = @{inputName: inputFeature}; + id inputProvider = [[MLDictionaryFeatureProvider alloc] + initWithDictionary:inputDict + error:&error]; + + if (error) { + CACTUS_LOG_ERROR("npu", "ANE create feature provider failed: " << [[error localizedDescription] UTF8String]); + return nil; + } + + id outputProvider = nil; + + if (useCached && _hasOutputBackings && _predictionOptions) { + outputProvider = [_model predictionFromFeatures:inputProvider + options:_predictionOptions + error:&error]; + } else { + outputProvider = [_model predictionFromFeatures:inputProvider error:&error]; + } + + if (error) { + CACTUS_LOG_ERROR("npu", "ANE prediction failed: " << [[error localizedDescription] UTF8String]); + return nil; + } + + NSString* outName = outputName; + if (!outName || outName.length == 0) { + outName = useCached ? _cachedOutputName + : _modelDescription.outputDescriptionsByName.allKeys.firstObject; + } + + MLFeatureValue* outputFeature = [outputProvider featureValueForName:outName]; + static bool logged_output_feature_layout = false; + if (!logged_output_feature_layout && outputFeature && outputFeature.multiArrayValue) { + MLMultiArray* outArr = outputFeature.multiArrayValue; + CACTUS_LOG_DEBUG("npu", + "[CactusANE] output feature name=" + << [outName UTF8String] + << " shape=" << [[outArr.shape description] UTF8String] + << " strides=" << [[outArr.strides description] UTF8String] + << " dtype=" << (long)outArr.dataType); + logged_output_feature_layout = true; + } + return outputFeature.multiArrayValue; +} + +static size_t copyMLArrayToFP16(MLMultiArray* array, __fp16* output) { + size_t count = array.count; + if (!array || !output || count == 0) return 0; + + std::vector dims; + std::vector strides; + const size_t rank = array.shape.count; + bool have_layout = rank == array.strides.count && rank > 0; + if (have_layout) { + dims.resize(rank); + strides.resize(rank); + for (size_t i = 0; i < rank; ++i) { + NSInteger d = [array.shape[i] integerValue]; + NSInteger s = [array.strides[i] integerValue]; + if (d <= 0 || s < 0) { + have_layout = false; + break; + } + dims[i] = static_cast(d); + strides[i] = static_cast(s); + } + } + + if (!have_layout) { + if (array.dataType == MLMultiArrayDataTypeFloat16) { + if (output != (__fp16*)array.dataPointer) { + memcpy(output, array.dataPointer, count * sizeof(__fp16)); + } + } else { + const float* src = (const float*)array.dataPointer; + for (size_t i = 0; i < count; i++) output[i] = (__fp16)src[i]; + } + return count; + } + + auto is_contiguous = [&]() -> bool { + if (!have_layout) return false; + size_t expected = 1; + for (size_t i = rank; i-- > 0;) { + if (dims[i] > 1 && strides[i] != expected) return false; + expected *= dims[i]; + } + return true; + }; + + if (false && is_contiguous()) { + if (array.dataType == MLMultiArrayDataTypeFloat16) { + if (output != (__fp16*)array.dataPointer) { + memcpy(output, array.dataPointer, count * sizeof(__fp16)); + } + } else { + const float* src = (const float*)array.dataPointer; + for (size_t i = 0; i < count; i++) output[i] = (__fp16)src[i]; + } + return count; + } + + std::vector idx(rank, 0); + if (array.dataType == MLMultiArrayDataTypeFloat16) { + const __fp16* src = (const __fp16*)array.dataPointer; + for (size_t i = 0; i < count; ++i) { + size_t offset = 0; + for (size_t d = 0; d < rank; ++d) { + offset += idx[d] * strides[d]; + } + output[i] = src[offset]; + + for (size_t d = rank; d-- > 0;) { + idx[d]++; + if (idx[d] < dims[d]) break; + idx[d] = 0; + } + } + } else { + const float* src = (const float*)array.dataPointer; + for (size_t i = 0; i < count; ++i) { + size_t offset = 0; + for (size_t d = 0; d < rank; ++d) { + offset += idx[d] * strides[d]; + } + output[i] = (__fp16)src[offset]; + + for (size_t d = rank; d-- > 0;) { + idx[d]++; + if (idx[d] < dims[d]) break; + idx[d] = 0; + } + } + } + return count; +} + +static void copyFP16ToMLArray(const __fp16* data, size_t count, MLMultiArray* array) { + if (!array || !data || count == 0) return; + + std::vector dims; + std::vector strides; + const size_t rank = array.shape.count; + bool have_layout = rank == array.strides.count && rank > 0; + if (have_layout) { + dims.resize(rank); + strides.resize(rank); + for (size_t i = 0; i < rank; ++i) { + NSInteger d = [array.shape[i] integerValue]; + NSInteger s = [array.strides[i] integerValue]; + if (d <= 0 || s < 0) { + have_layout = false; + break; + } + dims[i] = static_cast(d); + strides[i] = static_cast(s); + } + } + + if (!have_layout) { + if (array.dataType == MLMultiArrayDataTypeFloat16) { + memcpy(array.dataPointer, data, count * sizeof(__fp16)); + } else { + float* dst = (float*)array.dataPointer; + for (size_t i = 0; i < count; i++) dst[i] = (float)data[i]; + } + return; + } + + auto is_contiguous = [&]() -> bool { + if (!have_layout) return false; + size_t expected = 1; + for (size_t i = rank; i-- > 0;) { + if (dims[i] > 1 && strides[i] != expected) return false; + expected *= dims[i]; + } + return true; + }; + + if (false && is_contiguous()) { + if (array.dataType == MLMultiArrayDataTypeFloat16) { + memcpy(array.dataPointer, data, count * sizeof(__fp16)); + } else { + float* dst = (float*)array.dataPointer; + for (size_t i = 0; i < count; i++) dst[i] = (float)data[i]; + } + return; + } + + std::vector idx(rank, 0); + if (array.dataType == MLMultiArrayDataTypeFloat16) { + __fp16* dst = (__fp16*)array.dataPointer; + for (size_t i = 0; i < count; ++i) { + size_t offset = 0; + for (size_t d = 0; d < rank; ++d) { + offset += idx[d] * strides[d]; + } + dst[offset] = data[i]; + + for (size_t d = rank; d-- > 0;) { + idx[d]++; + if (idx[d] < dims[d]) break; + idx[d] = 0; + } + } + } else { + float* dst = (float*)array.dataPointer; + for (size_t i = 0; i < count; ++i) { + size_t offset = 0; + for (size_t d = 0; d < rank; ++d) { + offset += idx[d] * strides[d]; + } + dst[offset] = (float)data[i]; + + for (size_t d = rank; d-- > 0;) { + idx[d]++; + if (idx[d] < dims[d]) break; + idx[d] = 0; + } + } + } +} + +- (BOOL)preallocateMultiInputBuffersWithOutputName:(NSString*)outputName { + if (!_model || !_modelDescription) return NO; + + NSError* error = nil; + _cachedMultiInputArrays = [NSMutableDictionary new]; + + for (NSString* inputName in _modelDescription.inputDescriptionsByName) { + MLFeatureDescription* desc = _modelDescription.inputDescriptionsByName[inputName]; + if (desc.type != MLFeatureTypeMultiArray) continue; + + MLMultiArray* array = [[MLMultiArray alloc] + initWithShape:desc.multiArrayConstraint.shape + dataType:desc.multiArrayConstraint.dataType + error:&error]; + if (!array || error) return NO; + _cachedMultiInputArrays[inputName] = array; + } + + NSString* outName = (outputName && outputName.length > 0) + ? outputName : _modelDescription.outputDescriptionsByName.allKeys.firstObject; + + MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outName]; + if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { + _cachedMultiOutputArray = [[MLMultiArray alloc] + initWithShape:outputDesc.multiArrayConstraint.shape + dataType:outputDesc.multiArrayConstraint.dataType + error:&error]; + if (!_cachedMultiOutputArray || error) return NO; + + _multiPredictionOptions = [[MLPredictionOptions alloc] init]; + _hasMultiOutputBackings = NO; + } + + _multiInputPreallocated = YES; + return YES; +} + +- (size_t)predictMultiInputWithDict:(NSMutableDictionary*)inputDict + outputData:(__fp16*)output + outputName:(NSString*)outputName { + if (!_model) return 0; + + NSError* error = nil; + MLDictionaryFeatureProvider* provider = + [[MLDictionaryFeatureProvider alloc] initWithDictionary:inputDict error:&error]; + if (!provider || error) return 0; + + id result = nil; + if (_hasMultiOutputBackings && _multiPredictionOptions) { + result = [_model predictionFromFeatures:provider options:_multiPredictionOptions error:&error]; + } else { + result = [_model predictionFromFeatures:provider error:&error]; + } + if (!result || error) return 0; + + static bool logged_multi_layout = false; + if (!logged_multi_layout) { + for (NSString* key in inputDict) { + MLFeatureValue* inFeature = inputDict[key]; + if (inFeature && inFeature.multiArrayValue) { + MLMultiArray* arr = inFeature.multiArrayValue; + CACTUS_LOG_WARN("npu", + "[CactusANE] multi input name=" + << [key UTF8String] + << " shape=" << [[arr.shape description] UTF8String] + << " strides=" << [[arr.strides description] UTF8String] + << " dtype=" << (long)arr.dataType); + } + } + } + + MLFeatureValue* outFeature = [result featureValueForName:outputName]; + if (!outFeature || !outFeature.multiArrayValue) return 0; + if (!logged_multi_layout) { + MLMultiArray* outArr = outFeature.multiArrayValue; + CACTUS_LOG_WARN("npu", + "[CactusANE] multi output name=" + << [outputName UTF8String] + << " shape=" << [[outArr.shape description] UTF8String] + << " strides=" << [[outArr.strides description] UTF8String] + << " dtype=" << (long)outArr.dataType); + logged_multi_layout = true; + } + return copyMLArrayToFP16(outFeature.multiArrayValue, output); +} + +@end + +namespace cactus { +namespace npu { + +static bool g_npu_enabled = true; + +ANEEncoder::ANEEncoder() : impl_(nullptr) {} + +ANEEncoder::~ANEEncoder() { + if (impl_) { + (void)(__bridge_transfer CactusANEImpl*)impl_; + impl_ = nullptr; + } +} + +ANEEncoder::ANEEncoder(ANEEncoder&& other) noexcept : impl_(other.impl_) { + other.impl_ = nullptr; +} + +ANEEncoder& ANEEncoder::operator=(ANEEncoder&& other) noexcept { + if (this != &other) { + if (impl_) { + (void)(__bridge_transfer CactusANEImpl*)impl_; + } + impl_ = other.impl_; + other.impl_ = nullptr; + } + return *this; +} + +bool ANEEncoder::load(const std::string& model_path) { + @autoreleasepool { + CACTUS_LOG_INFO("npu", "ANEEncoder loading model: " << model_path); + NSString* path = [NSString stringWithUTF8String:model_path.c_str()]; + + if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { + CACTUS_LOG_WARN("npu", "not using NPU"); + return false; + } + + CactusANEImpl* impl = [[CactusANEImpl alloc] initWithModelPath:path]; + + if (impl && impl.model) { + impl_ = (__bridge_retained void*)impl; + CACTUS_LOG_INFO("npu", "ANEEncoder model loaded successfully: " << model_path); + return true; + } + CACTUS_LOG_ERROR("npu", "ANEEncoder model load failed: " << model_path); + return false; + } +} + +bool ANEEncoder::preallocate(const std::vector& input_shape, + const std::string& input_name, + const std::string& output_name) { + if (!impl_) return false; + + @autoreleasepool { + CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; + + NSMutableArray* shapeArray = [NSMutableArray array]; + for (int dim : input_shape) { + [shapeArray addObject:@(dim)]; + } + + NSString* inName = [NSString stringWithUTF8String:input_name.c_str()]; + NSString* outName = output_name.empty() + ? nil + : [NSString stringWithUTF8String:output_name.c_str()]; + + return [impl preallocateBuffersWithInput:inName shape:shapeArray outputName:outName]; + } +} + +size_t ANEEncoder::encode(const __fp16* input, + __fp16* output, + const std::vector& shape, + const std::string& input_name, + const std::string& output_name) { + if (!impl_ || !input || !output) return 0; + + @autoreleasepool { + CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; + + NSArray* shapeArray = impl.cachedShape; + bool shapeMatches = (shapeArray && shapeArray.count == shape.size()); + if (shapeMatches) { + for (size_t i = 0; i < shape.size(); ++i) { + if ([shapeArray[i] intValue] != shape[i]) { + shapeMatches = false; + break; + } + } + } + + if (!shapeMatches) { + NSMutableArray* newShapeArray = [NSMutableArray arrayWithCapacity:shape.size()]; + for (int dim : shape) { + [newShapeArray addObject:@(dim)]; + } + shapeArray = newShapeArray; + } + + NSString* inName = impl.cachedInputName; + if (!inName) { + inName = [NSString stringWithUTF8String:input_name.c_str()]; + } + NSString* outName = impl.cachedOutputName; + if (!outName && !output_name.empty()) { + outName = [NSString stringWithUTF8String:output_name.c_str()]; + } + + MLMultiArray* mlOutput = [impl predictWithInput:inName + data:input + shape:shapeArray + outputName:outName]; + + if (mlOutput) { + return copyMLArrayToFP16(mlOutput, output); + } + } + + return 0; +} + +bool ANEEncoder::is_available() const { + if (!impl_) return false; + CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; + return impl.model != nil; +} + +std::vector ANEEncoder::get_input_shape() const { + std::vector result; + if (!impl_) return result; + CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; + NSArray* shape = [impl getInputShape]; + for (NSNumber* dim in shape) { + result.push_back([dim intValue]); + } + return result; +} + +std::vector ANEEncoder::get_output_shape() const { + std::vector result; + if (!impl_) return result; + CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; + NSArray* shape = [impl getOutputShape]; + for (NSNumber* dim in shape) { + result.push_back([dim intValue]); + } + return result; +} + +__fp16* ANEEncoder::get_output_buffer() { + if (!impl_) return nullptr; + CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; + if (!impl.hasOutputBackings || !impl.cachedOutputArray) return nullptr; + return (__fp16*)impl.cachedOutputArray.dataPointer; +} + +size_t ANEEncoder::get_output_buffer_size() const { + if (!impl_) return 0; + CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; + if (!impl.hasOutputBackings || !impl.cachedOutputArray) return 0; + return impl.cachedOutputSize; +} + +size_t ANEEncoder::encode_multimodal_input( + const std::vector& inputs, + __fp16* output, + const std::string& output_name) { + if (!impl_ || !output || inputs.empty()) return 0; + + @autoreleasepool { + CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; + + NSString* outName = output_name.empty() + ? impl.modelDescription.outputDescriptionsByName.allKeys.firstObject + : [NSString stringWithUTF8String:output_name.c_str()]; + + if (!impl.multiInputPreallocated) { + [impl preallocateMultiInputBuffersWithOutputName:outName]; + } + + NSMutableDictionary* inputDict = [NSMutableDictionary new]; + + for (const auto& input : inputs) { + NSString* nsName = [NSString stringWithUTF8String:input.name.c_str()]; + + size_t total = 1; + for (int dim : input.shape) total *= dim; + + MLMultiArray* array = impl.cachedMultiInputArrays[nsName]; + if (!array) { + MLMultiArrayDataType dataType = MLMultiArrayDataTypeFloat16; + MLFeatureDescription* desc = impl.modelDescription.inputDescriptionsByName[nsName]; + if (desc && desc.multiArrayConstraint) dataType = desc.multiArrayConstraint.dataType; + + NSMutableArray* shapeArray = [NSMutableArray arrayWithCapacity:input.shape.size()]; + for (int dim : input.shape) [shapeArray addObject:@(dim)]; + + NSError* arrayError = nil; + array = [[MLMultiArray alloc] initWithShape:shapeArray dataType:dataType error:&arrayError]; + if (!array || arrayError) return 0; + } + + copyFP16ToMLArray(input.data, total, array); + inputDict[nsName] = [MLFeatureValue featureValueWithMultiArray:array]; + } + + return [impl predictMultiInputWithDict:inputDict outputData:output outputName:outName]; + } +} + +std::unique_ptr create_encoder() { + return std::make_unique(); +} + +bool is_npu_available() { + CACTUS_LOG_INFO("npu", "is_npu_available: " << (g_npu_enabled ? "true" : "false")); + return g_npu_enabled; +} + +} // namespace npu +} // namespace cactus + +@interface CactusANEPrefillImpl : NSObject + +@property (nonatomic, strong) MLModel* model; +@property (nonatomic, strong) MLModelDescription* modelDescription; +@property (nonatomic, assign) int chunkSize; +@property (nonatomic, assign) int hiddenDim; +@property (nonatomic, assign) int numLayers; +@property (nonatomic, assign) int numKvHeads; +@property (nonatomic, assign) int headDim; + +@property (nonatomic, strong) MLMultiArray* cachedInputArray; +@property (nonatomic, strong) MLMultiArray* cachedOffsetArray; +@property (nonatomic, strong) NSMutableDictionary* cachedOutputArrays; +@property (nonatomic, strong) MLPredictionOptions* predictionOptions; +@property (nonatomic, assign) BOOL hasOutputBackings; + +- (instancetype)initWithModelPath:(NSString*)path; +- (void)preallocateBuffers; +- (BOOL)predictDirectWithInput:(NSString*)inputName + data:(const __fp16*)data + offset:(int)offset; +- (MLMultiArray*)getOutputArray:(NSString*)name; +@end + +@implementation CactusANEPrefillImpl + +- (instancetype)initWithModelPath:(NSString*)path { + self = [super init]; + if (self) { + NSError* error = nil; + NSURL* modelURL = resolve_or_compile_model_url(path, &error); + + MLModelConfiguration* config = [[MLModelConfiguration alloc] init]; + config.computeUnits = resolve_compute_units_for_model(path, true); + CACTUS_LOG_INFO("npu", + "ANEPrefill compute units: " + << compute_units_to_string(config.computeUnits) + << " for " << [path UTF8String]); + + _model = [MLModel modelWithContentsOfURL:modelURL configuration:config error:&error]; + if (_model) { + _modelDescription = _model.modelDescription; + [self inferModelDimensions]; + [self preallocateBuffers]; + } + if (error) { + CACTUS_LOG_ERROR("npu", "ANE prefill model load failed: " << [[error localizedDescription] UTF8String]); + } + } + return self; +} + +- (void)inferModelDimensions { + if (!_modelDescription) return; + + NSString* inputName = _modelDescription.inputDescriptionsByName.allKeys.firstObject; + MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; + if (inputDesc && inputDesc.type == MLFeatureTypeMultiArray) { + NSArray* shape = inputDesc.multiArrayConstraint.shape; + if (shape.count >= 2) { + _chunkSize = [shape[0] intValue]; + _hiddenDim = [shape[1] intValue]; + } + } + + int maxLayerIdx = -1; + for (NSString* outputName in _modelDescription.outputDescriptionsByName.allKeys) { + if ([outputName hasPrefix:@"k_"]) { + int layerIdx = [[outputName substringFromIndex:2] intValue]; + maxLayerIdx = MAX(maxLayerIdx, layerIdx); + + MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; + if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { + NSArray* shape = outputDesc.multiArrayConstraint.shape; + if (shape.count >= 3) { + _numKvHeads = [shape[1] intValue]; + _headDim = [shape[2] intValue]; + } + } + } + } + _numLayers = maxLayerIdx + 1; +} + +- (void)preallocateBuffers { + if (!_model || !_modelDescription || _chunkSize == 0 || _hiddenDim == 0) return; + + NSError* error = nil; + + NSArray* inputShape = @[@(_chunkSize), @(_hiddenDim)]; + _cachedInputArray = [[MLMultiArray alloc] initWithShape:inputShape + dataType:MLMultiArrayDataTypeFloat16 + error:&error]; + if (error) { + _cachedInputArray = nil; + return; + } + + if (_modelDescription.inputDescriptionsByName[@"offset"] != nil) { + _cachedOffsetArray = [[MLMultiArray alloc] initWithShape:@[@1] + dataType:MLMultiArrayDataTypeFloat16 + error:&error]; + if (error) { + _cachedOffsetArray = nil; + } + } + + _cachedOutputArrays = [NSMutableDictionary dictionary]; + NSMutableDictionary* outputBackings = [NSMutableDictionary dictionary]; + + for (NSString* outputName in _modelDescription.outputDescriptionsByName.allKeys) { + MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; + if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { + NSArray* outputShape = outputDesc.multiArrayConstraint.shape; + + MLMultiArray* outputArray = [[MLMultiArray alloc] initWithShape:outputShape + dataType:MLMultiArrayDataTypeFloat16 + error:&error]; + if (!error && outputArray) { + _cachedOutputArrays[outputName] = outputArray; + outputBackings[outputName] = outputArray; + } + } + } + + _predictionOptions = [[MLPredictionOptions alloc] init]; + _hasOutputBackings = NO; + if (@available(macOS 14.0, iOS 17.0, *)) { + _predictionOptions.outputBackings = outputBackings; + _hasOutputBackings = YES; + } +} + +- (BOOL)predictDirectWithInput:(NSString*)inputName + data:(const __fp16*)data + offset:(int)offset { + if (!_model || !_cachedInputArray) return NO; + + NSError* error = nil; + + NSUInteger totalElements = (NSUInteger)(_chunkSize * _hiddenDim); + __fp16* inputPtr = (__fp16*)_cachedInputArray.dataPointer; + memcpy(inputPtr, data, totalElements * sizeof(__fp16)); + + MLFeatureValue* inputFeature = [MLFeatureValue featureValueWithMultiArray:_cachedInputArray]; + NSMutableDictionary* inputDict = [NSMutableDictionary dictionaryWithObject:inputFeature forKey:inputName]; + + if (_cachedOffsetArray) { + ((__fp16*)_cachedOffsetArray.dataPointer)[0] = (__fp16)offset; + MLFeatureValue* offsetFeature = [MLFeatureValue featureValueWithMultiArray:_cachedOffsetArray]; + inputDict[@"offset"] = offsetFeature; + } + + id inputProvider = [[MLDictionaryFeatureProvider alloc] + initWithDictionary:inputDict + error:&error]; + if (error) return NO; + + id outputProvider = [_model predictionFromFeatures:inputProvider + options:_predictionOptions + error:&error]; + if (error || !outputProvider) return NO; + + if (!_hasOutputBackings) { + for (NSString* outputName in _cachedOutputArrays) { + MLFeatureValue* outFeature = [outputProvider featureValueForName:outputName]; + if (!outFeature || !outFeature.multiArrayValue) return NO; + + MLMultiArray* dst = _cachedOutputArrays[outputName]; + if (!dst) return NO; + + size_t copied = copyMLArrayToFP16(outFeature.multiArrayValue, (__fp16*)dst.dataPointer); + if (copied == 0) return NO; + } + } + + return YES; +} + +- (MLMultiArray*)getOutputArray:(NSString*)name { + return _cachedOutputArrays[name]; +} + +@end + +namespace cactus { +namespace npu { + +ANEPrefill::ANEPrefill() : impl_(nullptr) {} + +ANEPrefill::~ANEPrefill() { + if (impl_) { + (void)(__bridge_transfer CactusANEPrefillImpl*)impl_; + impl_ = nullptr; + } +} + +ANEPrefill::ANEPrefill(ANEPrefill&& other) noexcept : impl_(other.impl_), + chunk_size_(other.chunk_size_), hidden_dim_(other.hidden_dim_), + num_layers_(other.num_layers_), num_kv_heads_(other.num_kv_heads_), + head_dim_(other.head_dim_) { + other.impl_ = nullptr; +} + +ANEPrefill& ANEPrefill::operator=(ANEPrefill&& other) noexcept { + if (this != &other) { + if (impl_) { + (void)(__bridge_transfer CactusANEPrefillImpl*)impl_; + } + impl_ = other.impl_; + chunk_size_ = other.chunk_size_; + hidden_dim_ = other.hidden_dim_; + num_layers_ = other.num_layers_; + num_kv_heads_ = other.num_kv_heads_; + head_dim_ = other.head_dim_; + other.impl_ = nullptr; + } + return *this; +} + +bool ANEPrefill::load(const std::string& model_path) { + @autoreleasepool { + CACTUS_LOG_INFO("npu", "ANEPrefill loading model: " << model_path); + NSString* path = [NSString stringWithUTF8String:model_path.c_str()]; + + if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { + CACTUS_LOG_DEBUG("npu", "ANEPrefill model file not found: " << model_path); + return false; + } + + CactusANEPrefillImpl* impl = [[CactusANEPrefillImpl alloc] initWithModelPath:path]; + + if (impl && impl.model) { + impl_ = (__bridge_retained void*)impl; + chunk_size_ = impl.chunkSize; + hidden_dim_ = impl.hiddenDim; + num_layers_ = impl.numLayers; + num_kv_heads_ = impl.numKvHeads; + head_dim_ = impl.headDim; + CACTUS_LOG_INFO("npu", "ANEPrefill model loaded successfully: " << model_path); + return true; + } + CACTUS_LOG_ERROR("npu", "ANEPrefill model load failed: " << model_path); + return false; + } +} + +bool ANEPrefill::is_available() const { + if (!impl_) return false; + CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; + return impl.model != nil; +} + +int ANEPrefill::get_chunk_size() const { return chunk_size_; } +int ANEPrefill::get_hidden_dim() const { return hidden_dim_; } +int ANEPrefill::get_num_layers() const { return num_layers_; } +int ANEPrefill::get_num_kv_heads() const { return num_kv_heads_; } +int ANEPrefill::get_head_dim() const { return head_dim_; } + +NPUPrefillDirectResult ANEPrefill::prefill_chunk_direct( + const std::vector<__fp16>& embeddings, + int position_offset, + const std::string& input_name) { + + NPUPrefillDirectResult result; + result.valid = false; + result.hidden = {nullptr, 0}; + result.k_caches.resize(num_layers_, {nullptr, 0}); + result.v_caches.resize(num_layers_, {nullptr, 0}); + + if (!impl_) return result; + + @autoreleasepool { + CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; + + NSString* inName = [NSString stringWithUTF8String:input_name.c_str()]; + + BOOL success = [impl predictDirectWithInput:inName + data:embeddings.data() + offset:position_offset]; + + if (!success) return result; + + MLMultiArray* hiddenArray = [impl getOutputArray:@"hidden"]; + if (hiddenArray) { + result.hidden.data = (__fp16*)hiddenArray.dataPointer; + result.hidden.count = hiddenArray.count; + } + + for (int layer = 0; layer < num_layers_; layer++) { + NSString* kName = [NSString stringWithFormat:@"k_%d", layer]; + NSString* vName = [NSString stringWithFormat:@"v_%d", layer]; + + MLMultiArray* kArray = [impl getOutputArray:kName]; + MLMultiArray* vArray = [impl getOutputArray:vName]; + + if (kArray) { + result.k_caches[layer].data = (__fp16*)kArray.dataPointer; + result.k_caches[layer].count = kArray.count; + } + if (vArray) { + result.v_caches[layer].data = (__fp16*)vArray.dataPointer; + result.v_caches[layer].count = vArray.count; + } + } + + result.valid = true; + } + + return result; +} + +std::unique_ptr create_prefill() { + return std::make_unique(); +} + +} // namespace npu +} // namespace cactus + +#else // !CACTUS_HAS_ANE + +namespace cactus { +namespace npu { + +std::unique_ptr create_encoder() { + return nullptr; +} + +bool is_npu_available() { + return false; +} + +std::unique_ptr create_prefill() { + return nullptr; +} + +} // namespace npu +} // namespace cactus + +#endif // CACTUS_HAS_ANE diff --git a/cactus-transpile/telemetry/telemetry.cpp b/cactus-transpile/telemetry/telemetry.cpp new file mode 100644 index 000000000..f45a7431b --- /dev/null +++ b/cactus-transpile/telemetry/telemetry.cpp @@ -0,0 +1,1546 @@ +#include "telemetry/telemetry.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(__APPLE__) +#include +#include +#endif +#include +#include +#include +#include +#include + +namespace cactus { +namespace telemetry { + +enum EventType { INIT = 0, COMPLETION = 1, EMBEDDING = 2, TRANSCRIPTION = 3, STREAM_TRANSCRIBE = 4 }; + +struct Event { + EventType type; + char model[128]; + bool success; + bool cloud_handoff; + double ttft_ms; + double prefill_tps; + double decode_tps; + double tps; + double response_time_ms; + double confidence; + double ram_usage_mb; + int tokens; + int prefill_tokens; + int decode_tokens; + double session_ttft_ms; + double session_tps; + double session_time_ms; + int session_tokens; + char message[256]; + char error[256]; + char function_calls[1024]; + std::chrono::system_clock::time_point timestamp; +}; +static bool enabled = false; +static int inference_active = 0; +static bool shutdown_called = false; +static bool atexit_registered = false; +static bool curl_initialized = false; +static bool cloud_disabled = false; +static std::string supabase_url = "https://vlqqczxwyaodtcdmdmlw.supabase.co"; +static std::string supabase_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZscXFjenh3eWFvZHRjZG1kbWx3Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTE1MTg2MzIsImV4cCI6MjA2NzA5NDYzMn0.nBzqGuK9j6RZ6mOPWU2boAC_5H9XDs-fPpo5P3WZYbI"; +static std::string device_id; +static std::string project_id; +static std::string cloud_key; +static std::string project_scope; +static std::string device_model; +static std::string device_os; +static std::string device_os_version; +static std::string device_brand; +static std::string cactus_version; +#ifndef CACTUS_DEFAULT_FRAMEWORK +#define CACTUS_DEFAULT_FRAMEWORK "cpp" +#endif +static std::string framework = CACTUS_DEFAULT_FRAMEWORK; +static std::string custom_cache_location; +static std::string app_id; +static std::string device_registered_file; +static std::string project_registered_file; +static bool device_registered = false; +static bool project_registered = false; +static bool ids_ready = false; +static bool in_stream_mode = false; + +static std::mutex telemetry_mutex; +static std::condition_variable telemetry_lifecycle_cv; + +enum class TelemetryLifecycleState { + Stopped, + Running, + ShuttingDown, +}; + +static TelemetryLifecycleState lifecycle_state = TelemetryLifecycleState::Stopped; + +static bool can_record_event_locked() { + return enabled && ids_ready && lifecycle_state == TelemetryLifecycleState::Running; +} + +struct CloudSendResult { + bool payload_ok = false; + bool project_registered_ok = false; + bool device_registered_ok = false; +}; + +struct CloudConfigurationStateSnapshot { + std::string supabase_url; + std::string supabase_key; + std::string device_id; + std::string project_id; + std::string cloud_key; + std::string project_scope; + std::string framework; + std::string cactus_version; + std::string device_model; + std::string device_os; + std::string device_os_version; + std::string device_brand; + std::string app_id; + std::string device_registered_file; + std::string project_registered_file; + bool enabled = false; + bool cloud_disabled = false; + bool device_registered = false; + bool project_registered = false; +}; + +static std::string new_uuid(); +static std::string format_timestamp(const std::chrono::system_clock::time_point& tp); +static std::string model_basename(const char* model_path) { + if (!model_path) return {}; + std::string m(model_path); + size_t pos = m.find_last_of("/\\"); + if (pos != std::string::npos && pos + 1 < m.size()) { + m = m.substr(pos + 1); + } + return m; +} + +// Forward declarations for helpers used before definition +static std::string event_type_to_string(EventType t); +static bool event_type_from_string(const std::string& s, EventType& t); +static bool extract_string_field(const std::string& line, const std::string& key, std::string& out); +static bool extract_json_field(const std::string& line, const std::string& key, std::string& out); +static bool extract_bool_field(const std::string& line, const std::string& key, bool& out); +static bool extract_double_field(const std::string& line, const std::string& key, double& out); +static bool extract_int_field(const std::string& line, const std::string& key, int& out); +static bool extract_double_field_raw(const std::string& line, const std::string& key, double& out); +static void process_events(const std::vector& fresh_events); +static std::string get_telemetry_dir_locked(); +static CloudConfigurationStateSnapshot capture_cloud_configuration_state_snapshot_locked(); + +class TelemetryDispatcher { +public: + TelemetryDispatcher() = default; + + static TelemetryDispatcher& instance() { + static std::once_flag once; + static TelemetryDispatcher* singleton = nullptr; + std::call_once(once, [] { + singleton = new TelemetryDispatcher(); + }); + return *singleton; + } + + void start() { + std::lock_guard lock(queue_mutex_); + if (running_) return; + stop_ = false; + processing_ = false; + running_ = true; + worker_thread_ = std::thread([this] { worker_loop(); }); + } + + bool enqueue(const Event& event) { + std::lock_guard lock(queue_mutex_); + if (!running_) return false; + pending_events_.push_back(event); + enqueue_seq_ += 1; + queue_cv_.notify_one(); + return true; + } + + void flush() { + uint64_t target = 0; + { + std::unique_lock lock(queue_mutex_); + if (running_) { + target = enqueue_seq_; + queue_cv_.notify_one(); + flush_cv_.wait(lock, [this, target] { + return processed_seq_ >= target && !processing_; + }); + } + } + process_with_io_lock({}); + } + + void stop() { + { + std::lock_guard lock(queue_mutex_); + stop_ = true; + queue_cv_.notify_all(); + } + + if (worker_thread_.joinable()) { + worker_thread_.join(); + } + + std::lock_guard lock(queue_mutex_); + running_ = false; + stop_ = false; + processing_ = false; + pending_events_.clear(); + enqueue_seq_ = 0; + processed_seq_ = 0; + } + +private: + void process_with_io_lock(const std::vector& batch) { + std::lock_guard io_guard(io_mutex_); + process_events(batch); + } + + void worker_loop() { + while (true) { + std::vector batch; + uint64_t processed_count = 0; + { + std::unique_lock lock(queue_mutex_); + queue_cv_.wait(lock, [this] { + return stop_ || !pending_events_.empty(); + }); + if (stop_ && pending_events_.empty()) { + break; + } + batch.assign(pending_events_.begin(), pending_events_.end()); + processed_count = static_cast(batch.size()); + pending_events_.clear(); + processing_ = true; + } + + process_with_io_lock(batch); + + { + std::lock_guard lock(queue_mutex_); + processed_seq_ += processed_count; + processing_ = false; + flush_cv_.notify_all(); + } + } + + std::lock_guard lock(queue_mutex_); + running_ = false; + processing_ = false; + flush_cv_.notify_all(); + } + std::mutex queue_mutex_; + std::mutex io_mutex_; + std::condition_variable queue_cv_; + std::condition_variable flush_cv_; + std::deque pending_events_; + std::thread worker_thread_; + bool stop_ = false; + bool running_ = false; + bool processing_ = false; + uint64_t enqueue_seq_ = 0; + uint64_t processed_seq_ = 0; +}; + +static void mkdir_p(const std::string& path) { + if (path.empty()) return; + + std::string current; + for (size_t i = 0; i < path.size(); ++i) { + current += path[i]; + if (path[i] == '/' && i > 0) { + mkdir(current.c_str(), 0755); + } + } + mkdir(path.c_str(), 0755); +} + +static std::string get_telemetry_dir_locked() { + const std::string& cache_location = custom_cache_location; + if (!cache_location.empty()) { + mkdir_p(cache_location); + return cache_location; + } + +#if defined(__ANDROID__) + { + std::ifstream f("/proc/self/cmdline"); + if (f.is_open()) { + std::string pkg; + std::getline(f, pkg, '\0'); + size_t colon = pkg.find(':'); + if (colon != std::string::npos) pkg = pkg.substr(0, colon); + if (!pkg.empty()) { + uid_t uid = getuid(); + int user_id = static_cast((uid - 10000) / 100000); + std::string dir = "/data/user/" + std::to_string(user_id) + "/" + pkg + "/cache/cactus/telemetry"; + mkdir_p(dir); + struct stat st; + if (stat(dir.c_str(), &st) == 0) return dir; + dir = "/data/data/" + pkg + "/cache/cactus/telemetry"; + mkdir_p(dir); + if (stat(dir.c_str(), &st) == 0) return dir; + } + } + } +#endif + + const char* home = getenv("HOME"); + if (!home) home = "/tmp"; + std::string dir = std::string(home) + "/Library/Caches/cactus/telemetry"; + mkdir((std::string(home) + "/Library").c_str(), 0755); + mkdir((std::string(home) + "/Library/Caches").c_str(), 0755); + mkdir((std::string(home) + "/Library/Caches/cactus").c_str(), 0755); + mkdir(dir.c_str(), 0755); + return dir; +} + +static std::string cloud_api_key_cache_file_locked() { + return get_telemetry_dir_locked() + "/cloud_api_key"; +} + +static std::string load_or_create_id(const std::string& file) { + std::ifstream in(file); + if (in.is_open()) { + std::string line; + if (std::getline(in, line) && !line.empty()) { + return line; + } + } + std::string id = new_uuid(); + std::ofstream out(file, std::ios::trunc); + if (out.is_open()) { + out << id; + } + return id; +} + +static bool load_registered_flag(const std::string& file) { + std::ifstream in(file); + if (!in.is_open()) return false; + std::string line; + if (std::getline(in, line) && !line.empty()) { + return line[0] == '1'; + } + return false; +} + +static void persist_registered_flag(const std::string& file) { + std::ofstream out(file, std::ios::trunc); + if (out.is_open()) { + out << "1"; + } +} + +static std::string sysctl_string(const char* key) { +#if defined(__APPLE__) + size_t size = 0; + if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) return {}; + std::string out(size, '\0'); + if (sysctlbyname(key, out.data(), &size, nullptr, 0) != 0) return {}; + if (!out.empty() && out.back() == '\0') out.pop_back(); + return out; +#else + (void)key; + return {}; +#endif +} + +static Event make_event(EventType type, const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message) { + Event e; + e.type = type; + e.success = success; + e.cloud_handoff = false; + e.ttft_ms = ttft_ms; + e.prefill_tps = 0.0; + e.decode_tps = tps; + e.tps = tps; + e.response_time_ms = response_time_ms; + e.confidence = 0.0; + e.ram_usage_mb = 0.0; + e.tokens = tokens; + e.prefill_tokens = 0; + e.decode_tokens = tokens; + e.session_ttft_ms = 0.0; + e.session_tps = 0.0; + e.session_time_ms = 0.0; + e.session_tokens = 0; + e.timestamp = std::chrono::system_clock::now(); + std::memset(e.model, 0, sizeof(e.model)); + std::memset(e.message, 0, sizeof(e.message)); + std::memset(e.error, 0, sizeof(e.error)); + std::memset(e.function_calls, 0, sizeof(e.function_calls)); + std::string safe_model = model_basename(model); + if (!safe_model.empty()) std::strncpy(e.model, safe_model.c_str(), sizeof(e.model)-1); + if (message) std::strncpy(e.message, message, sizeof(e.message)-1); + return e; +} + +static Event make_event_extended(EventType type, const char* model, const CompletionMetrics& metrics) { + Event e; + e.type = type; + e.success = metrics.success; + e.cloud_handoff = metrics.cloud_handoff; + e.ttft_ms = metrics.ttft_ms; + e.prefill_tps = metrics.prefill_tps; + e.decode_tps = metrics.decode_tps; + e.tps = metrics.decode_tps; + e.response_time_ms = metrics.response_time_ms; + e.confidence = metrics.confidence; + e.ram_usage_mb = metrics.ram_usage_mb; + e.tokens = static_cast(metrics.prefill_tokens + metrics.decode_tokens); + e.prefill_tokens = static_cast(metrics.prefill_tokens); + e.decode_tokens = static_cast(metrics.decode_tokens); + e.session_ttft_ms = 0.0; + e.session_tps = 0.0; + e.session_time_ms = 0.0; + e.session_tokens = 0; + e.timestamp = std::chrono::system_clock::now(); + std::memset(e.model, 0, sizeof(e.model)); + std::memset(e.message, 0, sizeof(e.message)); + std::memset(e.error, 0, sizeof(e.error)); + std::memset(e.function_calls, 0, sizeof(e.function_calls)); + std::string safe_model = model_basename(model); + if (!safe_model.empty()) std::strncpy(e.model, safe_model.c_str(), sizeof(e.model)-1); + if (!metrics.success && metrics.error_message) std::strncpy(e.error, metrics.error_message, sizeof(e.error)-1); + if (metrics.function_calls_json) std::strncpy(e.function_calls, metrics.function_calls_json, sizeof(e.function_calls)-1); + return e; +} + +static bool parse_event_line(const std::string& line, Event& out) { + std::string type_str; + if (!extract_string_field(line, "event_type", type_str)) return false; + EventType et; + if (!event_type_from_string(type_str, et)) return false; + + std::string model; + extract_string_field(line, "model", model); + bool success = false; + extract_bool_field(line, "success", success); + bool cloud_handoff = false; + extract_bool_field(line, "cloud_handoff", cloud_handoff); + double ttft = 0.0; + extract_double_field(line, "ttft", ttft); + double prefill_tps = 0.0; + extract_double_field(line, "prefill_tps", prefill_tps); + double decode_tps = 0.0; + extract_double_field(line, "decode_tps", decode_tps); + double tps = 0.0; + extract_double_field(line, "tps", tps); + if (decode_tps == 0.0 && tps != 0.0) decode_tps = tps; + double response_time = 0.0; + extract_double_field(line, "response_time", response_time); + double confidence = 0.0; + extract_double_field(line, "confidence", confidence); + double ram_usage_mb = 0.0; + extract_double_field(line, "ram_usage_mb", ram_usage_mb); + int tokens = 0; + extract_int_field(line, "tokens", tokens); + int prefill_tokens = 0; + extract_int_field(line, "prefill_tokens", prefill_tokens); + int decode_tokens = 0; + extract_int_field(line, "decode_tokens", decode_tokens); + double session_ttft_ms = 0.0; + extract_double_field(line, "session_ttft", session_ttft_ms); + double session_tps = 0.0; + extract_double_field(line, "session_tps", session_tps); + double session_time_ms = 0.0; + extract_double_field(line, "session_time_ms", session_time_ms); + int session_tokens = 0; + extract_int_field(line, "session_tokens", session_tokens); + std::string message; + extract_string_field(line, "message", message); + std::string error; + extract_string_field(line, "error", error); + std::string function_calls; + extract_json_field(line, "function_calls", function_calls); + double ts_ms = 0.0; + if (!extract_double_field_raw(line, "ts_ms", ts_ms)) { + extract_double_field(line, "ts_ms", ts_ms); + } + auto ts_point = std::chrono::system_clock::now(); + if (ts_ms > 0.0) { + ts_point = std::chrono::system_clock::time_point(std::chrono::milliseconds(static_cast(ts_ms))); + } + + out = make_event(et, + model.empty() ? nullptr : model.c_str(), + success, + ttft, + tps, + response_time, + tokens, + message.empty() ? nullptr : message.c_str()); + out.timestamp = ts_point; + out.cloud_handoff = cloud_handoff; + out.prefill_tps = prefill_tps; + out.decode_tps = decode_tps; + out.confidence = confidence; + out.ram_usage_mb = ram_usage_mb; + out.prefill_tokens = prefill_tokens; + out.decode_tokens = decode_tokens; + out.session_ttft_ms = session_ttft_ms; + out.session_tps = session_tps; + out.session_time_ms = session_time_ms; + out.session_tokens = session_tokens; + if (!error.empty()) std::strncpy(out.error, error.c_str(), sizeof(out.error)-1); + if (!function_calls.empty()) std::strncpy(out.function_calls, function_calls.c_str(), sizeof(out.function_calls)-1); + return true; +} + +static std::string event_type_to_string(EventType t) { + switch (t) { + case INIT: return "init"; + case COMPLETION: return "completion"; + case EMBEDDING: return "embedding"; + case TRANSCRIPTION: return "transcription"; + case STREAM_TRANSCRIBE: return "stream_transcribe"; + default: return "unknown"; + } +} + +static bool event_type_from_string(const std::string& s, EventType& t) { + if (s == "init") { t = INIT; return true; } + if (s == "completion") { t = COMPLETION; return true; } + if (s == "embedding") { t = EMBEDDING; return true; } + if (s == "transcription") { t = TRANSCRIPTION; return true; } + if (s == "stream_transcribe") { t = STREAM_TRANSCRIBE; return true; } + return false; +} + +static bool extract_string_field(const std::string& line, const std::string& key, std::string& out) { + std::string needle = "\"" + key + "\":"; + size_t pos = line.find(needle); + if (pos == std::string::npos) return false; + pos += needle.size(); + while (pos < line.size() && line[pos] == ' ') pos++; + if (pos >= line.size()) return false; + if (line[pos] == '"') { + pos++; + size_t end = line.find('"', pos); + if (end == std::string::npos) return false; + out = line.substr(pos, end - pos); + return true; + } + size_t end = line.find_first_of(",}", pos); + if (end == std::string::npos) end = line.size(); + out = line.substr(pos, end - pos); + if (out == "null") out.clear(); + return true; +} + +static bool extract_json_field(const std::string& line, const std::string& key, std::string& out) { + std::string needle = "\"" + key + "\":"; + size_t pos = line.find(needle); + if (pos == std::string::npos) return false; + pos += needle.size(); + while (pos < line.size() && line[pos] == ' ') pos++; + if (pos >= line.size()) return false; + + char start_char = line[pos]; + if (start_char == '[' || start_char == '{') { + char end_char = (start_char == '[') ? ']' : '}'; + int depth = 0; + size_t end = pos; + bool in_string = false; + bool escape_next = false; + + for (; end < line.size(); end++) { + if (escape_next) { + escape_next = false; + continue; + } + if (line[end] == '\\') { + escape_next = true; + continue; + } + if (line[end] == '"') { + in_string = !in_string; + continue; + } + if (in_string) continue; + + if (line[end] == start_char) { + depth++; + } else if (line[end] == end_char) { + depth--; + if (depth == 0) { + out = line.substr(pos, end - pos + 1); + return true; + } + } + } + return false; + } + + size_t end = line.find_first_of(",}", pos); + if (end == std::string::npos) end = line.size(); + out = line.substr(pos, end - pos); + if (out == "null") out.clear(); + return true; +} + +static bool extract_bool_field(const std::string& line, const std::string& key, bool& out) { + std::string raw; + if (!extract_string_field(line, key, raw)) return false; + if (raw == "true") { out = true; return true; } + if (raw == "false") { out = false; return true; } + return false; +} + +static bool extract_double_field(const std::string& line, const std::string& key, double& out) { + std::string raw; + if (!extract_string_field(line, key, raw)) return false; + try { + out = std::stod(raw); + return true; + } catch (...) { + return false; + } +} + +// Helper used when the field is already numeric (no quotes) in the log cache. +static bool extract_double_field_raw(const std::string& line, const std::string& key, double& out) { + std::string needle = "\"" + key + "\":"; + size_t pos = line.find(needle); + if (pos == std::string::npos) return false; + pos += needle.size(); + while (pos < line.size() && line[pos] == ' ') pos++; + if (pos >= line.size()) return false; + size_t end = line.find_first_of(",}", pos); + if (end == std::string::npos) end = line.size(); + std::string raw = line.substr(pos, end - pos); + try { + out = std::stod(raw); + return true; + } catch (...) { + return false; + } +} + +static bool extract_int_field(const std::string& line, const std::string& key, int& out) { + std::string raw; + if (!extract_string_field(line, key, raw)) return false; + try { + out = std::stoi(raw); + return true; + } catch (...) { + return false; + } +} + +static std::string new_uuid() { + static thread_local std::mt19937_64 rng(std::random_device{}()); + uint64_t a = rng(); + uint64_t b = rng(); + a = (a & 0xffffffffffff0fffULL) | 0x0000000000004000ULL; + b = (b & 0x3fffffffffffffffULL) | 0x8000000000000000ULL; + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << std::setw(8) << ((a >> 32) & 0xffffffffULL); + oss << "-" << std::setw(4) << ((a >> 16) & 0xffffULL); + oss << "-" << std::setw(4) << (a & 0xffffULL); + oss << "-" << std::setw(4) << ((b >> 48) & 0xffffULL); + oss << "-" << std::setw(12) << (b & 0xffffffffffffULL); + return oss.str(); +} + +static uint64_t fnv1a_64(const std::string& s) { + uint64_t h = 14695981039346656037ULL; + for (unsigned char c : s) { h ^= c; h *= 1099511628211ULL; } + return h; +} + +static std::string uuid_from_string(const std::string& input) { + uint64_t a = fnv1a_64(input); + uint64_t b = fnv1a_64(input + "\xff"); + a = (a & 0xffffffffffff0fffULL) | 0x0000000000004000ULL; + b = (b & 0x3fffffffffffffffULL) | 0x8000000000000000ULL; + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + oss << std::setw(8) << ((a >> 32) & 0xffffffffULL); + oss << "-" << std::setw(4) << ((a >> 16) & 0xffffULL); + oss << "-" << std::setw(4) << (a & 0xffffULL); + oss << "-" << std::setw(4) << ((b >> 48) & 0xffffULL); + oss << "-" << std::setw(12) << (b & 0xffffffffffffULL); + return oss.str(); +} + +static std::string read_git_remote_url() { + const char* cwd_env = std::getenv("PWD"); + if (!cwd_env) return ""; + std::string dir = cwd_env; + while (!dir.empty()) { + std::ifstream f(dir + "/.git/config"); + if (f.is_open()) { + std::string line; + bool in_origin = false; + while (std::getline(f, line)) { + if (line.find("[remote \"origin\"]") != std::string::npos) { + in_origin = true; + } else if (in_origin) { + size_t pos = line.find("url = "); + if (pos != std::string::npos) return line.substr(pos + 6); + if (!line.empty() && line[0] == '[') break; + } + } + return ""; + } + size_t pos = dir.rfind('/'); + if (pos == 0 || pos == std::string::npos) break; + dir = dir.substr(0, pos); + } + return ""; +} + +#if defined(__APPLE__) +static std::string detect_app_id() { + CFBundleRef bundle = CFBundleGetMainBundle(); + if (!bundle) return ""; + CFStringRef id = (CFStringRef)CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("CFBundleIdentifier")); + if (!id) return ""; + char buf[256]; + return CFStringGetCString(id, buf, sizeof(buf), kCFStringEncodingUTF8) ? buf : ""; +} +#elif defined(__linux__) || defined(__ANDROID__) +static std::string detect_app_id() { + std::ifstream f("/proc/self/cmdline"); + if (!f.is_open()) return ""; + std::string s; + std::getline(f, s, '\0'); + size_t colon = s.find(':'); + if (colon != std::string::npos) s = s.substr(0, colon); + return s; +} +#else +static std::string detect_app_id() { return ""; } +#endif + +static std::string extract_android_package_from_path(const std::string& path) { + for (const char* prefix : {"/data/user/", "/data/user_de/", "/data/data/"}) { + size_t pos = path.find(prefix); + if (pos == std::string::npos) continue; + size_t start = pos + strlen(prefix); + if (strcmp(prefix, "/data/data/") != 0) { + start = path.find('/', start); + if (start == std::string::npos) continue; + start++; + } + size_t end = path.find('/', start); + if (end == std::string::npos) end = path.size(); + std::string pkg = path.substr(start, end - start); + if (!pkg.empty()) return pkg; + } + return ""; +} + +static std::string format_timestamp(const std::chrono::system_clock::time_point& tp) { + using namespace std::chrono; + auto secs = time_point_cast(tp); + auto ms = duration_cast(tp - secs).count(); + std::time_t t = system_clock::to_time_t(secs); + std::tm tm{}; +#if defined(_WIN32) + gmtime_s(&tm, &t); +#else + gmtime_r(&t, &tm); +#endif + char buf[32]; + std::snprintf(buf, sizeof(buf), "%04d-%02d-%02dT%02d:%02d:%02d.%03lldZ", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, + static_cast(ms)); + return std::string(buf); +} + +[[maybe_unused]] static bool is_valid_uuid(const std::string& s) { + // Simple check used only for validation when accepting user-provided IDs. + if (s.size() != 36) return false; + for (size_t i = 0; i < s.size(); ++i) { + char c = s[i]; + if (i == 8 || i == 13 || i == 18 || i == 23) { + if (c != '-') return false; + } else { + bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + if (!hex) return false; + } + } + return true; +} + +static std::string escape_json_string(const char* str) { + if (!str) return ""; + std::string result; + result.reserve(std::strlen(str)); + for (const char* p = str; *p; ++p) { + switch (*p) { + case '"': result += "\\\""; break; + case '\\': result += "\\\\"; break; + case '\b': result += "\\b"; break; + case '\f': result += "\\f"; break; + case '\n': result += "\\n"; break; + case '\r': result += "\\r"; break; + case '\t': result += "\\t"; break; + default: + if (static_cast(*p) < 0x20) { + char buf[7]; + snprintf(buf, sizeof(buf), "\\u%04x", static_cast(*p)); + result += buf; + } else { + result += *p; + } + break; + } + } + return result; +} + +static void collect_device_info() { + struct utsname u; + if (uname(&u) == 0) { + const std::string sysname = u.sysname; + device_os = (sysname == "Darwin") ? "macos" : sysname; + device_model = u.machine; + device_os_version = u.release; + device_brand = "unknown"; +#if defined(__APPLE__) + // Prefer user-facing macOS info over Darwin kernel version. + std::string hw_model = sysctl_string("hw.model"); + std::string os_product = sysctl_string("kern.osproductversion"); + if (!hw_model.empty()) device_model = hw_model; + if (!os_product.empty()) device_os_version = os_product; + device_brand = "apple"; +#elif defined(__ANDROID__) + device_brand = "android"; +#endif + } +} + +static bool parse_version_line(const std::string& line) { + size_t start = line.find_first_not_of(" \t\r\n"); + size_t end = line.find_last_not_of(" \t\r\n"); + if (start != std::string::npos && end != std::string::npos) { + cactus_version = line.substr(start, end - start + 1); + return true; + } + return false; +} + +static void read_cactus_version() { + const char* version_paths[] = { + "CACTUS_VERSION", + "../CACTUS_VERSION", + "../../CACTUS_VERSION", + "../../../CACTUS_VERSION" + }; + + for (const char* path : version_paths) { + std::ifstream file(path); + if (file.is_open()) { + std::string line; + if (std::getline(file, line) && parse_version_line(line)) return; + } + } + +#if defined(__APPLE__) + CFBundleRef bundle = CFBundleGetMainBundle(); + if (bundle) { + CFURLRef url = CFBundleCopyResourceURL(bundle, CFSTR("CACTUS_VERSION"), NULL, NULL); + if (url) { + char fspath[4096]; + if (CFURLGetFileSystemRepresentation(url, true, (UInt8*)fspath, sizeof(fspath))) { + std::ifstream file(fspath); + if (file.is_open()) { + std::string line; + if (std::getline(file, line) && parse_version_line(line)) { + CFRelease(url); + return; + } + } + } + CFRelease(url); + } + } +#endif + +#ifdef CACTUS_COMPILE_TIME_VERSION + cactus_version = CACTUS_COMPILE_TIME_VERSION; + return; +#endif + + cactus_version = ""; +} + +static void apply_curl_tls_trust(CURL* curl) { + if (!curl) return; + const char* ca_bundle = std::getenv("CACTUS_CA_BUNDLE"); + if (ca_bundle && ca_bundle[0] != '\0') { + curl_easy_setopt(curl, CURLOPT_CAINFO, ca_bundle); + } +#if defined(__ANDROID__) + const char* ca_path = std::getenv("CACTUS_CA_PATH"); + if (ca_path && ca_path[0] != '\0') { + curl_easy_setopt(curl, CURLOPT_CAPATH, ca_path); + } else { + curl_easy_setopt(curl, CURLOPT_CAPATH, "/system/etc/security/cacerts"); + } +#endif +} + +static bool ensure_project_row_remote(CURL* curl, const CloudConfigurationStateSnapshot& snapshot) { + if (snapshot.project_id.empty() || snapshot.project_registered) return true; + std::string url = snapshot.supabase_url + "/rest/v1/projects"; + std::ostringstream payload; + payload << "[{"; + payload << "\"project_key\":\"" << snapshot.project_id << "\""; + if (!snapshot.project_scope.empty()) { + payload << ",\"name\":\"" << snapshot.project_scope << "\""; + } + payload << "}]"; + struct curl_slist* headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; + std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; + headers = curl_slist_append(headers, apikey_hdr.c_str()); + headers = curl_slist_append(headers, auth_hdr.c_str()); + headers = curl_slist_append(headers, "Prefer: resolution=ignore-duplicates"); + headers = curl_slist_append(headers, "Content-Profile: cactus"); + headers = curl_slist_append(headers, "Accept-Profile: cactus"); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + std::string body = payload.str(); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); + apply_curl_tls_trust(curl); + CURLcode res = curl_easy_perform(curl); + long code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + if (headers) curl_slist_free_all(headers); + bool ok = (res == CURLE_OK) && ((code >= 200 && code < 300) || code == 409); + return ok; +} + +static bool ensure_device_row_remote(CURL* curl, const CloudConfigurationStateSnapshot& snapshot) { + if (snapshot.device_registered) return true; + if (snapshot.device_id.empty()) return false; + std::string url = snapshot.supabase_url + "/rest/v1/devices"; + std::ostringstream payload; + payload << "[{"; + payload << "\"id\":\"" << snapshot.device_id << "\""; + payload << ",\"device_id\":\"" << snapshot.device_id << "\""; + if (!snapshot.device_model.empty()) payload << ",\"model\":\"" << snapshot.device_model << "\""; + if (!snapshot.device_os.empty()) payload << ",\"os\":\"" << snapshot.device_os << "\""; + if (!snapshot.device_os_version.empty()) payload << ",\"os_version\":\"" << snapshot.device_os_version << "\""; + if (!snapshot.device_brand.empty()) payload << ",\"brand\":\"" << snapshot.device_brand << "\""; + payload << "}]"; + struct curl_slist* headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; + std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; + headers = curl_slist_append(headers, apikey_hdr.c_str()); + headers = curl_slist_append(headers, auth_hdr.c_str()); + headers = curl_slist_append(headers, "Prefer: resolution=ignore-duplicates"); + headers = curl_slist_append(headers, "Content-Profile: cactus"); + headers = curl_slist_append(headers, "Accept-Profile: cactus"); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + std::string body = payload.str(); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); + apply_curl_tls_trust(curl); + CURLcode res = curl_easy_perform(curl); + long code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + if (headers) curl_slist_free_all(headers); + bool ok = (res == CURLE_OK) && ((code >= 200 && code < 300) || code == 409); + return ok; +} + +static bool send_payload(CURL* curl, const std::string& url, const std::string& body, const CloudConfigurationStateSnapshot& snapshot) { + struct curl_slist* headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; + std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; + headers = curl_slist_append(headers, apikey_hdr.c_str()); + headers = curl_slist_append(headers, auth_hdr.c_str()); + headers = curl_slist_append(headers, "Prefer: return=minimal"); + headers = curl_slist_append(headers, "Content-Profile: cactus"); + headers = curl_slist_append(headers, "Accept-Profile: cactus"); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); + apply_curl_tls_trust(curl); + CURLcode res = curl_easy_perform(curl); + long code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + if (headers) curl_slist_free_all(headers); + return (res == CURLE_OK) && (code >= 200 && code < 300); +} + +static CloudSendResult send_batch_to_cloud(const std::vector& local, const CloudConfigurationStateSnapshot& snapshot) { + CloudSendResult result; + if (!snapshot.enabled) return result; + if (local.empty()) { + result.payload_ok = true; + return result; + } + if (snapshot.cloud_disabled) return result; + if (snapshot.device_id.empty()) return result; + CURL* curl = curl_easy_init(); + if (!curl) return result; + result.project_registered_ok = ensure_project_row_remote(curl, snapshot); + if (!snapshot.device_registered) { + result.device_registered_ok = ensure_device_row_remote(curl, snapshot); + } else { + result.device_registered_ok = true; + } + std::string url = snapshot.supabase_url + "/rest/v1/logs"; + std::ostringstream payload; + payload << "["; + for (size_t i = 0; i < local.size(); ++i) { + const Event& e = local[i]; + payload << "{"; + payload << "\"event_type\":\"" << event_type_to_string(e.type) << "\","; + payload << "\"model\":\"" << escape_json_string(e.model) << "\","; + payload << "\"success\":" << (e.success ? "true" : "false") << ","; + payload << "\"cloud_handoff\":" << (e.cloud_handoff ? "true" : "false") << ","; + if (e.type == INIT || !e.success) { + payload << "\"ttft\":null,"; + payload << "\"prefill_tps\":null,"; + payload << "\"decode_tps\":null,"; + payload << "\"tps\":null,"; + } else { + payload << "\"ttft\":" << e.ttft_ms << ","; + payload << "\"prefill_tps\":" << e.prefill_tps << ","; + payload << "\"decode_tps\":" << e.decode_tps << ","; + payload << "\"tps\":" << e.tps << ","; + } + if (!e.success) { + payload << "\"response_time\":null,"; + payload << "\"ram_usage_mb\":null,"; + } else { + payload << "\"response_time\":" << e.response_time_ms << ","; + payload << "\"ram_usage_mb\":" << e.ram_usage_mb << ","; + } + if (e.type == INIT || !e.success) { + payload << "\"confidence\":null,"; + payload << "\"tokens\":null,"; + payload << "\"prefill_tokens\":null,"; + payload << "\"decode_tokens\":null,"; + } else { + payload << "\"confidence\":" << e.confidence << ","; + payload << "\"tokens\":" << e.tokens << ","; + payload << "\"prefill_tokens\":" << e.prefill_tokens << ","; + payload << "\"decode_tokens\":" << e.decode_tokens << ","; + } + if (e.type == STREAM_TRANSCRIBE) { + payload << "\"session_ttft\":" << e.session_ttft_ms << ","; + payload << "\"session_tps\":" << e.session_tps << ","; + payload << "\"session_time_ms\":" << e.session_time_ms << ","; + payload << "\"session_tokens\":" << e.session_tokens << ","; + } else { + payload << "\"session_ttft\":null,"; + payload << "\"session_tps\":null,"; + payload << "\"session_time_ms\":null,"; + payload << "\"session_tokens\":null,"; + } + payload << "\"created_at\":\"" << format_timestamp(e.timestamp) << "\","; + if (!snapshot.project_id.empty()) { + payload << "\"project_id\":\"" << snapshot.project_id << "\","; + } + if (!snapshot.cloud_key.empty()) { + payload << "\"key_hash\":\"" << snapshot.cloud_key << "\","; + } + payload << "\"framework\":\"" << snapshot.framework << "\","; + if (!snapshot.cactus_version.empty()) { + payload << "\"framework_version\":\"" << snapshot.cactus_version << "\","; + } + payload << "\"device_id\":\"" << snapshot.device_id << "\""; + if (!snapshot.app_id.empty()) { + payload << ",\"app_id\":\"" << escape_json_string(snapshot.app_id.c_str()) << "\""; + } else { + payload << ",\"app_id\":null"; + } + if (e.message[0] != '\0') { + payload << ",\"message\":\"" << escape_json_string(e.message) << "\""; + } else { + payload << ",\"message\":null"; + } + if (e.error[0] != '\0') { + payload << ",\"error\":\"" << escape_json_string(e.error) << "\""; + } else { + payload << ",\"error\":null"; + } + payload << ",\"function_calls\":null"; + payload << "}"; + if (i + 1 < local.size()) payload << ","; + } + payload << "]"; + result.payload_ok = send_payload(curl, url, payload.str(), snapshot); + curl_easy_cleanup(curl); + return result; +} + +static void write_events_to_cache_in_dir(const std::vector& local, const std::string& dir) { + for (const auto &e : local) { + std::ostringstream oss; + oss << "{\"event_type\":\"" << event_type_to_string(e.type) << "\","; + oss << "\"model\":\"" << escape_json_string(e.model) << "\","; + oss << "\"success\":" << (e.success ? "true" : "false") << ","; + oss << "\"cloud_handoff\":" << (e.cloud_handoff ? "true" : "false") << ","; + if (e.type == INIT || !e.success) { + oss << "\"ttft\":null,"; + oss << "\"prefill_tps\":null,"; + oss << "\"decode_tps\":null,"; + oss << "\"tps\":null,"; + } else { + oss << "\"ttft\":" << e.ttft_ms << ","; + oss << "\"prefill_tps\":" << e.prefill_tps << ","; + oss << "\"decode_tps\":" << e.decode_tps << ","; + oss << "\"tps\":" << e.tps << ","; + } + if (!e.success) { + oss << "\"response_time\":null,"; + oss << "\"ram_usage_mb\":null,"; + } else { + oss << "\"response_time\":" << e.response_time_ms << ","; + oss << "\"ram_usage_mb\":" << e.ram_usage_mb << ","; + } + if (e.type == INIT || !e.success) { + oss << "\"confidence\":null,"; + oss << "\"tokens\":null,"; + oss << "\"prefill_tokens\":null,"; + oss << "\"decode_tokens\":null,"; + } else { + oss << "\"confidence\":" << e.confidence << ","; + oss << "\"tokens\":" << e.tokens << ","; + oss << "\"prefill_tokens\":" << e.prefill_tokens << ","; + oss << "\"decode_tokens\":" << e.decode_tokens << ","; + } + if (e.type == STREAM_TRANSCRIBE) { + oss << "\"session_ttft\":" << e.session_ttft_ms << ","; + oss << "\"session_tps\":" << e.session_tps << ","; + oss << "\"session_time_ms\":" << e.session_time_ms << ","; + oss << "\"session_tokens\":" << e.session_tokens; + } else { + oss << "\"session_ttft\":null,"; + oss << "\"session_tps\":null,"; + oss << "\"session_time_ms\":null,"; + oss << "\"session_tokens\":null"; + } + oss << ",\"ts_ms\":" << std::chrono::duration_cast(e.timestamp.time_since_epoch()).count(); + if (e.message[0] != '\0') { + oss << ",\"message\":\"" << escape_json_string(e.message) << "\""; + } else { + oss << ",\"message\":null"; + } + if (e.error[0] != '\0') { + oss << ",\"error\":\"" << escape_json_string(e.error) << "\""; + } else { + oss << ",\"error\":null"; + } + oss << ",\"function_calls\":null"; + if (!app_id.empty()) { + oss << ",\"app_id\":\"" << escape_json_string(app_id.c_str()) << "\""; + } else { + oss << ",\"app_id\":null"; + } + oss << "}"; + std::string file = dir + "/" + event_type_to_string(e.type) + ".log"; + std::ofstream out(file, std::ios::app); + if (out.is_open()) { + out << oss.str() << "\n"; + out.close(); + } + } +} + +static std::vector load_cached_events_in_dir(const std::string& dir) { + std::vector events; + DIR* d = opendir(dir.c_str()); + if (!d) return events; + struct dirent* ent; + while ((ent = readdir(d)) != nullptr) { + std::string name = ent->d_name; + if (name.size() < 5 || name.substr(name.size() - 4) != ".log") continue; + std::ifstream in(dir + "/" + name); + std::string line; + while (std::getline(in, line)) { + Event e; + if (parse_event_line(line, e)) { + events.push_back(e); + } + } + } + closedir(d); + return events; +} + +static void clear_cache_files_in_dir(const std::string& dir) { + DIR* d = opendir(dir.c_str()); + if (!d) return; + struct dirent* ent; + while ((ent = readdir(d)) != nullptr) { + std::string name = ent->d_name; + if (name.size() < 5 || name.substr(name.size() - 4) != ".log") continue; + std::remove((dir + "/" + name).c_str()); + } + closedir(d); +} + +static CloudConfigurationStateSnapshot capture_cloud_configuration_state_snapshot_locked() { + CloudConfigurationStateSnapshot snapshot; + snapshot.supabase_url = supabase_url; + snapshot.supabase_key = supabase_key; + snapshot.device_id = device_id; + snapshot.project_id = project_id; + snapshot.cloud_key = cloud_key; + snapshot.project_scope = project_scope; + snapshot.framework = framework; + snapshot.cactus_version = cactus_version; + snapshot.device_model = device_model; + snapshot.device_os = device_os; + snapshot.device_os_version = device_os_version; + snapshot.device_brand = device_brand; + snapshot.app_id = app_id; + snapshot.device_registered_file = device_registered_file; + snapshot.project_registered_file = project_registered_file; + snapshot.enabled = enabled; + snapshot.cloud_disabled = cloud_disabled; + snapshot.device_registered = device_registered; + snapshot.project_registered = project_registered; + return snapshot; +} + +static void process_events(const std::vector& fresh_events) { + std::vector events; + CloudConfigurationStateSnapshot snapshot; + std::string telemetry_dir; + + { + std::lock_guard guard(telemetry_mutex); + telemetry_dir = get_telemetry_dir_locked(); + events = load_cached_events_in_dir(telemetry_dir); + if (!fresh_events.empty()) { + events.insert(events.end(), fresh_events.begin(), fresh_events.end()); + } + if (events.empty()) return; + snapshot = capture_cloud_configuration_state_snapshot_locked(); + } + + CloudSendResult send_result = send_batch_to_cloud(events, snapshot); + + { + std::lock_guard guard(telemetry_mutex); + if (send_result.project_registered_ok && !project_registered) { + if (!project_registered_file.empty()) { + persist_registered_flag(project_registered_file); + } + project_registered = true; + } + if (send_result.device_registered_ok && !device_registered) { + if (!device_registered_file.empty()) { + persist_registered_flag(device_registered_file); + } + device_registered = true; + } + + if (send_result.payload_ok) { + clear_cache_files_in_dir(telemetry_dir); + } else if (!fresh_events.empty()) { + write_events_to_cache_in_dir(fresh_events, telemetry_dir); + } + } +} + +void init(const char* project_id_param, const char* project_scope_param, const char* cloud_key_param) { + std::unique_lock lifecycle_guard(telemetry_mutex); + telemetry_lifecycle_cv.wait(lifecycle_guard, [] { + return lifecycle_state != TelemetryLifecycleState::ShuttingDown; + }); + + std::string scope = project_scope_param ? project_scope_param : "default"; + + bool cloud_disabled_from_env = false; + if (const char* env = std::getenv("CACTUS_NO_CLOUD_TELE")) { + if (env[0] != '\0' && !(env[0] == '0' && env[1] == '\0')) { + cloud_disabled_from_env = true; + } + } + const char* env_url = std::getenv("CACTUS_SUPABASE_URL"); + const char* env_key = std::getenv("CACTUS_SUPABASE_KEY"); + const char* env_project = std::getenv("CACTUS_PROJECT_ID"); + const char* env_cloud = std::getenv("CACTUS_CLOUD_KEY"); + + std::string dir = get_telemetry_dir_locked(); + + std::string device_file = dir + "/device_id"; + std::string device_flag_file = dir + "/device_registered"; + bool device_was_registered = load_registered_flag(device_flag_file); + std::string resolved_device_id = load_or_create_id(device_file); + + std::string resolved_project_id; + if (project_id_param && *project_id_param) { + resolved_project_id = project_id_param; + } else if (env_project && *env_project) { + resolved_project_id = env_project; + } else { + std::string remote_url = read_git_remote_url(); + resolved_project_id = remote_url.empty() ? resolved_device_id : uuid_from_string(remote_url); + } + + std::string project_flag_file = dir + "/project_reg_" + resolved_project_id.substr(0, 8); + bool project_was_registered = load_registered_flag(project_flag_file); + + std::string resolved_cloud_key; + if (cloud_key_param && *cloud_key_param) { + resolved_cloud_key = cloud_key_param; + } else if (env_cloud && *env_cloud) { + resolved_cloud_key = env_cloud; + } + + collect_device_info(); + if (cactus_version.empty()) read_cactus_version(); + + if (env_url && *env_url) supabase_url = env_url; + if (env_key && *env_key) supabase_key = env_key; + project_id = resolved_project_id; + project_scope = scope; + project_registered_file = project_flag_file; + cloud_key = resolved_cloud_key; + device_registered_file = device_flag_file; + device_id = resolved_device_id; + if (app_id.empty()) app_id = detect_app_id(); + + project_registered = project_was_registered; + device_registered = device_was_registered; + if (cloud_disabled_from_env) { + cloud_disabled = true; + } + + if (!curl_initialized) { + if (curl_global_init(CURL_GLOBAL_DEFAULT) == CURLE_OK) { + curl_initialized = true; + } + } + + if (!atexit_registered) { + atexit_registered = true; + std::atexit([](){ shutdown(); }); + } + + shutdown_called = false; + ids_ready = true; + enabled = true; + lifecycle_state = TelemetryLifecycleState::Running; + TelemetryDispatcher::instance().start(); +} + +void setEnabled(bool en) { + std::lock_guard guard(telemetry_mutex); + enabled = en; +} + +void setCloudDisabled(bool disabled) { + std::lock_guard guard(telemetry_mutex); + cloud_disabled = disabled; +} + +void setAppId(const char* id) { + std::lock_guard guard(telemetry_mutex); + if (id && id[0] != '\0') app_id = id; +} + +void setTelemetryEnvironment(const char* framework_str, const char* cache_location_str, const char* version_str) { + std::lock_guard guard(telemetry_mutex); + if (framework_str && framework_str[0] != '\0') { + framework = framework_str; + } + if (cache_location_str && cache_location_str[0] != '\0') { + custom_cache_location = cache_location_str; + if (app_id.empty()) { + std::string pkg = extract_android_package_from_path(custom_cache_location); + if (!pkg.empty()) app_id = pkg; + } + } + if (version_str && version_str[0] != '\0') { + cactus_version = version_str; + } +} + +void setCloudKey(const char* key) { + if (!key || key[0] == '\0') return; + std::lock_guard guard(telemetry_mutex); + cloud_key = key; +} + +void cacheCloudApiKey(const char* key) { + if (!key || key[0] == '\0') return; + std::lock_guard guard(telemetry_mutex); + const std::string file = cloud_api_key_cache_file_locked(); + std::ofstream out(file, std::ios::trunc); + if (out.is_open()) { + out << key; + } +} + +std::string loadCachedCloudApiKey() { + std::lock_guard guard(telemetry_mutex); + const std::string file = cloud_api_key_cache_file_locked(); + std::ifstream in(file); + if (!in.is_open()) return {}; + std::string line; + if (std::getline(in, line) && !line.empty()) { + return line; + } + return {}; +} + +void recordInit(const char* model, bool success, double response_time_ms, const char* message) { + double nan = std::numeric_limits::quiet_NaN(); + Event e = make_event(INIT, model, success, nan, nan, response_time_ms, 0, message); + std::lock_guard guard(telemetry_mutex); + if (!can_record_event_locked()) return; + TelemetryDispatcher::instance().enqueue(e); +} + +void recordCompletion(const char* model, const CompletionMetrics& metrics) { + Event e = make_event_extended(COMPLETION, model, metrics); + std::lock_guard guard(telemetry_mutex); + if (!can_record_event_locked()) return; + TelemetryDispatcher::instance().enqueue(e); +} + +void recordCompletion(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message) { + Event e = make_event(COMPLETION, model, success, ttft_ms, tps, response_time_ms, tokens, message); + std::lock_guard guard(telemetry_mutex); + if (!can_record_event_locked()) return; + TelemetryDispatcher::instance().enqueue(e); +} + +void recordEmbedding(const char* model, bool success, const char* message) { + Event e = make_event(EMBEDDING, model, success, 0.0, 0.0, 0.0, 0, message); + std::lock_guard guard(telemetry_mutex); + if (!can_record_event_locked()) return; + TelemetryDispatcher::instance().enqueue(e); +} + +void recordTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double ram_usage_mb, const char* message) { + std::lock_guard guard(telemetry_mutex); + if (in_stream_mode) return; + Event e = make_event(TRANSCRIPTION, model, success, ttft_ms, tps, response_time_ms, tokens, message); + e.ram_usage_mb = ram_usage_mb; + if (!can_record_event_locked()) return; + TelemetryDispatcher::instance().enqueue(e); +} + +void recordStreamTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double session_ttft_ms, double session_tps, double session_time_ms, int session_tokens, const char* message) { + Event e = make_event(STREAM_TRANSCRIBE, model, success, ttft_ms, tps, response_time_ms, tokens, message); + e.session_ttft_ms = session_ttft_ms; + e.session_tps = session_tps; + e.session_time_ms = session_time_ms; + e.session_tokens = session_tokens; + std::lock_guard guard(telemetry_mutex); + if (!can_record_event_locked()) return; + TelemetryDispatcher::instance().enqueue(e); +} + +void setStreamMode(bool in_stream) { + std::lock_guard guard(telemetry_mutex); + in_stream_mode = in_stream; +} + +bool isStreamMode() { + std::lock_guard guard(telemetry_mutex); + return in_stream_mode; +} + +void markInference(bool active) { + std::lock_guard guard(telemetry_mutex); + if (active) { + inference_active += 1; + } else if (inference_active > 0) { + inference_active -= 1; + } +} + +void flush() { + TelemetryDispatcher::instance().flush(); +} + +void shutdown() { + { + std::lock_guard lifecycle_guard(telemetry_mutex); + if (shutdown_called) { + return; + } + + shutdown_called = true; + lifecycle_state = TelemetryLifecycleState::ShuttingDown; + } + + flush(); + + { + std::lock_guard lifecycle_guard(telemetry_mutex); + enabled = false; + ids_ready = false; + } + + TelemetryDispatcher::instance().stop(); + + { + std::lock_guard lifecycle_guard(telemetry_mutex); + if (curl_initialized) { + curl_initialized = false; + curl_global_cleanup(); + } + lifecycle_state = TelemetryLifecycleState::Stopped; + telemetry_lifecycle_cv.notify_all(); + } +} + +} // namespace telemetry +} // namespace cactus diff --git a/cactus-transpile/telemetry/telemetry.h b/cactus-transpile/telemetry/telemetry.h new file mode 100644 index 000000000..cd7b49f45 --- /dev/null +++ b/cactus-transpile/telemetry/telemetry.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include + +namespace cactus { +namespace telemetry { + +struct CompletionMetrics { + bool success; + bool cloud_handoff; + double ttft_ms; + double prefill_tps; + double decode_tps; + double response_time_ms; + double confidence; + double ram_usage_mb; + size_t prefill_tokens; + size_t decode_tokens; + const char* error_message; + const char* function_calls_json; +}; + +void init(const char* project_id = nullptr, const char* project_scope = nullptr, const char* cloud_key = nullptr); +void setEnabled(bool enabled); +void setCloudDisabled(bool disabled); +void setAppId(const char* app_id); +void setTelemetryEnvironment(const char* framework, const char* cache_location, const char* version = nullptr); +void setCloudKey(const char* key); +void cacheCloudApiKey(const char* key); +std::string loadCachedCloudApiKey(); +void recordInit(const char* model, bool success, double response_time_ms, const char* message); +void recordCompletion(const char* model, const CompletionMetrics& metrics); +void recordCompletion(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message); +void recordEmbedding(const char* model, bool success, const char* message); +void recordTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double ram_usage_mb, const char* message); +void recordStreamTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double session_ttft_ms, double session_tps, double session_time_ms, int session_tokens, const char* message); +void setStreamMode(bool in_stream); +bool isStreamMode(); +void markInference(bool active); +void flush(); +void shutdown(); + +} // namespace telemetry +} // namespace cactus diff --git a/python/examples/compare_gemma4_multimodal.py b/python/examples/compare_gemma4_multimodal.py new file mode 100644 index 000000000..c176500b8 --- /dev/null +++ b/python/examples/compare_gemma4_multimodal.py @@ -0,0 +1,933 @@ +from __future__ import annotations + +import argparse +import copy +import gc +import json +import os +import sys +import time +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +EXAMPLES_DIR = Path(__file__).resolve().parent +PYTHON_ROOT = EXAMPLES_DIR.parent +PROJECT_ROOT = PYTHON_ROOT.parent + +sys.path.insert(0, str(PYTHON_ROOT)) +sys.path.insert(0, str(EXAMPLES_DIR)) + +from src.cactus import cactus_complete +from src.cactus import cactus_destroy +from src.cactus import cactus_init +from src.cactus import cactus_reset +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph +from transpile_hf_model import TranspileWrapper +from transpile_hf_model import _graph_to_dict +from transpile_hf_model import _load_transformers_bundle +from transpile_hf_model import _lower_preoptimized_ir +from transpile_hf_model import _parse_dtype +from transpile_hf_model import _prepare_gemma4_multimodal_inputs +from transpile_hf_model import _serialize_json_compatible +from transpile_hf_model import _validate_weights_dir +from transpile_hf_model import _write_json + + +_DEFAULT_STOP_SEQUENCES = ("", "", "", "<|im_end|>") + + +@dataclass +class NumericSummary: + mean: float + min: float + max: float + median: float + + +@dataclass +class HandwrittenRun: + wall_ms: float + total_time_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + total_tokens: int + response: str + response_json: dict[str, Any] + + +@dataclass +class TranspiledRun: + preprocess_ms: float + execute_ms: float + total_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + total_tokens: int + response: str + stop_reason: str + generated_token_ids: list[int] + input_shapes: dict[str, list[int]] + + +def _numeric_summary(values: list[float]) -> NumericSummary: + arr = np.asarray(values, dtype=np.float64) + return NumericSummary( + mean=float(np.mean(arr)), + min=float(np.min(arr)), + max=float(np.max(arr)), + median=float(np.median(arr)), + ) + + +def _roundtrip_jsonable(value: Any) -> Any: + return json.loads(json.dumps(_serialize_json_compatible(value))) + + +def _resolve_default_weights_dir(model_id: str) -> Path | None: + env_path = os.environ.get("CACTUS_TEST_GEMMA4_MODEL") + if env_path: + candidate = Path(env_path).resolve() + if (candidate / "config.txt").exists(): + return candidate + + candidates = ( + PROJECT_ROOT / "weights" / "gemma4_int4", + PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower(), + PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower().replace("_", "-"), + ) + for candidate in candidates: + if (candidate / "config.txt").exists(): + return candidate + return None + + +def _require_processor_tokenizer(processor_or_tokenizer: object) -> object: + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + if not hasattr(tokenizer_like, "decode") or not hasattr(tokenizer_like, "encode"): + raise RuntimeError( + "Gemma4 compare script requires a tokenizer-like object with encode/decode support." + ) + return tokenizer_like + + +def _build_messages_payload( + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + system_prompt: str = "", +) -> list[dict[str, object]]: + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system: + messages.append({"role": "system", "content": normalized_system}) + + user_message: dict[str, object] = { + "role": "user", + "content": prompt, + } + if image_files: + user_message["images"] = [str(Path(path).resolve()) for path in image_files] + if audio_file: + user_message["audio"] = [str(Path(audio_file).resolve())] + messages.append(user_message) + return messages + + +def _encode_stop_sequences(tokenizer: object, stop_sequences: tuple[str, ...]) -> list[list[int]]: + encoded: list[list[int]] = [] + encode = getattr(tokenizer, "encode", None) + if not callable(encode): + return encoded + for stop_sequence in stop_sequences: + try: + token_ids = list(encode(stop_sequence, add_special_tokens=False)) + except TypeError: + token_ids = list(encode(stop_sequence)) + if token_ids: + encoded.append([int(token_id) for token_id in token_ids]) + return encoded + + +def _has_suffix(token_ids: list[int], suffix: list[int]) -> bool: + if not suffix or len(token_ids) < len(suffix): + return False + return token_ids[-len(suffix) :] == suffix + + +def _trim_stop_suffix(token_ids: list[int], stop_sequences: list[list[int]]) -> bool: + for stop_sequence in stop_sequences: + if _has_suffix(token_ids, stop_sequence): + del token_ids[-len(stop_sequence) :] + return True + return False + + +def _decode_generated_text(tokenizer: object, token_ids: list[int], *, skip_special_tokens: bool) -> str: + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + raise RuntimeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") + try: + return str(decode(token_ids, skip_special_tokens=skip_special_tokens)) + except TypeError: + return str(decode(token_ids)) + + +def _summarize_handwritten_runs( + runs: list[HandwrittenRun], + *, + weights_dir: Path, +) -> dict[str, Any]: + if not runs: + raise ValueError("expected at least one handwritten run") + return { + "weights_dir": str(weights_dir), + "latest_response": runs[-1].response, + "total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), + "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), + "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], + } + + +def _summarize_transpiled_runs( + runs: list[TranspiledRun], + *, + compile_time_ms: float, + model_source: str, + weight_bindings: int, + prepared_input_shapes: dict[str, list[int]], +) -> dict[str, Any]: + if not runs: + raise ValueError("expected at least one transpiled run") + return { + "compile_time_ms": compile_time_ms, + "model_source": model_source, + "weight_bindings": weight_bindings, + "prepared_input_shapes": prepared_input_shapes, + "latest_response": runs[-1].response, + "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), + "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), + "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "total_tokens": asdict(_numeric_summary([float(run.total_tokens) for run in runs])), + "runs": [_roundtrip_jsonable(asdict(run)) for run in runs], + } + + +def _prepare_repeat_inputs( + processor: object, + *, + tokenizer: object, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str, + enable_thinking_if_supported: bool, + max_new_tokens: int, +) -> tuple[object, int, float]: + start = time.perf_counter() + prepared = _prepare_gemma4_multimodal_inputs( + processor, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + use_gemma4_chat_template=True, + ) + pad_token_id = getattr(tokenizer, "pad_token_id", None) + if pad_token_id is None: + pad_token_id = 0 + prompt_tokens = 0 + for name, tensor in zip(prepared.names, prepared.tensors): + if name == "input_ids": + prompt_tokens = int(tensor.shape[1]) + break + if prompt_tokens <= 0: + raise RuntimeError("Gemma4 multimodal prepared inputs did not include a non-empty input_ids tensor") + target_tokens = prompt_tokens + int(max_new_tokens) + + input_tensors: list[torch.Tensor] = [] + for name, tensor in zip(prepared.names, prepared.tensors): + if name in {"input_ids", "attention_mask", "token_type_ids"}: + padded_shape = list(tensor.shape) + padded_shape[1] = target_tokens + if name == "input_ids": + padded = torch.full( + padded_shape, + int(pad_token_id), + dtype=tensor.dtype, + device=tensor.device, + ) + else: + padded = torch.zeros( + padded_shape, + dtype=tensor.dtype, + device=tensor.device, + ) + padded[:, : tensor.shape[1]] = tensor + input_tensors.append(padded) + else: + input_tensors.append(tensor) + metadata = dict(prepared.metadata) + metadata["prompt_token_count"] = prompt_tokens + metadata["target_token_count"] = int(target_tokens) + metadata["input_shapes"] = { + name: list(tensor.shape) + for name, tensor in zip(prepared.names, input_tensors) + } + prepared = type(prepared)( + names=prepared.names, + tensors=tuple(input_tensors), + metadata=metadata, + ) + preprocess_ms = (time.perf_counter() - start) * 1000.0 + return prepared, prompt_tokens, preprocess_ms + + +def _run_handwritten_once( + *, + model_handle: Any, + messages_json: str, + options_json: str, +) -> HandwrittenRun: + cactus_reset(model_handle) + streamed_token_ids: list[int] = [] + + def _collect(_text: str, token_id: int) -> None: + streamed_token_ids.append(int(token_id)) + + start = time.perf_counter() + response_json_text = cactus_complete( + model_handle, + messages_json, + options_json, + "[]", + _collect, + pcm_data=None, + ) + wall_ms = (time.perf_counter() - start) * 1000.0 + response_payload = json.loads(response_json_text) + return HandwrittenRun( + wall_ms=wall_ms, + total_time_ms=float(response_payload.get("total_time_ms", wall_ms)), + time_to_first_token_ms=float(response_payload.get("time_to_first_token_ms", 0.0)), + prefill_tps=float(response_payload.get("prefill_tps", 0.0)), + decode_tps=float(response_payload.get("decode_tps", 0.0)), + decode_tokens=len(streamed_token_ids), + total_tokens=int(response_payload.get("total_tokens", len(streamed_token_ids))), + response=str(response_payload.get("response", "")), + response_json=response_payload, + ) + + +def _run_transpiled_generation_once( + *, + tg, + processor: object, + tokenizer: object, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str, + enable_thinking_if_supported: bool, + max_new_tokens: int, + stop_sequences: tuple[str, ...], + expected_input_shapes: dict[str, list[int]] | None = None, + prepared_override: object | None = None, + progress_label: str = "transpiled", + progress_every_tokens: int = 8, +) -> TranspiledRun: + if prepared_override is not None: + prepared = type(prepared_override)( + names=prepared_override.names, + tensors=tuple(tensor.clone() for tensor in prepared_override.tensors), + metadata=dict(prepared_override.metadata), + ) + prompt_tokens = int(prepared.metadata.get("prompt_token_count", 0)) + preprocess_ms = 0.0 + else: + prepared, prompt_tokens, preprocess_ms = _prepare_repeat_inputs( + processor, + tokenizer=tokenizer, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + max_new_tokens=max_new_tokens, + ) + inputs_by_name = { + name: tensor.detach().cpu().numpy() + for name, tensor in zip(prepared.names, prepared.tensors) + } + if expected_input_shapes is not None: + for name, tensor in zip(prepared.names, prepared.tensors): + expected_shape = expected_input_shapes.get(name) + if expected_shape is None: + continue + actual_shape = list(tensor.shape) + if actual_shape != list(expected_shape): + raise RuntimeError( + f"runtime prepared input shape mismatch for {name}: " + f"compiled with {list(expected_shape)}, got {actual_shape}" + ) + if "input_ids" not in inputs_by_name: + raise RuntimeError("transpiled Gemma4 generation requires input_ids") + + target_tokens = int(prepared.metadata["target_token_count"]) + + eos_token_id = getattr(tokenizer, "eos_token_id", None) + + padded_inputs: dict[str, np.ndarray] = dict(inputs_by_name) + generated_ids: list[int] = [] + encoded_stop_sequences = _encode_stop_sequences(tokenizer, stop_sequences) + current_length = prompt_tokens + first_token_ms = 0.0 + execute_start = time.perf_counter() + stop_reason = "max_new_tokens" + progress_interval = max(1, int(progress_every_tokens)) + + for step_index in range(max_new_tokens): + runtime_inputs = [padded_inputs[name] for name in prepared.names] + if step_index == 0: + print("transpiled_step0_set_inputs_begin=true", flush=True) + tg.set_inputs(runtime_inputs) + if step_index == 0: + print("transpiled_step0_set_inputs_done=true", flush=True) + print("transpiled_step0_execute_begin=true", flush=True) + logits = tg.execute()[0].numpy() + if step_index == 0: + print("transpiled_step0_execute_done=true", flush=True) + if logits.ndim == 3: + token_position = current_length - 1 + if logits.shape[1] == 1: + token_position = 0 + next_token_id = int(np.argmax(logits[0, token_position])) + elif logits.ndim == 2: + next_token_id = int(np.argmax(logits[0])) + else: + raise RuntimeError(f"unexpected transpiled logits rank: {logits.ndim}") + generated_ids.append(next_token_id) + + if step_index == 0: + first_token_ms = preprocess_ms + (time.perf_counter() - execute_start) * 1000.0 + + if eos_token_id is not None and next_token_id == int(eos_token_id): + stop_reason = "eos_token" + break + if _trim_stop_suffix(generated_ids, encoded_stop_sequences): + stop_reason = "stop_sequence" + break + + if current_length >= target_tokens: + stop_reason = "context_limit" + break + + padded_inputs["input_ids"][0, current_length] = next_token_id + padded_inputs["attention_mask"][0, current_length] = 1 + padded_inputs["token_type_ids"][0, current_length] = 0 + if step_index == 0: + print("transpiled_step0_state_update_done=true", flush=True) + current_length += 1 + generated_count = len(generated_ids) + if generated_count % progress_interval == 0 or generated_count == max_new_tokens: + elapsed_ms = (time.perf_counter() - execute_start) * 1000.0 + print( + f"{progress_label}_decode_progress={generated_count}/{max_new_tokens} " + f"elapsed_ms={elapsed_ms:.3f}", + flush=True, + ) + + execute_ms = (time.perf_counter() - execute_start) * 1000.0 + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=True).strip() + if not response: + response = _decode_generated_text(tokenizer, generated_ids, skip_special_tokens=False).strip() + decode_time_ms = max(0.0, execute_ms + preprocess_ms - first_token_ms) + decode_tps = ( + ((len(generated_ids) - 1) * 1000.0) / decode_time_ms + if len(generated_ids) > 1 and decode_time_ms > 0.0 + else 0.0 + ) + prefill_tps = (prompt_tokens * 1000.0) / first_token_ms if first_token_ms > 0.0 else 0.0 + print( + f"{progress_label}_decode_done=true tokens={len(generated_ids)} " + f"stop_reason={stop_reason} execute_ms={execute_ms:.3f}", + flush=True, + ) + + return TranspiledRun( + preprocess_ms=preprocess_ms, + execute_ms=execute_ms, + total_ms=preprocess_ms + execute_ms, + time_to_first_token_ms=first_token_ms, + prefill_tps=prefill_tps, + decode_tps=decode_tps, + decode_tokens=len(generated_ids), + total_tokens=prompt_tokens + len(generated_ids), + response=response, + stop_reason=stop_reason, + generated_token_ids=generated_ids, + input_shapes={ + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + ) + + +def _print_summary(summary: dict[str, Any]) -> None: + print(f"model_id={summary['model_id']}") + print(f"prompt={summary['prompt']!r}") + if summary.get("image_files"): + print(f"image_files={summary['image_files']}") + if summary.get("audio_file"): + print(f"audio_file={summary['audio_file']}") + + handwritten = summary.get("handwritten") + if handwritten is not None: + hw_total = handwritten["total_time_ms"]["mean"] + hw_ttft = handwritten["time_to_first_token_ms"]["mean"] + hw_tokens = handwritten["decode_tokens"]["mean"] + hw_tps = hw_tokens * 1000.0 / max(hw_total, 1e-9) + print("handwritten:") + print(f" total_ms mean={hw_total:.3f}") + print(f" time_to_first_token_ms mean={hw_ttft:.3f}") + print(f" decode_tokens mean={hw_tokens:.3f}") + print(f" output_tokens_per_sec={hw_tps:.3f}") + print(f" latest_response={handwritten['latest_response']!r}") + + transpiled = summary.get("transpiled") + if transpiled is not None: + tr_total = transpiled["total_ms"]["mean"] + tr_ttft = transpiled["time_to_first_token_ms"]["mean"] + tr_tokens = transpiled["decode_tokens"]["mean"] + tr_tps = tr_tokens * 1000.0 / max(tr_total, 1e-9) + print("transpiled:") + print(f" compile_time_ms={transpiled['compile_time_ms']:.3f}") + print(f" total_ms mean={tr_total:.3f}") + print(f" time_to_first_token_ms mean={tr_ttft:.3f}") + print(f" decode_tokens mean={tr_tokens:.3f}") + print(f" output_tokens_per_sec={tr_tps:.3f}") + print(f" weight_bindings={transpiled['weight_bindings']}") + print(f" latest_response={transpiled['latest_response']!r}") + + comparison = summary.get("comparison") + if comparison is not None: + print("comparison:") + print( + " end_to_end_speedup_vs_handwritten=" + f"{comparison['end_to_end_speedup_vs_handwritten']:.6f}" + ) + print(f" responses_match={comparison['responses_match']}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Compare handwritten Cactus Gemma4 multimodal generation against the " + "transpiled Gemma4 multimodal graph on the same prompt, image, and audio." + ) + ) + parser.add_argument("--model-id", default="google/gemma-4-E2B-it") + parser.add_argument("--prompt", required=True) + parser.add_argument("--system-prompt", default="") + parser.add_argument("--image-file", action="append", default=[]) + parser.add_argument("--audio-file", default="") + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus weights directory. Required for handwritten comparison and reused for transpiled mmap bindings.", + ) + parser.add_argument("--artifact-dir", default="", help="Optional directory for summary and transpile artifacts.") + parser.add_argument("--graph-filename", default="graph.cactus") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=1) + parser.add_argument("--max-new-tokens", type=int, default=96) + parser.add_argument( + "--transpiled-progress-every-tokens", + type=int, + default=1, + help=( + "Emit transpiled decode progress every N generated tokens. " + "Use 1 for per-token visibility on long CPU-bound Gemma4 runs." + ), + ) + parser.add_argument("--torch-dtype", default="float16") + parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--skip-handwritten", action="store_true") + parser.add_argument("--skip-transpiled", action="store_true") + parser.add_argument("--enable-thinking", action="store_true") + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--top-p", type=float, default=1.0) + parser.add_argument("--top-k", type=int, default=1) + parser.add_argument("--stop-sequence", action="append", default=[]) + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + args = parser.parse_args() + + if args.skip_handwritten and args.skip_transpiled: + raise RuntimeError("cannot skip both handwritten and transpiled runs") + + image_files = tuple(str(Path(path).resolve()) for path in args.image_file if str(path).strip()) + audio_file = str(Path(args.audio_file).resolve()) if args.audio_file.strip() else None + if audio_file and not Path(audio_file).exists(): + raise FileNotFoundError(f"audio file does not exist: {audio_file}") + for image_file in image_files: + if not Path(image_file).exists(): + raise FileNotFoundError(f"image file does not exist: {image_file}") + + torch_dtype = _parse_dtype(args.torch_dtype) + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + + weights_dir_path = None + if args.weights_dir.strip(): + weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) + else: + weights_dir_path = _resolve_default_weights_dir(args.model_id) + + if not args.skip_handwritten and weights_dir_path is None: + raise RuntimeError( + "handwritten Gemma4 comparison requires a converted weights directory.\n" + "\n" + "Create one with something like:\n" + f" cactus convert {args.model_id} /path/to/gemma4_weights\n" + "\n" + "Then rerun with --weights-dir /path/to/gemma4_weights, or rerun with --skip-handwritten." + ) + + stop_sequences = tuple(args.stop_sequence) if args.stop_sequence else _DEFAULT_STOP_SEQUENCES + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + summary: dict[str, Any] = { + "model_id": args.model_id, + "prompt": args.prompt, + "system_prompt": args.system_prompt, + "image_files": list(image_files), + "audio_file": audio_file, + "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, + "warmup": args.warmup, + "repeats": args.repeats, + "max_new_tokens": args.max_new_tokens, + "transpiled_progress_every_tokens": int(args.transpiled_progress_every_tokens), + "torch_dtype": args.torch_dtype, + "trust_remote_code": bool(args.trust_remote_code), + "local_files_only": bool(args.local_files_only), + "enable_thinking": bool(args.enable_thinking), + "temperature": float(args.temperature), + "top_p": float(args.top_p), + "top_k": int(args.top_k), + "stop_sequences": list(stop_sequences), + } + + messages_payload = _build_messages_payload( + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + system_prompt=args.system_prompt, + ) + messages_json = json.dumps(messages_payload) + options_json = json.dumps( + { + "temperature": float(args.temperature), + "top_p": float(args.top_p), + "top_k": int(args.top_k), + "max_tokens": int(args.max_new_tokens), + "stop_sequences": list(stop_sequences), + "enable_thinking_if_supported": bool(args.enable_thinking), + "auto_handoff": False, + "telemetry_enabled": False, + } + ) + + transpiled_runs: list[TranspiledRun] = [] + if not args.skip_transpiled: + model_source, processor_or_tokenizer, model, _ = _load_transformers_bundle( + model_id=args.model_id, + task="multimodal_causal_lm_logits", + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + processor = processor_or_tokenizer + tokenizer = _require_processor_tokenizer(processor_or_tokenizer) + initial_prepared, _, initial_prepare_ms = _prepare_repeat_inputs( + processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + ) + canonical = canonicalize_model_interface( + model, + task="multimodal_causal_lm_logits", + input_names=initial_prepared.names, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + ) + if hasattr(canonical.module, "last_token_logits_only"): + canonical.module.last_token_logits_only = True + prime_static_features = getattr(canonical.module, "prime_static_multimodal_features", None) + if callable(prime_static_features): + prime_static_features(*initial_prepared.tensors) + wrapper = TranspileWrapper( + canonical.module, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + ).eval() + + compile_start = time.perf_counter() + print("transpile_capture_begin=true", flush=True) + prepare_cpu_float32_capture = getattr(canonical.module, "prepare_cpu_float32_capture", None) + restore_cpu_float32_capture = getattr(canonical.module, "restore_cpu_float32_capture", None) + if callable(prepare_cpu_float32_capture): + prepare_cpu_float32_capture() + try: + captured = capture_model(wrapper, initial_prepared.tensors) + finally: + if callable(restore_cpu_float32_capture): + restore_cpu_float32_capture() + print("transpile_capture_done=true", flush=True) + raw_ir_graph = copy.deepcopy(captured.ir_graph) + print("transpile_canonicalize_begin=true", flush=True) + canonicalize_exported_graph(captured.ir_graph) + print("transpile_canonicalize_done=true", flush=True) + print("transpile_optimize_begin=true", flush=True) + optimize_graph(captured.ir_graph, config=fusion_config) + print("transpile_optimize_done=true", flush=True) + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + print("transpile_lower_begin=true", flush=True) + tg = _lower_preoptimized_ir(captured.ir_graph) + print("transpile_lower_done=true", flush=True) + compile_time_ms = (time.perf_counter() - compile_start) * 1000.0 + + op_counts: dict[str, int] = {} + for node_id in optimized_ir_graph.order: + op = optimized_ir_graph.nodes[node_id].op + op_counts[op] = op_counts.get(op, 0) + 1 + weight_bindings = sum( + 1 + for value in optimized_ir_graph.values.values() + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) + ) + + summary["transpile_compile"] = { + "model_source": model_source, + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": weight_bindings, + "compile_time_ms": compile_time_ms, + "initial_prepare_ms": initial_prepare_ms, + "ops": op_counts, + } + prompt_token_count = int(initial_prepared.metadata.get("prompt_token_count", 0)) + target_token_count = int(initial_prepared.metadata.get("target_token_count", 0)) + total_transpiled_runs = int(args.warmup) + int(args.repeats) + print( + "transpiled_decode_plan=" + f"prompt_tokens:{prompt_token_count} " + f"target_tokens:{target_token_count} " + f"max_new_tokens:{int(args.max_new_tokens)} " + f"runs:{total_transpiled_runs} " + f"progress_every_tokens:{int(args.transpiled_progress_every_tokens)}", + flush=True, + ) + + if weights_dir_path is not None and weight_bindings == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir_path}\n" + "\n" + "The converted weights folder exists, but none of the captured constants matched weights_manifest.json.\n" + "Re-convert Gemma4 with the current converter before benchmarking." + ) + + if artifact_dir is not None: + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": "multimodal_causal_lm_logits", + "inputs": _serialize_json_compatible(initial_prepared.metadata), + "graph": _graph_to_dict(raw_ir_graph), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": "multimodal_causal_lm_logits", + "inputs": _serialize_json_compatible(initial_prepared.metadata), + "graph": _graph_to_dict(optimized_ir_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + if weight_bindings > 0: + skip_reason = ( + "skipped saved_graph because graph serialization does not yet preserve " + "mmap-bound weights/embeddings required by this transpiled Gemma4 graph" + ) + summary["transpile_compile"]["saved_graph_skipped_reason"] = skip_reason + print(f"note={skip_reason}") + else: + graph_path = artifact_dir / args.graph_filename + tg.graph.save(graph_path) + print(f"saved_graph={graph_path}") + + for _ in range(args.warmup): + warmup_index = _ + 1 + print(f"transpiled_warmup_begin={warmup_index}/{args.warmup}", flush=True) + _run_transpiled_generation_once( + tg=tg, + processor=processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + stop_sequences=stop_sequences, + expected_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + prepared_override=initial_prepared, + progress_label=f"transpiled_warmup{warmup_index}", + progress_every_tokens=args.transpiled_progress_every_tokens, + ) + for _ in range(args.repeats): + repeat_index = _ + 1 + print(f"transpiled_repeat_begin={repeat_index}/{args.repeats}", flush=True) + transpiled_runs.append( + _run_transpiled_generation_once( + tg=tg, + processor=processor, + tokenizer=tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + max_new_tokens=args.max_new_tokens, + stop_sequences=stop_sequences, + expected_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + prepared_override=initial_prepared, + progress_label=f"transpiled_repeat{repeat_index}", + progress_every_tokens=args.transpiled_progress_every_tokens, + ) + ) + + summary["transpiled"] = _summarize_transpiled_runs( + transpiled_runs, + compile_time_ms=compile_time_ms, + model_source=model_source, + weight_bindings=weight_bindings, + prepared_input_shapes=dict(initial_prepared.metadata.get("input_shapes", {})), + ) + + # Keep the HF/export capture stack alive until the transpiled graph has + # completed execution. Gemma4 teardown before first execute has been + # unstable in practice, even though the lowered graph is already built. + del model + del wrapper + del canonical + del captured + gc.collect() + + handwritten_runs: list[HandwrittenRun] = [] + if not args.skip_handwritten and weights_dir_path is not None: + model_handle = cactus_init(str(weights_dir_path), None, False) + try: + for _ in range(args.warmup): + warmup_index = _ + 1 + print(f"handwritten_warmup_begin={warmup_index}/{args.warmup}", flush=True) + _run_handwritten_once( + model_handle=model_handle, + messages_json=messages_json, + options_json=options_json, + ) + for _ in range(args.repeats): + repeat_index = _ + 1 + print(f"handwritten_repeat_begin={repeat_index}/{args.repeats}", flush=True) + handwritten_runs.append( + _run_handwritten_once( + model_handle=model_handle, + messages_json=messages_json, + options_json=options_json, + ) + ) + finally: + cactus_destroy(model_handle) + summary["handwritten"] = _summarize_handwritten_runs( + handwritten_runs, + weights_dir=weights_dir_path, + ) + + if handwritten_runs and transpiled_runs: + handwritten_latest = handwritten_runs[-1].response.strip() + transpiled_latest = transpiled_runs[-1].response.strip() + summary["comparison"] = { + "end_to_end_speedup_vs_handwritten": ( + summary["handwritten"]["total_time_ms"]["mean"] + / max(summary["transpiled"]["total_ms"]["mean"], 1e-9) + ), + "responses_match": handwritten_latest == transpiled_latest, + "handwritten_latest_response": handwritten_latest, + "transpiled_latest_response": transpiled_latest, + } + + if artifact_dir is not None: + summary_path = artifact_dir / "summary.json" + summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") + print(f"saved_summary={summary_path}") + + _print_summary(summary) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/compare_parakeet_performance.py b/python/examples/compare_parakeet_performance.py new file mode 100644 index 000000000..621f1e1f5 --- /dev/null +++ b/python/examples/compare_parakeet_performance.py @@ -0,0 +1,793 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from collections import defaultdict +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from scipy.io import wavfile + +EXAMPLES_DIR = Path(__file__).resolve().parent +PYTHON_ROOT = EXAMPLES_DIR.parent +PROJECT_ROOT = PYTHON_ROOT.parent + +sys.path.insert(0, str(PYTHON_ROOT)) +sys.path.insert(0, str(EXAMPLES_DIR)) + +from src.cactus import cactus_destroy +from src.cactus import cactus_init +from src.cactus import cactus_transcribe +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph +from transpile_hf_model import TranspileWrapper +from transpile_hf_model import _ctc_greedy_decode_token_ids +from transpile_hf_model import _decode_token_ids +from transpile_hf_model import _infer_task_from_config +from transpile_hf_model import _load_optional_json +from transpile_hf_model import _load_optional_tokenizer +from transpile_hf_model import _load_transformers_bundle +from transpile_hf_model import _lower_preoptimized_ir +from transpile_hf_model import _parse_dtype +from transpile_hf_model import _prepare_audio_inputs +from transpile_hf_model import _validate_weights_dir + + +@dataclass +class NumericSummary: + mean: float + min: float + max: float + median: float + + +@dataclass +class ProfileEntry: + op: str + time_ms: float + shape: str + raw_line: str + + +@dataclass +class ProfileSection: + entries: list[ProfileEntry] + total_ms: float + + +@dataclass +class HandwrittenRun: + wall_ms: float + total_time_ms: float + time_to_first_token_ms: float + prefill_tps: float + decode_tps: float + decode_tokens: int + prefill_tokens: int + total_tokens: int + confidence: float + transcript: str + response_json: dict[str, Any] + + +@dataclass +class TranspiledRun: + preprocess_ms: float + execute_ms: float + decode_ms: float + total_ms: float + graph_only_ms: float + output_frames: int + output_vocab: int + decode_tokens: int + transcript: str | None + input_shapes: dict[str, list[int]] + + +def _numeric_summary(values: list[float]) -> NumericSummary: + arr = np.asarray(values, dtype=np.float64) + return NumericSummary( + mean=float(np.mean(arr)), + min=float(np.min(arr)), + max=float(np.max(arr)), + median=float(np.median(arr)), + ) + + +def _resolve_default_weights_dir(model_id: str) -> Path | None: + candidate = PROJECT_ROOT / "weights" / model_id.split("/")[-1].lower() + if (candidate / "config.txt").exists(): + return candidate + return None + + +def _audio_duration_seconds(audio_file: str) -> float: + sample_rate, samples = wavfile.read(audio_file) + if sample_rate <= 0: + raise ValueError(f"invalid sample rate in {audio_file}: {sample_rate}") + return float(len(samples)) / float(sample_rate) + + +def _profile_env(profile_file: Path | None): + class _ProfileContext: + def __enter__(self_inner): + self_inner.old_profile = os.environ.get("CACTUS_PROFILE") + self_inner.old_profile_file = os.environ.get("CACTUS_PROFILE_FILE") + if profile_file is None: + os.environ.pop("CACTUS_PROFILE", None) + os.environ.pop("CACTUS_PROFILE_FILE", None) + return self_inner + profile_file.parent.mkdir(parents=True, exist_ok=True) + if profile_file.exists(): + profile_file.unlink() + os.environ.pop("CACTUS_PROFILE", None) + os.environ["CACTUS_PROFILE_FILE"] = str(profile_file) + return self_inner + + def __exit__(self_inner, exc_type, exc, tb): + if self_inner.old_profile is None: + os.environ.pop("CACTUS_PROFILE", None) + else: + os.environ["CACTUS_PROFILE"] = self_inner.old_profile + + if self_inner.old_profile_file is None: + os.environ.pop("CACTUS_PROFILE_FILE", None) + else: + os.environ["CACTUS_PROFILE_FILE"] = self_inner.old_profile_file + return False + + return _ProfileContext() + + +def _parse_profile_file(path: Path) -> list[ProfileSection]: + if not path.exists(): + return [] + + lines = path.read_text().splitlines() + sections: list[ProfileSection] = [] + idx = 0 + while idx < len(lines): + if lines[idx].strip() != "=== Graph Execution Profile ===": + idx += 1 + continue + + entries: list[ProfileEntry] = [] + total_ms = 0.0 + idx += 1 + while idx < len(lines): + line = lines[idx].rstrip() + stripped = line.strip() + if stripped.startswith("Total execution time:"): + try: + total_ms = float(stripped.split(":", 1)[1].strip().split()[0]) + except Exception: + total_ms = 0.0 + elif stripped == "================================": + break + elif ( + stripped + and not stripped.startswith("Operation") + and not set(stripped).issubset({"-"}) + and not stripped.startswith("Total execution time:") + ): + parts = stripped.split() + if len(parts) >= 3: + op = parts[0] + try: + time_ms = float(parts[1]) + except ValueError: + time_ms = 0.0 + shape = parts[2] if parts[2].startswith("[") else "" + entries.append(ProfileEntry(op=op, time_ms=time_ms, shape=shape, raw_line=stripped)) + idx += 1 + + sections.append(ProfileSection(entries=entries, total_ms=total_ms)) + idx += 1 + + return sections + + +def _summarize_profile_sections(sections: list[ProfileSection]) -> dict[str, Any]: + op_total_ms: dict[str, float] = defaultdict(float) + op_count: dict[str, int] = defaultdict(int) + all_entries: list[ProfileEntry] = [] + total_ms_values: list[float] = [] + + for section in sections: + total_ms_values.append(section.total_ms) + all_entries.extend(section.entries) + for entry in section.entries: + op_total_ms[entry.op] += entry.time_ms + op_count[entry.op] += 1 + + by_op = [] + for op, total_ms in sorted(op_total_ms.items(), key=lambda item: item[1], reverse=True): + count = op_count[op] + by_op.append( + { + "op": op, + "count": count, + "total_ms": round(total_ms, 6), + "mean_ms": round(total_ms / max(count, 1), 6), + } + ) + + slowest_nodes = [ + { + "op": entry.op, + "time_ms": round(entry.time_ms, 6), + "shape": entry.shape, + "raw_line": entry.raw_line, + } + for entry in sorted(all_entries, key=lambda item: item.time_ms, reverse=True)[:50] + ] + + return { + "runs_profiled": len(sections), + "graph_total_ms": asdict(_numeric_summary(total_ms_values)) if total_ms_values else None, + "entries_per_run": [len(section.entries) for section in sections], + "by_op": by_op, + "slowest_nodes": slowest_nodes, + } + + +def _run_handwritten_once( + *, + model_handle: Any, + audio_file: str, +) -> HandwrittenRun: + options_json = json.dumps( + { + "use_vad": False, + "max_tokens": 4096, + } + ) + start = time.perf_counter() + response = cactus_transcribe( + model_handle, + audio_file, + "", + options_json, + None, + None, + ) + end = time.perf_counter() + + payload = json.loads(response) + return HandwrittenRun( + wall_ms=(end - start) * 1000.0, + total_time_ms=float(payload.get("total_time_ms", 0.0)), + time_to_first_token_ms=float(payload.get("time_to_first_token_ms", 0.0)), + prefill_tps=float(payload.get("prefill_tps", 0.0)), + decode_tps=float(payload.get("decode_tps", 0.0)), + decode_tokens=int(payload.get("decode_tokens", 0)), + prefill_tokens=int(payload.get("prefill_tokens", 0)), + total_tokens=int(payload.get("total_tokens", 0)), + confidence=float(payload.get("confidence", 0.0)), + transcript=str(payload.get("response", "")), + response_json=payload, + ) + + +def _run_transpiled_once( + *, + tg: Any, + processor: object | None, + input_names: tuple[str, ...], + model_config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + torch_dtype: torch.dtype, + audio_file: str, + tokenizer: object | None, + blank_token_id: int | None, +) -> TranspiledRun: + start_pre = time.perf_counter() + prepared = _prepare_audio_inputs( + processor, + input_names=input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task="ctc_logits", + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + end_pre = time.perf_counter() + + input_arrays = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] + + start_exec = time.perf_counter() + tg.set_inputs(input_arrays) + outputs = tg.execute() + end_exec = time.perf_counter() + + start_decode = time.perf_counter() + logits = outputs[0].numpy().astype(np.float32) + token_ids = _ctc_greedy_decode_token_ids(logits, blank_token_id=blank_token_id) + transcript = _decode_token_ids(tokenizer, token_ids) if tokenizer is not None else None + end_decode = time.perf_counter() + + return TranspiledRun( + preprocess_ms=(end_pre - start_pre) * 1000.0, + execute_ms=(end_exec - start_exec) * 1000.0, + decode_ms=(end_decode - start_decode) * 1000.0, + total_ms=(end_decode - start_pre) * 1000.0, + graph_only_ms=(end_exec - start_exec) * 1000.0, + output_frames=int(logits.shape[1]), + output_vocab=int(logits.shape[2]), + decode_tokens=len(token_ids), + transcript=transcript, + input_shapes={ + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + ) + + +def _run_transpiled_graph_only_once( + *, + tg: Any, + cached_inputs: list[np.ndarray], +) -> float: + start = time.perf_counter() + tg.set_inputs(cached_inputs) + tg.execute() + end = time.perf_counter() + return (end - start) * 1000.0 + + +def _warmup(fn, count: int) -> None: + for _ in range(max(count, 0)): + fn() + + +def _roundtrip_jsonable(obj: Any) -> Any: + if isinstance(obj, dict): + return {str(key): _roundtrip_jsonable(value) for key, value in obj.items()} + if isinstance(obj, list): + return [_roundtrip_jsonable(value) for value in obj] + if isinstance(obj, tuple): + return [_roundtrip_jsonable(value) for value in obj] + if isinstance(obj, Path): + return str(obj) + if isinstance(obj, NumericSummary): + return asdict(obj) + if hasattr(obj, "__dataclass_fields__"): + return asdict(obj) + return obj + + +def _build_transpiled_graph( + *, + model_id: str, + audio_file: str, + weights_dir: str | None, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, + fusion_config: FusionConfig, +): + task = _infer_task_from_config(model_id) + if task != "ctc_logits": + raise RuntimeError(f"expected a CTC model, got task={task}") + + model_source, processor, model, model_config = _load_transformers_bundle( + model_id=model_id, + task=task, + torch_dtype=torch_dtype, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) + + preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") + if not preprocessor_config: + preprocessor_config = _load_optional_json(model_id, "preprocessor_config.json") + + tokenizer = _load_optional_tokenizer( + model_id=model_id, + model_source=model_source, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) + + canonical = canonicalize_model_interface(model, task=task) + prepared = _prepare_audio_inputs( + processor, + input_names=canonical.input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface(model, task=task, input_names=prepared.names) + + wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + + capture_started = time.perf_counter() + captured = capture_model(wrapper, prepared.tensors) + canonicalize_exported_graph(captured.ir_graph) + optimize_graph(captured.ir_graph, config=fusion_config) + tg = _lower_preoptimized_ir(captured.ir_graph) + capture_finished = time.perf_counter() + + cached_inputs = [tensor.detach().cpu().numpy() for tensor in prepared.tensors] + blank_token_id = getattr(getattr(model, "config", None), "pad_token_id", None) + + return { + "tg": tg, + "model": model, + "processor": processor, + "model_config": model_config, + "preprocessor_config": preprocessor_config, + "tokenizer": tokenizer, + "blank_token_id": int(blank_token_id) if blank_token_id is not None else None, + "compile_time_ms": (capture_finished - capture_started) * 1000.0, + "cached_inputs": cached_inputs, + "input_names": tuple(prepared.names), + "prepared_input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(prepared.names, prepared.tensors) + }, + "model_source": model_source, + "weight_bindings": sum( + 1 + for value in captured.ir_graph.values.values() + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str) + ), + } + + +def _benchmark_handwritten( + *, + weights_dir: Path, + audio_file: str, + warmup: int, + repeats: int, + profile_repeats: int, + artifact_dir: Path | None, +) -> dict[str, Any]: + model_handle = cactus_init(str(weights_dir), None, False) + try: + _warmup(lambda: _run_handwritten_once(model_handle=model_handle, audio_file=audio_file), warmup) + + runs: list[HandwrittenRun] = [] + for _ in range(max(repeats, 1)): + runs.append(_run_handwritten_once(model_handle=model_handle, audio_file=audio_file)) + + profile_path = ( + artifact_dir / "handwritten_profile.txt" + if artifact_dir is not None and profile_repeats > 0 + else None + ) + with _profile_env(profile_path): + for _ in range(max(profile_repeats, 0)): + _run_handwritten_once(model_handle=model_handle, audio_file=audio_file) + + profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] + profile_summary = _summarize_profile_sections(profile_sections) + + return { + "weights_dir": str(weights_dir), + "runs": [_roundtrip_jsonable(run) for run in runs], + "wall_ms": asdict(_numeric_summary([run.wall_ms for run in runs])), + "internal_total_time_ms": asdict(_numeric_summary([run.total_time_ms for run in runs])), + "time_to_first_token_ms": asdict(_numeric_summary([run.time_to_first_token_ms for run in runs])), + "decode_tps": asdict(_numeric_summary([run.decode_tps for run in runs])), + "prefill_tps": asdict(_numeric_summary([run.prefill_tps for run in runs])), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "confidence": asdict(_numeric_summary([run.confidence for run in runs])), + "latest_transcript": runs[-1].transcript if runs else "", + "profile_file": str(profile_path) if profile_path is not None else None, + "profile_summary": profile_summary, + } + finally: + cactus_destroy(model_handle) + + +def _benchmark_transpiled( + *, + model_id: str, + audio_file: str, + weights_dir: str | None, + warmup: int, + repeats: int, + profile_repeats: int, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, + artifact_dir: Path | None, + fusion_config: FusionConfig, +) -> dict[str, Any]: + built = _build_transpiled_graph( + model_id=model_id, + audio_file=audio_file, + weights_dir=weights_dir, + torch_dtype=torch_dtype, + token=token, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + fusion_config=fusion_config, + ) + + tg = built["tg"] + model = built["model"] + processor = built["processor"] + input_names = built["input_names"] + model_config = built["model_config"] + preprocessor_config = built["preprocessor_config"] + tokenizer = built["tokenizer"] + blank_token_id = built["blank_token_id"] + cached_inputs = built["cached_inputs"] + + _warmup( + lambda: _run_transpiled_once( + tg=tg, + processor=processor, + input_names=input_names, + model_config=model_config, + preprocessor_config=preprocessor_config, + model=model, + torch_dtype=torch_dtype, + audio_file=audio_file, + tokenizer=tokenizer, + blank_token_id=blank_token_id, + ), + warmup, + ) + _warmup(lambda: _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs), warmup) + + runs: list[TranspiledRun] = [] + graph_only_ms: list[float] = [] + for _ in range(max(repeats, 1)): + runs.append( + _run_transpiled_once( + tg=tg, + processor=processor, + input_names=input_names, + model_config=model_config, + preprocessor_config=preprocessor_config, + model=model, + torch_dtype=torch_dtype, + audio_file=audio_file, + tokenizer=tokenizer, + blank_token_id=blank_token_id, + ) + ) + graph_only_ms.append(_run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs)) + + profile_path = ( + artifact_dir / "transpiled_profile.txt" + if artifact_dir is not None and profile_repeats > 0 + else None + ) + with _profile_env(profile_path): + for _ in range(max(profile_repeats, 0)): + _run_transpiled_graph_only_once(tg=tg, cached_inputs=cached_inputs) + + profile_sections = _parse_profile_file(profile_path) if profile_path is not None else [] + profile_summary = _summarize_profile_sections(profile_sections) + + return { + "model_source": built["model_source"], + "compile_time_ms": built["compile_time_ms"], + "weight_bindings": built["weight_bindings"], + "prepared_input_shapes": built["prepared_input_shapes"], + "runs": [_roundtrip_jsonable(run) for run in runs], + "preprocess_ms": asdict(_numeric_summary([run.preprocess_ms for run in runs])), + "execute_ms": asdict(_numeric_summary([run.execute_ms for run in runs])), + "decode_ms": asdict(_numeric_summary([run.decode_ms for run in runs])), + "total_ms": asdict(_numeric_summary([run.total_ms for run in runs])), + "graph_only_ms": asdict(_numeric_summary(graph_only_ms)), + "decode_tokens": asdict(_numeric_summary([float(run.decode_tokens) for run in runs])), + "output_frames": asdict(_numeric_summary([float(run.output_frames) for run in runs])), + "latest_transcript": runs[-1].transcript if runs else "", + "profile_file": str(profile_path) if profile_path is not None else None, + "profile_summary": profile_summary, + } + + +def _print_summary( + *, + audio_duration_sec: float, + handwritten: dict[str, Any] | None, + transpiled: dict[str, Any] | None, +) -> None: + print() + print(f"audio_duration_sec={audio_duration_sec:.3f}") + + if handwritten is not None: + hw_total = handwritten["internal_total_time_ms"]["mean"] + hw_graph = None + profile_total = handwritten.get("profile_summary", {}).get("graph_total_ms") + if isinstance(profile_total, dict): + hw_graph = profile_total.get("mean") + hw_tok = handwritten["decode_tokens"]["mean"] + hw_rtf = hw_total / max(audio_duration_sec * 1000.0, 1e-9) + hw_xrt = (audio_duration_sec * 1000.0) / max(hw_total, 1e-9) + hw_tok_s = hw_tok * 1000.0 / max(hw_total, 1e-9) + print("handwritten:") + print(f" total_ms mean={hw_total:.3f}") + print(f" graph_profile_ms mean={hw_graph:.3f}" if hw_graph is not None else " graph_profile_ms unavailable") + print(f" decode_tokens mean={hw_tok:.3f}") + print(f" output_tokens_per_sec={hw_tok_s:.3f}") + print(f" real_time_factor={hw_rtf:.3f}") + print(f" audio_x_realtime={hw_xrt:.3f}") + + if transpiled is not None: + tr_total = transpiled["total_ms"]["mean"] + tr_graph = transpiled["graph_only_ms"]["mean"] + tr_tok = transpiled["decode_tokens"]["mean"] + tr_rtf = tr_total / max(audio_duration_sec * 1000.0, 1e-9) + tr_xrt = (audio_duration_sec * 1000.0) / max(tr_total, 1e-9) + tr_tok_s = tr_tok * 1000.0 / max(tr_total, 1e-9) + tr_graph_tok_s = tr_tok * 1000.0 / max(tr_graph, 1e-9) + print("transpiled:") + print(f" total_ms mean={tr_total:.3f}") + print(f" graph_only_ms mean={tr_graph:.3f}") + print(f" decode_tokens mean={tr_tok:.3f}") + print(f" output_tokens_per_sec={tr_tok_s:.3f}") + print(f" graph_only_tokens_per_sec={tr_graph_tok_s:.3f}") + print(f" real_time_factor={tr_rtf:.3f}") + print(f" audio_x_realtime={tr_xrt:.3f}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Compare handwritten Cactus Parakeet performance against the transpiled " + "Parakeet graph on the same WAV input, including per-op graph profiles." + ) + ) + parser.add_argument("--model-id", default="nvidia/parakeet-ctc-1.1b") + parser.add_argument("--audio-file", required=True) + parser.add_argument( + "--weights-dir", + default="", + help="Converted Cactus weights directory. Reused for handwritten runtime and transpiled mmap bindings.", + ) + parser.add_argument("--artifact-dir", default="", help="Optional directory for JSON summaries and raw profiles.") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--profile-repeats", type=int, default=1) + parser.add_argument("--torch-dtype", default="float16") + parser.add_argument("--token", default=os.environ.get("HF_TOKEN")) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--skip-handwritten", action="store_true") + parser.add_argument("--skip-transpiled", action="store_true") + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + args = parser.parse_args() + + audio_file = str(Path(args.audio_file).resolve()) + if not Path(audio_file).exists(): + raise FileNotFoundError(f"audio file does not exist: {audio_file}") + + torch_dtype = _parse_dtype(args.torch_dtype) + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else None + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + + weights_dir_path = None + if args.weights_dir.strip(): + weights_dir_path = _validate_weights_dir(args.weights_dir.strip(), model_id=args.model_id) + else: + weights_dir_path = _resolve_default_weights_dir(args.model_id) + + if not args.skip_handwritten and weights_dir_path is None: + raise RuntimeError( + "handwritten comparison requires a converted weights directory.\n" + "\n" + f"Create one with:\n" + f" cactus convert {args.model_id} {PROJECT_ROOT / 'weights' / args.model_id.split('/')[-1].lower()}\n" + "\n" + "Or rerun with --skip-handwritten." + ) + + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + summary: dict[str, Any] = { + "model_id": args.model_id, + "audio_file": audio_file, + "audio_duration_sec": _audio_duration_seconds(audio_file), + "weights_dir": str(weights_dir_path) if weights_dir_path is not None else None, + "warmup": args.warmup, + "repeats": args.repeats, + "profile_repeats": args.profile_repeats, + "torch_dtype": args.torch_dtype, + "trust_remote_code": bool(args.trust_remote_code), + "local_files_only": bool(args.local_files_only), + } + + handwritten_summary = None + if not args.skip_handwritten and weights_dir_path is not None: + handwritten_summary = _benchmark_handwritten( + weights_dir=weights_dir_path, + audio_file=audio_file, + warmup=args.warmup, + repeats=args.repeats, + profile_repeats=args.profile_repeats, + artifact_dir=artifact_dir, + ) + summary["handwritten"] = handwritten_summary + + transpiled_summary = None + if not args.skip_transpiled: + transpiled_summary = _benchmark_transpiled( + model_id=args.model_id, + audio_file=audio_file, + weights_dir=str(weights_dir_path) if weights_dir_path is not None else None, + warmup=args.warmup, + repeats=args.repeats, + profile_repeats=args.profile_repeats, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + artifact_dir=artifact_dir, + fusion_config=fusion_config, + ) + summary["transpiled"] = transpiled_summary + + if handwritten_summary is not None and transpiled_summary is not None: + hw_total = handwritten_summary["internal_total_time_ms"]["mean"] + tr_total = transpiled_summary["total_ms"]["mean"] + hw_graph = handwritten_summary.get("profile_summary", {}).get("graph_total_ms") + tr_graph = transpiled_summary["graph_only_ms"]["mean"] + summary["comparison"] = { + "end_to_end_speedup_vs_handwritten": hw_total / max(tr_total, 1e-9), + "graph_only_speedup_vs_handwritten": ( + float(hw_graph["mean"]) / max(tr_graph, 1e-9) + if isinstance(hw_graph, dict) and "mean" in hw_graph + else None + ), + } + + if artifact_dir is not None: + summary_path = artifact_dir / "summary.json" + summary_path.write_text(json.dumps(_roundtrip_jsonable(summary), indent=2, sort_keys=True) + "\n") + print(f"saved_summary={summary_path}") + if handwritten_summary is not None and handwritten_summary.get("profile_file"): + print(f"saved_handwritten_profile={handwritten_summary['profile_file']}") + if transpiled_summary is not None and transpiled_summary.get("profile_file"): + print(f"saved_transpiled_profile={transpiled_summary['profile_file']}") + + _print_summary( + audio_duration_sec=float(summary["audio_duration_sec"]), + handwritten=handwritten_summary, + transpiled=transpiled_summary, + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/inspect_gemma4_cleanup.py b/python/examples/inspect_gemma4_cleanup.py new file mode 100644 index 000000000..ec9e78d05 --- /dev/null +++ b/python/examples/inspect_gemma4_cleanup.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +import torch +from transformers import AutoModelForCausalLM + +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.model_adapters import canonicalize_model_interface + + +class Gemma4FullModelWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = canonicalize_model_interface(model, task="causal_lm_logits").module + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model(input_ids) + + +class Gemma4FirstBlockCheckpointWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module, checkpoint_name: str): + super().__init__() + adapter = canonicalize_model_interface(model, task="causal_lm_logits") + if adapter.family != "gemma4": + raise ValueError(f"expected gemma4 adapter, got family={adapter.family}") + if not hasattr(adapter.module, "debug_first_block"): + raise ValueError("Gemma4 adapter does not expose debug_first_block()") + self.adapter = adapter.module + self.checkpoint_name = checkpoint_name + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + checkpoints = self.adapter.debug_first_block(input_ids) + if self.checkpoint_name not in checkpoints: + available = ", ".join(sorted(checkpoints.keys())) + raise KeyError(f"unknown checkpoint {self.checkpoint_name!r}; available: {available}") + return checkpoints[self.checkpoint_name] + + +def format_ir_graph(graph: IRGraph, *, max_nodes: int | None = None) -> str: + lines: list[str] = [] + lines.append("IR Summary") + lines.append(f" inputs={graph.inputs}") + lines.append(f" outputs={graph.outputs}") + lines.append(f" nodes={len(graph.order)}") + if graph.meta: + lines.append(f" meta={graph.meta}") + lines.append("") + lines.append("IR Nodes") + + node_ids = graph.order if max_nodes is None else graph.order[:max_nodes] + for index, node_id in enumerate(node_ids): + node = graph.nodes[node_id] + lines.append(f"[{index}] {node.id} op={node.op}") + lines.append(f" inputs={node.inputs}") + lines.append(f" outputs={node.outputs}") + if node.attrs: + lines.append(f" attrs={node.attrs}") + if node.kind != "generic": + lines.append(f" kind={node.kind}") + if node.meta: + lines.append(f" meta={node.meta}") + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + lines.append( + f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" + ) + + if max_nodes is not None and len(graph.order) > max_nodes: + lines.append(f"... ({len(graph.order) - max_nodes} more nodes omitted)") + + if graph.constants: + lines.append("") + lines.append("IR Constants") + constant_ids = sorted(graph.constants.keys()) + preview_ids = constant_ids if max_nodes is None else constant_ids[: max(8, min(len(constant_ids), max_nodes))] + for value_id in preview_ids: + value = graph.constants[value_id] + value_meta = graph.values.get(value_id) + shape = None if value_meta is None else value_meta.shape + dtype = None if value_meta is None else value_meta.dtype + lines.append(f"- {value_id}: type={type(value).__name__} shape={shape} dtype={dtype}") + if len(preview_ids) < len(constant_ids): + lines.append(f"... ({len(constant_ids) - len(preview_ids)} more constants omitted)") + + return "\n".join(lines) + + +def format_unsupported_ops(graph: IRGraph) -> str: + counts = graph.meta.get("canonical_unsupported_op_counts", {}) + if not isinstance(counts, dict): + counts = {} + + examples: dict[str, list[str]] = {} + for node_id in graph.order: + node = graph.nodes[node_id] + print(f"node {node_id} op={node.op}") + if node.op not in counts: + continue + bucket = examples.setdefault(node.op, []) + if len(bucket) < 5: + bucket.append(node.id) + + total = sum(int(count) for count in counts.values()) + lines = ["Unsupported Ops", f" unique={len(counts)}", f" total_nodes={total}"] + if not counts: + lines.append(" none") + return "\n".join(lines) + + for op_name, count in counts.items(): + sample_nodes = ", ".join(examples.get(op_name, ())) + lines.append(f"- {op_name}: {count}") + if sample_nodes: + lines.append(f" examples={sample_nodes}") + return "\n".join(lines) + + +def resolve_local_model_path(model_id: str) -> str: + model_path = Path(model_id) + if model_path.exists(): + return str(model_path) + + cache_root = Path.home() / ".cache" / "huggingface" / "hub" / ("models--" + model_id.replace("/", "--")) + snapshots_dir = cache_root / "snapshots" + if not snapshots_dir.exists(): + raise FileNotFoundError( + f"could not find local snapshot for {model_id!r} under {snapshots_dir}" + ) + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + raise FileNotFoundError(f"no snapshots found for {model_id!r} under {snapshots_dir}") + return str(snapshots[-1]) + + +def load_model(model_id: str) -> torch.nn.Module: + token = os.environ.get("HF_TOKEN") + common_kwargs: dict[str, object] = { + "local_files_only": True, + } + if token: + common_kwargs["token"] = token + + local_model_path = resolve_local_model_path(model_id) + model = AutoModelForCausalLM.from_pretrained( + local_model_path, + torch_dtype=torch.float16, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model + + +def build_module( + model: torch.nn.Module, + *, + mode: str, + checkpoint: str, +) -> torch.nn.Module: + if mode == "full": + return Gemma4FullModelWrapper(model).eval() + if mode == "first_block": + return Gemma4FirstBlockCheckpointWrapper(model, checkpoint).eval() + raise ValueError(f"unknown mode: {mode}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture Gemma4, run canonicalize/cleanup.py, and report unsupported ops.", + ) + parser.add_argument( + "--model-id", + default=os.environ.get("CACTUS_GEMMA_HF_MODEL_ID", "google/gemma-4-E2B"), + help="Hugging Face model id. Uses local cache only.", + ) + parser.add_argument( + "--mode", + choices=("first_block", "full"), + default="first_block", + help="Capture a first-block checkpoint or the full Gemma4 adapter graph.", + ) + parser.add_argument( + "--checkpoint", + default="after_ffn_residual", + help="First-block checkpoint name when --mode=first_block.", + ) + parser.add_argument( + "--input-ids", + default="2,818,5279,529,7001,563", + help="Comma-separated integer token ids to capture.", + ) + parser.add_argument( + "--max-nodes", + type=int, + default=120, + help="Maximum IR nodes to print when --print-ir is enabled.", + ) + parser.add_argument( + "--print-ir", + action="store_true", + help="Also print the IR before and after canonical cleanup.", + ) + args = parser.parse_args() + + model = load_model(args.model_id) + module = build_module(model, mode=args.mode, checkpoint=args.checkpoint) + token_ids = [int(token.strip()) for token in args.input_ids.split(",") if token.strip()] + input_ids = torch.tensor([token_ids], dtype=torch.long) + + print(f"Model: {args.model_id}") + print(f"Mode: {args.mode}") + print(f"Checkpoint: {args.checkpoint if args.mode == 'first_block' else '(full model)'}") + print(f"Input IDs shape: {tuple(input_ids.shape)}") + print("") + + captured = capture_model(module, (input_ids,)) + + if args.print_ir: + print("=== IR Before Canonical Cleanup ===") + print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) + print("") + + canonicalize_exported_graph(captured.ir_graph) + + print("=== Unsupported Ops After Canonical Cleanup ===") + print(format_unsupported_ops(captured.ir_graph)) + + if args.print_ir: + print("") + print("=== IR After Canonical Cleanup ===") + print(format_ir_graph(captured.ir_graph, max_nodes=args.max_nodes)) + + +if __name__ == "__main__": + main() diff --git a/python/examples/inspect_transpile_cleanup.py b/python/examples/inspect_transpile_cleanup.py new file mode 100644 index 000000000..366765f62 --- /dev/null +++ b/python/examples/inspect_transpile_cleanup.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import argparse + +import torch +import torch.nn as nn + +from src.transpile.capture_pytorch import capture_model +from src.transpile.capture_pytorch import dump_graph +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph + + +class Toy(nn.Module): + def forward(self, x, y): + z = x + y + z = z * 0.5 + z = torch.nn.functional.gelu(z) + z = torch.softmax(z, dim=-1) + return z + + +class AttentionBlockToy(nn.Module): + def __init__(self, hidden_size: int = 8): + super().__init__() + self.out_proj = nn.Linear(hidden_size, hidden_size, bias=False, dtype=torch.float16) + + def forward(self, q, k, v, gate): + attn = torch.nn.functional.scaled_dot_product_attention(q, k, v, is_causal=True, scale=1.0) + attn = attn.transpose(1, 2).reshape(q.shape[0], q.shape[2], -1) + attn = attn * torch.sigmoid(gate) + attn = attn.to(self.out_proj.weight.dtype) + return self.out_proj(attn) + + +def build_case(name: str) -> tuple[nn.Module, tuple[torch.Tensor, ...]]: + if name == "toy": + model = Toy().eval() + args = ( + torch.randn(2, 4, dtype=torch.float16), + torch.randn(2, 4, dtype=torch.float16), + ) + return model, args + + if name == "attention_block": + model = AttentionBlockToy().eval() + args = ( + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 2, 3, 4, dtype=torch.float16), + torch.randn(1, 3, 8, dtype=torch.float16), + ) + return model, args + + raise ValueError(f"unknown case: {name}") + + +def format_ir_graph(graph: IRGraph) -> str: + lines: list[str] = [] + lines.append("IR Summary") + lines.append(f" inputs={graph.inputs}") + lines.append(f" outputs={graph.outputs}") + lines.append(f" nodes={len(graph.order)}") + if graph.meta: + lines.append(f" meta={graph.meta}") + lines.append("") + lines.append("IR Nodes") + + for index, node_id in enumerate(graph.order): + node = graph.nodes[node_id] + lines.append(f"[{index}] {node.id} op={node.op}") + lines.append(f" inputs={node.inputs}") + lines.append(f" outputs={node.outputs}") + if node.attrs: + lines.append(f" attrs={node.attrs}") + if node.kind != "generic": + lines.append(f" kind={node.kind}") + if node.meta: + lines.append(f" meta={node.meta}") + for output_id in node.outputs: + value = graph.values.get(output_id) + if value is None: + continue + lines.append( + f" value[{output_id}] shape={value.shape} dtype={value.dtype} users={value.users}" + ) + + if graph.constants: + lines.append("") + lines.append("IR Constants") + for value_id, value in graph.constants.items(): + value_meta = graph.values.get(value_id) + shape = None if value_meta is None else value_meta.shape + dtype = None if value_meta is None else value_meta.dtype + typename = type(value).__name__ + lines.append(f"- {value_id}: type={typename} shape={shape} dtype={dtype}") + + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Capture a small PyTorch model, dump the exported graph, and print IR before/after cleanup.", + ) + parser.add_argument( + "--case", + choices=("toy", "attention_block"), + default="toy", + help="Which built-in model to inspect.", + ) + parser.add_argument( + "--no-meta", + action="store_true", + help="Hide exported graph metadata to keep the dump shorter.", + ) + args = parser.parse_args() + + model, example_args = build_case(args.case) + captured = capture_model(model, example_args) + + print(f"Case: {args.case}") + print(f"Model: {type(model).__name__}") + print("") + print("=== Exported Graph ===") + print(dump_graph(captured, include_meta=not args.no_meta)) + print("") + print("=== IR Before Cleanup ===") + print(format_ir_graph(captured.ir_graph)) + print("") + + canonicalize_exported_graph(captured.ir_graph) + + print("=== IR After Cleanup ===") + print(format_ir_graph(captured.ir_graph)) + + +if __name__ == "__main__": + main() diff --git a/python/examples/test_torch_capture.py b/python/examples/test_torch_capture.py new file mode 100644 index 000000000..c2ff748b4 --- /dev/null +++ b/python/examples/test_torch_capture.py @@ -0,0 +1,70 @@ +import torch +import torch.nn as nn + +from src.transpile.capture_pytorch import ( + capture_model_with_fallback, + dump_graph, + get_dtype, + get_shape, +) +from src.transpile.lower import transpile_captured + + +class TinyBlock(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(8, 16) + self.fc2 = nn.Linear(16, 4) + + def forward(self, x): + h = self.fc1(x) + h = torch.relu(h) + y = self.fc2(h) + return y + + +def main(): + model = TinyBlock().eval() + x = torch.randn(2, 8) + + captured = capture_model_with_fallback(model, args=(x,)) + + print("strict:", captured.strict) + print("graph_module type:", type(captured.graph_module).__name__) + print("state_dict keys:", list(captured.state_dict.keys())) + print() + + print("=== FX Graph ===") + print(captured.graph) + print() + + print("=== Node Walk ===") + for i, node in enumerate(captured.graph.nodes): + print(f"[{i}] op={node.op} name={node.name} target={node.target}") + print(f" args={node.args}") + print(f" kwargs={node.kwargs}") + print(f" shape={get_shape(node)} dtype={get_dtype(node)}") + print() + + print("=== Full Dump ===") + print(dump_graph(captured)) + print() + + print("=== Lower To Cactus ===") + try: + transpiled = transpile_captured(captured) + print("lowering succeeded") + print("lowered graph type:", type(transpiled.graph).__name__) + print("runtime inputs:", transpiled.runtime_inputs) + print("bound constants:", transpiled.bound_constants) + print("outputs:", transpiled.outputs) + print() + print("binding runtime input 0 from example tensor") + transpiled.set_input(0, x) + except NotImplementedError as exc: + print("lowering stopped on unimplemented case:") + print(f" {exc}") + + +if __name__ == "__main__": + main() diff --git a/python/examples/transpile_hf_causal_lm.py b/python/examples/transpile_hf_causal_lm.py new file mode 100644 index 000000000..05e70b8c9 --- /dev/null +++ b/python/examples/transpile_hf_causal_lm.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import runpy +import sys +from pathlib import Path + + +def main() -> int: + if "--task" not in sys.argv[1:]: + sys.argv[1:1] = ["--task", "causal_lm_logits"] + script_path = Path(__file__).with_name("transpile_hf_model.py") + runpy.run_path(str(script_path), run_name="__main__") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/transpile_hf_model.py b/python/examples/transpile_hf_model.py new file mode 100644 index 000000000..fc3664671 --- /dev/null +++ b/python/examples/transpile_hf_model.py @@ -0,0 +1,2832 @@ +from __future__ import annotations + +import argparse +import builtins +import copy +import gc +import importlib.util +import json +import os +import re +import sys +from collections import Counter +from dataclasses import dataclass +from dataclasses import fields +from functools import lru_cache +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.graph import Graph +from src.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features +from src.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform +from src.transpile.audio_preprocess import prepare_cactus_audio_features +from src.transpile.audio_preprocess import prepare_native_gemma4_audio_features +from src.transpile.capture_pytorch import capture_model +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.component_partition import extract_component_subgraphs +from src.transpile.component_partition import summarize_ir_components +from src.transpile.component_pipeline import capture_component_spec +from src.transpile.component_pipeline import execute_component_pipeline +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import verify_ir +from src.transpile.lower import TranspiledGraph +from src.transpile.lower import _lower_constant_value +from src.transpile.lower import _lower_input_value +from src.transpile.lower import _lower_ir_node +from src.transpile.lower import _lookup_weight_binding +from src.transpile.model_adapters import build_component_module_specs +from src.transpile.model_adapters import canonicalize_model_interface +from src.transpile.optimize_graph import FusionConfig +from src.transpile.optimize_graph import optimize_graph +from src.transpile.parakeet_tdt_local import greedy_decode_parakeet_tdt_token_ids +from src.transpile.parakeet_tdt_local import load_parakeet_tdt_local_model +from src.transpile.parakeet_tdt_local import prepare_parakeet_tdt_audio_features + +_TORCHVISION_COMPAT_LIBRARIES: list[object] = [] + + +@dataclass +class PreparedInputs: + names: tuple[str, ...] + tensors: tuple[torch.Tensor, ...] + metadata: dict[str, object] + + +def _transformers_supports_model_module(module_name: str) -> bool: + try: + return importlib.util.find_spec(module_name) is not None + except Exception: + return False + + +def _candidate_external_site_packages() -> list[Path]: + candidates: list[Path] = [] + major_minor = f"python{sys.version_info.major}.{sys.version_info.minor}" + pyenv_versions = Path.home() / ".pyenv" / "versions" + if pyenv_versions.exists(): + for version_dir in sorted(pyenv_versions.iterdir(), reverse=True): + site_packages = version_dir / "lib" / major_minor / "site-packages" + if site_packages.exists(): + candidates.append(site_packages) + return candidates + + +def _ensure_transformers_supports_model_type(model_type: str) -> str | None: + normalized = str(model_type or "").strip().lower() + target_module = { + "gemma4": "transformers.models.gemma4.modeling_gemma4", + }.get(normalized) + if not target_module: + return None + if _transformers_supports_model_module(target_module): + return None + + for site_packages in _candidate_external_site_packages(): + candidate = site_packages / Path(*target_module.split(".")) + if not candidate.with_suffix(".py").exists() and not candidate.is_dir(): + continue + sys.path.insert(0, str(site_packages)) + for module_name in list(sys.modules): + root_name = module_name.split(".", 1)[0] + if root_name in {"transformers", "huggingface_hub", "tokenizers"}: + del sys.modules[module_name] + if _transformers_supports_model_module(target_module): + return str(site_packages) + try: + sys.path.remove(str(site_packages)) + except ValueError: + pass + return None + + +def _resolve_local_snapshot(model_id_or_path: str) -> str | None: + explicit = Path(model_id_or_path) + if explicit.exists(): + return str(explicit) + + snapshots_dir = ( + Path.home() + / ".cache" + / "huggingface" + / "hub" + / ("models--" + model_id_or_path.replace("/", "--")) + / "snapshots" + ) + if not snapshots_dir.exists(): + return None + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + return None + return str(snapshots[-1]) + + +def _snapshot_has_model_weights(path: str | Path) -> bool: + root = Path(path) + if not root.exists() or not root.is_dir(): + return False + candidates = ( + "model.safetensors", + "model.safetensors.index.json", + "pytorch_model.bin", + "pytorch_model.bin.index.json", + ) + return any((root / name).exists() for name in candidates) + + +def _validate_weights_dir(weights_dir: str | None, *, model_id: str) -> Path | None: + if not weights_dir: + return None + + root = Path(weights_dir).resolve() + if not root.exists(): + raise RuntimeError( + f"weights_dir does not exist: {root}\n" + "\n" + f"Create the folder first with:\n" + f" cactus convert {model_id} {root}\n" + ) + + manifest_path = root / "weights_manifest.json" + if not manifest_path.exists(): + raise RuntimeError( + f"weights_dir is missing weights_manifest.json: {manifest_path}\n" + "\n" + f"Re-convert with the current converter:\n" + f" cactus convert {model_id} {root}\n" + ) + + return root + + +def _serialize_json_compatible(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _serialize_json_compatible(inner) for key, inner in value.items()} + if isinstance(value, (list, tuple)): + return [_serialize_json_compatible(inner) for inner in value] + if isinstance(value, Path): + return str(value) + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.dtype): + return str(value) + if isinstance(value, torch.Tensor): + return { + "type": "torch.Tensor", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if isinstance(value, np.ndarray): + return { + "type": "numpy.ndarray", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if hasattr(value, "__dataclass_fields__"): + return { + field.name: _serialize_json_compatible(getattr(value, field.name)) + for field in fields(value) + } + try: + return repr(value) + except Exception: + return f"<{type(value).__module__}.{type(value).__name__}>" + + +def _slugify_model_artifact_name(value: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("._-") or "model" + + +def _default_artifact_dir_for_model(model_id: str) -> Path: + return (Path.cwd() / "transpiled" / _slugify_model_artifact_name(model_id)).resolve() + + +def _graph_to_dict(graph) -> dict[str, object]: + return { + "meta": _serialize_json_compatible(graph.meta), + "inputs": list(graph.inputs), + "outputs": list(graph.outputs), + "constants": { + value_id: _serialize_json_compatible(constant) + for value_id, constant in graph.constants.items() + }, + "values": { + value_id: _serialize_json_compatible(value) + for value_id, value in graph.values.items() + }, + "nodes": [ + _serialize_json_compatible(graph.nodes[node_id]) + for node_id in graph.order + ], + } + + +def _write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def _component_artifact_name(prefix: str, component: str) -> str: + return f"{prefix}_{component}.json" + + +def _component_graphs_to_payload(component_graphs: dict[str, IRGraph]) -> dict[str, object]: + return { + component: _graph_to_dict(graph) + for component, graph in component_graphs.items() + } + + +def _binding_entries_by_node_id( + bindings: list[dict[str, object]], +) -> dict[int, dict[str, object]]: + result: dict[int, dict[str, object]] = {} + for binding in bindings: + try: + result[int(binding["node_id"])] = binding + except Exception: + continue + return result + + +def _write_graph_binding_manifest( + *, + artifact_dir: Path, + filename: str, + model_id: str, + model_source: str, + task: str, + family: str, + inputs_metadata: dict[str, object], + transpiled_graph: TranspiledGraph, +) -> Path: + manifest_path = artifact_dir / filename + _write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(inputs_metadata), + "runtime_input_node_ids": [int(tensor.id) for tensor in transpiled_graph.runtime_inputs], + "output_node_ids": [int(tensor.id) for tensor in transpiled_graph.outputs], + "bound_constant_bindings": _serialize_json_compatible(transpiled_graph.bound_constant_bindings), + }, + ) + return manifest_path + + +def _write_component_bundle( + *, + artifact_dir: Path, + model_id: str, + model_source: str, + task: str, + family: str, + inputs_metadata: dict[str, object], + raw_component_graphs: dict[str, IRGraph], + optimized_component_graphs: dict[str, IRGraph], + transpiled_component_graphs: dict[str, TranspiledGraph] | None = None, + component_io_signatures: dict[str, dict[str, tuple[str, ...]]] | None = None, + graph_filename: str = "graph.cactus", +) -> Path: + bundle_dir = artifact_dir / "components" + component_order = [ + component + for component in ( + "audio_encoder", + "vision_encoder", + "lm_encoder", + "decoder", + "unspecified", + ) + if component in raw_component_graphs or component in optimized_component_graphs + ] + extra_components = sorted( + component + for component in set(raw_component_graphs) | set(optimized_component_graphs) + if component not in component_order + ) + component_order.extend(extra_components) + + manifest_components: list[dict[str, object]] = [] + for component in component_order: + raw_graph = raw_component_graphs.get(component) + optimized_graph = optimized_component_graphs.get(component) + transpiled_graph = None if transpiled_component_graphs is None else transpiled_component_graphs.get(component) + component_dir = bundle_dir / component + raw_relpath = None + optimized_relpath = None + graph_relpath = None + if raw_graph is not None: + raw_relpath = Path(component) / "raw_ir.json" + _write_json( + bundle_dir / raw_relpath, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(inputs_metadata), + "graph": _graph_to_dict(raw_graph), + }, + ) + if optimized_graph is not None: + optimized_relpath = Path(component) / "optimized_ir.json" + _write_json( + bundle_dir / optimized_relpath, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(inputs_metadata), + "graph": _graph_to_dict(optimized_graph), + }, + ) + if transpiled_graph is not None: + graph_relpath = Path(component) / graph_filename + component_dir.mkdir(parents=True, exist_ok=True) + transpiled_graph.graph.save(bundle_dir / graph_relpath) + + materialized_constant_bindings: list[dict[str, object]] = [] + if transpiled_graph is not None: + existing_bindings = list(_serialize_json_compatible(transpiled_graph.bound_constant_bindings)) + existing_by_node_id = _binding_entries_by_node_id(existing_bindings) + constant_dir = component_dir / "bound_constants" + for constant_tensor in transpiled_graph.bound_constants: + if int(constant_tensor.id) in existing_by_node_id: + continue + constant_dir.mkdir(parents=True, exist_ok=True) + constant_filename = f"node_{int(constant_tensor.id)}.npy" + constant_relpath = Path(component) / "bound_constants" / constant_filename + constant_array = constant_tensor.numpy().copy() + np.save(bundle_dir / constant_relpath, constant_array, allow_pickle=False) + materialized_constant_bindings.append( + { + "node_id": int(constant_tensor.id), + "value_id": f"materialized_constant_{int(constant_tensor.id)}", + "path": str((bundle_dir / constant_relpath).relative_to(artifact_dir)), + "kind": "saved_constant", + "source_name": f"materialized_constant_{int(constant_tensor.id)}", + "format": "npy", + "precision": int(constant_tensor.dtype), + } + ) + + graph_for_signature = optimized_graph or raw_graph + if graph_for_signature is None: + continue + manifest_components.append( + { + "component": component, + "directory": str(component_dir.relative_to(artifact_dir)), + "raw_ir": None if raw_relpath is None else str((bundle_dir / raw_relpath).relative_to(artifact_dir)), + "optimized_ir": None if optimized_relpath is None else str((bundle_dir / optimized_relpath).relative_to(artifact_dir)), + "graph": None if graph_relpath is None else str((bundle_dir / graph_relpath).relative_to(artifact_dir)), + "inputs": list(graph_for_signature.inputs), + "outputs": list(graph_for_signature.outputs), + "logical_inputs": list((component_io_signatures or {}).get(component, {}).get("input_keys", ())), + "logical_outputs": list((component_io_signatures or {}).get(component, {}).get("output_keys", ())), + "node_count": len(graph_for_signature.order), + "weight_binding_count": _count_weight_bindings(graph_for_signature), + "runtime_input_node_ids": [] if transpiled_graph is None else [int(tensor.id) for tensor in transpiled_graph.runtime_inputs], + "output_node_ids": [] if transpiled_graph is None else [int(tensor.id) for tensor in transpiled_graph.outputs], + "bound_constant_bindings": [] if transpiled_graph is None else ( + list(_serialize_json_compatible(transpiled_graph.bound_constant_bindings)) + + materialized_constant_bindings + ), + } + ) + + manifest_path = bundle_dir / "manifest.json" + _write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component_order": component_order, + "inputs": _serialize_json_compatible(inputs_metadata), + "components": manifest_components, + }, + ) + return manifest_path + + +def _named_tensor_store(prepared: PreparedInputs) -> dict[str, torch.Tensor]: + return { + name: tensor + for name, tensor in zip(prepared.names, prepared.tensors, strict=True) + } + + +def _aggregate_component_counts(component_graphs: dict[str, IRGraph]) -> dict[str, int]: + return { + component: len(graph.order) + for component, graph in sorted(component_graphs.items()) + } + + +def _aggregate_component_op_counts(component_graphs: dict[str, IRGraph]) -> Counter: + counter: Counter[str] = Counter() + for graph in component_graphs.values(): + counter.update(graph.nodes[node_id].op for node_id in graph.order) + return counter + + +def _count_component_weight_bindings(component_graphs: dict[str, IRGraph]) -> int: + return sum(_count_weight_bindings(graph) for graph in component_graphs.values()) + + +def _execute_gemma4_component_pipeline( + *, + component_graphs, + prepared: PreparedInputs, +) -> np.ndarray: + store, _ = execute_component_pipeline( + [component_graphs[name] for name in ("vision_encoder", "audio_encoder", "lm_encoder", "decoder")], + initial_store=_named_tensor_store(prepared), + ) + logits = store["logits"] + return np.asarray(logits, dtype=np.float32) + + +def _run_parakeet_tdt_component_decode( + *, + component_graphs, + model, + prepared: PreparedInputs, +) -> dict[str, object]: + store, _ = execute_component_pipeline( + [component_graphs["audio_encoder"]], + initial_store=_named_tensor_store(prepared), + ) + encoder_hidden_states = np.asarray(store["encoder_hidden_states"]) + batch_size = int(encoder_hidden_states.shape[0]) + if batch_size != 1: + raise ValueError("Parakeet TDT component decode currently expects batch size 1") + + hidden_dtype = prepared.tensors[0].dtype + initial_states = model.initial_decoder_state( + batch_size=batch_size, + device=torch.device("cpu"), + dtype=hidden_dtype, + ) + initial_state_arrays = tuple(state.detach().cpu().numpy() for state in initial_states) + decoder_component = component_graphs["decoder"] + + def _step( + frame: np.ndarray, + token_id: int, + state_values: tuple[np.ndarray, ...], + ) -> tuple[np.ndarray, tuple[np.ndarray, ...]]: + input_store: dict[str, object] = { + "encoder_frame": np.ascontiguousarray(frame), + "token_ids": np.full((batch_size,), token_id, dtype=np.int64), + } + for index in range(model.config.predictor_num_layers): + input_store[f"state_h_{index}"] = np.ascontiguousarray(state_values[index * 2]) + input_store[f"state_c_{index}"] = np.ascontiguousarray(state_values[index * 2 + 1]) + + runtime_inputs = [input_store[key] for key in decoder_component.input_keys] + decoder_component.transpiled_graph.set_inputs(runtime_inputs) + outputs = decoder_component.transpiled_graph.execute() + logits = outputs[0].numpy().astype(np.float32, copy=False) + next_states = tuple(output.numpy() for output in outputs[1:]) + return logits, next_states + + emitted = greedy_decode_parakeet_tdt_token_ids( + config=model.config, + encoder_hidden_states=encoder_hidden_states, + initial_states=initial_state_arrays, + step=_step, + ) + + return { + "token_ids": emitted, + "transcript": model.decode_token_ids(emitted), + "encoder_hidden_shape": list(encoder_hidden_states.shape), + } + + +def _run_component_pipeline_transpile( + *, + args, + task: str, + family: str, + model_source: str, + model, + prepared: PreparedInputs, + component_specs, + fusion_config: FusionConfig, + weights_dir: str | None, + artifact_dir: Path | None, + processor_or_tokenizer, + canonical, +) -> int: + print(f"model_id={args.model_id}") + print(f"model_source={model_source}") + print(f"task={task}") + print(f"adapter_family={family}") + print("adapter_module=component_pipeline") + print(f"input_names={','.join(prepared.names)}") + for name, tensor in zip(prepared.names, prepared.tensors, strict=True): + print(f"input_{name}_shape={list(tensor.shape)}") + if weights_dir: + print(f"weights_dir={weights_dir}") + + print("capture_begin=true", flush=True) + captured_components = {} + for spec in component_specs: + print(f"capture_component_begin={spec.component}", flush=True) + captured_components[spec.component] = capture_component_spec(spec, fusion_config=fusion_config) + print(f"capture_component_done={spec.component}", flush=True) + print("capture_done=true", flush=True) + + raw_component_graphs = { + name: captured.raw_ir_graph + for name, captured in captured_components.items() + } + optimized_component_graphs = { + name: captured.optimized_ir_graph + for name, captured in captured_components.items() + } + transpiled_component_graphs = { + name: captured.transpiled_graph + for name, captured in captured_components.items() + } + component_io_signatures = { + name: { + "input_keys": tuple(captured.input_keys), + "output_keys": tuple(captured.output_keys), + } + for name, captured in captured_components.items() + } + + raw_ir_nodes = sum(len(graph.order) for graph in raw_component_graphs.values()) + optimized_ir_nodes = sum(len(graph.order) for graph in optimized_component_graphs.values()) + binding_count = _count_component_weight_bindings(optimized_component_graphs) + raw_component_counts = _aggregate_component_counts(raw_component_graphs) + optimized_component_counts = _aggregate_component_counts(optimized_component_graphs) + op_counts = _aggregate_component_op_counts(optimized_component_graphs) + + print(f"raw_ir_nodes={raw_ir_nodes}") + print(f"optimized_ir_nodes={optimized_ir_nodes}") + print(f"weight_bindings={binding_count}") + if raw_component_counts: + print( + "raw_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(raw_component_counts.items())) + ) + if optimized_component_counts: + print( + "optimized_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(optimized_component_counts.items())) + ) + print( + "ops=" + f"attention:{op_counts.get('attention', 0)} " + f"conv1d:{op_counts.get('conv1d', 0)} " + f"conv2d:{op_counts.get('conv2d', 0)} " + f"batch_norm:{op_counts.get('batch_norm', 0)} " + f"layer_norm:{op_counts.get('layer_norm', 0)} " + f"rms_norm:{op_counts.get('rms_norm', 0)} " + f"rope:{op_counts.get('rope', 0)} " + f"linear:{op_counts.get('linear', 0)}" + ) + + if weights_dir and binding_count == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir}\n" + "\n" + "The weights folder exists, but none of the component graphs matched entries in weights_manifest.json.\n" + "\n" + f"Recommended fix:\n" + f" cactus convert {args.model_id} {weights_dir}\n" + ) + + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "component_order": [spec.component for spec in component_specs], + "components": _component_graphs_to_payload(raw_component_graphs), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "component_order": [spec.component for spec in component_specs], + "components": _component_graphs_to_payload(optimized_component_graphs), + }, + ) + for component, component_graph in raw_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("raw_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + for component, component_graph in optimized_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("optimized_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + for component in raw_component_graphs: + print(f"saved_raw_component_ir_{component}={artifact_dir / _component_artifact_name('raw_ir', component)}") + for component in optimized_component_graphs: + print( + f"saved_optimized_component_ir_{component}=" + f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" + ) + component_manifest_path = _write_component_bundle( + artifact_dir=artifact_dir, + model_id=args.model_id, + model_source=model_source, + task=task, + family=family, + inputs_metadata=prepared.metadata, + raw_component_graphs=raw_component_graphs, + optimized_component_graphs=optimized_component_graphs, + transpiled_component_graphs=transpiled_component_graphs, + component_io_signatures=component_io_signatures, + graph_filename=args.graph_filename, + ) + print(f"saved_component_bundle_manifest={component_manifest_path}") + for component in transpiled_component_graphs: + print( + f"saved_component_graph_{component}=" + f"{artifact_dir / 'components' / component / args.graph_filename}" + ) + print("note=saved split component graphs; use the component bundle manifest to rebind mmap weights/embeddings when loading") + + if args.skip_execute: + return 0 + + if task == "multimodal_causal_lm_logits": + print("execute_begin=true") + transpiled_output = _execute_gemma4_component_pipeline( + component_graphs=captured_components, + prepared=prepared, + ) + print("execute_done=true") + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + if args.skip_reference_compare: + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(transpiled_output.shape), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "reference_compare_skipped": True, + } + print(f"output_shape={list(transpiled_output.shape)}") + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"transpiled_next_token_id={transpiled_next}") + result_payload["transpiled_next_token_id"] = transpiled_next + if hasattr(tokenizer_like, "decode"): + transpiled_token = tokenizer_like.decode([transpiled_next]) + print(f"transpiled_next_token={transpiled_token!r}") + result_payload["transpiled_next_token"] = transpiled_token + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + print("reference_begin=true") + with torch.no_grad(): + reference_output = canonical.module(*prepared.tensors).detach().float().cpu().numpy() + print("reference_done=true") + max_abs_diff = float(np.max(np.abs(reference_output - transpiled_output))) + mean_abs_diff = float(np.mean(np.abs(reference_output - transpiled_output))) + hf_next = int(np.argmax(reference_output[0, -1])) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"hf_next_token_id={hf_next}") + print(f"transpiled_next_token_id={transpiled_next}") + print(f"logits_max_abs_diff={max_abs_diff:.6f}") + print(f"logits_mean_abs_diff={mean_abs_diff:.6f}") + if hasattr(tokenizer_like, "decode"): + print(f"hf_next_token={tokenizer_like.decode([hf_next])!r}") + print(f"transpiled_next_token={tokenizer_like.decode([transpiled_next])!r}") + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(reference_output.shape), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "max_abs_diff": max_abs_diff, + "mean_abs_diff": mean_abs_diff, + "hf_next_token_id": hf_next, + "transpiled_next_token_id": transpiled_next, + } + if hasattr(tokenizer_like, "decode"): + result_payload["hf_next_token"] = tokenizer_like.decode([hf_next]) + result_payload["transpiled_next_token"] = tokenizer_like.decode([transpiled_next]) + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + if task == "tdt_transcription": + print("execute_begin=true") + transpiled_decode = _run_parakeet_tdt_component_decode( + component_graphs=captured_components, + model=model, + prepared=prepared, + ) + print("execute_done=true") + print(f"transpiled_transcript={transpiled_decode['transcript']!r}") + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": family, + "inputs": _serialize_json_compatible(prepared.metadata), + "raw_ir_nodes": raw_ir_nodes, + "optimized_ir_nodes": optimized_ir_nodes, + "weight_bindings": binding_count, + "transpiled_token_ids": transpiled_decode["token_ids"], + "transpiled_transcript": transpiled_decode["transcript"], + "encoder_hidden_shape": transpiled_decode["encoder_hidden_shape"], + } + if args.skip_reference_compare: + result_payload["reference_compare_skipped"] = True + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + print("reference_begin=true") + reference_token_ids = model.greedy_decode_token_ids(prepared.tensors[0]) + reference_transcript = model.decode_token_ids(reference_token_ids) + print("reference_done=true") + print(f"reference_transcript={reference_transcript!r}") + print(f"transcript_match={reference_transcript == transpiled_decode['transcript']}") + result_payload["reference_token_ids"] = reference_token_ids + result_payload["reference_transcript"] = reference_transcript + result_payload["transcript_match"] = bool(reference_transcript == transpiled_decode["transcript"]) + result_payload["token_id_match"] = bool(reference_token_ids == transpiled_decode["token_ids"]) + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + return 0 + + raise NotImplementedError(f"component pipeline execution is not implemented for task={task}") + + +def _parse_dtype(name: str) -> torch.dtype: + normalized = name.strip().lower() + mapping = { + "float16": torch.float16, + "fp16": torch.float16, + "float32": torch.float32, + "fp32": torch.float32, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + } + if normalized not in mapping: + raise ValueError(f"unsupported torch dtype: {name}") + return mapping[normalized] + + +class TranspileWrapper(torch.nn.Module): + def __init__(self, adapter_module: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__() + self.adapter = adapter_module + self.weights_dir = weights_dir + + def forward(self, *bound_inputs: torch.Tensor) -> torch.Tensor: + return self.adapter(*bound_inputs) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.adapter, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta: dict[str, object] = {} + base_graph = metadata.get("graph", {}) + if isinstance(base_graph, dict): + graph_meta.update(base_graph) + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir + metadata["graph"] = graph_meta + return metadata + + +def _patch_transformers_torchvision_probe() -> str | None: + has_torchvision = importlib.util.find_spec("torchvision") is not None + has_lzma = importlib.util.find_spec("_lzma") is not None + + if not has_torchvision: + return None + + if has_lzma: + nms_patch_note = _patch_torchvision_missing_nms_op() + return nms_patch_note + + base_note: str | None = None + try: + import backports.lzma as backports_lzma # type: ignore + + sys.modules.setdefault("lzma", backports_lzma) + base_note = "using backports.lzma because this Python build is missing _lzma" + nms_patch_note = _patch_torchvision_missing_nms_op() + if nms_patch_note: + return f"{base_note}; {nms_patch_note}" + return base_note + except Exception: + pass + + class _InterpolationModeStub: + NEAREST = "nearest" + BILINEAR = "bilinear" + BICUBIC = "bicubic" + LANCZOS = "lanczos" + + class _TorchvisionFunctionalStub: + InterpolationMode = _InterpolationModeStub + + def __getattr__(self, name: str): + raise RuntimeError( + "torchvision functionality is unavailable because this Python build is missing _lzma; " + f"attempted to access torchvision.transforms.functional.{name}" + ) + + # Some Transformers multimodal image/video modules reference `F.InterpolationMode` + # or `tvF.InterpolationMode` in annotations even when torchvision probing is disabled. + # Installing a tiny builtins-level stub lets those modules import so the PIL fallback + # classes can be selected instead. + builtins.F = _TorchvisionFunctionalStub() + builtins.tvF = builtins.F + + import transformers.utils as tf_utils # type: ignore + import transformers.utils.import_utils as tf_import_utils # type: ignore + + @lru_cache + def _disabled() -> bool: + return False + + tf_import_utils.is_torchvision_available = _disabled + tf_import_utils.is_torchvision_v2_available = _disabled + tf_utils.is_torchvision_available = _disabled + tf_utils.is_torchvision_v2_available = _disabled + return "disabled torchvision import checks because this Python build is missing _lzma" + + +def _patch_torchvision_missing_nms_op() -> str | None: + """Keep mismatched torch/torchvision installs importable for processors. + + Some local environments have torchvision installed but built without the + custom operator registrations matching the active torch wheel. Transformers + imports torchvision just to access image interpolation enums, and that import + can fail before we ever touch model code because `torchvision::nms` is + missing. Defining the operator schema is enough for torchvision's fake/meta + registration path to complete; the transpiler never calls NMS. + """ + + try: + import torchvision # type: ignore # noqa: F401 + return None + except RuntimeError as exc: + if "operator torchvision::nms does not exist" not in str(exc): + return None + except Exception: + return None + + try: + import torchvision.extension as tv_extension # type: ignore + if bool(getattr(tv_extension, "_HAS_OPS", False)): + return None + except Exception: + pass + + try: + library = torch.library.Library("torchvision", "DEF") + library.define("nms(Tensor dets, Tensor scores, float iou_threshold) -> Tensor") + _TORCHVISION_COMPAT_LIBRARIES.append(library) + return "defined missing torchvision::nms operator for torchvision import compatibility" + except Exception: + return None + + +def _patch_torch_flex_attention_compat() -> str | None: + try: + import torch.nn.attention.flex_attention as flex_attention # type: ignore + except Exception: + return None + + if hasattr(flex_attention, "AuxRequest"): + return None + + class _AuxRequest: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + flex_attention.AuxRequest = _AuxRequest # type: ignore[attr-defined] + return "installed torch flex_attention AuxRequest compatibility stub" + + +def _tie_lfm2_vl_lm_head_if_needed(model: torch.nn.Module) -> str | None: + if str(getattr(getattr(model, "config", None), "model_type", "") or "").lower() != "lfm2_vl": + return None + lm_head = getattr(model, "lm_head", None) + model_root = getattr(model, "model", None) + language_model = getattr(model_root, "language_model", None) + embed_tokens = getattr(language_model, "embed_tokens", None) + if not isinstance(lm_head, torch.nn.Linear) or not isinstance(embed_tokens, torch.nn.Embedding): + return None + if tuple(lm_head.weight.shape) != tuple(embed_tokens.weight.shape): + return None + if lm_head.weight.data_ptr() == embed_tokens.weight.data_ptr(): + return None + lm_head.weight = embed_tokens.weight + return "tied LFM2-VL lm_head.weight to language_model.embed_tokens.weight" + + +def _load_local_torch_state_dict(model_source: str) -> dict[str, torch.Tensor] | None: + root = Path(model_source) + if not root.exists() or not root.is_dir(): + return None + + safetensors_path = root / "model.safetensors" + if safetensors_path.exists(): + try: + from safetensors.torch import load_file # type: ignore + + return load_file(str(safetensors_path)) + except Exception: + return None + + pytorch_path = root / "pytorch_model.bin" + if pytorch_path.exists(): + try: + loaded = torch.load(str(pytorch_path), map_location="cpu", weights_only=True) + except TypeError: + loaded = torch.load(str(pytorch_path), map_location="cpu") + return loaded if isinstance(loaded, dict) else None + + return None + + +def _remap_gemma4_checkpoint_state_dict(state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + remapped: dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + if "audio_tower" in new_key: + new_key = re.sub(r"subsample_conv_projection\.layer(\d+)\.", r"subsample_conv_projection.conv_\1.", new_key) + new_key = new_key.replace("audio_tower.layers.", "audio_tower.conformer.") + new_key = new_key.replace(".feed_forward1.", ".ffw_layer_start.") + new_key = new_key.replace(".feed_forward2.", ".ffw_layer_end.") + new_key = re.sub(r"\.self_attn\.(q_proj|k_proj|v_proj)\.", r".attention.attn.\1.", new_key) + new_key = new_key.replace(".self_attn.per_dim_scale", ".attention.attn.per_dim_scale") + new_key = new_key.replace(".self_attn.relative_k_proj.", ".attention.attn.relative_position_embedding.pos_proj.") + new_key = new_key.replace(".self_attn.post.", ".attention.post.") + new_key = new_key.replace(".norm_pre_attn.", ".attention.pre_attn_norm.") + new_key = new_key.replace(".norm_post_attn.", ".attention.post_norm.") + new_key = new_key.replace(".norm_out.", ".norm.") + new_key = new_key.replace(".linear.weight", ".weight") + remapped[new_key] = value + if new_key.endswith(".attention.attn.per_dim_scale"): + remapped[new_key.replace(".per_dim_scale", ".per_dim_key_scale")] = value + if "lm_head.weight" not in remapped: + tied_embedding = remapped.get("model.language_model.embed_tokens.weight") + if tied_embedding is not None: + remapped["lm_head.weight"] = tied_embedding + return remapped + + +def _repair_gemma4_checkpoint_weights(model: torch.nn.Module, model_source: str) -> dict[str, object]: + raw_state_dict = _load_local_torch_state_dict(model_source) + if raw_state_dict is None: + return {"applied": False, "reason": "no local checkpoint state_dict"} + + remapped_state_dict = _remap_gemma4_checkpoint_state_dict(raw_state_dict) + load_result = model.load_state_dict(remapped_state_dict, strict=False) + return { + "applied": True, + "missing_keys": list(load_result.missing_keys), + "unexpected_keys": list(load_result.unexpected_keys), + } + + +def _load_config_json(model_id_or_path: str) -> dict[str, object]: + local_snapshot = _resolve_local_snapshot(model_id_or_path) + config_source = Path(local_snapshot) / "config.json" if local_snapshot else None + if config_source is not None and config_source.exists(): + return json.loads(config_source.read_text()) + explicit = Path(model_id_or_path) / "config.json" + if explicit.exists(): + return json.loads(explicit.read_text()) + return {} + + +def _load_optional_json(model_id_or_path: str, filename: str) -> dict[str, object]: + local_snapshot = _resolve_local_snapshot(model_id_or_path) + candidate = Path(local_snapshot) / filename if local_snapshot else None + if candidate is not None and candidate.exists(): + return json.loads(candidate.read_text()) + explicit = Path(model_id_or_path) / filename + if explicit.exists(): + return json.loads(explicit.read_text()) + return {} + + +def _load_gemma4_tokenizer_fallback(source_candidates: list[str]) -> object | None: + try: + from transformers import PreTrainedTokenizerFast # type: ignore + except Exception: + return None + + for source in source_candidates: + root = Path(source) + if not root.exists() or not root.is_dir(): + continue + tokenizer_json = root / "tokenizer.json" + tokenizer_config_path = root / "tokenizer_config.json" + if not tokenizer_json.exists() or not tokenizer_config_path.exists(): + continue + + tokenizer_config = json.loads(tokenizer_config_path.read_text()) + tokenizer = PreTrainedTokenizerFast( + tokenizer_file=str(tokenizer_json), + bos_token=tokenizer_config.get("bos_token"), + eos_token=tokenizer_config.get("eos_token"), + unk_token=tokenizer_config.get("unk_token"), + pad_token=tokenizer_config.get("pad_token"), + mask_token=tokenizer_config.get("mask_token"), + padding_side=str(tokenizer_config.get("padding_side", "right")), + additional_special_tokens=list(tokenizer_config.get("extra_special_tokens", []) or []), + ) + model_max_length = tokenizer_config.get("model_max_length") + if isinstance(model_max_length, int): + tokenizer.model_max_length = model_max_length + + chat_template_path = root / "chat_template.jinja" + if chat_template_path.exists(): + tokenizer.chat_template = chat_template_path.read_text() + + for token_attr in ( + "image_token", + "audio_token", + "boi_token", + "eoi_token", + "boa_token", + "eoa_token", + ): + token_value = tokenizer_config.get(token_attr) + if isinstance(token_value, str): + setattr(tokenizer, token_attr, token_value) + setattr(tokenizer, f"{token_attr}_id", tokenizer.convert_tokens_to_ids(token_value)) + return tokenizer + + return None + + +def _load_gemma4_processor_fallback( + *, + source_candidates: list[str], + common_kwargs: dict[str, object], +) -> object | None: + try: + from transformers.models.gemma4.feature_extraction_gemma4 import Gemma4AudioFeatureExtractor # type: ignore + from transformers.models.gemma4.image_processing_gemma4 import Gemma4ImageProcessor # type: ignore + from transformers.models.gemma4.processing_gemma4 import Gemma4Processor # type: ignore + except Exception: + return None + + tokenizer = None + processor_config: dict[str, object] = {} + for source in source_candidates: + try: + try: + from transformers import AutoTokenizer # type: ignore + + tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) + except Exception: + tokenizer = _load_gemma4_tokenizer_fallback([source]) + processor_config = _load_optional_json(source, "processor_config.json") + if tokenizer is not None and processor_config: + break + except Exception: + continue + if tokenizer is None or not processor_config: + return None + + feature_config = dict(processor_config.get("feature_extractor", {}) or {}) + image_config = dict(processor_config.get("image_processor", {}) or {}) + feature_config.pop("feature_extractor_type", None) + image_config.pop("image_processor_type", None) + + feature_extractor = Gemma4AudioFeatureExtractor(**feature_config) + image_processor = Gemma4ImageProcessor(**image_config) + + processor_kwargs = {} + for key in ("image_seq_length", "audio_seq_length", "audio_ms_per_token"): + if key in processor_config: + processor_kwargs[key] = processor_config[key] + + return Gemma4Processor( + feature_extractor=feature_extractor, + image_processor=image_processor, + tokenizer=tokenizer, + **processor_kwargs, + ) + + +def _infer_task_from_config(model_id_or_path: str) -> str: + config = _load_config_json(model_id_or_path) + architectures = [str(value) for value in config.get("architectures", []) if isinstance(value, str)] + model_type = str(config.get("model_type", "") or "").lower() + decoding_cfg = config.get("decoding") + if isinstance(decoding_cfg, dict) and str(decoding_cfg.get("model_type", "") or "").lower() == "tdt": + return "tdt_transcription" + loss_cfg = config.get("loss") + if isinstance(loss_cfg, dict) and str(loss_cfg.get("loss_name", "") or "").lower() == "tdt": + return "tdt_transcription" + + if any("CausalLM" in value for value in architectures): + return "causal_lm_logits" + if any("CTC" in value for value in architectures): + return "ctc_logits" + if model_type == "whisper": + return "encoder_hidden_states" + if any("ConditionalGeneration" in value and "Whisper" in value for value in architectures): + return "encoder_hidden_states" + + lowered_id = model_id_or_path.lower() + if "parakeet-tdt" in lowered_id: + return "tdt_transcription" + if "whisper" in lowered_id: + return "encoder_hidden_states" + if "ctc" in lowered_id: + return "ctc_logits" + if any(token in lowered_id for token in ("qwen", "gemma", "llama", "mistral", "lfm")): + return "causal_lm_logits" + + raise RuntimeError( + f"Could not infer transpile task for {model_id_or_path}.\n" + "\n" + "Pass one explicitly with --task, for example:\n" + " --task causal_lm_logits\n" + " --task tdt_transcription\n" + " --task ctc_logits\n" + " --task encoder_hidden_states\n" + ) + + +def _resolve_audio_sample_rate(processor: object) -> int: + for attr_name in ("feature_extractor", "tokenizer"): + child = getattr(processor, attr_name, None) + sample_rate = getattr(child, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + sample_rate = getattr(processor, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + return 16000 + + +def _infer_fallback_audio_input_names(config: dict[str, object], task: str) -> tuple[str, ...]: + model_type = str(config.get("model_type", "") or "").lower() + if model_type == "whisper": + return ("input_features",) + if task == "encoder_hidden_states": + return ("input_features",) + if task == "tdt_transcription": + return ("input_features",) + audio_cfg = config.get("audio_config") + if isinstance(audio_cfg, dict) and any(key in audio_cfg for key in ("features", "input_feat_size", "num_mel_bins")): + return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) + encoder_cfg = config.get("encoder") + if isinstance(encoder_cfg, dict) and any(key in encoder_cfg for key in ("feat_in", "num_mel_bins")): + return ("input_features", "attention_mask") if task == "ctc_logits" else ("input_features",) + return ("input_values", "attention_mask") if task == "ctc_logits" else ("input_values",) + + +def _resolve_encoder_module(model: torch.nn.Module) -> torch.nn.Module | None: + get_encoder = getattr(model, "get_encoder", None) + if callable(get_encoder): + encoder = get_encoder() + if isinstance(encoder, torch.nn.Module): + return encoder + encoder = getattr(model, "encoder", None) + if isinstance(encoder, torch.nn.Module): + return encoder + model_attr = getattr(model, "model", None) + if model_attr is not None: + encoder = getattr(model_attr, "encoder", None) + if isinstance(encoder, torch.nn.Module): + return encoder + return None + + +def _infer_expected_input_feature_frames(model: torch.nn.Module) -> int | None: + config = getattr(model, "config", None) + max_source_positions = getattr(config, "max_source_positions", None) + if not isinstance(max_source_positions, int) or max_source_positions <= 0: + return None + + encoder = _resolve_encoder_module(model) + if encoder is None: + return None + + stride_product = 1 + found_conv = False + for child in encoder.children(): + if isinstance(child, torch.nn.Conv1d): + stride = child.stride[0] if isinstance(child.stride, tuple) else child.stride + stride_product *= int(stride) + found_conv = True + + if not found_conv: + return None + return int(max_source_positions) * stride_product + + +def _prepare_fallback_audio_inputs( + *, + input_names: tuple[str, ...], + config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + task: str, + audio_file: str, + torch_dtype: torch.dtype, +) -> PreparedInputs: + if not input_names: + input_names = _infer_fallback_audio_input_names(config, task) + preprocessor_root = config.get("preprocessor") + preprocessor_root = preprocessor_root if isinstance(preprocessor_root, dict) else {} + encoder_root = config.get("encoder") + encoder_root = encoder_root if isinstance(encoder_root, dict) else {} + + sample_rate = int( + preprocessor_config.get( + "sampling_rate", + preprocessor_root.get("sample_rate", config.get("sampling_rate", 16000)), + ) + or 16000 + ) + num_mels = int( + preprocessor_config.get( + "feature_size", + preprocessor_root.get( + "features", + encoder_root.get("feat_in", config.get("num_mel_bins", config.get("feature_size", 80))), + ), + ) + or 80 + ) + encoder_cfg = config.get("encoder_config") + if isinstance(encoder_cfg, dict): + num_mels = int(encoder_cfg.get("num_mel_bins", encoder_cfg.get("feat_in", num_mels)) or num_mels) + raw_hop_length = preprocessor_config.get( + "hop_length", + preprocessor_root.get("window_stride", config.get("hop_length", 160)), + ) + if raw_hop_length is None: + raw_hop_length = 160 + if isinstance(raw_hop_length, float) and raw_hop_length > 0.0 and raw_hop_length < 1.0: + hop_length = max(1, int(round(float(sample_rate) * raw_hop_length))) + else: + hop_length = int(raw_hop_length) + + n_fft = int(preprocessor_config.get("n_fft", config.get("n_fft", preprocessor_root.get("n_fft", 400))) or 400) + raw_frame_length = preprocessor_config.get( + "win_length", + preprocessor_config.get( + "frame_length", + preprocessor_root.get("window_size", preprocessor_root.get("n_window_size", config.get("frame_length", n_fft))), + ), + ) + if raw_frame_length is None: + raw_frame_length = n_fft + if isinstance(raw_frame_length, float) and raw_frame_length > 0.0 and raw_frame_length < 1.0: + frame_length = max(1, int(round(float(sample_rate) * raw_frame_length))) + else: + frame_length = int(raw_frame_length) + preemphasis = preprocessor_config.get("preemphasis", preprocessor_root.get("preemph")) + if preemphasis is not None: + preemphasis = float(preemphasis) + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + + tensors: list[torch.Tensor] = [] + if input_names and input_names[0] == "input_features": + model_type = str(config.get("model_type", "") or "").lower() + is_parakeet_like = ( + task == "tdt_transcription" + or "parakeet" in model_type + or "parakeet" in type(model).__module__.lower() + ) + if is_parakeet_like: + expected_frames = _infer_expected_input_feature_frames(model) + parakeet_features, _ = prepare_parakeet_tdt_audio_features( + audio_file, + expected_frames=expected_frames, + expected_mels=num_mels, + torch_dtype=torch_dtype, + ) + tensors.append(parakeet_features) + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "native_parakeet_audio_preprocessor": True, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + is_whisper_like = ( + str(config.get("model_type", "") or "").lower() == "whisper" + or "transformers.models.whisper." in type(model).__module__ + ) + if is_whisper_like: + expected_frames = _infer_expected_input_feature_frames(model) or 3000 + try: + whisper_features, active_frames = prepare_cactus_audio_features( + audio_file, + model_type="whisper", + expected_frames=expected_frames, + expected_mels=num_mels, + torch_dtype=torch_dtype, + layout="mels_frames", + ) + tensors.append(whisper_features) + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "native_cactus_audio_preprocessor": True, + "active_feature_frames": active_frames, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + except Exception: + pass + features, feature_length = _generic_log_mel_features( + waveform, + sample_rate=sample_rate, + num_mels=num_mels, + n_fft=n_fft, + hop_length=hop_length, + frame_length=frame_length, + preemphasis=preemphasis, + ) + expected_frames = _infer_expected_input_feature_frames(model) + attention_mask: np.ndarray | None = None + target_frames = feature_length + if isinstance(expected_frames, int) and expected_frames > 0: + target_frames = expected_frames + + active_frames = min(feature_length, target_frames) + features = features[:active_frames, :] + if target_frames > active_frames: + pad_width = target_frames - active_frames + features = np.pad(features, ((0, pad_width), (0, 0)), mode="constant") + attention_mask = np.zeros((target_frames,), dtype=np.bool_) + attention_mask[:active_frames] = True + + if is_whisper_like: + features = np.ascontiguousarray(features.T) + + tensors.append(torch.from_numpy(features).unsqueeze(0).to(dtype=torch_dtype)) + if attention_mask is not None and len(input_names) > 1 and input_names[1] == "attention_mask": + tensors.append(torch.from_numpy(attention_mask).unsqueeze(0)) + else: + input_values = torch.from_numpy(waveform).unsqueeze(0).to(dtype=torch_dtype) + tensors.append(input_values) + if len(input_names) > 1 and input_names[1] == "attention_mask": + tensors.append(torch.ones_like(input_values, dtype=torch.float32)) + + return PreparedInputs( + names=input_names[: len(tensors)], + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": True, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(input_names, tensors) + }, + }, + ) + + +def _prepare_audio_inputs( + processor: object | None, + *, + input_names: tuple[str, ...], + config: dict[str, object], + preprocessor_config: dict[str, object], + model: torch.nn.Module, + task: str, + audio_file: str, + torch_dtype: torch.dtype, +) -> PreparedInputs: + if processor is None: + return _prepare_fallback_audio_inputs( + input_names=input_names, + config=config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + + model_type = str(config.get("model_type", "") or "").lower() + if model_type == "whisper" or "transformers.models.whisper." in type(model).__module__: + return _prepare_fallback_audio_inputs( + input_names=input_names, + config=config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=audio_file, + torch_dtype=torch_dtype, + ) + + sample_rate = _resolve_audio_sample_rate(processor) + waveform = _load_audio_waveform(audio_file, target_sample_rate=sample_rate) + batch = processor( + waveform, + sampling_rate=sample_rate, + return_tensors="pt", + ) + + preferred_keys = tuple(input_names) + tuple( + key for key in ("input_features", "input_values", "attention_mask") if key not in input_names + ) + tensor_keys = [key for key, value in batch.items() if isinstance(value, torch.Tensor)] + ordered_keys = [key for key in preferred_keys if key in tensor_keys] + ordered_keys.extend(key for key in tensor_keys if key not in ordered_keys) + if not ordered_keys: + raise RuntimeError(f"processor did not return tensor inputs for audio file: {audio_file}") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata={ + "audio_file": str(Path(audio_file).resolve()), + "sample_rate": sample_rate, + "fallback_audio_preprocessor": False, + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + }, + ) + + +def _prepare_text_inputs(tokenizer: object, *, prompt: str, input_ids_text: str | None) -> PreparedInputs: + if input_ids_text: + token_ids = [int(part.strip()) for part in input_ids_text.split(",") if part.strip()] + if not token_ids: + raise ValueError("--input-ids was provided but no ids were parsed") + input_ids = torch.tensor([token_ids], dtype=torch.long) + else: + input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"] + return PreparedInputs( + names=("input_ids",), + tensors=(input_ids,), + metadata={ + "prompt": prompt, + "input_ids": input_ids.tolist(), + }, + ) + + +_GEMMA4_MULTIMODAL_INPUT_ORDER = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", +) + + +def _normalize_multimodal_prompt( + prompt: str, + *, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, +) -> str: + normalized = prompt.strip() + prefixes: list[str] = [] + + if image_token and num_images > 0: + image_count = normalized.count(image_token) + if image_count < num_images: + prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) + if audio_token and num_audio_segments > 0: + audio_count = normalized.count(audio_token) + if audio_count < num_audio_segments: + prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) + + if prefixes: + prefix = "\n".join(part for part in prefixes if part) + if normalized: + return f"{prefix}\n{normalized}" + return prefix + return normalized + + +def _build_gemma4_chat_prompt( + *, + prompt: str, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> str: + result = "" + normalized_system = system_prompt.strip() + if enable_thinking_if_supported or normalized_system: + result += "<|turn>system\n" + if enable_thinking_if_supported: + result += "<|think|>" + result += normalized_system + result += "\n" + + result += "<|turn>user\n" + if image_token and num_images > 0: + for _ in range(num_images): + # Gemma4Processor expands the raw placeholder into the native + # <|image> ... soft-token region. Supplying wrappers here + # double-wraps the prompt and shifts the multimodal prefill. + result += f"\n\n{image_token}\n\n" + result += prompt + if audio_token and num_audio_segments > 0: + result += "".join( + audio_token + for _ in range(num_audio_segments) + ) + result += "\n" + result += "<|turn>model\n" + return result + + +def _gemma4_split_cactus_newline_token_merges(batch: object) -> None: + """Match native Cactus Gemma4 tokenization for newline runs. + + The HF fast tokenizer merges ``\n\n`` and ``\n\n\n`` into dedicated tokens + (108 and 109). Cactus' converted BPE tokenizer emits repeated newline token + 107 instead. Gemma4's multimodal prefill is sensitive enough that this one + token of drift changes the first generated token, so normalize only this + Gemma4-specific quirk after the processor has expanded image/audio regions. + """ + + input_ids = batch.get("input_ids") if hasattr(batch, "get") else None + if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: + return + + expansions = { + 108: (107, 107), + 109: (107, 107, 107), + } + if not any(int(token) in expansions for token in input_ids.reshape(-1).tolist()): + return + + lengths: list[list[int]] = [] + max_len = 0 + for row in input_ids.detach().cpu().tolist(): + row_lengths = [len(expansions.get(int(token), (int(token),))) for token in row] + lengths.append(row_lengths) + max_len = max(max_len, sum(row_lengths)) + + for key in ("input_ids", "attention_mask", "token_type_ids"): + value = batch.get(key) if hasattr(batch, "get") else None + if not isinstance(value, torch.Tensor) or value.ndim != 2: + continue + pad_value = 0 + expanded = torch.full( + (value.shape[0], max_len), + pad_value, + dtype=value.dtype, + device=value.device, + ) + for row_idx, row in enumerate(value.detach().cpu().tolist()): + out: list[int] = [] + for token_idx, item in enumerate(row): + if key == "input_ids": + out.extend(expansions.get(int(item), (int(item),))) + else: + out.extend([int(item)] * lengths[row_idx][token_idx]) + expanded[row_idx, : len(out)] = torch.tensor(out, dtype=value.dtype, device=value.device) + batch[key] = expanded + + +def _build_gemma4_chat_messages( + *, + prompt: str, + num_images: int, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> list[dict[str, object]]: + messages: list[dict[str, object]] = [] + normalized_system = system_prompt.strip() + if normalized_system or enable_thinking_if_supported: + system_text = normalized_system + if enable_thinking_if_supported: + system_text = f"<|think|>{system_text}" + messages.append({"role": "system", "content": system_text}) + + user_content: list[dict[str, str]] = [] + for _ in range(num_images): + user_content.append({"type": "image"}) + user_content.append({"type": "text", "text": prompt.strip()}) + for _ in range(num_audio_segments): + user_content.append({"type": "audio"}) + messages.append({"role": "user", "content": user_content}) + return messages + + +def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: + if not image_files: + return [] + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for --image-file: {exc}") from exc + + images: list[object] = [] + for image_file in image_files: + path = Path(image_file).resolve() + if not path.exists(): + raise RuntimeError(f"image_file does not exist: {path}") + with Image.open(path) as image: + images.append(image.convert("RGB").copy()) + return images + + +def _get_processor_image_attr(processor: object, name: str, default: object) -> object: + image_processor = getattr(processor, "image_processor", None) + if image_processor is not None and hasattr(image_processor, name): + return getattr(image_processor, name) + if isinstance(image_processor, dict) and name in image_processor: + return image_processor[name] + return default + + +def _prepare_gemma4_native_image_tensors( + processor: object, + image_files: tuple[str, ...], +) -> tuple[torch.Tensor, torch.Tensor] | None: + if not image_files: + return None + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for Gemma4 native image preprocessing: {exc}") from exc + + patch_size = int(_get_processor_image_attr(processor, "patch_size", 16)) + pooling_kernel_size = int(_get_processor_image_attr(processor, "pooling_kernel_size", 3)) + max_soft_tokens = int(_get_processor_image_attr(processor, "max_soft_tokens", 280)) + rescale_factor = float(_get_processor_image_attr(processor, "rescale_factor", 1.0 / 255.0)) + max_patches = max_soft_tokens * pooling_kernel_size * pooling_kernel_size + side_multiple = pooling_kernel_size * patch_size + patch_dim = 3 * patch_size * patch_size + if patch_size <= 0 or pooling_kernel_size <= 0 or max_patches <= 0: + return None + + try: + resample_bilinear = Image.Resampling.BILINEAR + except AttributeError: # pragma: no cover + resample_bilinear = Image.BILINEAR + + pixel_batches: list[np.ndarray] = [] + position_batches: list[np.ndarray] = [] + for image_file in image_files: + path = Path(image_file).resolve() + with Image.open(path) as image: + rgb = image.convert("RGB") + width, height = rgb.size + target_pixels = float(max_patches * patch_size * patch_size) + factor = float(np.sqrt(target_pixels / max(1.0, float(width * height)))) + target_h = int(np.floor(factor * height / side_multiple)) * side_multiple + target_w = int(np.floor(factor * width / side_multiple)) * side_multiple + if target_h == 0: + target_h = side_multiple + if target_w == 0: + target_w = side_multiple + if (target_w, target_h) != rgb.size: + rgb = rgb.resize((target_w, target_h), resample=resample_bilinear) + array = np.asarray(rgb, dtype=np.float32) * rescale_factor + + patch_h = target_h // patch_size + patch_w = target_w // patch_size + num_patches = patch_h * patch_w + if num_patches > max_patches: + raise RuntimeError( + f"Gemma4 native image preprocessing produced {num_patches} patches, " + f"but max_patches={max_patches}" + ) + chw = np.transpose(array, (2, 0, 1)) + patches = ( + chw.reshape(3, patch_h, patch_size, patch_w, patch_size) + .transpose(1, 3, 2, 4, 0) + .reshape(num_patches, patch_dim) + ) + + padded_patches = np.zeros((max_patches, patch_dim), dtype=np.float32) + padded_patches[:num_patches] = patches + positions = np.full((max_patches, 2), -1, dtype=np.int64) + valid_positions = np.zeros((num_patches, 2), dtype=np.int64) + for patch_y in range(patch_h): + row_start = patch_y * patch_w + valid_positions[row_start: row_start + patch_w, 0] = np.arange(patch_w, dtype=np.int64) + valid_positions[row_start: row_start + patch_w, 1] = patch_y + positions[:num_patches] = valid_positions + + pixel_batches.append(padded_patches) + position_batches.append(positions) + + return ( + torch.from_numpy(np.stack(pixel_batches, axis=0)), + torch.from_numpy(np.stack(position_batches, axis=0)), + ) + + +def _prepare_gemma4_multimodal_inputs( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, + use_gemma4_chat_template: bool = False, +) -> PreparedInputs: + if processor is None: + raise RuntimeError("multimodal Gemma4 transpile requires an AutoProcessor with image and audio support") + + images = _load_image_inputs(image_files) + audio_waveforms: list[np.ndarray] = [] + sample_rate: int | None = None + if audio_file: + sample_rate = _resolve_audio_sample_rate(processor) + audio_waveforms.append(_load_audio_waveform(audio_file, target_sample_rate=sample_rate)) + + image_token = getattr(processor, "image_token", None) + audio_token = getattr(processor, "audio_token", None) + processor_prompt = _normalize_multimodal_prompt( + prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + ) + normalized_prompt = processor_prompt + if use_gemma4_chat_template: + normalized_prompt = prompt.strip() + # Match the native Cactus Gemma4 prompt builder instead of the HF chat + # template. The HF template omits the begin/end multimodal wrapper tokens, + # which shifts the multimodal prefill by four tokens for image+audio prompts. + processor_prompt = _build_gemma4_chat_prompt( + prompt=normalized_prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + + batch = processor( + text=processor_prompt, + images=images or None, + audio=audio_waveforms or None, + return_tensors="pt", + ) + if use_gemma4_chat_template: + _gemma4_split_cactus_newline_token_merges(batch) + native_image_tensors = _prepare_gemma4_native_image_tensors(processor, image_files) + if native_image_tensors is not None: + batch["pixel_values"], batch["pixel_position_ids"] = native_image_tensors + if audio_file and isinstance(batch.get("input_features"), torch.Tensor): + feature_tensor = batch["input_features"] + expected_mels = int(feature_tensor.shape[-1]) + native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( + audio_file, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + batch["input_features"] = native_audio + batch["input_features_mask"] = native_audio_mask + batch["native_audio_frames"] = native_audio_frames + + ordered_keys = [ + key + for key in _GEMMA4_MULTIMODAL_INPUT_ORDER + if isinstance(batch.get(key), torch.Tensor) + ] + if not ordered_keys: + raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + metadata: dict[str, object] = { + "prompt": normalized_prompt, + "processor_prompt": processor_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + } + if audio_file: + metadata["audio_file"] = str(Path(audio_file).resolve()) + if sample_rate is not None: + metadata["sample_rate"] = sample_rate + native_audio_frames = batch.get("native_audio_frames") + if isinstance(native_audio_frames, int): + metadata["native_audio_frames"] = native_audio_frames + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata=metadata, + ) + + +def _load_model_source(model_id: str, *, local_files_only: bool) -> str: + local_snapshot = _resolve_local_snapshot(model_id) + if local_snapshot and _snapshot_has_model_weights(local_snapshot): + return local_snapshot + if local_snapshot and local_files_only: + raise RuntimeError( + f"Found local snapshot for {model_id}, but it is incomplete and has no model weights:\n" + f" {local_snapshot}\n" + "\n" + "Re-run without --local-files-only to let transformers download the missing weights." + ) + return model_id + + +def _load_transformers_bundle( + *, + model_id: str, + task: str, + torch_dtype: torch.dtype, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, +): + config = _load_config_json(model_id) + config_model_type = str(config.get("model_type", "") or "").lower() + external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) + patch_note = _patch_transformers_torchvision_probe() + if patch_note: + print(f"note={patch_note}") + flex_patch_note = _patch_torch_flex_attention_compat() + if flex_patch_note: + print(f"note={flex_patch_note}") + if external_transformers_site_packages: + print(f"note=using external transformers install for {config_model_type}: {external_transformers_site_packages}") + + try: + from transformers import AutoFeatureExtractor # type: ignore + from transformers import AutoModel # type: ignore + from transformers import AutoModelForCTC # type: ignore + from transformers import AutoModelForCausalLM # type: ignore + from transformers import AutoModelForImageTextToText # type: ignore + from transformers import AutoModelForSeq2SeqLM # type: ignore + from transformers import AutoModelForSpeechSeq2Seq # type: ignore + from transformers import AutoProcessor # type: ignore + from transformers import AutoTokenizer # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"transformers is not available: {exc}") from exc + + model_source = _load_model_source(model_id, local_files_only=local_files_only) + source_candidates = [] + for candidate in (model_source, model_id): + if candidate not in source_candidates: + source_candidates.append(candidate) + common_kwargs: dict[str, object] = { + "trust_remote_code": trust_remote_code, + "local_files_only": local_files_only, + } + if token: + common_kwargs["token"] = token + + if task == "tdt_transcription": + model = load_parakeet_tdt_local_model(model_source, torch_dtype=torch_dtype).eval() + return model_source, None, model, config + + if task == "causal_lm_logits": + tokenizer = None + tokenizer_errors: list[str] = [] + for source in source_candidates: + try: + tokenizer = AutoTokenizer.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + tokenizer_errors.append(f"{source}: {exc}") + if tokenizer is None: + raise RuntimeError( + f"Could not load tokenizer for {model_id}:\n" + + "\n".join(tokenizer_errors) + ) + if config_model_type == "lfm2_vl": + model = AutoModelForImageTextToText.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + tie_note = _tie_lfm2_vl_lm_head_if_needed(model) + if tie_note: + print(f"note={tie_note}") + else: + model = AutoModelForCausalLM.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model_source, tokenizer, model, config + if task == "multimodal_causal_lm_logits": + processor = None + processor_errors: list[str] = [] + for source in source_candidates: + try: + processor = AutoProcessor.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + processor_errors.append(f"{source}: {exc}") + if processor is None and config_model_type == "gemma4": + processor = _load_gemma4_processor_fallback( + source_candidates=source_candidates, + common_kwargs=common_kwargs, + ) + if processor is not None: + print("note=using manual gemma4 processor fallback") + if processor is None: + processor_config_hint = "" + if config_model_type == "gemma4": + processor_config_hint = ( + "\n" + "Gemma4 multimodal transpile needs a processor bundle, not just tokenizer/model weights.\n" + "Your local snapshot may be missing files such as `processor_config.json` or modality-specific\n" + "preprocessor configs. Use an official Gemma4 snapshot that includes the processor, or let\n" + "transformers download one by re-running without `--local-files-only`.\n" + ) + raise RuntimeError( + f"Could not load processor for {model_id}:\n" + + "\n".join(processor_errors) + + processor_config_hint + ) + + model = AutoModelForCausalLM.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + if config_model_type == "gemma4": + repair_result = _repair_gemma4_checkpoint_weights(model, model_source) + if repair_result.get("applied"): + missing = repair_result.get("missing_keys", []) + unexpected = repair_result.get("unexpected_keys", []) + print( + "note=applied gemma4 checkpoint key remap" + f" missing_after={len(missing)} unexpected_after={len(unexpected)}" + ) + return model_source, processor, model, config + + processor = None + processor_errors: list[str] = [] + missing_optional_audio_dep: str | None = None + for source in source_candidates: + for loader in (AutoProcessor, AutoFeatureExtractor): + try: + processor = loader.from_pretrained(source, **common_kwargs) + break + except Exception as exc: + processor_errors.append(f"{loader.__name__}@{source}: {exc}") + if isinstance(exc, ImportError) and "requires the librosa library" in str(exc): + missing_optional_audio_dep = "librosa" + break + if processor is not None: + break + if missing_optional_audio_dep is not None: + break + if processor is None: + if missing_optional_audio_dep == "librosa": + print("note=falling back to built-in audio preprocessing because the HF feature extractor requires librosa") + else: + print("note=falling back to built-in audio preprocessing because no HF processor/feature extractor was available") + + if task == "ctc_logits": + model_loaders = (AutoModelForCTC, AutoModel) + elif task == "encoder_hidden_states": + model_loaders = (AutoModelForSpeechSeq2Seq, AutoModelForSeq2SeqLM, AutoModel) + else: + raise NotImplementedError(f"unsupported generic HF task: {task}") + + load_errors: list[str] = [] + for loader in model_loaders: + try: + model = loader.from_pretrained( + model_source, + dtype=torch_dtype, + device_map=None, + low_cpu_mem_usage=True, + **common_kwargs, + ).eval() + return model_source, processor, model, config + except Exception as exc: + load_errors.append(f"{loader.__name__}: {exc}") + + raise RuntimeError( + f"Could not load model for task={task} from {model_source}.\n" + "\n".join(load_errors) + ) + + +def _load_optional_tokenizer( + *, + model_id: str, + model_source: str, + token: str | None, + trust_remote_code: bool, + local_files_only: bool, +): + try: + from transformers import AutoTokenizer # type: ignore + except Exception: + return None + + source_candidates = [] + for candidate in (model_source, model_id): + if candidate not in source_candidates: + source_candidates.append(candidate) + + common_kwargs: dict[str, object] = { + "trust_remote_code": trust_remote_code, + "local_files_only": local_files_only, + } + if token: + common_kwargs["token"] = token + + for source in source_candidates: + try: + return AutoTokenizer.from_pretrained(source, **common_kwargs) + except Exception: + continue + return None + + +def _ctc_greedy_decode_token_ids(logits: np.ndarray, *, blank_token_id: int | None) -> list[int]: + if logits.ndim != 3 or logits.shape[0] < 1: + raise ValueError(f"expected CTC logits with shape [batch, time, vocab], got {list(logits.shape)}") + + raw_ids = np.argmax(logits[0], axis=-1).tolist() + collapsed: list[int] = [] + previous: int | None = None + for token_id in raw_ids: + if token_id != previous: + collapsed.append(int(token_id)) + previous = int(token_id) + + if blank_token_id is None: + return collapsed + return [token_id for token_id in collapsed if int(token_id) != int(blank_token_id)] + + +def _decode_token_ids(tokenizer: object, token_ids: list[int]) -> str: + decode = getattr(tokenizer, "decode", None) + if not callable(decode): + raise TypeError(f"tokenizer does not expose decode(): {type(tokenizer).__name__}") + try: + return str(decode(token_ids, skip_special_tokens=True)) + except TypeError: + return str(decode(token_ids)) + + +def _count_weight_bindings(ir_graph: IRGraph) -> int: + count = 0 + for value in ir_graph.values.values(): + if isinstance(value.meta, dict) and isinstance(value.meta.get("path"), str): + count += 1 + return count + + +def _lower_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + verify_ir(ir) + graph = Graph() + graph._transpile_materialized_constants = [] # type: ignore[attr-defined] + env: dict[str, Any] = {} + runtime_inputs = [] + bound_constants = [] + bound_constant_bindings = [] + + for value_id in ir.inputs: + value = ir.values[value_id] + tensor = _lower_input_value(graph, value) + env[value_id] = tensor + runtime_inputs.append(tensor) + + for value_id, const in ir.constants.items(): + value = ir.values[value_id] + binding = _lookup_weight_binding(value) + lowered_const = _lower_constant_value(graph, value, const) + env[value_id] = lowered_const + if hasattr(lowered_const, "g") and hasattr(lowered_const, "id"): + bound_constants.append(lowered_const) + if binding is not None: + bound_constant_bindings.append( + { + "node_id": int(lowered_const.id), + "value_id": str(value_id), + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + ) + + for node_id in ir.order: + node = ir.nodes[node_id] + outputs = _lower_ir_node(graph, node, env, ir) + if len(outputs) != len(node.outputs): + raise ValueError( + f"node {node.id} produced {len(outputs)} outputs, expected {len(node.outputs)}" + ) + for output_id, tensor in zip(node.outputs, outputs): + env[output_id] = tensor + + outputs = [env[value_id] for value_id in ir.outputs] + seen_bound_constant_ids = {int(tensor.id) for tensor in bound_constants} + for tensor in getattr(graph, "_transpile_materialized_constants", []): + tensor_id = int(tensor.id) + if tensor_id in seen_bound_constant_ids: + continue + bound_constants.append(tensor) + seen_bound_constant_ids.add(tensor_id) + return TranspiledGraph( + graph=graph, + runtime_inputs=runtime_inputs, + bound_constants=bound_constants, + bound_constant_bindings=bound_constant_bindings, + outputs=outputs, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Load a Hugging Face model, canonicalize it into a generic transpile task, " + "capture it with the Cactus transpiler, lower it to a Cactus Graph, and " + "optionally save artifacts or run the lowered graph." + ) + ) + parser.add_argument( + "--model-id", + required=True, + help="Hugging Face model id or local snapshot path.", + ) + parser.add_argument( + "--task", + default="auto", + choices=( + "auto", + "causal_lm_logits", + "multimodal_causal_lm_logits", + "ctc_logits", + "encoder_hidden_states", + "tdt_transcription", + ), + help="Transpile task. Use auto to infer from config/model id.", + ) + parser.add_argument( + "--prompt", + default="The capital of France is", + help="Prompt used for causal_lm_logits or multimodal_causal_lm_logits when --input-ids is not set.", + ) + parser.add_argument( + "--system-prompt", + default="", + help="Optional system prompt used for Gemma4 multimodal chat-style prompt construction.", + ) + parser.add_argument( + "--enable-thinking", + action="store_true", + help="Enable model-specific thinking markers when the multimodal prompt format supports them.", + ) + parser.add_argument( + "--input-ids", + default="", + help="Optional comma-separated token ids for causal_lm_logits.", + ) + parser.add_argument( + "--audio-file", + default="", + help="Path to a WAV file for audio or multimodal tasks.", + ) + parser.add_argument( + "--image-file", + action="append", + default=[], + help="Path to an image file for multimodal tasks. Repeat to pass multiple images.", + ) + parser.add_argument( + "--torch-dtype", + default="float16", + help="Torch dtype for model loading: float16, float32, or bfloat16.", + ) + parser.add_argument( + "--token", + default=os.environ.get("HF_TOKEN"), + help="Optional Hugging Face token. Defaults to HF_TOKEN.", + ) + parser.add_argument( + "--trust-remote-code", + action="store_true", + help="Pass trust_remote_code=True to transformers loaders.", + ) + parser.add_argument( + "--local-files-only", + action="store_true", + help="Require the model/processor to already exist locally.", + ) + parser.add_argument( + "--weights-dir", + default="", + help="Optional converted Cactus weights directory for mmap weight binding.", + ) + parser.add_argument( + "--artifact-dir", + default="", + help="Directory where raw_ir.json, optimized_ir.json, graph.cactus, component graphs, and result.json are saved. Defaults to ./transpiled//.", + ) + parser.add_argument( + "--graph-filename", + default="graph.cactus", + help="Filename to use for Graph.save() inside --artifact-dir.", + ) + parser.add_argument( + "--component-pipeline", + default="auto", + choices=("auto", "on", "off"), + help="Use a split component pipeline when the model adapter supports it.", + ) + parser.add_argument("--no-fuse-gated-deltanet", action="store_true") + parser.add_argument("--no-fuse-rms-norm", action="store_true") + parser.add_argument("--no-fuse-rope", action="store_true") + parser.add_argument("--no-fuse-attention", action="store_true") + parser.add_argument("--no-fuse-attention-block", action="store_true") + parser.add_argument("--no-fuse-add-clipped", action="store_true") + parser.add_argument( + "--skip-execute", + action="store_true", + help="Stop after lowering instead of running the transpiled graph.", + ) + parser.add_argument( + "--skip-reference-compare", + action="store_true", + help="Run the transpiled graph but skip the follow-up PyTorch reference pass.", + ) + args = parser.parse_args() + + image_files = tuple(str(path) for path in args.image_file if str(path).strip()) + if args.task == "auto": + inferred_task = _infer_task_from_config(args.model_id) + config_for_auto = _load_config_json(args.model_id) + model_type_for_auto = str(config_for_auto.get("model_type", "") or "").lower() + has_multimodal_config = ( + model_type_for_auto == "gemma4" + and ( + isinstance(config_for_auto.get("vision_config"), dict) + or isinstance(config_for_auto.get("audio_config"), dict) + ) + ) + if image_files or (args.audio_file and has_multimodal_config and inferred_task == "causal_lm_logits"): + task = "multimodal_causal_lm_logits" + else: + task = inferred_task + else: + task = args.task + torch_dtype = _parse_dtype(args.torch_dtype) + validated_weights_dir = _validate_weights_dir(args.weights_dir.strip() or None, model_id=args.model_id) + weights_dir = str(validated_weights_dir) if validated_weights_dir is not None else None + artifact_dir = Path(args.artifact_dir).resolve() if args.artifact_dir else _default_artifact_dir_for_model(args.model_id) + + model_source, processor_or_tokenizer, model, model_config = _load_transformers_bundle( + model_id=args.model_id, + task=task, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json") + if not preprocessor_config: + preprocessor_config = _load_optional_json(args.model_id, "preprocessor_config.json") + auxiliary_tokenizer = None + if task == "ctc_logits": + auxiliary_tokenizer = _load_optional_tokenizer( + model_id=args.model_id, + model_source=model_source, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, + ) + + canonical = None + if task == "causal_lm_logits": + prepared = _prepare_text_inputs( + processor_or_tokenizer, + prompt=args.prompt, + input_ids_text=args.input_ids.strip() or None, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + weights_dir=weights_dir, + ) + elif task == "multimodal_causal_lm_logits": + prepared = _prepare_gemma4_multimodal_inputs( + processor_or_tokenizer, + prompt=args.prompt, + image_files=image_files, + audio_file=args.audio_file.strip() or None, + torch_dtype=torch_dtype, + system_prompt=args.system_prompt, + enable_thinking_if_supported=args.enable_thinking, + use_gemma4_chat_template=True, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + weights_dir=weights_dir, + ) + prime_static_features = getattr(canonical.module, "prime_static_multimodal_features", None) + if callable(prime_static_features): + prime_static_features(*prepared.tensors) + elif task == "tdt_transcription": + if not args.audio_file: + raise RuntimeError(f"--audio-file is required for task={task}") + prepared = _prepare_audio_inputs( + processor_or_tokenizer, + input_names=("input_features",), + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=args.audio_file, + torch_dtype=torch_dtype, + ) + else: + if not args.audio_file: + raise RuntimeError(f"--audio-file is required for task={task}") + canonical = canonicalize_model_interface(model, task=task) + prepared = _prepare_audio_inputs( + processor_or_tokenizer, + input_names=canonical.input_names, + config=model_config, + preprocessor_config=preprocessor_config, + model=model, + task=task, + audio_file=args.audio_file, + torch_dtype=torch_dtype, + ) + canonical = canonicalize_model_interface( + model, + task=task, + input_names=prepared.names, + ) + component_specs = build_component_module_specs( + model, + task=task, + named_tensors=_named_tensor_store(prepared), + weights_dir=weights_dir, + ) + use_component_pipeline = False + if args.component_pipeline == "on": + if component_specs is None: + raise RuntimeError( + f"--component-pipeline=on was requested, but {type(model).__name__} does not expose component specs for task={task}" + ) + use_component_pipeline = True + elif args.component_pipeline == "auto" and component_specs is not None: + use_component_pipeline = True + + if use_component_pipeline: + family = canonical.family if canonical is not None else getattr(model, "family", type(model).__name__) + return _run_component_pipeline_transpile( + args=args, + task=task, + family=str(family), + model_source=model_source, + model=model, + prepared=prepared, + component_specs=component_specs, + fusion_config=FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ), + weights_dir=weights_dir, + artifact_dir=artifact_dir, + processor_or_tokenizer=processor_or_tokenizer, + canonical=canonical, + ) + + if canonical is None: + raise RuntimeError(f"task={task} requires a component pipeline but none was selected") + wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + + print(f"model_id={args.model_id}") + print(f"model_source={model_source}") + print(f"task={task}") + print(f"adapter_family={canonical.family}") + print(f"adapter_module={type(canonical.module).__name__}") + print(f"input_names={','.join(prepared.names)}") + for name, tensor in zip(prepared.names, prepared.tensors): + print(f"input_{name}_shape={list(tensor.shape)}") + if weights_dir: + print(f"weights_dir={weights_dir}") + + print("capture_begin=true", flush=True) + prepare_cpu_float32_capture = getattr(canonical.module, "prepare_cpu_float32_capture", None) + restore_cpu_float32_capture = getattr(canonical.module, "restore_cpu_float32_capture", None) + if callable(prepare_cpu_float32_capture): + prepare_cpu_float32_capture() + try: + captured = capture_model(wrapper, prepared.tensors) + finally: + if callable(restore_cpu_float32_capture): + restore_cpu_float32_capture() + print("capture_done=true", flush=True) + raw_ir_graph = copy.deepcopy(captured.ir_graph) + + fusion_config = FusionConfig( + enable_gated_deltanet=not args.no_fuse_gated_deltanet, + enable_rms_norm=not args.no_fuse_rms_norm, + enable_rope=not args.no_fuse_rope, + enable_attention=not args.no_fuse_attention, + enable_attention_block=not args.no_fuse_attention_block, + enable_add_clipped=not args.no_fuse_add_clipped, + ) + + print("canonicalize_begin=true", flush=True) + canonicalize_exported_graph(captured.ir_graph) + print("canonicalize_done=true", flush=True) + print("optimize_begin=true", flush=True) + optimize_graph(captured.ir_graph, config=fusion_config) + print("optimize_done=true", flush=True) + print("lower_begin=true", flush=True) + tg = _lower_preoptimized_ir(captured.ir_graph) + print("lower_done=true", flush=True) + + optimized_ir_graph = copy.deepcopy(captured.ir_graph) + binding_count = _count_weight_bindings(optimized_ir_graph) + op_counts = Counter(optimized_ir_graph.nodes[node_id].op for node_id in optimized_ir_graph.order) + raw_component_counts = summarize_ir_components(raw_ir_graph) + optimized_component_counts = summarize_ir_components(optimized_ir_graph) + raw_component_graphs = extract_component_subgraphs(raw_ir_graph) + optimized_component_graphs = extract_component_subgraphs(optimized_ir_graph) + transpiled_component_graphs: dict[str, TranspiledGraph] | None = None + + print(f"raw_ir_nodes={len(raw_ir_graph.order)}") + print(f"optimized_ir_nodes={len(optimized_ir_graph.order)}") + print(f"weight_bindings={binding_count}") + if raw_component_counts: + print( + "raw_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(raw_component_counts.items())) + ) + if optimized_component_counts: + print( + "optimized_components=" + + ",".join(f"{name}:{count}" for name, count in sorted(optimized_component_counts.items())) + ) + print( + "ops=" + f"attention:{op_counts.get('attention', 0)} " + f"conv1d:{op_counts.get('conv1d', 0)} " + f"conv2d:{op_counts.get('conv2d', 0)} " + f"batch_norm:{op_counts.get('batch_norm', 0)} " + f"layer_norm:{op_counts.get('layer_norm', 0)} " + f"rms_norm:{op_counts.get('rms_norm', 0)} " + f"rope:{op_counts.get('rope', 0)} " + f"linear:{op_counts.get('linear', 0)}" + ) + + if weights_dir and binding_count == 0: + raise RuntimeError( + f"No weight bindings were resolved from {weights_dir}\n" + "\n" + "The weights folder exists, but none of the captured constants matched entries in weights_manifest.json.\n" + "\n" + f"Recommended fix:\n" + f" cactus convert {args.model_id} {weights_dir}\n" + ) + + if artifact_dir is not None: + artifact_dir.mkdir(parents=True, exist_ok=True) + transpiled_component_graphs = { + component: _lower_preoptimized_ir(copy.deepcopy(component_graph)) + for component, component_graph in optimized_component_graphs.items() + } + _write_json( + artifact_dir / "raw_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(raw_ir_graph), + }, + ) + _write_json( + artifact_dir / "optimized_ir.json", + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(optimized_ir_graph), + }, + ) + for component, component_graph in raw_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("raw_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + for component, component_graph in optimized_component_graphs.items(): + _write_json( + artifact_dir / _component_artifact_name("optimized_ir", component), + { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "component": component, + "inputs": _serialize_json_compatible(prepared.metadata), + "graph": _graph_to_dict(component_graph), + }, + ) + print(f"saved_raw_ir={artifact_dir / 'raw_ir.json'}") + print(f"saved_optimized_ir={artifact_dir / 'optimized_ir.json'}") + for component in raw_component_graphs: + print(f"saved_raw_component_ir_{component}={artifact_dir / _component_artifact_name('raw_ir', component)}") + for component in optimized_component_graphs: + print( + f"saved_optimized_component_ir_{component}=" + f"{artifact_dir / _component_artifact_name('optimized_ir', component)}" + ) + component_manifest_path = _write_component_bundle( + artifact_dir=artifact_dir, + model_id=args.model_id, + model_source=model_source, + task=task, + family=canonical.family, + inputs_metadata=prepared.metadata, + raw_component_graphs=raw_component_graphs, + optimized_component_graphs=optimized_component_graphs, + transpiled_component_graphs=transpiled_component_graphs, + graph_filename=args.graph_filename, + ) + print(f"saved_component_bundle_manifest={component_manifest_path}") + for component in optimized_component_graphs: + print( + f"saved_component_graph_{component}=" + f"{artifact_dir / 'components' / component / args.graph_filename}" + ) + graph_path = artifact_dir / args.graph_filename + tg.graph.save(graph_path) + print(f"saved_graph={graph_path}") + graph_binding_manifest_path = _write_graph_binding_manifest( + artifact_dir=artifact_dir, + filename="graph_bindings.json", + model_id=args.model_id, + model_source=model_source, + task=task, + family=canonical.family, + inputs_metadata=prepared.metadata, + transpiled_graph=tg, + ) + print(f"saved_graph_bindings={graph_binding_manifest_path}") + if binding_count > 0: + print( + "note=saved graph structure without embedded mmap bindings; use the saved component/full-graph manifests " + "to rebind weights/embeddings from the weights folder when loading" + ) + + if args.skip_execute: + return 0 + + tg.set_inputs([tensor.cpu().numpy() for tensor in prepared.tensors]) + print("execute_begin=true") + transpiled_output = tg.execute()[0].numpy().astype(np.float32) + print("execute_done=true") + + if args.skip_reference_compare: + result_payload = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(transpiled_output.shape), + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": binding_count, + "reference_compare_skipped": True, + } + print(f"output_shape={list(transpiled_output.shape)}") + if task in {"causal_lm_logits", "multimodal_causal_lm_logits"}: + tokenizer_like = getattr(processor_or_tokenizer, "tokenizer", processor_or_tokenizer) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"transpiled_next_token_id={transpiled_next}") + result_payload["transpiled_next_token_id"] = transpiled_next + if hasattr(tokenizer_like, "decode"): + transpiled_token = tokenizer_like.decode([transpiled_next]) + print(f"transpiled_next_token={transpiled_token!r}") + result_payload["transpiled_next_token"] = transpiled_token + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + del model + del wrapper + gc.collect() + return 0 + + print("reference_begin=true") + with torch.no_grad(): + reference_output = wrapper(*prepared.tensors).detach().float().cpu().numpy() + print("reference_done=true") + + max_abs_diff = float(np.max(np.abs(reference_output - transpiled_output))) + mean_abs_diff = float(np.mean(np.abs(reference_output - transpiled_output))) + result_payload: dict[str, object] = { + "model_id": args.model_id, + "model_source": model_source, + "task": task, + "family": canonical.family, + "inputs": _serialize_json_compatible(prepared.metadata), + "output_shape": list(reference_output.shape), + "raw_ir_nodes": len(raw_ir_graph.order), + "optimized_ir_nodes": len(optimized_ir_graph.order), + "weight_bindings": binding_count, + "max_abs_diff": max_abs_diff, + "mean_abs_diff": mean_abs_diff, + } + + if task == "causal_lm_logits": + tokenizer = processor_or_tokenizer + hf_next = int(np.argmax(reference_output[0, -1])) + transpiled_next = int(np.argmax(transpiled_output[0, -1])) + print(f"hf_next_token_id={hf_next}") + print(f"transpiled_next_token_id={transpiled_next}") + print(f"logits_max_abs_diff={max_abs_diff:.6f}") + print(f"logits_mean_abs_diff={mean_abs_diff:.6f}") + print(f"hf_next_token={tokenizer.decode([hf_next])!r}") + print(f"transpiled_next_token={tokenizer.decode([transpiled_next])!r}") + result_payload.update( + { + "hf_next_token_id": hf_next, + "transpiled_next_token_id": transpiled_next, + "hf_next_token": tokenizer.decode([hf_next]), + "transpiled_next_token": tokenizer.decode([transpiled_next]), + } + ) + elif task == "ctc_logits": + blank_token_id = getattr(model.config, "pad_token_id", None) + if blank_token_id is None and auxiliary_tokenizer is not None: + blank_token_id = getattr(auxiliary_tokenizer, "pad_token_id", None) + print(f"output_shape={list(reference_output.shape)}") + print(f"output_max_abs_diff={max_abs_diff:.6f}") + print(f"output_mean_abs_diff={mean_abs_diff:.6f}") + if auxiliary_tokenizer is not None: + hf_token_ids = _ctc_greedy_decode_token_ids(reference_output, blank_token_id=blank_token_id) + transpiled_token_ids = _ctc_greedy_decode_token_ids(transpiled_output, blank_token_id=blank_token_id) + hf_transcript = _decode_token_ids(auxiliary_tokenizer, hf_token_ids) + transpiled_transcript = _decode_token_ids(auxiliary_tokenizer, transpiled_token_ids) + print(f"hf_transcript={hf_transcript!r}") + print(f"transpiled_transcript={transpiled_transcript!r}") + result_payload.update( + { + "blank_token_id": None if blank_token_id is None else int(blank_token_id), + "hf_transcript_token_ids": hf_token_ids, + "transpiled_transcript_token_ids": transpiled_token_ids, + "hf_transcript": hf_transcript, + "transpiled_transcript": transpiled_transcript, + } + ) + else: + print(f"output_shape={list(reference_output.shape)}") + print(f"output_max_abs_diff={max_abs_diff:.6f}") + print(f"output_mean_abs_diff={mean_abs_diff:.6f}") + + if artifact_dir is not None: + _write_json(artifact_dir / "result.json", result_payload) + print(f"saved_result={artifact_dir / 'result.json'}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/src/cactus.py b/python/src/cactus.py index 15479c0cd..1485d9044 100644 --- a/python/src/cactus.py +++ b/python/src/cactus.py @@ -1,12 +1,16 @@ """Cactus Python FFI bindings.""" import ctypes +import os import platform from pathlib import Path TokenCallback = ctypes.CFUNCTYPE(None, ctypes.c_char_p, ctypes.c_uint32, ctypes.c_void_p) _DIR = Path(__file__).parent.parent.parent -if platform.system() == "Darwin": +_OVERRIDE_LIB_PATH = os.environ.get("CACTUS_LIB_PATH") +if _OVERRIDE_LIB_PATH: + _LIB_PATH = Path(_OVERRIDE_LIB_PATH).expanduser().resolve() +elif platform.system() == "Darwin": _LIB_PATH = _DIR / "cactus" / "build" / "libcactus.dylib" else: _LIB_PATH = _DIR / "cactus" / "build" / "libcactus.so" @@ -88,11 +92,20 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_divide.restype = ctypes.c_int +_lib.cactus_graph_not_equal.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_not_equal.restype = ctypes.c_int _lib.cactus_graph_precision_cast.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_precision_cast.restype = ctypes.c_int +_lib.cactus_graph_quantize_activations.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_quantize_activations.restype = ctypes.c_int + _lib.cactus_graph_scalar_add.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] @@ -109,6 +122,14 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_scalar_divide.restype = ctypes.c_int +_lib.cactus_graph_scalar_floor_divide.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_scalar_floor_divide.restype = ctypes.c_int +_lib.cactus_graph_scalar_not_equal.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_scalar_not_equal.restype = ctypes.c_int _lib.cactus_graph_scalar_exp.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] @@ -129,6 +150,14 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_scalar_log.restype = ctypes.c_int +_lib.cactus_graph_masked_select_prefix.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_masked_select_prefix.restype = ctypes.c_int +_lib.cactus_graph_masked_scatter.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_masked_scatter.restype = ctypes.c_int _lib.cactus_graph_abs.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) @@ -154,6 +183,10 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_reshape.restype = ctypes.c_int +_lib.cactus_graph_expand.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_expand.restype = ctypes.c_int _lib.cactus_graph_transpose.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] @@ -182,11 +215,11 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_graph_cat.restype = ctypes.c_int _lib.cactus_graph_matmul.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, ctypes.c_int32, ctypes.POINTER(cactus_node_t) + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, ctypes.c_int32, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_matmul.restype = ctypes.c_int _lib.cactus_graph_gather.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_gather.restype = ctypes.c_int _lib.cactus_graph_embedding_from_tensor.argtypes = [ @@ -209,6 +242,14 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_bilinear_interpolation.restype = ctypes.c_int +_lib.cactus_graph_set_grouped_scales.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_void_p +] +_lib.cactus_graph_set_grouped_scales.restype = ctypes.c_int +_lib.cactus_graph_set_interleaved.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_bool, ctypes.c_size_t +] +_lib.cactus_graph_set_interleaved.restype = ctypes.c_int _lib.cactus_graph_release_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] _lib.cactus_graph_release_weight_pages.restype = ctypes.c_int _lib.cactus_graph_prefetch_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] @@ -236,6 +277,10 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_max.restype = ctypes.c_int +_lib.cactus_graph_cumsum.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_cumsum.restype = ctypes.c_int _lib.cactus_graph_relu.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) @@ -351,6 +396,15 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_conv2d_pointwise_1x1.restype = ctypes.c_int +_lib.cactus_graph_conv2d_k3s1p1.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv2d_k3s1p1.restype = ctypes.c_int +_lib.cactus_graph_conv2d.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, + ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv2d.restype = ctypes.c_int _lib.cactus_graph_lstm_cell.argtypes = [ cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] @@ -467,6 +521,17 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_audio_embed.restype = ctypes.c_int +try: + _lib.cactus_preprocess_audio_features.argtypes = [ + ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, + ctypes.POINTER(ctypes.c_size_t), ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t) + ] + _lib.cactus_preprocess_audio_features.restype = ctypes.c_int +except AttributeError: + pass + _lib.cactus_vad.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t @@ -507,6 +572,17 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_tokenize.restype = ctypes.c_int +_lib.cactus_decode_tokens.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_uint32), + ctypes.c_size_t, + ctypes.c_float, + ctypes.c_float, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_uint32), +] +_lib.cactus_decode_tokens.restype = ctypes.c_int + _lib.cactus_score_window.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), @@ -710,8 +786,6 @@ def cactus_prefill(model, messages_json, options_json, tools_json, pcm_data=None ) if rc < 0: raise RuntimeError(_err("Prefill failed")) - return buf.value.decode("utf-8", errors="ignore") - def cactus_detect_language(model, audio_path, options_json, pcm_data): """Detects the spoken language in audio. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) @@ -785,6 +859,29 @@ def cactus_audio_embed(model, audio_path): return list(buf[:dim.value]) +def cactus_preprocess_audio_features(audio_path, model_type, mel_bins, capacity): + """Returns Cactus-native audio frontend features as (values, mel_bins, frames).""" + if not hasattr(_lib, "cactus_preprocess_audio_features"): + raise RuntimeError("cactus_preprocess_audio_features is unavailable; rebuild with cactus build --python") + buf = (ctypes.c_float * int(capacity))() + feature_count = ctypes.c_size_t() + out_mels = ctypes.c_size_t() + out_frames = ctypes.c_size_t() + rc = _lib.cactus_preprocess_audio_features( + _enc(audio_path), + _enc(model_type), + ctypes.c_size_t(int(mel_bins)), + buf, + ctypes.sizeof(buf), + ctypes.byref(feature_count), + ctypes.byref(out_mels), + ctypes.byref(out_frames), + ) + if rc < 0: + raise RuntimeError(_err("Audio feature preprocessing failed")) + return list(buf[:feature_count.value]), int(out_mels.value), int(out_frames.value) + + def cactus_vad(model, audio_path, options_json, pcm_data): """Runs voice activity detection. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) @@ -858,6 +955,26 @@ def cactus_tokenize(model, text): return list(arr[:n.value]) +def cactus_decode_tokens(model, tokens, temperature=0.0, top_p=1.0, top_k=1): + """Returns the next token ID from native Cactus for exact raw token IDs.""" + if not tokens: + raise ValueError("tokens must be non-empty") + arr = (ctypes.c_uint32 * len(tokens))(*[int(token) for token in tokens]) + out = ctypes.c_uint32(0) + rc = _lib.cactus_decode_tokens( + model, + arr, + len(tokens), + ctypes.c_float(float(temperature)), + ctypes.c_float(float(top_p)), + ctypes.c_size_t(int(top_k)), + ctypes.byref(out), + ) + if rc < 0: + raise RuntimeError(_err("Raw token decode failed")) + return int(out.value) + + def cactus_score_window(model, tokens, start, end, context): """Scores a window of tokens. Returns JSON string.""" buf = ctypes.create_string_buffer(65536) diff --git a/python/src/cli.py b/python/src/cli.py index a3db11c21..fbb10f2f8 100644 --- a/python/src/cli.py +++ b/python/src/cli.py @@ -13,11 +13,10 @@ def _looks_like_project_root(path: Path) -> bool: - return ( - (path / "python" / "src" / "cli.py").exists() - and (path / "cactus").exists() - and (path / "tests").exists() - ) + has_cli = (path / "python" / "src" / "cli.py").exists() + has_engine = (path / "cactus").exists() or (path / "cactus-engine").exists() + has_tests = (path / "tests").exists() or (path / "python" / "tests").exists() + return has_cli and has_engine and has_tests def _resolve_project_root() -> Path: @@ -48,6 +47,62 @@ def _resolve_project_root() -> Path: DEFAULT_TEST_MODEL_ID = "google/gemma-4-E2B-it" WEIGHTS_VARIANT_CHOICES = ["auto", "apple", "standard"] + +def _transpile_runtime_root() -> Path: + return PROJECT_ROOT / "cactus-transpile" + + +def _transpile_runtime_library_path() -> Path: + suffix = ".dylib" if platform.system() == "Darwin" else ".so" + return _transpile_runtime_root() / "build" / f"libcactus{suffix}" + + +def _ensure_transpile_runtime_library() -> Path: + runtime_root = _transpile_runtime_root() + cmake_lists = runtime_root / "CMakeLists.txt" + if not cmake_lists.exists(): + raise RuntimeError( + "The vendored transpiler runtime is missing.\n" + f"Expected: {cmake_lists}" + ) + + library_path = _transpile_runtime_library_path() + if library_path.exists(): + return library_path + + build_dir = runtime_root / "build" + build_dir.mkdir(parents=True, exist_ok=True) + jobs = str(os.cpu_count() or 4) + + print_color(YELLOW, "Building vendored transpiler runtime...") + configure = subprocess.run( + [ + "cmake", + "-S", + str(runtime_root), + "-B", + str(build_dir), + "-DCMAKE_RULE_MESSAGES=OFF", + "-DCMAKE_VERBOSE_MAKEFILE=OFF", + ], + cwd=PROJECT_ROOT, + ) + if configure.returncode != 0: + raise RuntimeError("Failed to configure the vendored transpiler runtime") + + build = subprocess.run( + ["cmake", "--build", str(build_dir), "-j", jobs], + cwd=PROJECT_ROOT, + ) + if build.returncode != 0: + raise RuntimeError("Failed to build the vendored transpiler runtime") + if not library_path.exists(): + raise RuntimeError( + "The vendored transpiler runtime build completed, but the library was not produced.\n" + f"Expected: {library_path}" + ) + return library_path + # dont fail if models.json is missing or malformed - just have an empty registry and rely on HF download_args try: with open(PROJECT_ROOT / "models.json", "r", encoding="utf-8") as f: @@ -1023,8 +1078,67 @@ def prompt_for_api_key(config): return api_key +def _resolve_transpiled_manifest(path_value): + candidate = Path(path_value).expanduser().resolve() + if not candidate.exists(): + return None + if candidate.is_file() and candidate.name == "manifest.json": + return candidate + manifest = candidate / "components" / "manifest.json" + if manifest.exists(): + return manifest + manifest = candidate / "manifest.json" + if manifest.exists(): + return manifest + return None + + +def cmd_run_transpiled(args): + """Run a saved transpiled component bundle.""" + transpile_lib = _ensure_transpile_runtime_library() + os.environ["CACTUS_LIB_PATH"] = str(transpile_lib) + from .transpile.component_bundle_runtime import run_transpiled_bundle + + bundle_dir = getattr(args, "bundle_dir", None) or getattr(args, "model_id", None) + result = run_transpiled_bundle( + bundle_dir, + audio_file=getattr(args, "audio_file", None) or getattr(args, "audio", None), + prompt=getattr(args, "prompt", None), + input_ids=getattr(args, "input_ids", None), + weights_dir=getattr(args, "weights_dir", None), + ) + print(json.dumps(result, indent=2, sort_keys=True)) + if getattr(args, "result_json", None): + result_path = Path(args.result_json).expanduser().resolve() + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print_color(GREEN, f"Saved result to {result_path}") + return 0 + + +def cmd_transpile(args): + """Invoke the saved-model transpiler entrypoint.""" + script_path = PROJECT_ROOT / "python" / "examples" / "transpile_hf_model.py" + if not script_path.exists(): + print_color(RED, f"Error: transpiler script not found at {script_path}") + return 1 + + transpile_lib = _ensure_transpile_runtime_library() + command = [sys.executable, str(script_path), "--model-id", args.model_id] + command.extend(getattr(args, "extra_args", []) or []) + env = os.environ.copy() + env["CACTUS_LIB_PATH"] = str(transpile_lib) + result = subprocess.run(command, cwd=PROJECT_ROOT, env=env) + return result.returncode + + def cmd_run(args): """Download model if needed and start interactive chat.""" + manifest_path = _resolve_transpiled_manifest(args.model_id) + if manifest_path is not None: + args.bundle_dir = str(manifest_path.parent.parent if manifest_path.parent.name == "components" else manifest_path.parent) + return cmd_run_transpiled(args) + from .config_utils import CactusConfig config = CactusConfig() @@ -2080,13 +2194,38 @@ def create_parser(): help='Path to image file for VLM inference (attached to first message)') run_parser.add_argument('--audio', help='Path to audio file (WAV) for audio chat (attached to first message)') + run_parser.add_argument('--file', dest='audio_file', default=None, + help='Audio file for transpiled bundles when model_id points to a transpiled folder') + run_parser.add_argument('--weights-dir', default=None, + help='Converted weights directory for transpiled bundles with bound weights') run_parser.add_argument('--system', help='System prompt to prepend to all messages') run_parser.add_argument('--prompt', help='Initial prompt to send immediately') + run_parser.add_argument('--input-ids', default=None, + help='Comma-separated token ids for transpiled causal-LM bundles') + run_parser.add_argument('--result-json', default=None, + help='Optional path to save transpiled bundle results as JSON') run_parser.add_argument('--thinking', action='store_true', help='Enable thinking/reasoning for models that support it') + transpile_parser = subparsers.add_parser('transpile', help='Transpile a HuggingFace model into Cactus component graphs') + transpile_parser.add_argument('model_id', help='HuggingFace model ID to transpile') + + run_transpiled_parser = subparsers.add_parser('run-transpiled', help='Run a saved transpiled component bundle') + run_transpiled_parser.add_argument('bundle_dir', + help='Bundle root directory or manifest.json path from transpile_hf_model.py') + run_transpiled_parser.add_argument('--file', dest='audio_file', default=None, + help='Audio file for transcription bundles such as parakeet-tdt') + run_transpiled_parser.add_argument('--weights-dir', default=None, + help='Optional converted weights directory for bound-weight bundles') + run_transpiled_parser.add_argument('--prompt', default=None, + help='Prompt for causal-LM bundles') + run_transpiled_parser.add_argument('--input-ids', default=None, + help='Comma-separated token ids for causal-LM bundles') + run_transpiled_parser.add_argument('--result-json', default=None, + help='Optional path to save the result payload as JSON') + transcribe_parser = subparsers.add_parser('transcribe', help='Download ASR model and run transcription') transcribe_parser.add_argument('model_id', nargs='?', default=DEFAULT_ASR_MODEL_ID, help=f'HuggingFace model ID (default: {DEFAULT_ASR_MODEL_ID})') @@ -2182,6 +2321,9 @@ def preprocess_eval_args(parser, argv): if getattr(args, 'command', None) == 'eval': setattr(args, 'extra_args', unknown) return args + if getattr(args, 'command', None) == 'transpile': + setattr(args, 'extra_args', unknown) + return args if unknown: parser.error(f"unrecognized arguments: {' '.join(unknown)}") @@ -2202,8 +2344,12 @@ def main(): sys.exit(cmd_build(args)) elif args.command == 'run': sys.exit(cmd_run(args)) + elif args.command == 'transpile': + sys.exit(cmd_transpile(args)) elif args.command == 'transcribe': sys.exit(cmd_transcribe(args)) + elif args.command == 'run-transpiled': + sys.exit(cmd_run_transpiled(args)) elif args.command == 'test': sys.exit(cmd_test(args)) elif args.command == 'eval': diff --git a/python/src/converter.py b/python/src/converter.py index 30720b120..e03e027a8 100644 --- a/python/src/converter.py +++ b/python/src/converter.py @@ -1,3 +1,4 @@ +import json import os import re from pathlib import Path @@ -74,10 +75,23 @@ def _gemma_tower_output_name(hf_key, strip_prefix, add_prefix): return add_prefix + name + ext -def convert_hf_model_weights(model, output_dir, precision='FP16', args=None): +def _gemma_weight_kind(source_name: str) -> str: + if source_name in { + 'model.language_model.embed_tokens_per_layer.weight', + 'model.embed_vision.embedding.weight', + 'model.embed_audio.embedding.weight', + }: + return "embedding" + return "weight" + + +def convert_hf_model_weights(model, output_dir, precision='INT8', args=None): """Convert HuggingFace model weights to Cactus binary format.""" import gc quantization_stats = create_quantization_stats() + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + weights_manifest: dict[str, dict[str, object]] = {} state_dict = model.state_dict() root_config = model.config @@ -138,6 +152,33 @@ def convert_hf_model_weights(model, output_dir, precision='FP16', args=None): print(" Normalized gemma4 audio tower key naming for conversion") state_dict = remapped_state_dict + def save_and_record( + tensor, + output_path, + tensor_precision, + *, + transpose=False, + source_name: str | None = None, + source_names: list[str] | tuple[str, ...] | None = None, + kind: str = "weight", + ): + save_tensor_with_header( + tensor, + output_path, + tensor_precision, + transpose=transpose, + stats_tracker=quantization_stats, + args=args, + model_type=detected_model_type, + ) + names: list[str] = [] + if source_name is not None: + names.append(source_name) + if source_names: + names.extend(str(name) for name in source_names) + for name in names: + weights_manifest[str(name)] = {"filename": Path(output_path).name, "kind": kind} + model_config = extract_base_config(config, root_config) model_config['tie_word_embeddings'] = tie_word_embeddings model_config['model_type'] = detected_model_type @@ -279,7 +320,14 @@ class _Args: for name in EMBED_NAMES: if name in state_dict: embedding_tensor = state_dict[name] - save_tensor_with_header(embedding_tensor, output_dir / "token_embeddings.weights", precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + embedding_tensor, + output_dir / "token_embeddings.weights", + precision, + transpose=False, + source_name=name, + kind="embedding", + ) saved_tensor_full_names.add(name) embedding_found = True break @@ -287,7 +335,14 @@ class _Args: if model_type_str == 'nomic_bert': if 'embeddings.word_embeddings.weight' in state_dict: fused_embedding_tensor = state_dict['embeddings.word_embeddings.weight'] + state_dict.get('embeddings.token_type_embeddings.weight', torch.zeros([1])) - save_tensor_with_header(fused_embedding_tensor, output_dir / "token_embeddings.weights", precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + fused_embedding_tensor, + output_dir / "token_embeddings.weights", + precision, + transpose=False, + source_names=['embeddings.word_embeddings.weight', 'embeddings.token_type_embeddings.weight'], + kind="embedding", + ) saved_tensor_full_names.add('embeddings.word_embeddings.weight') if 'embeddings.token_type_embeddings.weight' in state_dict: saved_tensor_full_names.add('embeddings.token_type_embeddings.weight') @@ -296,7 +351,7 @@ class _Args: elif model_type_str == 'whisper': for name, save_name in WHISPER_GLOBAL_WEIGHTS: if name in state_dict: - save_tensor_with_header(state_dict[name], output_dir / save_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(state_dict[name], output_dir / save_name, precision, transpose=False, source_name=name) saved_tensor_full_names.add(name) embedding_found = True model_config['num_mel_bins'] = int(cfg_get(config, 'num_mel_bins', 80)) @@ -309,8 +364,8 @@ class _Args: tensor = state_dict[name] if name == 'model.encoder.conv2.weight': tensor = tensor.permute(1, 2, 0).contiguous() - - save_tensor_with_header(tensor, output_dir / save_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + + save_and_record(tensor, output_dir / save_name, precision, transpose=False, source_name=name) saved_tensor_full_names.add(name) embedding_found = True model_config['dec_hidden_act'] = config.decoder_hidden_act @@ -330,7 +385,7 @@ class _Args: embedding_norm_names = {'emb_ln.weight': 'embedding_layernorm.weight', 'emb_ln.bias': 'embedding_layernorm.bias'} for name, file_name in embedding_norm_names.items(): if name in state_dict: - save_tensor_with_header(state_dict[name], output_dir / file_name, precision, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(state_dict[name], output_dir / file_name, precision, source_name=name) saved_tensor_full_names.add(name) if tie_word_embeddings: @@ -342,14 +397,14 @@ class _Args: for name in OUTPUT_NAMES: if name in state_dict: tensor = state_dict[name] - save_tensor_with_header(tensor, output_dir / "output_weight.weights", precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / "output_weight.weights", precision, transpose=False, source_name=name) saved_tensor_full_names.add(name) break for name in OUTPUT_NORM_NAMES: if name in state_dict: tensor = state_dict[name] - save_tensor_with_header(tensor, output_dir / "output_norm.weights", precision, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / "output_norm.weights", precision, source_name=name) saved_tensor_full_names.add(name) break @@ -361,10 +416,7 @@ class _Args: ] for key, out_name in mtp_global_mappings: if key in state_dict: - save_tensor_with_header( - state_dict[key], output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type - ) + save_and_record(state_dict[key], output_dir / out_name, precision, transpose=False, source_name=key) saved_tensor_full_names.add(key) mtp_layer_mappings = [ @@ -382,10 +434,7 @@ class _Args: ] for key, out_name in mtp_layer_mappings: if key in state_dict: - save_tensor_with_header( - state_dict[key], output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type - ) + save_and_record(state_dict[key], output_dir / out_name, precision, transpose=False, source_name=key) saved_tensor_full_names.add(key) if is_vlm: @@ -472,26 +521,52 @@ class _Args: for name, save_name in GEMMA4_GLOBAL_WEIGHTS: if name in state_dict: - save_tensor_with_header(state_dict[name], output_dir / save_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + state_dict[name], + output_dir / save_name, + precision, + transpose=False, + source_name=name, + kind=_gemma_weight_kind(name), + ) saved_tensor_full_names.add(name) text_hidden = int(model_config.get('hidden_dim', 0)) assert text_hidden > 0, "Hidden dim must be specified in config for gemma4 model" proj_norm = np.ones(text_hidden, dtype=np.float32) - save_tensor_with_header(proj_norm, output_dir / 'embed_vision_post_proj_norm.weights', 'FP16', - stats_tracker=quantization_stats, model_type=detected_model_type) + save_and_record( + proj_norm, + output_dir / 'embed_vision_post_proj_norm.weights', + 'FP16', + source_names=( + 'model.embed_vision.post_projection_norm.weight', + 'model.embed_vision.embedding_post_projection_norm.weight', + ), + ) for hf_key in sorted(state_dict.keys()): if hf_key.startswith(GEMMA4_VISION_TOWER_PREFIX) and hf_key not in saved_tensor_full_names: out_name = _gemma_tower_output_name(hf_key, GEMMA4_VISION_TOWER_PREFIX, 'vision_') - save_tensor_with_header(state_dict[hf_key], output_dir / out_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + state_dict[hf_key], + output_dir / out_name, + precision, + transpose=False, + source_name=hf_key, + ) saved_tensor_full_names.add(hf_key) del state_dict[hf_key] for hf_key in sorted(state_dict.keys()): if hf_key.startswith(GEMMA4_AUDIO_TOWER_PREFIX) and hf_key not in saved_tensor_full_names: out_name = _gemma_tower_output_name(hf_key, GEMMA4_AUDIO_TOWER_PREFIX, 'audio_') - save_tensor_with_header(state_dict[hf_key], output_dir / out_name, precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record( + state_dict[hf_key], + output_dir / out_name, + precision, + transpose=False, + source_name=hf_key, + ) saved_tensor_full_names.add(hf_key) del state_dict[hf_key] @@ -536,9 +611,12 @@ class _Args: is_hwio_pw = tensor.shape[1] == 1 and tensor.shape[2] == 1 and tensor.shape[3] > 1 if is_hwio_k3 or is_hwio_pw: tensor = tensor.permute(0, 3, 1, 2).contiguous() - save_tensor_with_header( - tensor, output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + tensor, + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) @@ -594,9 +672,12 @@ class _Args: continue missing_tensors.append((i, out_name, [key])) continue - save_tensor_with_header( - state_dict[key], output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + state_dict[key], + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) tracked_key = layer_prefix + 'conv.norm.num_batches_tracked' @@ -638,9 +719,12 @@ class _Args: is_hwio_pw = tensor.shape[1] == 1 and tensor.shape[2] == 1 and tensor.shape[3] > 1 if is_hwio_k3 or is_hwio_pw: tensor = tensor.permute(0, 3, 1, 2).contiguous() - save_tensor_with_header( - tensor, output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + tensor, + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) @@ -660,6 +744,7 @@ class _Args: state_dict[key], output_dir / out_name, "FP16", transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type ) + weights_manifest[key] = {"filename": Path(out_name).name, "kind": "weight"} saved_tensor_full_names.add(key) for i in range(predictor_layers): @@ -673,9 +758,12 @@ class _Args: if key is None: missing_tensors.append((-1, out_name, candidate_keys)) continue - save_tensor_with_header( - state_dict[key], output_dir / out_name, "FP16", transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + state_dict[key], + output_dir / out_name, + "FP16", + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) @@ -692,9 +780,12 @@ class _Args: if key is None: missing_tensors.append((-1, out_name, candidate_keys)) continue - save_tensor_with_header( - state_dict[key], output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + state_dict[key], + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) @@ -845,9 +936,12 @@ class _Args: elif out_suffix == 'conv_depthwise.weights': if hasattr(tensor, 'shape') and len(tensor.shape) == 3 and tensor.shape[1] == 9 and tensor.shape[2] == 1: tensor = tensor.permute(0, 2, 1).contiguous() - save_tensor_with_header( - tensor, output_dir / out_name, precision, transpose=False, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type + save_and_record( + tensor, + output_dir / out_name, + precision, + transpose=False, + source_name=key, ) saved_tensor_full_names.add(key) tracked_keys = [ @@ -892,7 +986,7 @@ class _Args: channel_output_name = output_name.replace('{channel}', str(channel_idx)) tensor = state_dict[full_name] - save_tensor_with_header(tensor, output_dir / channel_output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / channel_output_name, tensor_precision, transpose=should_transpose, source_name=full_name) saved_tensor_full_names.add(full_name) matched_any_channel = True @@ -923,8 +1017,8 @@ class _Args: if 'encoder' in layer_prefix: save_name_prefix = "encoder_" + save_name_prefix - save_tensor_with_header(w_gate, output_dir / (save_name_prefix + "ffn_gate.weights"), precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) - save_tensor_with_header(w_up, output_dir / (save_name_prefix + "ffn_up.weights"), precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(w_gate, output_dir / (save_name_prefix + "ffn_gate.weights"), precision, transpose=should_transpose, source_name=full_name) + save_and_record(w_up, output_dir / (save_name_prefix + "ffn_up.weights"), precision, transpose=should_transpose, source_name=full_name) if b is not None: b_up = b[:half_dim] @@ -947,10 +1041,7 @@ class _Args: v_dim = int(model_config.get('linear_v_proj_dim', 0)) row_dim = int(tensor.shape[0]) - save_tensor_with_header( - tensor, output_dir / output_name, tensor_precision, transpose=should_transpose, - stats_tracker=quantization_stats, args=args, model_type=detected_model_type - ) + save_and_record(tensor, output_dir / output_name, tensor_precision, transpose=should_transpose, source_name=full_name) if q_dim > 0 and k_dim > 0 and v_dim > 0 and row_dim == (q_dim + k_dim + v_dim): q_weight = tensor[:q_dim, :] @@ -983,7 +1074,7 @@ class _Args: raise ValueError(f"Invalid tensor shape: {tensor.shape}") for j, ch in enumerate(['q', 'k', 'v']): channel_output_name = output_name.replace('{channel}', ch) - save_tensor_with_header(tensor[j], output_dir / channel_output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor[j], output_dir / channel_output_name, tensor_precision, transpose=should_transpose, source_name=full_name) saved_tensor_full_names.add(full_name) found = True break @@ -995,18 +1086,18 @@ class _Args: for expert_idx in range(num_experts): expert_tensor = tensor[expert_idx] expert_output_name = output_name.replace('{channel}', str(expert_idx)) - save_tensor_with_header(expert_tensor, output_dir / expert_output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(expert_tensor, output_dir / expert_output_name, tensor_precision, transpose=should_transpose, source_name=full_name) saved_tensor_full_names.add(full_name) found = True break if model_type_str == 'whisper': temp = layer_prefix[:layer_prefix.find('.')] + "." + output_name - save_tensor_with_header(tensor, output_dir / temp, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / temp, tensor_precision, transpose=should_transpose, source_name=full_name) elif model_type_str == 'moonshine' and 'encoder' in layer_prefix: enc_output_name = "encoder_" + output_name - save_tensor_with_header(tensor, output_dir / enc_output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / enc_output_name, tensor_precision, transpose=should_transpose, source_name=full_name) else: - save_tensor_with_header(tensor, output_dir / output_name, tensor_precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(tensor, output_dir / output_name, tensor_precision, transpose=should_transpose, source_name=full_name) saved_tensor_full_names.add(full_name) found = True break @@ -1039,8 +1130,8 @@ class _Args: if 'encoder' in layer_prefix: save_name_prefix = "encoder_" + save_name_prefix - save_tensor_with_header(w_gate, output_dir / (save_name_prefix + "ffn_gate.weights"), precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) - save_tensor_with_header(w_up, output_dir / (save_name_prefix + "ffn_up.weights"), precision, transpose=should_transpose, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(w_gate, output_dir / (save_name_prefix + "ffn_gate.weights"), precision, transpose=should_transpose, source_name=w_name) + save_and_record(w_up, output_dir / (save_name_prefix + "ffn_up.weights"), precision, transpose=should_transpose, source_name=w_name) if b is not None: b_up = b[:half_dim] @@ -1063,9 +1154,9 @@ class _Args: k_weight = combined_weight[:, hidden_size:2*hidden_size] v_weight = combined_weight[:, 2*hidden_size:] - save_tensor_with_header(q_weight, output_dir / f'layer_{i}_attn_q.weights', precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) - save_tensor_with_header(k_weight, output_dir / f'layer_{i}_attn_k.weights', precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) - save_tensor_with_header(v_weight, output_dir / f'layer_{i}_attn_v.weights', precision, transpose=False, stats_tracker=quantization_stats, args=args, model_type=detected_model_type) + save_and_record(q_weight, output_dir / f'layer_{i}_attn_q.weights', precision, transpose=False, source_name=attn_name) + save_and_record(k_weight, output_dir / f'layer_{i}_attn_k.weights', precision, transpose=False, source_name=attn_name) + save_and_record(v_weight, output_dir / f'layer_{i}_attn_v.weights', precision, transpose=False, source_name=attn_name) saved_tensor_full_names.add(attn_name) found = True @@ -1092,6 +1183,10 @@ class _Args: fh.write(f"layer={layer_idx}, output={output_name}, patterns=[{pattern_list}]\n") print(f"Warning: {len(missing_tensors)} tensors were not exported. See {missing_report.name} for details.") + manifest_path = output_dir / "weights_manifest.json" + with open(manifest_path, "w") as fh: + json.dump(dict(sorted(weights_manifest.items())), fh, indent=2, sort_keys=True) + print_quantization_summary(quantization_stats, args) if detected_model_type in ['whisper', 'moonshine', 'parakeet', 'parakeet_tdt']: @@ -1340,7 +1435,7 @@ def _build_needle_config(model_cfg, params, precision): hidden_dim = int(model_cfg["d_model"]) heads = int(model_cfg["num_heads"]) decoder_layers = int(model_cfg["num_decoder_layers"]) - compute_precision = "FP16" + compute_precision = "FP16" if precision in ("INT4", "INT8") else precision return { "model_type": "needle", "precision": compute_precision, diff --git a/python/src/graph.py b/python/src/graph.py index 52f706c6e..541c74608 100644 --- a/python/src/graph.py +++ b/python/src/graph.py @@ -1,17 +1,15 @@ import ctypes +from os import wait import numpy as np -from .cactus import _lib, cactus_node_t, cactus_tensor_info_t +from .cactus import _err, _lib, cactus_node_t, cactus_tensor_info_t class Graph: INT8 = 0 FP16 = 1 FP32 = 2 - CQ1 = 3 - CQ2 = 4 - CQ3 = 5 - CQ4 = 6 + INT4 = 3 CPU = 0 NPU = 1 ACT_SILU = 0 @@ -52,7 +50,7 @@ def input(self, shape, dtype=FP16): out = cactus_node_t() rc = _lib.cactus_graph_input(self.h, arr, len(shape), int(dtype), ctypes.byref(out)) if rc != 0: - raise RuntimeError("graph_input failed") + raise RuntimeError(_err("graph_input failed")) return self._tensor_from_node(out.value) def set_input(self, tensor, data, dtype=None): @@ -62,6 +60,25 @@ def set_input(self, tensor, data, dtype=None): raise ValueError("tensor belongs to a different graph") target_dtype = int(tensor.dtype if dtype is None else dtype) arr = self._coerce_input_array(data, target_dtype) + info = self._get_output_info(tensor.id) + expected_shape = tuple(int(x) for x in info["shape"]) + expected_num_elements = int(info["num_elements"]) + expected_byte_size = int(info["byte_size"]) + if tuple(int(x) for x in arr.shape) != expected_shape: + raise ValueError( + "graph input shape mismatch for node " + f"{tensor.id}: expected {expected_shape}, got {tuple(int(x) for x in arr.shape)}" + ) + if int(arr.size) != expected_num_elements: + raise ValueError( + "graph input element-count mismatch for node " + f"{tensor.id}: expected {expected_num_elements}, got {int(arr.size)}" + ) + if int(arr.nbytes) != expected_byte_size: + raise ValueError( + "graph input byte-size mismatch for node " + f"{tensor.id}: expected {expected_byte_size}, got {int(arr.nbytes)}" + ) rc = _lib.cactus_graph_set_input( self.h, cactus_node_t(tensor.id), @@ -69,7 +86,7 @@ def set_input(self, tensor, data, dtype=None): target_dtype, ) if rc != 0: - raise RuntimeError("graph_set_input failed") + raise RuntimeError(_err("graph_set_input failed")) def set_external_input(self, tensor, data_ptr, dtype=None): if not isinstance(tensor, Tensor): @@ -85,7 +102,7 @@ def set_external_input(self, tensor, data_ptr, dtype=None): target_dtype, ) if rc != 0: - raise RuntimeError("graph_set_external_input failed") + raise RuntimeError(_err("graph_set_external_input failed")) def hard_reset(self): rc = _lib.cactus_graph_hard_reset(self.h) @@ -95,7 +112,7 @@ def hard_reset(self): def execute(self): rc = _lib.cactus_graph_execute(self.h) if rc != 0: - raise RuntimeError("graph_execute failed") + raise RuntimeError(_err("graph_execute failed")) def add(self, a, b): return self._binary("cactus_graph_add", a, b) @@ -112,6 +129,9 @@ def multiply(self, a, b): def divide(self, a, b): return self._binary("cactus_graph_divide", a, b) + def not_equal(self, a, b): + return self._binary("cactus_graph_not_equal", a, b) + def abs(self, x): x = self._ensure_tensor(x) out = cactus_node_t() @@ -136,6 +156,13 @@ def precision_cast(self, x, dtype): raise RuntimeError("graph_precision_cast failed") return self._tensor_from_node(out.value) + def quantize_activations(self, x): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_quantize_activations(self.h, cactus_node_t(x.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_quantize_activations failed") + return self._tensor_from_node(out.value) def _scalar(self, fn_name, x, value=None): x = self._ensure_tensor(x) @@ -161,6 +188,12 @@ def scalar_multiply(self, x, value): def scalar_divide(self, x, value): return self._scalar("cactus_graph_scalar_divide", x, value) + def scalar_floor_divide(self, x, value): + return self._scalar("cactus_graph_scalar_floor_divide", x, value) + + def scalar_not_equal(self, x, value): + return self._scalar("cactus_graph_scalar_not_equal", x, value) + def scalar_exp(self, x): return self._scalar("cactus_graph_scalar_exp", x) @@ -176,6 +209,25 @@ def scalar_sin(self, x): def scalar_log(self, x): return self._scalar("cactus_graph_scalar_log", x) + def masked_select_prefix(self, x, mask): + return self._binary("cactus_graph_masked_select_prefix", x, mask) + + def masked_scatter(self, x, mask, source): + x = self._ensure_tensor(x) + mask = self._ensure_tensor(mask) + source = self._ensure_tensor(source) + out = cactus_node_t() + rc = _lib.cactus_graph_masked_scatter( + self.h, + cactus_node_t(x.id), + cactus_node_t(mask.id), + cactus_node_t(source.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_masked_scatter failed")) + return self._tensor_from_node(out.value) + def view(self, x, shape): x = self._ensure_tensor(x) shape = tuple(int(v) for v in shape) @@ -196,6 +248,16 @@ def reshape(self, x, shape): raise RuntimeError("graph_reshape failed") return self._tensor_from_node(out.value) + def expand(self, x, shape): + x = self._ensure_tensor(x) + shape = tuple(int(v) for v in shape) + arr = (ctypes.c_size_t * len(shape))(*shape) + out = cactus_node_t() + rc = _lib.cactus_graph_expand(self.h, cactus_node_t(x.id), arr, len(shape), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_expand failed") + return self._tensor_from_node(out.value) + def flatten(self, x, start_dim=0, end_dim=-1): x = self._ensure_tensor(x) out = cactus_node_t() @@ -269,27 +331,35 @@ def permute(self, x, permutation, backend=CPU): raise RuntimeError("graph_transpose_n failed") return self._tensor_from_node(out.value) - def matmul(self, a, b, pretransposed_rhs=False, backend=CPU): + def matmul(self, a, b, pretransposed_rhs=False, backend=CPU, output_dtype=None): a = self._ensure_tensor(a) b = self._ensure_tensor(b) out = cactus_node_t() + target_dtype = int(a.dtype if output_dtype is None else output_dtype) rc = _lib.cactus_graph_matmul( self.h, cactus_node_t(a.id), cactus_node_t(b.id), ctypes.c_bool(bool(pretransposed_rhs)), ctypes.c_int32(int(backend)), + ctypes.c_int32(target_dtype), ctypes.byref(out), ) if rc != 0: - raise RuntimeError("graph_matmul failed") + raise RuntimeError(_err("graph_matmul failed")) return self._tensor_from_node(out.value) - def gather(self, tensor, indices): + def gather(self, tensor, indices, axis=0): tensor = self._ensure_tensor(tensor) indices = self._ensure_tensor(indices) out = cactus_node_t() - rc = _lib.cactus_graph_gather(self.h, cactus_node_t(tensor.id), cactus_node_t(indices.id), ctypes.byref(out)) + rc = _lib.cactus_graph_gather( + self.h, + cactus_node_t(tensor.id), + cactus_node_t(indices.id), + ctypes.c_int32(int(axis)), + ctypes.byref(out), + ) if rc != 0: raise RuntimeError("graph_gather failed") return self._tensor_from_node(out.value) @@ -337,6 +407,27 @@ def bilinear_interpolation(self, pos_embeds, dst_height, dst_width): raise RuntimeError("graph_bilinear_interpolation failed") return self._tensor_from_node(out.value) + def set_grouped_scales(self, tensor, group_size, num_groups, scales): + tensor = self._ensure_tensor(tensor) + arr = np.ascontiguousarray(scales, dtype=np.float16) + rc = _lib.cactus_graph_set_grouped_scales( + self.h, + cactus_node_t(tensor.id), + ctypes.c_size_t(int(group_size)), + ctypes.c_size_t(int(num_groups)), + arr.ctypes.data_as(ctypes.c_void_p), + ) + if rc != 0: + raise RuntimeError("graph_set_grouped_scales failed") + + def set_interleaved(self, tensor, interleaved=True, original_n=0): + tensor = self._ensure_tensor(tensor) + rc = _lib.cactus_graph_set_interleaved( + self.h, cactus_node_t(tensor.id), ctypes.c_bool(bool(interleaved)), ctypes.c_size_t(int(original_n)) + ) + if rc != 0: + raise RuntimeError("graph_set_interleaved failed") + def release_weight_pages(self, tensor): tensor = self._ensure_tensor(tensor) rc = _lib.cactus_graph_release_weight_pages(self.h, cactus_node_t(tensor.id)) @@ -373,18 +464,13 @@ def cat(self, tensors, axis=0): tensors = [self._ensure_tensor(t) for t in tensors] if not tensors: raise ValueError("cat requires at least one tensor") - ids = (cactus_node_t * len(tensors))(*(cactus_node_t(t.id) for t in tensors)) - out = cactus_node_t() - rc = _lib.cactus_graph_cat( - self.h, - ids, - ctypes.c_size_t(len(tensors)), - ctypes.c_int32(int(axis)), - ctypes.byref(out), - ) - if rc != 0: - raise RuntimeError("graph_cat failed") - return self._tensor_from_node(out.value) + if len(tensors) == 1: + return tensors[0] + + result = tensors[0] + for tensor in tensors[1:]: + result = self.concat(result, tensor, axis=axis) + return result def groupnorm(self, x, weight, bias, num_groups, eps=1e-5): x = self._ensure_tensor(x) @@ -524,6 +610,9 @@ def min(self, x, axis): def max(self, x, axis): return self._reduce("cactus_graph_max", x, axis) + + def cumsum(self, x, axis): + return self._reduce("cactus_graph_cumsum", x, axis) def softmax(self, x, axis=-1): x = self._ensure_tensor(x) @@ -719,6 +808,21 @@ def conv2d_depthwise_k3s2p1(self, x, weight, bias=None): def conv2d_pointwise_1x1(self, x, weight, bias=None): return self._conv_with_optional_bias("cactus_graph_conv2d_pointwise_1x1", x, weight, bias) + def conv2d_k3s1p1(self, x, weight, bias=None): + return self._conv_with_optional_bias("cactus_graph_conv2d_k3s1p1", x, weight, bias) + + def conv2d(self, x, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + return self._conv_with_optional_bias( + "cactus_graph_conv2d", + x, + weight, + bias, + ctypes.c_size_t(int(stride)), + ctypes.c_size_t(int(padding)), + ctypes.c_size_t(int(dilation)), + ctypes.c_size_t(int(groups)), + ) + def _conv_with_optional_bias(self, fn_name, x, weight, bias=None, *extra): x = self._ensure_tensor(x) weight = self._ensure_tensor(weight) @@ -934,6 +1038,8 @@ def _coerce_input_array(self, data, precision): arr = np.ascontiguousarray(arr, dtype=np.float16) elif precision == self.FP32: arr = np.ascontiguousarray(arr, dtype=np.float32) + elif precision == self.INT4: + arr = np.ascontiguousarray(arr, dtype=np.uint8) else: raise RuntimeError("unsupported precision") return arr @@ -958,6 +1064,11 @@ def __mul__(self, other): def __truediv__(self, other): return self.g.divide(self, other) + def __ne__(self, other): + if isinstance(other, Tensor): + return self.g.not_equal(self, other) + return self.g.scalar_not_equal(self, other) + def abs(self): return self.g.abs(self) @@ -967,6 +1078,8 @@ def pow(self, exponent): def precision_cast(self, dtype): return self.g.precision_cast(self, dtype) + def quantize_activations(self): + return self.g.quantize_activations(self) def scalar_add(self, value): return self.g.scalar_add(self, value) @@ -980,6 +1093,12 @@ def scalar_multiply(self, value): def scalar_divide(self, value): return self.g.scalar_divide(self, value) + def scalar_floor_divide(self, value): + return self.g.scalar_floor_divide(self, value) + + def scalar_not_equal(self, value): + return self.g.scalar_not_equal(self, value) + def scalar_exp(self): return self.g.scalar_exp(self) @@ -1019,6 +1138,9 @@ def view(self, shape): def reshape(self, shape): return self.g.reshape(self, shape) + def expand(self, shape): + return self.g.expand(self, shape) + def flatten(self, start_dim=0, end_dim=-1): return self.g.flatten(self, start_dim=start_dim, end_dim=end_dim) @@ -1067,8 +1189,14 @@ def softmax(self, axis=-1): def glu(self, axis=-1): return self.g.glu(self, axis=axis) - def matmul(self, other, pretransposed_rhs=False, backend=Graph.CPU): - return self.g.matmul(self, other, pretransposed_rhs=pretransposed_rhs, backend=backend) + def matmul(self, other, pretransposed_rhs=False, backend=Graph.CPU, output_dtype=None): + return self.g.matmul( + self, + other, + pretransposed_rhs=pretransposed_rhs, + backend=backend, + output_dtype=output_dtype, + ) def sum(self, axis): return self.g.sum(self, axis) @@ -1085,6 +1213,9 @@ def min(self, axis): def max(self, axis): return self.g.max(self, axis) + def cumsum(self, axis): + return self.g.cumsum(self, axis) + def numpy(self): info = cactus_tensor_info_t() rc = _lib.cactus_graph_get_output_info(self.g.h, cactus_node_t(self.id), ctypes.byref(info)) @@ -1107,6 +1238,9 @@ def numpy(self): arr = np.ctypeslib.as_array((ctypes.c_float * num_elements).from_address(out_ptr.value)) elif precision == Graph.INT8: arr = np.ctypeslib.as_array((ctypes.c_int8 * num_elements).from_address(out_ptr.value)) + elif precision == Graph.INT4: + arr = np.ctypeslib.as_array((ctypes.c_uint8 * int(info.byte_size)).from_address(out_ptr.value)) + return arr.copy() else: raise RuntimeError("unsupported precision") diff --git a/python/src/tensor_io.py b/python/src/tensor_io.py index 25512269a..726903482 100644 --- a/python/src/tensor_io.py +++ b/python/src/tensor_io.py @@ -2,6 +2,7 @@ import struct from pathlib import Path from typing import Optional, Dict, Any, List +from .weight_patterns import EMBED_NAMES try: import torch @@ -9,11 +10,16 @@ torch = None +GROUP_SIZE = 32 +INTERLEAVE_BLOCK = 4 + CACTUS_MAGIC = b'CACT' CACTUS_ALIGNMENT = 32 +FLAG_HAS_SCALES = 1 << 0 FLAG_PAGE_ALIGNED = 1 << 1 FLAG_TRANSPOSED = 1 << 2 +FLAG_INTERLEAVED = 1 << 3 def align_offset(offset: int, alignment: int) -> int: @@ -85,6 +91,25 @@ def interleave_scales(scales: np.ndarray, block_size: int = INTERLEAVE_BLOCK) -> return scales.reshape(-1), original_N +def pack_int4_pairs(data: np.ndarray) -> np.ndarray: + """Pack INT4 values (stored as int8) into bytes using planar layout. + + Input: array of int8 values in range [-8, 7], length must be a multiple of 32 + Output: array of uint8 with half the length + + Packing format (planar, groups of 32): + For each group of 32 values, the first 16 are stored in the low nibbles + and the next 16 in the high nibbles of 16 consecutive bytes. + Nibbles are stored in two's complement form (value & 0x0F). + """ + assert len(data) % 32 == 0, "Data length must be a multiple of 32 for INT4 planar packing" + + groups = data.reshape(-1, 32) + low = (groups[:, :16].astype(np.int8).view(np.uint8) & 0x0F).astype(np.uint8) + high = ((groups[:, 16:].astype(np.int8).view(np.uint8) & 0x0F).astype(np.uint8)) << 4 + + return (low | high).astype(np.uint8).reshape(-1) + def fold_bn_into_conv(conv_w, bn_weight, bn_bias, bn_mean, bn_var, eps=1e-5): gamma = bn_weight.float().numpy() @@ -101,13 +126,18 @@ def fold_bn_into_conv(conv_w, bn_weight, bn_bias, bn_mean, bn_var, eps=1e-5): return new_w, new_b -def save_tensor_with_header(tensor, output_path, precision='FP16', transpose=False, stats_tracker=None, args=None, model_type=None): - """Save a tensor to binary format with header metadata. +def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=False, stats_tracker=None, args=None, model_type=None): + """Save a tensor to binary format with header metadata and group-wise quantization. + + For 2D tensors with INT8/INT4 precision: + - INT4 is packed (2 values per byte) for 50% storage savings, unpacked to INT8 at load time + - Weights are interleaved in blocks of 4 rows for SIMD efficiency + - Layout: [N/4, K/4, 4, 4] enables vdotq_laneq_s32 for efficient GEMV/GEMM Args: tensor: The tensor to save (PyTorch or NumPy) output_path: Path to save the tensor - precision: Storage precision ('FP16') + precision: Quantization precision ('INT4', 'INT8', 'FP16') transpose: Whether to transpose 2D tensors stats_tracker: Optional dict to track quantization statistics args: Optional args object with additional settings @@ -141,6 +171,21 @@ def save_tensor_with_header(tensor, output_path, precision='FP16', transpose=Fal elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE + if precision in ('INT8', 'INT4'): + filename = output_path.name + if any(x in filename for x in ['norm', 'bias', 'vision', 'position_embeddings', 'embed_positions']): + precision = 'FP16' + elif model_type == 'gemma4' and ( + filename.startswith('audio_') or + filename in ('embed_audio_proj.weights', 'embed_audio_embedding.weights') + ): + precision = 'FP16' + elif precision == 'INT4' and ( + any(x in filename for x in EMBED_NAMES) + or filename in ('token_embeddings.weights', 'embed_tokens_per_layer.weights') + ): + precision = 'INT8' + shape = list(data.shape) if transpose and len(shape) == 2: data = data.T @@ -155,6 +200,253 @@ def save_tensor_with_header(tensor, output_path, precision='FP16', transpose=Fal data = data.transpose(0, 2, 1) shape = [shape[0], shape[2], shape[1]] + if precision == 'INT8' and len(shape) == 2: + N, K = shape + original_N = N + + if K % GROUP_SIZE != 0: + pad_k = GROUP_SIZE - (K % GROUP_SIZE) + data = np.pad(data, ((0, 0), (0, pad_k)), mode='constant', constant_values=0) + K = data.shape[1] + + num_groups = K // GROUP_SIZE + + data_grouped = data.reshape(N, num_groups, GROUP_SIZE) + group_abs_max = np.max(np.abs(data_grouped), axis=2) + scales = (group_abs_max / 127.0).astype(np.float32) + scales = np.maximum(scales, 1e-10) + + quantized = np.clip( + np.round(data_grouped / scales[:, :, np.newaxis]), + -128, 127 + ).astype(np.int8) + quantized_2d = quantized.reshape(N, K) + + dequantized = (quantized.astype(np.float32) * scales[:, :, np.newaxis]).reshape(N, K) + mse_error = np.mean((data[:original_N, :] - dequantized[:original_N, :]) ** 2) + snr_db = 10 * np.log10(np.var(data[:original_N, :]) / mse_error) if mse_error > 0 else float('inf') + data_flat = data[:original_N, :].flatten() + dequant_flat = dequantized[:original_N, :].flatten() + cos_sim = np.dot(data_flat, dequant_flat) / (np.linalg.norm(data_flat) * np.linalg.norm(dequant_flat) + 1e-10) + del data_flat, dequantized + + quantized_interleaved, _ = interleave_weights(quantized_2d, INTERLEAVE_BLOCK) + scales_interleaved, _ = interleave_scales(scales, INTERLEAVE_BLOCK) + scales_fp16 = scales_interleaved.astype(np.float16) + + N_padded = ((N + INTERLEAVE_BLOCK - 1) // INTERLEAVE_BLOCK) * INTERLEAVE_BLOCK + + if stats_tracker: + stats_tracker['int8_tensors'] += 1 + stats_tracker['quantized_parameters'] += original_N * K + stats_tracker['mse_values'].append(mse_error) + stats_tracker['snr_values'].append(snr_db) + stats_tracker['cos_sim_values'].append(cos_sim) + stats_tracker['total_tensors'] += 1 + stats_tracker['total_parameters'] += original_N * K + + del data + + with open(output_path, 'wb') as f: + ndim = 2 + data_bytes = quantized_interleaved.size + scales_bytes = scales_fp16.size * 2 + flags = FLAG_HAS_SCALES | FLAG_INTERLEAVED + if transpose: + flags |= FLAG_TRANSPOSED + + f.write(CACTUS_MAGIC) # 4 bytes + f.write(struct.pack(' 0 else float('inf') + data_flat = data[:original_N, :].flatten() + dequant_flat = dequantized[:original_N, :].flatten() + cos_sim = np.dot(data_flat, dequant_flat) / (np.linalg.norm(data_flat) * np.linalg.norm(dequant_flat) + 1e-10) + del data_flat, dequantized + + quantized_interleaved, _ = interleave_weights(quantized_2d, INTERLEAVE_BLOCK) + scales_interleaved, _ = interleave_scales(scales, INTERLEAVE_BLOCK) + scales_fp16 = scales_interleaved.astype(np.float16) + + packed_data = pack_int4_pairs(quantized_interleaved) + + N_padded = ((N + INTERLEAVE_BLOCK - 1) // INTERLEAVE_BLOCK) * INTERLEAVE_BLOCK + + if stats_tracker: + stats_tracker['int4_tensors'] += 1 + stats_tracker['quantized_parameters'] += original_N * K + stats_tracker['mse_values'].append(mse_error) + stats_tracker['snr_values'].append(snr_db) + stats_tracker['cos_sim_values'].append(cos_sim) + stats_tracker['total_tensors'] += 1 + stats_tracker['total_parameters'] += original_N * K + + del data + + with open(output_path, 'wb') as f: + ndim = 2 + data_bytes = packed_data.size + scales_bytes = scales_fp16.size * 2 + flags = FLAG_HAS_SCALES | FLAG_INTERLEAVED + if transpose: + flags |= FLAG_TRANSPOSED + + f.write(CACTUS_MAGIC) # 4 bytes + f.write(struct.pack(' 0 else float('inf') + cos_sim = np.dot(data, dequantized) / (np.linalg.norm(data) * np.linalg.norm(dequantized) + 1e-10) + del dequantized + + scales_fp16 = scales.flatten().astype(np.float16) + + if stats_tracker: + stats_tracker['int8_tensors'] += 1 + stats_tracker['quantized_parameters'] += data.size + stats_tracker['mse_values'].append(mse_error) + stats_tracker['snr_values'].append(snr_db) + stats_tracker['cos_sim_values'].append(cos_sim) + stats_tracker['total_tensors'] += 1 + stats_tracker['total_parameters'] += data.size + + with open(output_path, 'wb') as f: + ndim = len(shape) + data_bytes = quantized_flat.size + scales_bytes = scales_fp16.size * 2 + flags = FLAG_HAS_SCALES + if transpose: + flags |= FLAG_TRANSPOSED + + f.write(CACTUS_MAGIC) + f.write(struct.pack(' 0: - print(f"\nConversion Summary: {fp16_count} FP16 tensors, {params} total parameters") + quantized_count = int8_count + int4_count + + if quantized_count > 0: + mse_values = np.array(quantization_stats['mse_values']) + snr_values = np.array(quantization_stats['snr_values']) + cos_sim_values = np.array(quantization_stats['cos_sim_values']) + + print("\nQuantization Summary:") + print(f"MSE - Mean: {np.mean(mse_values):.2e}, Max: {np.max(mse_values):.2e}, Median: {np.median(mse_values):.2e}, Min: {np.min(mse_values):.2e}") + print(f"SNR - Mean: {np.mean(snr_values):.1f}dB, Max: {np.max(snr_values):.1f}dB, Median: {np.median(snr_values):.1f}dB, Min: {np.min(snr_values):.1f}dB") + print(f"CosSim - Mean: {np.mean(cos_sim_values):.6f}, Max: {np.max(cos_sim_values):.6f}, Median: {np.median(cos_sim_values):.6f}, Min: {np.min(cos_sim_values):.6f}") + print(f"Processed {int8_count} INT8 tensors, {int4_count} INT4 tensors, {fp16_count} FP16 tensors") From 487b5c29c244b79d56c4415d428ec72cf9e39ef2 Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Tue, 12 May 2026 09:14:48 -0700 Subject: [PATCH 06/11] Removed debug files --- cactus-transpile/CMakeLists.txt | 211 -- cactus-transpile/cactus.h | 12 - cactus-transpile/engine/engine.h | 1079 ------- cactus-transpile/engine/engine_bpe.cpp | 538 ---- cactus-transpile/engine/engine_cache.cpp | 691 ----- .../engine/engine_constraints.cpp | 1137 ------- cactus-transpile/engine/engine_image.cpp | 600 ---- cactus-transpile/engine/engine_index.cpp | 780 ----- cactus-transpile/engine/engine_model.cpp | 1098 ------- cactus-transpile/engine/engine_signals.cpp | 775 ----- cactus-transpile/engine/engine_sp.cpp | 485 --- cactus-transpile/engine/engine_tokenizer.cpp | 866 ------ cactus-transpile/ffi/cactus_cloud.cpp | 452 --- cactus-transpile/ffi/cactus_cloud.h | 48 - cactus-transpile/ffi/cactus_complete.cpp | 1177 ------- cactus-transpile/ffi/cactus_diarize.cpp | 200 -- cactus-transpile/ffi/cactus_embed.cpp | 273 -- cactus-transpile/ffi/cactus_embed_speaker.cpp | 125 - cactus-transpile/ffi/cactus_ffi.h | 489 --- cactus-transpile/ffi/cactus_graph.cpp | 1432 --------- cactus-transpile/ffi/cactus_index.cpp | 458 --- cactus-transpile/ffi/cactus_init.cpp | 489 --- cactus-transpile/ffi/cactus_log.cpp | 23 - cactus-transpile/ffi/cactus_rag.cpp | 459 --- cactus-transpile/ffi/cactus_stream.cpp | 2714 ---------------- cactus-transpile/ffi/cactus_telemetry.cpp | 22 - cactus-transpile/ffi/cactus_transcribe.cpp | 829 ----- cactus-transpile/ffi/cactus_utils.h | 1850 ----------- cactus-transpile/ffi/cactus_vad.cpp | 176 -- cactus-transpile/ffi/gemma_tools.h | 576 ---- cactus-transpile/graph/graph.h | 805 ----- cactus-transpile/graph/graph_builder.cpp | 1879 ----------- cactus-transpile/graph/graph_core.cpp | 265 -- cactus-transpile/graph/graph_execute.cpp | 948 ------ cactus-transpile/graph/graph_io.cpp | 809 ----- cactus-transpile/graph/graph_ops_math.cpp | 737 ----- cactus-transpile/graph/graph_ops_nn.cpp | 2754 ----------------- cactus-transpile/graph/graph_ops_sample.cpp | 134 - cactus-transpile/graph/graph_ops_tensor.cpp | 813 ----- cactus-transpile/graph/graph_param_io.cpp | 394 --- cactus-transpile/graph/graph_param_io.h | 11 - cactus-transpile/kernel/kernel.h | 488 --- cactus-transpile/kernel/kernel_altup.cpp | 180 -- cactus-transpile/kernel/kernel_attention.cpp | 1365 -------- cactus-transpile/kernel/kernel_bilstm.cpp | 370 --- cactus-transpile/kernel/kernel_blas.cpp | 639 ---- cactus-transpile/kernel/kernel_conv.cpp | 1663 ---------- cactus-transpile/kernel/kernel_deltanet.cpp | 677 ---- cactus-transpile/kernel/kernel_i8mm.cpp | 189 -- cactus-transpile/kernel/kernel_lstm.cpp | 117 - cactus-transpile/kernel/kernel_matmul.cpp | 787 ----- cactus-transpile/kernel/kernel_nn.cpp | 874 ------ cactus-transpile/kernel/kernel_quants.cpp | 312 -- cactus-transpile/kernel/kernel_reduce.cpp | 430 --- cactus-transpile/kernel/kernel_scalar.cpp | 119 - cactus-transpile/kernel/kernel_sme2.cpp | 616 ---- cactus-transpile/kernel/kernel_utils.h | 724 ----- .../models/gemma4/model_gemma4.cpp | 594 ---- cactus-transpile/models/gemma4/model_gemma4.h | 404 --- .../models/gemma4/model_gemma4_audio.cpp | 777 ----- .../models/gemma4/model_gemma4_mm.cpp | 459 --- .../models/gemma4/model_gemma4_vision.cpp | 540 ---- cactus-transpile/models/model.h | 1773 ----------- cactus-transpile/models/model_gemma.cpp | 192 -- cactus-transpile/models/model_gemma3n.cpp | 475 --- cactus-transpile/models/model_lfm2.cpp | 389 --- cactus-transpile/models/model_lfm2moe.cpp | 456 --- cactus-transpile/models/model_lfm2vl.cpp | 607 ---- cactus-transpile/models/model_moonshine.cpp | 596 ---- cactus-transpile/models/model_needle.cpp | 337 -- cactus-transpile/models/model_nomic.cpp | 201 -- cactus-transpile/models/model_parakeet.cpp | 860 ----- .../models/model_parakeet_tdt.cpp | 1107 ------- cactus-transpile/models/model_pyannote.cpp | 201 -- cactus-transpile/models/model_qwen.cpp | 171 - cactus-transpile/models/model_qwen3p5.cpp | 771 ----- cactus-transpile/models/model_siglip2.cpp | 387 --- cactus-transpile/models/model_silero_vad.cpp | 338 -- cactus-transpile/models/model_wespeaker.cpp | 204 -- cactus-transpile/models/model_whisper.cpp | 778 ----- cactus-transpile/models/model_youtu.cpp | 246 -- cactus-transpile/npu/npu.cpp | 22 - cactus-transpile/npu/npu.h | 89 - cactus-transpile/npu/npu_ane.h | 154 - cactus-transpile/npu/npu_ane.mm | 1305 -------- cactus-transpile/telemetry/telemetry.cpp | 1546 --------- cactus-transpile/telemetry/telemetry.h | 44 - 87 files changed, 55186 deletions(-) delete mode 100644 cactus-transpile/CMakeLists.txt delete mode 100644 cactus-transpile/cactus.h delete mode 100644 cactus-transpile/engine/engine.h delete mode 100644 cactus-transpile/engine/engine_bpe.cpp delete mode 100644 cactus-transpile/engine/engine_cache.cpp delete mode 100644 cactus-transpile/engine/engine_constraints.cpp delete mode 100644 cactus-transpile/engine/engine_image.cpp delete mode 100644 cactus-transpile/engine/engine_index.cpp delete mode 100644 cactus-transpile/engine/engine_model.cpp delete mode 100644 cactus-transpile/engine/engine_signals.cpp delete mode 100644 cactus-transpile/engine/engine_sp.cpp delete mode 100644 cactus-transpile/engine/engine_tokenizer.cpp delete mode 100644 cactus-transpile/ffi/cactus_cloud.cpp delete mode 100644 cactus-transpile/ffi/cactus_cloud.h delete mode 100644 cactus-transpile/ffi/cactus_complete.cpp delete mode 100644 cactus-transpile/ffi/cactus_diarize.cpp delete mode 100644 cactus-transpile/ffi/cactus_embed.cpp delete mode 100644 cactus-transpile/ffi/cactus_embed_speaker.cpp delete mode 100644 cactus-transpile/ffi/cactus_ffi.h delete mode 100644 cactus-transpile/ffi/cactus_graph.cpp delete mode 100644 cactus-transpile/ffi/cactus_index.cpp delete mode 100644 cactus-transpile/ffi/cactus_init.cpp delete mode 100644 cactus-transpile/ffi/cactus_log.cpp delete mode 100644 cactus-transpile/ffi/cactus_rag.cpp delete mode 100644 cactus-transpile/ffi/cactus_stream.cpp delete mode 100644 cactus-transpile/ffi/cactus_telemetry.cpp delete mode 100644 cactus-transpile/ffi/cactus_transcribe.cpp delete mode 100644 cactus-transpile/ffi/cactus_utils.h delete mode 100644 cactus-transpile/ffi/cactus_vad.cpp delete mode 100644 cactus-transpile/ffi/gemma_tools.h delete mode 100644 cactus-transpile/graph/graph.h delete mode 100644 cactus-transpile/graph/graph_builder.cpp delete mode 100644 cactus-transpile/graph/graph_core.cpp delete mode 100644 cactus-transpile/graph/graph_execute.cpp delete mode 100644 cactus-transpile/graph/graph_io.cpp delete mode 100644 cactus-transpile/graph/graph_ops_math.cpp delete mode 100644 cactus-transpile/graph/graph_ops_nn.cpp delete mode 100644 cactus-transpile/graph/graph_ops_sample.cpp delete mode 100644 cactus-transpile/graph/graph_ops_tensor.cpp delete mode 100644 cactus-transpile/graph/graph_param_io.cpp delete mode 100644 cactus-transpile/graph/graph_param_io.h delete mode 100644 cactus-transpile/kernel/kernel.h delete mode 100644 cactus-transpile/kernel/kernel_altup.cpp delete mode 100644 cactus-transpile/kernel/kernel_attention.cpp delete mode 100644 cactus-transpile/kernel/kernel_bilstm.cpp delete mode 100644 cactus-transpile/kernel/kernel_blas.cpp delete mode 100644 cactus-transpile/kernel/kernel_conv.cpp delete mode 100644 cactus-transpile/kernel/kernel_deltanet.cpp delete mode 100644 cactus-transpile/kernel/kernel_i8mm.cpp delete mode 100644 cactus-transpile/kernel/kernel_lstm.cpp delete mode 100644 cactus-transpile/kernel/kernel_matmul.cpp delete mode 100644 cactus-transpile/kernel/kernel_nn.cpp delete mode 100644 cactus-transpile/kernel/kernel_quants.cpp delete mode 100644 cactus-transpile/kernel/kernel_reduce.cpp delete mode 100644 cactus-transpile/kernel/kernel_scalar.cpp delete mode 100644 cactus-transpile/kernel/kernel_sme2.cpp delete mode 100644 cactus-transpile/kernel/kernel_utils.h delete mode 100644 cactus-transpile/models/gemma4/model_gemma4.cpp delete mode 100644 cactus-transpile/models/gemma4/model_gemma4.h delete mode 100644 cactus-transpile/models/gemma4/model_gemma4_audio.cpp delete mode 100644 cactus-transpile/models/gemma4/model_gemma4_mm.cpp delete mode 100644 cactus-transpile/models/gemma4/model_gemma4_vision.cpp delete mode 100644 cactus-transpile/models/model.h delete mode 100644 cactus-transpile/models/model_gemma.cpp delete mode 100644 cactus-transpile/models/model_gemma3n.cpp delete mode 100644 cactus-transpile/models/model_lfm2.cpp delete mode 100644 cactus-transpile/models/model_lfm2moe.cpp delete mode 100644 cactus-transpile/models/model_lfm2vl.cpp delete mode 100644 cactus-transpile/models/model_moonshine.cpp delete mode 100644 cactus-transpile/models/model_needle.cpp delete mode 100644 cactus-transpile/models/model_nomic.cpp delete mode 100644 cactus-transpile/models/model_parakeet.cpp delete mode 100644 cactus-transpile/models/model_parakeet_tdt.cpp delete mode 100644 cactus-transpile/models/model_pyannote.cpp delete mode 100644 cactus-transpile/models/model_qwen.cpp delete mode 100644 cactus-transpile/models/model_qwen3p5.cpp delete mode 100644 cactus-transpile/models/model_siglip2.cpp delete mode 100644 cactus-transpile/models/model_silero_vad.cpp delete mode 100644 cactus-transpile/models/model_wespeaker.cpp delete mode 100644 cactus-transpile/models/model_whisper.cpp delete mode 100644 cactus-transpile/models/model_youtu.cpp delete mode 100644 cactus-transpile/npu/npu.cpp delete mode 100644 cactus-transpile/npu/npu.h delete mode 100644 cactus-transpile/npu/npu_ane.h delete mode 100644 cactus-transpile/npu/npu_ane.mm delete mode 100644 cactus-transpile/telemetry/telemetry.cpp delete mode 100644 cactus-transpile/telemetry/telemetry.h diff --git a/cactus-transpile/CMakeLists.txt b/cactus-transpile/CMakeLists.txt deleted file mode 100644 index 3901084e2..000000000 --- a/cactus-transpile/CMakeLists.txt +++ /dev/null @@ -1,211 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(Cactus LANGUAGES CXX) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED True) - -set(_CACTUS_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../CACTUS_VERSION") -if(EXISTS "${_CACTUS_VERSION_FILE}") - file(READ "${_CACTUS_VERSION_FILE}" _CACTUS_VERSION_CONTENT) - string(STRIP "${_CACTUS_VERSION_CONTENT}" CACTUS_VERSION_STR) - message(STATUS "Cactus version: ${CACTUS_VERSION_STR}") -else() - set(CACTUS_VERSION_STR "") -endif() - -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release) - message(STATUS "CMAKE_BUILD_TYPE was not set, defaulting to Release.") -endif() - -find_package(CURL QUIET) - -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -set(CACTUS_CURL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../libs/curl" CACHE PATH "Path to vendored libcurl artifacts") - -function(cactus_require_vendored_curl include_dir library_path platform_label) - if(NOT EXISTS "${include_dir}/curl/curl.h" OR NOT EXISTS "${library_path}") - message(WARNING "==============================================================================") - message(WARNING " Vendored libcurl is required for ${platform_label} builds but is missing.") - message(WARNING " Expected header: ${include_dir}/curl/curl.h") - message(WARNING " Expected library: ${library_path}") - message(WARNING " Populate libs/curl from your local curl build before running cactus build.") - message(WARNING "==============================================================================") - message(FATAL_ERROR "Missing vendored libcurl artifacts for ${platform_label}") - endif() -endfunction() - -if(APPLE) - set(CMAKE_OSX_ARCHITECTURES "arm64") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64 -march=armv8.2-a+fp16+simd+dotprod+i8mm -pthread -Wall -Wextra -pedantic -O3 -Wno-missing-field-initializers") - add_compile_definitions( - __ARM_NEON=1 - __ARM_FEATURE_FP16_VECTOR_ARITHMETIC=1 - __ARM_FEATURE_DOTPROD=1 - __ARM_FEATURE_MATMUL_INT8=1 - ACCELERATE_NEW_LAPACK - ) - - enable_language(OBJCXX) - set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") - set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") - cactus_require_vendored_curl("${CACTUS_CURL_INCLUDE_DIR}" "${CACTUS_CURL_LIBRARY_MACOS}" "macOS") - find_library(COREML_FRAMEWORK CoreML REQUIRED) - find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) - find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) - find_library(SECURITY_FRAMEWORK Security REQUIRED) - find_library(SYSTEMCONFIGURATION_FRAMEWORK SystemConfiguration REQUIRED) - find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) -else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8.2-a+fp16+simd+dotprod+i8mm -pthread -Wall -Wextra -pedantic -O3 -Wno-missing-field-initializers") - add_compile_definitions( - __ARM_NEON=1 - __ARM_FEATURE_FP16_VECTOR_ARITHMETIC=1 - __ARM_FEATURE_DOTPROD=1 - __ARM_FEATURE_MATMUL_INT8=1 - ) -endif() - -file(GLOB ENGINE_SOURCES CONFIGURE_DEPENDS "engine/*.cpp") -file(GLOB GRAPH_SOURCES CONFIGURE_DEPENDS "graph/*.cpp") -file(GLOB KERNEL_SOURCES CONFIGURE_DEPENDS "kernel/*.cpp") -file(GLOB FFI_SOURCES CONFIGURE_DEPENDS "ffi/*.cpp") -file(GLOB MODEL_SOURCES CONFIGURE_DEPENDS "models/*.cpp" "models/gemma4/*.cpp") -file(GLOB TELEMETRY_SOURCES CONFIGURE_DEPENDS "telemetry/*.cpp") - -set(SME2_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/kernel/kernel_sme2.cpp") -set(I8MM_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/kernel/kernel_i8mm.cpp") -list(REMOVE_ITEM KERNEL_SOURCES "${SME2_SOURCE}") -list(REMOVE_ITEM KERNEL_SOURCES "${I8MM_SOURCE}") - -if(APPLE) - set(NPU_SOURCES "npu/npu_ane.mm") - set_source_files_properties(npu/npu_ane.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc") - message(STATUS "Apple platform detected - NPU/ANE acceleration enabled") -else() - set(NPU_SOURCES "npu/npu.cpp") -endif() - -set(COMMON_SOURCES - ${ENGINE_SOURCES} - ${GRAPH_SOURCES} - ${KERNEL_SOURCES} - ${FFI_SOURCES} - ${MODEL_SOURCES} - ${NPU_SOURCES} - ${TELEMETRY_SOURCES} -) - -set(COMMON_INCLUDES - ${CMAKE_CURRENT_SOURCE_DIR} - engine - graph - kernel - ffi - models - npu - telemetry -) - -function(configure_cactus_target target_name) - target_include_directories(${target_name} PUBLIC ${COMMON_INCLUDES}) - - if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - target_compile_options(${target_name} PRIVATE -Wno-c99-extensions) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(${target_name} PRIVATE -Wno-pedantic) - endif() - - if(NOT "${CACTUS_VERSION_STR}" STREQUAL "") - target_compile_definitions(${target_name} PRIVATE CACTUS_COMPILE_TIME_VERSION="${CACTUS_VERSION_STR}") - endif() - - if(APPLE) - target_link_libraries(${target_name} PUBLIC - ${COREML_FRAMEWORK} - ${FOUNDATION_FRAMEWORK} - ${ACCELERATE_FRAMEWORK} - ${SECURITY_FRAMEWORK} - ${SYSTEMCONFIGURATION_FRAMEWORK} - ${CFNETWORK_FRAMEWORK} - ${CACTUS_CURL_LIBRARY_MACOS} - ) - target_include_directories(${target_name} PUBLIC ${CACTUS_CURL_INCLUDE_DIR}) - target_compile_definitions(${target_name} PRIVATE CACTUS_USE_CURL=1) - elseif(CURL_FOUND) - target_include_directories(${target_name} PRIVATE ${CURL_INCLUDE_DIRS}) - target_link_libraries(${target_name} PUBLIC ${CURL_LIBRARIES}) - target_compile_definitions(${target_name} PRIVATE CACTUS_USE_CURL) - endif() -endfunction() - -set(ENABLE_SME2 "AUTO" CACHE STRING "SME2 build mode: AUTO or OFF") -set_property(CACHE ENABLE_SME2 PROPERTY STRINGS AUTO OFF) - -set(BUILD_SME2 0) -set(SME2_MARCH_FLAG "-march=armv9.2-a+nosve+nosve2+sme2") -if(ENABLE_SME2 STREQUAL "AUTO") - if(APPLE AND CMAKE_OSX_ARCHITECTURES STREQUAL "arm64") - include(CheckCXXSourceCompiles) - set(CMAKE_REQUIRED_FLAGS "-arch arm64 ${SME2_MARCH_FLAG}") - check_cxx_source_compiles(" - #include - int main() { return 0; } - " COMPILER_SUPPORTS_SME2) - unset(CMAKE_REQUIRED_FLAGS) - - if(COMPILER_SUPPORTS_SME2) - set(BUILD_SME2 1) - message(STATUS "Toolchain supports SME2 - building SME2 kernels") - else() - message(STATUS "Compiler does not support SME2 - building without SME2") - endif() - - else() - message(STATUS "Non-arm64 target - building without SME2") - endif() - -endif() - -if(BUILD_SME2) - add_library(cactus_sme2_obj OBJECT ${SME2_SOURCE}) - target_include_directories(cactus_sme2_obj PRIVATE ${COMMON_INCLUDES}) - target_compile_options(cactus_sme2_obj PRIVATE ${SME2_MARCH_FLAG}) - set_target_properties(cactus_sme2_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) - target_compile_definitions(cactus_sme2_obj PRIVATE CACTUS_COMPILE_SME2) -endif() - -# I8MM support: compile kernel_i8mm.cpp with +i8mm -set(I8MM_MARCH_FLAG "-march=armv8.2-a+fp16+simd+dotprod+i8mm") -add_library(cactus_i8mm_obj OBJECT ${I8MM_SOURCE}) -target_include_directories(cactus_i8mm_obj PRIVATE ${COMMON_INCLUDES}) -target_compile_options(cactus_i8mm_obj PRIVATE ${I8MM_MARCH_FLAG}) -set_target_properties(cactus_i8mm_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) -target_compile_definitions(cactus_i8mm_obj PRIVATE CACTUS_COMPILE_I8MM __ARM_FEATURE_MATMUL_INT8=1) - -set(EXTRA_OBJECTS $) -if(BUILD_SME2) - set(EXTRA_OBJECTS ${EXTRA_OBJECTS} $) -endif() - -add_library(cactus STATIC ${COMMON_SOURCES} ${EXTRA_OBJECTS}) -configure_cactus_target(cactus) -target_compile_definitions(cactus PRIVATE CACTUS_COMPILE_I8MM) - -if(BUILD_SME2) - target_compile_definitions(cactus PRIVATE CACTUS_COMPILE_SME2) -endif() - -add_library(cactus_ffi SHARED ${COMMON_SOURCES} ${EXTRA_OBJECTS}) -configure_cactus_target(cactus_ffi) -set_target_properties(cactus_ffi PROPERTIES - POSITION_INDEPENDENT_CODE ON - OUTPUT_NAME "cactus" -) -target_compile_definitions(cactus_ffi PRIVATE CACTUS_COMPILE_I8MM) - -if(BUILD_SME2) - target_compile_definitions(cactus_ffi PRIVATE CACTUS_COMPILE_SME2) -endif() diff --git a/cactus-transpile/cactus.h b/cactus-transpile/cactus.h deleted file mode 100644 index b5b6d803f..000000000 --- a/cactus-transpile/cactus.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef CACTUS_H -#define CACTUS_H - -#include "graph/graph.h" -#include "kernel/kernel.h" -#include "kernel/kernel_utils.h" -#include "engine/engine.h" -#include "models/model.h" -#include "ffi/cactus_ffi.h" -#include "npu/npu.h" - -#endif // CACTUS_H \ No newline at end of file diff --git a/cactus-transpile/engine/engine.h b/cactus-transpile/engine/engine.h deleted file mode 100644 index c289466f6..000000000 --- a/cactus-transpile/engine/engine.h +++ /dev/null @@ -1,1079 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "../graph/graph.h" - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wc99-extensions" -#pragma clang diagnostic ignored "-Wunused-parameter" -#elif defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#endif - -extern "C" { - #include "../../libs/stb/stb_image.h" - #include "../../libs/stb/stb_image_resize2.h" -} - -#ifdef __clang__ -#pragma clang diagnostic pop -#elif defined(__GNUC__) -#pragma GCC diagnostic pop -#endif - -class CactusGraph; - -namespace cactus { -namespace npu { - class NPUPrefill; -} -namespace engine { - -class Siglip2Preprocessor; - -struct Config { - uint32_t vocab_size = 151936; - uint32_t bos_token_id = 151643; - uint32_t eos_token_id = 151645; - uint32_t num_layers = 28; - uint32_t hidden_dim = 1024; - uint32_t ffn_intermediate_dim = 3072; - uint32_t attention_heads = 16; - uint32_t attention_kv_heads = 8; - uint32_t attention_head_dim = 128; - float layer_norm_eps = 1e-6f; - float rope_theta = 1000000.0f; - uint32_t num_experts = 0; - uint32_t num_shared_experts = 0; - uint32_t num_top_experts = 0; - uint32_t moe_every_n_layers = 0; - uint32_t moe_intermediate_dim = 0; - uint32_t num_dense_layers = 0; - uint32_t num_experts_per_tok = 0; - bool norm_topk_prob = false; - bool use_expert_bias = false; - float routed_scaling_factor = 1.0f; - bool tie_word_embeddings = true; - - uint32_t vision_hidden_dim = 0; - uint32_t vision_num_layers = 0; - uint32_t vision_attention_heads = 0; - uint32_t vision_image_size = 0; - uint32_t vision_patch_size = 0; - uint32_t vision_num_channels = 3; - uint32_t vision_embed_dim = 0; - uint32_t visual_tokens_per_img = 0; - bool use_pixel_shuffle = false; - uint32_t pixel_shuffle_factor = 1; - bool use_image_tokens = false; - uint32_t image_token_id = 0; - bool use_layout_tags = false; - uint32_t image_seq_len = 64; - - uint32_t global_image_size = 2048; - uint32_t max_tile_size = 512; - float rescale_factor = 0.00392156862745098f; - float image_mean = 0.5f; - float image_std = 0.5f; - - uint32_t downsample_factor = 2; - uint32_t min_tiles = 2; - uint32_t max_tiles = 10; - bool use_thumbnail = true; - uint32_t min_image_tokens = 64; - uint32_t max_image_tokens = 256; - uint32_t max_num_patches = 1024; - uint32_t tile_size = 512; - float max_pixels_tolerance = 2.0f; - bool do_image_splitting = true; - bool encoder_act_gelu = false; - bool decoder_act_gelu = false; - uint32_t num_encoder_layers = 0; - uint32_t num_decoder_layers = 0; - float partial_rotary_factor = 0.0f; - uint32_t pad_token_id = 0; - uint32_t conv_kernel_size = 0; - uint32_t subsampling_conv_kernel_size = 0; - uint32_t subsampling_conv_stride = 0; - uint32_t subsampling_conv_channels = 0; - uint32_t subsampling_factor = 0; - uint32_t num_mel_bins = 80; - std::string encoder_hidden_act = "silu"; - uint32_t linear_num_key_heads = 0; - uint32_t linear_key_head_dim = 0; - uint32_t linear_num_value_heads = 0; - uint32_t linear_value_head_dim = 0; - uint32_t linear_q_proj_dim = 0; - uint32_t linear_k_proj_dim = 0; - uint32_t linear_v_proj_dim = 0; - - uint32_t kv_lora_rank = 0; - uint32_t q_lora_rank = 0; - uint32_t qk_head_dim = 0; - uint32_t qk_nope_head_dim = 0; - uint32_t qk_rope_head_dim = 0; - uint32_t v_head_dim = 0; - uint32_t rope_interleave = 0; - bool attention_bias = false; - float rope_scaling_factor = 1.0f; - float rope_mscale_all_dim = 0.0f; - - enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, PYANNOTE = 16, WESPEAKER = 17, NEEDLE = 18}; - uint32_t predictor_hidden_dim = 0; - uint32_t predictor_num_layers = 0; - uint32_t tdt_joint_dim = 0; - uint32_t tdt_num_durations = 0; - uint32_t tdt_blank_id = 0; - std::vector tdt_durations; - - ModelType model_type = ModelType::QWEN; - - enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3}; - ModelVariant model_variant = ModelVariant::DEFAULT; - - enum class Activation {GELU = 0, SILU = 1}; - Activation activation = Activation::SILU; - - enum class Backend {CPU = 0, NPU = 1}; - Backend default_backend = Backend::CPU; - - enum class Precision {INT8 = 0, FP16 = 1, FP32 = 2}; - Precision precision = Precision::FP32; - - float default_temperature = 0.6f; - float default_top_p = 0.95f; - size_t default_top_k = 20; - float default_max_tps = -1.0f; - float default_cloud_handoff_threshold = 0.0f; - - std::vector layer_types; - size_t conv_L_cache = 0; - - uint32_t altup_num_inputs = 4; - uint32_t laurel_rank = 64; - uint32_t hidden_size_per_layer_input = 256; - uint32_t num_kv_shared_layers = 0; - uint32_t sliding_window = 512; - float rope_local_base_freq = 10000.0f; - float final_logit_softcapping = 0.0f; - float global_partial_rotary_factor = 1.0f; - uint32_t expert_intermediate_size = 0; - uint32_t global_head_dim = 0; - uint32_t num_global_kv_heads = 0; - bool attention_k_eq_v = false; - bool enable_moe_block = false; - std::vector activation_sparsity_ppf; - - uint32_t vision_head_dim = 64; - uint32_t vision_kv_heads = 12; - uint32_t vision_intermediate_size = 3072; - uint32_t vision_position_embedding_size = 10240; - uint32_t vision_pooling_kernel_size = 3; - uint32_t vision_default_output_length = 280; - float vision_rope_theta = 100.0f; - - uint32_t audio_hidden_dim = 0; - uint32_t audio_num_layers = 0; - uint32_t audio_num_heads = 0; - uint32_t audio_head_dim = 0; - uint32_t audio_input_feat_size = 128; - uint32_t audio_conf_conv_kernel_size = 5; - uint32_t audio_chunk_size = 12; - uint32_t audio_context_left = 13; - uint32_t audio_context_right = 0; - float audio_logit_cap = 50.0f; - float audio_residual_weight = 0.5f; - uint32_t audio_output_proj_dims = 0; - uint32_t audio_vocab_size = 128; - uint32_t audio_vocab_offset = 0; - uint32_t audio_soft_tokens = 188; - uint32_t audio_sscp_conv0_channels = 128; - uint32_t audio_sscp_conv1_channels = 32; - float audio_sscp_conv_eps = 1e-3f; - float audio_rms_norm_eps = 1e-6f; - uint32_t audio_fft_length = 1024; - uint32_t audio_token_id = 0; - bool audio_fft_overdrive = false; - uint32_t channel_open_token_id = 100; - uint32_t channel_close_token_id = 101; - - static bool is_gemma_family(ModelType t) { - return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; - } - - bool from_json(const std::string& json_path); - std::string to_json() const; -}; - - - -struct MergeRule { - std::string first; - std::string second; - std::string merged; - uint32_t priority; - - MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) - : first(f), second(s), merged(m), priority(p) {} -}; - - -struct ToolCallInfo { - std::string name; - std::string arguments; -}; - -struct ChatMessage { - std::string role; - std::string content; - std::string name; - std::vector images; - std::vector audio; - size_t audio_soft_token_count = 0; - std::vector tool_calls; -}; - -inline std::string format_needle_query_text(const std::vector& messages) { - std::string system_text; - std::string user_query; - - for (const auto& msg : messages) { - if (msg.role == "system") { - if (!system_text.empty()) { - system_text += "\n"; - } - system_text += msg.content; - } else if (msg.role == "user") { - user_query = msg.content; - } - } - - if (user_query.empty() && !messages.empty()) { - user_query = messages.back().content; - } - if (system_text.empty()) { - return user_query; - } - if (user_query.empty()) { - return system_text; - } - return system_text + "\n\n" + user_query; -} - -struct ToolConstraintSpec { - std::string name; - std::vector parameter_names; - std::vector required_parameter_names; -}; - -struct TokenizerRuntimeConfig { - enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE }; - enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN }; - enum class Normalizer { NONE, METASPACE, BYTE_LEVEL }; - enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL }; - - TokenizerType tokenizer_type = TokenizerType::UNKNOWN; - VocabFormat vocab_format = VocabFormat::UNKNOWN; - Normalizer normalizer = Normalizer::NONE; - Decoder decoder = Decoder::NONE; - bool byte_fallback = false; - bool has_chat_template = false; -}; - -TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); -void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); -std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); - -class Tokenizer { -public: - virtual ~Tokenizer() = default; - - virtual std::vector encode(const std::string& text) const = 0; - virtual std::string decode(const std::vector& tokens) const = 0; - - virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; - virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = true) const; - - virtual uint32_t get_vocab_size() const = 0; - virtual uint32_t get_unk_token() const = 0; - virtual uint32_t get_bos_token() const = 0; - virtual uint32_t get_eos_token() const = 0; - virtual bool has_chat_template() const { return has_chat_template_; } - std::string get_default_stop_sequence() const; - - virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; - - uint32_t get_image_token_id() const { return image_token_id_; } - uint32_t get_fake_token_id() const { return fake_token_id_; } - uint32_t get_global_img_token_id() const { return global_img_token_id_; } - -protected: - enum class ModelType { UNKNOWN, QWEN, QWEN3P5, GEMMA, GEMMA4, LFM2, BERT, WHISPER, PARAKEET, YOUTU, NEEDLE}; - ModelType model_type_ = ModelType::UNKNOWN; - enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG}; - ModelVariant model_variant_ = ModelVariant::DEFAULT; - bool has_chat_template_ = false; - std::string chat_template_; - - uint32_t image_token_id_ = 396; - uint32_t fake_token_id_ = 49189; - uint32_t global_img_token_id_ = 49152; - - - uint32_t vision_patch_size_ = 16; - uint32_t vision_pooling_kernel_size_ = 3; - uint32_t vision_default_output_length_ = 280; - uint32_t vision_image_size_ = 768; - TokenizerRuntimeConfig runtime_config_; - - void detect_model_type(const std::string& config_path); - void load_chat_template(const std::string& template_file); - std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = true) const; - std::string format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = true) const; - std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_lfm2_vl_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_needle_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; -}; - -class BPETokenizer : public Tokenizer { -public: - BPETokenizer(); - ~BPETokenizer(); - - bool load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file); - bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; - - std::vector encode(const std::string& text) const override; - std::string decode(const std::vector& tokens) const override; - - uint32_t get_vocab_size() const override { return vocab_size_; } - uint32_t get_unk_token() const override { return unk_token_id_; } - uint32_t get_bos_token() const override { return bos_token_id_; } - uint32_t get_eos_token() const override { return eos_token_id_; } - -private: - std::unordered_map token_to_id_; - std::vector id_to_token_; - std::vector merge_rules_; - std::unordered_map merge_map_; - - uint32_t vocab_size_; - uint32_t unk_token_id_; - uint32_t bos_token_id_; - uint32_t eos_token_id_; - - void* vocab_mmap_ptr_; - size_t vocab_mmap_size_; - - void* merges_mmap_ptr_; - size_t merges_mmap_size_; - - std::vector apply_bpe(const std::vector& tokens) const; - std::pair find_best_merge_fast(const std::vector& tokens) const; - - std::string bytes_to_unicode(const std::string& text) const; - std::string unicode_to_bytes(const std::string& text) const; - std::vector byte_level_split(const std::string& text) const; - std::vector utf8_split(const std::string& text) const; - - void cleanup_mmap(); - -private: - mutable std::unordered_map byte_to_unicode_; - mutable std::unordered_map unicode_to_byte_; - void init_byte_mappings() const; - - std::unordered_map special_tokens_; - std::vector split_with_special_tokens(const std::string& text) const; - void load_special_tokens(const std::string& config_file); -}; - -class SPTokenizer : public Tokenizer { -public: - SPTokenizer(); - ~SPTokenizer(); - - bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; - - std::vector encode(const std::string& text) const override; - std::string decode(const std::vector& tokens) const override; - - uint32_t get_vocab_size() const override { return vocab_size_; } - uint32_t get_unk_token() const override { return unk_token_id_; } - uint32_t get_bos_token() const override { return bos_token_id_; } - uint32_t get_eos_token() const override { return eos_token_id_; } - -private: - struct TrieNode { - std::unordered_map> children; - int32_t token_id = -1; - float score = 0.0f; - }; - - std::unique_ptr trie_root_; - std::unordered_map token_to_id_; - std::vector id_to_token_; - std::vector token_scores_; - - uint32_t vocab_size_; - uint32_t unk_token_id_; - uint32_t bos_token_id_; - uint32_t eos_token_id_; - uint32_t pad_token_id_; - - bool sp_bpe_mode_ = false; - bool sp_add_dummy_prefix_ = false; - bool sp_byte_fallback_ = false; - - void* vocab_mmap_ptr_; - size_t vocab_mmap_size_; - - void build_trie(); - std::vector> tokenize_with_trie(const std::string& text) const; - std::vector tokenize_with_bpe(const std::string& text) const; - std::string preprocess_text(const std::string& text) const; - std::string postprocess_text(const std::string& text) const; - std::vector split_by_unicode_spaces(const std::string& text) const; - - void cleanup_mmap(); - - std::unordered_map special_tokens_; - std::vector split_with_special_tokens(const std::string& text) const; - void load_special_tokens(const std::string& config_file); -}; - -class ConvCache { -public: - struct CircularView { - const void* ptr1; - size_t len1; - const void* ptr2; - size_t len2; - size_t total_len; - }; - - void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); - CircularView get_window(size_t layer) const; - void update(CactusGraph* gb, size_t layer, const size_t latest_token); - void reset(); - - bool is_empty() const { return num_layers == 0; } - - size_t num_layers = 0; - size_t hidden_size = 0; - size_t window_size = 0; - Precision precision = Precision::FP32; - size_t element_size = 4; - -private: - struct LayerState { - std::vector data; - size_t head = 0; - size_t count = 0; - }; - - std::vector layer_states; -}; - -struct KVCache { - static constexpr size_t DEFAULT_WINDOW_SIZE = 1024; - static constexpr size_t DEFAULT_SINK_SIZE = 4; - - struct LayerCache { - std::vector keys; - std::vector values; - std::vector key_scales; - std::vector value_scales; - size_t head_dim = 0; - size_t kv_heads = 0; - }; - - std::vector layer_caches; - - size_t window_size = DEFAULT_WINDOW_SIZE; - size_t sink_size = DEFAULT_SINK_SIZE; - size_t current_seq_len = 0; - size_t total_seq_len = 0; - size_t max_seq_len = 2048; - size_t num_layers = 0; - Precision precision; - size_t element_size = 4; - - void set_window_size(size_t window, size_t sink = DEFAULT_SINK_SIZE); - size_t get_effective_seq_len() const { return current_seq_len; } - size_t get_total_seq_len() const { return total_seq_len; } - size_t get_layer_head_dim(size_t layer_idx) const { return layer_caches[layer_idx].head_dim; } - size_t get_layer_kv_heads(size_t layer_idx) const { return layer_caches[layer_idx].kv_heads; } - - void init(size_t num_layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision); - void reset(); - void update_from_graph(CactusGraph* gb, const std::vector& k_nodes, - const std::vector& v_nodes, size_t seq_len, - size_t num_layers); - - void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, - size_t num_tokens, size_t kv_heads, size_t head_dim); - - bool is_empty() const { return current_seq_len == 0; } - void* get_key_ptr(size_t layer); - void* get_value_ptr(size_t layer); - - struct CircularView { - const void* ptr1; - const void* ptr2; - size_t len1; - size_t len2; - size_t total_len; - }; - - CircularView get_key_view(size_t layer); - CircularView get_value_view(size_t layer); - - const int8_t* get_keys_int8(size_t layer) const; - const int8_t* get_values_int8(size_t layer) const; - const float* get_key_scales(size_t layer) const; - const float* get_value_scales(size_t layer) const; - - void remove_token_range(size_t start, size_t count); - void compact_to_windows(const std::vector& target_windows); -}; - -class ToolCallConstrainer { -public: - enum class State { - DONE, - - QWEN_START, - QWEN_EXPECT_OPEN_BRACE, - QWEN_EXPECT_NAME_KEY, - QWEN_EXPECT_NAME_COLON, - QWEN_EXPECT_NAME_VALUE, - QWEN_EXPECT_COMMA, - QWEN_EXPECT_ARGS_KEY, - QWEN_EXPECT_ARGS_COLON, - QWEN_IN_ARGUMENTS, - QWEN_EXPECT_CLOSE_BRACE, - QWEN_EXPECT_END, - - NEEDLE_START, - - LFM_START, - LFM_EXPECT_BRACKET, - LFM_IN_FUNC_NAME, - LFM_EXPECT_PAREN, - LFM_IN_ARGUMENTS, - LFM_EXPECT_BRACKET_CLOSE, - LFM_EXPECT_END, - - GEMMA_START, - GEMMA_EXPECT_CALL, - GEMMA_IN_FUNC_NAME, - GEMMA_EXPECT_BRACE, - GEMMA_IN_ARGUMENTS, - GEMMA_EXPECT_END - }; - - void init(Config::ModelType model_type, - const std::vector& tools, - Tokenizer* tokenizer); - - const std::unordered_map& get_bias() const { return current_bias_; } - - void update(uint32_t token_id, const std::string& decoded_text); - - void reset(); - - bool is_active() const { return active_; } - -private: - bool active_ = false; - State state_ = State::QWEN_START; - Config::ModelType model_type_ = Config::ModelType::QWEN; - Tokenizer* tokenizer_ = nullptr; - - bool is_gemma_family() const { return Config::is_gemma_family(model_type_); } - bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } - - enum class NeedleJsonState { - FREE, - IN_NAME, - IN_ARG_KEY, - }; - - struct NeedleTrieNode { - std::unordered_map> children; - bool is_terminal = false; - }; - - std::vector tool_specs_; - std::vector function_names_; - std::string generated_text_; - int brace_depth_ = 0; - - std::string call_start_tag_; - std::string call_end_tag_; - - std::unordered_set qwen_tool_call_start_tokens_; - std::unordered_set qwen_tool_call_end_tokens_; - std::unordered_set open_brace_tokens_; - std::unordered_set close_brace_tokens_; - std::unordered_set colon_tokens_; - std::unordered_set comma_tokens_; - std::unordered_set name_key_tokens_; - std::unordered_set args_key_tokens_; - std::unordered_set quote_tokens_; - std::unordered_set backtick_tokens_; - std::unordered_set all_func_name_tokens_; - std::unordered_map> func_name_sequences_; - NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; - std::string needle_buffer_; - std::string needle_constrained_buf_; - std::string needle_current_function_; - bool needle_in_arguments_ = false; - int needle_arguments_depth_ = 0; - int needle_nesting_depth_ = 0; - bool needle_in_string_value_ = false; - bool needle_between_pairs_ = false; - bool needle_prev_char_escape_ = false; - std::unique_ptr needle_name_trie_; - std::unordered_map> needle_param_tries_; - std::unordered_map> needle_required_params_; - std::unordered_set needle_seen_arg_keys_; - std::vector needle_token_strings_; - std::unordered_map> needle_token_index_; - std::unordered_set needle_arg_close_tokens_; - - std::unordered_set tool_start_tokens_; - std::unordered_set tool_end_tokens_; - std::unordered_set bracket_open_tokens_; - std::unordered_set bracket_close_tokens_; - std::unordered_set paren_open_tokens_; - std::unordered_set paren_close_tokens_; - std::unordered_set equals_tokens_; - - std::unordered_set gemma_call_start_tokens_; - std::unordered_set gemma_call_end_tokens_; - std::unordered_set gemma_response_start_tokens_; - std::unordered_set gemma_call_prefix_tokens_; - std::unordered_set escape_tokens_; - - std::unordered_map current_bias_; - - void compute_bias(); - void tokenize_grammar_elements(); - void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); - void tokenize_function_names(bool quote_names); - void init_common_tokens(); - void init_needle_constraints(); - void reset_needle_constraints(); - void feed_needle_text(const std::string& text); - void feed_needle_char(char ch); - bool needle_at_arg_key_start() const; - bool needle_is_value_string_start() const; - void needle_insert_word(NeedleTrieNode* root, const std::string& word); - const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; - bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; - bool needle_has_unseen_completion(const NeedleTrieNode* node, std::string& partial) const; -}; - -class Model { -public: - struct DebugNode { - uint32_t layer_idx; - std::string name; - size_t node_id; - }; - - Model(); - explicit Model(const Config& config); - virtual ~Model(); - - const Config& get_config() const { return config_; } - Tokenizer* get_tokenizer() const { return tokenizer_.get(); } - const std::vector& get_debug_nodes() const; - - virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); - - virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true); - - virtual uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, - size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f); - - virtual void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); - - virtual void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file = ""); - - virtual uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, - float temperature = -1.0f, float top_p = -1.0f, - size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f); - - virtual uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, float temperature = 0.0f, float top_p = 0.0f, - size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); - - std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); - - virtual std::vector get_image_embeddings(const std::string& image_path); - - virtual std::vector get_audio_embeddings(const std::vector& audio_features); - - virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } - void record_sampled_token(uint32_t token) { - if (token_history_.size() >= MAX_TOKEN_HISTORY) { - token_history_.erase(token_history_.begin(), token_history_.begin() + (MAX_TOKEN_HISTORY / 2)); - } - token_history_.push_back(token); - } - - double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored); - - - - void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); } - - bool load_npu_prefill(const std::string& model_path); - bool has_npu_prefill() const; - size_t get_prefill_chunk_size() const; - - virtual void remove_thinking_tokens(const std::vector>& ranges); - virtual void compact_kv_cache() {} - - void set_tool_constraints(const std::vector& tools); - void clear_tool_constraints(); - void update_tool_constraints(uint32_t token_id); - - void* graph_handle_; - - void set_vocab_bias(const std::unordered_map& bias) { - vocab_bias_ = bias; - } - - void clear_vocab_bias() { - vocab_bias_.clear(); - } - - bool has_vocab_bias() const { - return !vocab_bias_.empty(); - } - - const std::unordered_map& get_vocab_bias() const { - return vocab_bias_; - } - -protected: - size_t sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, - float min_p, float repetition_penalty, - const std::unordered_map* extra_bias = nullptr) const; - - static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); - - virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; - - virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); - - virtual void load_weights_to_graph(CactusGraph* gb) = 0; - - virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; - - virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const = 0; - virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; - void update_kv_cache(CactusGraph* gb, size_t seq_len); - virtual std::vector get_kv_layer_dims() const { - return std::vector(config_.num_layers, config_.attention_head_dim); - } - virtual std::vector get_kv_layer_heads() const { - return std::vector(config_.num_layers, config_.attention_kv_heads); - } - virtual void post_init() {} - virtual void post_execute_updates(CactusGraph*, size_t) {} - Config config_; - std::unique_ptr tokenizer_; - - bool initialized_; - float attention_scale_; - -protected: - KVCache kv_cache_; - std::vector cache_k_output_nodes_; - std::vector cache_v_output_nodes_; - - std::string embedding_file_path_; - size_t embedding_node_id_; - std::string model_folder_path_; - size_t output_weight_node_id_; - - mutable std::vector debug_nodes_; - - void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const; - void clear_debug_nodes(); - - bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup); - bool owns_graph_; - - std::unique_ptr npu_prefill_; - void prefill_npu(const std::vector& tokens); - virtual std::vector<__fp16> get_token_embeddings(const std::vector& tokens); - - static constexpr size_t MAX_TOKEN_HISTORY = 128; - ToolCallConstrainer tool_constrainer_; - std::vector token_history_; - -private: - std::unordered_map vocab_bias_; -}; - -std::unique_ptr create_model(const std::string& model_folder); - -class Siglip2Preprocessor { -public: - struct Config { - int patch_size = 16; - int downsample_factor = 2; - int min_tiles = 2; - int max_tiles = 10; - bool use_thumbnail = true; - int min_image_tokens = 64; - int max_image_tokens = 256; - int max_num_patches = 1024; - int tile_size = 512; - float max_pixels_tolerance = 2.0f; - bool do_resize = true; - bool do_rescale = true; - bool do_normalize = true; - bool do_convert_rgb = true; - bool do_image_splitting = true; - float rescale_factor = 1.0f / 255.0f; - float image_mean[3] = {0.5f, 0.5f, 0.5f}; - float image_std[3] = {0.5f, 0.5f, 0.5f}; - }; - - struct PreprocessedImage { - std::vector pixel_values; - std::vector pixel_attention_mask; - std::vector> spatial_shapes; - std::vector pixel_values_shape; - std::vector pixel_attention_mask_shape; - std::vector spatial_shapes_shape; - int num_patches_height; - int num_patches_width; - int actual_num_patches; - int num_tiles; - int patch_dim; - int max_patches_per_tile; - - int image_rows; - int image_cols; - int image_height; - int image_width; - int tokens_per_tile; - int thumbnail_tokens; - - ~PreprocessedImage(); - }; - - struct SpatialShapeResult { - std::vector> shapes; - int grid_rows; - int grid_cols; - }; - - explicit Siglip2Preprocessor(const Config& config); - Siglip2Preprocessor(); - ~Siglip2Preprocessor(); - - PreprocessedImage preprocess_from_file(const std::string& image_path); - PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels); - SpatialShapeResult compute_spatial_shapes(int height, int width); - -private: - Config config_; - - std::pair compute_pixel_limits() const; - std::vector convert_to_rgb(const unsigned char* img_data, int width, int height, int channels); - std::pair smart_resize(int height, int width); - bool is_image_too_large(int height, int width); - std::pair get_grid_layout(int height, int width); - std::pair find_closest_aspect_ratio(float aspect_ratio, int width, int height); - std::vector resize_image(const unsigned char* img_data, int src_width, int src_height, - int dst_width, int dst_height, int channels); - std::vector normalize_image(const float* img_data, int width, int height, int channels); - std::vector> convert_image_to_patches( - const std::vector& image, int width, int height, int channels, int patch_size); - PreprocessedImage pad_patches(const std::vector>& tile_patches, - const std::vector>& spatial_shapes, - int patch_dim, - int max_patches_per_tile); - int round_by_factor(int number, int factor); -}; - -class AudioProcessor { -public: - struct SpectrogramConfig { - size_t n_fft = 400; - size_t hop_length = 160; - size_t frame_length = 400; - float power = 2.0f; - bool center = true; - const char* pad_mode = "reflect"; - bool onesided = true; - float dither = 0.0f; - float mel_floor = 1e-10f; - const char* log_mel = nullptr; - float reference = 1.0f; - float min_value = 1e-10f; - bool remove_dc_offset = false; - float preemphasis = 0.0f; - bool hann_periodic = true; - float window_a0 = 0.5f; - size_t fft_override = 0; - bool mel_floor_additive = false; - }; - - AudioProcessor(); - ~AudioProcessor(); - - void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters, - float min_freq, float max_freq, size_t sampling_rate, - const char* norm = "slaney", const char* mel_scale = "slaney"); - - std::vector compute_spectrogram( - const std::vector& waveform, - const SpectrogramConfig& config); - - static std::vector compute_irfft( - const std::vector& complex_input, - size_t n, - const char* norm = "backward"); - - const std::vector& get_mel_filters() const { return mel_filters_; } - - size_t get_num_mel_filters() const { return num_mel_filters_; } - size_t get_num_frequency_bins() const { return num_frequency_bins_; } - -private: - std::vector mel_filters_; - size_t num_frequency_bins_; - size_t num_mel_filters_; -}; - -namespace index { - constexpr uint32_t MAGIC = 0x43414354; - constexpr uint32_t VERSION = 1; - - struct Document { - int id; - std::vector embedding; - std::string content; - std::string metadata; - }; - - struct QueryResult { - int doc_id; - float score; - - QueryResult(int doc_id, float score) : doc_id(doc_id), score(score) {} - }; - - struct QueryOptions { - size_t top_k = 10; - float score_threshold = -1.0f; - }; - - class Index { - public: - Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim); - ~Index(); - - Index(const Index&) = delete; - Index& operator=(const Index&) = delete; - Index(Index&&) = delete; - Index& operator=(Index&&) = delete; - - void add_documents(const std::vector& documents); - void delete_documents(const std::vector& doc_ids); - std::vector get_documents(const std::vector& doc_ids); - std::vector> query(const std::vector>& embeddings, const QueryOptions& options); - void compact(); - - private: - struct IndexHeader { - uint32_t magic; - uint32_t version; - uint32_t embedding_dim; - uint32_t num_documents; - }; - - struct IndexEntry { - int32_t doc_id; - uint64_t data_offset; - uint8_t flags; // bit 0: tombstone - - const __fp16* embedding() const { - return reinterpret_cast(this + 1); - } - - static size_t size(size_t embedding_dim) { - return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16); - } - }; - - struct DataHeader { - uint32_t magic; - uint32_t version; - }; - - struct DataEntry { - uint16_t content_len; - uint16_t metadata_len; - - const char* content() const { - return reinterpret_cast(this + 1); - } - - const char* metadata() const { - return content() + content_len; - } - }; - - void parse_index_header(); - void parse_data_header(); - void build_doc_id_map(); - void validate_documents(const std::vector& documents); - void validate_doc_ids(const std::vector& doc_ids); - ssize_t write_full(int fd, const void* buf, size_t count); - - std::unordered_map doc_id_map_; - - std::string index_path_, data_path_; - size_t embedding_dim_; - size_t index_entry_size_; - uint32_t num_documents_; - - int index_fd_, data_fd_; - void *mapped_index_, *mapped_data_; - size_t index_file_size_, data_file_size_; - }; -} // namespace index - -} -} diff --git a/cactus-transpile/engine/engine_bpe.cpp b/cactus-transpile/engine/engine_bpe.cpp deleted file mode 100644 index 6a882c6f1..000000000 --- a/cactus-transpile/engine/engine_bpe.cpp +++ /dev/null @@ -1,538 +0,0 @@ -#include "engine.h" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -namespace { -constexpr const char* kMetaspace = "\xE2\x96\x81"; -} // namespace - -BPETokenizer::BPETokenizer() - : vocab_size_(0), unk_token_id_(0), bos_token_id_(1), eos_token_id_(2), - vocab_mmap_ptr_(nullptr), vocab_mmap_size_(0), - merges_mmap_ptr_(nullptr), merges_mmap_size_(0) { - has_chat_template_ = false; -} - -BPETokenizer::~BPETokenizer() { - cleanup_mmap(); -} - -void BPETokenizer::cleanup_mmap() { - if (vocab_mmap_ptr_ && vocab_mmap_ptr_ != MAP_FAILED) { - munmap(vocab_mmap_ptr_, vocab_mmap_size_); - vocab_mmap_ptr_ = nullptr; - } - if (merges_mmap_ptr_ && merges_mmap_ptr_ != MAP_FAILED) { - munmap(merges_mmap_ptr_, merges_mmap_size_); - merges_mmap_ptr_ = nullptr; - } -} - -bool BPETokenizer::load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file) { - int vocab_fd = open(vocab_file.c_str(), O_RDONLY); - if (vocab_fd == -1) return false; - - struct stat vocab_stat; - if (fstat(vocab_fd, &vocab_stat) == -1) { - close(vocab_fd); - return false; - } - - vocab_mmap_size_ = vocab_stat.st_size; - vocab_mmap_ptr_ = mmap(nullptr, vocab_mmap_size_, PROT_READ, MAP_PRIVATE, vocab_fd, 0); - close(vocab_fd); - - if (vocab_mmap_ptr_ == MAP_FAILED) return false; - - std::string vocab_content(static_cast(vocab_mmap_ptr_), vocab_mmap_size_); - std::istringstream vocab_stream(vocab_content); - - auto rtrim_cr = [](std::string& s) { - if (!s.empty() && s.back() == '\r') s.pop_back(); - }; - - std::string line; - token_to_id_.clear(); - id_to_token_.clear(); - special_tokens_.clear(); - const bool use_id_tab_vocab = - runtime_config_.vocab_format == TokenizerRuntimeConfig::VocabFormat::ID_TAB_TOKEN; - - if (use_id_tab_vocab) { - while (std::getline(vocab_stream, line)) { - rtrim_cr(line); - - std::string token; - uint32_t id = UINT32_MAX; - - std::istringstream iss(line); - if (iss >> id) { - if (std::getline(iss, token) && !token.empty() && token[0] == '\t') { - token = token.substr(1); - } - - if (token.empty()) { - auto last_pos = vocab_stream.tellg(); - while (std::getline(vocab_stream, line)) { - rtrim_cr(line); - if (!line.empty()) { - break; - } - token += '\n'; - last_pos = vocab_stream.tellg(); - } - vocab_stream.seekg(last_pos); - } - } - - if (!token.empty() && id != UINT32_MAX) { - token_to_id_[token] = id; - if (id >= id_to_token_.size()) { - id_to_token_.resize(id + 1); - } - id_to_token_[id] = token; - } - } - vocab_size_ = static_cast(id_to_token_.size()); - } else { - uint32_t id = 0; - while (std::getline(vocab_stream, line)) { - rtrim_cr(line); - token_to_id_[line] = id; - id_to_token_.push_back(line); - ++id; - } - vocab_size_ = id; - } - - int merges_fd = open(merges_file.c_str(), O_RDONLY); - if (merges_fd == -1) return false; - - struct stat merges_stat; - if (fstat(merges_fd, &merges_stat) == -1) { - close(merges_fd); - return false; - } - - merges_mmap_size_ = merges_stat.st_size; - merges_mmap_ptr_ = mmap(nullptr, merges_mmap_size_, PROT_READ, MAP_PRIVATE, merges_fd, 0); - close(merges_fd); - - if (merges_mmap_ptr_ == MAP_FAILED) return false; - - std::string merges_content(static_cast(merges_mmap_ptr_), merges_mmap_size_); - std::istringstream merges_stream(merges_content); - - merge_rules_.clear(); - uint32_t priority = 0; - - while (std::getline(merges_stream, line)) { - rtrim_cr(line); - if (line.empty() || line[0] == '#') continue; - - std::istringstream iss(line); - std::string first, second; - if (iss >> first >> second) { - rtrim_cr(first); - rtrim_cr(second); - - std::string merged = first + second; - merge_rules_.emplace_back(first, second, merged, priority); - - std::string key = first + "\x00" + second; - auto it = merge_map_.find(key); - if (it == merge_map_.end() || priority < it->second) { - merge_map_[key] = priority; - } - priority++; - } - } - - std::sort(merge_rules_.begin(), merge_rules_.end(), - [](const MergeRule& a, const MergeRule& b) { - return a.priority < b.priority; - }); - - return true; -} - -bool BPETokenizer::load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) { - runtime_config_ = load_tokenizer_runtime_config(config_file); - if (!load_vocabulary_mmap(vocab_file, merges_file)) { - return false; - } - - std::ifstream config_stream(config_file); - if (!config_stream.is_open()) { - return true; - } - - std::string line; - while (std::getline(config_stream, line)) { - if (line.empty() || line[0] == '#') continue; - - size_t eq_pos = line.find('='); - if (eq_pos == std::string::npos) continue; - - std::string key = line.substr(0, eq_pos); - std::string value = line.substr(eq_pos + 1); - - key.erase(0, key.find_first_not_of(" \t")); - key.erase(key.find_last_not_of(" \t") + 1); - value.erase(0, value.find_first_not_of(" \t")); - value.erase(value.find_last_not_of(" \t") + 1); - - if (key == "eos_token_id") { - eos_token_id_ = std::stoul(value); - } else if (key == "pad_token_id") { - if (unk_token_id_ == 0) { - unk_token_id_ = std::stoul(value); - } - } else if (key == "unk_token_id" && value != "null") { - unk_token_id_ = std::stoul(value); - } else if (key == "bos_token_id" && value != "null") { - bos_token_id_ = std::stoul(value); - } else if (key == "vocab_size") { - if (std::stoul(value) != vocab_size_) { - } - } - } - - std::string dir = config_file.substr(0, config_file.find_last_of("/\\")); - std::string special_tokens_path = dir + "/special_tokens.json"; - load_special_tokens(special_tokens_path); - - std::string template_path = dir + "/chat_template.jinja2"; - load_chat_template(template_path); - - std::string config_path = dir + "/config.txt"; - detect_model_type(config_path); - - return true; -} - -void BPETokenizer::load_special_tokens(const std::string& config_file) { - load_special_tokens_map(config_file, special_tokens_); -} - -std::vector BPETokenizer::split_with_special_tokens(const std::string& text) const { - return cactus::engine::split_with_special_tokens(text, special_tokens_); -} - -void BPETokenizer::init_byte_mappings() const { - if (!byte_to_unicode_.empty()) return; - - std::vector bytes; - - for (int i = 33; i <= 126; ++i) { - bytes.push_back(i); - } - - - for (int i = 161; i <= 255; ++i) { - bytes.push_back(i); - } - - std::vector remaining_bytes; - for (int i = 0; i <= 32; ++i) remaining_bytes.push_back(i); - remaining_bytes.push_back(127); - for (int i = 128; i <= 160; ++i) remaining_bytes.push_back(i); - - int unicode_start = 256; - for (int byte : remaining_bytes) { - bytes.push_back(byte); - } - - for (size_t i = 0; i < bytes.size(); ++i) { - uint8_t byte = static_cast(bytes[i]); - - if (byte >= 33 && byte <= 126) { - std::string unicode_char(1, static_cast(byte)); - byte_to_unicode_[byte] = unicode_char; - unicode_to_byte_[unicode_char] = byte; - } else if (byte >= 161 && byte <= 255) { - std::string unicode_char; - unicode_char += static_cast(0xC0 | (byte >> 6)); - unicode_char += static_cast(0x80 | (byte & 0x3F)); - byte_to_unicode_[byte] = unicode_char; - unicode_to_byte_[unicode_char] = byte; - } else { - int unicode_point = unicode_start++; - std::string unicode_char; - if (unicode_point < 0x800) { - unicode_char += static_cast(0xC0 | (unicode_point >> 6)); - unicode_char += static_cast(0x80 | (unicode_point & 0x3F)); - } else { - unicode_char += static_cast(0xE0 | (unicode_point >> 12)); - unicode_char += static_cast(0x80 | ((unicode_point >> 6) & 0x3F)); - unicode_char += static_cast(0x80 | (unicode_point & 0x3F)); - } - byte_to_unicode_[byte] = unicode_char; - unicode_to_byte_[unicode_char] = byte; - } - } -} - -std::string BPETokenizer::bytes_to_unicode(const std::string& text) const { - init_byte_mappings(); - - std::string result; - for (uint8_t byte : text) { - result += byte_to_unicode_.at(byte); - } - return result; -} - -std::string BPETokenizer::unicode_to_bytes(const std::string& text) const { - init_byte_mappings(); - - std::string result; - size_t i = 0; - while (i < text.length()) { - std::string unicode_char; - - if ((text[i] & 0x80) == 0) { - unicode_char = text.substr(i, 1); - i += 1; - } else if ((text[i] & 0xE0) == 0xC0) { - unicode_char = text.substr(i, 2); - i += 2; - } else if ((text[i] & 0xF0) == 0xE0) { - unicode_char = text.substr(i, 3); - i += 3; - } else { - unicode_char = text.substr(i, 1); - i += 1; - } - - auto it = unicode_to_byte_.find(unicode_char); - if (it != unicode_to_byte_.end()) { - result += static_cast(it->second); - } else if (unicode_char.size() == 1 && (unsigned char)unicode_char[0] < 128) { - result += unicode_char[0]; - } else { - result += '?'; - } - } - return result; -} - -std::vector BPETokenizer::byte_level_split(const std::string& text) const { - std::string unicode_text = bytes_to_unicode(text); - - std::vector chars; - size_t i = 0; - while (i < unicode_text.length()) { - size_t char_len = 1; - - if ((unicode_text[i] & 0x80) == 0) { - char_len = 1; - } else if ((unicode_text[i] & 0xE0) == 0xC0) { - char_len = 2; - } else if ((unicode_text[i] & 0xF0) == 0xE0) { - char_len = 3; - } else if ((unicode_text[i] & 0xF8) == 0xF0) { - char_len = 4; - } - - if (i + char_len <= unicode_text.length()) { - chars.push_back(unicode_text.substr(i, char_len)); - } - i += char_len; - } - - return chars; -} - -std::vector BPETokenizer::utf8_split(const std::string& text) const { - std::vector chars; - size_t i = 0; - while (i < text.length()) { - size_t char_len = 1; - const unsigned char byte = static_cast(text[i]); - - if ((byte & 0x80) == 0) { - char_len = 1; - } else if ((byte & 0xE0) == 0xC0) { - char_len = 2; - } else if ((byte & 0xF0) == 0xE0) { - char_len = 3; - } else if ((byte & 0xF8) == 0xF0) { - char_len = 4; - } - - if (i + char_len <= text.length()) { - chars.push_back(text.substr(i, char_len)); - } - i += char_len; - } - - return chars; -} - - -std::pair BPETokenizer::find_best_merge_fast(const std::vector& tokens) const { - int best_pos = -1; - uint32_t best_priority = UINT32_MAX; - - for (size_t i = 0; i < tokens.size() - 1; ++i) { - std::string key = tokens[i] + "\x00" + tokens[i + 1]; - auto it = merge_map_.find(key); - if (it != merge_map_.end()) { - if (it->second < best_priority) { - best_priority = it->second; - best_pos = static_cast(i); - } - } - } - - return {best_pos, best_priority}; -} - -std::vector BPETokenizer::apply_bpe(const std::vector& tokens) const { - if (tokens.size() <= 1) return tokens; - - std::vector current_tokens = tokens; - - - while (true) { - auto [merge_pos, priority] = find_best_merge_fast(current_tokens); - if (merge_pos == -1) break; - - - std::vector new_tokens; - new_tokens.reserve(current_tokens.size() - 1); - - for (int i = 0; i < static_cast(current_tokens.size()); ++i) { - if (i == merge_pos) { - std::string merged = current_tokens[i] + current_tokens[i + 1]; - new_tokens.push_back(merged); - i++; - } else { - new_tokens.push_back(current_tokens[i]); - } - } - current_tokens = std::move(new_tokens); - } - - return current_tokens; -} - -std::vector BPETokenizer::encode(const std::string& text) const { - if (text.empty()) return {}; - - - auto text_segments = split_with_special_tokens(text); - - - std::vector token_ids; - - for (const auto& segment : text_segments) { - auto special_it = special_tokens_.find(segment); - if (special_it != special_tokens_.end()) { - token_ids.push_back(special_it->second); - } else { - std::string normalized_segment = segment; - std::vector chars; - if (runtime_config_.normalizer == TokenizerRuntimeConfig::Normalizer::METASPACE) { - size_t pos = 0; - while ((pos = normalized_segment.find(' ', pos)) != std::string::npos) { - normalized_segment.replace(pos, 1, kMetaspace); - pos += 3; - } - chars = utf8_split(normalized_segment); - } else { - chars = byte_level_split(segment); - } - auto bpe_tokens = apply_bpe(chars); - - for (const auto& token : bpe_tokens) { - auto it = token_to_id_.find(token); - if (it != token_to_id_.end()) { - token_ids.push_back(it->second); - } else { - if (!runtime_config_.byte_fallback) { - token_ids.push_back(unk_token_id_); - continue; - } - - std::vector fallback_ids; - fallback_ids.reserve(token.size()); - for (unsigned char byte : token) { - char fallback_token[7]; - std::snprintf(fallback_token, sizeof(fallback_token), "<0x%02X>", byte); - auto fallback_it = token_to_id_.find(fallback_token); - if (fallback_it == token_to_id_.end()) { - fallback_ids.clear(); - break; - } - fallback_ids.push_back(fallback_it->second); - } - - if (fallback_ids.empty()) { - token_ids.push_back(unk_token_id_); - } else { - token_ids.insert(token_ids.end(), fallback_ids.begin(), fallback_ids.end()); - } - } - } - } - } - - return token_ids; -} - -std::string BPETokenizer::decode(const std::vector& tokens) const { - if (runtime_config_.decoder == TokenizerRuntimeConfig::Decoder::REPLACE_METASPACE) { - std::string result; - result.reserve(tokens.size() * 4); - - for (uint32_t token_id : tokens) { - if (token_id >= id_to_token_.size()) continue; - result += id_to_token_[token_id]; - } - - size_t pos = 0; - while ((pos = result.find(kMetaspace, pos)) != std::string::npos) { - result.replace(pos, 3, " "); - pos += 1; - } - return result; - } - - std::string unicode_result; - unicode_result.reserve(tokens.size() * 4); - - for (uint32_t token_id : tokens) { - if (token_id >= id_to_token_.size()) continue; - const std::string& tok = id_to_token_[token_id]; - - size_t pos = 0; - while (pos < tok.size()) { - if (pos + 3 <= tok.size() && - (unsigned char)tok[pos] == 0xE2 && - (unsigned char)tok[pos+1] == 0x96 && - (unsigned char)tok[pos+2] == 0x81) { - unicode_result.push_back(' '); - pos += 3; - } else { - unicode_result.push_back(tok[pos++]); - } - } - } - - return unicode_to_bytes(unicode_result); -} - - -} -} diff --git a/cactus-transpile/engine/engine_cache.cpp b/cactus-transpile/engine/engine_cache.cpp deleted file mode 100644 index 77a7381b4..000000000 --- a/cactus-transpile/engine/engine_cache.cpp +++ /dev/null @@ -1,691 +0,0 @@ -#include "engine.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include -#include -#include - -namespace cactus { -namespace engine { - -void KVCache::init(size_t layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision) { - num_layers = layers; - max_seq_len = max_seq; - precision = model_precision; - element_size = PrecisionTraits::size_of(precision); - - layer_caches.resize(num_layers); - for (size_t i = 0; i < num_layers; i++) { - layer_caches[i].head_dim = layer_dims[i]; - layer_caches[i].kv_heads = layer_kv_heads[i]; - } - - current_seq_len = 0; - total_seq_len = 0; -} - -void KVCache::set_window_size(size_t window, size_t sink) { - window_size = window; - sink_size = sink; - - if (window_size > 0) { - for (size_t layer_idx = 0; layer_idx < num_layers; layer_idx++) { - size_t head_dim = get_layer_head_dim(layer_idx); - if (head_dim == 0) continue; - size_t layer_kv = get_layer_kv_heads(layer_idx); - size_t cache_bytes = window_size * layer_kv * head_dim * element_size; - size_t num_groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t num_scales = window_size * layer_kv * num_groups; - auto& cache = layer_caches[layer_idx]; - - cache.keys.resize(cache_bytes); - cache.values.resize(cache_bytes); - std::memset(cache.keys.data(), 0, cache_bytes); - std::memset(cache.values.data(), 0, cache_bytes); - - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - std::fill(cache.key_scales.begin(), cache.key_scales.end(), 1.0f); - std::fill(cache.value_scales.begin(), cache.value_scales.end(), 1.0f); - } - } -} - -void KVCache::reset() { - current_seq_len = 0; - total_seq_len = 0; -} - -void* KVCache::get_key_ptr(size_t layer) { - if (current_seq_len == 0 || layer >= num_layers) return nullptr; - return layer_caches[layer].keys.data(); -} - -void* KVCache::get_value_ptr(size_t layer) { - if (current_seq_len == 0 || layer >= num_layers) return nullptr; - return layer_caches[layer].values.data(); -} - -KVCache::CircularView KVCache::get_key_view(size_t layer) { - CircularView view; - if (layer >= num_layers || current_seq_len == 0) { - view.ptr1 = nullptr; - view.ptr2 = nullptr; - view.len1 = 0; - view.len2 = 0; - view.total_len = 0; - return view; - } - - view.ptr1 = layer_caches[layer].keys.data(); - view.ptr2 = nullptr; - view.len1 = current_seq_len; - view.len2 = 0; - view.total_len = current_seq_len; - return view; -} - -KVCache::CircularView KVCache::get_value_view(size_t layer) { - CircularView view; - if (layer >= num_layers || current_seq_len == 0) { - view.ptr1 = nullptr; - view.ptr2 = nullptr; - view.len1 = 0; - view.len2 = 0; - view.total_len = 0; - return view; - } - - view.ptr1 = layer_caches[layer].values.data(); - view.ptr2 = nullptr; - view.len1 = current_seq_len; - view.len2 = 0; - view.total_len = current_seq_len; - return view; -} - -void KVCache::update_from_graph(CactusGraph* gb, const std::vector& k_nodes, - const std::vector& v_nodes, size_t seq_len, - size_t layers) { - size_t old_seq_len = current_seq_len; - size_t new_total_len = old_seq_len + seq_len; - - total_seq_len += seq_len; - - size_t effective_seq_len; - bool use_sliding_window = (window_size > 0 && new_total_len > window_size); - - if (use_sliding_window) { - effective_seq_len = window_size; - } else { - effective_seq_len = new_total_len; - } - - bool any_layer_updated = false; - - for (size_t layer_idx = 0; layer_idx < layers; layer_idx++) { - if (k_nodes[layer_idx] == 0 || v_nodes[layer_idx] == 0) - continue; - - auto& cache = layer_caches[layer_idx]; - - void* k_output = gb->get_output(k_nodes[layer_idx]); - void* v_output = gb->get_output(v_nodes[layer_idx]); - - if (k_output && v_output) { - const auto& k_buffer = gb->get_output_buffer(k_nodes[layer_idx]); - const auto& v_buffer = gb->get_output_buffer(v_nodes[layer_idx]); - - size_t dim = get_layer_head_dim(layer_idx); - size_t kv_heads = get_layer_kv_heads(layer_idx); - size_t elements_per_token = kv_heads * dim; - size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t scales_per_token = kv_heads * num_groups; - size_t bytes_per_token = elements_per_token * element_size; - - size_t expected_elements = new_total_len * elements_per_token; - - if (k_buffer.total_size == expected_elements && v_buffer.total_size == expected_elements) { - any_layer_updated = true; - - if (!use_sliding_window) { - size_t total_bytes = new_total_len * bytes_per_token; - cache.keys.resize(total_bytes); - cache.values.resize(total_bytes); - - if (precision == Precision::INT8) { - size_t num_scales = new_total_len * scales_per_token; - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - - cactus_quantize_kv_fp16_to_int8( - static_cast(k_output), - reinterpret_cast(cache.keys.data()), - cache.key_scales.data(), - new_total_len, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - static_cast(v_output), - reinterpret_cast(cache.values.data()), - cache.value_scales.data(), - new_total_len, kv_heads, dim); - } else { - std::memcpy(cache.keys.data(), k_output, total_bytes); - std::memcpy(cache.values.data(), v_output, total_bytes); - } - } else { - size_t cache_bytes = window_size * bytes_per_token; - size_t remaining_window = window_size - sink_size; - size_t skip_tokens = new_total_len - window_size; - - bool first_slide = (cache.keys.size() != cache_bytes); - - if (first_slide) { - cache.keys.resize(cache_bytes); - cache.values.resize(cache_bytes); - } - - if (precision == Precision::INT8) { - size_t num_scales = window_size * scales_per_token; - if (first_slide) { - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - } - - const __fp16* k_fp16 = static_cast(k_output); - const __fp16* v_fp16 = static_cast(v_output); - - if (first_slide) { - cactus_quantize_kv_fp16_to_int8( - k_fp16, - reinterpret_cast(cache.keys.data()), - cache.key_scales.data(), - sink_size, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_fp16, - reinterpret_cast(cache.values.data()), - cache.value_scales.data(), - sink_size, kv_heads, dim); - } - - size_t src_offset = (sink_size + skip_tokens) * elements_per_token; - size_t dst_offset = sink_size * elements_per_token; - size_t scale_dst_offset = sink_size * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - k_fp16 + src_offset, - reinterpret_cast(cache.keys.data()) + dst_offset, - cache.key_scales.data() + scale_dst_offset, - remaining_window, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_fp16 + src_offset, - reinterpret_cast(cache.values.data()) + dst_offset, - cache.value_scales.data() + scale_dst_offset, - remaining_window, kv_heads, dim); - } else { - if (first_slide) { - size_t sink_bytes = sink_size * bytes_per_token; - std::memcpy(cache.keys.data(), k_output, sink_bytes); - std::memcpy(cache.values.data(), v_output, sink_bytes); - } - - const uint8_t* k_src = static_cast(k_output) + - (sink_size + skip_tokens) * bytes_per_token; - const uint8_t* v_src = static_cast(v_output) + - (sink_size + skip_tokens) * bytes_per_token; - size_t sink_bytes = sink_size * bytes_per_token; - size_t recent_bytes = remaining_window * bytes_per_token; - - std::memcpy(cache.keys.data() + sink_bytes, k_src, recent_bytes); - std::memcpy(cache.values.data() + sink_bytes, v_src, recent_bytes); - } - } - } - else if (seq_len * elements_per_token == k_buffer.total_size && - seq_len * elements_per_token == v_buffer.total_size) { - any_layer_updated = true; - - if (!use_sliding_window) { - size_t old_bytes = old_seq_len * bytes_per_token; - size_t new_bytes = seq_len * bytes_per_token; - size_t total_bytes = old_bytes + new_bytes; - cache.keys.resize(total_bytes); - cache.values.resize(total_bytes); - - if (precision == Precision::INT8) { - size_t num_scales = (old_seq_len + seq_len) * scales_per_token; - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - - size_t dst_offset = old_seq_len * elements_per_token; - size_t scale_offset = old_seq_len * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - static_cast(k_output), - reinterpret_cast(cache.keys.data()) + dst_offset, - cache.key_scales.data() + scale_offset, - seq_len, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - static_cast(v_output), - reinterpret_cast(cache.values.data()) + dst_offset, - cache.value_scales.data() + scale_offset, - seq_len, kv_heads, dim); - } else { - std::memcpy(cache.keys.data() + old_bytes, k_output, new_bytes); - std::memcpy(cache.values.data() + old_bytes, v_output, new_bytes); - } - } else { - size_t cache_bytes = window_size * bytes_per_token; - - if (cache.keys.size() != cache_bytes) { - cache.keys.resize(cache_bytes); - cache.values.resize(cache_bytes); - } - - size_t tokens_to_shift = window_size - sink_size - seq_len; - size_t sink_bytes = sink_size * bytes_per_token; - - if (tokens_to_shift > 0 && old_seq_len > sink_size) { - size_t shift_src = old_seq_len - tokens_to_shift; - if (shift_src > sink_size) { - size_t shift_bytes = tokens_to_shift * bytes_per_token; - - std::memmove(cache.keys.data() + sink_bytes, - cache.keys.data() + shift_src * bytes_per_token, - shift_bytes); - std::memmove(cache.values.data() + sink_bytes, - cache.values.data() + shift_src * bytes_per_token, - shift_bytes); - - if (precision == Precision::INT8) { - size_t sink_scale_offset = sink_size * scales_per_token; - size_t shift_src_scale_offset = shift_src * scales_per_token; - size_t shift_scale_count = tokens_to_shift * scales_per_token; - - std::memmove(cache.key_scales.data() + sink_scale_offset, - cache.key_scales.data() + shift_src_scale_offset, - shift_scale_count * sizeof(float)); - std::memmove(cache.value_scales.data() + sink_scale_offset, - cache.value_scales.data() + shift_src_scale_offset, - shift_scale_count * sizeof(float)); - } - } - } - - size_t append_token_offset = window_size - seq_len; - size_t append_bytes_offset = append_token_offset * bytes_per_token; - size_t new_bytes = seq_len * bytes_per_token; - - if (precision == Precision::INT8) { - size_t append_offset = append_token_offset * elements_per_token; - size_t scale_offset = append_token_offset * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - static_cast(k_output), - reinterpret_cast(cache.keys.data()) + append_offset, - cache.key_scales.data() + scale_offset, - seq_len, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - static_cast(v_output), - reinterpret_cast(cache.values.data()) + append_offset, - cache.value_scales.data() + scale_offset, - seq_len, kv_heads, dim); - } else { - std::memcpy(cache.keys.data() + append_bytes_offset, k_output, new_bytes); - std::memcpy(cache.values.data() + append_bytes_offset, v_output, new_bytes); - } - } - } - } - } - - if (any_layer_updated) { - current_seq_len = effective_seq_len; - } -} - -void KVCache::update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, - size_t num_tokens, size_t kv_heads, size_t dim) { - if (layer_idx >= num_layers || !k_data || !v_data || num_tokens == 0) { - return; - } - - auto& cache = layer_caches[layer_idx]; - size_t old_seq_len = current_seq_len; - size_t new_total_len = old_seq_len + num_tokens; - size_t elements_per_token = kv_heads * dim; - size_t bytes_per_token = elements_per_token * element_size; - size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t scales_per_token = kv_heads * num_groups; - - if (layer_idx == 0) { - total_seq_len += num_tokens; - } - - bool use_sliding_window = (window_size > 0 && new_total_len > window_size); - size_t effective_seq_len = use_sliding_window ? window_size : new_total_len; - - if (!use_sliding_window) { - size_t total_bytes = new_total_len * bytes_per_token; - cache.keys.resize(total_bytes); - cache.values.resize(total_bytes); - - size_t num_scales = new_total_len * scales_per_token; - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - - size_t dst_offset = old_seq_len * elements_per_token; - size_t scale_offset = old_seq_len * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - k_data, - reinterpret_cast(cache.keys.data()) + dst_offset, - cache.key_scales.data() + scale_offset, - num_tokens, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_data, - reinterpret_cast(cache.values.data()) + dst_offset, - cache.value_scales.data() + scale_offset, - num_tokens, kv_heads, dim); - } else { - size_t cache_bytes = window_size * bytes_per_token; - - if (cache.keys.size() != cache_bytes) { - cache.keys.resize(cache_bytes); - cache.values.resize(cache_bytes); - - size_t num_scales = window_size * scales_per_token; - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - } - - size_t remaining_window = window_size - sink_size; - - if (num_tokens >= remaining_window) { - size_t skip_tokens = num_tokens - remaining_window; - size_t dst_offset = sink_size * elements_per_token; - size_t scale_offset = sink_size * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - k_data + skip_tokens * elements_per_token, - reinterpret_cast(cache.keys.data()) + dst_offset, - cache.key_scales.data() + scale_offset, - remaining_window, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_data + skip_tokens * elements_per_token, - reinterpret_cast(cache.values.data()) + dst_offset, - cache.value_scales.data() + scale_offset, - remaining_window, kv_heads, dim); - } else { - size_t tokens_to_shift = remaining_window - num_tokens; - - if (tokens_to_shift > 0 && old_seq_len > sink_size) { - size_t shift_src = old_seq_len - tokens_to_shift; - if (shift_src > sink_size) { - size_t sink_offset = sink_size * elements_per_token; - size_t shift_src_offset = shift_src * elements_per_token; - size_t shift_bytes = tokens_to_shift * bytes_per_token; - - std::memmove(cache.keys.data() + sink_offset, - cache.keys.data() + shift_src_offset, - shift_bytes); - std::memmove(cache.values.data() + sink_offset, - cache.values.data() + shift_src_offset, - shift_bytes); - - size_t sink_scale_offset = sink_size * scales_per_token; - size_t shift_src_scale_offset = shift_src * scales_per_token; - size_t shift_scale_count = tokens_to_shift * scales_per_token; - - std::memmove(cache.key_scales.data() + sink_scale_offset, - cache.key_scales.data() + shift_src_scale_offset, - shift_scale_count * sizeof(float)); - std::memmove(cache.value_scales.data() + sink_scale_offset, - cache.value_scales.data() + shift_src_scale_offset, - shift_scale_count * sizeof(float)); - } - } - - size_t append_token_offset = window_size - num_tokens; - size_t append_offset = append_token_offset * elements_per_token; - size_t scale_offset = append_token_offset * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - k_data, - reinterpret_cast(cache.keys.data()) + append_offset, - cache.key_scales.data() + scale_offset, - num_tokens, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_data, - reinterpret_cast(cache.values.data()) + append_offset, - cache.value_scales.data() + scale_offset, - num_tokens, kv_heads, dim); - } - } - - if (layer_idx == num_layers - 1) { - current_seq_len = effective_seq_len; - } -} - -void KVCache::remove_token_range(size_t start, size_t count) { - if (count == 0 || start >= current_seq_len || start + count > current_seq_len) return; - - size_t tail_tokens = current_seq_len - start - count; - - auto erase_bytes = [](std::vector& buf, size_t offset, size_t remove, size_t tail) { - std::memmove(buf.data() + offset, buf.data() + offset + remove, tail); - buf.resize(buf.size() - remove); - }; - - auto erase_floats = [](std::vector& buf, size_t offset, size_t remove, size_t tail) { - std::memmove(buf.data() + offset, buf.data() + offset + remove, tail * sizeof(float)); - buf.resize(buf.size() - remove); - }; - - for (size_t i = 0; i < layer_caches.size(); i++) { - size_t dim = get_layer_head_dim(i); - if (dim == 0) continue; - auto& layer = layer_caches[i]; - size_t num_kv_heads = get_layer_kv_heads(i); - size_t bytes_per_tok = num_kv_heads * dim * element_size; - - erase_bytes(layer.keys, start * bytes_per_tok, count * bytes_per_tok, tail_tokens * bytes_per_tok); - erase_bytes(layer.values, start * bytes_per_tok, count * bytes_per_tok, tail_tokens * bytes_per_tok); - - if (precision == Precision::INT8 && !layer.key_scales.empty()) { - size_t scaler_per_tok = num_kv_heads * dim / KV_QUANT_GROUP_SIZE; - erase_floats(layer.key_scales, start * scaler_per_tok, count * scaler_per_tok, tail_tokens * scaler_per_tok); - erase_floats(layer.value_scales, start * scaler_per_tok, count * scaler_per_tok, tail_tokens * scaler_per_tok); - } - } - - current_seq_len -= count; - total_seq_len -= count; -} - -void KVCache::compact_to_windows(const std::vector& target_windows) { - for (size_t i = 0; i < layer_caches.size(); i++) { - size_t dim = get_layer_head_dim(i); - if (dim == 0) continue; - size_t target = (i < target_windows.size()) ? target_windows[i] : 0; - if (target == 0) continue; - - auto& layer = layer_caches[i]; - size_t num_kv_heads = get_layer_kv_heads(i); - size_t layer_tokens = layer.keys.size() / (num_kv_heads * dim * element_size); - if (layer_tokens <= target) continue; - - size_t bytes_per_token = num_kv_heads * dim * element_size; - size_t keep_bytes = target * bytes_per_token; - size_t discard_bytes = (layer_tokens - target) * bytes_per_token; - - std::memmove(layer.keys.data(), layer.keys.data() + discard_bytes, keep_bytes); - layer.keys.resize(keep_bytes); - std::memmove(layer.values.data(), layer.values.data() + discard_bytes, keep_bytes); - layer.values.resize(keep_bytes); - - if (precision == Precision::INT8 && !layer.key_scales.empty()) { - size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t scales_per_token = num_kv_heads * num_groups; - size_t keep_scales = target * scales_per_token; - size_t discard_scales = (layer_tokens - target) * scales_per_token; - - std::memmove(layer.key_scales.data(), layer.key_scales.data() + discard_scales, keep_scales * sizeof(float)); - layer.key_scales.resize(keep_scales); - std::memmove(layer.value_scales.data(), layer.value_scales.data() + discard_scales, keep_scales * sizeof(float)); - layer.value_scales.resize(keep_scales); - } - } -} - -const int8_t* KVCache::get_keys_int8(size_t layer) const { - if (layer >= num_layers || current_seq_len == 0) { - return nullptr; - } - return reinterpret_cast(layer_caches[layer].keys.data()); -} - -const int8_t* KVCache::get_values_int8(size_t layer) const { - if (layer >= num_layers || current_seq_len == 0) { - return nullptr; - } - return reinterpret_cast(layer_caches[layer].values.data()); -} - -const float* KVCache::get_key_scales(size_t layer) const { - if (layer >= num_layers || current_seq_len == 0) { - return nullptr; - } - return layer_caches[layer].key_scales.data(); -} - -const float* KVCache::get_value_scales(size_t layer) const { - if (layer >= num_layers || current_seq_len == 0) { - return nullptr; - } - return layer_caches[layer].value_scales.data(); -} - -void ConvCache::init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision) { - num_layers = layers; - hidden_size = hidden_dim; - window_size = window_len; - precision = model_precision; - element_size = PrecisionTraits::size_of(precision); - - size_t state_bytes = window_size * hidden_size * element_size; - layer_states.resize(num_layers); - for (auto& state : layer_states) { - state.data.resize(state_bytes); - std::memset(state.data.data(), 0, state_bytes); - state.head = 0; - state.count = 0; - } -} - -ConvCache::CircularView ConvCache::get_window(size_t layer) const { - CircularView view; - if (layer >= num_layers) { - view.ptr1 = nullptr; - view.len1 = 0; - view.ptr2 = nullptr; - view.len2 = 0; - view.total_len = 0; - return view; - } - - const auto& state = layer_states[layer]; - if (state.count == 0) { - view.ptr1 = nullptr; - view.len1 = 0; - view.ptr2 = nullptr; - view.len2 = 0; - view.total_len = 0; - return view; - } - - size_t stride = hidden_size * element_size; - - if (state.count < window_size) { - view.ptr1 = state.data.data(); - view.len1 = state.count; - view.ptr2 = nullptr; - view.len2 = 0; - view.total_len = state.count; - return view; - } - - view.ptr1 = state.data.data(); - view.len1 = state.head; - view.ptr2 = state.data.data() + state.head * stride; - view.len2 = window_size - state.head; - view.total_len = window_size; - return view; -} - -void ConvCache::update(CactusGraph* gb, size_t layer, const size_t bx_node) { - if (layer >= num_layers || !bx_node || window_size == 0 || hidden_size == 0) { - return; - } - - auto& state = layer_states[layer]; - const void* output_ptr = gb->get_output(bx_node); - if (!output_ptr) { - return; - } - - const auto& buffer = gb->get_output_buffer(bx_node); - const size_t stride_bytes = hidden_size * element_size; - - size_t rows = 1; - if (!buffer.shape.empty()) { - if (buffer.shape.size() == 1) { - rows = 1; - } else { - rows = buffer.shape[0]; - } - } - - if (buffer.total_size > 0 && hidden_size > 0) { - size_t inferred = buffer.total_size / hidden_size; - if (inferred > 0) { - rows = inferred; - } - } - - if (rows == 0) { - return; - } - - size_t copy_rows = std::min(rows, window_size); - size_t start_row = rows > window_size ? rows - window_size : 0; - - const uint8_t* src = static_cast(output_ptr) + start_row * stride_bytes; - - for (size_t i = 0; i < copy_rows; ++i) { - std::memcpy(state.data.data() + state.head * stride_bytes, src + i * stride_bytes, stride_bytes); - state.head = (state.head + 1) % window_size; - if (state.count < window_size) { - ++state.count; - } - } -} - -void ConvCache::reset() { - for (auto& state : layer_states) { - std::fill(state.data.begin(), state.data.end(), 0); - state.head = 0; - state.count = 0; - } -} - -} -} \ No newline at end of file diff --git a/cactus-transpile/engine/engine_constraints.cpp b/cactus-transpile/engine/engine_constraints.cpp deleted file mode 100644 index 2e18c7c14..000000000 --- a/cactus-transpile/engine/engine_constraints.cpp +++ /dev/null @@ -1,1137 +0,0 @@ -#include "engine.h" - -namespace cactus { -namespace engine { - -constexpr float FORCE_BIAS = 500.0f; -constexpr float BLOCK_BIAS = -500.0f; - -void ToolCallConstrainer::add_tokens_for_string(const std::string& str, std::unordered_set& token_set) { - if (!tokenizer_) return; - auto tokens = tokenizer_->encode(str); - for (uint32_t t : tokens) { - token_set.insert(t); - } -} - -void ToolCallConstrainer::tokenize_function_names(bool quote_names) { - all_func_name_tokens_.clear(); - func_name_sequences_.clear(); - - for (const auto& name : function_names_) { - std::string name_to_encode = quote_names ? ("\"" + name + "\"") : name; - auto tokens = tokenizer_->encode(name_to_encode); - func_name_sequences_[name] = tokens; - for (uint32_t t : tokens) { - all_func_name_tokens_.insert(t); - } - if (quote_names) { - auto unquoted_tokens = tokenizer_->encode(name); - for (uint32_t t : unquoted_tokens) { - all_func_name_tokens_.insert(t); - } - } - } -} - -void ToolCallConstrainer::init_common_tokens() { - backtick_tokens_.clear(); - add_tokens_for_string("`", backtick_tokens_); - add_tokens_for_string("``", backtick_tokens_); - add_tokens_for_string("```", backtick_tokens_); - add_tokens_for_string("````", backtick_tokens_); - add_tokens_for_string("```json", backtick_tokens_); - add_tokens_for_string("```JSON", backtick_tokens_); - add_tokens_for_string("``` json", backtick_tokens_); - add_tokens_for_string("```\n", backtick_tokens_); - add_tokens_for_string("` ", backtick_tokens_); -} - -void ToolCallConstrainer::needle_insert_word(NeedleTrieNode* root, const std::string& word) { - if (!root) return; - - NeedleTrieNode* node = root; - for (char ch : word) { - auto& child = node->children[ch]; - if (!child) { - child = std::make_unique(); - } - node = child.get(); - } - node->is_terminal = true; -} - -const ToolCallConstrainer::NeedleTrieNode* ToolCallConstrainer::needle_get_trie_node( - const NeedleTrieNode* root, - const std::string& prefix -) const { - const NeedleTrieNode* node = root; - for (char ch : prefix) { - if (!node) return nullptr; - auto it = node->children.find(ch); - if (it == node->children.end()) { - return nullptr; - } - node = it->second.get(); - } - return node; -} - -bool ToolCallConstrainer::needle_has_unseen_completion(const NeedleTrieNode* node, std::string& partial) const { - if (!node) return false; - if (node->is_terminal && !needle_seen_arg_keys_.count(partial)) return true; - for (const auto& [ch, child] : node->children) { - partial.push_back(ch); - if (needle_has_unseen_completion(child.get(), partial)) { - partial.pop_back(); - return true; - } - partial.pop_back(); - } - return false; -} - -bool ToolCallConstrainer::needle_check_token_valid(const std::string& token_text, - const NeedleTrieNode* trie_node) const { - const NeedleTrieNode* node = trie_node; - if (!node) return false; - - for (char ch : token_text) { - if (ch == '"') { - return node->is_terminal; - } - auto it = node->children.find(ch); - if (it == node->children.end()) { - return false; - } - node = it->second.get(); - } - return true; -} - -void ToolCallConstrainer::reset_needle_constraints() { - needle_json_state_ = NeedleJsonState::FREE; - needle_buffer_.clear(); - needle_constrained_buf_.clear(); - needle_current_function_.clear(); - needle_seen_arg_keys_.clear(); - needle_in_arguments_ = false; - needle_arguments_depth_ = 0; - needle_nesting_depth_ = 0; - needle_in_string_value_ = false; - needle_between_pairs_ = false; - needle_prev_char_escape_ = false; -} - -bool ToolCallConstrainer::needle_at_arg_key_start() const { - if (needle_buffer_.size() < 2) return false; - return needle_buffer_.compare(needle_buffer_.size() - 2, 2, "{\"") == 0 || - needle_buffer_.compare(needle_buffer_.size() - 2, 2, ",\"") == 0; -} - -bool ToolCallConstrainer::needle_is_value_string_start() const { - if (needle_buffer_.empty()) return false; - for (size_t i = needle_buffer_.size() - 1; i-- > 0;) { - char ch = needle_buffer_[i]; - if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r') { - continue; - } - return ch == ':'; - } - return false; -} - -void ToolCallConstrainer::feed_needle_char(char ch) { - if (needle_json_state_ == NeedleJsonState::IN_NAME || - needle_json_state_ == NeedleJsonState::IN_ARG_KEY) { - if (ch == '"') { - if (needle_json_state_ == NeedleJsonState::IN_NAME) { - needle_current_function_ = needle_constrained_buf_; - needle_seen_arg_keys_.clear(); - } else { - needle_seen_arg_keys_.insert(needle_constrained_buf_); - } - needle_constrained_buf_.clear(); - needle_json_state_ = NeedleJsonState::FREE; - } else { - needle_constrained_buf_.push_back(ch); - } - needle_buffer_.push_back(ch); - return; - } - - needle_buffer_.push_back(ch); - - if (needle_in_string_value_) { - if (needle_prev_char_escape_) { - needle_prev_char_escape_ = false; - return; - } - if (ch == '\\') { - needle_prev_char_escape_ = true; - return; - } - if (ch == '"') { - needle_in_string_value_ = false; - if (needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { - needle_between_pairs_ = true; - } - } - return; - } - - if (ch == '{' || ch == '[') { - needle_nesting_depth_++; - } else if (ch == '}' || ch == ']') { - needle_nesting_depth_ = std::max(0, needle_nesting_depth_ - 1); - if (ch == '}' && needle_in_arguments_ && needle_nesting_depth_ < needle_arguments_depth_) { - needle_in_arguments_ = false; - } - } else if (ch == ',' && needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { - needle_between_pairs_ = true; - } - - if (needle_buffer_.size() >= 8 && - needle_buffer_.compare(needle_buffer_.size() - 8, 8, "\"name\":\"") == 0) { - needle_json_state_ = NeedleJsonState::IN_NAME; - needle_constrained_buf_.clear(); - return; - } - - if (needle_buffer_.size() >= 13 && - needle_buffer_.compare(needle_buffer_.size() - 13, 13, "\"arguments\":{") == 0) { - needle_in_arguments_ = true; - needle_arguments_depth_ = needle_nesting_depth_; - return; - } - - if (needle_in_arguments_ && needle_at_arg_key_start()) { - needle_json_state_ = NeedleJsonState::IN_ARG_KEY; - needle_constrained_buf_.clear(); - needle_between_pairs_ = false; - return; - } - - if (ch == '"' && needle_is_value_string_start()) { - needle_in_string_value_ = true; - } -} - -void ToolCallConstrainer::feed_needle_text(const std::string& text) { - for (char ch : text) { - feed_needle_char(ch); - } -} - -void ToolCallConstrainer::init_needle_constraints() { - reset_needle_constraints(); - - needle_name_trie_ = std::make_unique(); - needle_param_tries_.clear(); - needle_required_params_.clear(); - for (const auto& tool : tool_specs_) { - if (!tool.name.empty()) { - needle_insert_word(needle_name_trie_.get(), tool.name); - } - - auto param_root = std::make_unique(); - for (const auto& param_name : tool.parameter_names) { - if (!param_name.empty()) { - needle_insert_word(param_root.get(), param_name); - } - } - needle_param_tries_[tool.name] = std::move(param_root); - needle_required_params_[tool.name] = tool.required_parameter_names; - } - - const uint32_t vocab_size = tokenizer_ ? tokenizer_->get_vocab_size() : 0; - if (needle_token_strings_.size() != vocab_size) { - needle_token_strings_.assign(vocab_size, ""); - needle_token_index_.clear(); - needle_arg_close_tokens_.clear(); - - auto would_close_args = [](const std::string& tok, bool start_in_str) -> bool { - bool in_str = start_in_str; - bool esc = false; - int depth = 0; - for (char ch : tok) { - if (in_str) { - if (esc) { esc = false; } - else if (ch == '\\') { esc = true; } - else if (ch == '"') { in_str = false; } - continue; - } - if (ch == '"') { in_str = true; continue; } - if (ch == '{' || ch == '[') { depth++; continue; } - if (ch == '}') { if (depth == 0) return true; depth--; } - else if (ch == ']' && depth > 0) depth--; - } - return false; - }; - - for (uint32_t token_id = 0; token_id < vocab_size; ++token_id) { - std::string decoded = tokenizer_->decode({token_id}); - needle_token_strings_[token_id] = decoded; - if (decoded.empty()) continue; - needle_token_index_[decoded.front()].push_back(token_id); - if (would_close_args(decoded, false) || would_close_args(decoded, true)) { - needle_arg_close_tokens_.insert(token_id); - } - } - } -} - -void ToolCallConstrainer::tokenize_grammar_elements() { - if (!tokenizer_) return; - - // Clear all token sets - open_brace_tokens_.clear(); - close_brace_tokens_.clear(); - colon_tokens_.clear(); - comma_tokens_.clear(); - name_key_tokens_.clear(); - args_key_tokens_.clear(); - quote_tokens_.clear(); - tool_start_tokens_.clear(); - tool_end_tokens_.clear(); - bracket_open_tokens_.clear(); - bracket_close_tokens_.clear(); - paren_open_tokens_.clear(); - paren_close_tokens_.clear(); - equals_tokens_.clear(); - - init_common_tokens(); - - if (model_type_ == Config::ModelType::LFM2) { - add_tokens_for_string("<|tool_call_start|>", tool_start_tokens_); - add_tokens_for_string("<|tool_call_end|>", tool_end_tokens_); - add_tokens_for_string("[", bracket_open_tokens_); - add_tokens_for_string("]", bracket_close_tokens_); - add_tokens_for_string("(", paren_open_tokens_); - add_tokens_for_string(")", paren_close_tokens_); - add_tokens_for_string("=", equals_tokens_); - add_tokens_for_string(",", comma_tokens_); - add_tokens_for_string("\"", quote_tokens_); - - tokenize_function_names(false); - } else if (is_gemma_family()) { - gemma_call_start_tokens_.clear(); - gemma_call_end_tokens_.clear(); - gemma_call_prefix_tokens_.clear(); - escape_tokens_.clear(); - gemma_response_start_tokens_.clear(); - - add_tokens_for_string(call_start_tag_, gemma_call_start_tokens_); - add_tokens_for_string(call_end_tag_, gemma_call_end_tokens_); - if (model_type_ == Config::ModelType::GEMMA4) { - add_tokens_for_string("<|tool_response>", gemma_response_start_tokens_); - } else { - add_tokens_for_string("", gemma_response_start_tokens_); - add_tokens_for_string("", escape_tokens_); - } - add_tokens_for_string("call:", gemma_call_prefix_tokens_); - - add_tokens_for_string("{", open_brace_tokens_); - add_tokens_for_string("}", close_brace_tokens_); - add_tokens_for_string(":", colon_tokens_); - add_tokens_for_string(",", comma_tokens_); - - tokenize_function_names(false); - } else if (is_needle()) { - qwen_tool_call_start_tokens_.clear(); - add_tokens_for_string("", qwen_tool_call_start_tokens_); - } else { - qwen_tool_call_start_tokens_.clear(); - qwen_tool_call_end_tokens_.clear(); - - add_tokens_for_string("", qwen_tool_call_start_tokens_); - add_tokens_for_string("", qwen_tool_call_end_tokens_); - - add_tokens_for_string("{", open_brace_tokens_); - add_tokens_for_string("}", close_brace_tokens_); - add_tokens_for_string(":", colon_tokens_); - add_tokens_for_string(",", comma_tokens_); - add_tokens_for_string("\"", quote_tokens_); - - add_tokens_for_string("name", name_key_tokens_); - - add_tokens_for_string("arguments", args_key_tokens_); - - tokenize_function_names(true); - } -} - -void ToolCallConstrainer::init(Config::ModelType model_type, - const std::vector& tools, - Tokenizer* tokenizer) { - model_type_ = model_type; - tool_specs_ = tools; - function_names_.clear(); - function_names_.reserve(tool_specs_.size()); - for (const auto& tool : tool_specs_) { - function_names_.push_back(tool.name); - } - tokenizer_ = tokenizer; - generated_text_.clear(); - brace_depth_ = 0; - active_ = !function_names_.empty() && tokenizer != nullptr; - - if (model_type_ == Config::ModelType::LFM2) { - state_ = State::LFM_START; - } else if (is_gemma_family()) { - state_ = State::GEMMA_START; - if (model_type_ == Config::ModelType::GEMMA4) { - call_start_tag_ = "<|tool_call>"; - call_end_tag_ = ""; - } else { - call_start_tag_ = ""; - call_end_tag_ = ""; - } - } else if (is_needle()) { - state_ = State::NEEDLE_START; - } else { - state_ = State::QWEN_START; - } - - if (!active_) { - return; - } - - tokenize_grammar_elements(); - if (is_needle()) { - init_needle_constraints(); - } - compute_bias(); -} - -void ToolCallConstrainer::update(uint32_t /*token_id*/, const std::string& decoded_text) { - if (!active_) return; - - generated_text_ += decoded_text; - - if (model_type_ == Config::ModelType::LFM2) { - switch (state_) { - case State::LFM_START: - if (generated_text_.find("<|tool_call_start|>") != std::string::npos) { - state_ = State::LFM_EXPECT_BRACKET; - generated_text_.clear(); - } - break; - - case State::LFM_EXPECT_BRACKET: - if (generated_text_.find("[") != std::string::npos) { - state_ = State::LFM_IN_FUNC_NAME; - generated_text_.clear(); - } - break; - - case State::LFM_IN_FUNC_NAME: - for (const auto& name : function_names_) { - if (generated_text_.find(name) != std::string::npos) { - state_ = State::LFM_EXPECT_PAREN; - generated_text_.clear(); - break; - } - } - break; - - case State::LFM_EXPECT_PAREN: - if (generated_text_.find("(") != std::string::npos) { - state_ = State::LFM_IN_ARGUMENTS; - generated_text_.clear(); - } - break; - - case State::LFM_IN_ARGUMENTS: - if (decoded_text.find(")") != std::string::npos) { - state_ = State::LFM_EXPECT_BRACKET_CLOSE; - generated_text_.clear(); - } else { - generated_text_.clear(); - } - break; - - case State::LFM_EXPECT_BRACKET_CLOSE: - if (generated_text_.find("]") != std::string::npos) { - state_ = State::LFM_EXPECT_END; - generated_text_.clear(); - } - break; - - case State::LFM_EXPECT_END: - if (generated_text_.find("<|tool_call_end|>") != std::string::npos) { - state_ = State::DONE; - generated_text_.clear(); - } - break; - - default: - break; - } - } else if (is_gemma_family()) { - switch (state_) { - case State::GEMMA_START: - if (generated_text_.find(call_start_tag_) != std::string::npos) { - state_ = State::GEMMA_EXPECT_CALL; - generated_text_.clear(); - } - break; - - case State::GEMMA_EXPECT_CALL: - if (generated_text_.find("call:") != std::string::npos) { - state_ = State::GEMMA_IN_FUNC_NAME; - generated_text_.clear(); - } - break; - - case State::GEMMA_IN_FUNC_NAME: - for (const auto& name : function_names_) { - if (generated_text_.find(name) != std::string::npos) { - state_ = State::GEMMA_EXPECT_BRACE; - generated_text_.clear(); - break; - } - } - break; - - case State::GEMMA_EXPECT_BRACE: - if (generated_text_.find("{") != std::string::npos) { - state_ = State::GEMMA_IN_ARGUMENTS; - brace_depth_ = 1; - generated_text_.clear(); - } - break; - - case State::GEMMA_IN_ARGUMENTS: - generated_text_.clear(); - for (char c : decoded_text) { - if (c == '{') brace_depth_++; - else if (c == '}') { - brace_depth_--; - if (brace_depth_ == 0) { - state_ = State::GEMMA_EXPECT_END; - break; - } - } - } - break; - - case State::GEMMA_EXPECT_END: - if (generated_text_.find(call_end_tag_) != std::string::npos) { - state_ = State::DONE; - generated_text_.clear(); - } - break; - - default: - break; - } - } else if (is_needle()) { - switch (state_) { - case State::NEEDLE_START: - if (generated_text_.find("") != std::string::npos) { - state_ = State::DONE; - generated_text_.clear(); - } - break; - - case State::DONE: - feed_needle_text(decoded_text); - generated_text_.clear(); - break; - - default: - break; - } - } else { - switch (state_) { - case State::QWEN_START: - if (generated_text_.find("") != std::string::npos) { - state_ = State::QWEN_EXPECT_OPEN_BRACE; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_OPEN_BRACE: { - size_t pos = generated_text_.find("{"); - if (pos != std::string::npos) { - state_ = State::QWEN_EXPECT_NAME_KEY; - generated_text_ = generated_text_.substr(pos + 1); - } - break; - } - - case State::QWEN_EXPECT_NAME_KEY: - if (generated_text_.find("name\"") != std::string::npos) { - state_ = State::QWEN_EXPECT_NAME_COLON; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_NAME_COLON: - if (generated_text_.find(":") != std::string::npos) { - state_ = State::QWEN_EXPECT_NAME_VALUE; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_NAME_VALUE: - for (const auto& name : function_names_) { - if (generated_text_.find(name + "\"") != std::string::npos) { - state_ = State::QWEN_EXPECT_COMMA; - generated_text_.clear(); - break; - } - } - break; - - case State::QWEN_EXPECT_COMMA: - if (generated_text_.find(",") != std::string::npos) { - state_ = State::QWEN_EXPECT_ARGS_KEY; - generated_text_.clear(); - } else if (generated_text_.find("}") != std::string::npos) { - state_ = State::QWEN_EXPECT_END; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_ARGS_KEY: - if (generated_text_.find("arguments\"") != std::string::npos) { - state_ = State::QWEN_EXPECT_ARGS_COLON; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_ARGS_COLON: - if (generated_text_.find(":") != std::string::npos) { - state_ = State::QWEN_IN_ARGUMENTS; - brace_depth_ = 0; - generated_text_.clear(); - } - break; - - case State::QWEN_IN_ARGUMENTS: - generated_text_.clear(); - for (char c : decoded_text) { - if (c == '{') brace_depth_++; - else if (c == '}') { - if (brace_depth_ > 0) { - brace_depth_--; - } else { - state_ = State::QWEN_EXPECT_CLOSE_BRACE; - generated_text_.clear(); - break; - } - } - } - break; - - case State::QWEN_EXPECT_CLOSE_BRACE: - if (generated_text_.find("}") != std::string::npos) { - state_ = State::QWEN_EXPECT_END; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_END: - if (generated_text_.find("") != std::string::npos) { - state_ = State::DONE; - generated_text_.clear(); - } - break; - - case State::DONE: - if (generated_text_.find("") != std::string::npos) { - state_ = State::QWEN_EXPECT_OPEN_BRACE; - generated_text_.clear(); - } - break; - - default: - break; - } - } - - compute_bias(); -} - -void ToolCallConstrainer::compute_bias() { - current_bias_.clear(); - - if (!active_) return; - - for (uint32_t t : backtick_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - - if (model_type_ == Config::ModelType::LFM2) { - switch (state_) { - case State::LFM_START: - for (uint32_t t : tool_start_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : bracket_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_EXPECT_BRACKET: - for (uint32_t t : bracket_open_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_IN_FUNC_NAME: - for (uint32_t t : all_func_name_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : paren_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : equals_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_EXPECT_PAREN: - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : equals_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_IN_ARGUMENTS: - for (uint32_t t : paren_close_tokens_) { - current_bias_[t] = 15.0f; - } - for (uint32_t t : equals_tokens_) { - current_bias_[t] = 10.0f; - } - for (uint32_t t : comma_tokens_) { - current_bias_[t] = 8.0f; - } - for (uint32_t t : quote_tokens_) { - current_bias_[t] = 5.0f; - } - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : tool_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_EXPECT_BRACKET_CLOSE: - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : equals_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_EXPECT_END: - for (uint32_t t : tool_end_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : bracket_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - default: - break; - } - } else if (is_gemma_family()) { - for (uint32_t t : gemma_response_start_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - - switch (state_) { - case State::GEMMA_START: - for (uint32_t t : gemma_call_start_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_EXPECT_CALL: - for (uint32_t t : gemma_call_prefix_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_IN_FUNC_NAME: - for (uint32_t t : all_func_name_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_EXPECT_BRACE: - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_IN_ARGUMENTS: - for (uint32_t t : colon_tokens_) { - current_bias_[t] = 10.0f; - } - for (uint32_t t : comma_tokens_) { - current_bias_[t] = 8.0f; - } - for (uint32_t t : escape_tokens_) { - current_bias_[t] = 5.0f; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_EXPECT_END: - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : gemma_call_start_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - default: - break; - } - } else if (is_needle()) { - switch (state_) { - case State::NEEDLE_START: - for (uint32_t t : qwen_tool_call_start_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - if (tokenizer_) { - current_bias_[tokenizer_->get_eos_token()] = FORCE_BIAS; - } - break; - - default: - if (needle_json_state_ == NeedleJsonState::FREE) { - if (needle_in_arguments_ && needle_nesting_depth_ == needle_arguments_depth_) { - auto param_it = needle_param_tries_.find(needle_current_function_); - if (param_it != needle_param_tries_.end()) { - std::string partial; - bool all_seen = !needle_has_unseen_completion(param_it->second.get(), partial); - if (all_seen && needle_between_pairs_) { - for (uint32_t t : needle_arg_close_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - } else if (!all_seen && (needle_between_pairs_ || needle_in_string_value_)) { - auto req_it = needle_required_params_.find(needle_current_function_); - if (req_it != needle_required_params_.end()) { - for (const auto& req : req_it->second) { - if (!needle_seen_arg_keys_.count(req)) { - for (uint32_t t : needle_arg_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - } - } - } - } - } - } - break; - } - - const NeedleTrieNode* trie_node = nullptr; - if (needle_json_state_ == NeedleJsonState::IN_NAME) { - trie_node = needle_get_trie_node(needle_name_trie_.get(), needle_constrained_buf_); - } else if (needle_json_state_ == NeedleJsonState::IN_ARG_KEY) { - auto it = needle_param_tries_.find(needle_current_function_); - if (it != needle_param_tries_.end()) { - trie_node = needle_get_trie_node(it->second.get(), needle_constrained_buf_); - } - } - - if (!trie_node) { - break; - } - - std::vector valid_tokens(needle_token_strings_.size(), false); - bool has_valid = false; - auto mark_valid_tokens = [&](char first_char) { - auto idx_it = needle_token_index_.find(first_char); - if (idx_it == needle_token_index_.end()) { - return; - } - for (uint32_t token_id : idx_it->second) { - if (!valid_tokens[token_id] && - needle_check_token_valid(needle_token_strings_[token_id], trie_node)) { - valid_tokens[token_id] = true; - has_valid = true; - } - } - }; - - for (const auto& [first_char, _] : trie_node->children) { - mark_valid_tokens(first_char); - } - if (trie_node->is_terminal) { - mark_valid_tokens('"'); - } - - if (!has_valid) { - break; - } - - if (needle_json_state_ == NeedleJsonState::IN_ARG_KEY && !needle_seen_arg_keys_.empty()) { - for (size_t token_id = 0; token_id < valid_tokens.size(); ++token_id) { - if (!valid_tokens[token_id]) continue; - const std::string& tok = needle_token_strings_[token_id]; - size_t quote_pos = tok.find('"'); - if (quote_pos != std::string::npos) { - if (needle_seen_arg_keys_.count(needle_constrained_buf_ + tok.substr(0, quote_pos))) { - valid_tokens[token_id] = false; - } - } else { - std::string partial = needle_constrained_buf_ + tok; - const NeedleTrieNode* new_node = needle_get_trie_node(trie_node, tok); - if (!needle_has_unseen_completion(new_node, partial)) { - valid_tokens[token_id] = false; - } - } - } - } - - for (size_t token_id = 0; token_id < valid_tokens.size(); ++token_id) { - if (!valid_tokens[token_id]) { - current_bias_[static_cast(token_id)] = BLOCK_BIAS; - } - } - break; - } - } else { - switch (state_) { - case State::QWEN_START: - for (uint32_t t : qwen_tool_call_start_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::QWEN_EXPECT_OPEN_BRACE: - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : qwen_tool_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::QWEN_EXPECT_NAME_KEY: { - bool has_name = generated_text_.find("name") != std::string::npos; - bool has_quote = generated_text_.find("\"") != std::string::npos; - if (has_name) { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } else if (has_quote) { - for (uint32_t t : name_key_tokens_) { current_bias_[t] = FORCE_BIAS; } - for (uint32_t t : quote_tokens_) { current_bias_[t] = BLOCK_BIAS; } - } else { - for (uint32_t t : all_func_name_tokens_) { current_bias_[t] = BLOCK_BIAS; } - for (uint32_t t : args_key_tokens_) { current_bias_[t] = BLOCK_BIAS; } - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } - break; - } - - case State::QWEN_EXPECT_NAME_COLON: - for (uint32_t t : colon_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - break; - - case State::QWEN_EXPECT_NAME_VALUE: { - bool name_complete = false; - for (const auto& name : function_names_) { - if (generated_text_.find(name) != std::string::npos) { name_complete = true; break; } - } - bool has_open_quote = generated_text_.find("\"") != std::string::npos; - if (name_complete) { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } else if (has_open_quote) { - for (uint32_t t : all_func_name_tokens_) { current_bias_[t] = FORCE_BIAS; } - for (uint32_t t : quote_tokens_) { current_bias_[t] = 5.0f; } - } else { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } - break; - } - - case State::QWEN_EXPECT_COMMA: - for (uint32_t t : comma_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = 5.0f; - } - break; - - case State::QWEN_EXPECT_ARGS_KEY: { - bool has_args = generated_text_.find("arguments") != std::string::npos; - bool has_quote = generated_text_.find("\"") != std::string::npos; - if (has_args) { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } else if (has_quote) { - for (uint32_t t : args_key_tokens_) { current_bias_[t] = FORCE_BIAS; } - for (uint32_t t : quote_tokens_) { current_bias_[t] = BLOCK_BIAS; } - } else { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } - break; - } - - case State::QWEN_EXPECT_ARGS_COLON: - for (uint32_t t : colon_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - break; - - case State::QWEN_IN_ARGUMENTS: - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : colon_tokens_) { - current_bias_[t] = 2.0f; - } - for (uint32_t t : comma_tokens_) { - current_bias_[t] = 2.0f; - } - for (uint32_t t : quote_tokens_) { - current_bias_[t] = 2.0f; - } - for (uint32_t t : qwen_tool_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::QWEN_EXPECT_CLOSE_BRACE: - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - break; - - case State::QWEN_EXPECT_END: - for (uint32_t t : qwen_tool_call_end_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : qwen_tool_call_start_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - default: - break; - } - } -} - -void ToolCallConstrainer::reset() { - generated_text_.clear(); - current_bias_.clear(); - brace_depth_ = 0; - reset_needle_constraints(); - - if (model_type_ == Config::ModelType::LFM2) { - state_ = State::LFM_START; - } else if (is_gemma_family()) { - state_ = State::GEMMA_START; - } else if (is_needle()) { - state_ = State::NEEDLE_START; - } else { - state_ = State::QWEN_START; - } - - if (active_) { - compute_bias(); - } -} - - -void Model::set_tool_constraints(const std::vector& tools) { - tool_constrainer_.init(config_.model_type, tools, tokenizer_.get()); -} - -void Model::clear_tool_constraints() { - tool_constrainer_.reset(); - tool_constrainer_.init(config_.model_type, {}, tokenizer_.get()); -} - -void Model::update_tool_constraints(uint32_t token_id) { - if (tool_constrainer_.is_active() && tokenizer_) { - std::string decoded = tokenizer_->decode({token_id}); - tool_constrainer_.update(token_id, decoded); - } -} - -} // namespace engine -} // namespace cactus diff --git a/cactus-transpile/engine/engine_image.cpp b/cactus-transpile/engine/engine_image.cpp deleted file mode 100644 index cf066e99e..000000000 --- a/cactus-transpile/engine/engine_image.cpp +++ /dev/null @@ -1,600 +0,0 @@ -#define STB_IMAGE_IMPLEMENTATION -#define STB_IMAGE_RESIZE_IMPLEMENTATION - -#define STBI_NO_BMP -#define STBI_NO_PSD -#define STBI_NO_HDR -#define STBI_NO_PIC -#define STBI_NO_PNM -#define STBI_NO_TGA - -#include "engine.h" -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Siglip2Preprocessor::PreprocessedImage::~PreprocessedImage() { - pixel_values.clear(); - pixel_attention_mask.clear(); -} - -Siglip2Preprocessor::Siglip2Preprocessor(const Config& config) - : config_(config) {} - -Siglip2Preprocessor::Siglip2Preprocessor() : config_() {} - -Siglip2Preprocessor::~Siglip2Preprocessor() = default; - -std::pair Siglip2Preprocessor::compute_pixel_limits() const { - int64_t min_pixels = static_cast(config_.min_image_tokens) * - config_.patch_size * config_.patch_size * - config_.downsample_factor * config_.downsample_factor; - int64_t max_pixels = static_cast(config_.max_image_tokens) * - config_.patch_size * config_.patch_size * - config_.downsample_factor * config_.downsample_factor; - return {min_pixels, max_pixels}; -} - -int Siglip2Preprocessor::round_by_factor(int number, int factor) { - if (factor == 0) { - return number; - } - double scaled = static_cast(number) / static_cast(factor); - long long rounded = std::llround(scaled); - - return static_cast(rounded * factor); -} - -std::pair Siglip2Preprocessor::smart_resize(int height, int width) { - const int total_factor = config_.patch_size * config_.downsample_factor; - auto [min_pixels, max_pixels] = compute_pixel_limits(); - - int h_bar = std::max(total_factor, round_by_factor(height, total_factor)); - int w_bar = std::max(total_factor, round_by_factor(width, total_factor)); - - - if (static_cast(h_bar) * static_cast(w_bar) > max_pixels) { - double beta = std::sqrt((static_cast(height) * static_cast(width)) / - static_cast(max_pixels)); - h_bar = std::max(total_factor, - static_cast(std::floor(height / beta / total_factor)) * total_factor); - w_bar = std::max(total_factor, - static_cast(std::floor(width / beta / total_factor)) * total_factor); - } else if (static_cast(h_bar) * static_cast(w_bar) < min_pixels) { - double beta = std::sqrt(static_cast(min_pixels) / - (static_cast(height) * static_cast(width))); - h_bar = std::max(total_factor, - static_cast(std::ceil(height * beta / total_factor)) * total_factor); - w_bar = std::max(total_factor, - static_cast(std::ceil(width * beta / total_factor)) * total_factor); - } - - return {w_bar, h_bar}; -} - - -bool Siglip2Preprocessor::is_image_too_large(int height, int width) { - const int total_factor = config_.patch_size * config_.downsample_factor; - const int h_bar = std::max(config_.patch_size, round_by_factor(height, total_factor)); - const int w_bar = std::max(config_.patch_size, round_by_factor(width, total_factor)); - - const int64_t pixels = static_cast(h_bar) * static_cast(w_bar); - auto [min_pixels, max_pixels] = compute_pixel_limits(); - const double max_pixels_with_tolerance = static_cast(max_pixels) * config_.max_pixels_tolerance; - - bool result = static_cast(pixels) > max_pixels_with_tolerance; - - return result; -} - - -std::pair Siglip2Preprocessor::find_closest_aspect_ratio(float aspect_ratio, int width, int height) { - float best_ratio_diff = std::numeric_limits::infinity(); - std::pair best_ratio = {1, 1}; - int area = width * height; - - std::vector> target_ratios; - for (int n = config_.min_tiles; n <= config_.max_tiles; ++n) { - for (int w = 1; w <= n; ++w) { - for (int h = 1; h <= n; ++h) { - int total_tiles = w * h; - if (total_tiles >= config_.min_tiles && total_tiles <= config_.max_tiles) { - target_ratios.push_back({w, h}); - } - } - } - } - - std::sort(target_ratios.begin(), target_ratios.end()); - target_ratios.erase(std::unique(target_ratios.begin(), target_ratios.end()), target_ratios.end()); - std::sort(target_ratios.begin(), target_ratios.end(), [](const auto& a, const auto& b) { - return (a.first * a.second) < (b.first * b.second); - }); - - for (const auto& ratio : target_ratios) { - float target_aspect_ratio = static_cast(ratio.first) / ratio.second; - float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio); - - if (ratio_diff < best_ratio_diff) { - best_ratio_diff = ratio_diff; - best_ratio = ratio; - } else if (ratio_diff == best_ratio_diff) { - int target_area = config_.tile_size * config_.tile_size * ratio.first * ratio.second; - if (area > 0.5f * target_area) { - best_ratio = ratio; - } - } - } - - - return best_ratio; -} - -std::pair Siglip2Preprocessor::get_grid_layout(int height, int width) { - float aspect_ratio = (float)width / (float)height; - auto [grid_width, grid_height] = find_closest_aspect_ratio(aspect_ratio, width, height); - - int target_width = config_.tile_size * grid_width; - int target_height = config_.tile_size * grid_height; - - return {target_width, target_height}; -} - -Siglip2Preprocessor::SpatialShapeResult Siglip2Preprocessor::compute_spatial_shapes(int height, int width) { - if (height <= 0 || width <= 0) { - throw std::runtime_error("Image dimensions must be positive"); - } - - if (config_.patch_size <= 0) { - throw std::runtime_error("Patch size must be positive"); - } - - const int patch = config_.patch_size; - auto [resized_width, resized_height] = smart_resize(height, width); - const bool should_split = config_.do_image_splitting && is_image_too_large(height, width); - - - SpatialShapeResult result; - result.grid_rows = 1; - result.grid_cols = 1; - - if (should_split) { - if (config_.tile_size % patch != 0) { - throw std::runtime_error("Tile size must be divisible by patch size"); - } - - auto [grid_target_width, grid_target_height] = get_grid_layout(height, width); - result.grid_cols = grid_target_width / config_.tile_size; - result.grid_rows = grid_target_height / config_.tile_size; - - - const int patches_per_tile_side = config_.tile_size / patch; - const auto tile_shape = std::make_pair(patches_per_tile_side, patches_per_tile_side); - - result.shapes.reserve(static_cast(result.grid_rows) * result.grid_cols + 1); - for (int row = 0; row < result.grid_rows; ++row) { - for (int col = 0; col < result.grid_cols; ++col) { - result.shapes.push_back(tile_shape); - - } - } - - if (config_.use_thumbnail && result.grid_rows * result.grid_cols != 1) { - if (resized_height % patch != 0 || resized_width % patch != 0) { - throw std::runtime_error("Resized thumbnail dimensions must be divisible by patch size"); - } - result.shapes.emplace_back(resized_height / patch, resized_width / patch); - - } - } else { - int target_width = resized_width; - int target_height = resized_height; - if (!config_.do_resize) { - target_width = width; - target_height = height; - } - - if (target_height % patch != 0 || target_width % patch != 0) { - throw std::runtime_error("Target dimensions must be divisible by patch size"); - } - - result.shapes.emplace_back(target_height / patch, target_width / patch); - - } - - return result; -} - - -Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::preprocess_from_file(const std::string& image_path) { - int width, height, channels; - unsigned char* img_data = stbi_load(image_path.c_str(), &width, &height, &channels, 0); - - if (!img_data) { - throw std::runtime_error("Failed to load image: " + image_path + " - " + std::string(stbi_failure_reason())); - } - - PreprocessedImage result; - result = preprocess_from_memory(img_data, width, height, channels); - - stbi_image_free(img_data); - return result; -} - -Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::preprocess_from_memory( - const unsigned char* img_data, int width, int height, int channels) { - if (!img_data) { - throw std::runtime_error("Invalid image data pointer"); - } - - const int expected_channels = 3; - std::vector rgb_data; - const unsigned char* source_data = img_data; - int source_channels = channels; - - if (config_.do_convert_rgb && channels != expected_channels) { - rgb_data = convert_to_rgb(img_data, width, height, channels); - source_data = rgb_data.data(); - source_channels = expected_channels; - - } - - if (source_channels != expected_channels) { - throw std::runtime_error("Image must have 3 channels (RGB)"); - } - - const int patch = config_.patch_size; - const int downsample = config_.downsample_factor; - const int patch_dim = patch * patch * expected_channels; - - if (patch <= 0) { - throw std::runtime_error("Patch size must be positive"); - } - if (config_.tile_size % patch != 0) { - throw std::runtime_error("Tile size must be divisible by patch size"); - } - - auto [resized_width, resized_height] = smart_resize(height, width); - - - const int patches_per_tile_side = config_.tile_size / patch; - const int tile_patch_count = patches_per_tile_side * patches_per_tile_side; - const int thumbnail_patch_cap = config_.max_image_tokens * downsample * downsample; - int max_patches_per_tile = config_.max_num_patches; - max_patches_per_tile = std::max(max_patches_per_tile, tile_patch_count); - max_patches_per_tile = std::max(max_patches_per_tile, thumbnail_patch_cap); - - const bool allow_splitting = config_.do_image_splitting; - const bool should_split = allow_splitting && is_image_too_large(height, width); - - - size_t expected_tiles = should_split ? static_cast(config_.max_tiles) : 1; - if (should_split && config_.use_thumbnail) { - expected_tiles += 1; - } - - std::vector> tile_patches; - tile_patches.reserve(expected_tiles); - std::vector> spatial_shapes; - spatial_shapes.reserve(expected_tiles); - - auto normalize_and_patchify = [&](const float* data_ptr, int img_width, int img_height) { - if (img_height % patch != 0 || img_width % patch != 0) { - throw std::runtime_error("Image dimensions must be divisible by patch size"); - } - std::vector normalized = normalize_image(data_ptr, img_width, img_height, expected_channels); - auto patches = convert_image_to_patches(normalized, img_width, img_height, expected_channels, patch); - std::vector flattened(patches.size() * patch_dim); - for (size_t idx = 0; idx < patches.size(); ++idx) { - std::copy(patches[idx].begin(), patches[idx].end(), flattened.begin() + idx * patch_dim); - } - tile_patches.push_back(std::move(flattened)); - spatial_shapes.emplace_back(img_height / patch, img_width / patch); - }; - - int grid_rows = 1; - int grid_cols = 1; - bool thumbnail_added = false; - - if (should_split) { - auto [grid_target_width, grid_target_height] = get_grid_layout(height, width); - grid_cols = grid_target_width / config_.tile_size; - grid_rows = grid_target_height / config_.tile_size; - - std::vector resized_grid = resize_image( - source_data, width, height, grid_target_width, grid_target_height, expected_channels); - - std::vector tile_buffer( - static_cast(config_.tile_size) * config_.tile_size * expected_channels); - - for (int row = 0; row < grid_rows; ++row) { - for (int col = 0; col < grid_cols; ++col) { - for (int y = 0; y < config_.tile_size; ++y) { - const float* src_row = resized_grid.data() + - ((row * config_.tile_size + y) * grid_target_width + col * config_.tile_size) * - expected_channels; - float* dst_row = tile_buffer.data() + (y * config_.tile_size) * expected_channels; - std::copy_n(src_row, - static_cast(config_.tile_size) * expected_channels, - dst_row); - } - normalize_and_patchify(tile_buffer.data(), config_.tile_size, config_.tile_size); - - } - } - - if (config_.use_thumbnail && grid_rows * grid_cols != 1) { - std::vector thumbnail_bytes = resize_image( - source_data, width, height, resized_width, resized_height, expected_channels); - normalize_and_patchify(thumbnail_bytes.data(), resized_width, resized_height); - thumbnail_added = true; - - } - } else { - int target_width = resized_width; - int target_height = resized_height; - - const bool needs_resize = config_.do_resize && (width != target_width || height != target_height); - - std::vector resized_image; - if (needs_resize) { - resized_image = resize_image(source_data, width, height, target_width, target_height, expected_channels); - normalize_and_patchify(resized_image.data(), target_width, target_height); - - } else { - resized_image.resize(static_cast(width) * height * expected_channels); - for (size_t idx = 0; idx < resized_image.size(); ++idx) { - resized_image[idx] = static_cast(source_data[idx]); - } - normalize_and_patchify(resized_image.data(), width, height); - target_width = width; - target_height = height; - resized_width = target_width; - resized_height = target_height; - - } - - grid_rows = 1; - grid_cols = 1; - } - - PreprocessedImage result = pad_patches(tile_patches, spatial_shapes, patch_dim, max_patches_per_tile); - - - result.image_rows = grid_rows; - result.image_cols = grid_cols; - result.image_width = resized_width; - result.image_height = resized_height; - - auto compute_tokens = [&](int patches_h, int patches_w) -> int { - int tokens_h = (patches_h + downsample - 1) / downsample; - int tokens_w = (patches_w + downsample - 1) / downsample; - return tokens_h * tokens_w; - }; - - result.tokens_per_tile = spatial_shapes.empty() - ? 0 - : compute_tokens(spatial_shapes.front().first, spatial_shapes.front().second); - - - if (thumbnail_added && !spatial_shapes.empty()) { - const auto& thumb_shape = spatial_shapes.back(); - result.thumbnail_tokens = compute_tokens(thumb_shape.first, thumb_shape.second); - - } else { - result.thumbnail_tokens = 0; - } - - return result; -} - -std::vector Siglip2Preprocessor::convert_to_rgb( - const unsigned char* img_data, int width, int height, int channels) { - - std::vector rgb_data(width * height * 3); - - if (channels == 1) { - for (int i = 0; i < width * height; ++i) { - rgb_data[i * 3 + 0] = img_data[i]; - rgb_data[i * 3 + 1] = img_data[i]; - rgb_data[i * 3 + 2] = img_data[i]; - } - } else if (channels == 4) { - for (int i = 0; i < width * height; ++i) { - rgb_data[i * 3 + 0] = img_data[i * 4 + 0]; - rgb_data[i * 3 + 1] = img_data[i * 4 + 1]; - rgb_data[i * 3 + 2] = img_data[i * 4 + 2]; - } - } else if (channels == 2) { - for (int i = 0; i < width * height; ++i) { - rgb_data[i * 3 + 0] = img_data[i * 2 + 0]; - rgb_data[i * 3 + 1] = img_data[i * 2 + 0]; - rgb_data[i * 3 + 2] = img_data[i * 2 + 0]; - } - } else { - throw std::runtime_error("Unsupported number of channels: " + std::to_string(channels)); - } - - return rgb_data; -} - -std::vector Siglip2Preprocessor::resize_image( - const unsigned char* img_data, int src_width, int src_height, - int dst_width, int dst_height, int channels) { - - const size_t src_elements = static_cast(src_width) * src_height * channels; - std::vector src_float(src_elements); - for (size_t idx = 0; idx < src_elements; ++idx) { - src_float[idx] = static_cast(img_data[idx]); - } - - - std::vector resized_data(static_cast(dst_width) * dst_height * channels); - - stbir_pixel_layout layout = (channels == 1) ? STBIR_1CHANNEL : - (channels == 3) ? STBIR_RGB : STBIR_RGBA; - - float* result = stbir_resize_float_linear( - src_float.data(), src_width, src_height, 0, - resized_data.data(), dst_width, dst_height, 0, - layout - ); - - if (!result) { - throw std::runtime_error("Failed to resize image"); - } - - - return resized_data; -} - -std::vector Siglip2Preprocessor::normalize_image( - const float* img_data, int width, int height, int channels) { - - size_t total_pixels = width * height * channels; - std::vector normalized(total_pixels); - - for (size_t i = 0; i < static_cast(width * height); ++i) { - for (int c = 0; c < channels; ++c) { - size_t idx = i * channels + c; - float pixel = img_data[idx]; - - if (config_.do_rescale) { - pixel *= config_.rescale_factor; - } - - if (config_.do_normalize) { - pixel = (pixel - config_.image_mean[c]) / config_.image_std[c]; - } - - normalized[idx] = pixel; - } - } - - - return normalized; -} - -std::vector> Siglip2Preprocessor::convert_image_to_patches( - const std::vector& image, int width, int height, int channels, int patch_size) { - - int num_patches_height = height / patch_size; - int num_patches_width = width / patch_size; - int num_patches = num_patches_height * num_patches_width; - int patch_elements = patch_size * patch_size * channels; - - std::vector> patches(num_patches, std::vector(patch_elements)); - - for (int ph = 0; ph < num_patches_height; ++ph) { - for (int pw = 0; pw < num_patches_width; ++pw) { - int patch_idx = ph * num_patches_width + pw; - - for (int y = 0; y < patch_size; ++y) { - for (int x = 0; x < patch_size; ++x) { - int img_y = ph * patch_size + y; - int img_x = pw * patch_size + x; - int img_idx = (img_y * width + img_x) * channels; - int patch_offset = (y * patch_size + x) * channels; - - for (int c = 0; c < channels; ++c) { - patches[patch_idx][patch_offset + c] = image[img_idx + c]; - } - } - } - } - } - - return patches; -} - -Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::pad_patches( - const std::vector>& tile_patches, - const std::vector>& spatial_shapes, - int patch_dim, - int max_patches_per_tile) { - - if (tile_patches.size() != spatial_shapes.size()) { - throw std::runtime_error("Mismatch between tile data and spatial shapes"); - } - - PreprocessedImage result; - - const int num_tiles = static_cast(tile_patches.size()); - result.num_tiles = num_tiles; - result.patch_dim = patch_dim; - result.max_patches_per_tile = max_patches_per_tile; - result.spatial_shapes = spatial_shapes; - result.actual_num_patches = 0; - - const size_t total_values = static_cast(num_tiles) * max_patches_per_tile * patch_dim; - result.pixel_values.assign(total_values, 0.0f); - result.pixel_attention_mask.assign(static_cast(num_tiles) * max_patches_per_tile, 0); - - - for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { - const auto& [patches_h, patches_w] = spatial_shapes[tile_idx]; - const int actual_patches = patches_h * patches_w; - - if (actual_patches > max_patches_per_tile) { - throw std::runtime_error("Actual patches exceed max_patches_per_tile"); - } - - const auto& flattened = tile_patches[tile_idx]; - const size_t expected_size = static_cast(actual_patches) * patch_dim; - if (flattened.size() != expected_size) { - throw std::runtime_error("Tile patch data has unexpected size"); - } - - float* destination = result.pixel_values.data() + - static_cast(tile_idx) * max_patches_per_tile * patch_dim; - std::memcpy(destination, flattened.data(), expected_size * sizeof(float)); - - - int mask_offset = tile_idx * max_patches_per_tile; - for (int p = 0; p < actual_patches; ++p) { - result.pixel_attention_mask[mask_offset + p] = 1; - } - - result.actual_num_patches += actual_patches; - } - - if (!spatial_shapes.empty()) { - result.num_patches_height = spatial_shapes.front().first; - result.num_patches_width = spatial_shapes.front().second; - } else { - result.num_patches_height = 0; - result.num_patches_width = 0; - } - - result.pixel_values_shape = { - static_cast(num_tiles), - static_cast(max_patches_per_tile), - static_cast(patch_dim) - }; - - - result.pixel_attention_mask_shape = { - static_cast(num_tiles), - static_cast(max_patches_per_tile) - }; - - result.spatial_shapes_shape = { - static_cast(num_tiles), - static_cast(2) - }; - - - return result; -} - -} // namespace engine -} // namespace cactus - - diff --git a/cactus-transpile/engine/engine_index.cpp b/cactus-transpile/engine/engine_index.cpp deleted file mode 100644 index 13d08920a..000000000 --- a/cactus-transpile/engine/engine_index.cpp +++ /dev/null @@ -1,780 +0,0 @@ -#include "engine.h" -#include "kernel/kernel.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { -namespace index { - - __fp16 dot_product(const __fp16 *a, const __fp16 *b, size_t dim) { - __fp16 result; - cactus_matmul_f16(a, b, &result, 1, dim, 1); - return result; - } - - void normalize(__fp16 *v, size_t dim) { - __fp16 x = dot_product(v, v, dim); - if (x < 1e-6f) { - throw std::runtime_error("Cannot normalize zero vector"); - } - cactus_scalar_op_f16(&x, &x, 1, 0, ScalarOpType::SQRT); - cactus_scalar_op_f16(v, v, dim, x, ScalarOpType::DIVIDE); - } - - Index::Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim): - index_path_(index_path), data_path_(data_path), embedding_dim_(embedding_dim), - index_entry_size_(IndexEntry::size(embedding_dim)), num_documents_(0), - index_fd_(-1), data_fd_(-1), - mapped_index_(nullptr), mapped_data_(nullptr) { - - bool index_exists = (access(index_path.c_str(), F_OK) == 0); - bool data_exists = (access(data_path.c_str(), F_OK) == 0); - - if (index_exists != data_exists) { - throw std::runtime_error("Index and data files must both exist or both not exist"); - } - - index_fd_ = open(index_path.c_str(), O_RDWR | O_CREAT, 0644); - if (index_fd_ < 0) { - throw std::runtime_error("Cannot open index file: " + index_path); - } - - data_fd_ = open(data_path.c_str(), O_RDWR | O_CREAT, 0644); - if (data_fd_ < 0) { - close(index_fd_); - throw std::runtime_error("Cannot open data file: " + data_path); - } - - auto cleanup_and_throw = [this](const std::string& msg) { - close(index_fd_); - close(data_fd_); - throw std::runtime_error(msg); - }; - - struct stat index_st, data_st; - if (fstat(index_fd_, &index_st)) { - cleanup_and_throw("Cannot get index file size: " + index_path); - } - index_file_size_ = index_st.st_size; - - if (fstat(data_fd_, &data_st)) { - cleanup_and_throw("Cannot get data file size: " + data_path); - } - data_file_size_ = data_st.st_size; - - if (!index_exists) { - index_file_size_ = sizeof(IndexHeader); - data_file_size_ = sizeof(DataHeader); - - if (ftruncate(index_fd_, index_file_size_) != 0) { - cleanup_and_throw("Failed to resize index file"); - } - - if (ftruncate(data_fd_, data_file_size_) != 0) { - cleanup_and_throw("Failed to resize data file"); - } - } - - mapped_index_ = mmap(nullptr, index_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, index_fd_, 0); - if (mapped_index_ == MAP_FAILED) { - cleanup_and_throw("Cannot map file: " + index_path); - } - - mapped_data_ = mmap(nullptr, data_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, data_fd_, 0); - if (mapped_data_ == MAP_FAILED) { - munmap(mapped_index_, index_file_size_); - close(index_fd_); - close(data_fd_); - throw std::runtime_error("Cannot map file: " + data_path); - } - - if (!index_exists) { - IndexHeader index_header = { - MAGIC, - VERSION, - static_cast(embedding_dim), - 0 - }; - memcpy(mapped_index_, &index_header, sizeof(IndexHeader)); - - DataHeader data_header = { MAGIC, VERSION }; - memcpy(mapped_data_, &data_header, sizeof(DataHeader)); - - if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { - cleanup_and_throw("Failed to sync index file"); - } - - if (msync(mapped_data_, data_file_size_, MS_SYNC) != 0) { - cleanup_and_throw("Failed to sync data file"); - } - } - - parse_index_header(); - parse_data_header(); - - build_doc_id_map(); - } - - Index::~Index() { - if (mapped_index_ != nullptr && mapped_index_ != MAP_FAILED) { - madvise(mapped_index_, index_file_size_, MADV_DONTNEED); - munmap(mapped_index_, index_file_size_); - } - if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { - madvise(mapped_data_, data_file_size_, MADV_DONTNEED); - munmap(mapped_data_, data_file_size_); - } - if (index_fd_ != -1) { - close(index_fd_); - } - if (data_fd_ != -1) { - close(data_fd_); - } - } - - void Index::add_documents(const std::vector& documents) { - validate_documents(documents); - - size_t added_data_size = 0; - for (const auto& doc : documents) { - added_data_size += sizeof(DataEntry) + doc.content.size() + doc.metadata.size(); - } - - size_t new_index_size = index_file_size_ + documents.size() * index_entry_size_; - size_t new_data_size = data_file_size_ + added_data_size; - - if (ftruncate(index_fd_, new_index_size) != 0) { - throw std::runtime_error("Failed to resize index file"); - } - - if (ftruncate(data_fd_, new_data_size) != 0) { - throw std::runtime_error("Failed to resize data file"); - } - - munmap(mapped_index_, index_file_size_); - munmap(mapped_data_, data_file_size_); - - mapped_index_ = mmap(nullptr, new_index_size, PROT_READ | PROT_WRITE, MAP_SHARED, index_fd_, 0); - if (mapped_index_ == MAP_FAILED) { - throw std::runtime_error("Failed to remap index file"); - } - - mapped_data_ = mmap(nullptr, new_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, data_fd_, 0); - if (mapped_data_ == MAP_FAILED) { - munmap(mapped_index_, new_index_size); - throw std::runtime_error("Failed to remap data file"); - } - - size_t old_index_size = index_file_size_; - size_t old_data_size = data_file_size_; - - index_file_size_ = new_index_size; - data_file_size_ = new_data_size; - - char* index_write_pos = static_cast(mapped_index_) + old_index_size; - char* data_write_pos = static_cast(mapped_data_) + old_data_size; - - uint64_t data_offset = old_data_size; - doc_id_map_.reserve(doc_id_map_.size() + documents.size()); - - for (size_t i = 0; i < documents.size(); ++i) { - const auto& doc = documents[i]; - - DataEntry data_entry{ - static_cast(doc.content.size()), - static_cast(doc.metadata.size()) - }; - - memcpy(data_write_pos, &data_entry, sizeof(DataEntry)); - data_write_pos += sizeof(DataEntry); - memcpy(data_write_pos, doc.content.data(), doc.content.size()); - data_write_pos += doc.content.size(); - memcpy(data_write_pos, doc.metadata.data(), doc.metadata.size()); - data_write_pos += doc.metadata.size(); - - IndexEntry index_entry{doc.id, data_offset, 0}; - memcpy(index_write_pos, &index_entry, sizeof(IndexEntry)); - index_write_pos += sizeof(IndexEntry); - - std::vector<__fp16> normalized_embedding(embedding_dim_); - cactus_fp32_to_fp16(doc.embedding.data(), normalized_embedding.data(), embedding_dim_); - normalize(normalized_embedding.data(), embedding_dim_); - - size_t embedding_bytes = embedding_dim_ * sizeof(__fp16); - memcpy(index_write_pos, normalized_embedding.data(), embedding_bytes); - index_write_pos += embedding_bytes; - - doc_id_map_[doc.id] = num_documents_ + static_cast(i); - data_offset += sizeof(DataEntry) + doc.content.size() + doc.metadata.size(); - } - - num_documents_ += static_cast(documents.size()); - IndexHeader* header = reinterpret_cast(mapped_index_); - header->num_documents = num_documents_; - - if (msync(mapped_data_, data_file_size_, MS_SYNC) != 0) { - throw std::runtime_error("Failed to sync data file to disk"); - } - - if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { - throw std::runtime_error("Failed to sync index file to disk"); - } - } - - void Index::delete_documents(const std::vector& doc_ids) { - validate_doc_ids(doc_ids); - - char* index_ptr = static_cast(mapped_index_); - char* entries = index_ptr + sizeof(IndexHeader); - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("File corrupted: index entry extends beyond file size"); - } - } - - for (int doc_id : doc_ids) { - uint32_t i = doc_id_map_.at(doc_id); - IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); - - entry.flags |= 0x1; - doc_id_map_.erase(doc_id); - } - - if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { - throw std::runtime_error("Failed to sync index file to disk"); - } - } - - std::vector Index::get_documents(const std::vector& doc_ids) { - validate_doc_ids(doc_ids); - - const char* index_ptr = static_cast(mapped_index_); - const char* entries = index_ptr + sizeof(IndexHeader); - const char* data_ptr = static_cast(mapped_data_); - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("File corrupted: index entry extends beyond file size"); - } - } - - std::vector results; - results.reserve(doc_ids.size()); - - for (int doc_id : doc_ids) { - uint32_t i = doc_id_map_.at(doc_id); - const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); - - if (static_cast(entry.data_offset) + sizeof(DataEntry) > data_file_size_) { - throw std::runtime_error("File corrupted: data entry extends beyond file size"); - } - - const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); - size_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; - - if (static_cast(entry.data_offset) + data_entry_size > data_file_size_) { - throw std::runtime_error("File corrupted: data entry extends beyond file size"); - } - - std::vector embedding_f32(embedding_dim_); - cactus_fp16_to_fp32(entry.embedding(), embedding_f32.data(), embedding_dim_); - - results.emplace_back( - doc_id, - std::move(embedding_f32), - std::string(data_entry->content(), data_entry->content_len), - std::string(data_entry->metadata(), data_entry->metadata_len) - ); - } - - return results; - } - - std::vector> Index::query(const std::vector>& embeddings, const QueryOptions& options) { - if (embeddings.empty()) { - return {}; - } - - if (options.top_k == 0) { - return std::vector>(embeddings.size()); - } - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("File corrupted: index entry extends beyond file size"); - } - } - - std::vector> normalized_embeddings; - normalized_embeddings.reserve(embeddings.size()); - - for (const auto& embedding : embeddings) { - std::vector<__fp16> normalized_embedding(embedding_dim_); - cactus_fp32_to_fp16(embedding.data(), normalized_embedding.data(), embedding_dim_); - normalize(normalized_embedding.data(), embedding_dim_); - normalized_embeddings.emplace_back(std::move(normalized_embedding)); - } - - std::vector> all_results; - all_results.reserve(embeddings.size()); - - const char* index_ptr = static_cast(mapped_index_); - const char* entries = index_ptr + sizeof(IndexHeader); - - auto cmp = [](const QueryResult& a, const QueryResult& b) { - return a.score > b.score; - }; - - for (const auto& normalized_embedding : normalized_embeddings) { - std::priority_queue, decltype(cmp)> top_results(cmp); - - for (uint32_t i = 0; i < num_documents_; ++i) { - const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); - - if (entry.flags & 0x1) { - continue; - } - - float score = static_cast( - dot_product(normalized_embedding.data(), entry.embedding(), embedding_dim_) - ); - - if (score < options.score_threshold) { - continue; - } - - if (top_results.size() < options.top_k) { - top_results.emplace(entry.doc_id, score); - } else if (score > top_results.top().score) { - top_results.pop(); - top_results.emplace(entry.doc_id, score); - } - } - - std::vector results; - results.reserve(top_results.size()); - - while(!top_results.empty()) { - results.emplace_back(top_results.top()); - top_results.pop(); - } - std::reverse(results.begin(), results.end()); - - all_results.emplace_back(std::move(results)); - } - - return all_results; - } - - void Index::compact() { - std::string temp_index_path = index_path_ + ".tmp"; - std::string temp_data_path = data_path_ + ".tmp"; - - const char* index_ptr = static_cast(mapped_index_); - const char* entries = index_ptr + sizeof(IndexHeader); - const char* data_ptr = static_cast(mapped_data_); - - uint32_t compacted_count = static_cast(doc_id_map_.size()); - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("Compaction failed: File corrupted: index entry extends beyond file size"); - } - } - - off_t new_data_offset = sizeof(DataHeader); - size_t new_data_size = sizeof(DataHeader); - - for (const auto& [doc_id, index] : doc_id_map_) { - const IndexEntry& entry = *reinterpret_cast(entries + index * index_entry_size_); - - if (static_cast(entry.data_offset) + sizeof(DataEntry) > data_file_size_) { - throw std::runtime_error("Compaction failed: File corrupted: data entry extends beyond file size"); - } - - const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); - uint32_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; - - if (static_cast(entry.data_offset) + data_entry_size > data_file_size_) { - throw std::runtime_error("Compaction failed: File corrupted: data entry extends beyond file size"); - } - - new_data_size += data_entry_size; - } - - size_t new_index_size = sizeof(IndexHeader) + compacted_count * index_entry_size_; - - int temp_index_fd = open(temp_index_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644); - if (temp_index_fd < 0) { - throw std::runtime_error("Cannot create temporary index file: " + temp_index_path); - } - - int temp_data_fd = open(temp_data_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644); - if (temp_data_fd < 0) { - close(temp_index_fd); - unlink(temp_index_path.c_str()); - throw std::runtime_error("Cannot create temporary data file: " + temp_data_path); - } - - auto cleanup_and_throw = [&temp_index_fd, &temp_data_fd, &temp_index_path, &temp_data_path](const std::string& msg) { - close(temp_index_fd); - close(temp_data_fd); - unlink(temp_index_path.c_str()); - unlink(temp_data_path.c_str()); - throw std::runtime_error(msg); - }; - - if (ftruncate(temp_index_fd, new_index_size) != 0) { - cleanup_and_throw("Failed to resize temporary index file"); - } - - if (ftruncate(temp_data_fd, new_data_size) != 0) { - cleanup_and_throw("Failed to resize temporary data file"); - } - - void* temp_index_map = mmap(nullptr, new_index_size, PROT_READ | PROT_WRITE, MAP_SHARED, temp_index_fd, 0); - if (temp_index_map == MAP_FAILED) { - cleanup_and_throw("Cannot map temporary index file"); - } - - void* temp_data_map = mmap(nullptr, new_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, temp_data_fd, 0); - if (temp_data_map == MAP_FAILED) { - munmap(temp_index_map, new_index_size); - cleanup_and_throw("Cannot map temporary data file"); - } - - IndexHeader new_header = { - MAGIC, - VERSION, - static_cast(embedding_dim_), - compacted_count - }; - - DataHeader new_data_header = { - MAGIC, - VERSION - }; - - memcpy(temp_index_map, &new_header, sizeof(IndexHeader)); - memcpy(temp_data_map, &new_data_header, sizeof(DataHeader)); - - char* temp_index_ptr = static_cast(temp_index_map); - char* temp_data_ptr = static_cast(temp_data_map); - char* index_write_pos = temp_index_ptr + sizeof(IndexHeader); - char* data_write_pos = temp_data_ptr + sizeof(DataHeader); - - std::unordered_map new_doc_id_map; - new_doc_id_map.reserve(compacted_count); - - uint32_t new_index = 0; - new_data_offset = sizeof(DataHeader); - - for (const auto& [doc_id, index] : doc_id_map_) { - const IndexEntry& entry = *reinterpret_cast(entries + index * index_entry_size_); - const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); - uint32_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; - - memcpy(data_write_pos, data_entry, data_entry_size); - data_write_pos += data_entry_size; - - IndexEntry new_entry = {entry.doc_id, static_cast(new_data_offset), 0}; - - memcpy(index_write_pos, &new_entry, sizeof(IndexEntry)); - index_write_pos += sizeof(IndexEntry); - - size_t embedding_bytes = embedding_dim_ * sizeof(__fp16); - memcpy(index_write_pos, entry.embedding(), embedding_bytes); - index_write_pos += embedding_bytes; - - new_doc_id_map[doc_id] = new_index; - ++new_index; - - if (static_cast(data_entry_size) > std::numeric_limits::max() - new_data_offset) { - munmap(temp_index_map, new_index_size); - munmap(temp_data_map, new_data_size); - cleanup_and_throw("Data offset overflow during compaction"); - } - new_data_offset += data_entry_size; - } - - if (msync(temp_data_map, new_data_size, MS_SYNC) != 0) { - munmap(temp_index_map, new_index_size); - munmap(temp_data_map, new_data_size); - cleanup_and_throw("Failed to sync temporary data file"); - } - - if (msync(temp_index_map, new_index_size, MS_SYNC) != 0) { - munmap(temp_index_map, new_index_size); - munmap(temp_data_map, new_data_size); - cleanup_and_throw("Failed to sync temporary index file"); - } - - munmap(temp_index_map, new_index_size); - munmap(temp_data_map, new_data_size); - - std::string backup_index = index_path_ + ".backup"; - std::string backup_data = data_path_ + ".backup"; - - if (access(backup_index.c_str(), F_OK) == 0 || access(backup_data.c_str(), F_OK) == 0) { - cleanup_and_throw("Backup files already exist, previous compaction may have failed"); - } - - close(temp_index_fd); - close(temp_data_fd); - - munmap(mapped_index_, index_file_size_); - munmap(mapped_data_, data_file_size_); - close(index_fd_); - close(data_fd_); - - if (rename(index_path_.c_str(), backup_index.c_str()) != 0) { - cleanup_and_throw("Failed to backup index file"); - } - if (rename(data_path_.c_str(), backup_data.c_str()) != 0) { - rename(backup_index.c_str(), index_path_.c_str()); - cleanup_and_throw("Failed to backup data file"); - } - - if (rename(temp_data_path.c_str(), data_path_.c_str()) != 0) { - rename(backup_data.c_str(), data_path_.c_str()); - rename(backup_index.c_str(), index_path_.c_str()); - cleanup_and_throw("Failed to rename data file"); - } - - if (rename(temp_index_path.c_str(), index_path_.c_str()) != 0) { - unlink(data_path_.c_str()); - rename(backup_data.c_str(), data_path_.c_str()); - rename(backup_index.c_str(), index_path_.c_str()); - cleanup_and_throw("Failed to rename index file"); - } - - auto sync_dir = [](const std::string& file_path) { - size_t last_slash = file_path.rfind('/'); - std::string dir_path = (last_slash != std::string::npos) ? file_path.substr(0, last_slash) : "."; - int dir_fd = open(dir_path.c_str(), O_RDONLY); - if (dir_fd < 0) { - throw std::runtime_error("Cannot open directory for fsync: " + dir_path); - } - if (fsync(dir_fd) != 0) { - close(dir_fd); - throw std::runtime_error("Failed to fsync directory: " + dir_path); - } - close(dir_fd); - }; - - sync_dir(index_path_); - sync_dir(data_path_); - - unlink(backup_index.c_str()); - unlink(backup_data.c_str()); - - int new_index_fd = open(index_path_.c_str(), O_RDWR); - if (new_index_fd < 0) { - throw std::runtime_error("Cannot open new index file: " + index_path_); - } - - int new_data_fd = open(data_path_.c_str(), O_RDWR); - if (new_data_fd < 0) { - close(new_index_fd); - throw std::runtime_error("Cannot open new data file: " + data_path_); - } - - auto cleanup_fds = [&new_index_fd, &new_data_fd](const std::string& msg) { - close(new_index_fd); - close(new_data_fd); - throw std::runtime_error(msg); - }; - - struct stat index_st, data_st; - if (fstat(new_index_fd, &index_st)) { - cleanup_fds("Cannot get index file size: " + index_path_); - } - index_file_size_ = index_st.st_size; - - if (fstat(new_data_fd, &data_st)) { - cleanup_fds("Cannot get data file size: " + data_path_); - } - data_file_size_ = data_st.st_size; - - mapped_index_ = mmap(nullptr, index_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, new_index_fd, 0); - if (mapped_index_ == MAP_FAILED) { - cleanup_fds("Cannot map file: " + index_path_); - } - - mapped_data_ = mmap(nullptr, data_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, new_data_fd, 0); - if (mapped_data_ == MAP_FAILED) { - munmap(mapped_index_, index_file_size_); - close(new_index_fd); - close(new_data_fd); - throw std::runtime_error("Cannot map file: " + data_path_); - } - - index_fd_ = new_index_fd; - data_fd_ = new_data_fd; - - num_documents_ = compacted_count; - doc_id_map_ = std::move(new_doc_id_map); - } - - void Index::parse_index_header() { - const char* index_ptr = static_cast(mapped_index_); - size_t offset = 0; - - if (index_file_size_ < sizeof(IndexHeader)) { - throw std::runtime_error("Index file too small: insufficient data for header"); - } - - IndexHeader header; - header.magic = *reinterpret_cast(index_ptr + offset); - offset += sizeof(header.magic); - - if (header.magic != MAGIC) { - throw std::runtime_error("Invalid index file magic number"); - } - - header.version = *reinterpret_cast(index_ptr + offset); - offset += sizeof(header.version); - - if (header.version != VERSION) { - throw std::runtime_error("Index file version mismatch"); - } - - header.embedding_dim = *reinterpret_cast(index_ptr + offset); - offset += sizeof(header.embedding_dim); - - if (static_cast(header.embedding_dim) != embedding_dim_) { - throw std::runtime_error("Embedding dimension mismatch"); - } - - num_documents_ = *reinterpret_cast(index_ptr + offset); - } - - void Index::parse_data_header() { - const char* data_ptr = static_cast(mapped_data_); - size_t offset = 0; - - if (data_file_size_ < sizeof(DataHeader)) { - throw std::runtime_error("Data file too small: insufficient data for header"); - } - - DataHeader header; - header.magic = *reinterpret_cast(data_ptr + offset); - offset += sizeof(header.magic); - - if (header.magic != MAGIC) { - throw std::runtime_error("Invalid data file magic number"); - } - - header.version = *reinterpret_cast(data_ptr + offset); - offset += sizeof(header.version); - - if (header.version != VERSION) { - throw std::runtime_error("Data file version mismatch"); - } - } - - void Index::build_doc_id_map() { - const char* index_ptr = static_cast(mapped_index_); - const char* entries = index_ptr + sizeof(IndexHeader); - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("File corrupted: index entry extends beyond file size"); - } - } - - doc_id_map_.reserve(static_cast(num_documents_)); - - for (uint32_t i = 0; i < num_documents_; ++i) { - const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); - - if (entry.flags & 0x1) { - continue; - } - - doc_id_map_[entry.doc_id] = i; - } - } - - void Index::validate_documents(const std::vector& documents) { - if (documents.empty()) { - throw std::runtime_error("Documents vector is empty"); - } - - std::unordered_set seen_ids; - for (const auto& doc : documents) { - if (doc.content.size() > 65535) { - throw std::runtime_error("Document content too long (max 65535 bytes) for ID: " + std::to_string(doc.id)); - } - - if (doc.metadata.size() > 65535) { - throw std::runtime_error("Document metadata too long (max 65535 bytes) for ID: " + std::to_string(doc.id)); - } - - if (doc.embedding.size() != embedding_dim_) { - throw std::runtime_error("Document embedding dimension mismatch for ID: " + std::to_string(doc.id)); - } - - if (seen_ids.find(doc.id) != seen_ids.end()) { - throw std::runtime_error("Duplicate document ID in input: " + std::to_string(doc.id)); - } - seen_ids.insert(doc.id); - - if (doc_id_map_.find(doc.id) != doc_id_map_.end()) { - throw std::runtime_error("Document ID already exists: " + std::to_string(doc.id)); - } - } - } - - void Index::validate_doc_ids(const std::vector& doc_ids) { - if (doc_ids.empty()) { - throw std::runtime_error("Document IDs vector is empty"); - } - - std::unordered_set seen_ids; - for (int doc_id : doc_ids) { - if (seen_ids.find(doc_id) != seen_ids.end()) { - throw std::runtime_error("Duplicate document ID in input: " + std::to_string(doc_id)); - } - seen_ids.insert(doc_id); - - if (doc_id_map_.find(doc_id) == doc_id_map_.end()) { - throw std::runtime_error("Document ID not found: " + std::to_string(doc_id)); - } - } - } - - ssize_t Index::write_full(int fd, const void* buf, size_t count) { - size_t written = 0; - const char* ptr = static_cast(buf); - while (written < count) { - ssize_t n = write(fd, ptr + written, count - written); - if (n < 0) { - if (errno == EINTR) continue; - return -1; - } - if (n == 0) return written; - written += n; - } - return written; - } - -} // namespace index -} // namespace engine -} // namespace cactus diff --git a/cactus-transpile/engine/engine_model.cpp b/cactus-transpile/engine/engine_model.cpp deleted file mode 100644 index 42575da0b..000000000 --- a/cactus-transpile/engine/engine_model.cpp +++ /dev/null @@ -1,1098 +0,0 @@ -#include "engine.h" -#include "../models/model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - - -Model::Model() - : graph_handle_(nullptr), - config_(), - tokenizer_(nullptr), - initialized_(false), - attention_scale_(0.0f), - output_weight_node_id_(0), - owns_graph_(false) { -} - -Model::Model(const Config& config) - : graph_handle_(nullptr), - config_(config), - tokenizer_(nullptr), - initialized_(false), - attention_scale_(0.0f), - output_weight_node_id_(0), - owns_graph_(false) { -} - -Model::~Model() { - if (graph_handle_ && owns_graph_) { - delete static_cast(graph_handle_); - } -} - -bool Model::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - if (initialized_) { - return true; - } - auto* gb = new CactusGraph(); - graph_handle_ = gb; - owns_graph_ = true; - embedding_file_path_ = model_folder + "/token_embeddings.weights"; - return init_internal(gb, model_folder, context_size, system_prompt, do_warmup); -} - -bool Model::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!external_graph) { - throw std::invalid_argument("External graph pointer must not be null"); - } - if (initialized_) { - graph_handle_ = external_graph; - owns_graph_ = false; - return true; - } - - owns_graph_ = false; - graph_handle_ = external_graph; - return init_internal(external_graph, model_folder, context_size, system_prompt, do_warmup); -} - -bool Model::init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - - CACTUS_LOG_DEBUG("model", "Initializing model from: " << model_folder); - model_folder_path_ = model_folder; - std::string config_path = model_folder + "/config.txt"; - - if (!config_.from_json(config_path)) { - CACTUS_LOG_ERROR("model", "Model initialization failed - config not loaded from: " << model_folder); - return false; - } - - std::string vocab_file = model_folder + "/vocab.txt"; - std::string merges_file = model_folder + "/merges.txt"; - std::string tokenizer_config_file = model_folder + "/tokenizer_config.txt"; - TokenizerRuntimeConfig tokenizer_runtime_config = load_tokenizer_runtime_config(tokenizer_config_file); - - std::ifstream merges_check(merges_file); - bool has_merges = false; - if (merges_check.is_open()) { - std::string line; - int line_count = 0; - while (std::getline(merges_check, line) && line_count < 10) { - if (!line.empty() && line[0] != '#') { - has_merges = true; - break; - } - line_count++; - } - merges_check.close(); - } - - if (tokenizer_runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::BPE || - (tokenizer_runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::UNKNOWN && has_merges)) { - tokenizer_ = std::make_unique(); - } else { - tokenizer_ = std::make_unique(); - } - - if (!tokenizer_->load_vocabulary_with_config(vocab_file, merges_file, tokenizer_config_file)) { - return false; - } - - graph_handle_ = gb; - - if(config_.model_type == Config::ModelType::WHISPER){ - embedding_file_path_ = model_folder+"/decoder_token_embeddings.weights"; - } - else{ - embedding_file_path_ = model_folder + "/token_embeddings.weights"; - } - - load_weights_to_graph(gb); - - if (config_.model_type == Config::ModelType::GEMMA3N || config_.model_type == Config::ModelType::GEMMA4) { - attention_scale_ = 1.0f; - } else if (config_.model_type == Config::ModelType::GEMMA) { - attention_scale_ = 1.0f / std::sqrt(256.0f); - } else { - attention_scale_ = 1.0f / std::sqrt(static_cast(config_.attention_head_dim)); - } - - Precision cache_precision = (config_.model_type == Config::ModelType::WHISPER || - config_.model_type == Config::ModelType::MOONSHINE || - config_.model_type == Config::ModelType::PARAKEET || - config_.model_type == Config::ModelType::PARAKEET_TDT || - config_.model_type == Config::ModelType::NEEDLE) - ? Precision::FP16 - : Precision::INT8; - kv_cache_.init(config_.num_layers, context_size, get_kv_layer_dims(), get_kv_layer_heads(), cache_precision); - - size_t window_size = std::min(context_size, size_t(512)); - size_t sink_size = 4; - const char* env_window = std::getenv("CACTUS_KV_WINDOW_SIZE"); - const char* env_sink = std::getenv("CACTUS_KV_SINK_SIZE"); - if (env_window) { - window_size = std::stoul(env_window); - } - if (env_sink) { - sink_size = std::stoul(env_sink); - } - kv_cache_.set_window_size(window_size, sink_size); - cache_k_output_nodes_.resize(config_.num_layers); - cache_v_output_nodes_.resize(config_.num_layers); - - post_init(); - - initialized_ = true; - - if (do_warmup && - config_.model_type != Config::ModelType::WHISPER && - config_.model_type != Config::ModelType::MOONSHINE && - config_.model_type != Config::ModelType::PARAKEET && - config_.model_type != Config::ModelType::PARAKEET_TDT && - config_.model_type != Config::ModelType::NEEDLE) { - std::string warmup_text = system_prompt.empty() ? "Hello" : system_prompt; - auto warmup_tokens = tokenizer_->encode(warmup_text); - if (config_.model_type == Config::ModelType::GEMMA4) { - warmup_tokens = {2}; - } - forward(warmup_tokens); - auto* gb = static_cast(graph_handle_); - gb->execute(); - } - - reset_cache(); - return true; -} - -size_t Model::forward(const std::vector& /*mel_bins*/, const std::vector& tokens, bool use_cache){ - return forward(tokens, use_cache); -} - -void Model::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (tokens.empty()) { - return; - } - - if (has_npu_prefill()) { - size_t npu_chunk_size = static_cast(npu_prefill_->get_chunk_size()); - if (tokens.size() > npu_chunk_size) { - prefill_npu(tokens); - return; - } - } - - auto* gb = static_cast(graph_handle_); - - auto process_chunk = [&](const std::vector& chunk) { - forward(chunk, true); - gb->execute(profile_file); - post_execute_updates(gb, chunk.size()); - update_kv_cache(gb, chunk.size()); - }; - - if (tokens.size() <= chunk_size) { - process_chunk(tokens); - return; - } - - size_t num_full_chunks = (tokens.size() - 1) / chunk_size; - - for (size_t chunk_idx = 0; chunk_idx < num_full_chunks; ++chunk_idx) { - size_t start = chunk_idx * chunk_size; - size_t end = start + chunk_size; - std::vector chunk(tokens.begin() + start, tokens.begin() + end); - if (chunk_idx == 1) { - gb->set_prefill_mode(true); - } - process_chunk(chunk); - } - - gb->set_prefill_mode(false); - size_t final_start = num_full_chunks * chunk_size; - std::vector final_chunk(tokens.begin() + final_start, tokens.end()); - process_chunk(final_chunk); -} - -void Model::prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file) { - (void)image_paths; - prefill(tokens, get_prefill_chunk_size(), profile_file); -} - -uint32_t Model::decode(const std::vector& tokens, float temperature, float top_p, - size_t top_k, const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - - if (temperature < 0) { - temperature = config_.default_temperature; - } - if (top_p < 0) { - top_p = config_.default_top_p; - } - if (top_k == 0) { - top_k = config_.default_top_k; - } - auto final_hidden = forward(tokens, true); - - auto* gb = static_cast(graph_handle_); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto last_hidden = gb->index(final_hidden, tokens.size() - 1, 0); - const auto& last_hidden_buf = gb->get_output_buffer(last_hidden); - size_t hidden_dim = last_hidden_buf.shape[0]; - last_hidden = gb->reshape(last_hidden, {1, hidden_dim}); - - auto logits_node_id = gb->matmul(last_hidden, output_weight_node_id_, true, backend); - - if (config_.final_logit_softcapping > 0.0f) { - float inv_cap = 1.0f / config_.final_logit_softcapping; - logits_node_id = gb->scalar_multiply(logits_node_id, inv_cap); - logits_node_id = gb->tanh(logits_node_id); - logits_node_id = gb->scalar_multiply(logits_node_id, config_.final_logit_softcapping); - } - auto sampled_token_id = sample_token(gb, logits_node_id, temperature, top_p, top_k, min_p, repetition_penalty); - - gb->execute(profile_file); - - compute_entropy(gb, logits_node_id, out_entropy); - - post_execute_updates(gb, tokens.size()); - update_kv_cache(gb, tokens.size()); - - auto* output_ptr = gb->get_output(sampled_token_id); - uint32_t result_token = *static_cast(output_ptr); - record_sampled_token(result_token); - return result_token; -} - -size_t Model::sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, - float min_p, float repetition_penalty, - const std::unordered_map* extra_bias) const { - auto combined_bias = tool_constrainer_.get_bias(); - for (const auto& [token_id, boost] : vocab_bias_) { - combined_bias[token_id] += boost; - } - if (extra_bias) { - for (const auto& [token_id, boost] : *extra_bias) { - combined_bias[token_id] += boost; - } - } - if (!token_history_.empty() && repetition_penalty > 1.0f && std::isfinite(repetition_penalty)) { - float log_penalty = std::log(repetition_penalty); - for (uint32_t tok : token_history_) { - combined_bias[tok] -= log_penalty; - } - } - return gb->sample_with_options(logits_node_id, temperature, top_p, min_p, 1.0f, top_k, combined_bias); -} - -void Model::compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy) { - if (!out_entropy) return; - - const auto& logits_buf = gb->get_output_buffer(logits_node_id); - void* logits_ptr = gb->get_output(logits_node_id); - size_t vocab_size = logits_buf.shape.back(); - size_t seq_len = 1; - if (logits_buf.shape.size() >= 2) - seq_len = logits_buf.shape[logits_buf.shape.size() - 2]; - size_t row_offset = (seq_len > 0 ? (seq_len - 1) * vocab_size : 0); - - std::vector logits(vocab_size); - if (logits_buf.precision == Precision::FP32) { - float* src = static_cast(logits_ptr) + row_offset; - std::copy(src, src + vocab_size, logits.begin()); - } else if (logits_buf.precision == Precision::FP16) { - __fp16* src = static_cast<__fp16*>(logits_ptr) + row_offset; - Quantization::fp16_to_fp32(src, logits.data(), vocab_size); - } else { - int8_t* src = static_cast(logits_ptr) + row_offset; - Quantization::int8_to_fp32(src, logits.data(), vocab_size, 1.0f); - } - - float max_logit = *std::max_element(logits.begin(), logits.end()); - double sum_exp = 0.0; - for (size_t i = 0; i < vocab_size; ++i) - sum_exp += std::exp(static_cast(logits[i] - max_logit)); - double log_sum_exp = static_cast(max_logit) + std::log(sum_exp); - - double entropy = 0.0; - for (size_t i = 0; i < vocab_size; ++i) { - double log_prob = static_cast(logits[i]) - log_sum_exp; - double prob = std::exp(log_prob); - if (prob > 1e-10) - entropy -= prob * log_prob; - } - - double max_entropy = std::log(static_cast(vocab_size)); - *out_entropy = static_cast(entropy / max_entropy); -} - -uint32_t Model::decode_with_audio(const std::vector& tokens, const std::vector& /*mel_bins*/, float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty, - float* /*out_token_time_start*/, float* /*out_token_time_end*/){ - return decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -uint32_t Model::decode_with_images(const std::vector& tokens, const std::vector& image_paths, - float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - (void)image_paths; - return decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -std::vector Model::get_image_embeddings(const std::string& /*image_path*/) { - throw std::runtime_error("Image embeddings not supported for this model type"); -} - -std::vector Model::get_audio_embeddings(const std::vector& /*mel_bins*/) { - throw std::runtime_error("Audio embeddings not supported for this model type"); -} - -void Model::update_kv_cache(CactusGraph* gb, size_t seq_len) { - kv_cache_.update_from_graph(gb, cache_k_output_nodes_, cache_v_output_nodes_, - seq_len, config_.num_layers); -} - -void Model::remove_thinking_tokens(const std::vector>& ranges) { - for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) - kv_cache_.remove_token_range(it->first, it->second); -} - -std::vector Model::get_embeddings(const std::vector& tokens, bool pooled, bool normalize, const std::string& profile_file) { - std::vector embeddings; - auto final_hidden = forward(tokens); - - auto* gb = static_cast(graph_handle_); - auto* output_ptr = gb->get_output(final_hidden); - const auto& output_buffer = gb->get_output_buffer(final_hidden); - - if (pooled) { - auto pooled_hidden = gb->mean(final_hidden, 0); - - if (!profile_file.empty()) { - gb->execute(profile_file); - } else { - gb->execute(); - } - post_execute_updates(gb, tokens.size()); - auto* pooled_ptr = gb->get_output(pooled_hidden); - const auto& pooled_buffer = gb->get_output_buffer(pooled_hidden); - - size_t hidden_dim = pooled_buffer.total_size; - embeddings.resize(hidden_dim); - - if (pooled_buffer.precision == Precision::FP32) { - float* pooled_data = static_cast(pooled_ptr); - std::copy(pooled_data, pooled_data + hidden_dim, embeddings.begin()); - } else if (pooled_buffer.precision == Precision::FP16) { - __fp16* pooled_data = static_cast<__fp16*>(pooled_ptr); - Quantization::fp16_to_fp32(pooled_data, embeddings.data(), hidden_dim); - } else if (pooled_buffer.precision == Precision::INT8) { - int8_t* pooled_data = static_cast(pooled_ptr); - Quantization::int8_to_fp32(pooled_data, embeddings.data(), hidden_dim, 1.0f); - } - } else { - if (!profile_file.empty()) { - gb->execute(profile_file); - } else { - gb->execute(); - } - post_execute_updates(gb, tokens.size()); - - size_t total_size = output_buffer.total_size; - embeddings.resize(total_size); - - if (output_buffer.precision == Precision::FP32) { - float* hidden_states = static_cast(output_ptr); - std::copy(hidden_states, hidden_states + total_size, embeddings.begin()); - } else if (output_buffer.precision == Precision::FP16) { - __fp16* hidden_states = static_cast<__fp16*>(output_ptr); - for (size_t i = 0; i < total_size; i++) { - embeddings[i] = static_cast(hidden_states[i]); - } - } else if (output_buffer.precision == Precision::INT8) { - int8_t* hidden_states = static_cast(output_ptr); - for (size_t i = 0; i < total_size; i++) { - embeddings[i] = static_cast(hidden_states[i]); - } - } - } - - if (normalize && !embeddings.empty()) { - float norm_sq = 0.0f; - for (float v : embeddings) { - norm_sq += v * v; - } - float norm = std::sqrt(norm_sq); - if (norm > 1e-12f) { - float inv_norm = 1.0f / norm; - for (float& v : embeddings) { - v *= inv_norm; - } - } - } - - kv_cache_.reset(); - - return embeddings; -} - -bool Config::from_json(const std::string& config_path) { - std::ifstream file(config_path); - if (!file) { - CACTUS_LOG_ERROR("config", "Failed to open config file: " << config_path); - return false; - } - - std::string line; - while (std::getline(file, line)) { - if (line.empty() || line[0] == '#') continue; - - size_t eq_pos = line.find('='); - if (eq_pos == std::string::npos) continue; - - std::string key = line.substr(0, eq_pos); - std::string value = line.substr(eq_pos + 1); - - key.erase(0, key.find_first_not_of(" \t")); - key.erase(key.find_last_not_of(" \t") + 1); - value.erase(0, value.find_first_not_of(" \t")); - value.erase(value.find_last_not_of(" \t") + 1); - - if (key == "vocab_size") vocab_size = static_cast(std::stoul(value)); - else if (key == "bos_token_id") bos_token_id = static_cast(std::stoul(value)); - else if (key == "eos_token_id") eos_token_id = static_cast(std::stoul(value)); - else if (key == "num_layers") num_layers = static_cast(std::stoul(value)); - else if (key == "hidden_dim") hidden_dim = static_cast(std::stoul(value)); - else if (key == "ffn_intermediate_dim") ffn_intermediate_dim = static_cast(std::stoul(value)); - else if (key == "attention_heads") attention_heads = static_cast(std::stoul(value)); - else if (key == "attention_kv_heads") attention_kv_heads = static_cast(std::stoul(value)); - else if (key == "attention_head_dim") attention_head_dim = static_cast(std::stoul(value)); - else if (key == "layer_norm_eps") layer_norm_eps = std::stof(value); - else if (key == "rope_theta") rope_theta = std::stof(value); - else if (key == "num_experts") num_experts = static_cast(std::stoul(value)); - else if (key == "num_shared_experts") num_shared_experts = static_cast(std::stoul(value)); - else if (key == "num_top_experts") num_top_experts = static_cast(std::stoul(value)); - else if (key == "moe_every_n_layers") moe_every_n_layers = static_cast(std::stoul(value)); - else if (key == "moe_intermediate_dim" || key == "moe_intermediate_size") moe_intermediate_dim = static_cast(std::stoul(value)); - else if (key == "num_dense_layers") num_dense_layers = static_cast(std::stoul(value)); - else if (key == "num_experts_per_tok") num_experts_per_tok = static_cast(std::stoul(value)); - else if (key == "norm_topk_prob") norm_topk_prob = (value == "true" || value == "1"); - else if (key == "use_expert_bias") use_expert_bias = (value == "true" || value == "1"); - else if (key == "routed_scaling_factor") routed_scaling_factor = std::stof(value); - else if (key == "tie_word_embeddings") tie_word_embeddings = (value == "true" || value == "1"); - else if (key == "vision_hidden_dim" || key == "vision_hidden_size") vision_hidden_dim = static_cast(std::stoul(value)); - else if (key == "vision_num_layers") vision_num_layers = static_cast(std::stoul(value)); - else if (key == "vision_attention_heads") vision_attention_heads = static_cast(std::stoul(value)); - else if (key == "vision_image_size") vision_image_size = static_cast(std::stoul(value)); - else if (key == "vision_patch_size") vision_patch_size = static_cast(std::stoul(value)); - else if (key == "vision_num_channels") vision_num_channels = static_cast(std::stoul(value)); - else if (key == "vision_embed_dim") vision_embed_dim = static_cast(std::stoul(value)); - else if (key == "visual_tokens_per_img") visual_tokens_per_img = static_cast(std::stoul(value)); - else if (key == "use_pixel_shuffle") use_pixel_shuffle = (value == "true" || value == "1"); - else if (key == "pixel_shuffle_factor") pixel_shuffle_factor = static_cast(std::stoul(value)); - else if (key == "use_image_tokens") use_image_tokens = (value == "true" || value == "1"); - else if (key == "image_token_id") image_token_id = static_cast(std::stoul(value)); - else if (key == "use_layout_tags") use_layout_tags = (value == "true" || value == "1"); - else if (key == "image_seq_len") image_seq_len = static_cast(std::stoul(value)); - else if (key == "global_image_size") global_image_size = static_cast(std::stoul(value)); - else if (key == "max_tile_size") max_tile_size = static_cast(std::stoul(value)); - else if (key == "rescale_factor") rescale_factor = std::stof(value); - else if (key == "image_mean") image_mean = std::stof(value); - else if (key == "image_std") image_std = std::stof(value); - else if (key == "downsample_factor") downsample_factor = static_cast(std::stoul(value)); - else if (key == "min_tiles") min_tiles = static_cast(std::stoul(value)); - else if (key == "max_tiles") max_tiles = static_cast(std::stoul(value)); - else if (key == "use_thumbnail") use_thumbnail = (value == "true" || value == "1"); - else if (key == "min_image_tokens") min_image_tokens = static_cast(std::stoul(value)); - else if (key == "max_image_tokens") max_image_tokens = static_cast(std::stoul(value)); - else if (key == "tile_size") tile_size = static_cast(std::stoul(value)); - else if (key == "max_pixels_tolerance") max_pixels_tolerance = std::stof(value); - else if (key == "do_image_splitting") do_image_splitting = (value == "true" || value == "1"); - else if (key == "precision") { - if (value == "INT8") precision = Precision::INT8; - else if (value == "FP16") precision = Precision::FP16; - else precision = Precision::FP32; - } - else if (key == "model_type") { - std::string model_type_value = value; - std::transform(model_type_value.begin(), model_type_value.end(), model_type_value.begin(), ::tolower); - if (value == "gemma" || value == "GEMMA") model_type = ModelType::GEMMA; - else if (value == "lfm2" || value == "LFM2" || value == "lfm2_moe" || value == "LFM2_MOE") model_type = ModelType::LFM2; - else if (value == "bert" || value == "BERT") model_type = ModelType::NOMIC; - else if (value == "whisper" || value == "WHISPER") model_type = ModelType::WHISPER; - else if (value == "moonshine" || value == "MOONSHINE") model_type = ModelType::MOONSHINE; - else if (value == "silero_vad" || value == "SILERO_VAD") model_type = ModelType::SILERO_VAD; - else if (value == "parakeet" || value == "PARAKEET") model_type = ModelType::PARAKEET; - else if (model_type_value.rfind("qwen3_5", 0) == 0) model_type = ModelType::QWEN3P5; - else if (value == "parakeet_tdt" || value == "PARAKEET_TDT") model_type = ModelType::PARAKEET_TDT; - else if (value == "gemma3n" || value == "GEMMA3N") model_type = ModelType::GEMMA3N; - else if (value == "needle" || value == "NEEDLE") model_type = ModelType::NEEDLE; - else if (value == "gemma4" || value == "GEMMA4" || value == "tinyllama" || value == "TINYLLAMA") model_type = ModelType::GEMMA4; - else if (value == "youtu" || value == "YOUTU") model_type = ModelType::YOUTU; - else if (value == "pyannote" || value == "PYANNOTE") model_type = ModelType::PYANNOTE; - else if (value == "wespeaker" || value == "WESPEAKER") model_type = ModelType::WESPEAKER; - else model_type = ModelType::QWEN; - } - else if (key == "model_variant") { - std::string v = value; - std::transform(v.begin(), v.end(), v.begin(), ::tolower); - if (v == "vlm") model_variant = ModelVariant::VLM; - else if (v == "extract") model_variant = ModelVariant::EXTRACT; - else if (v == "rag") model_variant = ModelVariant::RAG; - else model_variant = ModelVariant::DEFAULT; - } - else if (key == "conv_L_cache") conv_L_cache = static_cast(std::stoul(value)); - else if (key == "layer_types") { - layer_types.clear(); - std::string sanitized; - sanitized.reserve(value.size()); - for (char c : value) { - if (c == '[' || c == ']' || c == '\'' || c == '"') { - continue; - } - sanitized.push_back(c); - } - std::stringstream ss(sanitized); - std::string item; - while (std::getline(ss, item, ',')) { - if (!item.empty()) { - item.erase(0, item.find_first_not_of(" \t")); - item.erase(item.find_last_not_of(" \t") + 1); - if (!item.empty()) layer_types.push_back(item); - } - } - } - else if (key == "enc_hidden_act") encoder_act_gelu = (value == "gelu"); - else if (key == "dec_hidden_act") decoder_act_gelu = (value == "gelu"); - else if (key == "num_encoder_layers") num_encoder_layers = static_cast(std::stoul(value)); - else if (key == "num_decoder_layers") num_decoder_layers = static_cast(std::stoul(value)); - else if (key == "partial_rotary_factor") partial_rotary_factor = std::stof(value); - else if (key == "pad_token_id") pad_token_id = static_cast(std::stoul(value)); - else if (key == "conv_kernel_size") conv_kernel_size = static_cast(std::stoul(value)); - else if (key == "subsampling_conv_kernel_size") subsampling_conv_kernel_size = static_cast(std::stoul(value)); - else if (key == "subsampling_conv_stride") subsampling_conv_stride = static_cast(std::stoul(value)); - else if (key == "subsampling_conv_channels") subsampling_conv_channels = static_cast(std::stoul(value)); - else if (key == "subsampling_factor") subsampling_factor = static_cast(std::stoul(value)); - else if (key == "num_mel_bins") num_mel_bins = static_cast(std::stoul(value)); - else if (key == "encoder_hidden_act") encoder_hidden_act = value; - else if (key == "linear_num_key_heads") linear_num_key_heads = static_cast(std::stoul(value)); - else if (key == "linear_key_head_dim") linear_key_head_dim = static_cast(std::stoul(value)); - else if (key == "linear_num_value_heads") linear_num_value_heads = static_cast(std::stoul(value)); - else if (key == "linear_value_head_dim") linear_value_head_dim = static_cast(std::stoul(value)); - else if (key == "linear_q_proj_dim") linear_q_proj_dim = static_cast(std::stoul(value)); - else if (key == "kv_lora_rank") kv_lora_rank = static_cast(std::stoul(value)); - else if (key == "q_lora_rank") q_lora_rank = static_cast(std::stoul(value)); - else if (key == "qk_head_dim") qk_head_dim = static_cast(std::stoul(value)); - else if (key == "qk_nope_head_dim") qk_nope_head_dim = static_cast(std::stoul(value)); - else if (key == "qk_rope_head_dim") qk_rope_head_dim = static_cast(std::stoul(value)); - else if (key == "v_head_dim") v_head_dim = static_cast(std::stoul(value)); - else if (key == "rope_interleave") rope_interleave = (value == "true" || value == "1"); - else if (key == "attention_bias") attention_bias = (value == "true" || value == "1"); - else if (key == "rope_scaling_factor") rope_scaling_factor = std::stof(value); - else if (key == "rope_mscale_all_dim") rope_mscale_all_dim = std::stof(value); - else if (key == "linear_k_proj_dim") linear_k_proj_dim = static_cast(std::stoul(value)); - else if (key == "linear_v_proj_dim") linear_v_proj_dim = static_cast(std::stoul(value)); - else if (key == "predictor_hidden_dim") predictor_hidden_dim = static_cast(std::stoul(value)); - else if (key == "predictor_num_layers") predictor_num_layers = static_cast(std::stoul(value)); - else if (key == "tdt_joint_dim") tdt_joint_dim = static_cast(std::stoul(value)); - else if (key == "tdt_num_durations") tdt_num_durations = static_cast(std::stoul(value)); - else if (key == "tdt_blank_id") tdt_blank_id = static_cast(std::stoul(value)); - else if (key == "tdt_durations") { - tdt_durations.clear(); - std::stringstream ss(value); - std::string item; - while (std::getline(ss, item, ',')) { - size_t first = item.find_first_not_of(" \t"); - if (first == std::string::npos) continue; - size_t last = item.find_last_not_of(" \t"); - item = item.substr(first, last - first + 1); - tdt_durations.push_back(static_cast(std::stoul(item))); - } - } - else if (key == "altup_num_inputs") altup_num_inputs = static_cast(std::stoul(value)); - else if (key == "laurel_rank") laurel_rank = static_cast(std::stoul(value)); - else if (key == "hidden_size_per_layer_input") hidden_size_per_layer_input = static_cast(std::stoul(value)); - else if (key == "num_kv_shared_layers") num_kv_shared_layers = static_cast(std::stoul(value)); - else if (key == "sliding_window") sliding_window = static_cast(std::stoul(value)); - else if (key == "rope_local_base_freq") rope_local_base_freq = std::stof(value); - else if (key == "final_logit_softcapping") final_logit_softcapping = std::stof(value); - else if (key == "global_partial_rotary_factor") global_partial_rotary_factor = std::stof(value); - else if (key == "expert_intermediate_size") expert_intermediate_size = static_cast(std::stoul(value)); - else if (key == "global_head_dim") global_head_dim = static_cast(std::stoul(value)); - else if (key == "num_global_kv_heads" || key == "num_global_key_value_heads") num_global_kv_heads = static_cast(std::stoul(value)); - else if (key == "attention_k_eq_v") attention_k_eq_v = (value == "true" || value == "1"); - else if (key == "enable_moe_block") enable_moe_block = (value == "true" || value == "1"); - else if (key == "vision_head_dim") vision_head_dim = static_cast(std::stoul(value)); - else if (key == "vision_kv_heads") vision_kv_heads = static_cast(std::stoul(value)); - else if (key == "vision_intermediate_size") vision_intermediate_size = static_cast(std::stoul(value)); - else if (key == "vision_position_embedding_size") vision_position_embedding_size = static_cast(std::stoul(value)); - else if (key == "vision_pooling_kernel_size") vision_pooling_kernel_size = static_cast(std::stoul(value)); - else if (key == "vision_default_output_length") vision_default_output_length = static_cast(std::stoul(value)); - else if (key == "vision_rope_theta") vision_rope_theta = std::stof(value); - else if (key == "audio_hidden_dim") audio_hidden_dim = static_cast(std::stoul(value)); - else if (key == "audio_num_layers") audio_num_layers = static_cast(std::stoul(value)); - else if (key == "audio_num_heads") audio_num_heads = static_cast(std::stoul(value)); - else if (key == "audio_head_dim") audio_head_dim = static_cast(std::stoul(value)); - else if (key == "audio_input_feat_size") audio_input_feat_size = static_cast(std::stoul(value)); - else if (key == "audio_conf_conv_kernel_size") audio_conf_conv_kernel_size = static_cast(std::stoul(value)); - else if (key == "audio_chunk_size") audio_chunk_size = static_cast(std::stoul(value)); - else if (key == "audio_context_left") audio_context_left = static_cast(std::stoul(value)); - else if (key == "audio_context_right") audio_context_right = static_cast(std::stoul(value)); - else if (key == "audio_logit_cap") audio_logit_cap = std::stof(value); - else if (key == "audio_residual_weight") audio_residual_weight = std::stof(value); - else if (key == "audio_output_proj_dims") audio_output_proj_dims = static_cast(std::stoul(value)); - else if (key == "audio_vocab_size") audio_vocab_size = static_cast(std::stoul(value)); - else if (key == "audio_vocab_offset") audio_vocab_offset = static_cast(std::stoul(value)); - else if (key == "audio_soft_tokens") audio_soft_tokens = static_cast(std::stoul(value)); - else if (key == "audio_sscp_conv0_channels") audio_sscp_conv0_channels = static_cast(std::stoul(value)); - else if (key == "audio_sscp_conv1_channels") audio_sscp_conv1_channels = static_cast(std::stoul(value)); - else if (key == "audio_sscp_conv_eps") audio_sscp_conv_eps = std::stof(value); - else if (key == "audio_rms_norm_eps") audio_rms_norm_eps = std::stof(value); - else if (key == "audio_fft_length") audio_fft_length = static_cast(std::stoul(value)); - else if (key == "audio_fft_overdrive") { - audio_fft_overdrive = (value == "true" || value == "1"); - audio_fft_length = audio_fft_overdrive ? 1024u : 512u; - } - else if (key == "audio_token_id") audio_token_id = static_cast(std::stoul(value)); - else if (key == "channel_open_token_id") channel_open_token_id = static_cast(std::stoul(value)); - else if (key == "channel_close_token_id") channel_close_token_id = static_cast(std::stoul(value)); - else if (key == "activation_sparsity_ppf") { - activation_sparsity_ppf.clear(); - std::stringstream ss(value); - std::string item; - while (std::getline(ss, item, ',')) { - size_t first = item.find_first_not_of(" \t"); - if (first == std::string::npos) continue; - size_t last = item.find_last_not_of(" \t"); - item = item.substr(first, last - first + 1); - activation_sparsity_ppf.push_back(std::stof(item)); - } - } - } - - if (is_gemma_family(model_type)) { - default_temperature = 1.0f; - default_top_p = 0.95f; - default_top_k = 64; - } else if (model_type == ModelType::LFM2) { - default_temperature = 0.3f; - default_top_p = 0.95f; - default_top_k = 20; - } else if (model_type == ModelType::QWEN) { - default_temperature = 0.6f; - default_top_p = 0.95f; - default_top_k = 20; - } else if (model_type == ModelType::QWEN3P5) { - default_temperature = 0.7f; - default_top_p = 0.8f; - default_top_k = 20; - } else if (model_type == ModelType::WHISPER) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - default_cloud_handoff_threshold = 0.4f; - } else if (model_type == ModelType::MOONSHINE) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - default_max_tps = 6.5f; - default_cloud_handoff_threshold = 0.35f; - } else if (model_type == ModelType::PARAKEET) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - default_max_tps = 8.0f; - default_cloud_handoff_threshold = 0.35f; - } else if (model_type == ModelType::PARAKEET_TDT) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - default_max_tps = 8.0f; - default_cloud_handoff_threshold = 0.35f; - } else if (model_type == ModelType::NEEDLE) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - } else if (model_type == ModelType::YOUTU) { - default_temperature = 1.0f; - default_top_p = 0.95f; - default_top_k = 20; - } - - return true; -} - -std::string Config::to_json() const { - return "{}"; -} - -std::unique_ptr create_model(const std::string& model_folder) { - CACTUS_LOG_DEBUG("model", "Creating model from: " << model_folder); - Config config; - std::string config_path = model_folder + "/config.txt"; - - if (!config.from_json(config_path)) { - CACTUS_LOG_ERROR("model", "Failed to create model - cannot load config from: " << model_folder); - return nullptr; - } - - const bool has_vision_support = - config.use_image_tokens || - config.vision_num_layers > 0 || - config.vision_embed_dim > 0 || - config.vision_hidden_dim > 0 || - config.visual_tokens_per_img > 0; - - if (config.model_type == Config::ModelType::LFM2 && has_vision_support) { - return std::make_unique(config); - } - - const bool has_audio_support = - config.audio_num_layers > 0 || - config.audio_hidden_dim > 0; - - if (config.model_type == Config::ModelType::GEMMA4 && (has_vision_support || has_audio_support)) { - return std::make_unique(config); - } - - switch (config.model_type) { - case Config::ModelType::QWEN: - return std::make_unique(config); - case Config::ModelType::QWEN3P5: - return std::make_unique(config); - case Config::ModelType::GEMMA: - return std::make_unique(config); - case Config::ModelType::GEMMA3N: - return std::make_unique(config); - case Config::ModelType::LFM2: - if (config.num_experts > 0 && config.moe_intermediate_dim > 0 && config.num_experts_per_tok > 0) { - return std::make_unique(config); - } - return std::make_unique(config); - case Config::ModelType::NOMIC: - return std::make_unique(config); - case Config::ModelType::WHISPER: - return std::make_unique(config); - case Config::ModelType::MOONSHINE: - return std::make_unique(config); - case Config::ModelType::SILERO_VAD: - return std::make_unique(config); - case Config::ModelType::PARAKEET: - return std::make_unique(config); - case Config::ModelType::PARAKEET_TDT: - return std::make_unique(config); - case Config::ModelType::NEEDLE: - return std::make_unique(config); - case Config::ModelType::GEMMA4: - return std::make_unique(config); - case Config::ModelType::YOUTU: - return std::make_unique(config); - case Config::ModelType::PYANNOTE: - return std::make_unique(config); - case Config::ModelType::WESPEAKER: - return std::make_unique(config); - default: - return std::make_unique(config); - } -} - -void Model::capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const { - auto* graph = static_cast(graph_handle_); - if (!graph) { - return; - } - graph->capture_debug_node(layer_idx, name, node_id); -} - -void Model::clear_debug_nodes() { - auto* graph = static_cast(graph_handle_); - if (!graph) { - return; - } - graph->clear_debug_nodes(); -} - -const std::vector& Model::get_debug_nodes() const { - auto* graph = static_cast(graph_handle_); - debug_nodes_.clear(); - if (!graph) { - return debug_nodes_; - } - - const auto& entries = graph->get_debug_nodes(); - debug_nodes_.reserve(entries.size()); - for (const auto& entry : entries) { - debug_nodes_.push_back({entry.layer_idx, entry.name, entry.node_id}); - } - return debug_nodes_; -} - -bool Model::load_npu_prefill(const std::string& model_path) { - CACTUS_LOG_DEBUG("npu", "Attempting to load NPU prefill from: " << model_path); - - npu_prefill_ = npu::create_prefill(); - if (!npu_prefill_) { - CACTUS_LOG_DEBUG("npu", "NPU prefill creation failed (not supported on this device)"); - return false; - } - - bool loaded = npu_prefill_->load(model_path); - if (loaded) { - CACTUS_LOG_INFO("npu", "NPU prefill loaded successfully from: " << model_path); - } else { - CACTUS_LOG_DEBUG("npu", "NPU prefill model not found at: " << model_path); - } - return loaded; -} - -bool Model::has_npu_prefill() const { - return npu_prefill_ && npu_prefill_->is_available(); -} - -size_t Model::get_prefill_chunk_size() const { - if (has_npu_prefill()) { - return static_cast(npu_prefill_->get_chunk_size()); - } - return 256; // default chunk size -} - -std::vector<__fp16> Model::get_token_embeddings(const std::vector& tokens) { - auto* gb = static_cast(graph_handle_); - if (!gb || tokens.empty()) { - return {}; - } - - gb->soft_reset(); - - size_t tok_input = gb->input({tokens.size()}, Precision::FP32); - std::vector tok_f(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) { - tok_f[i] = static_cast(tokens[i]); - } - gb->set_input(tok_input, tok_f.data(), Precision::FP32); - - size_t embedding_node = gb->embedding(embedding_node_id_, tok_input); - - gb->execute(); - - const auto& emb_buf = gb->get_output_buffer(embedding_node); - void* emb_ptr = gb->get_output(embedding_node); - - size_t num_tokens = tokens.size(); - size_t hidden_dim = config_.hidden_dim; - std::vector<__fp16> embeddings(num_tokens * hidden_dim); - - if (emb_buf.precision == Precision::FP16) { - __fp16* src = static_cast<__fp16*>(emb_ptr); - std::copy(src, src + num_tokens * hidden_dim, embeddings.begin()); - } else if (emb_buf.precision == Precision::FP32) { - float* src = static_cast(emb_ptr); - for (size_t i = 0; i < num_tokens * hidden_dim; i++) { - embeddings[i] = static_cast<__fp16>(src[i]); - } - } else if (emb_buf.precision == Precision::INT8) { - int8_t* src = static_cast(emb_ptr); - for (size_t i = 0; i < num_tokens * hidden_dim; i++) { - embeddings[i] = static_cast<__fp16>(src[i]); - } - } - - return embeddings; -} - -void Model::prefill_npu(const std::vector& tokens) { - if (!npu_prefill_ || !npu_prefill_->is_available()) { - throw std::runtime_error("NPU prefill not available"); - } - - const int chunk_size = npu_prefill_->get_chunk_size(); - const int hidden_dim = npu_prefill_->get_hidden_dim(); - const int num_layers = npu_prefill_->get_num_layers(); - const int fallback_num_kv_heads = npu_prefill_->get_num_kv_heads(); - const int fallback_head_dim = npu_prefill_->get_head_dim(); - - const std::vector layer_dims = get_kv_layer_dims(); - const std::vector layer_heads = get_kv_layer_heads(); - const int layers_to_update = std::min(num_layers, static_cast(config_.num_layers)); - - std::vector<__fp16> all_embeddings = get_token_embeddings(tokens); - if (all_embeddings.empty()) { - throw std::runtime_error("Failed to get token embeddings for NPU prefill"); - } - - if (Config::is_gemma_family(config_.model_type)) { - float scale = std::sqrt(static_cast(hidden_dim)); - for (size_t i = 0; i < all_embeddings.size(); i++) { - all_embeddings[i] = __fp16(static_cast(all_embeddings[i]) * scale); - } - } - - size_t num_tokens = tokens.size(); - size_t num_chunks = (num_tokens + chunk_size - 1) / chunk_size; - - for (size_t c = 0; c < num_chunks; c++) { - size_t start = c * chunk_size; - size_t actual_tokens = std::min(static_cast(chunk_size), num_tokens - start); - - std::vector<__fp16> chunk_embeddings(chunk_size * hidden_dim, __fp16(0)); - std::copy(all_embeddings.begin() + start * hidden_dim, - all_embeddings.begin() + (start + actual_tokens) * hidden_dim, - chunk_embeddings.begin()); - - int position_offset = static_cast(start); - - npu::NPUPrefillDirectResult direct_result = npu_prefill_->prefill_chunk_direct(chunk_embeddings, position_offset); - - if (direct_result.valid) { - for (int layer_idx = 0; layer_idx < layers_to_update; layer_idx++) { - const auto& k_ref = direct_result.k_caches[layer_idx]; - const auto& v_ref = direct_result.v_caches[layer_idx]; - - if (k_ref.data && v_ref.data) { - size_t layer_kv_heads = layer_idx < static_cast(layer_heads.size()) - ? layer_heads[layer_idx] - : static_cast(fallback_num_kv_heads); - size_t layer_head_dim = layer_idx < static_cast(layer_dims.size()) - ? layer_dims[layer_idx] - : static_cast(fallback_head_dim); - - size_t expected = static_cast(chunk_size) * layer_kv_heads * layer_head_dim; - if (expected > 0 && (k_ref.count < expected || v_ref.count < expected)) { - CACTUS_LOG_WARN( - "npu", - "NPU prefill cache output too small for layer " << layer_idx - << " (expected>=" << expected - << ", got k=" << k_ref.count << ", v=" << v_ref.count << "); skipping layer"); - continue; - } - - kv_cache_.update_from_npu(layer_idx, k_ref.data, v_ref.data, - actual_tokens, layer_kv_heads, layer_head_dim); - } - } - } - } -} - -double Model::score_tokens_window_logprob( - const std::vector& tokens, - size_t start, - size_t end, - size_t context, - size_t* tokens_scored -) { - if (tokens_scored) - *tokens_scored = 0; - - if (tokens.empty()) - return 0.0; - - if (end > tokens.size()) - end = tokens.size(); - - if (start >= end) - return 0.0; - - if (start == 0) - start = 1; - - if (start >= end) - return 0.0; - - const size_t target_len = end - start; - const size_t ctx_begin = (start > context) ? (start - context) : 0; - - if (end < 2) return 0.0; - const size_t input_end = end - 1; - - if (input_end <= ctx_begin) - return 0.0; - - std::vector input_tokens(tokens.begin() + ctx_begin,tokens.begin() + input_end); - - if (tokens_scored) - *tokens_scored = target_len; - - reset_cache(); - - auto* gb = static_cast(graph_handle_); - const auto backend = (config_.default_backend == Config::Backend::CPU) ? ComputeBackend::CPU : ComputeBackend::NPU; - - const size_t hidden_node = forward(input_tokens, /*use_cache=*/false); - const auto& hidden_buf = gb->get_output_buffer(hidden_node); - - if (hidden_buf.shape.size() != 2) { - throw std::runtime_error("Expected hidden to be rank-2 [L, hidden_dim]"); - } - - - const size_t first_pos = start - ctx_begin - 1; - const size_t hidden_slice = gb->slice(hidden_node, /*axis=*/0, first_pos, target_len); - bool transpose_w = true; - const size_t logits_node = gb->matmul(hidden_slice, output_weight_node_id_, transpose_w, backend); - gb->execute(); - - const auto& logits_buf = gb->get_output_buffer(logits_node); - if (logits_buf.shape.size() != 2) - throw std::runtime_error("Expected logits to be rank-2 [T, vocab]"); - - const size_t T = logits_buf.shape[0]; - const size_t vocab_size = logits_buf.shape[1]; - - if (T != target_len) - throw std::runtime_error("Logits T dimension does not match target_len"); - - void* logits_ptr = gb->get_output(logits_node); - std::vector row(vocab_size); - double total_logprob = 0.0; - - for (size_t i = 0; i < target_len; ++i) { - const uint32_t y = tokens[start + i]; - if (y >= vocab_size) - throw std::runtime_error("Target token out of vocab range"); - - if (logits_buf.precision == Precision::FP32) { - const float* src = static_cast(logits_ptr) + i * vocab_size; - std::memcpy(row.data(), src, vocab_size * sizeof(float)); - } - else if (logits_buf.precision == Precision::FP16) { - const __fp16* src = static_cast(logits_ptr) + i * vocab_size; - Quantization::fp16_to_fp32(const_cast<__fp16*>(src), row.data(), vocab_size); - } - else { - const int8_t* src = static_cast(logits_ptr) + i * vocab_size; - Quantization::int8_to_fp32(const_cast(src), row.data(), vocab_size, 1.0f); - } - - float max_logit = *std::max_element(row.begin(), row.end()); - double sum = 0.0; - - for (size_t j = 0; j < vocab_size; ++j) - sum += std::exp(double(row[j] - max_logit)); - - const double lse = double(max_logit) + std::log(sum); - total_logprob += double(row[y]) - lse; - } - - return total_logprob; -} -} -} diff --git a/cactus-transpile/engine/engine_signals.cpp b/cactus-transpile/engine/engine_signals.cpp deleted file mode 100644 index 38468b4c0..000000000 --- a/cactus-transpile/engine/engine_signals.cpp +++ /dev/null @@ -1,775 +0,0 @@ -#include "engine.h" -#include "../kernel/kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#endif - -#ifndef M_PI -#define M_PI 3.14159265358979323846 -#endif - -namespace cactus { -namespace engine { - -static void to_db( - float* spectrogram, - size_t size, - float reference, - float min_value, - const float* db_range, - float multiplier) -{ - if (reference <= 0.0f) { - throw std::invalid_argument("reference must be greater than zero"); - } - if (min_value <= 0.0f) { - throw std::invalid_argument("min_value must be greater than zero"); - } - - reference = std::max(min_value, reference); - const float log_ref = std::log10(reference); - - CactusThreading::parallel_for(size, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { - for (size_t i = start; i < end; i++) { - float value = std::max(min_value, spectrogram[i]); - spectrogram[i] = multiplier * (std::log10(value) - log_ref); - } - }); - - if (db_range != nullptr) { - if (*db_range <= 0.0f) { - throw std::invalid_argument("db_range must be greater than zero"); - } - - float max_db = CactusThreading::parallel_reduce, float, std::function>( - size, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) { - float local_max = -std::numeric_limits::infinity(); - for (size_t i = start; i < end; i++) { - local_max = std::max(local_max, spectrogram[i]); - } - return local_max; - }, - -std::numeric_limits::infinity(), - [](float a, float b) { return std::max(a, b); } - ); - - float min_db = max_db - *db_range; - CactusThreading::parallel_for(size, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { - for (size_t i = start; i < end; i++) { - spectrogram[i] = std::max(min_db, spectrogram[i]); - } - }); - } -} - -static size_t bit_reverse(size_t x, size_t log2n) { - size_t result = 0; - for (size_t i = 0; i < log2n; i++) { - result = (result << 1) | (x & 1); - x >>= 1; - } - return result; -} - -static bool is_power_of_two(size_t n) { - return n != 0 && (n & (n - 1)) == 0; -} - -static size_t log2_power_of_two(size_t n) { - size_t log2n = 0; - for (size_t temp = n; temp > 1; temp >>= 1) { - log2n++; - } - return log2n; -} - -static void fft_radix2(float* re, float* im, size_t n, bool inverse) { - if (!is_power_of_two(n)) return; - - const size_t log2n = log2_power_of_two(n); - - for (size_t i = 0; i < n; i++) { - size_t j = bit_reverse(i, log2n); - if (i < j) { - std::swap(re[i], re[j]); - std::swap(im[i], im[j]); - } - } - - for (size_t s = 1; s <= log2n; s++) { - size_t m = size_t{1} << s; - size_t m2 = m >> 1; - float w_re = 1.0f; - float w_im = 0.0f; - const float angle = static_cast(M_PI) / static_cast(m2); - const float wm_re = std::cos(angle); - const float wm_im = inverse ? std::sin(angle) : -std::sin(angle); - - for (size_t j = 0; j < m2; j++) { - for (size_t k = j; k < n; k += m) { - size_t k_m2 = k + m2; - float t_re = w_re * re[k_m2] - w_im * im[k_m2]; - float t_im = w_re * im[k_m2] + w_im * re[k_m2]; - float u_re = re[k]; - float u_im = im[k]; - re[k] = u_re + t_re; - im[k] = u_im + t_im; - re[k_m2] = u_re - t_re; - im[k_m2] = u_im - t_im; - } - float new_w_re = w_re * wm_re - w_im * wm_im; - float new_w_im = w_re * wm_im + w_im * wm_re; - w_re = new_w_re; - w_im = new_w_im; - } - } -} - -enum class FFTNorm { - BACKWARD, - FORWARD, - ORTHO -}; - -static FFTNorm parse_fft_norm(const char* norm) { - if (!norm || std::strcmp(norm, "backward") == 0) { - return FFTNorm::BACKWARD; - } - if (std::strcmp(norm, "forward") == 0) { - return FFTNorm::FORWARD; - } - if (std::strcmp(norm, "ortho") == 0) { - return FFTNorm::ORTHO; - } - throw std::invalid_argument("norm must be one of {\"backward\",\"forward\",\"ortho\"}"); -} - -static size_t next_power_of_two(size_t n) { - size_t padded_n = 1; - while (padded_n < n) { - padded_n <<= 1; - } - return padded_n; -} - -static float fft_norm_factor(size_t n, FFTNorm norm_mode, bool inverse) { - if (norm_mode == FFTNorm::ORTHO) { - return 1.0f / std::sqrt(static_cast(n)); - } - if (norm_mode == FFTNorm::FORWARD) { - return inverse ? 1.0f : (1.0f / static_cast(n)); - } - return inverse ? (1.0f / static_cast(n)) : 1.0f; -} - -static void ensure_irfft_scratch(std::vector& re, std::vector& im, size_t n) { - if (re.size() < n) { - re.resize(n); - } - if (im.size() < n) { - im.resize(n); - } -} - -static void idft_r_f32_1d_dft( - const float* input, - float* output, - const size_t n, - const float norm_factor) { - const size_t in_len = n / 2 + 1; - const float two_pi_over_n = (2.0f * static_cast(M_PI)) / static_cast(n); - for (size_t t = 0; t < n; ++t) { - float sum = input[0]; - for (size_t k = 1; k < in_len; ++k) { - const float re = input[k * 2]; - const float im = input[k * 2 + 1]; - const float angle = two_pi_over_n * static_cast(k * t); - const float c = std::cos(angle); - const float s = std::sin(angle); - const bool self_conjugate = (k * 2 == n); - if (self_conjugate) { - sum += re * c; - } else { - sum += 2.0f * (re * c - im * s); - } - } - output[t] = sum * norm_factor; - } -} - -static void rfft_f32_1d(const float* input, float* output, const size_t n, const char* norm) { - const size_t out_len = n / 2 + 1; - const FFTNorm norm_mode = parse_fft_norm(norm); - const float norm_factor = fft_norm_factor(n, norm_mode, false); - - if (n == 1) { - output[0] = input[0] * norm_factor; - output[1] = 0.0f; - return; - } - -#ifdef __APPLE__ - { - size_t log2n = 0; - size_t padded_n = 1; - while (padded_n < n) { - padded_n <<= 1; - log2n++; - } - - FFTSetup fft_setup = vDSP_create_fftsetup(log2n, FFT_RADIX2); - if (fft_setup) { - std::vector buffer(padded_n, 0.0f); - std::copy(input, input + n, buffer.begin()); - - DSPSplitComplex split; - std::vector real_part(padded_n / 2); - std::vector imag_part(padded_n / 2); - split.realp = real_part.data(); - split.imagp = imag_part.data(); - - vDSP_ctoz(reinterpret_cast(buffer.data()), 2, &split, 1, padded_n / 2); - - vDSP_fft_zrip(fft_setup, &split, 1, log2n, FFT_FORWARD); - - float scale = 0.5f * norm_factor; - - output[0] = split.realp[0] * scale * 2.0f; - output[1] = 0.0f; - - if (out_len > 1) { - size_t nyquist_idx = padded_n / 2; - if (nyquist_idx < out_len) { - output[nyquist_idx * 2] = split.imagp[0] * scale * 2.0f; - output[nyquist_idx * 2 + 1] = 0.0f; - } - } - - for (size_t i = 1; i < out_len && i < padded_n / 2; i++) { - output[i * 2] = split.realp[i] * scale * 2.0f; - output[i * 2 + 1] = split.imagp[i] * scale * 2.0f; - } - - vDSP_destroy_fftsetup(fft_setup); - return; - } - } -#endif - - const size_t padded_n = next_power_of_two(n); - - std::vector re(padded_n, 0.0f), im(padded_n, 0.0f); - std::copy(input, input + n, re.begin()); - - fft_radix2(re.data(), im.data(), padded_n, false); - - for (size_t i = 0; i < out_len; i++) { - output[i * 2] = re[i] * norm_factor; - output[i * 2 + 1] = im[i] * norm_factor; - } -} - -static void idft_r_f32_1d(const float* input, float* output, const size_t n, const char* norm) { - const size_t in_len = n / 2 + 1; - const FFTNorm norm_mode = parse_fft_norm(norm); - const float norm_factor = fft_norm_factor(n, norm_mode, true); - - if (n == 1) { - output[0] = input[0] * norm_factor; - return; - } - - if (!is_power_of_two(n)) { - idft_r_f32_1d_dft(input, output, n, norm_factor); - return; - } - - const size_t padded_n = next_power_of_two(n); - const size_t padded_half_plus_one = padded_n / 2 + 1; - thread_local std::vector re_scratch; - thread_local std::vector im_scratch; - ensure_irfft_scratch(re_scratch, im_scratch, padded_n); - std::fill(re_scratch.begin(), re_scratch.begin() + padded_n, 0.0f); - std::fill(im_scratch.begin(), im_scratch.begin() + padded_n, 0.0f); - float* re = re_scratch.data(); - float* im = im_scratch.data(); - for (size_t i = 0; i < in_len; ++i) { - re[i] = input[i * 2]; - im[i] = input[i * 2 + 1]; - } - - im[0] = 0.0f; - const bool has_padded_nyquist_bin = in_len == padded_half_plus_one; - if (has_padded_nyquist_bin) { - im[padded_n / 2] = 0.0f; - } - - const size_t mirror_limit = has_padded_nyquist_bin ? (in_len - 1) : in_len; - for (size_t i = 1; i < mirror_limit; ++i) { - const size_t mirrored = padded_n - i; - re[mirrored] = re[i]; - im[mirrored] = -im[i]; - } - - fft_radix2(re, im, padded_n, true); - - for (size_t i = 0; i < n; ++i) { - output[i] = re[i] * norm_factor; - } -} - -static float hertz_to_mel(float freq, const char* mel_scale) { - if (std::strcmp(mel_scale, "htk") == 0) { - return 2595.0f * std::log10(1.0f + (freq / 700.0f)); - } else if (std::strcmp(mel_scale, "kaldi") == 0) { - return 1127.0f * std::log(1.0f + (freq / 700.0f)); - } - - const float min_log_hertz = 1000.0f; - const float min_log_mel = 15.0f; - const float logstep = 27.0f / std::log(6.4f); - float mels = 3.0f * freq / 200.0f; - - if (freq >= min_log_hertz) { - mels = min_log_mel + std::log(freq / min_log_hertz) * logstep; - } - - return mels; -} - -static float mel_to_hertz(float mels, const char* mel_scale) { - if (std::strcmp(mel_scale, "htk") == 0) { - return 700.0f * (std::pow(10.0f, mels / 2595.0f) - 1.0f); - } else if (std::strcmp(mel_scale, "kaldi") == 0) { - return 700.0f * (std::exp(mels / 1127.0f) - 1.0f); - } - - const float min_log_hertz = 1000.0f; - const float min_log_mel = 15.0f; - const float logstep = std::log(6.4f) / 27.0f; - float freq = 200.0f * mels / 3.0f; - - if (mels >= min_log_mel) { - freq = min_log_hertz * std::exp(logstep * (mels - min_log_mel)); - } - - return freq; -} - -static void generate_mel_filter_bank( - float* mel_filters, - const int num_frequency_bins, - const int num_mel_filters, - const float min_frequency, - const float max_frequency, - const int sampling_rate, - const char* norm, - const char* mel_scale, - const bool triangularize_in_mel_space) -{ - if (norm != nullptr && std::strcmp(norm, "slaney") != 0) { - throw std::invalid_argument("norm must be one of None or \"slaney\""); - } - - if (std::strcmp(mel_scale, "htk") != 0 && std::strcmp(mel_scale, "kaldi") != 0 && std::strcmp(mel_scale, "slaney") != 0) { - throw std::invalid_argument("mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\"."); - } - - if (num_frequency_bins < 2) { - throw std::invalid_argument( - "Require num_frequency_bins: " + std::to_string(num_frequency_bins) + " >= 2"); - } - - if (min_frequency > max_frequency) { - throw std::invalid_argument( - "Require min_frequency: " + std::to_string(min_frequency) + - " <= max_frequency: " + std::to_string(max_frequency)); - } - - const float mel_min = hertz_to_mel(min_frequency, mel_scale); - const float mel_max = hertz_to_mel(max_frequency, mel_scale); - - std::vector mel_freqs(num_mel_filters + 2); - for (int i = 0; i < num_mel_filters + 2; i++) { - mel_freqs[i] = mel_min + (mel_max - mel_min) * i / (num_mel_filters + 1); - } - - std::vector filter_freqs(num_mel_filters + 2); - for (int i = 0; i < num_mel_filters + 2; i++) { - filter_freqs[i] = mel_to_hertz(mel_freqs[i], mel_scale); - } - - std::vector fft_freqs(num_frequency_bins); - if (triangularize_in_mel_space) { - float fft_bin_width = static_cast(sampling_rate) / ((num_frequency_bins - 1) * 2); - for (int i = 0; i < num_frequency_bins; i++) { - fft_freqs[i] = hertz_to_mel(fft_bin_width * i, mel_scale); - } - filter_freqs = mel_freqs; - } else { - for (int i = 0; i < num_frequency_bins; i++) { - fft_freqs[i] = (static_cast(sampling_rate) / 2.0f) * i / (num_frequency_bins - 1); - } - } - - for (int i = 0; i < num_mel_filters; i++) { - float left_edge = filter_freqs[i]; - float center = filter_freqs[i + 1]; - float right_edge = filter_freqs[i + 2]; - - for (int j = 0; j < num_frequency_bins; j++) { - float freq = fft_freqs[j]; - float down_slope = (freq - left_edge) / (center - left_edge); - float up_slope = (right_edge - freq) / (right_edge - center); - - mel_filters[i * num_frequency_bins + j] = std::max(0.0f, std::min(down_slope, up_slope)); - } - } - - if (norm != nullptr && std::strcmp(norm, "slaney") == 0) { - for (int i = 0; i < num_mel_filters; i++) { - float enorm = 2.0f / (filter_freqs[i + 2] - filter_freqs[i]); - for (int j = 0; j < num_frequency_bins; j++) { - mel_filters[i * num_frequency_bins + j] *= enorm; - } - } - } -} - -static void compute_spectrogram_f32( - const float* waveform, - size_t waveform_length, - const float* window, - size_t window_length, - size_t frame_length, - size_t hop_length, - const size_t* fft_length, - float* spectrogram, - float power, - bool center, - const char* pad_mode, - bool onesided [[maybe_unused]], - float dither, - const float* preemphasis, - const float* mel_filters, - size_t mel_filters_size, - float mel_floor, - const char* log_mel, - float reference, - float min_value, - const float* db_range, - bool remove_dc_offset) -{ - size_t actual_fft_length; - if (fft_length == nullptr) { - actual_fft_length = frame_length; - } else { - actual_fft_length = *fft_length; - } - - if (frame_length > actual_fft_length) { - throw std::invalid_argument( - "frame_length (" + std::to_string(frame_length) + - ") may not be larger than fft_length (" + - std::to_string(actual_fft_length) + ")"); - } - - std::vector hann_window; - const float* actual_window = window; - - if (window == nullptr) { - size_t length = frame_length + 1; - hann_window.resize(frame_length); - for (size_t i = 0; i < frame_length; i++) { - hann_window[i] = 0.5f * (1.0f - std::cos(2.0f * static_cast(M_PI) * i / (length - 1))); - } - actual_window = hann_window.data(); - } else if (window_length != frame_length) { - throw std::invalid_argument( - "Length of the window (" + std::to_string(window_length) + - ") must equal frame_length (" + std::to_string(frame_length) + ")"); - } - - if (hop_length <= 0) { - throw std::invalid_argument("hop_length must be greater than zero"); - } - - if (power == 0.0f && mel_filters != nullptr) { - throw std::invalid_argument( - "You have provided `mel_filters` but `power` is `None`. " - "Mel spectrogram computation is not yet supported for complex-valued spectrogram. " - "Specify `power` to fix this issue."); - } - - std::vector padded_waveform; - const float* input_waveform = waveform; - size_t input_length = waveform_length; - - if (center) { - size_t pad_length = frame_length / 2; - size_t padded_length = waveform_length + 2 * pad_length; - padded_waveform.assign(padded_length, 0.0f); - - if (std::strcmp(pad_mode, "reflect") == 0) { - for (size_t i = 0; i < pad_length; i++) { - padded_waveform[i] = waveform[pad_length - i]; - } - - std::copy(waveform, waveform + waveform_length, padded_waveform.data() + pad_length); - - for (size_t i = 0; i < pad_length; i++) { - padded_waveform[pad_length + waveform_length + i] = waveform[waveform_length - 2 - i]; - } - } else if (std::strcmp(pad_mode, "constant") == 0) { - std::copy(waveform, waveform + waveform_length, padded_waveform.data() + pad_length); - } else { - throw std::invalid_argument("Unsupported pad_mode: " + std::string(pad_mode)); - } - - input_waveform = padded_waveform.data(); - input_length = padded_length; - } - - const size_t num_frames = 1 + (input_length - frame_length) / hop_length; - const size_t num_frequency_bins = (actual_fft_length / 2) + 1; - - std::vector buffer(actual_fft_length); - std::vector raw_complex_frequencies(num_frequency_bins * 2); - - const size_t num_mel_bins = mel_filters != nullptr ? mel_filters_size / num_frequency_bins : 0; - const size_t spectrogram_bins = mel_filters != nullptr ? num_mel_bins : num_frequency_bins; - - std::vector temp_spectrogram(num_frames * num_frequency_bins); - - CactusThreading::parallel_for(num_frames, CactusThreading::Thresholds::SCALAR_EXPENSIVE, [&](size_t start_frame, size_t end_frame) { - std::vector local_buffer(actual_fft_length); - std::vector local_complex_frequencies(num_frequency_bins * 2); - - for (size_t frame_idx = start_frame; frame_idx < end_frame; frame_idx++) { - size_t timestep = frame_idx * hop_length; - std::fill(local_buffer.begin(), local_buffer.end(), 0.0f); - - size_t available_length = std::min(frame_length, input_length - timestep); - std::copy(input_waveform + timestep, input_waveform + timestep + available_length, local_buffer.data()); - - if (dither != 0.0f) { - thread_local std::mt19937 rng(std::random_device{}()); - std::normal_distribution dist(0.0f, 1.0f); - for (size_t i = 0; i < frame_length; i++) { - local_buffer[i] += dither * dist(rng); - } - } - - if (remove_dc_offset) { - float mean = 0.0f; - for (size_t i = 0; i < frame_length; i++) { - mean += local_buffer[i]; - } - mean /= static_cast(frame_length); - - for (size_t i = 0; i < frame_length; i++) { - local_buffer[i] -= mean; - } - } - - if (preemphasis != nullptr) { - float preemph_coef = *preemphasis; - for (size_t i = frame_length - 1; i > 0; i--) { - local_buffer[i] -= preemph_coef * local_buffer[i - 1]; - } - local_buffer[0] *= (1.0f - preemph_coef); - } - - for (size_t i = 0; i < frame_length; i++) { - local_buffer[i] *= actual_window[i]; - } - - rfft_f32_1d(local_buffer.data(), local_complex_frequencies.data(), actual_fft_length, "backward"); - - for (size_t i = 0; i < num_frequency_bins; i++) { - float real = local_complex_frequencies[i * 2]; - float imag = local_complex_frequencies[i * 2 + 1]; - float magnitude = std::hypot(real, imag); - temp_spectrogram[frame_idx * num_frequency_bins + i] = std::pow(magnitude, power); - } - } - }); - - if (mel_filters != nullptr) { - CactusThreading::parallel_for_2d(num_mel_bins, num_frames, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t m, size_t t) { - float sum = 0.0f; - for (size_t f = 0; f < num_frequency_bins; f++) { - sum += mel_filters[m * num_frequency_bins + f] * temp_spectrogram[t * num_frequency_bins + f]; - } - spectrogram[m * num_frames + t] = std::max(mel_floor, sum); - }); - } else { - CactusThreading::parallel_for_2d(num_frames, num_frequency_bins, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t t, size_t f) { - spectrogram[f * num_frames + t] = temp_spectrogram[t * num_frequency_bins + f]; - }); - } - - if (power != 0.0f && log_mel != nullptr) { - const size_t total_elements = spectrogram_bins * num_frames; - - if (std::strcmp(log_mel, "log") == 0) { - CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { - for (size_t i = start; i < end; i++) { - spectrogram[i] = std::log(spectrogram[i]); - } - }); - } else if (std::strcmp(log_mel, "log10") == 0) { - CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { - for (size_t i = start; i < end; i++) { - spectrogram[i] = std::log10(spectrogram[i]); - } - }); - } else if (std::strcmp(log_mel, "dB") == 0) { - if (power == 1.0f) { - to_db(spectrogram, total_elements, reference, min_value, db_range, 20.0f); - } else if (power == 2.0f) { - to_db(spectrogram, total_elements, reference, min_value, db_range, 10.0f); - } else { - throw std::invalid_argument( - "Cannot use log_mel option 'dB' with power " + std::to_string(power)); - } - } else { - throw std::invalid_argument("Unknown log_mel option: " + std::string(log_mel)); - } - } -} - -AudioProcessor::AudioProcessor() - : num_frequency_bins_(0), num_mel_filters_(0) {} - -AudioProcessor::~AudioProcessor() = default; - -void AudioProcessor::init_mel_filters(size_t num_frequency_bins, - size_t num_mel_filters, - float min_freq, - float max_freq, - size_t sampling_rate, - const char* norm, - const char* mel_scale) { - num_frequency_bins_ = num_frequency_bins; - num_mel_filters_ = num_mel_filters; - mel_filters_.resize(num_mel_filters * num_frequency_bins); - - generate_mel_filter_bank( - mel_filters_.data(), - num_frequency_bins, - num_mel_filters, - min_freq, - max_freq, - sampling_rate, - norm, - mel_scale, - false - ); -} - -std::vector AudioProcessor::compute_spectrogram( - const std::vector& waveform, - const SpectrogramConfig& config) { - - if (mel_filters_.empty()) { - throw std::runtime_error("Mel filters not initialized. Call init_mel_filters() first."); - } - - const size_t n_samples = waveform.size(); - const size_t fft_size = config.fft_override > 0 ? config.fft_override : config.n_fft; - const size_t analysis_frame_length = config.n_fft; - const size_t window_length = std::min(config.frame_length, analysis_frame_length); - const size_t pad_length = config.center ? analysis_frame_length / 2 : 0; - const size_t padded_length = n_samples + 2 * pad_length; - if (analysis_frame_length == 0 || padded_length < analysis_frame_length || config.hop_length == 0) { - return {}; - } - const size_t num_frames = 1 + (padded_length - analysis_frame_length) / config.hop_length; - - std::vector output(num_mel_filters_ * num_frames); - std::vector window(analysis_frame_length, 0.0f); - if (window_length == 1) { - window[analysis_frame_length / 2] = 1.0f; - } else if (window_length > 1) { - const float denom = config.hann_periodic - ? static_cast(window_length) - : static_cast(window_length - 1); - const float a0 = config.window_a0; - const size_t left_pad = (analysis_frame_length - window_length) / 2; - for (size_t i = 0; i < window_length; ++i) { - const float w = a0 - (1.0f - a0) * std::cos(2.0f * static_cast(M_PI) * static_cast(i) / denom); - window[left_pad + i] = w; - } - } - - float effective_floor = config.mel_floor_additive ? 0.0f : config.mel_floor; - const char* effective_log = config.mel_floor_additive ? nullptr : config.log_mel; - - compute_spectrogram_f32( - waveform.data(), - waveform.size(), - window.data(), - window.size(), - analysis_frame_length, - config.hop_length, - &fft_size, - output.data(), - config.power, - config.center, - config.pad_mode, - config.onesided, - config.dither, - config.preemphasis != 0.0f ? &config.preemphasis : nullptr, - mel_filters_.data(), - mel_filters_.size(), - effective_floor, - effective_log, - config.reference, - config.min_value, - nullptr, - config.remove_dc_offset - ); - - if (config.mel_floor_additive) { - for (size_t i = 0; i < output.size(); i++) { -#ifdef __APPLE__ - // vDSP FFT returns 2x standard DFT magnitudes, so raw mel - // values are 2x too large. Halve before adding floor and log. - output[i] = std::log(output[i] * 0.5f + config.mel_floor); -#else - output[i] = std::log(output[i] + config.mel_floor); -#endif - } - } - - return output; -} - -std::vector AudioProcessor::compute_irfft( - const std::vector& complex_input, - size_t n, - const char* norm) { - if (n == 0) { - throw std::invalid_argument("compute_irfft: n must be greater than 0"); - } - const size_t expected_in_len = (n / 2 + 1) * 2; - if (complex_input.size() != expected_in_len) { - throw std::invalid_argument("compute_irfft: complex_input must contain exactly (n/2+1)*2 values"); - } - std::vector output(n); - idft_r_f32_1d(complex_input.data(), output.data(), n, norm); - return output; -} - -} -} diff --git a/cactus-transpile/engine/engine_sp.cpp b/cactus-transpile/engine/engine_sp.cpp deleted file mode 100644 index 41cd506c6..000000000 --- a/cactus-transpile/engine/engine_sp.cpp +++ /dev/null @@ -1,485 +0,0 @@ -#include "engine.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -SPTokenizer::SPTokenizer() - : trie_root_(std::make_unique()), - vocab_size_(0), - unk_token_id_(3), - bos_token_id_(2), - eos_token_id_(1), - pad_token_id_(0), - vocab_mmap_ptr_(nullptr), - vocab_mmap_size_(0) { - has_chat_template_ = false; -} - -SPTokenizer::~SPTokenizer() { - cleanup_mmap(); -} - -void SPTokenizer::cleanup_mmap() { - if (vocab_mmap_ptr_ && vocab_mmap_ptr_ != MAP_FAILED) { - munmap(vocab_mmap_ptr_, vocab_mmap_size_); - vocab_mmap_ptr_ = nullptr; - } -} - -bool SPTokenizer::load_vocabulary_with_config(const std::string& vocab_file, const std::string& /*merges_file*/, const std::string& config_file) { - runtime_config_ = load_tokenizer_runtime_config(config_file); - std::string config_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/config.txt"; - detect_model_type(config_path); - - std::ifstream vocab_stream(vocab_file); - if (!vocab_stream.is_open()) return false; - - token_to_id_.clear(); - id_to_token_.clear(); - token_scores_.clear(); - - std::string first_line; - std::getline(vocab_stream, first_line); - vocab_stream.seekg(0); - - bool is_id_token_format = false; - if (!first_line.empty()) { - is_id_token_format = (std::isdigit(first_line[0]) && - first_line.find('\t') != std::string::npos); - } - - if (is_id_token_format) { - std::string line = ""; - while (std::getline(vocab_stream, line)) { - std::string token = ""; - uint32_t id = UINT32_MAX; - - std::istringstream iss(line); - if (iss >> id) { - if (std::getline(iss, token)) { - if (!token.empty() && token[0] == '\t') { - token = token.substr(1); - } - } - - if (token.empty()) { - auto last_pos = vocab_stream.tellg(); - while (std::getline(vocab_stream, line)) { - if (!line.empty()) { - break; - } - token += '\n'; - last_pos = vocab_stream.tellg(); - } - vocab_stream.seekg(last_pos); - } - } - - if (!token.empty() && id != UINT32_MAX) { - float score = -static_cast(id); - size_t tab_in_token = token.find('\t'); - if (tab_in_token != std::string::npos) { - std::string score_str = token.substr(tab_in_token + 1); - if (!score_str.empty()) score = std::stof(score_str); - token = token.substr(0, tab_in_token); - } - token_to_id_[token] = id; - if (id >= id_to_token_.size()) { - id_to_token_.resize(id + 1); - token_scores_.resize(id + 1, 0.0f); - } - id_to_token_[id] = token; - token_scores_[id] = score; - } - } - vocab_size_ = id_to_token_.size(); - } else { - std::string line; - uint32_t id = 0; - - vocab_stream.seekg(0); - while (std::getline(vocab_stream, line)) { - token_to_id_[line] = id; - id_to_token_.push_back(line); - token_scores_.push_back(0.0f); - id++; - } - vocab_size_ = id; - } - - vocab_stream.close(); - - build_trie(); - - std::ifstream config_stream(config_file); - if (config_stream.is_open()) { - std::string config_line; - while (std::getline(config_stream, config_line)) { - if (config_line.empty() || config_line[0] == '#') continue; - - size_t eq_pos = config_line.find('='); - if (eq_pos == std::string::npos) continue; - - std::string key = config_line.substr(0, eq_pos); - std::string value = config_line.substr(eq_pos + 1); - - key.erase(0, key.find_first_not_of(" \t")); - key.erase(key.find_last_not_of(" \t") + 1); - value.erase(0, value.find_first_not_of(" \t")); - value.erase(value.find_last_not_of(" \t") + 1); - - if (key == "eos_token_id") { - eos_token_id_ = std::stoul(value); - } else if (key == "pad_token_id") { - pad_token_id_ = std::stoul(value); - } else if (key == "unk_token_id") { - unk_token_id_ = std::stoul(value); - } else if (key == "bos_token_id") { - bos_token_id_ = std::stoul(value); - } else if (key == "sp_model_type") { - sp_bpe_mode_ = (value == "bpe" || value == "BPE"); - } else if (key == "sp_add_dummy_prefix") { - sp_add_dummy_prefix_ = (value == "true" || value == "1"); - } else if (key == "sp_byte_fallback") { - sp_byte_fallback_ = (value == "true" || value == "1"); - } - } - } - - std::string special_tokens_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/special_tokens.json"; - load_special_tokens(special_tokens_path); - - std::string template_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/chat_template.jinja2"; - load_chat_template(template_path); - - return true; -} - -void SPTokenizer::build_trie() { - for (uint32_t id = 0; id < id_to_token_.size(); ++id) { - const std::string& token = id_to_token_[id]; - if (token.empty()) continue; - - std::u32string u32_token; - size_t pos = 0; - while (pos < token.length()) { - char32_t codepoint = 0; - unsigned char byte = token[pos]; - - if (byte < 0x80) { - codepoint = byte; - pos++; - } else if ((byte & 0xE0) == 0xC0) { - if (pos + 1 < token.length()) { - codepoint = ((byte & 0x1F) << 6) | (token[pos + 1] & 0x3F); - pos += 2; - } else break; - } else if ((byte & 0xF0) == 0xE0) { - if (pos + 2 < token.length()) { - codepoint = ((byte & 0x0F) << 12) | - ((token[pos + 1] & 0x3F) << 6) | - (token[pos + 2] & 0x3F); - pos += 3; - } else break; - } else if ((byte & 0xF8) == 0xF0) { - if (pos + 3 < token.length()) { - codepoint = ((byte & 0x07) << 18) | - ((token[pos + 1] & 0x3F) << 12) | - ((token[pos + 2] & 0x3F) << 6) | - (token[pos + 3] & 0x3F); - pos += 4; - } else break; - } else { - pos++; - continue; - } - - u32_token.push_back(codepoint); - } - - if (u32_token.empty()) continue; - - TrieNode* current = trie_root_.get(); - for (char32_t ch : u32_token) { - if (current->children.find(ch) == current->children.end()) { - current->children[ch] = std::make_unique(); - } - current = current->children[ch].get(); - } - current->token_id = static_cast(id); - current->score = token_scores_[id]; - } -} - -std::string SPTokenizer::preprocess_text(const std::string& text) const { - if (text.empty()) return text; - - if (sp_bpe_mode_) { - std::string processed; - if (sp_add_dummy_prefix_) processed += "\xE2\x96\x81"; - for (char c : text) { - if (c == ' ') processed += "\xE2\x96\x81"; - else processed += c; - } - return processed; - } - - std::string processed = ""; - if (model_type_ == ModelType::BERT) { - processed = "▁"; - } - - for (size_t i = text.find_first_not_of(" "); i < text.length(); i++) { - char c = text[i]; - if (c == ' ') { - processed += "▁"; - } else { - processed += c; - } - } - - return processed; -} - -std::string SPTokenizer::postprocess_text(const std::string& text) const { - std::string result; - size_t i = 0; - while (i < text.length()) { - if (i + 2 < text.length() && - static_cast(text[i]) == 0xE2 && - static_cast(text[i+1]) == 0x96 && - static_cast(text[i+2]) == 0x81) { - result += ' '; - i += 3; - } else { - result += text[i]; - i++; - } - } - if (!result.empty() && result[0] == ' ') { - result = result.substr(1); - } - return result; -} - -std::vector> SPTokenizer::tokenize_with_trie(const std::string& text) const { - std::vector> result; - - std::u32string u32_text; - size_t pos = 0; - while (pos < text.length()) { - char32_t codepoint = 0; - unsigned char byte = text[pos]; - - if (byte < 0x80) { - codepoint = byte; - pos++; - } else if ((byte & 0xE0) == 0xC0) { - if (pos + 1 < text.length()) { - codepoint = ((byte & 0x1F) << 6) | (text[pos + 1] & 0x3F); - pos += 2; - } else break; - } else if ((byte & 0xF0) == 0xE0) { - if (pos + 2 < text.length()) { - codepoint = ((byte & 0x0F) << 12) | - ((text[pos + 1] & 0x3F) << 6) | - (text[pos + 2] & 0x3F); - pos += 3; - } else break; - } else if ((byte & 0xF8) == 0xF0) { - if (pos + 3 < text.length()) { - codepoint = ((byte & 0x07) << 18) | - ((text[pos + 1] & 0x3F) << 12) | - ((text[pos + 2] & 0x3F) << 6) | - (text[pos + 3] & 0x3F); - pos += 4; - } else break; - } else { - pos++; - continue; - } - - u32_text.push_back(codepoint); - } - - if (u32_text.empty()) { - result.push_back({text, unk_token_id_}); - return result; - } - - pos = 0; - while (pos < u32_text.length()) { - TrieNode* current = trie_root_.get(); - size_t best_match_len = 0; - int32_t best_token_id = -1; - - for (size_t len = 0; pos + len < u32_text.length(); ++len) { - char32_t ch = u32_text[pos + len]; - if (current->children.find(ch) == current->children.end()) { - break; - } - current = current->children[ch].get(); - if (current->token_id >= 0) { - best_match_len = len + 1; - best_token_id = current->token_id; - } - } - - if (best_match_len > 0) { - std::u32string u32_token = u32_text.substr(pos, best_match_len); - - std::string token; - for (char32_t cp : u32_token) { - if (cp < 0x80) { - token.push_back(static_cast(cp)); - } else if (cp < 0x800) { - token.push_back(static_cast(0xC0 | (cp >> 6))); - token.push_back(static_cast(0x80 | (cp & 0x3F))); - } else if (cp < 0x10000) { - token.push_back(static_cast(0xE0 | (cp >> 12))); - token.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); - token.push_back(static_cast(0x80 | (cp & 0x3F))); - } else { - token.push_back(static_cast(0xF0 | (cp >> 18))); - token.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); - token.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); - token.push_back(static_cast(0x80 | (cp & 0x3F))); - } - } - result.push_back({token, static_cast(best_token_id)}); - pos += best_match_len; - } else { - char32_t cp = u32_text[pos]; - std::string char_str; - if (cp < 0x80) { - char_str.push_back(static_cast(cp)); - } else if (cp < 0x800) { - char_str.push_back(static_cast(0xC0 | (cp >> 6))); - char_str.push_back(static_cast(0x80 | (cp & 0x3F))); - } else if (cp < 0x10000) { - char_str.push_back(static_cast(0xE0 | (cp >> 12))); - char_str.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); - char_str.push_back(static_cast(0x80 | (cp & 0x3F))); - } else { - char_str.push_back(static_cast(0xF0 | (cp >> 18))); - char_str.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); - char_str.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); - char_str.push_back(static_cast(0x80 | (cp & 0x3F))); - } - result.push_back({char_str, unk_token_id_}); - pos++; - } - } - - return result; -} - -std::vector SPTokenizer::tokenize_with_bpe(const std::string& text) const { - std::vector symbols; - for (size_t i = 0; i < text.size(); ) { - size_t char_len = 1; - unsigned char byte = static_cast(text[i]); - if ((byte & 0xE0) == 0xC0) char_len = 2; - else if ((byte & 0xF0) == 0xE0) char_len = 3; - else if ((byte & 0xF8) == 0xF0) char_len = 4; - if (i + char_len <= text.size()) symbols.push_back(text.substr(i, char_len)); - i += char_len; - } - - while (symbols.size() > 1) { - int best_index = -1; - float best_score = -std::numeric_limits::infinity(); - for (size_t i = 0; i + 1 < symbols.size(); ++i) { - auto it = token_to_id_.find(symbols[i] + symbols[i + 1]); - if (it == token_to_id_.end()) continue; - float score = it->second < token_scores_.size() - ? token_scores_[it->second] - : -static_cast(it->second); - if (score > best_score) { best_score = score; best_index = static_cast(i); } - } - if (best_index < 0) break; - symbols[best_index] += symbols[best_index + 1]; - symbols.erase(symbols.begin() + best_index + 1); - } - - std::vector result; - for (const auto& symbol : symbols) { - auto it = token_to_id_.find(symbol); - if (it != token_to_id_.end()) { result.push_back(it->second); continue; } - if (sp_byte_fallback_) { - for (unsigned char b : symbol) { - char buf[7]; std::snprintf(buf, sizeof(buf), "<0x%02X>", b); - auto byte_it = token_to_id_.find(buf); - result.push_back(byte_it != token_to_id_.end() ? byte_it->second : unk_token_id_); - } - continue; - } - result.push_back(unk_token_id_); - } - return result; -} - -std::vector SPTokenizer::split_with_special_tokens(const std::string& text) const { - return cactus::engine::split_with_special_tokens(text, special_tokens_); -} - -std::vector SPTokenizer::encode(const std::string& text) const { - if (text.empty()) return {}; - - auto text_segments = split_with_special_tokens(text); - std::vector token_ids; - - for (const auto& segment : text_segments) { - auto special_it = special_tokens_.find(segment); - if (special_it != special_tokens_.end()) { - token_ids.push_back(special_it->second); - } else { - std::string processed = preprocess_text(segment); - if (processed.empty()) continue; - - if (sp_bpe_mode_) { - auto ids = tokenize_with_bpe(processed); - token_ids.insert(token_ids.end(), ids.begin(), ids.end()); - } else { - auto token_pairs = tokenize_with_trie(processed); - for (const auto& [token, id] : token_pairs) { - token_ids.push_back(id); - } - } - } - } - - return token_ids; -} - -std::string SPTokenizer::decode(const std::vector& tokens) const { - std::string raw; - for (uint32_t token_id : tokens) { - if (token_id >= id_to_token_.size()) continue; - const std::string& piece = id_to_token_[token_id]; - unsigned int byte_val; - if (piece.size() == 6 && std::sscanf(piece.c_str(), "<0x%02X>", &byte_val) == 1) { - raw.push_back(static_cast(byte_val)); - } else { - raw += piece; - } - } - return postprocess_text(raw); -} - -void SPTokenizer::load_special_tokens(const std::string& config_file) { - load_special_tokens_map(config_file, special_tokens_); -} - -} // namespace engine -} // namespace cactus diff --git a/cactus-transpile/engine/engine_tokenizer.cpp b/cactus-transpile/engine/engine_tokenizer.cpp deleted file mode 100644 index 566fea2f8..000000000 --- a/cactus-transpile/engine/engine_tokenizer.cpp +++ /dev/null @@ -1,866 +0,0 @@ -#include "engine.h" -#include -#include -#include -#include -#include - -extern "C" { - #include "../../libs/stb/stb_image.h" -} - -namespace cactus { -namespace engine { - -namespace { - -std::string trim_copy(const std::string& value) { - size_t start = value.find_first_not_of(" \t\r\n"); - if (start == std::string::npos) return ""; - size_t end = value.find_last_not_of(" \t\r\n"); - return value.substr(start, end - start + 1); -} - -TokenizerRuntimeConfig::TokenizerType parse_tokenizer_type(const std::string& value) { - if (value == "bpe") return TokenizerRuntimeConfig::TokenizerType::BPE; - if (value == "sentencepiece") return TokenizerRuntimeConfig::TokenizerType::SENTENCEPIECE; - return TokenizerRuntimeConfig::TokenizerType::UNKNOWN; -} - -TokenizerRuntimeConfig::VocabFormat parse_vocab_format(const std::string& value) { - if (value == "id_tab_token") return TokenizerRuntimeConfig::VocabFormat::ID_TAB_TOKEN; - if (value == "line_token") return TokenizerRuntimeConfig::VocabFormat::LINE_TOKEN; - return TokenizerRuntimeConfig::VocabFormat::UNKNOWN; -} - -TokenizerRuntimeConfig::Normalizer parse_normalizer(const std::string& value) { - if (value == "metaspace") return TokenizerRuntimeConfig::Normalizer::METASPACE; - if (value == "byte_level") return TokenizerRuntimeConfig::Normalizer::BYTE_LEVEL; - return TokenizerRuntimeConfig::Normalizer::NONE; -} - -TokenizerRuntimeConfig::Decoder parse_decoder(const std::string& value) { - if (value == "replace_metaspace") return TokenizerRuntimeConfig::Decoder::REPLACE_METASPACE; - if (value == "byte_level") return TokenizerRuntimeConfig::Decoder::BYTE_LEVEL; - return TokenizerRuntimeConfig::Decoder::NONE; -} - -} // namespace - -TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file) { - TokenizerRuntimeConfig config; - - std::ifstream file(config_file); - if (!file.is_open()) { - return config; - } - - std::string line; - while (std::getline(file, line)) { - if (line.empty() || line[0] == '#') continue; - - size_t eq_pos = line.find('='); - if (eq_pos == std::string::npos) continue; - - const std::string key = trim_copy(line.substr(0, eq_pos)); - const std::string value = trim_copy(line.substr(eq_pos + 1)); - - if (key == "tokenizer_type") { - config.tokenizer_type = parse_tokenizer_type(value); - } else if (key == "vocab_format") { - config.vocab_format = parse_vocab_format(value); - } else if (key == "normalizer") { - config.normalizer = parse_normalizer(value); - } else if (key == "decoder") { - config.decoder = parse_decoder(value); - } else if (key == "byte_fallback") { - config.byte_fallback = (value == "true" || value == "1"); - } else if (key == "has_chat_template") { - config.has_chat_template = (value == "true" || value == "1"); - } - } - - return config; -} - -void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens) { - special_tokens.clear(); - - std::ifstream file(config_file); - if (!file.is_open()) { - return; - } - - std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - - size_t pos = content.find("\"special_tokens\""); - if (pos == std::string::npos) return; - - pos = content.find("{", pos); - if (pos == std::string::npos) return; - - size_t end_pos = content.find("}", pos); - if (end_pos == std::string::npos) return; - - std::string special_tokens_section = content.substr(pos + 1, end_pos - pos - 1); - std::istringstream iss(special_tokens_section); - std::string line; - - while (std::getline(iss, line)) { - size_t colon_pos = line.find(":"); - if (colon_pos == std::string::npos) continue; - - std::string id_part = line.substr(0, colon_pos); - std::string token_part = line.substr(colon_pos + 1); - - size_t id_start = id_part.find("\""); - size_t id_end = id_part.find("\"", id_start + 1); - if (id_start == std::string::npos || id_end == std::string::npos) continue; - - uint32_t token_id = static_cast(std::stoul(id_part.substr(id_start + 1, id_end - id_start - 1))); - - size_t token_start = token_part.find("\""); - size_t token_end = token_part.rfind("\""); - if (token_start == std::string::npos || token_end == std::string::npos || token_start >= token_end) continue; - - std::string token_content = token_part.substr(token_start + 1, token_end - token_start - 1); - special_tokens[token_content] = token_id; - } -} - -std::vector split_with_special_tokens(const std::string& text, - const std::unordered_map& special_tokens) { - std::vector result; - size_t start = 0; - while (start < text.size()) { - size_t best_match_pos = text.size(); - size_t best_match_len = 0; - std::string best_special_token; - - for (const auto& [special_token, token_id] : special_tokens) { - size_t pos = text.find(special_token, start); - if (pos != std::string::npos && - (pos < best_match_pos || (pos == best_match_pos && special_token.length() > best_match_len))) { - best_match_pos = pos; - best_match_len = special_token.length(); - best_special_token = special_token; - } - } - - if (best_match_pos < text.size()) { - if (best_match_pos > start) - result.push_back(text.substr(start, best_match_pos - start)); - result.push_back(best_special_token); - start = best_match_pos + best_match_len; - } else { - if (start < text.size()) - result.push_back(text.substr(start)); - break; - } - } - return result; -} - -void Tokenizer::load_chat_template(const std::string& template_file) { - std::ifstream file(template_file); - if (!file.is_open()) { - has_chat_template_ = false; - return; - } - chat_template_ = std::string((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - has_chat_template_ = !chat_template_.empty(); -} - -void Tokenizer::detect_model_type(const std::string& config_path) { - std::ifstream file(config_path); - if (!file.is_open()) { - model_type_ = ModelType::UNKNOWN; - return; - } - - std::string line; - while (std::getline(file, line)) { - size_t pos = line.find("model_type"); - if (pos != std::string::npos) { - std::transform(line.begin(), line.end(), line.begin(), ::tolower); - - if (line.find("qwen3_5") != std::string::npos) { - model_type_ = ModelType::QWEN3P5; - break; - } else if (line.find("qwen") != std::string::npos) { - model_type_ = ModelType::QWEN; - break; - } else if (line.find("gemma4") != std::string::npos || line.find("tinyllama") != std::string::npos) { - model_type_ = ModelType::GEMMA4; - break; - } else if (line.find("gemma") != std::string::npos) { - model_type_ = ModelType::GEMMA; - break; - } else if(line.find("lfm2") != std::string::npos) { - model_type_ = ModelType::LFM2; - } else if (line.find("bert") != std::string::npos) { - model_type_ = ModelType::BERT; - break; - } else if (line.find("whisper") != std::string::npos) { - model_type_ = ModelType::WHISPER; - break; - } else if (line.find("parakeet") != std::string::npos) { - model_type_ = ModelType::PARAKEET; - break; - } else if (line.find("needle") != std::string::npos) { - model_type_ = ModelType::NEEDLE; - break; - } else if (line.find("youtu") != std::string::npos) { - model_type_ = ModelType::YOUTU; - break; - } else { - model_type_ = ModelType::UNKNOWN; - } - } - } - file.clear(); - file.seekg(0); - - while (std::getline(file, line)) { - size_t pos2 = line.find("model_variant"); - if (pos2 != std::string::npos) { - std::transform(line.begin(), line.end(), line.begin(), ::tolower); - - if (line.find("vlm") != std::string::npos) { - model_variant_ = ModelVariant::VLM; - break; - } else if (line.find("extract") != std::string::npos) { - model_variant_ = ModelVariant::EXTRACT; - break; - } else if (line.find("rag") != std::string::npos) { - model_variant_ = ModelVariant::RAG; - break; - } else { - model_variant_ = ModelVariant::DEFAULT; - } - } - } - - file.clear(); - file.seekg(0); - while (std::getline(file, line)) { - auto parse_uint = [&](const std::string& key, uint32_t& out) { - size_t p = line.find(key + "="); - if (p != std::string::npos) { - out = static_cast(std::stoul(line.substr(p + key.size() + 1))); - } - }; - parse_uint("vision_patch_size", vision_patch_size_); - parse_uint("vision_pooling_kernel_size", vision_pooling_kernel_size_); - parse_uint("vision_default_output_length", vision_default_output_length_); - parse_uint("vision_image_size", vision_image_size_); - } - - file.close(); -} - -std::string Tokenizer::get_default_stop_sequence() const { - switch (model_type_) { - case ModelType::GEMMA: - return ""; - case ModelType::GEMMA4: - return ""; - case ModelType::QWEN: - case ModelType::QWEN3P5: - case ModelType::LFM2: - return "<|im_end|>"; - case ModelType::NEEDLE: - return ""; - default: - return "<|im_end|>"; - } -} - -std::vector Tokenizer::apply_chat_template(const std::vector& messages, bool add_generation_prompt) const { - std::string formatted_prompt = format_chat_prompt(messages, add_generation_prompt); - return encode(formatted_prompt); -} - -std::string Tokenizer::format_chat_prompt(const std::vector& messages, bool add_generation_prompt, - const std::string& tools_json, bool enable_thinking_if_supported) const { - bool has_images = false; - for (const auto& msg : messages) { - if (!msg.images.empty()) { - has_images = true; - break; - } - } - if (model_type_ == ModelType::LFM2 && has_images) { - return format_lfm2_vl_style(messages, add_generation_prompt, tools_json); - } - - switch (model_type_) { - case ModelType::QWEN: - case ModelType::QWEN3P5: - return format_qwen_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); - case ModelType::GEMMA: - return format_gemma_style(messages, add_generation_prompt, tools_json); - case ModelType::GEMMA4: - return format_gemma4_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); - case ModelType::LFM2: - return format_lfm2_style(messages, add_generation_prompt, tools_json); - case ModelType::NEEDLE: - return format_needle_style(messages, add_generation_prompt, tools_json); - case ModelType::YOUTU: - return format_youtu_style(messages, add_generation_prompt, tools_json); - default: - return format_qwen_style(messages, add_generation_prompt, tools_json); - } -} - -std::string Tokenizer::format_needle_style(const std::vector& messages, - bool /*add_generation_prompt*/, - const std::string& tools_json) const { - std::string serialized_tools = tools_json.empty() ? "[]" : tools_json; - return format_needle_query_text(messages) + "" + serialized_tools + ""; -} - -std::string Tokenizer::format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported) const { - std::string result; - size_t start_idx = 0; - - if (!tools_json.empty()) { - result += "<|im_start|>system\n"; - if (!messages.empty() && messages[0].role == "system") { - result += messages[0].content + "\n\n"; - start_idx = 1; - } - result += "# Tools\n\n"; - result += "You may call one or more functions to assist with the user query.\n\n"; - result += "You are provided with function signatures within XML tags:\n"; - result += ""; - result += tools_json; - result += "\n\n\n"; - result += "For each function call, return a json object with function name and arguments within XML tags:\n"; - result += "\n"; - result += "{\"name\": , \"arguments\": }\n"; - result += ""; - result += "<|im_end|>\n"; - } else { - if (!messages.empty() && messages[0].role == "system") { - result += "<|im_start|>system\n" + messages[0].content + "<|im_end|>\n"; - start_idx = 1; - } - } - - size_t last_query_index = messages.empty() ? 0 : messages.size() - 1; - for (size_t idx = messages.size(); idx > 0; idx--) { - const auto& m = messages[idx - 1]; - if (m.role == "user") { - bool is_tool_response = m.content.size() >= 31 && - m.content.substr(0, 15) == "" && - m.content.substr(m.content.size() - 16) == ""; - if (!is_tool_response) { - last_query_index = idx - 1; - break; - } - } - } - - for (size_t i = start_idx; i < messages.size(); i++) { - const auto& msg = messages[i]; - if (msg.role == "user" || (msg.role == "system" && i > 0)) { - result += "<|im_start|>" + msg.role + "\n" + msg.content + "<|im_end|>\n"; - } else if (msg.role == "assistant") { - if (i > last_query_index && i == messages.size() - 1) { - result += "<|im_start|>assistant\n\n\n\n\n"; - std::string content = msg.content; - size_t lstrip = content.find_first_not_of('\n'); - if (lstrip != std::string::npos) { - result += content.substr(lstrip); - } - } else { - result += "<|im_start|>assistant\n" + msg.content; - } - if (!msg.tool_calls.empty()) { - for (size_t j = 0; j < msg.tool_calls.size(); j++) { - if ((j == 0 && !msg.content.empty()) || j > 0) { - result += "\n"; - } - result += "\n{\"name\": \"" + msg.tool_calls[j].name - + "\", \"arguments\": " + msg.tool_calls[j].arguments - + "}\n"; - } - } - result += "<|im_end|>\n"; - } else if (msg.role == "tool") { - bool first_tool = (i == 0 || messages[i - 1].role != "tool"); - bool last_tool = (i == messages.size() - 1 || messages[i + 1].role != "tool"); - if (first_tool) { - result += "<|im_start|>user"; - } - result += "\n\n" + msg.content + "\n"; - if (last_tool) { - result += "<|im_end|>\n"; - } - } - } - - if (add_generation_prompt) { - result += "<|im_start|>assistant\n"; - if (!enable_thinking_if_supported) { - result += "\n\n\n\n"; - } - } - - return result; -} - -std::string Tokenizer::format_lfm2_style(const std::vector& messages, - bool add_generation_prompt, - const std::string& tools_json) const -{ - std::string result = "<|startoftext|>"; - - std::string sys_content; - bool has_system_msg = false; - for (const auto& msg : messages) { - if (msg.role == "system") { - sys_content = msg.content; - has_system_msg = true; - break; - } - } - - if (!tools_json.empty()) { - if (!sys_content.empty()) { - sys_content += "\n"; - } - sys_content += "List of tools: <|tool_list_start|>["; - if (!tools_json.empty()) { - sys_content += "\n"; - sys_content += tools_json; - sys_content += "\n"; - } - sys_content += "]<|tool_list_end|>"; - sys_content += "\n\nWhen you need to call a tool, use this exact format:\n"; - sys_content += "<|tool_call_start|>[function_name(arg1=\"value1\", arg2=\"value2\")]<|tool_call_end|>\n"; - sys_content += "You can call multiple tools by using multiple tool call blocks."; - } - - if (!sys_content.empty()) { - result += "<|im_start|>system\n"; - result += sys_content; - result += "<|im_end|>\n"; - } - - for (const auto& msg : messages) { - if (msg.role == "system" && has_system_msg) { - has_system_msg = false; - continue; - } - result += "<|im_start|>" + msg.role + "\n"; - if (msg.role == "tool") { - result += "<|tool_response_start|>"; - result += msg.content; - result += "<|tool_response_end|>"; - } else { - result += msg.content; - if (msg.role == "assistant" && !msg.tool_calls.empty()) { - for (const auto& tc : msg.tool_calls) { - result += "<|tool_call_start|>[{\"name\": \"" + tc.name - + "\", \"arguments\": " + tc.arguments + "}]<|tool_call_end|>"; - } - } - } - result += "<|im_end|>\n"; - } - - if (add_generation_prompt) { - result += "<|im_start|>assistant\n"; - } - - return result; -} - -static std::string format_tool_call_for_prompt(const std::string& name, const std::string& args_json, bool use_pipe_tags) { - const char* tag_start = use_pipe_tags ? "<|tool_call>" : ""; - const char* tag_end = use_pipe_tags ? "" : ""; - - std::string formatted_args = "{"; - if (!args_json.empty()) { - std::map args; - size_t pos = 0; - while (pos < args_json.length() && args_json[pos] != '{') pos++; - if (pos < args_json.length()) pos++; - - while (pos < args_json.length()) { - while (pos < args_json.length() && std::isspace(static_cast(args_json[pos]))) pos++; - if (pos >= args_json.length() || args_json[pos] == '}') break; - if (args_json[pos] == ',') { pos++; continue; } - if (args_json[pos] != '"') break; - pos++; - - std::string key; - while (pos < args_json.length() && args_json[pos] != '"') { - if (args_json[pos] == '\\' && pos + 1 < args_json.length()) { pos++; } - key += args_json[pos++]; - } - if (pos < args_json.length()) pos++; - - while (pos < args_json.length() && (std::isspace(static_cast(args_json[pos])) || args_json[pos] == ':')) pos++; - - std::string value; - if (pos < args_json.length() && args_json[pos] == '"') { - pos++; - std::string str_val; - while (pos < args_json.length() && args_json[pos] != '"') { - if (args_json[pos] == '\\' && pos + 1 < args_json.length()) { - pos++; - if (args_json[pos] == 'n') str_val += '\n'; - else if (args_json[pos] == 't') str_val += '\t'; - else str_val += args_json[pos]; - } else { - str_val += args_json[pos]; - } - pos++; - } - if (pos < args_json.length()) pos++; - value = "" + str_val + ""; - } else { - size_t val_start = pos; - while (pos < args_json.length() && args_json[pos] != ',' && args_json[pos] != '}') pos++; - value = args_json.substr(val_start, pos - val_start); - while (!value.empty() && std::isspace(static_cast(value.back()))) value.pop_back(); - } - - if (!key.empty()) args[key] = value; - } - - bool first = true; - for (const auto& [k, v] : args) { - if (!first) formatted_args += ","; - first = false; - formatted_args += k + ":" + v; - } - } - formatted_args += "}"; - - return std::string(tag_start) + "call:" + name + formatted_args + tag_end; -} - -std::string Tokenizer::format_gemma4_style(const std::vector& messages, bool add_generation_prompt, - const std::string& tools_json, bool enable_thinking_if_supported) const { - std::string result = ""; - - std::string sys_content; - size_t first_msg = 0; - if (!messages.empty() && (messages[0].role == "system" || messages[0].role == "developer")) { - sys_content = messages[0].content; - first_msg = 1; - } - - if (enable_thinking_if_supported || !sys_content.empty() || !tools_json.empty()) { - result += "<|turn>system\n"; - if (enable_thinking_if_supported) { - result += "<|think|>"; - } - result += sys_content; - result += tools_json; - result += "\n"; - } - - auto strip_channel = [](const std::string& text) -> std::string { - const std::string open_tag = "<|channel>"; - const std::string close_tag = ""; - std::string out; - size_t pos = 0; - while (pos < text.size()) { - size_t open_pos = text.find(open_tag, pos); - if (open_pos == std::string::npos) { - out += text.substr(pos); - break; - } - out += text.substr(pos, open_pos - pos); - size_t close_pos = text.find(close_tag, open_pos + open_tag.size()); - if (close_pos == std::string::npos) { - break; - } - pos = close_pos + close_tag.size(); - } - return out; - }; - - auto compute_soft_tokens = [&](const std::string& image_path) -> size_t { - int w = 0, h = 0, c = 0; - unsigned char* data = stbi_load(image_path.c_str(), &w, &h, &c, 3); - if (!data) return 0; - stbi_image_free(data); - - uint32_t p = vision_patch_size_; - uint32_t k = vision_pooling_kernel_size_; - uint32_t side = k * p; - uint32_t max_patches = vision_default_output_length_ * k * k; - float factor = std::sqrt(static_cast(max_patches) * p * p / - (static_cast(h) * w)); - int th = static_cast(std::floor(factor * h / side)) * side; - int tw = static_cast(std::floor(factor * w / side)) * side; - if (th == 0) th = side; - if (tw == 0) tw = side; - return static_cast((th / p / k) * (tw / p / k)); - }; - - for (size_t i = first_msg; i < messages.size(); i++) { - const auto& msg = messages[i]; - std::string role = (msg.role == "assistant") ? "model" : msg.role; - result += "<|turn>" + role + "\n"; - if (role == "model") { - result += strip_channel(msg.content); - if (!msg.tool_calls.empty()) { - for (const auto& tc : msg.tool_calls) { - result += format_tool_call_for_prompt(tc.name, tc.arguments, true); - } - } - } else { - for (const auto& image_path : msg.images) { - size_t n = compute_soft_tokens(image_path); - if (n > 0) { - result += "\n\n<|image>"; - for (size_t j = 0; j < n; j++) - result += "<|image|>"; - result += "\n\n"; - } - } - result += msg.content; - if (msg.audio_soft_token_count > 0) { - result += "<|audio>"; - for (size_t j = 0; j < msg.audio_soft_token_count; j++) - result += "<|audio|>"; - result += ""; - } - } - result += "\n"; - } - - if (add_generation_prompt) { - result += "<|turn>model\n"; - } - - return result; -} - -std::string Tokenizer::format_lfm2_vl_style( - const std::vector& messages, - bool add_generation_prompt, - const std::string& tools_json) const -{ - if (!tools_json.empty()) { - return "ERROR: Tool calls are not supported for LFM2-VL models"; - } - - std::string result = "<|startoftext|>"; - - for (const auto& msg : messages) { - result += "<|im_start|>" + msg.role + "\n"; - for (const auto& image_path : msg.images) { - int width = 0, height = 0, channels = 0; - unsigned char* img_data = stbi_load(image_path.c_str(), &width, &height, &channels, 0); - - if (img_data) { - Siglip2Preprocessor preprocessor; - auto shape_result = preprocessor.compute_spatial_shapes(height, width); - int downsample_factor = 2; - bool use_thumbnail = true; - int grid_rows = shape_result.grid_rows; - int grid_cols = shape_result.grid_cols; - int num_tiles = grid_rows * grid_cols; - result += "<|image_start|>"; - - if (num_tiles > 1) { - for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { - int row = tile_idx / grid_cols; - int col = tile_idx % grid_cols; - - result += "<|img_row_" + std::to_string(row + 1) + "_col_" + std::to_string(col + 1) + "|>"; - auto [tile_height, tile_width] = shape_result.shapes[tile_idx]; - int tile_tokens = (tile_height * tile_width) / (downsample_factor * downsample_factor); - - for (int t = 0; t < tile_tokens; ++t) { - result += ""; - } - } - if (use_thumbnail && static_cast(num_tiles) < shape_result.shapes.size()) { - result += "<|img_thumbnail|>"; - - auto [thumb_height, thumb_width] = shape_result.shapes[num_tiles]; - int thumbnail_tokens = (thumb_height * thumb_width) / (downsample_factor * downsample_factor); - - for (int t = 0; t < thumbnail_tokens; ++t) { - result += ""; - } - } - } else if (num_tiles == 1) { - auto [thumb_height, thumb_width] = shape_result.shapes[0]; - int thumbnail_tokens = (thumb_height * thumb_width) / (downsample_factor * downsample_factor); - - for (int t = 0; t < thumbnail_tokens; ++t) { - result += ""; - } - } - - result += "<|image_end|>"; - - stbi_image_free(img_data); - } else { - result += ""; - } - } - result += msg.content; - - result += "<|im_end|>\n"; - } - - if (add_generation_prompt) { - result += "<|im_start|>assistant\n"; - } - - return result; -} - - -std::string Tokenizer::format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const { - auto trim = [](const std::string& s) -> std::string { - size_t start = 0; - while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; - size_t end = s.size(); - while (end > start && std::isspace(static_cast(s[end - 1]))) --end; - return s.substr(start, end - start); - }; - - std::string result = ""; - - std::string system_content; - size_t start_idx = 0; - - if (!messages.empty() && (messages[0].role == "system" || messages[0].role == "developer")) { - system_content = messages[0].content; - start_idx = 1; - } - - if (!tools_json.empty() || !system_content.empty()) { - result += "developer\n"; - if (!system_content.empty()) { - result += trim(system_content); - } - if (!tools_json.empty()) { - result += tools_json; - } - result += "\n"; - } - - std::string prev_message_type; - - for (size_t i = start_idx; i < messages.size(); i++) { - const auto& msg = messages[i]; - - if (msg.role == "tool") { - std::string func_name = msg.name.empty() ? "tool" : msg.name; - result += "response:" + func_name + "{" + msg.content + "}"; - prev_message_type = "tool_response"; - - } else if (msg.role == "user") { - if (prev_message_type != "tool_response") { - result += "user\n"; - } - result += trim(msg.content); - result += "\n"; - prev_message_type = "content"; - - } else if (msg.role == "assistant" || msg.role == "model") { - if (prev_message_type != "tool_response") { - result += "model\n"; - } - if (!msg.content.empty()) { - result += trim(msg.content); - } - if (!msg.tool_calls.empty()) { - for (const auto& tc : msg.tool_calls) { - result += format_tool_call_for_prompt(tc.name, tc.arguments, false); - } - if (i == messages.size() - 1) { - result += ""; - } - prev_message_type = "tool_call"; - } else { - result += "\n"; - prev_message_type = "content"; - } - } - } - - if (add_generation_prompt) { - if (prev_message_type != "tool_response" && prev_message_type != "tool_call") { - result += "model\n"; - } - } - - return result; -} - -std::string Tokenizer::format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const { - std::string result = "<|begin_of_text|>"; - - std::string system_block; - for (const auto& msg : messages) { - if (msg.role == "system") { system_block = msg.content; break; } - } - - if (!tools_json.empty()) { - std::string tool_desc = - "<|begin_of_tool_description|>Tool calling capabilities.\n" - "You may call one or more functions to assist with the user query. " - "You have the following functions available:\n" - "```json\n" + tools_json + "\n```\n" - "For tool call returns, you MUST use the following format:\n" - "{\"name\": \"function-name\", \"arguments\": {\"param1\": \"value1\", \"param2\": \"value2\"}}\n" - "<|end_of_tool_description|>"; - system_block = system_block.empty() ? tool_desc : system_block + "\n\n" + tool_desc; - } - - result += system_block; - - bool is_last_user = false; - bool is_tool = false; - bool is_output_first = true; - - for (const auto& msg : messages) { - if (msg.role == "system") { - continue; - } else if (msg.role == "user") { - is_last_user = true; is_tool = false; - result += "<|User|>" + msg.content; - } else if (msg.role == "tool") { - is_tool = true; is_last_user = false; - if (is_output_first) { - result += "<|User|>" + msg.content + ""; - is_output_first = false; - } else { - result += "\n" + msg.content + ""; - } - } else if (msg.role == "assistant" || msg.role == "model") { - is_last_user = false; is_tool = false; is_output_first = true; - result += "<|Assistant|>" + msg.content; - if (!msg.tool_calls.empty()) { - for (const auto& tc : msg.tool_calls) { - result += "{\"name\": \"" + tc.name - + "\", \"arguments\": " + tc.arguments + "}"; - } - } - result += "<|end_of_text|>"; - } - } - - if (add_generation_prompt && (is_last_user || is_tool)) { - result += "<|Assistant|>"; - } - - return result; -} - -} // namespace engine -} // namespace cactus diff --git a/cactus-transpile/ffi/cactus_cloud.cpp b/cactus-transpile/ffi/cactus_cloud.cpp deleted file mode 100644 index 2116f48f3..000000000 --- a/cactus-transpile/ffi/cactus_cloud.cpp +++ /dev/null @@ -1,452 +0,0 @@ -#include "cactus_cloud.h" -#include "telemetry/telemetry.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef CACTUS_USE_CURL -#include -#endif - -namespace cactus { -namespace ffi { - -namespace { - -std::string compact_error_detail(std::string detail) { - detail = trim_string(detail); - if (detail.empty()) return {}; - - for (char& c : detail) { - if (c == '\n' || c == '\r' || c == '\t') c = ' '; - } - while (detail.find(" ") != std::string::npos) { - detail.erase(detail.find(" "), 1); - } - - constexpr size_t kMaxLen = 160; - if (detail.size() > kMaxLen) { - detail = detail.substr(0, kMaxLen) + "..."; - } - return detail; -} - -} // namespace - -std::string resolve_cloud_api_key(const char* cloud_key_param) { - const char* env_cloud = std::getenv("CACTUS_CLOUD_KEY"); - const char* env_cloud_legacy = std::getenv("CACTUS_CLOUD_API_KEY"); // keeping this cuz Matthew Hayes had some code reliant on Cactus_cloud_api_key - std::string resolved_cloud_key; - bool key_from_param_or_env = false; - - if (cloud_key_param && *cloud_key_param) { - resolved_cloud_key = cloud_key_param; - key_from_param_or_env = true; - } else if (env_cloud && *env_cloud) { - resolved_cloud_key = env_cloud; - key_from_param_or_env = true; - } else if (env_cloud_legacy && *env_cloud_legacy) { - resolved_cloud_key = env_cloud_legacy; - key_from_param_or_env = true; - } else { - resolved_cloud_key = cactus::telemetry::loadCachedCloudApiKey(); - } - - resolved_cloud_key = trim_string(resolved_cloud_key); - - const bool should_cache = - !resolved_cloud_key.empty() && - key_from_param_or_env; - if (should_cache) { - cactus::telemetry::cacheCloudApiKey(resolved_cloud_key.c_str()); - } - - return resolved_cloud_key; -} - -namespace { - -#ifdef CACTUS_USE_CURL -static std::atomic g_warned_missing_cloud_api_key{false}; - -static void apply_curl_tls_trust(CURL* curl) { - if (!curl) return; - const char* ca_bundle = std::getenv("CACTUS_CA_BUNDLE"); - if (ca_bundle && ca_bundle[0] != '\0') { - curl_easy_setopt(curl, CURLOPT_CAINFO, ca_bundle); - } -#if defined(__ANDROID__) - const char* ca_path = std::getenv("CACTUS_CA_PATH"); - if (ca_path && ca_path[0] != '\0') { - curl_easy_setopt(curl, CURLOPT_CAPATH, ca_path); - } else { - curl_easy_setopt(curl, CURLOPT_CAPATH, "/system/etc/security/cacerts"); - } -#endif -} - -static size_t curl_write_cb(void* ptr, size_t size, size_t nmemb, void* userdata) { - auto* s = static_cast(userdata); - s->append(static_cast(ptr), size * nmemb); - return size * nmemb; -} - -static bool read_file_bytes(const std::string& path, std::vector& out) { - std::ifstream in(path, std::ios::binary); - if (!in.is_open()) return false; - in.seekg(0, std::ios::end); - std::streamsize size = in.tellg(); - if (size <= 0) return false; - in.seekg(0, std::ios::beg); - - out.resize(static_cast(size)); - if (!in.read(reinterpret_cast(out.data()), size)) return false; - return true; -} - -static std::string infer_mime_type(const std::string& path) { - auto dot = path.find_last_of('.'); - if (dot == std::string::npos) return "image/jpeg"; - std::string ext = path.substr(dot + 1); - std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); - - if (ext == "png") return "image/png"; - if (ext == "webp") return "image/webp"; - if (ext == "gif") return "image/gif"; - return "image/jpeg"; -} - -static std::string build_cloud_text_prompt(const CloudCompletionRequest& request) { - std::ostringstream oss; - oss << "You are continuing an existing assistant conversation.\\n"; - oss << "Output contract:\\n"; - oss << "1) Never include role prefixes like 'assistant:'.\\n"; - oss << "2) Never include markdown/code fences/backticks.\\n"; - oss << "3) Return only the final assistant answer text unless a tool call is required.\\n"; - oss << "4) If a tool call is required, return ONLY JSON with this exact shape:\\n"; - oss << "[{\"name\":\"tool_name\",\"arguments\":{\"arg\":\"value\"}}]\\n"; - oss << "5) Do not include any prose before or after that JSON tool-call output.\\n"; - oss << "\\nConversation:\\n"; - for (const auto& m : request.messages) { - oss << "[" << m.role << "] " << m.content << "\\n"; - } - - if (!request.tools.empty()) { - oss << "\\nAvailable tools JSON (use only these tool names and arguments):\\n"; - oss << serialize_tools_json(request.tools) << "\\n"; - oss << "If tools are relevant, prefer the strict JSON tool-call output contract above.\\n"; - } - - if (!request.local_output.empty()) { - oss << "\\nLocal model draft (useful fallback reference, may be low confidence):\\n"; - oss << request.local_output << "\\n"; - } - - return oss.str(); -} - -static std::string call_cloud_endpoint(const std::string& url, - const std::string& payload, - long timeout_ms, - const char* cloud_key_param, - std::string& err_out) { - std::string api_key = resolve_cloud_api_key(cloud_key_param); - if (api_key.empty()) { - if (!g_warned_missing_cloud_api_key.exchange(true)) { - CACTUS_LOG_WARN("cloud_handoff", "No cloud key found (cloud_key param, CACTUS_CLOUD_KEY, or CACTUS_CLOUD_API_KEY env); cloud handoff will fall back to local output"); - } - err_out = "missing_api_key"; - return {}; - } - - CURL* curl = curl_easy_init(); - if (!curl) { - err_out = "curl_init_failed"; - return {}; - } - - std::string response_body; - struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, ("X-API-Key: " + api_key).c_str()); - headers = curl_slist_append(headers, "Content-Type: application/json"); - - const char* extra_hdrs = std::getenv("CACTUS_CLOUD_HEADERS"); - if (extra_hdrs && extra_hdrs[0] != '\0') { - std::istringstream stream(extra_hdrs); - std::string pair; - while (std::getline(stream, pair, ',')) { - auto eq = pair.find('='); - if (eq != std::string::npos && eq > 0) { - std::string header = pair.substr(0, eq) + ": " + pair.substr(eq + 1); - headers = curl_slist_append(headers, header.c_str()); - } - } - } - - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.size())); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_body); - curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout_ms); - - if (!env_flag_enabled("CACTUS_CLOUD_STRICT_SSL")) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - } else { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); - apply_curl_tls_trust(curl); - } - - CURLcode res = curl_easy_perform(curl); - long http_status = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_status); - curl_slist_free_all(headers); - curl_easy_cleanup(curl); - - if (res != CURLE_OK) { - err_out = curl_easy_strerror(res); - return {}; - } - - if (http_status >= 400) { - std::string detail = json_string_field(response_body, "error"); - if (detail.empty()) detail = json_string_field(response_body, "message"); - if (detail.empty()) detail = json_string_field(response_body, "detail"); - detail = compact_error_detail(detail); - - err_out = "http_" + std::to_string(http_status); - if (!detail.empty()) { - err_out += ":" + detail; - } - return {}; - } - - return response_body; -} -#endif - -} // namespace - -std::string cloud_base64_encode(const uint8_t* data, size_t len) { - static const char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - std::string out; - out.reserve(((len + 2) / 3) * 4); - for (size_t i = 0; i < len; i += 3) { - uint32_t n = static_cast(data[i]) << 16; - if (i + 1 < len) n |= static_cast(data[i + 1]) << 8; - if (i + 2 < len) n |= static_cast(data[i + 2]); - out += table[(n >> 18) & 0x3F]; - out += table[(n >> 12) & 0x3F]; - out += (i + 1 < len) ? table[(n >> 6) & 0x3F] : '='; - out += (i + 2 < len) ? table[n & 0x3F] : '='; - } - return out; -} - -std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes) { - constexpr uint32_t sample_rate = 16000; - constexpr uint16_t channels = 1; - constexpr uint16_t bits = 16; - const uint32_t byte_rate = sample_rate * channels * bits / 8; - const uint16_t block_align = channels * bits / 8; - const uint32_t data_size = static_cast(pcm_bytes); - const uint32_t file_size = 36 + data_size; - - std::vector wav(44 + pcm_bytes); - auto w16 = [&](size_t off, uint16_t v) { - wav[off] = v & 0xFF; - wav[off + 1] = v >> 8; - }; - auto w32 = [&](size_t off, uint32_t v) { - wav[off] = v & 0xFF; - wav[off + 1] = (v >> 8) & 0xFF; - wav[off + 2] = (v >> 16) & 0xFF; - wav[off + 3] = (v >> 24) & 0xFF; - }; - - std::memcpy(wav.data(), "RIFF", 4); - w32(4, file_size); - std::memcpy(wav.data() + 8, "WAVE", 4); - std::memcpy(wav.data() + 12, "fmt ", 4); - w32(16, 16); - w16(20, 1); - w16(22, channels); - w32(24, sample_rate); - w32(28, byte_rate); - w16(32, block_align); - w16(34, bits); - std::memcpy(wav.data() + 36, "data", 4); - w32(40, data_size); - std::memcpy(wav.data() + 44, pcm, pcm_bytes); - return wav; -} - -CloudResponse cloud_transcribe_request(const std::string& audio_b64, - const std::string& fallback_text, - long timeout_seconds, - const char* cloud_key) { -#ifdef CACTUS_USE_CURL - std::string base = env_or_default("CACTUS_CLOUD_API_BASE", "https://104.198.76.3/api/v1"); - std::string endpoint = base + "/transcribe"; - - std::string payload = "{\"audio\":\"" + audio_b64 + "\",\"mime_type\":\"audio/wav\",\"language\":\"en-US\"}"; - - std::string err; - std::string body = call_cloud_endpoint(endpoint, payload, timeout_seconds * 1000L, cloud_key, err); - if (body.empty()) { - return {fallback_text, "", false, err.empty() ? "request_failed" : err}; - } - - std::string transcript = json_string_field(body, "transcript"); - if (transcript.empty()) { - transcript = json_string_field(body, "text"); - } - if (transcript.empty()) { - transcript = json_string_field(body, "response"); - } - if (transcript.empty()) { - transcript = json_string_field(body, "analysis"); - } - if (transcript.empty()) { - std::string detail = json_string_field(body, "error"); - if (detail.empty()) detail = json_string_field(body, "message"); - if (detail.empty()) detail = json_string_field(body, "detail"); - detail = compact_error_detail(detail); - if (!detail.empty()) { - return {fallback_text, "", false, "missing_transcript:" + detail}; - } - return {fallback_text, "", false, "missing_transcript"}; - } - - std::string api_key_hash = json_string_field(body, "api_key_hash"); - return {transcript, api_key_hash, true, ""}; -#else - (void)audio_b64; - (void)timeout_seconds; - return {fallback_text, "", false, "curl_not_enabled"}; -#endif -} - -CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, - long timeout_ms) { -#ifdef CACTUS_USE_CURL - std::string base = env_or_default("CACTUS_CLOUD_API_BASE", "https://104.198.76.3/api/v1"); - std::string text_model = env_or_default("CACTUS_CLOUD_TEXT_MODEL", "gemini-2.5-flash"); - std::string vlm_model = env_or_default("CACTUS_CLOUD_VLM_MODEL", "gemini-2.5-flash"); - - std::string endpoint; - std::string payload; - - if (request.has_images) { - std::vector image_paths; - for (const auto& message : request.messages) { - for (const auto& image : message.images) { - if (!image.empty()) { - image_paths.push_back(image); - } - } - } - - if (image_paths.size() > 1) { - return {false, false, "", {}, "multiple_images_not_supported"}; - } - - std::string image_path; - if (!image_paths.empty()) { - image_path = image_paths.back(); - } - if (image_path.empty()) { - return {false, false, "", {}, "missing_image_path"}; - } - - std::vector img_bytes; - if (!read_file_bytes(image_path, img_bytes)) { - return {false, false, "", {}, "image_read_failed"}; - } - - std::string img_b64 = cloud_base64_encode(img_bytes.data(), img_bytes.size()); - std::string mime = infer_mime_type(image_path); - std::string prompt = build_cloud_text_prompt(request); - - endpoint = base + "/vlm"; - payload = "{" - "\"image\":\"" + img_b64 + "\"," - "\"mime_type\":\"" + mime + "\"," - "\"prompt\":\"" + escape_json_string(prompt) + "\"," - "\"language\":\"en-US\"," - "\"model\":\"" + escape_json_string(vlm_model) + "\"" - "}"; - } else { - endpoint = base + "/text"; - std::string text = build_cloud_text_prompt(request); - payload = "{" - "\"text\":\"" + escape_json_string(text) + "\"," - "\"language\":\"en-US\"," - "\"model\":\"" + escape_json_string(text_model) + "\"" - "}"; - } - - std::string err; - const char* cloud_key = request.cloud_key.empty() ? nullptr : request.cloud_key.c_str(); - std::string body = call_cloud_endpoint(endpoint, payload, timeout_ms, cloud_key, err); - if (body.empty()) { - return {false, false, "", {}, err.empty() ? "request_failed" : err}; - } - - std::vector function_calls; - std::string calls_json = json_array_field(body, "function_calls"); - if (calls_json != "[]") { - function_calls = split_json_array(calls_json); - } - - std::string response = json_string_field(body, "text"); - if (response.empty()) { - response = json_string_field(body, "analysis"); - } - - if (!response.empty() && function_calls.empty()) { - std::string regular_response; - parse_function_calls_from_response(response, regular_response, function_calls); - response = regular_response; - } - - if (!response.empty() && function_calls.empty()) { - auto first = response.find_first_not_of(" \t\n\r"); - auto last = response.find_last_not_of(" \t\n\r"); - if (first != std::string::npos && last != std::string::npos) { - std::string trimmed = response.substr(first, last - first + 1); - if (!trimmed.empty() && trimmed.front() == '[' && trimmed.back() == ']' && - trimmed.find("\"name\"") != std::string::npos) { - function_calls = split_json_array(trimmed); - response.clear(); - } - } - } - - if (response.empty() && function_calls.empty()) { - return {false, false, "", {}, "missing_text"}; - } - - CloudCompletionResult out; - out.ok = true; - out.used_cloud = true; - out.response = response; - out.function_calls = std::move(function_calls); - return out; -#else - (void)request; - (void)timeout_ms; - return {false, false, "", {}, "curl_not_enabled"}; -#endif -} - -} // namespace ffi -} // namespace cactus diff --git a/cactus-transpile/ffi/cactus_cloud.h b/cactus-transpile/ffi/cactus_cloud.h deleted file mode 100644 index e61841d7c..000000000 --- a/cactus-transpile/ffi/cactus_cloud.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef CACTUS_CLOUD_H -#define CACTUS_CLOUD_H - -#include "cactus_utils.h" -#include -#include - -namespace cactus { -namespace ffi { - -struct CloudResponse { - std::string transcript; - std::string api_key_hash; - bool used_cloud = false; - std::string error; -}; - -struct CloudCompletionRequest { - std::vector messages; - std::vector tools; - std::string local_output; - std::vector local_function_calls; - bool has_images = false; - std::string cloud_key; -}; - -struct CloudCompletionResult { - bool ok = false; - bool used_cloud = false; - std::string response; - std::vector function_calls; - std::string error; -}; - -std::string cloud_base64_encode(const uint8_t* data, size_t len); -std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes); -std::string resolve_cloud_api_key(const char* cloud_key_param); -CloudResponse cloud_transcribe_request(const std::string& audio_b64, - const std::string& fallback_text, - long timeout_seconds = 15L, - const char* cloud_key = nullptr); -CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, - long timeout_ms); - -} // namespace ffi -} // namespace cactus - -#endif // CACTUS_CLOUD_H diff --git a/cactus-transpile/ffi/cactus_complete.cpp b/cactus-transpile/ffi/cactus_complete.cpp deleted file mode 100644 index cd25d88a1..000000000 --- a/cactus-transpile/ffi/cactus_complete.cpp +++ /dev/null @@ -1,1177 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_cloud.h" -#include "cactus_utils.h" -#include "telemetry/telemetry.h" -#include "../../libs/audio/wav.h" -#include -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - -static constexpr size_t ROLLING_ENTROPY_WINDOW = 10; - -namespace { - -std::vector> extract_schema_property_types(const std::string& schema); -std::vector extract_schema_required(const std::string& schema); - -std::string extract_last_user_query(const std::vector& messages) { - for (auto it = messages.rbegin(); it != messages.rend(); ++it) { - if (it->role == "user") { - return it->content; - } - } - return {}; -} - -std::vector encode_needle_tools_suffix(Tokenizer* tokenizer, const std::string& formatted_tools) { - auto prefix = tokenizer->encode(""); - auto body = tokenizer->encode(formatted_tools); - std::vector tokens; - tokens.reserve(prefix.size() + body.size() + 1); - tokens.insert(tokens.end(), prefix.begin(), prefix.end()); - tokens.insert(tokens.end(), body.begin(), body.end()); - tokens.push_back(tokenizer->get_eos_token()); - return tokens; -} - -void inject_rag_context(CactusModelHandle* handle, std::vector& messages) { - if (!handle->corpus_index) return; - - std::string query = extract_last_user_query(messages); - if (query.empty()) return; - - std::string rag_context = retrieve_rag_context(handle, query); - if (rag_context.empty()) return; - - if (!messages.empty() && messages[0].role == "system") { - messages[0].content = rag_context + messages[0].content; - } else { - ChatMessage system_msg; - system_msg.role = "system"; - system_msg.content = rag_context + "Answer the user's question using ONLY the context above. Do not use any prior knowledge. If the answer cannot be found in the context, respond with \"I don't have enough information to answer that.\""; - messages.insert(messages.begin(), system_msg); - } -} - -std::vector build_tool_constraint_specs(const std::vector& tools) { - std::vector specs; - specs.reserve(tools.size()); - - for (const auto& tool : tools) { - ToolConstraintSpec spec; - spec.name = tool.name; - - auto schema_it = tool.parameters.find("schema"); - if (schema_it != tool.parameters.end()) { - auto properties = extract_schema_property_types(schema_it->second); - spec.parameter_names.reserve(properties.size()); - for (const auto& [name, _] : properties) { - spec.parameter_names.push_back(name); - } - spec.required_parameter_names = extract_schema_required(schema_it->second); - } - - specs.push_back(std::move(spec)); - } - - return specs; -} - -void strip_thinking_from_cache(CactusModelHandle* handle, - const std::vector& generated_tokens, - size_t prompt_len) { - const auto& cfg = handle->model->get_config(); - uint32_t open_id = cfg.channel_open_token_id; - uint32_t close_id = cfg.channel_close_token_id; - auto ranges = find_channel_token_ranges(generated_tokens, prompt_len, - open_id, close_id); - if (ranges.empty()) return; - - handle->model->remove_thinking_tokens(ranges); - for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) { - auto start = handle->processed_tokens.begin() + it->first; - handle->processed_tokens.erase(start, start + it->second); - } -} - -void setup_tool_constraints(CactusModelHandle* handle, const std::vector& tools, - bool force_tools, float& temperature) { - if (!force_tools || tools.empty()) return; - - handle->model->set_tool_constraints(build_tool_constraint_specs(tools)); - - if (temperature == 0.0f) { - temperature = 0.01f; - } -} - -size_t find_json_block_end(const std::string& json, size_t start) { - if (start >= json.size() || json[start] != '{') { - return std::string::npos; - } - - int depth = 1; - bool in_string = false; - bool escaped = false; - size_t pos = start + 1; - while (pos < json.size() && depth > 0) { - char c = json[pos]; - if (in_string) { - if (escaped) { - escaped = false; - } else if (c == '\\') { - escaped = true; - } else if (c == '"') { - in_string = false; - } - } else { - if (c == '"') { - in_string = true; - } else if (c == '{') { - depth++; - } else if (c == '}') { - depth--; - } - } - ++pos; - } - - return depth == 0 ? pos : std::string::npos; -} - -std::string extract_json_object_field(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t key_pos = json.find(pattern); - if (key_pos == std::string::npos) { - return {}; - } - - size_t object_start = json.find('{', key_pos + pattern.size()); - if (object_start == std::string::npos) { - return {}; - } - - size_t object_end = find_json_block_end(json, object_start); - if (object_end == std::string::npos) { - return {}; - } - - return json.substr(object_start, object_end - object_start); -} - -std::vector> extract_schema_property_types(const std::string& schema) { - std::vector> properties; - std::string properties_object = extract_json_object_field(schema, "properties"); - if (properties_object.empty() || properties_object.size() < 2) { - return properties; - } - - size_t pos = 1; - while (pos + 1 < properties_object.size()) { - size_t key_start = properties_object.find('"', pos); - if (key_start == std::string::npos || key_start + 1 >= properties_object.size()) { - break; - } - size_t key_end = properties_object.find('"', key_start + 1); - if (key_end == std::string::npos) { - break; - } - - std::string name = properties_object.substr(key_start + 1, key_end - key_start - 1); - size_t value_start = properties_object.find('{', key_end); - if (value_start == std::string::npos) { - break; - } - size_t value_end = find_json_block_end(properties_object, value_start); - if (value_end == std::string::npos) { - break; - } - - std::string value = properties_object.substr(value_start, value_end - value_start); - std::string type = "string"; - std::string type_pattern = "\"type\":\""; - size_t type_pos = value.find(type_pattern); - if (type_pos != std::string::npos) { - size_t type_start = type_pos + type_pattern.size(); - size_t type_end = value.find('"', type_start); - if (type_end != std::string::npos) { - type = value.substr(type_start, type_end - type_start); - } - } else if (value.find("\"enum\"") != std::string::npos) { - type = "string"; - } else if (value.find("\"properties\"") != std::string::npos) { - type = "object"; - } - - properties.emplace_back(std::move(name), std::move(type)); - pos = value_end; - } - - return properties; -} - -std::vector extract_schema_required(const std::string& schema) { - std::vector required; - std::string key = "\"required\""; - size_t key_pos = schema.find(key); - if (key_pos == std::string::npos) return required; - size_t arr_start = schema.find('[', key_pos + key.size()); - if (arr_start == std::string::npos) return required; - size_t arr_end = schema.find(']', arr_start); - if (arr_end == std::string::npos) return required; - size_t pos = arr_start + 1; - while (pos < arr_end) { - size_t qs = schema.find('"', pos); - if (qs == std::string::npos || qs >= arr_end) break; - size_t qe = schema.find('"', qs + 1); - if (qe == std::string::npos || qe > arr_end) break; - required.push_back(schema.substr(qs + 1, qe - qs - 1)); - pos = qe + 1; - } - return required; -} - -std::string serialize_needle_tools_json(const std::vector& tools, const char* raw_tools_json) { - if (tools.empty()) { - return (raw_tools_json && std::strlen(raw_tools_json) > 0) ? raw_tools_json : "[]"; - } - - std::ostringstream oss; - oss << "["; - for (size_t i = 0; i < tools.size(); ++i) { - if (i > 0) { - oss << ","; - } - - oss << "{\"name\":\"" << escape_json_string(tools[i].name) << "\""; - if (!tools[i].description.empty()) { - oss << ",\"description\":\"" << escape_json_string(tools[i].description) << "\""; - } - - oss << ",\"parameters\":"; - auto schema_it = tools[i].parameters.find("schema"); - if (schema_it == tools[i].parameters.end()) { - oss << "{}"; - } else { - auto properties = extract_schema_property_types(schema_it->second); - if (properties.empty()) { - oss << schema_it->second; - } else { - oss << "{"; - for (size_t p = 0; p < properties.size(); ++p) { - if (p > 0) { - oss << ","; - } - oss << "\"" << escape_json_string(properties[p].first) << "\":" - << "\"" << escape_json_string(properties[p].second) << "\""; - } - oss << "}"; - } - } - - oss << "}"; - } - oss << "]"; - return oss.str(); -} - -std::vector> build_stop_sequences( - Tokenizer* tokenizer, - const std::vector& stop_sequences, - Config::ModelType model_type, - bool has_tools -) { - std::vector> stop_token_sequences; - stop_token_sequences.push_back({tokenizer->get_eos_token()}); - - std::vector sequences = stop_sequences; - if (sequences.empty()) { - std::string default_stop = tokenizer->get_default_stop_sequence(); - if (!default_stop.empty()) { - sequences.push_back(default_stop); - } - } - for (const auto& stop_seq : sequences) { - stop_token_sequences.push_back(tokenizer->encode(stop_seq)); - } - - if ((model_type == Config::ModelType::GEMMA || model_type == Config::ModelType::GEMMA3N) && has_tools) { - stop_token_sequences.push_back(tokenizer->encode("")); - stop_token_sequences.push_back(tokenizer->encode("")); - } - - if (model_type == Config::ModelType::GEMMA4) { - stop_token_sequences.push_back(tokenizer->encode("")); - if (has_tools) { - stop_token_sequences.push_back(tokenizer->encode("")); - stop_token_sequences.push_back(tokenizer->encode("<|tool_response>")); - } - } - - return stop_token_sequences; -} - -void trim_stop_suffix(std::vector& generated_tokens, - const std::vector>& stop_token_sequences, - bool include_stop_sequences) { - if (include_stop_sequences) return; - for (const auto& stop_seq : stop_token_sequences) { - if (stop_seq.empty()) continue; - if (generated_tokens.size() >= stop_seq.size() && - std::equal(stop_seq.rbegin(), stop_seq.rend(), generated_tokens.rbegin())) { - generated_tokens.resize(generated_tokens.size() - stop_seq.size()); - break; - } - } -} - -void reset_cache(CactusModelHandle* handle) { - handle->model->reset_cache(); - handle->processed_tokens.clear(); - handle->processed_images.clear(); -} - -struct PrefillResult { - std::vector remaining_tokens; - size_t prefilled_count = 0; - bool was_prefix = false; - bool was_exact_match = false; -}; - -struct EntropyState { - std::vector window; - float window_sum = 0.0f; - float total_sum = 0.0f; - size_t total_count = 0; - bool spike_handoff = false; - - void add(float entropy) { - window.push_back(entropy); - window_sum += entropy; - total_sum += entropy; - total_count++; - - if (window.size() > ROLLING_ENTROPY_WINDOW) { - window_sum -= window.front(); - window.erase(window.begin()); - } - } - - float rolling_confidence() const { - return 1.0f - (window_sum / window.size()); - } - - float mean_confidence() const { - return 1.0f - (total_sum / static_cast(total_count)); - } -}; - -struct PreparedPrompt { - InferenceOptions options; - Config::ModelType model_type = Config::ModelType::QWEN; - std::vector image_paths; - std::vector audio_paths; - std::vector messages; - std::vector tools; - std::vector tokens; - size_t context_token_count = 0; - std::vector> images; - - std::vector audio_features; - size_t audio_num_frames = 0; - - bool has_images() const { - return std::any_of(images.begin(), images.end(), - [](const auto& msg_imgs) { return !msg_imgs.empty(); }); - } - - bool has_audio() const { - return !audio_features.empty(); - } -}; - -CactusModelHandle::ProcessedImage image_signature(const std::string& image_path) { - std::filesystem::path normalized_path(image_path); - std::error_code ec; - - auto absolute_path = std::filesystem::absolute(normalized_path, ec); - if (!ec) { - normalized_path = absolute_path; - } - - CactusModelHandle::ProcessedImage image; - image.path = normalized_path.string(); - - ec.clear(); - auto status = std::filesystem::status(normalized_path, ec); - if (!ec && std::filesystem::is_regular_file(status)) { - std::error_code time_ec; - auto mtime = std::filesystem::last_write_time(normalized_path, time_ec); - if (!time_ec) { - image.last_modified_timestamp = static_cast(mtime.time_since_epoch().count()); - } - } - - return image; -} - -std::vector> images_from_message(const std::vector& messages) { - std::vector> message_signatures; - message_signatures.reserve(messages.size()); - - for (const auto& message : messages) { - std::vector image_signatures; - image_signatures.reserve(message.images.size()); - for (const auto& image_path : message.images) { - image_signatures.push_back(image_signature(image_path)); - } - message_signatures.push_back(std::move(image_signatures)); - } - - return message_signatures; -} - -bool image_context_prefix_matches( - const std::vector>& prefix, - const std::vector>& full -) { - return prefix.size() <= full.size() && - std::equal(prefix.begin(), prefix.end(), full.begin()); -} - -bool prompt_context_matches( - const CactusModelHandle* handle, - const PreparedPrompt& prompt -) { - if (handle->processed_tokens.empty()) { - return false; - } - if (handle->model && - handle->model->get_config().model_type == Config::ModelType::NEEDLE && - prompt.tokens.size() != handle->processed_tokens.size() + 1) { - return false; - } - if (prompt.context_token_count < handle->processed_tokens.size()) { - return false; - } - if (!std::equal(handle->processed_tokens.begin(), handle->processed_tokens.end(), prompt.tokens.begin())) { - return false; - } - if (prompt.has_images()) { - return image_context_prefix_matches(handle->processed_images, prompt.images); - } - return !prompt.has_images(); -} - -PreparedPrompt prepare_prompt( - CactusModelHandle* handle, - const char* messages_json, - const char* options_json, - const char* tools_json, - bool apply_tool_constraints, - bool add_generation_prompt, - const uint8_t* pcm_buffer = nullptr, - size_t pcm_buffer_size = 0 -) { - if (!handle || !handle->model) { - throw std::runtime_error("Invalid model handle"); - } - - PreparedPrompt prompt; - prompt.options = parse_inference_options_json(options_json ? options_json : ""); - prompt.messages = parse_messages_json(messages_json, prompt.image_paths, &prompt.audio_paths); - if (prompt.messages.empty()) { - throw std::runtime_error("No messages provided"); - } - - inject_rag_context(handle, prompt.messages); - - if (tools_json && std::strlen(tools_json) > 0) { - prompt.tools = parse_tools_json(tools_json); - } - - if (prompt.options.tool_rag_top_k > 0 && prompt.tools.size() > prompt.options.tool_rag_top_k) { - std::string query = extract_last_user_query(prompt.messages); - if (!query.empty()) { - prompt.tools = select_relevant_tools(handle, query, prompt.tools, prompt.options.tool_rag_top_k); - } - } - - if (prompt.model_type == Config::ModelType::NEEDLE && !prompt.tools.empty()) { - prompt.options.force_tools = true; - } - - if (apply_tool_constraints) { - setup_tool_constraints(handle, prompt.tools, prompt.options.force_tools, prompt.options.temperature); - } - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - throw std::runtime_error("Tokenizer unavailable"); - } - - prompt.model_type = handle->model->get_config().model_type; - - if (prompt.model_type == Config::ModelType::GEMMA4) { - std::vector audio_samples; - if (pcm_buffer != nullptr && pcm_buffer_size > 1) { - auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); - audio_samples = resample_to_16k_fp32(waveform_fp32, 16000); - } else if (!prompt.audio_paths.empty()) { - for (auto it = prompt.messages.rbegin(); it != prompt.messages.rend(); ++it) { - if (!it->audio.empty()) { - const std::string& audio_path = it->audio.back(); - AudioFP32 wav = load_wav(audio_path); - audio_samples = resample_to_16k_fp32(wav.samples, wav.sample_rate); - break; - } - } - } - if (!audio_samples.empty()) { - auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, handle->model->get_config()); - prompt.audio_features = std::move(audio_prep.features); - prompt.audio_num_frames = audio_prep.num_frames; - for (auto it = prompt.messages.rbegin(); it != prompt.messages.rend(); ++it) { - if (it->role == "user") { - it->audio_soft_token_count = audio_prep.num_soft_tokens; - break; - } - } - } - } - - std::string formatted_tools; - if (Config::is_gemma_family(prompt.model_type)) { - formatted_tools = gemma::format_tools(prompt.tools, prompt.model_type == Config::ModelType::GEMMA4); - } else if (prompt.model_type == Config::ModelType::NEEDLE) { - formatted_tools = serialize_needle_tools_json(prompt.tools, tools_json); - } else if (prompt.model_type == Config::ModelType::QWEN || prompt.model_type == Config::ModelType::QWEN3P5) { - formatted_tools = serialize_tools_for_template(prompt.tools); - } else { - formatted_tools = serialize_tools_json(prompt.tools); - } - - if (prompt.model_type == Config::ModelType::NEEDLE) { - std::string query_text = cactus::engine::format_needle_query_text(prompt.messages); - prompt.tokens = tokenizer->encode(query_text); - auto suffix_tokens = encode_needle_tools_suffix(tokenizer, formatted_tools); - prompt.tokens.insert(prompt.tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); - } else { - std::string full_prompt = tokenizer->format_chat_prompt( - prompt.messages, - add_generation_prompt, - formatted_tools, - prompt.options.enable_thinking_if_supported - ); - if (full_prompt.find("ERROR:") == 0) { - throw std::runtime_error(full_prompt.substr(6)); - } - prompt.tokens = tokenizer->encode(full_prompt); - } - prompt.context_token_count = prompt.tokens.size(); - prompt.images = images_from_message(prompt.messages); - return prompt; -} - -PrefillResult do_prefill( - CactusModelHandle* handle, - const PreparedPrompt& prompt, - const std::vector& target_tokens -) { - PrefillResult result = {}; - bool has_images = prompt.has_images(); - - result.was_prefix = prompt_context_matches(handle, prompt); - result.was_exact_match = result.was_prefix && - target_tokens.size() == handle->processed_tokens.size(); - - if (result.was_exact_match) { - return result; - } - - std::vector tokens_to_process; - if (!result.was_prefix) { - reset_cache(handle); - tokens_to_process = target_tokens; - } else { - tokens_to_process.assign( - target_tokens.begin() + handle->processed_tokens.size(), - target_tokens.end() - ); - } - - if (tokens_to_process.size() > 1) { - std::vector prefill_tokens(tokens_to_process.begin(), tokens_to_process.end() - 1); - result.prefilled_count = prefill_tokens.size(); - if (has_images) { - std::vector delta_image_paths; - if (result.was_prefix) { - size_t cached_image_count = 0; - for (const auto& msg_imgs : handle->processed_images) { - cached_image_count += msg_imgs.size(); - } - delta_image_paths.assign( - prompt.image_paths.begin() + cached_image_count, - prompt.image_paths.end() - ); - } else { - delta_image_paths = prompt.image_paths; - } - handle->model->prefill_with_images(prefill_tokens, delta_image_paths); - } else { - handle->model->prefill(prefill_tokens, handle->model->get_prefill_chunk_size()); - } - result.remaining_tokens = {tokens_to_process.back()}; - } else { - result.remaining_tokens = tokens_to_process; - } - - return result; -} - -uint32_t decode( - std::unique_ptr& model, - const std::vector& tokens, - const InferenceOptions& options, - float* out_entropy -) { - return model->decode(tokens, options.temperature, options.top_p, options.top_k, - "", out_entropy, options.min_p, options.repetition_penalty); -} - -uint32_t generate_first_token( - CactusModelHandle* handle, - const PrefillResult& prefill_result, - const PreparedPrompt& prompt, - float* first_token_entropy -) { - if (prefill_result.was_exact_match || prefill_result.remaining_tokens.empty()) { - if (handle->processed_tokens.empty()) { - throw std::runtime_error("Cannot generate from empty prompt"); - } - return decode(handle->model, {handle->processed_tokens.back()}, prompt.options, first_token_entropy); - } - return decode(handle->model, prefill_result.remaining_tokens, prompt.options, first_token_entropy); -} - -std::string construct_prefill_response_json( - bool success, - const std::string* error, - size_t prefill_tokens, - double prefill_tps, - double total_time_ms -) { - std::ostringstream json; - json << "{"; - json << "\"success\":" << (success ? "true" : "false") << ","; - if (error) { - json << "\"error\":\"" << escape_json_string(*error) << "\","; - } else { - json << "\"error\":null,"; - } - json << "\"prefill_tokens\":" << prefill_tokens << ","; - json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - return json.str(); -} - -} // anonymous namespace - -extern "C" { - -int cactus_complete( - cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const char* tools_json, - cactus_token_callback callback, - void* user_data, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (!model) { - std::string error_msg = last_error_message.empty() ? - "Model not initialized. Check model path and files." : last_error_message; - CACTUS_LOG_ERROR("complete", error_msg); - handle_error_response(error_msg, response_buffer, buffer_size); - return -1; - } - - if (!messages_json || !response_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("complete", "Invalid parameters: messages_json, response_buffer, or buffer_size"); - handle_error_response("Invalid parameters", response_buffer, buffer_size); - return -1; - } - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - - auto* handle = static_cast(model); - handle->should_stop = false; - auto* tokenizer = handle->model->get_tokenizer(); - auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, true, true, pcm_buffer, pcm_buffer_size); - - CACTUS_LOG_DEBUG("complete", "Prompt tokens: " << prompt.tokens.size() - << ", max_tokens: " << prompt.options.max_tokens); - - bool has_images = prompt.has_images(); - bool has_audio = prompt.has_audio(); - - auto stop_token_sequences = build_stop_sequences(tokenizer, prompt.options.stop_sequences, prompt.model_type, !prompt.tools.empty()); - - std::vector generated_tokens; - double time_to_first_token = 0.0; - float first_token_entropy = 0.0f; - uint32_t next_token; - size_t prompt_tokens; - - if (has_audio) { - prompt_tokens = prompt.tokens.size(); - next_token = handle->model->decode_with_audio( - prompt.tokens, prompt.audio_features, - prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, - "", &first_token_entropy, - prompt.options.min_p, prompt.options.repetition_penalty); - } else { - auto prefill_result = do_prefill(handle, prompt, prompt.tokens); - prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); - next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); - } - - handle->processed_tokens = prompt.tokens; - handle->processed_images = prompt.images; - - auto token_end = std::chrono::high_resolution_clock::now(); - time_to_first_token = std::chrono::duration_cast(token_end - start_time).count() / 1000.0; - - float confidence = 1.0f - first_token_entropy; - bool cloud_used = false; - std::string cloud_error; - std::future cloud_future; - bool cloud_future_started = false; - const bool cloud_eligible = prompt.options.auto_handoff && (!has_images || prompt.options.handoff_with_images); - - auto maybe_start_cloud_handoff = [&](const std::string& local_output_hint, - const std::vector& local_calls_hint) { - if (!cloud_eligible || cloud_future_started) { - return; - } - CloudCompletionRequest request; - request.messages = prompt.messages; - request.tools = prompt.tools; - request.local_output = local_output_hint; - request.local_function_calls = local_calls_hint; - request.has_images = has_images; - request.cloud_key = resolve_cloud_api_key(nullptr); - - cloud_future_started = true; - cloud_future = std::async(std::launch::async, [request, &prompt]() { - return cloud_complete_request(request, static_cast(prompt.options.cloud_timeout_ms)); - }); - }; - - if (confidence < prompt.options.confidence_threshold) { - maybe_start_cloud_handoff("", {}); - } - - generated_tokens.push_back(next_token); - handle->processed_tokens.push_back(next_token); - - if (prompt.options.force_tools && !prompt.tools.empty()) { - handle->model->update_tool_constraints(next_token); - } - - EntropyState entropy; - entropy.add(first_token_entropy); - - if (!matches_stop_sequence(generated_tokens, stop_token_sequences)) { - if (callback) { - std::string new_text = tokenizer->decode({next_token}); - callback(new_text.c_str(), next_token, user_data); - } - - for (size_t i = 1; i < prompt.options.max_tokens; i++) { - if (handle->should_stop) break; - - float token_entropy = 0.0f; - if (has_audio) { - next_token = handle->model->decode_with_audio( - handle->processed_tokens, prompt.audio_features, - prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, - "", &token_entropy, - prompt.options.min_p, prompt.options.repetition_penalty); - } else { - next_token = decode(handle->model, {next_token}, prompt.options, &token_entropy); - } - handle->processed_tokens.push_back(next_token); - generated_tokens.push_back(next_token); - - entropy.add(token_entropy); - - if (entropy.rolling_confidence() < prompt.options.confidence_threshold) { - entropy.spike_handoff = true; - maybe_start_cloud_handoff("", {}); - } - - if (prompt.options.force_tools && !prompt.tools.empty()) { - handle->model->update_tool_constraints(next_token); - } - - if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { - trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); - break; - } - - if (callback) { - std::string new_text = tokenizer->decode({next_token}); - callback(new_text.c_str(), next_token, user_data); - } - } - } else { - trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); - } - - confidence = entropy.mean_confidence(); - - if (prompt.options.force_tools && !prompt.tools.empty()) { - handle->model->clear_tool_constraints(); - } - - if (prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported && !generated_tokens.empty()) { - strip_thinking_from_cache(handle, generated_tokens, prompt.tokens.size()); - } - - if (prompt.model_type == Config::ModelType::GEMMA4) { - handle->model->compact_kv_cache(); - } - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - size_t completion_tokens = generated_tokens.size(); - double decode_time_ms = total_time_ms - time_to_first_token; - double prefill_tps = time_to_first_token > 0 ? (prompt_tokens * 1000.0) / time_to_first_token : 0.0; - double decode_tps = (completion_tokens > 1 && decode_time_ms > 0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; - - std::string response_text = tokenizer->decode(generated_tokens); - - std::string regular_response; - std::vector function_calls; - parse_function_calls_from_response(response_text, regular_response, function_calls); - - std::string thinking_text; - if (prompt.model_type == Config::ModelType::GEMMA4 || prompt.options.enable_thinking_if_supported) { - std::string stripped_content; - strip_thinking_block(regular_response, thinking_text, stripped_content); - regular_response = stripped_content; - if (!prompt.options.enable_thinking_if_supported) { - thinking_text.clear(); - } - } - - if (confidence < prompt.options.confidence_threshold) { - maybe_start_cloud_handoff(regular_response, function_calls); - } - - std::string local_completion = regular_response; - if (local_completion.empty() && function_calls.empty()) { - local_completion = response_text; - } - std::string primary_response = local_completion; - std::vector primary_function_calls = function_calls; - - if (cloud_future_started) { - auto status = cloud_future.wait_for(std::chrono::milliseconds(prompt.options.cloud_timeout_ms)); - if (status == std::future_status::ready) { - CloudCompletionResult cloud_result = cloud_future.get(); - if (cloud_result.ok && (!cloud_result.response.empty() || !cloud_result.function_calls.empty())) { - cloud_used = true; - if (!cloud_result.response.empty()) { - primary_response = cloud_result.response; - } - if (!cloud_result.function_calls.empty()) { - primary_function_calls = cloud_result.function_calls; - } - } else { - cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; - CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed, falling back to local output: " << cloud_error); - } - } else { - cloud_error = "timeout"; - CACTUS_LOG_WARN("cloud_handoff", "Cloud completion timed out, falling back to local output: " << cloud_error); - } - } - - const bool handoff_succeeded = cloud_used; - std::string result = construct_response_json(primary_response, primary_function_calls, time_to_first_token, - total_time_ms, prefill_tps, decode_tps, prompt_tokens, - completion_tokens, confidence, handoff_succeeded, - thinking_text); - - if (result.length() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, result.c_str()); - - std::string function_calls_json = serialize_function_calls(primary_function_calls); - cactus::telemetry::CompletionMetrics metrics{}; - metrics.success = true; - metrics.cloud_handoff = handoff_succeeded; - metrics.ttft_ms = time_to_first_token; - metrics.prefill_tps = prefill_tps; - metrics.decode_tps = decode_tps; - metrics.response_time_ms = total_time_ms; - metrics.confidence = confidence; - metrics.ram_usage_mb = get_ram_usage_mb(); - metrics.prefill_tokens = prompt_tokens; - metrics.decode_tokens = completion_tokens; - metrics.error_message = nullptr; - metrics.function_calls_json = nullptr; - cactus::telemetry::recordCompletion(handle->model_name.c_str(), metrics); - - return static_cast(result.length()); - - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("complete", "Exception: " << e.what()); - handle_error_response(e.what(), response_buffer, buffer_size); - - cactus::telemetry::CompletionMetrics metrics{}; - metrics.success = false; - metrics.cloud_handoff = false; - metrics.ttft_ms = 0.0; - metrics.prefill_tps = 0.0; - metrics.decode_tps = 0.0; - metrics.response_time_ms = 0.0; - metrics.confidence = 0.0; - metrics.ram_usage_mb = get_ram_usage_mb(); - metrics.prefill_tokens = 0; - metrics.decode_tokens = 0; - metrics.error_message = e.what(); - metrics.function_calls_json = nullptr; - auto* h = static_cast(model); - cactus::telemetry::recordCompletion(h ? h->model_name.c_str() : "unknown", metrics); - - return -1; - } catch (...) { - CACTUS_LOG_ERROR("complete", "Unknown exception during completion"); - handle_error_response("Unknown error during completion", response_buffer, buffer_size); - - cactus::telemetry::CompletionMetrics metrics{}; - metrics.success = false; - metrics.cloud_handoff = false; - metrics.ttft_ms = 0.0; - metrics.prefill_tps = 0.0; - metrics.decode_tps = 0.0; - metrics.response_time_ms = 0.0; - metrics.confidence = 0.0; - metrics.ram_usage_mb = get_ram_usage_mb(); - metrics.prefill_tokens = 0; - metrics.decode_tokens = 0; - metrics.error_message = "Unknown error during completion"; - metrics.function_calls_json = nullptr; - auto* h = static_cast(model); - cactus::telemetry::recordCompletion(h ? h->model_name.c_str() : "unknown", metrics); - - return -1; - } -} - -int cactus_prefill( - cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const char* tools_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (!model) { - std::string error_msg = last_error_message.empty() - ? "Model not initialized. Check model path and files." - : last_error_message; - if (response_buffer && buffer_size > 0) { - std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (result.size() < buffer_size) { - std::strcpy(response_buffer, result.c_str()); - } - } - return -1; - } - - if (!messages_json || !response_buffer || buffer_size == 0) { - std::string error_msg = "Invalid parameters"; - if (response_buffer && buffer_size > 0) { - std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (result.size() < buffer_size) { - std::strcpy(response_buffer, result.c_str()); - } - } - return -1; - } - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - - auto* handle = static_cast(model); - auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, false, false, pcm_buffer, pcm_buffer_size); - - std::vector context_tokens(prompt.tokens.begin(), prompt.tokens.begin() + prompt.context_token_count); - auto prefill_result = do_prefill(handle, prompt, context_tokens); - - if (!prefill_result.was_exact_match) { - handle->processed_tokens = context_tokens; - if (!handle->processed_tokens.empty()) { - handle->processed_tokens.pop_back(); - } - } - handle->processed_images = prompt.images; - - auto end_time = std::chrono::high_resolution_clock::now(); - double elapsed_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - double prefill_tps = (prefill_result.prefilled_count > 0 && elapsed_ms > 0.0) - ? (static_cast(prefill_result.prefilled_count) * 1000.0) / elapsed_ms - : 0.0; - - std::string result = construct_prefill_response_json(true, nullptr, prefill_result.prefilled_count, prefill_tps, elapsed_ms); - if (result.size() >= buffer_size) { - std::string error_msg = "Response buffer too small"; - std::string error_json = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (error_json.size() < buffer_size) { - std::strcpy(response_buffer, error_json.c_str()); - } - return -1; - } - - std::strcpy(response_buffer, result.c_str()); - return static_cast(result.size()); - } catch (const std::exception& e) { - std::string error_msg = e.what(); - std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (result.size() < buffer_size) { - std::strcpy(response_buffer, result.c_str()); - } - return -1; - } catch (...) { - std::string error_msg = "Unknown error during prefill"; - std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (result.size() < buffer_size) { - std::strcpy(response_buffer, result.c_str()); - } - return -1; - } -} - -int cactus_tokenize( - cactus_model_t model, - const char* text, - uint32_t* token_buffer, - size_t token_buffer_len, - size_t* out_token_len -) { - if (!model || !text || !out_token_len) return -1; - - try { - auto* handle = static_cast(model); - auto* tokenizer = handle->model->get_tokenizer(); - - std::vector toks = tokenizer->encode(std::string(text)); - *out_token_len = toks.size(); - - if (!token_buffer || token_buffer_len == 0) return 0; - if (token_buffer_len < toks.size()) return -2; - - std::memcpy(token_buffer, toks.data(), toks.size() * sizeof(uint32_t)); - return 0; - } catch (...) { - return -1; - } -} - -int cactus_decode_tokens( - cactus_model_t model, - const uint32_t* tokens, - size_t token_len, - float temperature, - float top_p, - size_t top_k, - uint32_t* out_token_id -) { - if (!model || !tokens || token_len == 0 || !out_token_id) { - last_error_message = "Invalid parameters"; - return -1; - } - - try { - auto* handle = static_cast(model); - std::vector vec(tokens, tokens + token_len); - handle->model->reset_cache(); - *out_token_id = handle->model->decode(vec, temperature, top_p, top_k); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } catch (...) { - last_error_message = "Unknown error during raw token decode"; - return -1; - } -} - -int cactus_score_window( - cactus_model_t model, - const uint32_t* tokens, - size_t token_len, - size_t start, - size_t end, - size_t context, - char* response_buffer, - size_t buffer_size -) { - if (!model || !tokens || token_len == 0 || !response_buffer || buffer_size == 0) { - handle_error_response("Invalid parameters", response_buffer, buffer_size); - return -1; - } - - try { - auto* handle = static_cast(model); - - std::vector vec(tokens, tokens + token_len); - - size_t scored = 0; - double logprob = handle->model->score_tokens_window_logprob(vec, start, end, context, &scored); - - std::ostringstream oss; - oss << "{" - << "\"success\":true," - << "\"logprob\":" << std::setprecision(10) << logprob << "," - << "\"tokens\":" << scored - << "}"; - - std::string result = oss.str(); - if (result.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, result.c_str()); - return (int)result.size(); - - } catch (const std::exception& e) { - handle_error_response(e.what(), response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus-transpile/ffi/cactus_diarize.cpp b/cactus-transpile/ffi/cactus_diarize.cpp deleted file mode 100644 index 358c7aa37..000000000 --- a/cactus-transpile/ffi/cactus_diarize.cpp +++ /dev/null @@ -1,200 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../models/model.h" -#include "../../libs/audio/wav.h" -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - -static constexpr int DIARIZE_MAX_SPEAKERS = 3; - -struct DiarizeOptions { - size_t step_samples = 16000; - float threshold = -1.0f; - int num_speakers = -1; - int min_speakers = -1; - int max_speakers = -1; - bool raw_powerset = false; -}; - -static DiarizeOptions parse_diarize_options(const std::string& json) { - DiarizeOptions opts; - if (json.empty()) return opts; - - size_t pos; - - pos = json.find("\"step_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - int v = std::stoi(json.substr(pos)); - if (v > 0) opts.step_samples = static_cast(v) * 16; - } - - pos = json.find("\"threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.threshold = std::stof(json.substr(pos)); - } - - pos = json.find("\"num_speakers\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.num_speakers = std::stoi(json.substr(pos)); - } - - pos = json.find("\"min_speakers\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.min_speakers = std::stoi(json.substr(pos)); - } - - pos = json.find("\"max_speakers\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.max_speakers = std::stoi(json.substr(pos)); - } - - pos = json.find("\"raw_powerset\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.raw_powerset = (json.substr(pos, 4) == "true"); - } - - return opts; -} - -static void apply_speaker_filter(std::vector& scores, size_t total_frames, const DiarizeOptions& opts) { - int keep = DIARIZE_MAX_SPEAKERS; - if (opts.num_speakers > 0) keep = std::min(opts.num_speakers, DIARIZE_MAX_SPEAKERS); - if (opts.max_speakers > 0) keep = std::min(keep, opts.max_speakers); - if (opts.min_speakers > 0) keep = std::max(keep, std::min(opts.min_speakers, DIARIZE_MAX_SPEAKERS)); - if (keep >= DIARIZE_MAX_SPEAKERS) return; - - float activity[DIARIZE_MAX_SPEAKERS] = {}; - for (size_t f = 0; f < total_frames; ++f) - for (int s = 0; s < DIARIZE_MAX_SPEAKERS; ++s) - activity[s] += scores[f * DIARIZE_MAX_SPEAKERS + s]; - - int order[DIARIZE_MAX_SPEAKERS] = {0, 1, 2}; - std::sort(order, order + DIARIZE_MAX_SPEAKERS, [&](int a, int b) { return activity[a] > activity[b]; }); - - bool active[DIARIZE_MAX_SPEAKERS] = {}; - for (int i = 0; i < keep; ++i) active[order[i]] = true; - - for (size_t f = 0; f < total_frames; ++f) - for (int s = 0; s < DIARIZE_MAX_SPEAKERS; ++s) - if (!active[s]) scores[f * DIARIZE_MAX_SPEAKERS + s] = 0.0f; -} - -static void apply_threshold(std::vector& scores, float threshold) { - for (float& v : scores) - v = (v >= threshold) ? v : 0.0f; -} - -extern "C" { - -int cactus_diarize( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (validate_audio_params("diarize", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) - return -1; - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - auto* pyannote = dynamic_cast(handle->model.get()); - if (!pyannote) { - CACTUS_LOG_ERROR("diarize", "Model is not a PyAnnote diarization model"); - handle_error_response("Model is not a PyAnnote diarization model", response_buffer, buffer_size); - return -1; - } - - std::vector audio; - if (audio_file_path == nullptr) { - audio = pcm_to_float(pcm_buffer, pcm_buffer_size); - } else { - AudioFP32 wav_audio = load_wav(audio_file_path); - audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); - } - - if (audio.empty()) { - last_error_message = "Failed to load audio or audio is empty"; - CACTUS_LOG_ERROR("diarize", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - const DiarizeOptions opts = parse_diarize_options(options_json ? options_json : ""); - auto scores = pyannote->diarize(audio.data(), audio.size(), opts.step_samples, opts.raw_powerset); - audio = {}; - - const int num_channels = opts.raw_powerset ? 7 : DIARIZE_MAX_SPEAKERS; - const size_t total_frames = scores.size() / num_channels; - - if (!opts.raw_powerset) { - if (opts.num_speakers > 0 || opts.min_speakers > 0 || opts.max_speakers > 0) - apply_speaker_filter(scores, total_frames, opts); - - if (opts.threshold >= 0.0f) - apply_threshold(scores, opts.threshold); - } - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"num_speakers\":" << num_channels << ","; - json << "\"scores\":["; - json << std::fixed << std::setprecision(6); - for (size_t i = 0; i < scores.size(); ++i) { - if (i > 0) json << ","; - json << scores[i]; - } - json << "],"; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - - std::string response = json.str(); - if (response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("diarize", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, response.c_str()); - return static_cast(response.length()); - - } catch (const std::exception& e) { - last_error_message = "Exception during diarization: " + std::string(e.what()); - CACTUS_LOG_ERROR("diarize", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } catch (...) { - last_error_message = "Unknown exception during diarization"; - CACTUS_LOG_ERROR("diarize", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus-transpile/ffi/cactus_embed.cpp b/cactus-transpile/ffi/cactus_embed.cpp deleted file mode 100644 index ba6da2420..000000000 --- a/cactus-transpile/ffi/cactus_embed.cpp +++ /dev/null @@ -1,273 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../../libs/audio/wav.h" -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; -using cactus::audio::WHISPER_SAMPLE_RATE; -using cactus::audio::apply_preemphasis; -using cactus::audio::get_parakeet_spectrogram_config; -using cactus::audio::get_whisper_spectrogram_config; -using cactus::audio::init_whisper_mel_filters; -using cactus::audio::normalize_parakeet_log_mel; -using cactus::audio::normalize_whisper_mel; -using cactus::audio::preprocess_audio_for_gemma4; -using cactus::audio::trim_mel_frames; - -static std::vector compute_mel_from_wav(const std::string& wav_path, bool is_parakeet, size_t mel_bins) { - AudioFP32 audio = load_wav(wav_path); - std::vector waveform_16k = resample_to_16k_fp32(audio.samples, audio.sample_rate); - - auto cfg = is_parakeet ? get_parakeet_spectrogram_config() : get_whisper_spectrogram_config(); - const size_t num_mel_filters = std::max(1, mel_bins); - - AudioProcessor ap; - if (is_parakeet) { - ap.init_mel_filters(cfg.n_fft / 2 + 1, num_mel_filters, 0.0f, 8000.0f, WHISPER_SAMPLE_RATE); - } else { - init_whisper_mel_filters(ap, cfg, num_mel_filters); - } - const size_t waveform_samples = waveform_16k.size(); - if (is_parakeet) { - apply_preemphasis(waveform_16k, 0.97f); - } - std::vector mel = ap.compute_spectrogram(waveform_16k, cfg); - - if (mel.empty()) return mel; - if (is_parakeet) { - normalize_parakeet_log_mel(mel, num_mel_filters); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(mel, num_mel_filters, valid_frames); - return mel; - } - - const bool is_v3 = num_mel_filters > 80; - return normalize_whisper_mel(mel, num_mel_filters, is_v3); -} - -extern "C" { - -int cactus_embed( - cactus_model_t model, - const char* text, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim, - bool normalize -) { - if (!model || !text || !embeddings_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("embed", "Invalid parameters for text embedding"); - return -1; - } - - try { - auto* handle = static_cast(model); - auto* tokenizer = handle->model->get_tokenizer(); - - std::vector tokens = tokenizer->encode(text); - if (tokens.empty()) { - CACTUS_LOG_ERROR("embed", "Tokenization produced empty result"); - return -1; - } - - std::vector embeddings = handle->model->get_embeddings(tokens, true, normalize); - if (embeddings.size() * sizeof(float) > buffer_size) { - CACTUS_LOG_ERROR("embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes, got " << buffer_size); - return -2; - } - - std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); - if (embedding_dim) *embedding_dim = embeddings.size(); - - return static_cast(embeddings.size()); - - } catch (const std::exception& e) { - last_error_message = e.what(); - CACTUS_LOG_ERROR("embed", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during embedding"; - CACTUS_LOG_ERROR("embed", last_error_message); - return -1; - } -} - -int cactus_image_embed( - cactus_model_t model, - const char* image_path, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim -) { - if (!model || !image_path || !embeddings_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("image_embed", "Invalid parameters for image embedding"); - return -1; - } - - try { - auto* handle = static_cast(model); - - CACTUS_LOG_DEBUG("image_embed", "Processing image: " << image_path); - std::vector embeddings = handle->model->get_image_embeddings(image_path); - if (embeddings.empty()) { - CACTUS_LOG_ERROR("image_embed", "Image embedding returned empty result"); - return -1; - } - if (embeddings.size() * sizeof(float) > buffer_size) { - CACTUS_LOG_ERROR("image_embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes"); - return -2; - } - - std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); - if (embedding_dim) *embedding_dim = embeddings.size(); - - return static_cast(embeddings.size()); - - } catch (const std::exception& e) { - last_error_message = e.what(); - CACTUS_LOG_ERROR("image_embed", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during image embedding"; - CACTUS_LOG_ERROR("image_embed", last_error_message); - return -1; - } -} - -int cactus_audio_embed( - cactus_model_t model, - const char* audio_path, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim -) { - if (!model || !audio_path || !embeddings_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("audio_embed", "Invalid parameters for audio embedding"); - return -1; - } - - try { - auto* handle = static_cast(model); - - CACTUS_LOG_DEBUG("audio_embed", "Processing audio: " << audio_path); - const bool is_parakeet = - handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET || - handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - const bool is_gemma4 = - handle->model->get_config().model_type == cactus::engine::Config::ModelType::GEMMA4; - std::vector mel_bins; - if (is_gemma4) { - AudioFP32 audio = load_wav(audio_path); - std::vector waveform_16k = resample_to_16k_fp32(audio.samples, audio.sample_rate); - auto prepared = preprocess_audio_for_gemma4(std::move(waveform_16k), handle->model->get_config()); - mel_bins = std::move(prepared.features); - } else { - mel_bins = compute_mel_from_wav(audio_path, is_parakeet, handle->model->get_config().num_mel_bins); - } - if (mel_bins.empty()) { - last_error_message = "Failed to compute mel spectrogram"; - CACTUS_LOG_ERROR("audio_embed", last_error_message << " for: " << audio_path); - return -1; - } - - std::vector embeddings = handle->model->get_audio_embeddings(mel_bins); - if (embeddings.empty()) { - CACTUS_LOG_ERROR("audio_embed", "Audio embedding returned empty result"); - return -1; - } - if (embeddings.size() * sizeof(float) > buffer_size) { - CACTUS_LOG_ERROR("audio_embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes"); - return -2; - } - - std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); - if (embedding_dim) *embedding_dim = embeddings.size(); - - return static_cast(embeddings.size()); - - } catch (const std::exception& e) { - last_error_message = e.what(); - CACTUS_LOG_ERROR("audio_embed", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during audio embedding"; - CACTUS_LOG_ERROR("audio_embed", last_error_message); - return -1; - } -} - -int cactus_preprocess_audio_features( - const char* audio_path, - const char* model_type, - size_t mel_bins, - float* features_buffer, - size_t buffer_size, - size_t* feature_count, - size_t* out_mel_bins, - size_t* out_frames -) { - if (!audio_path || !model_type || !features_buffer || buffer_size == 0 || mel_bins == 0) { - CACTUS_LOG_ERROR("audio_features", "Invalid parameters for audio feature preprocessing"); - return -1; - } - - try { - std::string type(model_type); - std::transform(type.begin(), type.end(), type.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - const bool is_parakeet = type.find("parakeet") != std::string::npos; - const bool is_gemma4 = type.find("gemma4") != std::string::npos; - - std::vector features; - if (is_gemma4) { - AudioFP32 audio = load_wav(audio_path); - std::vector waveform_16k = resample_to_16k_fp32(audio.samples, audio.sample_rate); - cactus::engine::Config config; - config.audio_input_feat_size = static_cast(mel_bins); - config.audio_fft_length = 512; - auto prepared = preprocess_audio_for_gemma4(std::move(waveform_16k), config); - features = std::move(prepared.features); - } else { - features = compute_mel_from_wav(audio_path, is_parakeet, mel_bins); - } - if (features.empty()) { - last_error_message = "Failed to compute audio features"; - CACTUS_LOG_ERROR("audio_features", last_error_message << " for: " << audio_path); - return -1; - } - if ((features.size() % mel_bins) != 0) { - last_error_message = "Computed audio features have invalid mel/frame shape"; - CACTUS_LOG_ERROR("audio_features", last_error_message); - return -1; - } - - if (feature_count) *feature_count = features.size(); - if (out_mel_bins) *out_mel_bins = mel_bins; - if (out_frames) *out_frames = features.size() / mel_bins; - - const size_t bytes = features.size() * sizeof(float); - if (bytes > buffer_size) { - CACTUS_LOG_ERROR("audio_features", "Buffer too small: need " << bytes << " bytes"); - return -2; - } - - std::memcpy(features_buffer, features.data(), bytes); - return static_cast(features.size()); - } catch (const std::exception& e) { - last_error_message = e.what(); - CACTUS_LOG_ERROR("audio_features", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during audio feature preprocessing"; - CACTUS_LOG_ERROR("audio_features", last_error_message); - return -1; - } -} - -} diff --git a/cactus-transpile/ffi/cactus_embed_speaker.cpp b/cactus-transpile/ffi/cactus_embed_speaker.cpp deleted file mode 100644 index ddb42707e..000000000 --- a/cactus-transpile/ffi/cactus_embed_speaker.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../models/model.h" -#include "../../libs/audio/wav.h" -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; -using cactus::audio::get_wespeaker_spectrogram_config; - -extern "C" { - -int cactus_embed_speaker( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - const float* mask_weights, - size_t mask_num_frames -) { - (void)options_json; - if (validate_audio_params("embed_speaker", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) - return -1; - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - auto* wespeaker = dynamic_cast(handle->model.get()); - if (!wespeaker) { - CACTUS_LOG_ERROR("embed_speaker", "Model is not a WeSpeaker embedding model"); - handle_error_response("Model is not a WeSpeaker embedding model", response_buffer, buffer_size); - return -1; - } - - std::vector audio; - if (audio_file_path == nullptr) { - audio = pcm_to_float(pcm_buffer, pcm_buffer_size); - } else { - AudioFP32 wav_audio = load_wav(audio_file_path); - audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); - } - - if (audio.empty()) { - last_error_message = "Failed to load audio or audio is empty"; - CACTUS_LOG_ERROR("embed_speaker", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - static const auto s_cfg = get_wespeaker_spectrogram_config(); - static AudioProcessor s_ap = []() { - AudioProcessor p; - p.init_mel_filters(get_wespeaker_spectrogram_config().n_fft / 2 + 1, - 80, 20.0f, 8000.0f, 16000, nullptr, "htk"); - return p; - }(); - - std::vector mel = s_ap.compute_spectrogram(audio, s_cfg); - audio = {}; - - static constexpr size_t NUM_MEL_BINS = 80; - const size_t num_frames = mel.size() / NUM_MEL_BINS; - - for (size_t m = 0; m < NUM_MEL_BINS; ++m) { - float* row = mel.data() + m * num_frames; - float sum = 0.0f; - for (size_t t = 0; t < num_frames; ++t) sum += row[t]; - const float mean = sum / static_cast(num_frames); - for (size_t t = 0; t < num_frames; ++t) row[t] -= mean; - } - - std::vector embedding; - if (mask_weights && mask_num_frames > 0) { - embedding = wespeaker->embed(mel.data(), mel.size(), mask_weights, mask_num_frames); - } else { - embedding = wespeaker->embed(mel.data(), mel.size()); - } - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"embedding\":["; - json << std::fixed << std::setprecision(6); - for (size_t i = 0; i < embedding.size(); ++i) { - if (i > 0) json << ","; - json << embedding[i]; - } - json << "],"; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - - std::string response = json.str(); - if (response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("embed_speaker", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, response.c_str()); - return static_cast(response.length()); - - } catch (const std::exception& e) { - last_error_message = "Exception during speaker embedding: " + std::string(e.what()); - CACTUS_LOG_ERROR("embed_speaker", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } catch (...) { - last_error_message = "Unknown exception during speaker embedding"; - CACTUS_LOG_ERROR("embed_speaker", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus-transpile/ffi/cactus_ffi.h b/cactus-transpile/ffi/cactus_ffi.h deleted file mode 100644 index 9a12e3afa..000000000 --- a/cactus-transpile/ffi/cactus_ffi.h +++ /dev/null @@ -1,489 +0,0 @@ -#ifndef CACTUS_FFI_H -#define CACTUS_FFI_H - -#include -#include -#include - -#if __GNUC__ >= 4 - #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) - #define CACTUS_FFI_LOCAL __attribute__((visibility("hidden"))) -#else - #define CACTUS_FFI_EXPORT - #define CACTUS_FFI_LOCAL -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void* cactus_model_t; -typedef void* cactus_index_t; -typedef void* cactus_stream_transcribe_t; - -typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); - -CACTUS_FFI_EXPORT cactus_model_t cactus_init( - const char* model_path, - const char* corpus_dir, // optional: NULL if no RAG corpus - bool cache_index // false = always rebuild index, true = load cached if available -); - -CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); -CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); -CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); - -CACTUS_FFI_EXPORT int cactus_complete( - cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, // optional - const char* tools_json, // optional - cactus_token_callback callback, // optional - void* user_data, // optional - const uint8_t* pcm_buffer, // optional: NULL when not used - size_t pcm_buffer_size // optional: 0 when not used -); - -CACTUS_FFI_EXPORT int cactus_prefill( - cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, // optional - const char* tools_json, // optional - const uint8_t* pcm_buffer, // optional: NULL when not used - size_t pcm_buffer_size // optional: 0 when not used -); - -CACTUS_FFI_EXPORT int cactus_tokenize( - cactus_model_t model, - const char* text, - uint32_t* token_buffer, - size_t token_buffer_len, - size_t* out_token_len -); - -CACTUS_FFI_EXPORT int cactus_decode_tokens( - cactus_model_t model, - const uint32_t* tokens, - size_t token_len, - float temperature, - float top_p, - size_t top_k, - uint32_t* out_token_id -); - -CACTUS_FFI_EXPORT int cactus_score_window( - cactus_model_t model, - const uint32_t* tokens, - size_t token_len, - size_t start, - size_t end, - size_t context, - char* response_buffer, - size_t buffer_size -); - -CACTUS_FFI_EXPORT int cactus_transcribe( - cactus_model_t model, - const char* audio_file_path, // NULL if using pcm_buffer - const char* prompt, - char* response_buffer, - size_t buffer_size, - const char* options_json, // optional - cactus_token_callback callback, // optional - void* user_data, // optional - const uint8_t* pcm_buffer, // NULL if using audio_file_path - size_t pcm_buffer_size -); - -CACTUS_FFI_EXPORT int cactus_detect_language( - cactus_model_t model, - const char* audio_file_path, // NULL if using pcm_buffer - char* response_buffer, - size_t buffer_size, - const char* options_json, // optional - const uint8_t* pcm_buffer, // NULL if using audio_file_path - size_t pcm_buffer_size -); - -CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_start( - cactus_model_t model, - const char* options_json // optional -); - -CACTUS_FFI_EXPORT int cactus_stream_transcribe_process( - cactus_stream_transcribe_t stream, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - char* response_buffer, - size_t buffer_size -); - -CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop( - cactus_stream_transcribe_t stream, - char* response_buffer, - size_t buffer_size -); - -CACTUS_FFI_EXPORT int cactus_embed( - cactus_model_t model, - const char* text, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim, - bool normalize -); - -CACTUS_FFI_EXPORT int cactus_image_embed( - cactus_model_t model, - const char* image_path, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim -); - -CACTUS_FFI_EXPORT int cactus_audio_embed( - cactus_model_t model, - const char* audio_path, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim -); - -CACTUS_FFI_EXPORT int cactus_preprocess_audio_features( - const char* audio_path, - const char* model_type, - size_t mel_bins, - float* features_buffer, - size_t buffer_size, - size_t* feature_count, - size_t* out_mel_bins, - size_t* out_frames -); - -CACTUS_FFI_EXPORT int cactus_vad( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -); - -CACTUS_FFI_EXPORT int cactus_diarize( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -); - -CACTUS_FFI_EXPORT int cactus_embed_speaker( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - const float* mask_weights, - size_t mask_num_frames -); - -CACTUS_FFI_EXPORT int cactus_rag_query( - cactus_model_t model, - const char* query, - char* response_buffer, - size_t buffer_size, - size_t top_k -); - -CACTUS_FFI_EXPORT cactus_index_t cactus_index_init( - const char* index_dir, - size_t embedding_dim -); - -CACTUS_FFI_EXPORT int cactus_index_add( - cactus_index_t index, - const int* ids, - const char** documents, - const char** metadatas, // optional: can be NULL - const float** embeddings, - size_t count, - size_t embedding_dim -); - -CACTUS_FFI_EXPORT int cactus_index_delete( - cactus_index_t index, - const int* ids, - size_t ids_count -); - -CACTUS_FFI_EXPORT int cactus_index_get( - cactus_index_t index, - const int* ids, - size_t ids_count, - char** document_buffers, - size_t* document_buffer_sizes, - char** metadata_buffers, - size_t* metadata_buffer_sizes, - float** embedding_buffers, - size_t* embedding_buffer_sizes -); - -CACTUS_FFI_EXPORT int cactus_index_query( - cactus_index_t index, - const float** embeddings, - size_t embeddings_count, - size_t embedding_dim, - const char* options_json, // optional - int** id_buffers, - size_t* id_buffer_sizes, - float** score_buffers, - size_t* score_buffer_sizes -); - -CACTUS_FFI_EXPORT int cactus_index_compact(cactus_index_t index); -CACTUS_FFI_EXPORT void cactus_index_destroy(cactus_index_t index); - -CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); - -// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE -CACTUS_FFI_EXPORT void cactus_log_set_level(int level); - -typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); -CACTUS_FFI_EXPORT void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); - -CACTUS_FFI_EXPORT void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); -CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); -CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); -CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); - -// cactus graph export -typedef void* cactus_graph_t; -typedef uint64_t cactus_node_t; - -typedef struct { - int32_t precision; - size_t rank; - size_t shape[8]; - size_t num_elements; - size_t byte_size; -} cactus_tensor_info_t; - -CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); -CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); -CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); - -CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); -CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); - -CACTUS_FFI_EXPORT int cactus_graph_input( - cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, -cactus_node_t* out_node); - -CACTUS_FFI_EXPORT int cactus_graph_set_input( - cactus_graph_t graph, cactus_node_t node, const void* data, int32_t -precision); -CACTUS_FFI_EXPORT int cactus_graph_set_external_input( - cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); - -CACTUS_FFI_EXPORT int cactus_graph_precision_cast( - cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_quantize_activations( - cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, -cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, -cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_not_equal(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_floor_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_not_equal(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_masked_select_prefix(cactus_graph_t graph, cactus_node_t input, cactus_node_t mask, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_masked_scatter(cactus_graph_t graph, cactus_node_t input, cactus_node_t mask, cactus_node_t source, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, -float exponent, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_view( - cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_flatten( - cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_reshape( - cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_expand( - cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_transpose( - cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_transpose_n( - cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_slice( - cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_index( - cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_concat( - cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, -cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_cat( - cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t -axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_matmul( - cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, int32_t output_precision, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gather( - cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( - cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( - cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( - cactus_graph_t graph, const char* filename, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( - cactus_graph_t graph, const char* filename, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( - cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_set_grouped_scales( - cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr); -CACTUS_FFI_EXPORT int cactus_graph_set_interleaved( - cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n); -CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); -CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); -CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); - -CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_layernorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_groupnorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_batchnorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rms_norm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rope( - cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( - cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_attention( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( - cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s1p1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, - size_t stride, size_t padding, size_t dilation, size_t groups, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( - cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_stft( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_altup_predict( - cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_altup_correct( - cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( - cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( - cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( - cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_sample( - cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( - cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_persistent( - cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_is_populated( - cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); -CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( - cactus_graph_t graph, cactus_node_t persistent_node); - -CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); -CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, -cactus_node_t node, void** out_ptr); -CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, -cactus_node_t node, cactus_tensor_info_t* out_info); - -#ifdef __cplusplus -} -#endif - -#endif // CACTUS_FFI_H diff --git a/cactus-transpile/ffi/cactus_graph.cpp b/cactus-transpile/ffi/cactus_graph.cpp deleted file mode 100644 index 5590e38aa..000000000 --- a/cactus-transpile/ffi/cactus_graph.cpp +++ /dev/null @@ -1,1432 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../graph/graph.h" -#include -#include - -using namespace cactus::ffi; - -struct GraphHandle { - CactusGraph graph; -}; - -static GraphHandle* as_graph(cactus_graph_t g) { - return reinterpret_cast(g); -} - -static int fail_invalid(const char* msg) { - last_error_message = msg; - return -1; -} - -#define GRAPH_WRAP_UNARY(fn_name, method_name) \ -int fn_name(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { \ - if (!graph || !out) return fail_invalid("Invalid args to " #fn_name); \ - try { \ - *out = static_cast(as_graph(graph)->graph.method_name(static_cast(x))); \ - return 0; \ - } catch (const std::exception& e) { \ - last_error_message = e.what(); \ - return -1; \ - } \ -} - -#define GRAPH_WRAP_BINARY(fn_name, method_name) \ -int fn_name(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { \ - if (!graph || !out) return fail_invalid("Invalid args to " #fn_name); \ - try { \ - *out = static_cast(as_graph(graph)->graph.method_name(static_cast(a), static_cast(b))); \ - return 0; \ - } catch (const std::exception& e) { \ - last_error_message = e.what(); \ - return -1; \ - } \ -} - -extern "C" { - -cactus_graph_t cactus_graph_create(void) { - try { - return reinterpret_cast(new GraphHandle()); - } catch (const std::exception& e) { - last_error_message = e.what(); - return nullptr; - } catch (...) { - last_error_message = "Unknown error creating graph"; - return nullptr; - } -} - -void cactus_graph_destroy(cactus_graph_t graph) { - delete as_graph(graph); -} - -int cactus_graph_hard_reset(cactus_graph_t graph) { - if (!graph) { - last_error_message = "Invalid args to cactus_graph_hard_reset"; - return -1; - } - try { - as_graph(graph)->graph.hard_reset(); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_save(cactus_graph_t graph, const char* filename) { - if (!graph || !filename) { - last_error_message = "Invalid args to cactus_graph_save"; - return -1; - } - try { - as_graph(graph)->graph.save(filename); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - - -cactus_graph_t cactus_graph_load(const char* filename) { - if (!filename) { - last_error_message = "Invalid args to cactus_graph_load"; - return nullptr; - } - try { - auto handle = std::make_unique(); - handle->graph = CactusGraph::load(std::string(filename)); - return reinterpret_cast(handle.release()); - } catch (const std::exception& e) { - last_error_message = e.what(); - return nullptr; - } catch (...) { - last_error_message = "Unknown error loading graph"; - return nullptr; - } -} - -int cactus_graph_input(cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, cactus_node_t* out_node) { - if (!graph || !shape || rank == 0 || !out_node) { - last_error_message = "Invalid args to cactus_graph_input"; - return -1; - } - try { - std::vector s(shape, shape + rank); - auto id = as_graph(graph)->graph.input(s, static_cast(precision)); - *out_node = static_cast(id); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_set_input(cactus_graph_t graph, cactus_node_t node, const void* data, int32_t precision) { - if (!graph || !data) { - last_error_message = "Invalid args to cactus_graph_set_input"; - return -1; - } - try { - as_graph(graph)->graph.set_input(static_cast(node), data, static_cast(precision)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_set_external_input(cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision) { - if (!graph || !data) { - return fail_invalid("Invalid args to cactus_graph_set_external_input"); - } - try { - as_graph(graph)->graph.set_external_input(static_cast(node), data, static_cast(precision)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_precision_cast(cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out) { - if (!graph || !out) { - return fail_invalid("Invalid args to cactus_graph_precision_cast"); - } - try { - *out = static_cast(as_graph(graph)->graph.precision_cast(static_cast(input), static_cast(target_precision))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_quantize_activations(cactus_graph_t graph, cactus_node_t input, cactus_node_t* out) { - if (!graph || !out) { - return fail_invalid("Invalid args to cactus_graph_quantize_activations"); - } - try { - *out = static_cast(as_graph(graph)->graph.quantize_activations(static_cast(input))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_add"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.add(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_add_clipped"); - try { - *out = static_cast(as_graph(graph)->graph.add_clipped(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_subtract"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.subtract(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_multiply"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.multiply(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_divide(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_divide"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.divide(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_not_equal(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_not_equal"); - try { - *out = static_cast(as_graph(graph)->graph.not_equal(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_add"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_add(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_subtract"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_subtract(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_multiply"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_multiply(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_divide"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_divide(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_floor_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_floor_divide"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_floor_divide(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_not_equal(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_not_equal"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_not_equal(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_masked_select_prefix(cactus_graph_t graph, cactus_node_t input, cactus_node_t mask, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_masked_select_prefix"); - try { - *out = static_cast( - as_graph(graph)->graph.masked_select_prefix(static_cast(input), static_cast(mask)) - ); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_masked_scatter( - cactus_graph_t graph, - cactus_node_t input, - cactus_node_t mask, - cactus_node_t source, - cactus_node_t* out -) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_masked_scatter"); - try { - *out = static_cast( - as_graph(graph)->graph.masked_scatter( - static_cast(input), - static_cast(mask), - static_cast(source) - ) - ); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_exp"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_exp(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_sqrt"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_sqrt(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_cos"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_cos(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_sin"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_sin(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_log"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_log(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_abs"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.abs(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, float exponent, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_pow"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.pow(static_cast(x), exponent)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_view(cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out) { - if (!graph || !shape || rank == 0 || !out) { - last_error_message = "Invalid args to cactus_graph_view"; - return -1; - } - try { - std::vector s(shape, shape + rank); - *out = static_cast(as_graph(graph)->graph.view(static_cast(x), s)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_flatten(cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_flatten"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.flatten(static_cast(x), start_dim, end_dim)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_reshape(cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out) { - if (!graph || !shape || rank == 0 || !out) return fail_invalid("Invalid args to cactus_graph_reshape"); - try { - std::vector s(shape, shape + rank); - *out = static_cast(as_graph(graph)->graph.reshape(static_cast(x), s)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_expand(cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out) { - if (!graph || !shape || rank == 0 || !out) return fail_invalid("Invalid args to cactus_graph_expand"); - try { - std::vector s(shape, shape + rank); - *out = static_cast(as_graph(graph)->graph.expand(static_cast(x), s)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_transpose(cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_transpose"); - try { - *out = static_cast(as_graph(graph)->graph.transpose(static_cast(x), static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_transpose_n(cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out) { - if (!graph || !permutation || rank == 0 || !out) return fail_invalid("Invalid args to cactus_graph_transpose_n"); - try { - std::vector p(permutation, permutation + rank); - *out = static_cast(as_graph(graph)->graph.transposeN(static_cast(x), p, static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_slice(cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_slice"); - try { - *out = static_cast(as_graph(graph)->graph.slice(static_cast(x), axis, start, length)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_index(cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_index"); - try { - *out = static_cast(as_graph(graph)->graph.index(static_cast(x), index_value, dim)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sum"); - try { - *out = static_cast(as_graph(graph)->graph.sum(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_mean"); - try { - *out = static_cast(as_graph(graph)->graph.mean(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_variance"); - try { - *out = static_cast(as_graph(graph)->graph.variance(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_min"); - try { - *out = static_cast(as_graph(graph)->graph.min(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_max"); - try { - *out = static_cast(as_graph(graph)->graph.max(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_cumsum"); - try { - *out = static_cast(as_graph(graph)->graph.cumsum(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_concat(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_concat"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.concat(static_cast(a), static_cast(b), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_cat(cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t axis, cactus_node_t* out) { - if (!graph || !nodes || !out || count == 0) { - last_error_message = "Invalid args to cactus_graph_cat"; - return -1; - } - try { - std::vector ns; - for (size_t i = 0; i < count; ++i) { - ns.push_back(static_cast(nodes[i])); - } - *out = static_cast(as_graph(graph)->graph.cat(ns, axis)); - return 0; - - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_matmul( - cactus_graph_t graph, - cactus_node_t a, - cactus_node_t b, - bool pretransposed_rhs, - int32_t backend, - int32_t output_precision, - cactus_node_t* out -) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_matmul"); - try { - *out = static_cast( - as_graph(graph)->graph.matmul( - static_cast(a), - static_cast(b), - pretransposed_rhs, - static_cast(backend), - static_cast(output_precision) - ) - ); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gather(cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gather"); - try { - *out = static_cast(as_graph(graph)->graph.gather( - static_cast(tensor), - static_cast(indices), - axis - )); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_embedding_from_tensor(cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_embedding_from_tensor"); - try { - *out = static_cast(as_graph(graph)->graph.embedding(static_cast(embedding_tensor), static_cast(indices))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_embedding_from_file(cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out) { - if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_embedding_from_file"); - try { - *out = static_cast(as_graph(graph)->graph.embedding(std::string(filename), static_cast(indices))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_mmap_embeddings(cactus_graph_t graph, const char* filename, cactus_node_t* out) { - if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_mmap_embeddings"); - try { - *out = static_cast(as_graph(graph)->graph.mmap_embeddings(std::string(filename))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_mmap_weights(cactus_graph_t graph, const char* filename, cactus_node_t* out) { - if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_mmap_weights"); - try { - *out = static_cast(as_graph(graph)->graph.mmap_weights(std::string(filename))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_bilinear_interpolation(cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_bilinear_interpolation"); - try { - *out = static_cast(as_graph(graph)->graph.bilinear_interpolation(static_cast(pos_embeds), dst_height, dst_width)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_set_grouped_scales(cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr) { - if (!graph || !scales_ptr) return fail_invalid("Invalid args to cactus_graph_set_grouped_scales"); - try { - as_graph(graph)->graph.set_grouped_scales(static_cast(node), group_size, num_groups, scales_ptr); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_set_interleaved(cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_set_interleaved"); - try { - as_graph(graph)->graph.set_interleaved(static_cast(node), interleaved, original_n); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_release_weight_pages"); - try { - as_graph(graph)->graph.release_weight_pages(static_cast(node)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_prefetch_weight_pages"); - try { - as_graph(graph)->graph.prefetch_weight_pages(static_cast(node)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_release_all_weight_pages(cactus_graph_t graph) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_release_all_weight_pages"); - try { - as_graph(graph)->graph.release_all_weight_pages(); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_relu"); - try { - *out = static_cast(as_graph(graph)->graph.relu(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_silu"); - try { - *out = static_cast(as_graph(graph)->graph.silu(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gelu"); - try { - *out = static_cast(as_graph(graph)->graph.gelu(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gelu_erf"); - try { - *out = static_cast(as_graph(graph)->graph.gelu_erf(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sigmoid"); - try { - *out = static_cast(as_graph(graph)->graph.sigmoid(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_tanh"); - try { - *out = static_cast(as_graph(graph)->graph.tanh(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_glu"); - try { - *out = static_cast(as_graph(graph)->graph.glu(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_layernorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_layernorm"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.layernorm(static_cast(input), static_cast(weight), static_cast(bias), epsilon)); - } else { - *out = static_cast(as_graph(graph)->graph.layernorm(static_cast(input), static_cast(weight), epsilon)); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_groupnorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_groupnorm"); - try { - *out = static_cast(as_graph(graph)->graph.groupnorm(static_cast(input), static_cast(weight), static_cast(bias), num_groups, epsilon)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_batchnorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_batchnorm"); - try { - *out = static_cast(as_graph(graph)->graph.batchnorm(static_cast(input), static_cast(weight), static_cast(bias), static_cast(running_mean), static_cast(running_var), axis, epsilon)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_topk"); - try { - *out = static_cast(as_graph(graph)->graph.topk(static_cast(input), k)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_rms_norm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rms_norm"); - try { - *out = static_cast(as_graph(graph)->graph.rms_norm(static_cast(input), static_cast(weight), epsilon)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_rope(cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rope"); - try { - *out = static_cast(as_graph(graph)->graph.rope(static_cast(input), theta, position_offset, static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_rope_gptj(cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rope_gptj"); - try { - *out = static_cast(as_graph(graph)->graph.rope_gptj(static_cast(input), theta, position_offset, rot_dim, static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_softmax"); - try { - *out = static_cast(as_graph(graph)->graph.softmax(static_cast(input), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_attention(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_attention"); - try { - if (use_mask) { - *out = static_cast(as_graph(graph)->graph.attention_masked(static_cast(query), static_cast(key), static_cast(value), static_cast(mask), scale, is_causal, static_cast(backend), additive_mask, position_offset, window_size)); - } else if (window_size > 0 || position_offset > 0) { - *out = static_cast(as_graph(graph)->graph.attention(static_cast(query), static_cast(key), static_cast(value), scale, position_offset, window_size, static_cast(backend))); - } else { - *out = static_cast(as_graph(graph)->graph.attention(static_cast(query), static_cast(key), static_cast(value), scale, is_causal, static_cast(backend))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_rel_pos_bias(cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rel_pos_bias"); - try { - *out = static_cast(as_graph(graph)->graph.rel_pos_bias(static_cast(query), static_cast(relative_key), scale)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_attention_int8_hybrid(cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_attention_int8_hybrid"); - try { - *out = static_cast(as_graph(graph)->graph.attention_int8_hybrid(static_cast(query), static_cast(key_new), static_cast(value_new), scale, position_offset, cached_keys, cached_values, k_scales, v_scales, cache_len, num_kv_heads, head_dim, window_size)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_causal(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_causal"); - try { - *out = static_cast(as_graph(graph)->graph.conv1d_causal(static_cast(input), static_cast(weight), kernel_size, dilation)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_k3(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_k3"); - try { - *out = static_cast(as_graph(graph)->graph.conv1d_k3(static_cast(input), static_cast(weight), stride)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_k7s3(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_k7s3"); - try { - *out = static_cast(as_graph(graph)->graph.conv1d_k7s3(static_cast(input), static_cast(weight), static_cast(bias))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv1d(static_cast(input), static_cast(weight), static_cast(bias), stride)); - } else { - *out = static_cast(as_graph(graph)->graph.conv1d(static_cast(input), static_cast(weight), stride)); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_same_depthwise_k9(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_same_depthwise_k9"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv1d_same_depthwise_k9(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv1d_same_depthwise_k9(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_pointwise(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_pointwise"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv1d_pointwise(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv1d_pointwise(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv2d_k3s2p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_k3s2p1"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv2d_k3s2p1(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv2d_k3s2p1(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv2d_depthwise_k3s2p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_depthwise_k3s2p1"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv2d_depthwise_k3s2p1(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv2d_depthwise_k3s2p1(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv2d_pointwise_1x1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_pointwise_1x1"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv2d_pointwise_1x1(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv2d_pointwise_1x1(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv2d_k3s1p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_k3s1p1"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv2d_k3s1p1(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv2d_k3s1p1(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv2d( - cactus_graph_t graph, - cactus_node_t input, - cactus_node_t weight, - bool has_bias, - cactus_node_t bias, - size_t stride, - size_t padding, - size_t dilation, - size_t groups, - cactus_node_t* out -) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d"); - try { - if (has_bias) { - *out = static_cast( - as_graph(graph)->graph.conv2d( - static_cast(input), - static_cast(weight), - static_cast(bias), - stride, - padding, - dilation, - groups - ) - ); - } else { - *out = static_cast( - as_graph(graph)->graph.conv2d( - static_cast(input), - static_cast(weight), - stride, - padding, - dilation, - groups - ) - ); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_lstm_cell(cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_lstm_cell"); - try { - *out = static_cast(as_graph(graph)->graph.lstm_cell(static_cast(input), static_cast(h_prev), static_cast(c_prev), static_cast(weight_ih), static_cast(weight_hh), static_cast(bias_ih), static_cast(bias_hh))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gated_deltanet_decode(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gated_deltanet_decode"); - try { - *out = static_cast(as_graph(graph)->graph.gated_deltanet_decode(static_cast(query), static_cast(key), static_cast(value), static_cast(gate_log), static_cast(beta), static_cast(initial_state), scale)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gated_deltanet_prefill(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gated_deltanet_prefill"); - try { - *out = static_cast(as_graph(graph)->graph.gated_deltanet_prefill(static_cast(query), static_cast(key), static_cast(value), static_cast(gate_log), static_cast(beta), static_cast(initial_state), chunk_size, scale)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_stft(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_stft"); - try { - *out = static_cast(as_graph(graph)->graph.stft(static_cast(input), static_cast(weight), stride, num_fft_bins)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_altup_predict(cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out) { - if (!graph || !streams || num_streams == 0 || !out) return fail_invalid("Invalid args to cactus_graph_altup_predict"); - try { - std::vector s(num_streams); - for (size_t i = 0; i < num_streams; ++i) s[i] = static_cast(streams[i]); - *out = static_cast(as_graph(graph)->graph.altup_predict(static_cast(coefs), s.data(), num_streams)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_altup_correct(cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out) { - if (!graph || !predictions || num_predictions == 0 || !out) return fail_invalid("Invalid args to cactus_graph_altup_correct"); - try { - std::vector p(num_predictions); - for (size_t i = 0; i < num_predictions; ++i) p[i] = static_cast(predictions[i]); - *out = static_cast(as_graph(graph)->graph.altup_correct(static_cast(coefs), static_cast(innovation), p.data(), num_predictions)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gaussian_topk(cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gaussian_topk"); - try { - *out = static_cast(as_graph(graph)->graph.gaussian_topk(static_cast(input), ppf)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_moe_layer_gated(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out) { - if (!graph || !w1_weights || !w3_weights || !w2_weights || !out) return fail_invalid("Invalid args to cactus_graph_moe_layer_gated"); - try { - std::vector w1(num_experts), w3(num_experts), w2(num_experts); - for (size_t i = 0; i < num_experts; ++i) { - w1[i] = static_cast(w1_weights[i]); - w3[i] = static_cast(w3_weights[i]); - w2[i] = static_cast(w2_weights[i]); - } - *out = static_cast(as_graph(graph)->graph.moe_layer(static_cast(hidden), static_cast(routing_probs), static_cast(topk_indices), w1, w3, w2, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_moe_layer_ungated(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out) { - if (!graph || !w1_weights || !w2_weights || !out) return fail_invalid("Invalid args to cactus_graph_moe_layer_ungated"); - try { - std::vector w1(num_experts), w2(num_experts); - for (size_t i = 0; i < num_experts; ++i) { - w1[i] = static_cast(w1_weights[i]); - w2[i] = static_cast(w2_weights[i]); - } - *out = static_cast(as_graph(graph)->graph.moe_layer(static_cast(hidden), static_cast(routing_probs), static_cast(topk_indices), w1, w2, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor, static_cast(activation))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_sample(cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sample"); - try { - std::unordered_map empty_bias; - *out = static_cast(as_graph(graph)->graph.sample(static_cast(logits), temperature, top_p, top_k, empty_bias)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scatter_topk(cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scatter_topk"); - try { - *out = static_cast(as_graph(graph)->graph.scatter_topk(static_cast(indices), static_cast(values), num_classes)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_persistent(cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_persistent"); - try { - *out = static_cast(as_graph(graph)->graph.persistent(static_cast(source_node))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_is_populated(cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated) { - if (!graph || !out_is_populated) return fail_invalid("Invalid args to cactus_graph_is_populated"); - try { - *out_is_populated = as_graph(graph)->graph.is_populated(static_cast(persistent_node)) ? 1 : 0; - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_invalidate_persistent(cactus_graph_t graph, cactus_node_t persistent_node) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_invalidate_persistent"); - try { - as_graph(graph)->graph.invalidate_persistent(static_cast(persistent_node)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_execute(cactus_graph_t graph) { - if (!graph) { - last_error_message = "Graph is null"; - return -1; - } - try { - as_graph(graph)->graph.execute(); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_get_output_ptr(cactus_graph_t graph, cactus_node_t node, void** out_ptr) { - if (!graph || !out_ptr) { - last_error_message = "Invalid args to cactus_graph_get_output_ptr"; - return -1; - } - try { - *out_ptr = as_graph(graph)->graph.get_output(static_cast(node)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_get_output_info(cactus_graph_t graph, cactus_node_t node, cactus_tensor_info_t* out_info) { - if (!graph || !out_info) { - last_error_message = "Invalid args to cactus_graph_get_output_info"; - return -1; - } - try { - const auto& buf = as_graph(graph)->graph.get_output_buffer(static_cast(node)); - out_info->precision = static_cast(buf.precision); - out_info->rank = buf.shape.size(); - if (out_info->rank > 8) { - last_error_message = "Rank exceeds cactus_tensor_info_t shape capacity"; - return -1; - } - for (size_t i = 0; i < out_info->rank; ++i) { - out_info->shape[i] = buf.shape[i]; - } - for (size_t i = out_info->rank; i < 8; ++i) { - out_info->shape[i] = 0; - } - out_info->num_elements = buf.total_size; - out_info->byte_size = buf.byte_size; - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} -} diff --git a/cactus-transpile/ffi/cactus_index.cpp b/cactus-transpile/ffi/cactus_index.cpp deleted file mode 100644 index a81f62377..000000000 --- a/cactus-transpile/ffi/cactus_index.cpp +++ /dev/null @@ -1,458 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include - -using namespace cactus::ffi; - -struct CactusIndexHandle { - std::unique_ptr index; -}; - -static cactus::engine::index::QueryOptions parse_query_options_json(const std::string& json) { - cactus::engine::index::QueryOptions options; - - if (json.empty()) return options; - - size_t pos = json.find("\"top_k\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.top_k = std::stoul(json.substr(pos)); - } - - pos = json.find("\"score_threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.score_threshold = std::stof(json.substr(pos)); - } - - return options; -} - -extern "C" { - -cactus_index_t cactus_index_init(const char* index_dir, size_t embedding_dim) { - if (!index_dir) { - last_error_message = "Index directory path cannot be null"; - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } - - if (embedding_dim == 0) { - last_error_message = "Embedding dimension must be greater than 0"; - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } - - std::string dir_path(index_dir); - if (dir_path.empty()) { - last_error_message = "Index directory path cannot be empty"; - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } - - std::string index_path_str = dir_path + "/index.bin"; - std::string data_path_str = dir_path + "/data.bin"; - - CACTUS_LOG_INFO("index_init", "Initializing index in directory: " << dir_path << ", dim: " << embedding_dim); - - try { - auto* handle = new CactusIndexHandle(); - handle->index = std::make_unique( - index_path_str, - data_path_str, - embedding_dim - ); - - if (!handle->index) { - last_error_message = "Failed to create index instance"; - CACTUS_LOG_ERROR("index_init", last_error_message); - - delete handle; - return nullptr; - } - - CACTUS_LOG_INFO("index_init", "Index initialized successfully"); - return handle; - - } catch (const std::exception& e) { - last_error_message = "Exception during index init: " + std::string(e.what()); - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } catch (...) { - last_error_message = "Unknown exception during index initialization"; - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } -} - -int cactus_index_add( - cactus_index_t index, - const int* ids, - const char** documents, - const char** metadatas, - const float** embeddings, - size_t count, - size_t embedding_dim -) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - if (!ids || !documents || !embeddings) { - last_error_message = "Invalid parameters: ids, documents, or embeddings is null"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - if (count == 0) { - last_error_message = "Invalid parameter: count must be greater than 0"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - if (embedding_dim == 0) { - last_error_message = "Invalid parameter: embedding_dim must be greater than 0"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - std::vector docs; - docs.reserve(count); - - for (size_t i = 0; i < count; ++i) { - if (!embeddings[i]) { - last_error_message = "Embedding at index " + std::to_string(i) + " is null"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - docs.emplace_back( - ids[i], - std::vector(embeddings[i], embeddings[i] + embedding_dim), - documents[i] ? std::string(documents[i]) : "", - metadatas && metadatas[i] ? std::string(metadatas[i]) : "" - ); - } - - CACTUS_LOG_INFO("index_add", "Adding " << docs.size() << " documents"); - handle->index->add_documents(docs); - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_add", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during add"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } -} - -int cactus_index_delete( - cactus_index_t index, - const int* ids, - size_t ids_count -) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_delete", last_error_message); - return -1; - } - - if (!ids || ids_count == 0) { - last_error_message = "Invalid parameters"; - CACTUS_LOG_ERROR("index_delete", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - std::vector doc_ids(ids, ids + ids_count); - - CACTUS_LOG_INFO("index_delete", "Deleting " << doc_ids.size() << " documents"); - handle->index->delete_documents(doc_ids); - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_delete", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during delete"; - CACTUS_LOG_ERROR("index_delete", last_error_message); - return -1; - } -} - -int cactus_index_get( - cactus_index_t index, - const int* ids, - size_t ids_count, - char** document_buffers, - size_t* document_buffer_sizes, - char** metadata_buffers, - size_t* metadata_buffer_sizes, - float** embedding_buffers, - size_t* embedding_buffer_sizes -) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (!ids) { - last_error_message = "Invalid parameter: ids is null"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (ids_count == 0) { - last_error_message = "Invalid parameter: ids_count must be greater than 0"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (document_buffers && !document_buffer_sizes) { - last_error_message = "Invalid parameters: document_buffers provided but document_buffer_sizes is null"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (metadata_buffers && !metadata_buffer_sizes) { - last_error_message = "Invalid parameters: metadata_buffers provided but metadata_buffer_sizes is null"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (embedding_buffers && !embedding_buffer_sizes) { - last_error_message = "Invalid parameters: embedding_buffers provided but embedding_buffer_sizes is null"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - std::vector doc_ids(ids, ids + ids_count); - - CACTUS_LOG_INFO("index_get", "Getting " << doc_ids.size() << " documents"); - std::vector docs = handle->index->get_documents(doc_ids); - - for (size_t i = 0; i < docs.size(); ++i) { - if (document_buffers) { - size_t document_size = docs[i].content.size() + 1; - if (document_buffer_sizes[i] < document_size) { - last_error_message = "Document buffer too small at index " + std::to_string(i) + - " (required: " + std::to_string(document_size) + - ", provided: " + std::to_string(document_buffer_sizes[i]) + ")"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - } - - if (metadata_buffers) { - size_t metadata_size = docs[i].metadata.size() + 1; - if (metadata_buffer_sizes[i] < metadata_size) { - last_error_message = "Metadata buffer too small at index " + std::to_string(i) + - " (required: " + std::to_string(metadata_size) + - ", provided: " + std::to_string(metadata_buffer_sizes[i]) + ")"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - } - - if (embedding_buffers) { - size_t embedding_size = docs[i].embedding.size(); - if (embedding_buffer_sizes[i] < embedding_size) { - last_error_message = "Embedding buffer too small at index " + std::to_string(i) + - " (required: " + std::to_string(embedding_size) + - ", provided: " + std::to_string(embedding_buffer_sizes[i]) + ")"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - } - } - - for (size_t i = 0; i < docs.size(); ++i) { - if (document_buffers) { - size_t document_size = docs[i].content.size() + 1; - std::memcpy(document_buffers[i], docs[i].content.c_str(), document_size); - document_buffer_sizes[i] = document_size; - } - - if (metadata_buffers) { - size_t metadata_size = docs[i].metadata.size() + 1; - std::memcpy(metadata_buffers[i], docs[i].metadata.c_str(), metadata_size); - metadata_buffer_sizes[i] = metadata_size; - } - - if (embedding_buffers) { - size_t embedding_size = docs[i].embedding.size(); - std::memcpy(embedding_buffers[i], docs[i].embedding.data(), embedding_size * sizeof(float)); - embedding_buffer_sizes[i] = embedding_size; - } - } - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_get", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during get"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } -} - -int cactus_index_query( - cactus_index_t index, - const float** embeddings, - size_t embeddings_count, - size_t embedding_dim, - const char* options_json, - int** id_buffers, - size_t* id_buffer_sizes, - float** score_buffers, - size_t* score_buffer_sizes -) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (!embeddings) { - last_error_message = "Invalid parameter: embeddings is null"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (!id_buffers || !score_buffers) { - last_error_message = "Invalid parameters: id_buffers or score_buffers is null"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (!id_buffer_sizes || !score_buffer_sizes) { - last_error_message = "Invalid parameters: buffer size arrays cannot be null"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (embeddings_count == 0) { - last_error_message = "Invalid parameter: embeddings_count must be greater than 0"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (embedding_dim == 0) { - last_error_message = "Invalid parameter: embedding_dim must be greater than 0"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - std::vector> embeddings_vec; - embeddings_vec.reserve(embeddings_count); - - for (size_t i = 0; i < embeddings_count; ++i) { - if (!embeddings[i]) { - last_error_message = "Embedding at index " + std::to_string(i) + " is null"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - embeddings_vec.emplace_back(embeddings[i], embeddings[i] + embedding_dim); - } - - cactus::engine::index::QueryOptions options; - if (options_json && std::strlen(options_json) > 0) { - options = parse_query_options_json(options_json); - } - - CACTUS_LOG_DEBUG("index_query", "Querying with " << embeddings_count << " queries, dim: " << embedding_dim << ", top_k: " << options.top_k); - std::vector> results = handle->index->query(embeddings_vec, options); - - for (size_t i = 0; i < results.size(); ++i) { - size_t result_count = results[i].size(); - if (id_buffer_sizes[i] < result_count || score_buffer_sizes[i] < result_count) { - last_error_message = "Result buffer too small at query index " + std::to_string(i) + - " (required: " + std::to_string(result_count) + - ", id_buffer: " + std::to_string(id_buffer_sizes[i]) + - ", score_buffer: " + std::to_string(score_buffer_sizes[i]) + ")"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - } - - for (size_t i = 0; i < results.size(); ++i) { - size_t result_count = results[i].size(); - - for (size_t j = 0; j < result_count; ++j) { - id_buffers[i][j] = results[i][j].doc_id; - score_buffers[i][j] = results[i][j].score; - } - - id_buffer_sizes[i] = result_count; - score_buffer_sizes[i] = result_count; - } - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_query", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during query"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } -} - -int cactus_index_compact(cactus_index_t index) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_compact", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - CACTUS_LOG_INFO("index_compact", "Compacting index"); - handle->index->compact(); - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_compact", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during compact"; - CACTUS_LOG_ERROR("index_compact", last_error_message); - return -1; - } -} - -void cactus_index_destroy(cactus_index_t index) { - if (index) delete static_cast(index); -} - -} diff --git a/cactus-transpile/ffi/cactus_init.cpp b/cactus-transpile/ffi/cactus_init.cpp deleted file mode 100644 index 13cc35851..000000000 --- a/cactus-transpile/ffi/cactus_init.cpp +++ /dev/null @@ -1,489 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "telemetry/telemetry.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - -static constexpr size_t RAG_MAX_CHUNK_TOKENS = 128; -static constexpr size_t RAG_MIN_CHUNK_TOKENS = 24; -static constexpr size_t RAG_CHUNK_OVERLAP = 32; - -static void apply_no_cloud_telemetry_env() { - if (cactus::ffi::env_flag_enabled("CACTUS_NO_CLOUD_TELE")) { - cactus::telemetry::setCloudDisabled(true); - } -} - -static time_t get_file_mtime(const std::string& path) { - struct stat st; - if (stat(path.c_str(), &st) == 0) { - return st.st_mtime; - } - return 0; -} - -static bool corpus_is_stale(const std::string& corpus_dir) { - std::string index_path = corpus_dir + "/index.bin"; - time_t index_mtime = get_file_mtime(index_path); - if (index_mtime == 0) return true; - - DIR* dir = opendir(corpus_dir.c_str()); - if (!dir) return true; - - struct dirent* entry; - while ((entry = readdir(dir)) != nullptr) { - std::string name = entry->d_name; - if (name == "." || name == "..") continue; - if (name == "index.bin" || name == "data.bin") continue; - - bool is_corpus_file = false; - if (name.size() > 4 && name.substr(name.size() - 4) == ".txt") is_corpus_file = true; - if (name.size() > 3 && name.substr(name.size() - 3) == ".md") is_corpus_file = true; - - if (is_corpus_file) { - std::string full_path = corpus_dir + "/" + name; - time_t file_mtime = get_file_mtime(full_path); - if (file_mtime > index_mtime) { - closedir(dir); - CACTUS_LOG_INFO("init", "Corpus file " << name << " is newer than index, rebuilding"); - return true; - } - } - } - closedir(dir); - return false; -} - -static std::string read_file_contents(const std::string& path) { - std::ifstream file(path); - if (!file.is_open()) return ""; - std::stringstream buffer; - buffer << file.rdbuf(); - return buffer.str(); -} - -static std::vector scan_corpus_files(const std::string& corpus_dir) { - std::vector files; - DIR* dir = opendir(corpus_dir.c_str()); - if (!dir) return files; - - struct dirent* entry; - while ((entry = readdir(dir)) != nullptr) { - std::string name = entry->d_name; - if (name == "." || name == "..") continue; - if (name == "index.bin" || name == "data.bin") continue; - - std::string full_path = corpus_dir + "/" + name; - struct stat st; - if (stat(full_path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) { - if (name.size() > 4 && (name.substr(name.size() - 4) == ".txt" || name.substr(name.size() - 3) == ".md")) { - files.push_back(full_path); - } - } - } - closedir(dir); - return files; -} - -static std::vector split_into_paragraphs(const std::string& content) { - std::vector paragraphs; - std::string current; - - size_t i = 0; - while (i < content.size()) { - // Check for markdown header - if (content[i] == '#' && (i == 0 || content[i-1] == '\n')) { - if (!current.empty()) { - paragraphs.push_back(current); - current.clear(); - } - // Include the header line - while (i < content.size() && content[i] != '\n') { - current += content[i++]; - } - if (i < content.size()) current += content[i++]; - continue; - } - - // Check for double newline (paragraph break) - if (content[i] == '\n' && i + 1 < content.size() && content[i+1] == '\n') { - current += content[i]; - if (!current.empty() && current != "\n") { - paragraphs.push_back(current); - current.clear(); - } - i++; - // Skip multiple blank lines - while (i < content.size() && content[i] == '\n') i++; - continue; - } - - current += content[i++]; - } - - if (!current.empty()) { - paragraphs.push_back(current); - } - - return paragraphs; -} - -static std::vector> chunk_corpus( - const std::vector& file_paths, - Tokenizer* tokenizer -) { - std::vector> chunks; - - for (const auto& path : file_paths) { - std::string content = read_file_contents(path); - if (content.empty()) continue; - - std::string filename = path; - size_t last_slash = path.find_last_of("/\\"); - if (last_slash != std::string::npos) { - filename = path.substr(last_slash + 1); - } - - auto paragraphs = split_into_paragraphs(content); - - std::string current_chunk; - size_t current_tokens = 0; - - for (const auto& para : paragraphs) { - std::vector para_tokens = tokenizer->encode(para); - size_t para_token_count = para_tokens.size(); - - // If paragraph alone is too large, split it by tokens - if (para_token_count > RAG_MAX_CHUNK_TOKENS) { - // Flush current chunk first - if (!current_chunk.empty()) { - chunks.emplace_back(current_chunk, filename); - current_chunk.clear(); - current_tokens = 0; - } - - // Split large paragraph by sentences or fixed tokens - size_t stride = RAG_MAX_CHUNK_TOKENS - RAG_CHUNK_OVERLAP; - for (size_t i = 0; i < para_tokens.size(); i += stride) { - size_t end = std::min(i + RAG_MAX_CHUNK_TOKENS, para_tokens.size()); - std::vector chunk_tokens(para_tokens.begin() + i, para_tokens.begin() + end); - std::string chunk_text = tokenizer->decode(chunk_tokens); - chunks.emplace_back(chunk_text, filename); - if (end >= para_tokens.size()) break; - } - continue; - } - - // Would adding this paragraph exceed max? - if (current_tokens + para_token_count > RAG_MAX_CHUNK_TOKENS && !current_chunk.empty()) { - chunks.emplace_back(current_chunk, filename); - current_chunk.clear(); - current_tokens = 0; - } - - // Add paragraph to current chunk - if (!current_chunk.empty()) current_chunk += "\n"; - current_chunk += para; - current_tokens += para_token_count; - } - - // Don't forget the last chunk - if (!current_chunk.empty() && current_tokens >= RAG_MIN_CHUNK_TOKENS) { - chunks.emplace_back(current_chunk, filename); - } else if (!current_chunk.empty() && !chunks.empty()) { - // Append small remaining chunk to previous - chunks.back().first += "\n" + current_chunk; - } else if (!current_chunk.empty()) { - chunks.emplace_back(current_chunk, filename); - } - } - - return chunks; -} - -static bool build_corpus_index(CactusModelHandle* handle, const std::string& corpus_dir) { - CACTUS_LOG_INFO("init", "Building corpus index from: " << corpus_dir); - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - CACTUS_LOG_ERROR("init", "No tokenizer available for corpus indexing"); - return false; - } - - auto file_paths = scan_corpus_files(corpus_dir); - if (file_paths.empty()) { - CACTUS_LOG_WARN("init", "No .txt or .md files found in corpus directory"); - return false; - } - - CACTUS_LOG_INFO("init", "Found " << file_paths.size() << " corpus files"); - - auto chunks = chunk_corpus(file_paths, tokenizer); - if (chunks.empty()) { - CACTUS_LOG_WARN("init", "No chunks generated from corpus"); - return false; - } - - CACTUS_LOG_INFO("init", "Generated " << chunks.size() << " chunks from corpus"); - - std::vector test_tokens = tokenizer->encode("test"); - std::vector test_embedding = handle->model->get_embeddings(test_tokens, true, true); - if (test_embedding.empty()) { - CACTUS_LOG_ERROR("init", "Failed to get embedding dimension"); - return false; - } - size_t embedding_dim = test_embedding.size(); - handle->corpus_embedding_dim = embedding_dim; - - CACTUS_LOG_INFO("init", "Embedding dimension: " << embedding_dim); - - std::string index_path = corpus_dir + "/index.bin"; - std::string data_path = corpus_dir + "/data.bin"; - - std::remove(index_path.c_str()); - std::remove(data_path.c_str()); - - try { - handle->corpus_index = std::make_unique(index_path, data_path, embedding_dim); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("init", "Failed to create index: " << e.what()); - return false; - } - - std::vector docs; - docs.reserve(chunks.size()); - - for (size_t i = 0; i < chunks.size(); ++i) { - const auto& [chunk_text, source_file] = chunks[i]; - - std::vector tokens = tokenizer->encode(chunk_text); - std::vector embedding = handle->model->get_embeddings(tokens, true, true); - - if (embedding.size() != embedding_dim) { - CACTUS_LOG_WARN("init", "Skipping chunk " << i << " - embedding dimension mismatch"); - continue; - } - - docs.push_back(index::Document{ - static_cast(i), - std::move(embedding), - chunk_text, - source_file - }); - - if ((i + 1) % 50 == 0) { - CACTUS_LOG_INFO("init", "Embedded " << (i + 1) << "/" << chunks.size() << " chunks"); - } - } - - if (docs.empty()) { - CACTUS_LOG_ERROR("init", "No documents to add to index"); - return false; - } - - try { - handle->corpus_index->add_documents(docs); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("init", "Failed to add documents to index: " << e.what()); - return false; - } - - CACTUS_LOG_INFO("init", "Corpus index built successfully with " << docs.size() << " chunks"); - return true; -} - -static bool load_corpus_index(CactusModelHandle* handle, const std::string& corpus_dir) { - std::string index_path = corpus_dir + "/index.bin"; - std::string data_path = corpus_dir + "/data.bin"; - - struct stat st; - if (stat(index_path.c_str(), &st) != 0 || stat(data_path.c_str(), &st) != 0) { - return false; - } - - if (corpus_is_stale(corpus_dir)) { - return false; - } - - auto* tokenizer = handle->model->get_tokenizer(); - std::vector test_tokens = tokenizer->encode("test"); - std::vector test_embedding = handle->model->get_embeddings(test_tokens, true, true); - if (test_embedding.empty()) { - CACTUS_LOG_ERROR("init", "Failed to get embedding dimension for index loading"); - return false; - } - size_t embedding_dim = test_embedding.size(); - handle->corpus_embedding_dim = embedding_dim; - - try { - handle->corpus_index = std::make_unique(index_path, data_path, embedding_dim); - CACTUS_LOG_INFO("init", "Loaded existing corpus index from: " << corpus_dir); - return true; - } catch (const std::exception& e) { - CACTUS_LOG_WARN("init", "Failed to load existing index: " << e.what()); - return false; - } -} - -std::string last_error_message; - -bool matches_stop_sequence(const std::vector& generated_tokens, - const std::vector>& stop_sequences) { - for (const auto& stop_seq : stop_sequences) { - if (stop_seq.empty()) continue; - if (generated_tokens.size() >= stop_seq.size()) { - if (std::equal(stop_seq.rbegin(), stop_seq.rend(), generated_tokens.rbegin())) - return true; - } - } - return false; -} - -extern "C" { - -const char* cactus_get_last_error() { - return last_error_message.c_str(); -} - -cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool cache_index) { - constexpr size_t DEFAULT_CONTEXT_SIZE = 512; // matches default sliding window size - - std::string model_path_str = model_path ? std::string(model_path) : "unknown"; - - std::string model_name = model_path_str; - size_t last_slash = model_path_str.find_last_of("/\\"); - if (last_slash != std::string::npos) { - model_name = model_path_str.substr(last_slash + 1); - } - - CACTUS_LOG_INFO("init", "Loading model: " << model_name << " from " << model_path_str); - - apply_no_cloud_telemetry_env(); - cactus::telemetry::init(nullptr, model_path_str.c_str(), nullptr); - - auto __cactus_init_start = std::chrono::steady_clock::now(); - - try { - auto* handle = new CactusModelHandle(); - handle->model = create_model(model_path); - handle->model_name = model_name; - - if (!handle->model) { - last_error_message = "Failed to create model - check config.txt exists at: " + model_path_str; - CACTUS_LOG_ERROR("init", last_error_message); - { - auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); - } - delete handle; - return nullptr; - } - - if (!handle->model->init(model_path, DEFAULT_CONTEXT_SIZE)) { - last_error_message = "Failed to initialize model - check weight files at: " + model_path_str; - CACTUS_LOG_ERROR("init", last_error_message); - { - auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); - } - delete handle; - return nullptr; - } - - auto model_type = handle->model->get_config().model_type; - if (model_type == Config::ModelType::WHISPER || - model_type == Config::ModelType::MOONSHINE || - model_type == Config::ModelType::PARAKEET || - model_type == Config::ModelType::PARAKEET_TDT) { - std::string vad_path = model_path_str + "/vad"; - const std::filesystem::path vad_config_path = std::filesystem::path(vad_path) / "config.txt"; - if (std::filesystem::exists(vad_config_path)) { - handle->vad_model = create_model(vad_path); - if (!handle->vad_model) { - last_error_message = "Failed to create VAD model - check VAD weights at: " + vad_path; - CACTUS_LOG_ERROR("init", last_error_message); - delete handle; - return nullptr; - } - if (!handle->vad_model->init(vad_path, 0, "", false)) { - last_error_message = "Failed to initialize VAD model - check VAD weight files at: " + vad_path; - CACTUS_LOG_ERROR("init", last_error_message); - delete handle; - return nullptr; - } - } else { - CACTUS_LOG_WARN("init", "VAD weights not found at " << vad_path << "; continuing without VAD"); - } - } - - if (corpus_dir != nullptr && strlen(corpus_dir) > 0) { - handle->corpus_dir = std::string(corpus_dir); - - bool loaded = false; - if (cache_index) { - loaded = load_corpus_index(handle, handle->corpus_dir); - } - - if (!loaded) { - CACTUS_LOG_INFO("init", (cache_index ? "No existing index found, building new corpus index" : "Building fresh corpus index (caching disabled)")); - if (!build_corpus_index(handle, handle->corpus_dir)) { - CACTUS_LOG_WARN("init", "Failed to build corpus index - RAG disabled"); - } - } - } - - CACTUS_LOG_INFO("init", "Model loaded successfully: " << model_name); - { - auto __cactus_init_ok_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), true, static_cast(__cactus_init_ok_dur), ""); - } - - return handle; - } catch (const std::exception& e) { - last_error_message = "Exception during init: " + std::string(e.what()); - CACTUS_LOG_ERROR("init", last_error_message); - { - auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); - } - return nullptr; - } catch (...) { - last_error_message = "Unknown exception during model initialization"; - CACTUS_LOG_ERROR("init", last_error_message); - { - auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); - } - return nullptr; - } -} - -void cactus_destroy(cactus_model_t model) { - if (model) delete static_cast(model); -} - -void cactus_reset(cactus_model_t model) { - if (!model) return; - auto* handle = static_cast(model); - handle->model->reset_cache(); - handle->processed_tokens.clear(); - handle->processed_images.clear(); -} - -void cactus_stop(cactus_model_t model) { - if (!model) return; - auto* handle = static_cast(model); - handle->should_stop = true; -} - -} diff --git a/cactus-transpile/ffi/cactus_log.cpp b/cactus-transpile/ffi/cactus_log.cpp deleted file mode 100644 index d067f9adf..000000000 --- a/cactus-transpile/ffi/cactus_log.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "cactus_ffi.h" -#include "graph/graph.h" - -extern "C" { - -void cactus_log_set_level(int level) { - cactus::Logger::instance().set_level(static_cast(level)); -} - -void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data) { - if (callback) { - cactus::Logger::instance().set_callback( - [callback, user_data](cactus::LogLevel level, - const std::string& component, - const std::string& message) { - callback(static_cast(level), component.c_str(), message.c_str(), user_data); - }); - } else { - cactus::Logger::instance().set_callback(nullptr); - } -} - -} diff --git a/cactus-transpile/ffi/cactus_rag.cpp b/cactus-transpile/ffi/cactus_rag.cpp deleted file mode 100644 index a0765f787..000000000 --- a/cactus-transpile/ffi/cactus_rag.cpp +++ /dev/null @@ -1,459 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - -static constexpr size_t RAG_TOP_K = 5; -static constexpr size_t RAG_CANDIDATE_K = 20; -static constexpr float RRF_K = 60.0f; -static constexpr float RRF_EMB_WEIGHT = 0.8f; -static constexpr float RRF_BM25_WEIGHT = 0.2f; -static constexpr float BM25_K1 = 1.5f; -static constexpr float BM25_B = 0.75f; - -static std::vector> compute_rrf_scores( - const std::vector>& emb_ranked, - const std::vector>& bm25_ranked, - size_t num_items -) { - std::vector> emb_sorted = emb_ranked; - std::vector> bm25_sorted = bm25_ranked; - - std::sort(emb_sorted.begin(), emb_sorted.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - std::sort(bm25_sorted.begin(), bm25_sorted.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - std::unordered_map emb_rank_map, bm25_rank_map; - for (size_t r = 0; r < emb_sorted.size(); ++r) { - emb_rank_map[emb_sorted[r].second] = r + 1; - } - for (size_t r = 0; r < bm25_sorted.size(); ++r) { - bm25_rank_map[bm25_sorted[r].second] = r + 1; - } - - std::vector> rrf_scored; - rrf_scored.reserve(num_items); - for (size_t i = 0; i < num_items; ++i) { - float rrf = RRF_EMB_WEIGHT / (RRF_K + emb_rank_map[i]) + - RRF_BM25_WEIGHT / (RRF_K + bm25_rank_map[i]); - rrf_scored.emplace_back(rrf, i); - } - - std::sort(rrf_scored.begin(), rrf_scored.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - return rrf_scored; -} - -static std::vector tokenize_words(const std::string& text) { - std::vector words; - std::string current; - for (char c : text) { - if (std::isalnum(static_cast(c))) { - current += std::tolower(static_cast(c)); - } else if (!current.empty()) { - if (current.length() > 2) { - words.push_back(current); - } - current.clear(); - } - } - if (!current.empty() && current.length() > 2) { - words.push_back(current); - } - return words; -} - -static float compute_bm25_score( - const std::vector& query_words, - const std::string& doc_content, - float avg_doc_len, - const std::unordered_map& doc_freqs, - int total_docs -) { - auto doc_words = tokenize_words(doc_content); - float doc_len = static_cast(doc_words.size()); - - std::unordered_map tf; - for (const auto& w : doc_words) { - tf[w]++; - } - - float score = 0.0f; - std::unordered_set query_set(query_words.begin(), query_words.end()); - - for (const auto& term : query_set) { - auto tf_it = tf.find(term); - if (tf_it == tf.end()) continue; - - float term_freq = static_cast(tf_it->second); - float df = 1.0f; - auto df_it = doc_freqs.find(term); - if (df_it != doc_freqs.end()) { - df = static_cast(df_it->second); - } - - float idf = std::log((total_docs - df + 0.5f) / (df + 0.5f) + 1.0f); - - float tf_component = (term_freq * (BM25_K1 + 1.0f)) / - (term_freq + BM25_K1 * (1.0f - BM25_B + BM25_B * (doc_len / avg_doc_len))); - - score += idf * tf_component; - } - - return score; -} - -std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query) { - if (!handle->corpus_index || handle->corpus_embedding_dim == 0) { - return ""; - } - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) return ""; - - std::vector query_tokens = tokenizer->encode(query); - if (query_tokens.empty()) return ""; - - std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); - if (query_embedding.size() != handle->corpus_embedding_dim) { - CACTUS_LOG_WARN("rag", "Query embedding dimension mismatch"); - return ""; - } - - index::QueryOptions options; - options.top_k = RAG_CANDIDATE_K; - options.score_threshold = 0.0f; - - std::vector> query_embeddings = {query_embedding}; - - try { - auto results = handle->corpus_index->query(query_embeddings, options); - if (results.empty() || results[0].empty()) { - return ""; - } - - std::vector doc_ids; - std::vector embedding_scores; - for (const auto& result : results[0]) { - doc_ids.push_back(result.doc_id); - embedding_scores.push_back(result.score); - } - - auto docs = handle->corpus_index->get_documents(doc_ids); - if (docs.empty()) { - return ""; - } - - auto query_words = tokenize_words(query); - - float total_len = 0.0f; - std::unordered_map doc_freqs; - for (const auto& doc : docs) { - auto words = tokenize_words(doc.content); - total_len += words.size(); - std::unordered_set unique_words(words.begin(), words.end()); - for (const auto& w : unique_words) { - doc_freqs[w]++; - } - } - float avg_doc_len = total_len / docs.size(); - - std::vector> emb_ranked; - std::vector> bm25_ranked; - for (size_t i = 0; i < docs.size(); ++i) { - emb_ranked.emplace_back(embedding_scores[i], i); - float bm25 = compute_bm25_score( - query_words, docs[i].content, avg_doc_len, doc_freqs, docs.size() - ); - bm25_ranked.emplace_back(bm25, i); - } - - auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, docs.size()); - - std::string context = "[Retrieved Context - Use ONLY this information to answer. If the answer is not in the context, say \"I don't have enough information to answer that.\"]\n"; - size_t count = std::min(RAG_TOP_K, rrf_scored.size()); - for (size_t i = 0; i < count; ++i) { - size_t idx = rrf_scored[i].second; - context += "---\n"; - context += docs[idx].content; - if (!docs[idx].metadata.empty()) { - context += "\n(Source: " + docs[idx].metadata + ")"; - } - context += "\n"; - } - context += "---\n\n"; - - CACTUS_LOG_DEBUG("rag", "Retrieved " << count << " RAG chunks (hybrid BM25+embedding)"); - return context; - - } catch (const std::exception& e) { - CACTUS_LOG_WARN("rag", "RAG retrieval failed: " << e.what()); - return ""; - } -} - -static std::string tool_to_text(const ToolFunction& tool) { - return tool.name + " " + tool.description; -} - -static float cosine_similarity(const std::vector& a, const std::vector& b) { - if (a.size() != b.size() || a.empty()) return 0.0f; - - float dot = 0.0f, norm_a = 0.0f, norm_b = 0.0f; - for (size_t i = 0; i < a.size(); ++i) { - dot += a[i] * b[i]; - norm_a += a[i] * a[i]; - norm_b += b[i] * b[i]; - } - - float denom = std::sqrt(norm_a) * std::sqrt(norm_b); - return denom > 0.0f ? dot / denom : 0.0f; -} - -std::vector select_relevant_tools( - CactusModelHandle* handle, - const std::string& query, - const std::vector& all_tools, - size_t top_k -) { - using cactus::ffi::ToolFunction; - if (!handle || all_tools.empty() || top_k == 0) return all_tools; - - // If we have fewer tools than top_k, just return all - if (all_tools.size() <= top_k) return all_tools; - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - CACTUS_LOG_WARN("tool_rag", "No tokenizer available, returning all tools"); - return all_tools; - } - - // Build tool texts and embeddings if not cached or tool set changed - bool need_recompute = (handle->tool_texts.size() != all_tools.size()); - if (!need_recompute) { - for (size_t i = 0; i < all_tools.size(); ++i) { - if (tool_to_text(all_tools[i]) != handle->tool_texts[i]) { - need_recompute = true; - break; - } - } - } - - if (need_recompute) { - CACTUS_LOG_DEBUG("tool_rag", "Computing embeddings for " << all_tools.size() << " tools"); - handle->tool_texts.clear(); - handle->tool_embeddings.clear(); - handle->tool_texts.reserve(all_tools.size()); - handle->tool_embeddings.reserve(all_tools.size()); - - for (const auto& tool : all_tools) { - std::string text = tool_to_text(tool); - handle->tool_texts.push_back(text); - - std::vector tokens = tokenizer->encode(text); - if (!tokens.empty()) { - std::vector emb = handle->model->get_embeddings(tokens, true, true); - handle->tool_embeddings.push_back(std::move(emb)); - } else { - handle->tool_embeddings.push_back({}); - } - } - } - - // Embed the query - std::vector query_tokens = tokenizer->encode(query); - if (query_tokens.empty()) { - CACTUS_LOG_WARN("tool_rag", "Empty query, returning all tools"); - return all_tools; - } - - std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); - if (query_embedding.empty()) { - CACTUS_LOG_WARN("tool_rag", "Failed to get query embedding, returning all tools"); - return all_tools; - } - - auto query_words = tokenize_words(query); - - float total_len = 0.0f; - std::unordered_map doc_freqs; - for (const auto& text : handle->tool_texts) { - auto words = tokenize_words(text); - total_len += words.size(); - std::unordered_set unique_words(words.begin(), words.end()); - for (const auto& w : unique_words) { - doc_freqs[w]++; - } - } - float avg_doc_len = total_len / handle->tool_texts.size(); - - std::vector> emb_ranked; - for (size_t i = 0; i < handle->tool_embeddings.size(); ++i) { - float sim = cosine_similarity(query_embedding, handle->tool_embeddings[i]); - emb_ranked.emplace_back(sim, i); - } - - std::vector> bm25_ranked; - for (size_t i = 0; i < handle->tool_texts.size(); ++i) { - float bm25 = compute_bm25_score( - query_words, handle->tool_texts[i], avg_doc_len, doc_freqs, handle->tool_texts.size() - ); - bm25_ranked.emplace_back(bm25, i); - } - - auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, all_tools.size()); - - // Select top-k tools - std::vector selected; - selected.reserve(top_k); - for (size_t i = 0; i < top_k && i < rrf_scored.size(); ++i) { - size_t idx = rrf_scored[i].second; - selected.push_back(all_tools[idx]); - CACTUS_LOG_DEBUG("tool_rag", "Selected tool: " << all_tools[idx].name << " (RRF: " << rrf_scored[i].first << ")"); - } - - CACTUS_LOG_INFO("tool_rag", "Selected " << selected.size() << " of " << all_tools.size() << " tools using hybrid ranking"); - return selected; -} - -extern "C" { - -int cactus_rag_query( - cactus_model_t model, - const char* query, - char* response_buffer, - size_t buffer_size, - size_t top_k -) { - if (!model || !query || !response_buffer || buffer_size == 0) { - return -1; - } - - auto* handle = static_cast(model); - - if (!handle->corpus_index || handle->corpus_embedding_dim == 0) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"No corpus index loaded\"}"); - return 0; - } - - try { - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"No tokenizer\"}"); - return 0; - } - - std::vector query_tokens = tokenizer->encode(query); - if (query_tokens.empty()) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Empty query\"}"); - return 0; - } - - std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); - if (query_embedding.size() != handle->corpus_embedding_dim) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Embedding dimension mismatch\"}"); - return 0; - } - - index::QueryOptions options; - options.top_k = RAG_CANDIDATE_K; - options.score_threshold = 0.0f; - - std::vector> query_embeddings = {query_embedding}; - auto results = handle->corpus_index->query(query_embeddings, options); - - if (results.empty() || results[0].empty()) { - std::strcpy(response_buffer, "{\"chunks\":[]}"); - return 0; - } - - std::vector doc_ids; - std::vector embedding_scores; - for (const auto& result : results[0]) { - doc_ids.push_back(result.doc_id); - embedding_scores.push_back(result.score); - } - - auto docs = handle->corpus_index->get_documents(doc_ids); - - auto query_words = tokenize_words(query); - - float total_len = 0.0f; - std::unordered_map doc_freqs; - for (const auto& doc : docs) { - auto words = tokenize_words(doc.content); - total_len += words.size(); - std::unordered_set unique_words(words.begin(), words.end()); - for (const auto& w : unique_words) { - doc_freqs[w]++; - } - } - float avg_doc_len = docs.size() > 0 ? total_len / docs.size() : 1.0f; - - std::vector> emb_ranked; - std::vector> bm25_ranked; - for (size_t i = 0; i < docs.size(); ++i) { - emb_ranked.emplace_back(embedding_scores[i], i); - float bm25 = compute_bm25_score( - query_words, docs[i].content, avg_doc_len, doc_freqs, docs.size() - ); - bm25_ranked.emplace_back(bm25, i); - } - - auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, docs.size()); - - size_t result_count = std::min(top_k > 0 ? top_k : RAG_TOP_K, rrf_scored.size()); - - std::ostringstream oss; - oss << "{\"chunks\":["; - for (size_t i = 0; i < result_count; ++i) { - size_t idx = rrf_scored[i].second; - float final_score = rrf_scored[i].first; - - if (i > 0) oss << ","; - oss << "{\"score\":" << std::setprecision(4) << final_score - << ",\"source\":\"" << docs[idx].metadata << "\"" - << ",\"content\":\""; - for (char c : docs[idx].content) { - switch (c) { - case '"': oss << "\\\""; break; - case '\\': oss << "\\\\"; break; - case '\n': oss << "\\n"; break; - case '\r': oss << "\\r"; break; - case '\t': oss << "\\t"; break; - default: oss << c; - } - } - oss << "\"}"; - } - oss << "]}"; - - std::string result = oss.str(); - if (result.size() >= buffer_size) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Buffer too small\"}"); - return -1; - } - - std::strcpy(response_buffer, result.c_str()); - return static_cast(result.size()); - - } catch (const std::exception& e) { - std::string error = "{\"chunks\":[],\"error\":\"" + std::string(e.what()) + "\"}"; - if (error.size() < buffer_size) { - std::strcpy(response_buffer, error.c_str()); - } - return -1; - } -} - -} diff --git a/cactus-transpile/ffi/cactus_stream.cpp b/cactus-transpile/ffi/cactus_stream.cpp deleted file mode 100644 index b451b8dfd..000000000 --- a/cactus-transpile/ffi/cactus_stream.cpp +++ /dev/null @@ -1,2714 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_cloud.h" -#include "cactus_utils.h" -#include "telemetry/telemetry.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::ffi; -using cactus::audio::WHISPER_SAMPLE_RATE; -using cactus::audio::apply_preemphasis; -using cactus::audio::get_parakeet_spectrogram_config; -using cactus::audio::normalize_parakeet_log_mel; -using cactus::audio::trim_mel_frames; - -double json_number(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return 0.0; - size_t start = pos + pattern.size(); - while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; - size_t end = start; - while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; - try { return std::stod(json.substr(start, end - start)); } - catch (...) { return 0.0; } -} - -std::string json_string(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return {}; - size_t start = pos + pattern.size(); - - while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; - if (start >= json.size() || json[start] != '"') return {}; - size_t q1 = start; - size_t q2 = json.find('"', q1 + 1); - if (q2 == std::string::npos) return {}; - return json.substr(q1 + 1, q2 - q1 - 1); -} - -inline std::string escape_json(const std::string& s) { - return escape_json_string(s); -} - -bool json_bool(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return false; - size_t start = pos + pattern.size(); - while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; - if (start + 4 <= json.size() && json.substr(start, 4) == "true") return true; - return false; -} - -static std::string ensure_json_bool_option(const std::string& json, - const std::string& key, - bool value) { - const std::string field = "\"" + key + "\""; - if (json.find(field) != std::string::npos) return json; - - const std::string bool_str = value ? "true" : "false"; - if (json.empty()) { - return "{\"" + key + "\":" + bool_str + "}"; - } - - std::string out = json; - const size_t close = out.rfind('}'); - const size_t open = out.find('{'); - if (open == std::string::npos || close == std::string::npos || close <= open) { - return json; - } - - size_t content_start = open + 1; - while (content_start < close && std::isspace(static_cast(out[content_start]))) { - ++content_start; - } - const bool has_entries = content_start < close; - const std::string insertion = (has_entries ? "," : "") + field + ":" + bool_str; - out.insert(close, insertion); - return out; -} - -std::string json_array(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return "[]"; - size_t start = pos + pattern.size(); - while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; - if (start >= json.size() || json[start] != '[') return "[]"; - int depth = 1; - size_t end = start + 1; - while (end < json.size() && depth > 0) { - if (json[end] == '[') depth++; - else if (json[end] == ']') depth--; - end++; - } - return json.substr(start, end - start); -} - -static std::string suppress_unwanted_text(const std::string& text) { - static const std::regex pattern(R"(\([^)]*\)|\[[^\]]*\]|\.\.\.)"); - std::string result = std::regex_replace(text, pattern, ""); - - size_t start = result.find_first_not_of(" \t\n\r"); - if (start == std::string::npos) return ""; - size_t end = result.find_last_not_of(" \t\n\r"); - return result.substr(start, end - start + 1); -} - -static bool is_pcm_chunk_silent(const uint8_t* pcm_buffer, size_t pcm_buffer_size, float rms_threshold) { - if (!pcm_buffer || pcm_buffer_size < sizeof(int16_t)) return true; - - const size_t sample_count = pcm_buffer_size / sizeof(int16_t); - const auto* samples = reinterpret_cast(pcm_buffer); - double sum_sq = 0.0; - for (size_t i = 0; i < sample_count; ++i) { - const double normalized = static_cast(samples[i]) / 32768.0; - sum_sq += normalized * normalized; - } - - const double rms = std::sqrt(sum_sq / static_cast(sample_count)); - return rms < static_cast(rms_threshold); -} - -static void parse_stream_transcribe_init_options(const std::string& json, - double& confirmation_threshold, - size_t& min_chunk_size, - bool& telemetry_enabled, - std::string& language) { - confirmation_threshold = 0.99; - min_chunk_size = 32000; - telemetry_enabled = true; - language = "en"; - - if (json.empty()) return; - - size_t pos = json.find("\"confirmation_threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - confirmation_threshold = std::stod(json.substr(pos)); - } - - pos = json.find("\"min_chunk_size\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - min_chunk_size = static_cast(std::stod(json.substr(pos))); - } - - pos = json.find("\"telemetry_enabled\""); - if (pos != std::string::npos) { - telemetry_enabled = json_bool(json, "telemetry_enabled"); - } - - language = json_string(json, "language"); - if (language.empty()) language = "en"; -} - -static bool build_parakeet_features_from_pcm( - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - const cactus::engine::Model* model, - std::vector& out_features) { - if (!pcm_buffer || pcm_buffer_size < sizeof(int16_t) || !model) { - return false; - } - - std::vector waveform = cactus::audio::pcm_buffer_to_float_samples( - pcm_buffer, pcm_buffer_size); - if (waveform.empty()) { - return false; - } - - const auto cfg = get_parakeet_spectrogram_config(); - const size_t mel_bins = std::max( - 1, static_cast(model->get_config().num_mel_bins)); - cactus::engine::AudioProcessor processor; - processor.init_mel_filters( - cfg.n_fft / 2 + 1, - mel_bins, - 0.0f, - 8000.0f, - WHISPER_SAMPLE_RATE); - - const size_t waveform_samples = waveform.size(); - apply_preemphasis(waveform, 0.97f); - out_features = processor.compute_spectrogram(waveform, cfg); - normalize_parakeet_log_mel(out_features, mel_bins); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(out_features, mel_bins, valid_frames); - return !out_features.empty(); -} - -static size_t parakeet_tdt_pcm_bytes_to_encoder_frames( - const cactus::engine::Model* model, - size_t pcm_bytes) { - if (!model) return 0; - const auto cfg = get_parakeet_spectrogram_config(); - const double sample_count = static_cast(pcm_bytes) / 2.0; - const double feature_frames = sample_count / static_cast(cfg.hop_length); - const double subsampling = std::max(1, model->get_config().subsampling_factor); - return static_cast(std::llround(feature_frames / subsampling)); -} - -struct CactusStreamTranscribeHandle { - CactusModelHandle* model_handle; - - struct CactusStreamTranscribeOptions { - double confirmation_threshold; - size_t min_chunk_size; - std::string language; - } options; - std::string transcribe_options_json; - bool has_custom_vocabulary_bias = false; - std::vector custom_vocabulary; - - std::vector audio_buffer; - std::string previous_parakeet_pending; - size_t previous_parakeet_audio_buffer_size = 0; - size_t previous_parakeet_pending_ticks = 0; - size_t parakeet_silence_run_bytes = 0; - size_t parakeet_speech_run_bytes = 0; - bool parakeet_resume_guard_active = true; - std::string parakeet_committed_text; - size_t parakeet_chunk_cursor_bytes = 0; - float parakeet_committed_until_sec = 0.0f; - bool parakeet_onset_active = false; - size_t parakeet_onset_start_bytes = 0; - bool parakeet_tdt_chunked_stream = false; - struct ParakeetTdtDecodeContext { - std::vector audio_samples; - size_t samples_decoded_up_to = 0; - float audio_time_offset_sec = 0.0f; - cactus::engine::ParakeetTDTModel::ChunkStreamState decoder_state; - cactus::engine::AudioProcessor audio_processor; - size_t mel_bins = 0; - bool initialized = false; - } parakeet_tdt_decode_context; - bool parakeet_tdt_chunked_blocked_until_pause = false; - cactus::engine::ParakeetTDTModel::ChunkStreamState parakeet_tdt_decoder_state; - std::string parakeet_tdt_pending_raw; - struct ParakeetTdtResumeCheckpoint { - size_t end_bytes = 0; - cactus::engine::ParakeetTDTModel::ChunkStreamState state; - }; - std::deque parakeet_tdt_resume_checkpoints; - - std::vector previous_segments; - bool previous_cloud_handoff = false; - float stream_audio_offset_sec = 0.0f; - uint64_t next_cloud_job_id = 1; - - struct CloudJob { - uint64_t id; - std::future result; - }; - std::vector pending_cloud_jobs; - std::vector> completed_cloud_results; - - char transcribe_response_buffer[65536]; - - std::chrono::steady_clock::time_point stream_start; - bool stream_first_token_seen; - double stream_first_token_ms; - int stream_total_tokens; - - std::chrono::steady_clock::time_point stream_session_start; - bool stream_session_first_token_seen; - double stream_session_first_token_ms; - int stream_cumulative_tokens; - double last_stream_decode_tps = 0.0; - double last_stream_decode_tokens = 0.0; - double last_stream_total_time_ms = 0.0; - double last_stream_raw_decoder_tps = 0.0; -}; - -static std::vector parse_segments(const std::string& transcribe_json); - -struct StreamWindowDecodeResult { - std::string raw_json; - std::string response; - std::vector segments; - double decode_tokens = 0.0; - bool cloud_handoff = false; -}; - -static double pcm_bytes_to_seconds(size_t pcm_bytes) { - return static_cast(pcm_bytes) / (16000.0 * 2.0); -} - -static size_t seconds_to_pcm_bytes(double seconds) { - if (seconds <= 0.0) return 0; - const size_t raw = static_cast(std::llround(seconds * 16000.0 * 2.0)); - return raw - (raw % sizeof(int16_t)); -} - -static std::string join_segments_by_end_time( - const std::vector& segments, - float offset_sec, - float min_end_exclusive_sec, - float max_end_inclusive_sec, - float* out_max_emitted_end_sec = nullptr) { - std::string joined; - float max_end_sec = min_end_exclusive_sec; - for (const auto& seg : segments) { - const float abs_end_sec = offset_sec + seg.end; - if (abs_end_sec <= min_end_exclusive_sec || abs_end_sec > max_end_inclusive_sec) { - continue; - } - if (!joined.empty()) joined += ' '; - joined += seg.text; - if (abs_end_sec > max_end_sec) { - max_end_sec = abs_end_sec; - } - } - - if (out_max_emitted_end_sec) { - *out_max_emitted_end_sec = max_end_sec; - } - return suppress_unwanted_text(joined); -} - -static bool run_stream_window_transcribe( - CactusStreamTranscribeHandle* handle, - const char* transcribe_prompt, - const std::string& effective_transcribe_options_json, - const uint8_t* decode_pcm, - size_t decode_pcm_size, - StreamWindowDecodeResult& out) { - if (!handle || !handle->model_handle || !decode_pcm || decode_pcm_size == 0) { - return false; - } - - const auto model_type = handle->model_handle->model->get_config().model_type; - const bool use_model_stream_mode = - model_type != cactus::engine::Config::ModelType::PARAKEET_TDT; - cactus::telemetry::setStreamMode(use_model_stream_mode); - const int result = cactus_transcribe( - handle->model_handle, - nullptr, - transcribe_prompt, - handle->transcribe_response_buffer, - sizeof(handle->transcribe_response_buffer), - effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), - nullptr, - nullptr, - decode_pcm, - decode_pcm_size); - cactus::telemetry::setStreamMode(false); - - if (result < 0) { - return false; - } - - out.raw_json.assign(handle->transcribe_response_buffer); - out.response = suppress_unwanted_text(json_string(out.raw_json, "response")); - out.segments = parse_segments(out.raw_json); - out.decode_tokens = std::max(0.0, json_number(out.raw_json, "decode_tokens")); - out.cloud_handoff = json_bool(out.raw_json, "cloud_handoff"); - handle->last_stream_decode_tps = std::max(0.0, json_number(out.raw_json, "decode_tps")); - handle->last_stream_decode_tokens = out.decode_tokens; - handle->last_stream_total_time_ms = std::max(0.0, json_number(out.raw_json, "total_time_ms")); - handle->last_stream_raw_decoder_tps = std::max(0.0, json_number(out.raw_json, "raw_decoder_tps")); - return true; -} - -static bool run_parakeet_tdt_chunk_decode( - CactusStreamTranscribeHandle* handle, - size_t window_start_bytes, - size_t window_end_bytes, - size_t chunk_start_bytes, - size_t chunk_end_bytes, - cactus::engine::ParakeetTDTModel::ChunkStreamResult& out, - cactus::engine::ParakeetTDTModel::ChunkStreamState& out_state) { - if (!handle || !handle->model_handle || !handle->model_handle->model) { - return false; - } - if (chunk_end_bytes <= chunk_start_bytes || - window_end_bytes <= window_start_bytes || - window_end_bytes > handle->audio_buffer.size() || - chunk_end_bytes > window_end_bytes || - window_start_bytes > chunk_start_bytes) { - return false; - } - - auto* model = static_cast( - handle->model_handle->model.get()); - size_t replay_start_bytes = window_start_bytes; - cactus::engine::ParakeetTDTModel::ChunkStreamState base_state; - bool found_checkpoint = false; - for (auto it = handle->parakeet_tdt_resume_checkpoints.rbegin(); - it != handle->parakeet_tdt_resume_checkpoints.rend(); - ++it) { - if (it->end_bytes <= window_start_bytes) { - replay_start_bytes = std::max(window_start_bytes, it->end_bytes); - base_state = it->state; - found_checkpoint = true; - break; - } - } - if (!found_checkpoint) { - for (auto it = handle->parakeet_tdt_resume_checkpoints.rbegin(); - it != handle->parakeet_tdt_resume_checkpoints.rend(); - ++it) { - if (it->end_bytes <= chunk_start_bytes) { - replay_start_bytes = std::max(window_start_bytes, it->end_bytes); - base_state = it->state; - found_checkpoint = true; - break; - } - } - } - if (!found_checkpoint) { - replay_start_bytes = window_start_bytes; - base_state = {}; - } - - std::vector audio_features; - if (!build_parakeet_features_from_pcm( - handle->audio_buffer.data() + window_start_bytes, - window_end_bytes - window_start_bytes, - model, - audio_features)) { - return false; - } - - const size_t replay_start_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( - model, replay_start_bytes - window_start_bytes); - const size_t start_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( - model, chunk_start_bytes - window_start_bytes); - size_t end_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( - model, chunk_end_bytes - window_start_bytes); - if (end_frame <= start_frame) { - end_frame = start_frame + 1; - } - - const auto decode_start = std::chrono::steady_clock::now(); - std::lock_guard lock(handle->model_handle->model_mutex); - cactus::telemetry::setStreamMode(true); - try { - out_state = base_state; - out = model->decode_chunk_stream( - audio_features, - replay_start_frame, - start_frame, - end_frame, - out_state); - } catch (...) { - cactus::telemetry::setStreamMode(false); - throw; - } - cactus::telemetry::setStreamMode(false); - const auto decode_end = std::chrono::steady_clock::now(); - const double decode_ms = std::chrono::duration_cast( - decode_end - decode_start).count() / 1000.0; - handle->last_stream_total_time_ms = decode_ms; - handle->last_stream_decode_tokens = static_cast(out.token_count); - handle->last_stream_decode_tps = - (out.token_count > 1 && decode_ms > 0.0) - ? ((static_cast(out.token_count) - 1.0) * 1000.0) / decode_ms - : 0.0; - handle->last_stream_raw_decoder_tps = out.raw_decoder_tps; - return true; -} - -static void reset_parakeet_tdt_chunk_stream(CactusStreamTranscribeHandle* handle) { - if (!handle) return; - handle->parakeet_tdt_decoder_state = {}; - handle->parakeet_tdt_pending_raw.clear(); - handle->parakeet_tdt_resume_checkpoints.clear(); -} - -static std::vector> collect_parakeet_word_spans(const std::string& text); - -static std::string strip_unwanted_text_preserving_edges(const std::string& text) { - static const std::regex pattern(R"(\([^)]*\)|\[[^\]]*\]|\.\.\.)"); - return std::regex_replace(text, pattern, ""); -} - -static void parakeet_append_monotonic_text(std::string& dst, const std::string& piece) { - const std::string normalized = suppress_unwanted_text(piece); - if (normalized.empty()) return; - if (dst.empty()) { - dst = normalized; - return; - } - - const unsigned char first = static_cast(normalized.front()); - const bool attach_without_space = - std::ispunct(first) && normalized.front() != '"' && - normalized.front() != '\'' && normalized.front() != '('; - if (!std::isspace(static_cast(dst.back())) && !attach_without_space) { - dst += " "; - } - dst += normalized; -} - -static std::vector parse_segments(const std::string& transcribe_json) { - std::vector segs; - std::string arr = json_array(transcribe_json, "segments"); - size_t i = 1; - while (i < arr.size() - 1) { - while (i < arr.size() && (arr[i] == ',' || std::isspace((unsigned char)arr[i]))) ++i; - if (i >= arr.size() - 1 || arr[i] != '{') break; - int depth = 1; - size_t obj_start = i++; - while (i < arr.size() && depth > 0) { - if (arr[i] == '{') ++depth; - else if (arr[i] == '}') --depth; - ++i; - } - std::string obj = arr.substr(obj_start, i - obj_start); - segs.emplace_back( - static_cast(json_number(obj, "start")), - static_cast(json_number(obj, "end")), - json_string(obj, "text") - ); - } - return segs; -} - -static bool is_parakeet_word_char(unsigned char ch) { - return std::isalnum(ch) || ch == '\''; -} - -static std::vector> collect_parakeet_word_spans(const std::string& text) { - std::vector> spans; - bool in_word = false; - size_t word_start = 0; - - for (size_t i = 0; i < text.size(); ++i) { - const unsigned char ch = static_cast(text[i]); - if (is_parakeet_word_char(ch)) { - if (!in_word) { - in_word = true; - word_start = i; - } - } else if (in_word) { - spans.emplace_back(word_start, i); - in_word = false; - } - } - - if (in_word) { - spans.emplace_back(word_start, text.size()); - } - return spans; -} - -static std::string parakeet_word_from_span( - const std::string& text, - const std::pair& span) { - if (span.second <= span.first || span.second > text.size()) return ""; - return text.substr(span.first, span.second - span.first); -} - -static bool parakeet_words_fuzzy_equal(const std::string& a, const std::string& b) { - if (a.empty() || b.empty()) return false; - const size_t dist = levenshtein_ci(a, b); - if (dist == 0) return true; - const size_t max_len = std::max(a.size(), b.size()); - return max_len >= 5 && dist <= 1; -} - -static size_t parakeet_common_prefix_words(const std::string& left, const std::string& right) { - const auto left_spans = collect_parakeet_word_spans(left); - const auto right_spans = collect_parakeet_word_spans(right); - const size_t max_words = std::min(left_spans.size(), right_spans.size()); - size_t common = 0; - for (; common < max_words; ++common) { - const std::string left_word = parakeet_word_from_span(left, left_spans[common]); - const std::string right_word = parakeet_word_from_span(right, right_spans[common]); - if (!parakeet_words_fuzzy_equal(left_word, right_word)) { - break; - } - } - return common; -} - -static std::string parakeet_take_first_words(const std::string& text, size_t words) { - if (text.empty() || words == 0) return ""; - const auto spans = collect_parakeet_word_spans(text); - if (spans.empty()) return ""; - if (words >= spans.size()) return suppress_unwanted_text(text); - const size_t end_idx = spans[words].first; - return suppress_unwanted_text(text.substr(0, end_idx)); -} - -static std::string parakeet_drop_first_words(const std::string& text, size_t words) { - if (text.empty()) return ""; - const auto spans = collect_parakeet_word_spans(text); - if (spans.empty()) return suppress_unwanted_text(text); - if (words >= spans.size()) return ""; - const size_t start_idx = spans[words].first; - return suppress_unwanted_text(text.substr(start_idx)); -} - -static size_t parakeet_word_count(const std::string& text) { - return collect_parakeet_word_spans(text).size(); -} - -static size_t parakeet_overlap_suffix_prefix_words( - const std::string& left, - const std::string& right, - size_t max_words = 64) { - const auto left_spans = collect_parakeet_word_spans(left); - const auto right_spans = collect_parakeet_word_spans(right); - if (left_spans.empty() || right_spans.empty()) return 0; - - const size_t max_overlap = std::min({left_spans.size(), right_spans.size(), max_words}); - size_t best = 0; - for (size_t k = 1; k <= max_overlap; ++k) { - bool matched = true; - for (size_t i = 0; i < k; ++i) { - const auto& left_span = left_spans[left_spans.size() - k + i]; - const auto& right_span = right_spans[i]; - const std::string left_word = parakeet_word_from_span(left, left_span); - const std::string right_word = parakeet_word_from_span(right, right_span); - if (!parakeet_words_fuzzy_equal(left_word, right_word)) { - matched = false; - break; - } - } - if (matched) best = k; - } - return best; -} - -static std::string parakeet_strip_recent_aligned_overlap( - const std::string& committed_text, - const std::string& hypothesis, - size_t max_recent_words = 24) { - const std::string candidate = suppress_unwanted_text(hypothesis); - if (candidate.empty() || committed_text.empty()) return candidate; - - const auto committed_spans = collect_parakeet_word_spans(committed_text); - if (committed_spans.empty()) return candidate; - - const size_t recent_words = std::min(committed_spans.size(), max_recent_words); - const size_t recent_start = committed_spans[committed_spans.size() - recent_words].first; - const std::string recent_tail = suppress_unwanted_text(committed_text.substr(recent_start)); - const auto recent_spans = collect_parakeet_word_spans(recent_tail); - const auto candidate_spans = collect_parakeet_word_spans(candidate); - const size_t recent_count = recent_spans.size(); - const size_t candidate_count = candidate_spans.size(); - if (recent_count == 0 || candidate_count == 0) return candidate; - - size_t best_run = 0; - size_t best_candidate_start = candidate_count; - size_t best_candidate_end = 0; - size_t best_recent_start = recent_count; - - for (size_t i = 0; i < recent_count; ++i) { - for (size_t j = 0; j < candidate_count; ++j) { - size_t run = 0; - while (i + run < recent_count && j + run < candidate_count) { - const std::string recent_word = parakeet_word_from_span( - recent_tail, recent_spans[i + run]); - const std::string candidate_word = parakeet_word_from_span( - candidate, candidate_spans[j + run]); - if (!parakeet_words_fuzzy_equal(recent_word, candidate_word)) { - break; - } - ++run; - } - - const size_t candidate_end = j + run; - const bool better_run = run > best_run; - const bool better_end = run == best_run && candidate_end > best_candidate_end; - const bool better_start = run == best_run && - candidate_end == best_candidate_end && - j < best_candidate_start; - if (better_run || better_end || better_start) { - best_run = run; - best_candidate_start = j; - best_candidate_end = candidate_end; - best_recent_start = i; - } - } - } - - if (best_run == 0 || best_candidate_end == 0) return candidate; - - const size_t min_overlap_words = candidate_count <= 5 ? 2 : 3; - if (best_run < min_overlap_words) return candidate; - - if (best_candidate_start > 1) return candidate; - if (best_candidate_start == 1) { - const std::string lead_word = parakeet_word_from_span(candidate, candidate_spans[0]); - const std::string overlap_word = parakeet_word_from_span(candidate, candidate_spans[1]); - const std::string recent_word = parakeet_word_from_span( - recent_tail, recent_spans[best_recent_start]); - if (!parakeet_words_fuzzy_equal(lead_word, overlap_word) && - !parakeet_words_fuzzy_equal(lead_word, recent_word)) { - return candidate; - } - } - - return parakeet_drop_first_words(candidate, best_candidate_end); -} - -static std::string parakeet_strip_recent_committed_prefix( - const std::string& committed_text, - const std::string& hypothesis); - -static std::string parakeet_emit_delta( - const std::string& committed_text, - const std::string& newly_confirmed) { - std::string candidate = parakeet_strip_recent_committed_prefix( - committed_text, newly_confirmed); - if (candidate.empty()) return ""; - if (committed_text.empty()) return candidate; - - const size_t overlap_words = parakeet_overlap_suffix_prefix_words(committed_text, candidate); - if (overlap_words > 0) { - return parakeet_drop_first_words(candidate, overlap_words); - } - - return candidate; -} - -static std::string parakeet_strip_recent_committed_prefix( - const std::string& committed_text, - const std::string& hypothesis) { - const std::string candidate = suppress_unwanted_text(hypothesis); - if (candidate.empty() || committed_text.empty()) return candidate; - - const auto committed_spans = collect_parakeet_word_spans(committed_text); - if (committed_spans.empty()) return candidate; - - const size_t recent_words = std::min(committed_spans.size(), 24); - const size_t recent_start = committed_spans[committed_spans.size() - recent_words].first; - const std::string recent_tail = suppress_unwanted_text(committed_text.substr(recent_start)); - const size_t candidate_words = parakeet_word_count(candidate); - const size_t overlap_words = parakeet_overlap_suffix_prefix_words(recent_tail, candidate, recent_words); - - // Be conservative for tiny hypotheses, but strip even a single repeated - // boundary word when the new hypothesis is a longer continuation. - const size_t required_overlap = candidate_words >= 5 ? 1 : 2; - if (overlap_words >= required_overlap) { - return parakeet_drop_first_words(candidate, overlap_words); - } - - const std::string aligned_trimmed = parakeet_strip_recent_aligned_overlap( - committed_text, candidate, recent_words); - if (aligned_trimmed != candidate) { - return aligned_trimmed; - } - - const auto recent_spans = collect_parakeet_word_spans(recent_tail); - const auto candidate_spans = collect_parakeet_word_spans(candidate); - const size_t recent_count = recent_spans.size(); - const size_t candidate_count = candidate_spans.size(); - if (recent_count == 0 || candidate_count < 4) return candidate; - - std::vector> dp( - recent_count + 1, std::vector(candidate_count + 1, 0)); - for (size_t i = 1; i <= recent_count; ++i) { - for (size_t j = 1; j <= candidate_count; ++j) { - const std::string recent_word = parakeet_word_from_span(recent_tail, recent_spans[i - 1]); - const std::string candidate_word = parakeet_word_from_span(candidate, candidate_spans[j - 1]); - if (parakeet_words_fuzzy_equal(recent_word, candidate_word)) { - dp[i][j] = dp[i - 1][j - 1] + 1; - } else { - dp[i][j] = std::max(dp[i - 1][j], dp[i][j - 1]); - } - } - } - - const size_t lcs_words = dp[recent_count][candidate_count]; - if (lcs_words < 3 || lcs_words * 2 < candidate_count) return candidate; - - size_t i = recent_count; - size_t j = candidate_count; - size_t last_matched_candidate_word = 0; - bool found_match = false; - bool captured_last_match = false; - while (i > 0 && j > 0) { - const std::string recent_word = parakeet_word_from_span(recent_tail, recent_spans[i - 1]); - const std::string candidate_word = parakeet_word_from_span(candidate, candidate_spans[j - 1]); - if (parakeet_words_fuzzy_equal(recent_word, candidate_word)) { - if (!captured_last_match) { - last_matched_candidate_word = j; - captured_last_match = true; - } - found_match = true; - --i; - --j; - } else if (dp[i - 1][j] >= dp[i][j - 1]) { - --i; - } else { - --j; - } - } - - if (!found_match || last_matched_candidate_word >= candidate_count) return candidate; - return parakeet_drop_first_words(candidate, last_matched_candidate_word); - - return candidate; -} - -static std::string serialize_segments_with_offset( - const std::vector& segs, float offset_sec) { - std::ostringstream out; - out << "["; - for (size_t i = 0; i < segs.size(); ++i) { - if (i > 0) out << ","; - out << std::fixed << std::setprecision(3) - << "{\"start\":" << (segs[i].start + offset_sec) - << ",\"end\":" << (segs[i].end + offset_sec) - << ",\"text\":\"" << escape_json(segs[i].text) << "\"}"; - } - out << "]"; - return out.str(); -} - -static std::string build_stream_response( - const std::string& raw_json_str, - const std::string& error_msg, - const std::string& confirmed, - const std::string& pending, - const std::string& segments_json, - bool cloud_handoff, - double buffer_duration_ms, - uint64_t cloud_job_id, - uint64_t cloud_result_job_id, - const CloudResponse& cloud_result, - double override_time_to_first_token_ms = -1.0, - double override_total_time_ms = -1.0, - double override_decode_tps = -1.0, - double override_decode_tokens = -1.0, - double override_total_tokens = -1.0, - double override_raw_decoder_tps = -1.0 -) { - std::string function_calls = json_array(raw_json_str, "function_calls"); - double confidence = json_number(raw_json_str, "confidence"); - double time_to_first_token_ms = json_number(raw_json_str, "time_to_first_token_ms"); - double total_time_ms = json_number(raw_json_str, "total_time_ms"); - double prefill_tps = json_number(raw_json_str, "prefill_tps"); - double decode_tps = json_number(raw_json_str, "decode_tps"); - double ram_usage_mb = json_number(raw_json_str, "ram_usage_mb"); - double prefill_tokens = json_number(raw_json_str, "prefill_tokens"); - double decode_tokens = json_number(raw_json_str, "decode_tokens"); - double total_tokens = json_number(raw_json_str, "total_tokens"); - double raw_decoder_tps = json_number(raw_json_str, "raw_decoder_tps"); - if (override_time_to_first_token_ms >= 0.0) { - time_to_first_token_ms = override_time_to_first_token_ms; - } - if (override_total_time_ms >= 0.0) { - total_time_ms = override_total_time_ms; - } - if (override_decode_tps >= 0.0) { - decode_tps = override_decode_tps; - } - if (override_decode_tokens >= 0.0) { - decode_tokens = override_decode_tokens; - } - if (override_total_tokens >= 0.0) { - total_tokens = override_total_tokens; - } - if (override_raw_decoder_tps >= 0.0) { - raw_decoder_tps = override_raw_decoder_tps; - } - std::string effective_confirmed = confirmed; - if (cloud_result.used_cloud && !cloud_result.transcript.empty()) { - effective_confirmed = cloud_result.transcript; - } - - std::ostringstream json_builder; - json_builder << "{"; - json_builder << "\"success\":true,"; - json_builder << "\"buffer_duration_ms\":" << buffer_duration_ms << ","; - json_builder << "\"error\":" << (error_msg.empty() ? "null" : "\"" + escape_json(error_msg) + "\"") << ","; - json_builder << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; - json_builder << "\"cloud_job_id\":" << cloud_job_id << ","; - json_builder << "\"cloud_result_job_id\":" << cloud_result_job_id << ","; - json_builder << "\"cloud_result\":\"" << escape_json(cloud_result.transcript) << "\","; - json_builder << "\"cloud_result_used_cloud\":" << (cloud_result.used_cloud ? "true" : "false") << ","; - json_builder << "\"cloud_result_error\":"; - if (cloud_result.error.empty()) { - json_builder << "null,"; - } else { - json_builder << "\"" << escape_json(cloud_result.error) << "\","; - } - json_builder << "\"cloud_result_source\":\"" << (cloud_result.used_cloud ? "cloud" : "fallback") << "\","; - json_builder << "\"confirmed_local\":\"" << escape_json(confirmed) << "\","; - json_builder << "\"confirmed\":\"" << escape_json(effective_confirmed) << "\","; - json_builder << "\"pending\":\"" << escape_json(pending) << "\","; - json_builder << "\"segments\":" << segments_json << ","; - json_builder << "\"function_calls\":" << function_calls << ","; - json_builder << "\"confidence\":" << confidence << ","; - json_builder << "\"time_to_first_token_ms\":" << time_to_first_token_ms << ","; - json_builder << "\"total_time_ms\":" << total_time_ms << ","; - json_builder << "\"prefill_tps\":" << prefill_tps << ","; - json_builder << "\"decode_tps\":" << decode_tps << ","; - json_builder << "\"ram_usage_mb\":" << ram_usage_mb << ","; - json_builder << "\"prefill_tokens\":" << prefill_tokens << ","; - json_builder << "\"decode_tokens\":" << decode_tokens << ","; - json_builder << "\"total_tokens\":" << total_tokens << ","; - json_builder << "\"raw_decoder_tps\":" << raw_decoder_tps; - json_builder << "}"; - return json_builder.str(); -} - -extern "C" { - -cactus_stream_transcribe_t cactus_stream_transcribe_start(cactus_model_t model, const char* options_json) { - if (!model) { - last_error_message = "Model not initialized. Check model path and files."; - CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); - return nullptr; - } - - try { - auto* model_handle = static_cast(model); - if (!model_handle->model) { - last_error_message = "Invalid model handle."; - CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); - return nullptr; - } - - auto* stream_handle = new CactusStreamTranscribeHandle(); - stream_handle->model_handle = model_handle; - stream_handle->transcribe_response_buffer[0] = '\0'; - - auto session_start_time = std::chrono::steady_clock::now(); - stream_handle->stream_start = session_start_time; - stream_handle->stream_first_token_seen = false; - stream_handle->stream_first_token_ms = 0.0; - stream_handle->stream_total_tokens = 0; - - stream_handle->stream_session_start = session_start_time; - stream_handle->stream_session_first_token_seen = false; - stream_handle->stream_session_first_token_ms = 0.0; - stream_handle->stream_cumulative_tokens = 0; - - double confirmation_threshold; - size_t min_chunk_size; - bool telemetry_enabled; - std::string language; - parse_stream_transcribe_init_options( - options_json ? options_json : "", - confirmation_threshold, - min_chunk_size, - telemetry_enabled, - language - ); - - stream_handle->options = { confirmation_threshold, min_chunk_size, language }; - stream_handle->transcribe_options_json = options_json ? options_json : ""; - stream_handle->parakeet_tdt_chunked_stream = - model_handle->model->get_config().model_type == - cactus::engine::Config::ModelType::PARAKEET_TDT; - { - float vocabulary_boost = 5.0f; - parse_custom_vocabulary_options(stream_handle->transcribe_options_json, - stream_handle->custom_vocabulary, vocabulary_boost); - auto vocab_bias = build_custom_vocabulary_bias( - model_handle->model->get_tokenizer(), - stream_handle->custom_vocabulary, - vocabulary_boost - ); - stream_handle->has_custom_vocabulary_bias = !vocab_bias.empty(); - if (stream_handle->has_custom_vocabulary_bias) { - model_handle->model->set_vocab_bias(vocab_bias); - } - } - - if (stream_handle->parakeet_tdt_chunked_stream) { - auto& ctx = stream_handle->parakeet_tdt_decode_context; - ctx.mel_bins = std::max( - 1, static_cast(model_handle->model->get_config().num_mel_bins)); - auto cfg = get_parakeet_spectrogram_config(); - ctx.audio_processor.init_mel_filters( - cfg.n_fft / 2 + 1, - ctx.mel_bins, - 0.0f, - 8000.0f, - WHISPER_SAMPLE_RATE); - ctx.initialized = true; - } - - CACTUS_LOG_INFO("stream_transcribe_start", - "Stream transcription initialized for model: " << model_handle->model_name); - - return stream_handle; - } catch (const std::exception& e) { - last_error_message = "Exception during stream_transcribe_start: " + std::string(e.what()); - CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); - return nullptr; - } catch (...) { - last_error_message = "Unknown exception during stream transcription initialization"; - CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); - return nullptr; - } -} - -int cactus_stream_transcribe_process( - cactus_stream_transcribe_t stream, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - char* response_buffer, - size_t buffer_size -) { - if (!stream) { - last_error_message = "Stream not initialized."; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - if (!pcm_buffer || pcm_buffer_size == 0) { - last_error_message = "Invalid parameters: pcm_buffer or pcm_buffer_size"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - if (!response_buffer || buffer_size == 0) { - last_error_message = "Invalid parameters: response_buffer or buffer_size"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - try { - auto* handle = static_cast(stream); - const auto model_type = handle->model_handle->model->get_config().model_type; - bool is_moonshine = model_type == cactus::engine::Config::ModelType::MOONSHINE; - bool is_parakeet_tdt = - model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - bool is_parakeet = - model_type == cactus::engine::Config::ModelType::PARAKEET || - is_parakeet_tdt; - bool is_gemma4 = model_type == cactus::engine::Config::ModelType::GEMMA4; - - if (is_parakeet_tdt && - handle->parakeet_tdt_chunked_stream && - handle->parakeet_tdt_decode_context.initialized) { - auto& ctx = handle->parakeet_tdt_decode_context; - auto* tdt_model = static_cast( - handle->model_handle->model.get()); - auto new_samples = cactus::audio::pcm_buffer_to_float_samples( - pcm_buffer, pcm_buffer_size); - ctx.audio_samples.insert(ctx.audio_samples.end(), new_samples.begin(), new_samples.end()); - - constexpr size_t kTdtLeftContextSamples = 8 * 16000; - constexpr size_t kTdtRightContextSamples = 16000; - constexpr size_t kTdtChunkSamples = 16000; - constexpr size_t kTdtColdStartSamples = 6 * 16000; - - const size_t total_samples = ctx.audio_samples.size(); - const size_t decodable_up_to = total_samples > kTdtRightContextSamples - ? total_samples - kTdtRightContextSamples - : 0; - const size_t min_chunk_samples = ctx.samples_decoded_up_to == 0 - ? kTdtColdStartSamples - : kTdtChunkSamples; - - if (decodable_up_to <= ctx.samples_decoded_up_to || - decodable_up_to - ctx.samples_decoded_up_to < min_chunk_samples) { - const std::string json_response = build_stream_response( - "{}", - "", - "", - handle->previous_parakeet_pending, - "[]", - false, - 0.0, - 0, - 0, - CloudResponse{}, - handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, - handle->last_stream_total_time_ms, - handle->last_stream_decode_tps, - handle->last_stream_decode_tokens, - handle->last_stream_decode_tokens, - handle->last_stream_raw_decoder_tps); - if (json_response.length() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - const size_t window_start_sample = ctx.samples_decoded_up_to > kTdtLeftContextSamples - ? ctx.samples_decoded_up_to - kTdtLeftContextSamples - : 0; - const size_t window_end_sample = std::min( - total_samples, decodable_up_to + kTdtRightContextSamples); - std::vector window_audio( - ctx.audio_samples.begin() + window_start_sample, - ctx.audio_samples.begin() + window_end_sample); - - auto cfg = get_parakeet_spectrogram_config(); - const size_t waveform_samples = window_audio.size(); - apply_preemphasis(window_audio, 0.97f); - window_audio = ctx.audio_processor.compute_spectrogram(window_audio, cfg); - normalize_parakeet_log_mel(window_audio, ctx.mel_bins); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(window_audio, ctx.mel_bins, valid_frames); - - const uint32_t subsampling = - std::max(1, handle->model_handle->model->get_config().subsampling_factor); - const size_t samples_per_enc_frame = cfg.hop_length * subsampling; - const size_t context_samples = ctx.samples_decoded_up_to - window_start_sample; - const size_t decode_start_frame = context_samples / samples_per_enc_frame; - const size_t new_samples_count = decodable_up_to - ctx.samples_decoded_up_to; - const size_t decode_end_frame = - decode_start_frame + (new_samples_count / samples_per_enc_frame); - - if (decode_end_frame <= decode_start_frame) { - const std::string json_response = build_stream_response( - "{}", - "", - "", - handle->previous_parakeet_pending, - "[]", - false, - 0.0, - 0, - 0, - CloudResponse{}, - handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, - handle->last_stream_total_time_ms, - handle->last_stream_decode_tps, - handle->last_stream_decode_tokens, - handle->last_stream_decode_tokens, - handle->last_stream_raw_decoder_tps); - if (json_response.length() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; - { - const auto decode_start = std::chrono::steady_clock::now(); - std::lock_guard lock(handle->model_handle->model_mutex); - cactus::telemetry::setStreamMode(true); - try { - chunk_decode = tdt_model->decode_chunk_stream( - window_audio, - decode_start_frame, - decode_start_frame, - decode_end_frame, - ctx.decoder_state); - cactus_reset(handle->model_handle); - } catch (...) { - cactus::telemetry::setStreamMode(false); - throw; - } - cactus::telemetry::setStreamMode(false); - const auto decode_end = std::chrono::steady_clock::now(); - const double decode_ms = - std::chrono::duration_cast( - decode_end - decode_start).count() / 1000.0; - handle->last_stream_total_time_ms = decode_ms; - handle->last_stream_decode_tokens = static_cast(chunk_decode.token_count); - handle->last_stream_decode_tps = - (chunk_decode.token_count > 1 && decode_ms > 0.0) - ? ((static_cast(chunk_decode.token_count) - 1.0) * 1000.0) / decode_ms - : 0.0; - handle->last_stream_raw_decoder_tps = chunk_decode.raw_decoder_tps; - } - - const float window_offset_sec = ctx.audio_time_offset_sec + - static_cast(window_start_sample) / static_cast(WHISPER_SAMPLE_RATE); - std::vector confirmed_segments; - if (!chunk_decode.confirmed_text.empty()) { - confirmed_segments.emplace_back( - window_offset_sec + chunk_decode.start_sec, - window_offset_sec + chunk_decode.confirmed_end_sec, - suppress_unwanted_text(chunk_decode.confirmed_text)); - } - std::string confirmed_text = suppress_unwanted_text(chunk_decode.confirmed_text); - std::string pending_text = suppress_unwanted_text(chunk_decode.pending_text); - if (!confirmed_text.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed_text, handle->custom_vocabulary); - } - if (!pending_text.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(pending_text, handle->custom_vocabulary); - } - - if (!confirmed_text.empty()) { - if (chunk_decode.token_count > 0) { - handle->stream_total_tokens += static_cast(chunk_decode.token_count); - handle->stream_cumulative_tokens += static_cast(chunk_decode.token_count); - } - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - parakeet_append_monotonic_text(handle->parakeet_committed_text, confirmed_text); - } - - handle->previous_parakeet_pending = pending_text; - handle->previous_parakeet_pending_ticks = pending_text.empty() ? 0 : 1; - - if (chunk_decode.confirmed_token_count > 0) { - const size_t confirmed_samples_in_window = static_cast( - chunk_decode.confirmed_end_sec * WHISPER_SAMPLE_RATE); - ctx.samples_decoded_up_to = window_start_sample + confirmed_samples_in_window; - } else if (chunk_decode.token_count == 0) { - ctx.samples_decoded_up_to = decodable_up_to; - } - - if (ctx.samples_decoded_up_to > kTdtLeftContextSamples * 2) { - const size_t trim_samples = ctx.samples_decoded_up_to - kTdtLeftContextSamples; - ctx.audio_samples.erase( - ctx.audio_samples.begin(), - ctx.audio_samples.begin() + trim_samples); - ctx.samples_decoded_up_to -= trim_samples; - ctx.audio_time_offset_sec += static_cast(trim_samples) / 16000.0f; - } - - handle->parakeet_committed_until_sec = std::max( - handle->parakeet_committed_until_sec, - window_offset_sec + - (chunk_decode.confirmed_token_count > 0 ? chunk_decode.confirmed_end_sec : 0.0f)); - - const std::string json_response = build_stream_response( - "{}", - "", - confirmed_text, - pending_text, - serialize_segments_with_offset(confirmed_segments, 0.0f), - false, - 0.0, - 0, - 0, - CloudResponse{}, - handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, - handle->last_stream_total_time_ms, - handle->last_stream_decode_tps, - handle->last_stream_decode_tokens, - handle->last_stream_decode_tokens, - handle->last_stream_raw_decoder_tps); - if (json_response.length() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - handle->audio_buffer.insert( - handle->audio_buffer.end(), - pcm_buffer, - pcm_buffer + pcm_buffer_size - ); - CACTUS_LOG_DEBUG("stream_transcribe_process", - "Inserted " << pcm_buffer_size << " bytes, buffer size: " << handle->audio_buffer.size()); - - if (handle->audio_buffer.size() < handle->options.min_chunk_size * sizeof(int16_t)) { - std::string json_response = "{\"success\":true,\"confirmed\":\"\",\"pending\":\"\"}"; - - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - std::string whisper_prompt = "<|startoftranscript|><|" + handle->options.language + "|><|transcribe|><|notimestamps|>"; - const char* transcribe_prompt = - is_gemma4 ? "Transcribe the audio." : - (is_moonshine || is_parakeet) ? "" : - whisper_prompt.c_str(); - std::string effective_transcribe_options_json = handle->transcribe_options_json; - if (is_parakeet && !is_parakeet_tdt) { - effective_transcribe_options_json = ensure_json_bool_option( - effective_transcribe_options_json, "use_vad", false); - } - - if (is_parakeet_tdt) { - constexpr double kParakeetTdtLeftContextSec = 2.0; - constexpr double kParakeetTdtChunkSec = 0.25; - constexpr double kParakeetTdtRightContextSec = 1.0; - constexpr size_t kParakeetTdtMaxChunksPerProcess = 3; - - const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); - const size_t chunk_bytes = std::max( - seconds_to_pcm_bytes(kParakeetTdtChunkSec), - handle->options.min_chunk_size * sizeof(int16_t)); - const size_t right_context_bytes = seconds_to_pcm_bytes(kParakeetTdtRightContextSec); - const std::string tdt_vad_options_json = ensure_json_bool_option( - handle->transcribe_options_json, "use_vad", true); - const std::string tdt_no_vad_options_json = ensure_json_bool_option( - handle->transcribe_options_json, "use_vad", false); - - if (handle->parakeet_tdt_chunked_stream && - !handle->parakeet_tdt_chunked_blocked_until_pause && - !handle->parakeet_committed_text.empty()) { - const size_t encoder_frame_bytes = - get_parakeet_spectrogram_config().hop_length * - std::max(1, handle->model_handle->model->get_config().subsampling_factor) * - sizeof(int16_t); - std::string confirmed; - size_t confirmed_audio_bytes = 0; - size_t processed_chunks = 0; - bool chunked_stalled_on_speech = false; - - while (processed_chunks < kParakeetTdtMaxChunksPerProcess) { - const bool chunked_warmup = - handle->parakeet_committed_text.empty() && - handle->parakeet_tdt_pending_raw.empty(); - const size_t available_chunk_bytes = - handle->audio_buffer.size() > handle->parakeet_chunk_cursor_bytes + right_context_bytes - ? handle->audio_buffer.size() - handle->parakeet_chunk_cursor_bytes - right_context_bytes - : 0; - const size_t chunk_decode_bytes = std::max( - encoder_frame_bytes, - ((chunked_warmup - ? available_chunk_bytes - : seconds_to_pcm_bytes(0.96)) / - encoder_frame_bytes) * encoder_frame_bytes); - if (handle->parakeet_chunk_cursor_bytes + chunk_decode_bytes + right_context_bytes > - handle->audio_buffer.size()) { - break; - } - - const size_t chunk_start_bytes = handle->parakeet_chunk_cursor_bytes; - const size_t chunk_end_bytes = chunk_start_bytes + chunk_decode_bytes; - const size_t window_start_bytes = chunk_start_bytes > left_context_bytes - ? chunk_start_bytes - left_context_bytes - : 0; - const size_t window_end_bytes = chunk_end_bytes + right_context_bytes; - - cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; - cactus::engine::ParakeetTDTModel::ChunkStreamState chunk_state; - if (!run_parakeet_tdt_chunk_decode( - handle, - window_start_bytes, - window_end_bytes, - chunk_start_bytes, - chunk_end_bytes, - chunk_decode, - chunk_state)) { - last_error_message = "Chunked TDT decode failed."; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription( - handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, - false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, - last_error_message.c_str()); - return -1; - } - - const bool chunk_silent = is_pcm_chunk_silent( - handle->audio_buffer.data() + chunk_start_bytes, - chunk_end_bytes - chunk_start_bytes, - 0.012f); - const std::string raw_chunk_text = - strip_unwanted_text_preserving_edges(chunk_decode.text); - const std::string raw_confirmed_text = - strip_unwanted_text_preserving_edges(chunk_decode.confirmed_text); - const std::string raw_pending_text = - strip_unwanted_text_preserving_edges(chunk_decode.pending_text); - - if (raw_chunk_text.empty() && !chunk_silent) { - chunked_stalled_on_speech = true; - break; - } - - if (!raw_chunk_text.empty()) { - std::string confirmed_delta = suppress_unwanted_text(raw_confirmed_text); - std::string pending_raw = raw_pending_text; - if (!confirmed_delta.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed_delta, handle->custom_vocabulary); - } - handle->parakeet_tdt_pending_raw = pending_raw; - if (!confirmed_delta.empty()) { - if (chunk_decode.token_count > 0) { - handle->stream_total_tokens += static_cast(chunk_decode.token_count); - handle->stream_cumulative_tokens += static_cast(chunk_decode.token_count); - } - - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - - parakeet_append_monotonic_text( - handle->parakeet_committed_text, - confirmed_delta); - parakeet_append_monotonic_text(confirmed, confirmed_delta); - const size_t confirmed_boundary_bytes = std::min( - chunk_end_bytes, - std::max( - chunk_start_bytes, - window_start_bytes + seconds_to_pcm_bytes(chunk_decode.confirmed_end_sec))); - if (confirmed_boundary_bytes > chunk_start_bytes) { - confirmed_audio_bytes += confirmed_boundary_bytes - chunk_start_bytes; - } - } - } - - handle->parakeet_chunk_cursor_bytes = chunk_end_bytes; - handle->parakeet_tdt_decoder_state = chunk_state; - const size_t checkpoint_end_bytes = std::min( - chunk_end_bytes, - std::max( - window_start_bytes, - window_start_bytes + seconds_to_pcm_bytes(chunk_decode.resume_end_sec))); - while (!handle->parakeet_tdt_resume_checkpoints.empty() && - handle->parakeet_tdt_resume_checkpoints.back().end_bytes >= checkpoint_end_bytes) { - handle->parakeet_tdt_resume_checkpoints.pop_back(); - } - if (handle->parakeet_tdt_resume_checkpoints.empty() || - handle->parakeet_tdt_resume_checkpoints.back().end_bytes < checkpoint_end_bytes) { - handle->parakeet_tdt_resume_checkpoints.push_back({checkpoint_end_bytes, chunk_state}); - } - while (handle->parakeet_tdt_resume_checkpoints.size() > 16) { - handle->parakeet_tdt_resume_checkpoints.pop_front(); - } - handle->parakeet_committed_until_sec = std::max( - handle->parakeet_committed_until_sec, - static_cast( - handle->stream_audio_offset_sec + - pcm_bytes_to_seconds(checkpoint_end_bytes))); - ++processed_chunks; - } - - const bool fallback_to_legacy_tdt = - chunked_stalled_on_speech && - confirmed.empty() && - handle->parakeet_tdt_pending_raw.empty(); - if (fallback_to_legacy_tdt) { - const bool bootstrap_reset = - handle->parakeet_committed_text.empty() && - handle->previous_parakeet_pending.empty(); - reset_parakeet_tdt_chunk_stream(handle); - handle->parakeet_tdt_chunked_blocked_until_pause = true; - if (bootstrap_reset) { - handle->parakeet_chunk_cursor_bytes = 0; - } - } - - if (!fallback_to_legacy_tdt && - handle->parakeet_chunk_cursor_bytes > left_context_bytes) { - const size_t trim_bytes = - handle->parakeet_chunk_cursor_bytes - left_context_bytes; - handle->stream_audio_offset_sec += static_cast( - pcm_bytes_to_seconds(trim_bytes)); - handle->audio_buffer.erase( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + trim_bytes); - handle->parakeet_chunk_cursor_bytes -= trim_bytes; - while (!handle->parakeet_tdt_resume_checkpoints.empty() && - handle->parakeet_tdt_resume_checkpoints.front().end_bytes <= trim_bytes) { - handle->parakeet_tdt_resume_checkpoints.pop_front(); - } - for (auto& checkpoint : handle->parakeet_tdt_resume_checkpoints) { - checkpoint.end_bytes -= trim_bytes; - } - } - - if (!fallback_to_legacy_tdt) { - std::string pending_output; - std::string raw_json = "{}"; - if (!handle->parakeet_tdt_pending_raw.empty()) { - pending_output = suppress_unwanted_text(handle->parakeet_tdt_pending_raw); - if (!pending_output.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction( - pending_output, handle->custom_vocabulary); - } - } - if (handle->audio_buffer.size() > handle->parakeet_chunk_cursor_bytes) { - const size_t preview_start_bytes = handle->parakeet_chunk_cursor_bytes > left_context_bytes - ? handle->parakeet_chunk_cursor_bytes - left_context_bytes - : 0; - StreamWindowDecodeResult preview_decode; - if (run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_no_vad_options_json, - handle->audio_buffer.data() + preview_start_bytes, - handle->audio_buffer.size() - preview_start_bytes, - preview_decode)) { - raw_json = preview_decode.raw_json; - if (pending_output.empty()) { - pending_output = parakeet_strip_recent_committed_prefix( - handle->parakeet_committed_text, - preview_decode.response); - if (!pending_output.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction( - pending_output, handle->custom_vocabulary); - } - } - } - } - - handle->previous_parakeet_pending = pending_output; - handle->previous_parakeet_pending_ticks = pending_output.empty() ? 0 : 1; - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - handle->previous_segments.clear(); - handle->previous_cloud_handoff = false; - handle->parakeet_onset_active = false; - handle->parakeet_resume_guard_active = false; - - CloudResponse empty_cloud_result; - const std::string json_response = build_stream_response( - raw_json, - "", - confirmed, - pending_output, - "[]", - false, - pcm_bytes_to_seconds(confirmed_audio_bytes) * 1000.0, - 0, - 0, - empty_cloud_result, - handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, - raw_json == "{}" ? handle->last_stream_total_time_ms : -1.0, - raw_json == "{}" ? handle->last_stream_decode_tps : -1.0, - raw_json == "{}" ? handle->last_stream_decode_tokens : -1.0, - raw_json == "{}" ? handle->last_stream_decode_tokens : -1.0, - raw_json == "{}" ? handle->last_stream_raw_decoder_tps : -1.0); - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - } - - auto decode_window = [&](size_t window_start_bytes, - size_t window_end_bytes, - StreamWindowDecodeResult& out, - float& out_offset_sec) -> bool { - if (window_end_bytes <= window_start_bytes || window_end_bytes > handle->audio_buffer.size()) { - return false; - } - out_offset_sec = static_cast( - handle->stream_audio_offset_sec + pcm_bytes_to_seconds(window_start_bytes)); - StreamWindowDecodeResult vad_decode; - const bool vad_ok = run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_vad_options_json, - handle->audio_buffer.data() + window_start_bytes, - window_end_bytes - window_start_bytes, - vad_decode); - if (vad_ok && (!vad_decode.response.empty() || !vad_decode.segments.empty())) { - out = std::move(vad_decode); - return true; - } - - return run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_no_vad_options_json, - handle->audio_buffer.data() + window_start_bytes, - window_end_bytes - window_start_bytes, - out); - }; - - constexpr size_t kParakeetHoldbackWords = 1; - constexpr size_t kParakeetSilenceFlushBytes = 16000; // ~0.5s silence - constexpr size_t kParakeetStableFlushWords = 10; - constexpr size_t kParakeetStableFlushTicks = 2; - constexpr size_t kParakeetResumeGuardBytes = 24000; // ~0.75s speech - constexpr size_t kParakeetOnsetLeadSilenceBytes = 16000; // ~0.5s lead-in - constexpr size_t kParakeetOnsetMinConfirmWords = 6; - - const bool chunk_silent = is_pcm_chunk_silent(pcm_buffer, pcm_buffer_size, 0.012f); - const size_t prior_silence_run_bytes = handle->parakeet_silence_run_bytes; - const bool speech_resumed = - !chunk_silent && prior_silence_run_bytes >= kParakeetSilenceFlushBytes; - const size_t speech_chunk_start_bytes = - handle->audio_buffer.size() >= pcm_buffer_size - ? handle->audio_buffer.size() - pcm_buffer_size - : 0; - - if (chunk_silent) { - handle->parakeet_silence_run_bytes += pcm_buffer_size; - handle->parakeet_speech_run_bytes = 0; - if (handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes) { - handle->parakeet_resume_guard_active = true; - handle->parakeet_onset_active = false; - reset_parakeet_tdt_chunk_stream(handle); - const size_t retained_silence_bytes = std::min( - handle->parakeet_silence_run_bytes, kParakeetOnsetLeadSilenceBytes); - handle->parakeet_onset_start_bytes = - handle->audio_buffer.size() > retained_silence_bytes - ? handle->audio_buffer.size() - retained_silence_bytes - : 0; - } - } else { - if (!handle->parakeet_onset_active && - handle->parakeet_committed_text.empty() && - handle->previous_parakeet_pending.empty()) { - handle->parakeet_onset_active = true; - handle->parakeet_onset_start_bytes = 0; - } - if (speech_resumed) { - handle->parakeet_resume_guard_active = true; - handle->parakeet_speech_run_bytes = 0; - handle->parakeet_onset_active = true; - const size_t retained_silence_bytes = std::min( - prior_silence_run_bytes, kParakeetOnsetLeadSilenceBytes); - handle->parakeet_onset_start_bytes = - speech_chunk_start_bytes > retained_silence_bytes - ? speech_chunk_start_bytes - retained_silence_bytes - : 0; - } - handle->parakeet_silence_run_bytes = 0; - if (handle->parakeet_resume_guard_active) { - handle->parakeet_speech_run_bytes += pcm_buffer_size; - if (handle->parakeet_speech_run_bytes >= kParakeetResumeGuardBytes) { - handle->parakeet_resume_guard_active = false; - handle->parakeet_speech_run_bytes = 0; - } - } - } - - const bool detected_pause = - handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes; - const bool should_hold_cursor = handle->parakeet_onset_active; - - size_t processed_chunks = 0; - if (!should_hold_cursor) { - while (handle->parakeet_chunk_cursor_bytes + chunk_bytes + right_context_bytes <= - handle->audio_buffer.size() && - processed_chunks < kParakeetTdtMaxChunksPerProcess) { - handle->parakeet_chunk_cursor_bytes += chunk_bytes; - ++processed_chunks; - } - } - - if (!handle->parakeet_onset_active && - handle->parakeet_chunk_cursor_bytes > left_context_bytes) { - const size_t trim_bytes = - handle->parakeet_chunk_cursor_bytes - left_context_bytes; - handle->stream_audio_offset_sec += static_cast( - pcm_bytes_to_seconds(trim_bytes)); - handle->audio_buffer.erase( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + trim_bytes); - handle->parakeet_chunk_cursor_bytes -= trim_bytes; - } - - const bool onset_mode_active = handle->parakeet_onset_active; - const size_t preview_start_bytes = onset_mode_active - ? std::min(handle->parakeet_onset_start_bytes, handle->audio_buffer.size()) - : (handle->parakeet_chunk_cursor_bytes > left_context_bytes - ? handle->parakeet_chunk_cursor_bytes - left_context_bytes - : 0); - const size_t preview_end_bytes = handle->audio_buffer.size(); - - StreamWindowDecodeResult response_decode; - float response_segments_offset_sec = 0.0f; - if (!decode_window(preview_start_bytes, preview_end_bytes, response_decode, response_segments_offset_sec)) { - last_error_message = "Transcription failed in stream process."; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription( - handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, - false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, - last_error_message.c_str()); - return -1; - } - - std::string confirmed; - const std::string previous_pending = handle->previous_parakeet_pending; - std::string pending_tail = previous_pending; - const std::string normalized_response = parakeet_strip_recent_committed_prefix( - handle->parakeet_committed_text, response_decode.response); - if (!normalized_response.empty()) { - if (previous_pending.empty() || (onset_mode_active && handle->parakeet_resume_guard_active)) { - pending_tail = normalized_response; - } else { - const size_t stable_words = parakeet_common_prefix_words( - previous_pending, normalized_response); - const size_t confirm_words = - stable_words > kParakeetHoldbackWords - ? stable_words - kParakeetHoldbackWords - : 0; - confirmed = parakeet_take_first_words(normalized_response, confirm_words); - pending_tail = confirm_words > 0 - ? parakeet_drop_first_words(normalized_response, confirm_words) - : normalized_response; - } - } - - const bool onset_commit_too_small = - onset_mode_active && - !confirmed.empty() && - !detected_pause && - parakeet_word_count(confirmed) < kParakeetOnsetMinConfirmWords; - if (onset_commit_too_small) { - confirmed.clear(); - pending_tail = normalized_response; - } - - const bool pending_equivalent = - pending_tail == previous_pending || - (!pending_tail.empty() && - !previous_pending.empty() && - levenshtein_ci(pending_tail, previous_pending) <= 1); - const bool pending_changed = !pending_equivalent; - if (pending_changed) { - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } - - const bool stable_short_pending = - !pending_tail.empty() && - !handle->parakeet_resume_guard_active && - pending_equivalent && - handle->previous_parakeet_pending_ticks >= kParakeetStableFlushTicks && - parakeet_word_count(pending_tail) <= kParakeetStableFlushWords; - if ((detected_pause || stable_short_pending) && !pending_tail.empty()) { - if (!confirmed.empty()) confirmed += " "; - confirmed += pending_tail; - pending_tail.clear(); - } - - double buffer_duration_ms = 0.0; - bool cloud_handoff_triggered = false; - uint64_t cloud_job_id = 0; - uint64_t cloud_result_job_id = 0; - CloudResponse cloud_result; - - if (!confirmed.empty()) { - if (!handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); - } - - std::string confirmed_delta = parakeet_emit_delta( - handle->parakeet_committed_text, confirmed); - if (!confirmed_delta.empty()) { - if (response_decode.decode_tokens > 0.0) { - handle->stream_total_tokens += static_cast(std::round(response_decode.decode_tokens)); - handle->stream_cumulative_tokens += static_cast(std::round(response_decode.decode_tokens)); - } - - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - - if (!handle->parakeet_committed_text.empty()) { - handle->parakeet_committed_text += " "; - } - handle->parakeet_committed_text += confirmed_delta; - confirmed = confirmed_delta; - if (onset_mode_active) { - handle->parakeet_onset_active = false; - handle->parakeet_onset_start_bytes = 0; - handle->parakeet_chunk_cursor_bytes = - handle->audio_buffer.size() > right_context_bytes - ? handle->audio_buffer.size() - right_context_bytes - : 0; - } - buffer_duration_ms = - static_cast(processed_chunks) * kParakeetTdtChunkSec * 1000.0; - } else { - confirmed.clear(); - } - } - - if (pending_tail.empty()) { - handle->previous_parakeet_pending_ticks = 0; - } else if (pending_equivalent) { - handle->previous_parakeet_pending_ticks = std::min( - handle->previous_parakeet_pending_ticks + 1, 1000000); - } else { - handle->previous_parakeet_pending_ticks = 1; - } - handle->previous_parakeet_pending = pending_tail; - if (pending_tail.empty()) { - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } - - std::vector current_segments = response_decode.segments; - const float segments_offset_sec = response_segments_offset_sec; - const std::string json_str = response_decode.raw_json; - const std::string pending_output = pending_tail; - handle->previous_cloud_handoff = false; - - for (auto it = handle->pending_cloud_jobs.begin(); it != handle->pending_cloud_jobs.end(); ) { - if (it->result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { - CloudResponse job_result = it->result.get(); - CACTUS_LOG_INFO( - "cloud_handoff", - "Completed stream cloud job id=" << it->id - << " used_cloud=" << (job_result.used_cloud ? "true" : "false") - << " error=" << (job_result.error.empty() ? "none" : job_result.error) - << " transcript_chars=" << job_result.transcript.size()); - handle->completed_cloud_results.push_back({it->id, std::move(job_result)}); - it = handle->pending_cloud_jobs.erase(it); - } else { - ++it; - } - } - - if (!handle->completed_cloud_results.empty()) { - cloud_result_job_id = handle->completed_cloud_results.front().first; - cloud_result = handle->completed_cloud_results.front().second; - handle->completed_cloud_results.erase(handle->completed_cloud_results.begin()); - - if (!cloud_result.api_key_hash.empty()) { - cactus::telemetry::setCloudKey(cloud_result.api_key_hash.c_str()); - } - } - - constexpr int STREAM_TOKENS_CAP = 20000; - constexpr double STREAM_DURATION_CAP_MS = 600000.0; - auto now = std::chrono::steady_clock::now(); - double elapsed_ms = - std::chrono::duration_cast( - now - handle->stream_start).count(); - if (handle->stream_total_tokens >= STREAM_TOKENS_CAP || - elapsed_ms >= STREAM_DURATION_CAP_MS) { - double period_tps = (elapsed_ms > 0.0) - ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms - : 0.0; - - double cumulative_elapsed_ms = - std::chrono::duration_cast( - now - handle->stream_session_start).count(); - double cumulative_tps = (cumulative_elapsed_ms > 0.0) - ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / - cumulative_elapsed_ms - : 0.0; - - cactus::telemetry::recordStreamTranscription( - handle->model_handle->model_name.c_str(), - true, - handle->stream_first_token_ms, - period_tps, - elapsed_ms, - handle->stream_total_tokens, - handle->stream_session_first_token_ms, - cumulative_tps, - cumulative_elapsed_ms, - handle->stream_cumulative_tokens, - "" - ); - - handle->stream_start = std::chrono::steady_clock::now(); - handle->stream_first_token_seen = false; - handle->stream_first_token_ms = 0.0; - handle->stream_total_tokens = 0; - } - - std::string json_response = build_stream_response( - json_str, - json_string(json_str, "error"), - confirmed, - pending_output, - serialize_segments_with_offset(current_segments, segments_offset_sec), - cloud_handoff_triggered, - buffer_duration_ms, - cloud_job_id, - cloud_result_job_id, - cloud_result - ); - - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription( - handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, - false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, - last_error_message.c_str()); - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - const uint8_t* decode_pcm = handle->audio_buffer.data(); - size_t decode_pcm_size = handle->audio_buffer.size(); - if (is_parakeet) { - constexpr size_t kParakeetDecodeWindowBytes = 160000; // ~5.0s - if (decode_pcm_size > kParakeetDecodeWindowBytes) { - decode_pcm += (decode_pcm_size - kParakeetDecodeWindowBytes); - decode_pcm_size = kParakeetDecodeWindowBytes; - } - } - - cactus::telemetry::setStreamMode(true); - int result = cactus_transcribe( - handle->model_handle, - nullptr, - transcribe_prompt, - handle->transcribe_response_buffer, - sizeof(handle->transcribe_response_buffer), - effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), - nullptr, - nullptr, - decode_pcm, - decode_pcm_size); - cactus::telemetry::setStreamMode(false); - if (result < 0) { - last_error_message = "Transcription failed in stream process."; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - std::string json_str(handle->transcribe_response_buffer); - std::string response = suppress_unwanted_text(json_string(json_str, "response")); - - cactus_reset(handle->model_handle); - - auto current_segments = parse_segments(json_str); - const float segments_offset_sec = static_cast(handle->stream_audio_offset_sec); - - std::string confirmed; - double buffer_duration_ms = 0.0; - bool cloud_handoff_triggered = false; - uint64_t cloud_job_id = 0; - uint64_t cloud_result_job_id = 0; - CloudResponse cloud_result; - double chunk_decode_tokens = json_number(json_str, "decode_tokens"); - if (chunk_decode_tokens < 0.0) { - chunk_decode_tokens = 0.0; - } - - if (is_parakeet) { - constexpr size_t kParakeetHoldbackWords = 1; - constexpr size_t kParakeetTailBytes = 64000; // ~2.0s left context - constexpr size_t kParakeetSilenceFlushBytes = 16000; // ~0.5s silence - constexpr size_t kParakeetStableFlushWords = 4; - constexpr size_t kParakeetStableFlushTicks = 2; - constexpr size_t kParakeetMediumTrimWords = 3; - constexpr size_t kParakeetAggressiveTrimWords = 6; - - const std::string previous_pending = handle->previous_parakeet_pending; - std::string pending_tail = previous_pending; - const bool chunk_silent = is_pcm_chunk_silent(pcm_buffer, pcm_buffer_size, 0.012f); - if (chunk_silent) { - handle->parakeet_silence_run_bytes += pcm_buffer_size; - } else { - handle->parakeet_silence_run_bytes = 0; - } - if (!response.empty()) { - const std::string normalized_response = parakeet_strip_recent_committed_prefix( - handle->parakeet_committed_text, response); - if (previous_pending.empty()) { - pending_tail = normalized_response; - } else { - const size_t stable_words = parakeet_common_prefix_words(previous_pending, normalized_response); - size_t confirm_words = 0; - if (stable_words > kParakeetHoldbackWords) { - confirm_words = stable_words - kParakeetHoldbackWords; - } - - confirmed = parakeet_take_first_words(normalized_response, confirm_words); - if (confirm_words > 0) { - pending_tail = parakeet_drop_first_words(normalized_response, confirm_words); - } else { - pending_tail = normalized_response; - } - } - } - - const bool pending_equivalent = - pending_tail == previous_pending || - (!pending_tail.empty() && - !previous_pending.empty() && - levenshtein_ci(pending_tail, previous_pending) <= 1); - const bool pending_changed = !pending_equivalent; - if (pending_changed) { - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } - - const bool detected_pause = - handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes; - const bool stable_short_pending = - !pending_tail.empty() && - pending_equivalent && - handle->previous_parakeet_pending_ticks >= kParakeetStableFlushTicks && - parakeet_word_count(pending_tail) <= kParakeetStableFlushWords && - handle->audio_buffer.size() >= - (handle->previous_parakeet_audio_buffer_size + (kParakeetSilenceFlushBytes / 2)); - if ((detected_pause || stable_short_pending) && !pending_tail.empty()) { - if (!confirmed.empty()) confirmed += " "; - confirmed += pending_tail; - pending_tail.clear(); - } - - if (!confirmed.empty()) { - if (!handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); - } - - std::string confirmed_delta = parakeet_emit_delta( - handle->parakeet_committed_text, confirmed); - - const size_t max_trim = handle->audio_buffer.size() > kParakeetTailBytes - ? handle->audio_buffer.size() - kParakeetTailBytes - : 0; - const size_t confirmed_words = parakeet_word_count(confirmed); - const size_t hop_bytes = std::clamp( - pcm_buffer_size, static_cast(4000), static_cast(16000)); - size_t confirmed_bytes = std::min(hop_bytes, max_trim); - if (confirmed_words >= kParakeetAggressiveTrimWords || pending_tail.empty()) { - confirmed_bytes = max_trim; - } else if (confirmed_words >= kParakeetMediumTrimWords) { - confirmed_bytes = std::min(max_trim, hop_bytes * 2); - } - buffer_duration_ms = (confirmed_bytes / 2.0) / 16000.0 * 1000.0; - - if (!confirmed_delta.empty()) { - if (chunk_decode_tokens > 0.0) { - handle->stream_total_tokens += static_cast(std::round(chunk_decode_tokens)); - handle->stream_cumulative_tokens += static_cast(std::round(chunk_decode_tokens)); - } - - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = std::chrono::duration_cast(now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - - if (!handle->parakeet_committed_text.empty()) { - handle->parakeet_committed_text += " "; - } - handle->parakeet_committed_text += confirmed_delta; - } - - if (confirmed_bytes > 0) { - if (handle->previous_cloud_handoff && !confirmed.empty()) { - cloud_handoff_triggered = true; - std::vector confirmed_audio( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + confirmed_bytes - ); - auto wav = cloud_build_wav(confirmed_audio.data(), confirmed_audio.size()); - std::string b64 = cloud_base64_encode(wav.data(), wav.size()); - cloud_job_id = handle->next_cloud_job_id++; - CACTUS_LOG_INFO( - "cloud_handoff", - "Queued stream cloud job id=" << cloud_job_id - << " confirmed_chars=" << confirmed.size() - << " audio_bytes=" << confirmed_audio.size()); - handle->pending_cloud_jobs.push_back({ - cloud_job_id, - std::async(std::launch::async, cloud_transcribe_request, b64, confirmed, 15L, nullptr) - }); - } - - handle->stream_audio_offset_sec += static_cast(confirmed_bytes) / (16000.0f * 2.0f); - handle->audio_buffer.erase( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + confirmed_bytes - ); - } - - confirmed = confirmed_delta; - handle->previous_cloud_handoff = false; - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } else { - handle->previous_cloud_handoff = json_bool(json_str, "cloud_handoff"); - } - - if (pending_tail.empty()) { - handle->previous_parakeet_pending_ticks = 0; - } else if (pending_equivalent) { - handle->previous_parakeet_pending_ticks = std::min( - handle->previous_parakeet_pending_ticks + 1, 1000000); - } else { - handle->previous_parakeet_pending_ticks = 1; - } - handle->previous_parakeet_pending = pending_tail; - if (pending_tail.empty()) { - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } - handle->previous_segments = current_segments; - } else { - size_t confirmed_segments = 0; - float confirmed_end_sec = 0.0f; - for (size_t i = 0; i < std::min(handle->previous_segments.size(), current_segments.size()); ++i) { - if (handle->previous_segments[i].text != current_segments[i].text) { - break; - } - confirmed_segments = i + 1; - confirmed_end_sec = std::min(handle->previous_segments[i].end, current_segments[i].end); - } - - if (confirmed_segments > 0 && confirmed_end_sec > 0.0f) { - buffer_duration_ms = confirmed_end_sec * 1000.0; - - for (size_t i = 0; i < confirmed_segments; ++i) { - if (!confirmed.empty()) confirmed += ' '; - confirmed += handle->previous_segments[i].text; - } - - if (!handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); - } - - if (chunk_decode_tokens > 0.0) { - handle->stream_total_tokens += static_cast(std::round(chunk_decode_tokens)); - handle->stream_cumulative_tokens += static_cast(std::round(chunk_decode_tokens)); - } - - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = std::chrono::duration_cast(now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - - const size_t confirmed_bytes = std::min( - static_cast(confirmed_end_sec * 16000.0) * 2, - handle->audio_buffer.size() - ); - - if (handle->previous_cloud_handoff && !confirmed.empty()) { - cloud_handoff_triggered = true; - std::vector confirmed_audio( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + confirmed_bytes - ); - auto wav = cloud_build_wav(confirmed_audio.data(), confirmed_audio.size()); - std::string b64 = cloud_base64_encode(wav.data(), wav.size()); - cloud_job_id = handle->next_cloud_job_id++; - CACTUS_LOG_INFO( - "cloud_handoff", - "Queued stream cloud job id=" << cloud_job_id - << " confirmed_chars=" << confirmed.size() - << " audio_bytes=" << confirmed_audio.size()); - handle->pending_cloud_jobs.push_back({ - cloud_job_id, - std::async(std::launch::async, cloud_transcribe_request, b64, confirmed, 15L, nullptr) - }); - } - - handle->stream_audio_offset_sec += confirmed_end_sec; - handle->audio_buffer.erase( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + confirmed_bytes - ); - constexpr float kSegmentEpsilonSec = 0.02f; - size_t current_tail_start = 0; - while (current_tail_start < current_segments.size() && - current_segments[current_tail_start].end <= confirmed_end_sec + kSegmentEpsilonSec) { - ++current_tail_start; - } - handle->previous_segments = std::vector( - current_segments.begin() + current_tail_start, current_segments.end()); - handle->previous_cloud_handoff = false; - } else { - handle->previous_segments = current_segments; - handle->previous_cloud_handoff = json_bool(json_str, "cloud_handoff"); - } - } - - for (auto it = handle->pending_cloud_jobs.begin(); it != handle->pending_cloud_jobs.end(); ) { - if (it->result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { - CloudResponse job_result = it->result.get(); - CACTUS_LOG_INFO( - "cloud_handoff", - "Completed stream cloud job id=" << it->id - << " used_cloud=" << (job_result.used_cloud ? "true" : "false") - << " error=" << (job_result.error.empty() ? "none" : job_result.error) - << " transcript_chars=" << job_result.transcript.size()); - handle->completed_cloud_results.push_back({it->id, std::move(job_result)}); - it = handle->pending_cloud_jobs.erase(it); - } else { - ++it; - } - } - - if (!handle->completed_cloud_results.empty()) { - cloud_result_job_id = handle->completed_cloud_results.front().first; - cloud_result = handle->completed_cloud_results.front().second; - handle->completed_cloud_results.erase(handle->completed_cloud_results.begin()); - - if (!cloud_result.api_key_hash.empty()) { - cactus::telemetry::setCloudKey(cloud_result.api_key_hash.c_str()); - } - } - - constexpr int STREAM_TOKENS_CAP = 20000; - constexpr double STREAM_DURATION_CAP_MS = 600000.0; - auto now = std::chrono::steady_clock::now(); - double elapsed_ms = std::chrono::duration_cast(now - handle->stream_start).count(); - if (handle->stream_total_tokens >= STREAM_TOKENS_CAP || elapsed_ms >= STREAM_DURATION_CAP_MS) { - double period_tps = (elapsed_ms > 0.0) ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms : 0.0; - - double cumulative_elapsed_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); - double cumulative_tps = (cumulative_elapsed_ms > 0.0) ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / cumulative_elapsed_ms : 0.0; - - cactus::telemetry::recordStreamTranscription( - handle->model_handle->model_name.c_str(), - true, - handle->stream_first_token_ms, - period_tps, - elapsed_ms, - handle->stream_total_tokens, - handle->stream_session_first_token_ms, - cumulative_tps, - cumulative_elapsed_ms, - handle->stream_cumulative_tokens, - "" - ); - - handle->stream_start = std::chrono::steady_clock::now(); - handle->stream_first_token_seen = false; - handle->stream_first_token_ms = 0.0; - handle->stream_total_tokens = 0; - } - - std::string pending_output = response; - if (is_parakeet && !handle->previous_parakeet_pending.empty()) { - pending_output = handle->previous_parakeet_pending; - } - - std::string json_response = build_stream_response( - json_str, - json_string(json_str, "error"), - confirmed, - pending_output, - serialize_segments_with_offset(current_segments, segments_offset_sec), - cloud_handoff_triggered, - buffer_duration_ms, - cloud_job_id, - cloud_result_job_id, - cloud_result - ); - - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } catch (const std::exception& e) { - last_error_message = "Exception during stream_transcribe_process: " + std::string(e.what()); - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(e.what(), response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown exception during stream transcription processing"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response("Unknown error during stream processing", response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, "Unknown error during stream processing"); - return -1; - } -} - -int cactus_stream_transcribe_stop( - cactus_stream_transcribe_t stream, - char* response_buffer, - size_t buffer_size -) { - if (!stream) { - last_error_message = "Stream not initialized."; - CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); - return -1; - } - - auto* handle = static_cast(stream); - auto clear_stream_vocab_bias = [&]() { - if (handle->has_custom_vocabulary_bias && - handle->model_handle && - handle->model_handle->model) { - handle->model_handle->model->clear_vocab_bias(); - } - }; - - if (!response_buffer || buffer_size == 0) { - clear_stream_vocab_bias(); - delete handle; - return 0; - } - - try { - const auto model_type = handle->model_handle->model->get_config().model_type; - bool is_moonshine = model_type == cactus::engine::Config::ModelType::MOONSHINE; - bool is_parakeet_tdt = - model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - bool is_parakeet = - model_type == cactus::engine::Config::ModelType::PARAKEET || - is_parakeet_tdt; - bool is_tinyllama = model_type == cactus::engine::Config::ModelType::GEMMA4; - - std::string final_confirmed; - if (is_parakeet_tdt) { - final_confirmed = handle->parakeet_committed_text; - if (!(handle->parakeet_tdt_chunked_stream && handle->parakeet_tdt_decode_context.initialized) && - !handle->previous_parakeet_pending.empty()) { - const std::string normalized_pending = parakeet_strip_recent_committed_prefix( - final_confirmed, handle->previous_parakeet_pending); - const std::string pending_delta = parakeet_emit_delta( - final_confirmed, normalized_pending); - if (!pending_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += pending_delta; - } - } - } else if (is_parakeet) { - final_confirmed = handle->parakeet_committed_text; - if (!handle->previous_parakeet_pending.empty()) { - const std::string normalized_pending = parakeet_strip_recent_committed_prefix( - final_confirmed, handle->previous_parakeet_pending); - const std::string pending_delta = parakeet_emit_delta( - final_confirmed, normalized_pending); - if (!pending_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += pending_delta; - } - } - } else { - for (const auto& seg : handle->previous_segments) { - if (!final_confirmed.empty()) final_confirmed += ' '; - final_confirmed += seg.text; - } - } - - if (is_parakeet_tdt && - handle->parakeet_tdt_chunked_stream && - handle->parakeet_tdt_decode_context.initialized) { - auto& ctx = handle->parakeet_tdt_decode_context; - auto* tdt_model = static_cast( - handle->model_handle->model.get()); - const size_t total_samples = ctx.audio_samples.size(); - if (total_samples > ctx.samples_decoded_up_to) { - constexpr size_t kTdtLeftContextSamples = 8 * 16000; - const size_t window_start_sample = ctx.samples_decoded_up_to > kTdtLeftContextSamples - ? ctx.samples_decoded_up_to - kTdtLeftContextSamples - : 0; - std::vector window_audio( - ctx.audio_samples.begin() + window_start_sample, - ctx.audio_samples.end()); - - auto cfg = get_parakeet_spectrogram_config(); - const size_t waveform_samples = window_audio.size(); - apply_preemphasis(window_audio, 0.97f); - window_audio = ctx.audio_processor.compute_spectrogram(window_audio, cfg); - normalize_parakeet_log_mel(window_audio, ctx.mel_bins); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(window_audio, ctx.mel_bins, valid_frames); - - const uint32_t subsampling = - std::max(1, handle->model_handle->model->get_config().subsampling_factor); - const size_t samples_per_enc_frame = cfg.hop_length * subsampling; - const size_t context_samples = ctx.samples_decoded_up_to - window_start_sample; - const size_t decode_start_frame = context_samples / samples_per_enc_frame; - const size_t total_window_samples = total_samples - window_start_sample; - const size_t decode_end_frame = total_window_samples / samples_per_enc_frame; - - if (decode_end_frame > decode_start_frame) { - cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; - { - std::lock_guard lock(handle->model_handle->model_mutex); - cactus::telemetry::setStreamMode(true); - try { - chunk_decode = tdt_model->decode_chunk_stream( - window_audio, - decode_start_frame, - decode_start_frame, - decode_end_frame, - ctx.decoder_state); - cactus_reset(handle->model_handle); - } catch (...) { - cactus::telemetry::setStreamMode(false); - throw; - } - cactus::telemetry::setStreamMode(false); - } - - std::string confirmed_flush = suppress_unwanted_text(chunk_decode.confirmed_text); - std::string pending_flush = suppress_unwanted_text(chunk_decode.pending_text); - if (!confirmed_flush.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed_flush, handle->custom_vocabulary); - } - if (!pending_flush.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(pending_flush, handle->custom_vocabulary); - } - parakeet_append_monotonic_text(final_confirmed, confirmed_flush); - parakeet_append_monotonic_text(final_confirmed, pending_flush); - } - } else if (!handle->previous_parakeet_pending.empty()) { - parakeet_append_monotonic_text(final_confirmed, handle->previous_parakeet_pending); - } - } - - if (!handle->audio_buffer.empty()) { - std::string whisper_prompt = "<|startoftranscript|><|" + handle->options.language + "|><|transcribe|><|notimestamps|>"; - const char* transcribe_prompt = - is_tinyllama ? "Transcribe the audio." : - (is_moonshine || is_parakeet) ? "" : - whisper_prompt.c_str(); - std::string effective_transcribe_options_json = handle->transcribe_options_json; - if (is_parakeet && !is_parakeet_tdt) { - effective_transcribe_options_json = ensure_json_bool_option( - effective_transcribe_options_json, "use_vad", false); - } - - if (is_parakeet_tdt) { - const bool use_chunked_stop = - handle->parakeet_tdt_chunked_stream && - !handle->parakeet_tdt_chunked_blocked_until_pause && - (handle->parakeet_tdt_decoder_state.initialized || - !handle->parakeet_tdt_resume_checkpoints.empty() || - !handle->parakeet_tdt_pending_raw.empty()); - if (use_chunked_stop) { - final_confirmed = handle->parakeet_committed_text; - constexpr double kParakeetTdtLeftContextSec = 2.0; - const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); - if (handle->parakeet_chunk_cursor_bytes < handle->audio_buffer.size()) { - const size_t chunk_start_bytes = handle->parakeet_chunk_cursor_bytes; - const size_t chunk_end_bytes = handle->audio_buffer.size(); - const size_t window_start_bytes = chunk_start_bytes > left_context_bytes - ? chunk_start_bytes - left_context_bytes - : 0; - cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; - cactus::engine::ParakeetTDTModel::ChunkStreamState chunk_state; - if (run_parakeet_tdt_chunk_decode( - handle, - window_start_bytes, - chunk_end_bytes, - chunk_start_bytes, - chunk_end_bytes, - chunk_decode, - chunk_state)) { - std::string confirmed_chunk = suppress_unwanted_text(chunk_decode.confirmed_text); - std::string pending_chunk = suppress_unwanted_text(chunk_decode.pending_text); - if (!confirmed_chunk.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed_chunk, handle->custom_vocabulary); - } - if (!pending_chunk.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(pending_chunk, handle->custom_vocabulary); - } - parakeet_append_monotonic_text(final_confirmed, confirmed_chunk); - handle->parakeet_tdt_pending_raw = - strip_unwanted_text_preserving_edges(chunk_decode.pending_text); - handle->parakeet_chunk_cursor_bytes = chunk_end_bytes; - handle->parakeet_tdt_decoder_state = chunk_state; - handle->parakeet_committed_until_sec = std::max( - handle->parakeet_committed_until_sec, - static_cast( - handle->stream_audio_offset_sec + - pcm_bytes_to_seconds( - std::min( - chunk_end_bytes, - std::max( - window_start_bytes, - window_start_bytes + seconds_to_pcm_bytes( - chunk_decode.resume_end_sec)))))); - } - } else if (!handle->previous_parakeet_pending.empty()) { - const std::string normalized_pending = parakeet_strip_recent_committed_prefix( - final_confirmed, handle->previous_parakeet_pending); - const std::string pending_delta = parakeet_emit_delta( - final_confirmed, normalized_pending); - if (!pending_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += pending_delta; - } - } - if (!handle->parakeet_tdt_pending_raw.empty()) { - std::string pending_flush = suppress_unwanted_text( - handle->parakeet_tdt_pending_raw); - if (!pending_flush.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction( - pending_flush, handle->custom_vocabulary); - } - parakeet_append_monotonic_text(final_confirmed, pending_flush); - handle->parakeet_tdt_pending_raw.clear(); - } - handle->parakeet_committed_text = final_confirmed; - } else { - constexpr double kParakeetTdtLeftContextSec = 2.0; - constexpr float kParakeetTdtCommitEpsilonSec = 0.03f; - const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); - const size_t flush_start_bytes = handle->parakeet_onset_active - ? std::min(handle->parakeet_onset_start_bytes, handle->audio_buffer.size()) - : (handle->parakeet_chunk_cursor_bytes > left_context_bytes - ? handle->parakeet_chunk_cursor_bytes - left_context_bytes - : 0); - const std::string tdt_vad_options_json = ensure_json_bool_option( - handle->transcribe_options_json, "use_vad", true); - const std::string tdt_no_vad_options_json = ensure_json_bool_option( - handle->transcribe_options_json, "use_vad", false); - - StreamWindowDecodeResult flush_decode; - bool flush_ok = run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_vad_options_json, - handle->audio_buffer.data() + flush_start_bytes, - handle->audio_buffer.size() - flush_start_bytes, - flush_decode); - if (flush_ok && flush_decode.response.empty() && flush_decode.segments.empty()) { - flush_ok = run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_no_vad_options_json, - handle->audio_buffer.data() + flush_start_bytes, - handle->audio_buffer.size() - flush_start_bytes, - flush_decode); - } - if (flush_ok) { - const float flush_offset_sec = static_cast( - handle->stream_audio_offset_sec + pcm_bytes_to_seconds(flush_start_bytes)); - float flushed_until_sec = handle->parakeet_committed_until_sec; - std::string flushed_text = join_segments_by_end_time( - flush_decode.segments, - flush_offset_sec, - handle->parakeet_committed_until_sec + kParakeetTdtCommitEpsilonSec, - std::numeric_limits::max(), - &flushed_until_sec); - if (flushed_text.empty()) { - flushed_text = parakeet_strip_recent_committed_prefix( - final_confirmed, flush_decode.response); - } - const std::string effective_flush_delta = - parakeet_emit_delta(final_confirmed, flushed_text); - if (!effective_flush_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += effective_flush_delta; - } - if (flushed_until_sec > handle->parakeet_committed_until_sec) { - handle->parakeet_committed_until_sec = flushed_until_sec; - } - } else { - CACTUS_LOG_WARN( - "stream_transcribe_stop", - "Final flush transcription failed; returning previously confirmed transcript."); - } - } - } else { - cactus::telemetry::setStreamMode(true); - const int flush_result = cactus_transcribe( - handle->model_handle, - nullptr, - transcribe_prompt, - handle->transcribe_response_buffer, - sizeof(handle->transcribe_response_buffer), - effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), - nullptr, - nullptr, - handle->audio_buffer.data(), - handle->audio_buffer.size()); - cactus::telemetry::setStreamMode(false); - - cactus_reset(handle->model_handle); - - if (flush_result >= 0) { - std::string flush_json(handle->transcribe_response_buffer); - std::string flushed_text = suppress_unwanted_text(json_string(flush_json, "response")); - if (flushed_text.empty()) { - auto flush_segments = parse_segments(flush_json); - for (const auto& seg : flush_segments) { - if (!flushed_text.empty()) flushed_text += ' '; - flushed_text += seg.text; - } - } - if (!flushed_text.empty()) { - if (is_parakeet) { - const std::string normalized_flush = parakeet_strip_recent_committed_prefix( - final_confirmed, flushed_text); - const std::string effective_flush_delta = - parakeet_emit_delta(final_confirmed, normalized_flush); - if (!effective_flush_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += effective_flush_delta; - } - } else { - final_confirmed = flushed_text; - } - } - } else { - CACTUS_LOG_WARN( - "stream_transcribe_stop", - "Final flush transcription failed; returning previously confirmed segments."); - } - } - } - - if (!handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(final_confirmed, handle->custom_vocabulary); - } - - std::string json_response = "{\"success\":true,\"confirmed\":\"" + - escape_json(final_confirmed) + "\"}"; - - auto now = std::chrono::steady_clock::now(); - double elapsed_ms = std::chrono::duration_cast(now - handle->stream_start).count(); - double period_tps = (elapsed_ms > 0.0) ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms : 0.0; - - double cumulative_elapsed_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); - double cumulative_tps = (cumulative_elapsed_ms > 0.0) ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / cumulative_elapsed_ms : 0.0; - - cactus::telemetry::recordStreamTranscription( - handle->model_handle->model_name.c_str(), - true, - handle->stream_first_token_ms, - period_tps, - elapsed_ms, - handle->stream_total_tokens, - handle->stream_session_first_token_ms, - cumulative_tps, - cumulative_elapsed_ms, - handle->stream_cumulative_tokens, - "" - ); - - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, last_error_message.c_str()); - clear_stream_vocab_bias(); - delete handle; - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - clear_stream_vocab_bias(); - delete handle; - return static_cast(json_response.length()); - } catch (const std::exception& e) { - last_error_message = "Exception during stream_transcribe_stop: " + std::string(e.what()); - CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); - handle_error_response(e.what(), response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, e.what()); - clear_stream_vocab_bias(); - delete handle; - return -1; - } catch (...) { - last_error_message = "Unknown exception during stream transcription stop"; - CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); - handle_error_response("Unknown error during stream stop", response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, "Unknown error during stream stop"); - clear_stream_vocab_bias(); - delete handle; - return -1; - } -} - -} diff --git a/cactus-transpile/ffi/cactus_telemetry.cpp b/cactus-transpile/ffi/cactus_telemetry.cpp deleted file mode 100644 index 1844f0e22..000000000 --- a/cactus-transpile/ffi/cactus_telemetry.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "cactus_ffi.h" -#include "telemetry/telemetry.h" - -extern "C" { - -void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version) { - cactus::telemetry::setTelemetryEnvironment(framework, cache_location, version); -} - -void cactus_set_app_id(const char* app_id) { - cactus::telemetry::setAppId(app_id); -} - -void cactus_telemetry_flush(void) { - cactus::telemetry::flush(); -} - -void cactus_telemetry_shutdown(void) { - cactus::telemetry::shutdown(); -} - -} diff --git a/cactus-transpile/ffi/cactus_transcribe.cpp b/cactus-transpile/ffi/cactus_transcribe.cpp deleted file mode 100644 index 539c0f155..000000000 --- a/cactus-transpile/ffi/cactus_transcribe.cpp +++ /dev/null @@ -1,829 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../models/model.h" -#include "telemetry/telemetry.h" -#include "../../libs/audio/wav.h" -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; -using cactus::audio::WHISPER_TARGET_FRAMES; -using cactus::audio::WHISPER_SAMPLE_RATE; -using cactus::audio::apply_preemphasis; -using cactus::audio::get_parakeet_spectrogram_config; -using cactus::audio::get_whisper_spectrogram_config; -using cactus::audio::init_whisper_mel_filters; -using cactus::audio::normalize_parakeet_log_mel; -using cactus::audio::normalize_whisper_mel; -using cactus::audio::trim_mel_frames; -using cactus::audio::get_htk_spectrogram_config; -using cactus::audio::get_gemma4_audio_spectrogram_config; -using cactus::audio::transpose_mel_to_frame_major; - -static constexpr size_t WHISPER_MAX_DECODER_POSITIONS = 448; -static constexpr size_t MAX_CHUNK_SAMPLES = WHISPER_SAMPLE_RATE * 30; -static constexpr size_t MAX_CONTEXT_WORDS = 64; - -static std::string extract_whisper_language_code(std::string token_text) { - const size_t open = token_text.find("<|"); - if (open == std::string::npos) return ""; - - const size_t close = token_text.find("|>", open + 2); - if (close == std::string::npos || close <= open + 2) return ""; - - std::string inner = token_text.substr(open + 2, close - (open + 2)); - std::transform(inner.begin(), inner.end(), inner.begin(), [](unsigned char c) { - return static_cast(std::tolower(c)); - }); - - static const std::vector non_language_tokens = { - "startoftranscript", "transcribe", "translate", "notimestamps", - "endoftext", "nospeech", "nocaptions", "sot", "sot_prev", "sot_lm" - }; - for (const auto& reserved : non_language_tokens) { - if (inner == reserved) return ""; - } - - if (inner.size() < 2 || inner.size() > 8) return ""; - for (char c : inner) { - const bool ok = (c >= 'a' && c <= 'z') || c == '-'; - if (!ok) return ""; - } - return inner; -} - -static bool is_terminal_transcription_piece(const std::string& piece) { - return piece == "<|endoftext|>" || - piece == "<|endoftranscript|>" || - piece == "" || - piece == ""; -} - -extern "C" { - -int cactus_transcribe( - cactus_model_t model, - const char* audio_file_path, - const char* prompt, - char* response_buffer, - size_t buffer_size, - const char* options_json, - cactus_token_callback callback, - void* user_data, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (validate_audio_params("transcribe", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) - return -1; - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - std::lock_guard lock(handle->model_mutex); - handle->should_stop = false; - - float cloud_handoff_threshold = handle->model->get_config().default_cloud_handoff_threshold; - const std::string opts = options_json ? options_json : ""; - InferenceOptions options = parse_inference_options_json(opts); - { - size_t pos = opts.find("\"cloud_handoff_threshold\""); - if (pos != std::string::npos) { - pos = opts.find(':', pos); - if (pos != std::string::npos) { - ++pos; - while (pos < opts.size() && std::isspace(static_cast(opts[pos]))) ++pos; - try { - cloud_handoff_threshold = std::stof(opts.c_str() + pos); - } catch (...) {} - } - } - } - - const char* force_handoff_env = std::getenv("CACTUS_FORCE_HANDOFF"); - if (force_handoff_env && force_handoff_env[0] == '1' && force_handoff_env[1] == '\0') { - cloud_handoff_threshold = 0.0001f; - } - - const bool request_has_custom_vocabulary_options = - opts.find("\"custom_vocabulary\"") != std::string::npos || - opts.find("\"vocabulary_boost\"") != std::string::npos; - const bool apply_request_scoped_vocabulary_bias = - request_has_custom_vocabulary_options && !handle->model->has_vocab_bias(); - - struct ScopedVocabularyBiasReset { - Model* model; - bool clear_on_exit; - ~ScopedVocabularyBiasReset() { - if (clear_on_exit && model) { - model->clear_vocab_bias(); - } - } - } scoped_vocabulary_bias_reset{ - handle->model.get(), - apply_request_scoped_vocabulary_bias - }; - std::vector custom_vocabulary; - float vocabulary_boost_unused = 5.0f; - parse_custom_vocabulary_options(opts, custom_vocabulary, vocabulary_boost_unused); - - if (apply_request_scoped_vocabulary_bias) { - apply_custom_vocabulary_options(handle->model.get(), opts); - } - - if (request_has_custom_vocabulary_options && - options.temperature == 0.0f && options.top_p <= 0.0f && options.top_k == 0) { - // Keep deterministic decoding while ensuring bias is applied in sampling. - options.top_k = 1; - } - - bool is_whisper = handle->model->get_config().model_type == cactus::engine::Config::ModelType::WHISPER; - bool is_moonshine = handle->model->get_config().model_type == cactus::engine::Config::ModelType::MOONSHINE; - bool is_parakeet_tdt = handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - bool is_parakeet = - handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET || - handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - bool is_gemma4 = handle->model->get_config().model_type == cactus::engine::Config::ModelType::GEMMA4; - - std::vector audio_samples; - if (audio_file_path == nullptr) { - auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); - audio_samples = resample_to_16k_fp32(waveform_fp32, WHISPER_SAMPLE_RATE); - } else { - AudioFP32 audio = load_wav(audio_file_path); - audio_samples = resample_to_16k_fp32(audio.samples, audio.sample_rate); - } - - if (opts.find("\"max_tokens\"") == std::string::npos) { - const float audio_length_sec = static_cast(audio_samples.size()) / static_cast(WHISPER_SAMPLE_RATE); - const float tps = is_parakeet ? 30.0f : (is_gemma4 ? 30.0f : 20.0f); - const size_t estimated = static_cast(audio_length_sec * tps); - options.max_tokens = std::max(estimated, 100); - } - - if (is_gemma4) { - if (audio_samples.empty()) { - handle_error_response("No audio input provided", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "No audio input"); - return -1; - } - - const auto& model_config = handle->model->get_config(); - uint32_t audio_token_id = model_config.audio_token_id; - if (audio_token_id == 0) { - CACTUS_LOG_WARN("transcribe", "audio_token_id not set in config, using default 258881"); - audio_token_id = 258881; - } - - auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, model_config); - std::vector& audio_features = audio_prep.features; - size_t num_soft_tokens = audio_prep.num_soft_tokens; - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - CACTUS_LOG_ERROR("transcribe", "Tokenizer unavailable"); - handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Tokenizer unavailable"); - return -1; - } - - std::string task_text = (prompt[0] != '\0') ? std::string(prompt) : "Transcribe the audio."; - auto prefix_tokens = tokenizer->encode("<|turn>user\n" + task_text + "<|audio>"); - auto suffix_tokens = tokenizer->encode("\n<|turn>model\n"); - - std::vector tokens; - tokens.reserve(prefix_tokens.size() + num_soft_tokens + suffix_tokens.size()); - tokens.insert(tokens.end(), prefix_tokens.begin(), prefix_tokens.end()); - for (size_t j = 0; j < num_soft_tokens; j++) - tokens.push_back(audio_token_id); - tokens.insert(tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); - - std::vector> stop_token_sequences = {{ tokenizer->get_eos_token() }}; - auto append_stop = [&](const char* stop_text) { - std::vector seq = tokenizer->encode(stop_text); - if (!seq.empty()) - stop_token_sequences.push_back(std::move(seq)); - }; - append_stop(""); - append_stop(""); - append_stop(""); - - const size_t prompt_token_count = tokens.size(); - double time_to_first_token = 0.0; - size_t completion_tokens = 0; - std::string final_text; - float total_entropy_sum = 0.0f; - float max_token_entropy_norm = 0.0f; - std::vector generated_tokens; - generated_tokens.reserve(options.max_tokens); - - for (size_t i = 0; i < options.max_tokens; ++i) { - if (handle->should_stop) break; - - float token_entropy = 0.0f; - uint32_t next_token = handle->model->decode_with_audio( - tokens, audio_features, - options.temperature, options.top_p, options.top_k, - "", &token_entropy, - options.min_p, options.repetition_penalty - ); - - if (completion_tokens == 0) [[unlikely]] { - auto t_first = std::chrono::high_resolution_clock::now(); - time_to_first_token = std::chrono::duration_cast(t_first - start_time).count() / 1000.0; - } - - total_entropy_sum += token_entropy; - if (token_entropy > max_token_entropy_norm) max_token_entropy_norm = token_entropy; - - generated_tokens.emplace_back(next_token); - if (matches_stop_sequence(generated_tokens, stop_token_sequences)) - break; - - std::string piece = tokenizer->decode({ next_token }); - tokens.emplace_back(next_token); - completion_tokens++; - final_text += piece; - if (callback) callback(piece.c_str(), next_token, user_data); - } - - cactus_reset(model); - - float mean_entropy = completion_tokens > 0 ? total_entropy_sum / static_cast(completion_tokens) : 0.0f; - float confidence = 1.0f - mean_entropy; - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - double prefill_tps = time_to_first_token > 0 ? (prompt_token_count * 1000.0) / time_to_first_token : 0.0; - double decode_time_ms = std::max(0.0, total_time_ms - time_to_first_token); - double decode_tps = (completion_tokens > 1 && decode_time_ms > 0.0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; - - if (!final_text.empty() && final_text[0] == ' ') - final_text.erase(0, 1); - - // Strip thinking channel content: <|channel>thought... - { - auto chan_start = final_text.find("<|channel>"); - auto chan_end = final_text.find(""); - if (chan_start != std::string::npos && chan_end != std::string::npos && chan_end > chan_start) { - size_t after = chan_end + std::string("").length(); - final_text = final_text.substr(after); - if (!final_text.empty() && final_text[0] == ' ') - final_text.erase(0, 1); - } - } - - const bool cloud_handoff = !final_text.empty() && final_text.length() > 5 && - cloud_handoff_threshold > 0.0f && max_token_entropy_norm > cloud_handoff_threshold; - - std::string json = construct_response_json(final_text, {}, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_token_count, completion_tokens, confidence, cloud_handoff); - - if (json.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); - return -1; - } - - cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, time_to_first_token, decode_tps, total_time_ms, static_cast(completion_tokens), get_ram_usage_mb(), ""); - std::strcpy(response_buffer, json.c_str()); - return static_cast(json.size()); - } - - auto to_sec = [](size_t samples) { - return static_cast(samples) / static_cast(WHISPER_SAMPLE_RATE); - }; - - struct AudioChunk { - std::vector audio; - float start_sec; - float end_sec; - std::vector> anchors; - }; - std::vector audio_chunks; - - if (options.use_vad) { - auto* vad = static_cast(handle->vad_model.get()); - auto vad_segments = vad->get_speech_timestamps(audio_samples, {}); - audio_chunks.reserve(vad_segments.size()); - - std::vector current; - std::vector> current_anchors; - size_t chunk_start_sample = 0; - size_t chunk_end_sample = 0; - float concat_cursor = 0.0f; - for (const auto& seg : vad_segments) { - size_t end = std::min(seg.end, audio_samples.size()); - if (current.size() + (end - seg.start) > MAX_CHUNK_SAMPLES) { - audio_chunks.emplace_back(std::move(current), to_sec(chunk_start_sample), to_sec(chunk_end_sample), std::move(current_anchors)); - current.clear(); - current_anchors.clear(); - concat_cursor = 0.0f; - } - if (current.empty()) chunk_start_sample = seg.start; - chunk_end_sample = end; - current_anchors.emplace_back(concat_cursor, to_sec(seg.start)); - concat_cursor += to_sec(end - seg.start); - current.insert( - current.end(), - audio_samples.begin() + seg.start, - audio_samples.begin() + end - ); - } - - if (!current.empty()) { - audio_chunks.emplace_back(std::move(current), to_sec(chunk_start_sample), to_sec(chunk_end_sample), std::move(current_anchors)); - } - - if (audio_chunks.empty()) { - CACTUS_LOG_DEBUG("transcribe", "VAD detected only silence, returning empty transcription"); - auto vad_end_time = std::chrono::high_resolution_clock::now(); - double vad_total_time_ms = std::chrono::duration_cast(vad_end_time - start_time).count() / 1000.0; - std::string json = construct_response_json("", {}, 0.0, vad_total_time_ms, 0.0, 0.0, 0, 0, 1.0f); - if (json.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); - return -1; - } - cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, 0.0, 0.0, vad_total_time_ms, 0, get_ram_usage_mb(), ""); - std::strcpy(response_buffer, json.c_str()); - return static_cast(json.size()); - } - } else { - audio_chunks.reserve((audio_samples.size() + MAX_CHUNK_SAMPLES - 1) / MAX_CHUNK_SAMPLES); - for (size_t start = 0; start < audio_samples.size(); start += MAX_CHUNK_SAMPLES) { - size_t end = std::min(start + MAX_CHUNK_SAMPLES, audio_samples.size()); - audio_chunks.emplace_back(std::vector(audio_samples.begin() + start, audio_samples.begin() + end), to_sec(start), to_sec(end)); - } - } - - auto cfg = is_parakeet ? get_parakeet_spectrogram_config() : get_whisper_spectrogram_config(); - size_t mel_bins = std::max(1, static_cast(handle->model->get_config().num_mel_bins)); - const bool is_whisper_v3 = is_whisper && mel_bins > 80; - AudioProcessor ap; - if (is_parakeet) { - ap.init_mel_filters(cfg.n_fft / 2 + 1, mel_bins, 0.0f, 8000.0f, WHISPER_SAMPLE_RATE); - } else if (!is_moonshine) { - init_whisper_mel_filters(ap, cfg, mel_bins); - } - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - CACTUS_LOG_ERROR("transcribe", "Tokenizer unavailable"); - handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Tokenizer unavailable"); - return -1; - } - - std::vector initial_tokens = tokenizer->encode( - prompt && prompt[0] != '\0' - ? std::string(prompt) - : (is_whisper ? "<|startoftranscript|>" : std::string()) - ); - if (initial_tokens.empty() && !is_moonshine && !is_parakeet) { - CACTUS_LOG_ERROR("transcribe", "Decoder input tokens empty after encoding prompt"); - handle_error_response("Decoder input tokens empty", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Decoder input tokens empty"); - return -1; - } - - float max_tps = handle->model->get_config().default_max_tps; - if (max_tps < 0) max_tps = 100; - - std::vector> stop_token_sequences = {{ tokenizer->get_eos_token() }}; - auto append_exact_stop_sequence = [&](const char* stop_text) { - std::vector seq = tokenizer->encode(stop_text); - if (!seq.empty() && tokenizer->decode(seq) == stop_text) { - stop_token_sequences.push_back(std::move(seq)); - } - }; - append_exact_stop_sequence("<|endoftext|>"); - append_exact_stop_sequence("<|endoftranscript|>"); - append_exact_stop_sequence(""); - append_exact_stop_sequence(""); - - double time_to_first_token = 0.0; - size_t completion_tokens = 0; - std::string final_text; - float total_entropy_sum = 0.0f; - float max_token_entropy_norm = 0.0f; - - const std::regex whisper_timestamp_re(R"(<\|(\d+(?:\.\d+)?)\|>)"); - std::vector segments; - - auto sop = tokenizer->encode("<|startofprev|>"); - auto sot = tokenizer->encode("<|startoftranscript|>"); - auto zero = tokenizer->encode("<|0.00|>"); - auto notimestamps = tokenizer->encode("<|notimestamps|>"); - const bool has_notimestamps = !notimestamps.empty() && - std::search(initial_tokens.begin(), initial_tokens.end(), - notimestamps.begin(), notimestamps.end()) != initial_tokens.end(); - auto sot_it = std::search(initial_tokens.begin(), initial_tokens.end(), sot.begin(), sot.end()); - const auto sot_begin = sot_it != initial_tokens.end() ? sot_it : initial_tokens.begin(); - - for (auto& audio_chunk : audio_chunks) { - if (handle->should_stop || completion_tokens >= options.max_tokens) break; - - std::vector chunk_audio = std::move(audio_chunk.audio); - const float audio_chunk_length_sec = static_cast(chunk_audio.size()) / static_cast(WHISPER_SAMPLE_RATE); - - std::vector tokens; - if (final_text.empty() || is_parakeet || is_moonshine) { - tokens = initial_tokens; - } else { - size_t word_count = 0, pos = final_text.size(); - while (pos > 0 && word_count < MAX_CONTEXT_WORDS) { - while (pos > 0 && std::isspace((unsigned char)final_text[pos - 1])) --pos; - while (pos > 0 && !std::isspace((unsigned char)final_text[pos - 1])) --pos; - ++word_count; - } - tokens = sop; - auto ctx = tokenizer->encode(final_text.substr(pos)); - tokens.insert(tokens.end(), ctx.begin(), ctx.end()); - tokens.insert(tokens.end(), sot_begin, initial_tokens.end()); - } - - if (is_whisper && !has_notimestamps && !zero.empty() && (tokens.empty() || tokens.back() != zero.back())) { - tokens.insert(tokens.end(), zero.begin(), zero.end()); - } - - if (!is_moonshine) { - if (is_parakeet) { - size_t waveform_samples = chunk_audio.size(); - apply_preemphasis(chunk_audio, 0.97f); - chunk_audio = ap.compute_spectrogram(chunk_audio, cfg); - normalize_parakeet_log_mel(chunk_audio, mel_bins); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(chunk_audio, mel_bins, valid_frames); - } else { - std::vector mel = ap.compute_spectrogram(chunk_audio, cfg); - chunk_audio = normalize_whisper_mel(mel, mel_bins, is_whisper_v3); - } - } - - if (chunk_audio.empty()) { - CACTUS_LOG_DEBUG("transcribe", "Chunk audio features empty, skipping"); - continue; - } - - CACTUS_LOG_DEBUG("transcribe", "Chunk audio features size: " << chunk_audio.size()); - - size_t chunk_max_tokens = options.max_tokens - completion_tokens; - if (!is_parakeet) { - size_t max_allowed = tokens.size() < WHISPER_MAX_DECODER_POSITIONS ? - WHISPER_MAX_DECODER_POSITIONS - tokens.size() : 0; - if (chunk_max_tokens > max_allowed) chunk_max_tokens = max_allowed; - } - size_t max_tps_tokens = std::max(1, static_cast(audio_chunk_length_sec * max_tps)); - if (chunk_max_tokens > max_tps_tokens) chunk_max_tokens = max_tps_tokens; - - tokens.reserve(tokens.size() + chunk_max_tokens); - std::vector generated_tokens; - generated_tokens.reserve(chunk_max_tokens); - - float segment_start = audio_chunk.start_sec; - float segment_end = audio_chunk.end_sec; - float orig_offset = audio_chunk.start_sec; - size_t anchor_idx = 0; - std::string segment_text; - - for (size_t i = 0; i < chunk_max_tokens; ++i) { - if (handle->should_stop) break; - - float token_entropy = 0.0f; - float tok_time_start = 0.0f; - float tok_time_end = 0.0f; - uint32_t next_token = handle->model->decode_with_audio( - tokens, - chunk_audio, - options.temperature, - options.top_p, - options.top_k, - "", - &token_entropy, - options.min_p, - options.repetition_penalty, - is_parakeet_tdt ? &tok_time_start : nullptr, - is_parakeet_tdt ? &tok_time_end : nullptr - ); - - if (completion_tokens == 0) [[unlikely]] { - auto t_first = std::chrono::high_resolution_clock::now(); - time_to_first_token = std::chrono::duration_cast(t_first - start_time).count() / 1000.0; - } - - total_entropy_sum += token_entropy; - if (token_entropy > max_token_entropy_norm) max_token_entropy_norm = token_entropy; - - generated_tokens.emplace_back(next_token); - if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { - break; - } - - std::string piece = tokenizer->decode({ next_token }); - if (is_terminal_transcription_piece(piece)) { - break; - } - tokens.emplace_back(next_token); - completion_tokens++; - - std::smatch timestamp_match; - if (is_parakeet_tdt) { - while (anchor_idx + 1 < audio_chunk.anchors.size() && audio_chunk.anchors[anchor_idx + 1].first <= tok_time_end) { - ++anchor_idx; - orig_offset = audio_chunk.anchors[anchor_idx].second - audio_chunk.anchors[anchor_idx].first; - } - const bool new_word = !piece.empty() && piece[0] == ' '; - if (new_word && !segment_text.empty()) { - final_text += segment_text; - segments.emplace_back( - segment_start, - segment_end, - segment_text[0] == ' ' - ? (segment_text.erase(0, 1), std::move(segment_text)) - : std::move(segment_text) - ); - segment_text.clear(); - } - if (segment_text.empty()) segment_start = tok_time_start + orig_offset; - segment_text += piece; - segment_end = tok_time_end + orig_offset; - if (callback) callback(piece.c_str(), next_token, user_data); - continue; - } else if (std::regex_match(piece, timestamp_match, whisper_timestamp_re)) { - const float ts_sec = std::stof(timestamp_match[1].str()); - while (anchor_idx + 1 < audio_chunk.anchors.size() && audio_chunk.anchors[anchor_idx + 1].first <= ts_sec) { - ++anchor_idx; - orig_offset = audio_chunk.anchors[anchor_idx].second - audio_chunk.anchors[anchor_idx].first; - } - segment_end = std::min(ts_sec + orig_offset, audio_chunk.end_sec); - if (!segment_text.empty()) { - final_text += segment_text; - segments.emplace_back( - segment_start, - segment_end, - segment_text[0] == ' ' - ? (segment_text.erase(0, 1), std::move(segment_text)) - : std::move(segment_text) - ); - segment_text.clear(); - } - segment_start = segment_end; - continue; - } - - segment_text += piece; - if (callback) callback(piece.c_str(), next_token, user_data); - } - - if (!segment_text.empty()) { - final_text += segment_text; - segments.emplace_back( - segment_start, - segment_end, - segment_text[0] == ' ' - ? (segment_text.erase(0, 1), std::move(segment_text)) - : std::move(segment_text) - ); - } - - cactus_reset(model); - } - - float mean_entropy = completion_tokens > 0 ? total_entropy_sum / static_cast(completion_tokens) : 0.0f; - float confidence = 1.0f - mean_entropy; - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - size_t prompt_tokens = initial_tokens.size(); - double prefill_tps = time_to_first_token > 0 ? (prompt_tokens * 1000.0) / time_to_first_token : 0.0; - double decode_time_ms = std::max(0.0, total_time_ms - time_to_first_token); - double decode_tps = (completion_tokens > 1 && decode_time_ms > 0.0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; - - if (!final_text.empty() && final_text[0] == ' ') final_text.erase(0, 1); - - if (!custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(final_text, custom_vocabulary); - for (auto& seg : segments) { - apply_vocabulary_spelling_correction(seg.text, custom_vocabulary); - } - } - - const bool cloud_handoff = !final_text.empty() && final_text.length() > 5 && - cloud_handoff_threshold > 0.0f && max_token_entropy_norm > cloud_handoff_threshold; - - std::string json = construct_response_json(final_text, {}, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_tokens, completion_tokens, confidence, cloud_handoff, "", segments); - - if (json.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); - return -1; - } - - cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, time_to_first_token, decode_tps, total_time_ms, static_cast(completion_tokens), get_ram_usage_mb(), ""); - - std::strcpy(response_buffer, json.c_str()); - - return static_cast(json.size()); - } - catch (const std::exception& e) { - CACTUS_LOG_ERROR("transcribe", "Exception: " << e.what()); - handle_error_response(e.what(), response_buffer, buffer_size); - cactus::telemetry::recordTranscription(model ? static_cast(model)->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, e.what()); - return -1; - } - catch (...) { - CACTUS_LOG_ERROR("transcribe", "Unknown exception during transcription"); - handle_error_response("Unknown error in transcribe", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(model ? static_cast(model)->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, "Unknown error in transcribe"); - return -1; - } -} - -int cactus_detect_language( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (!model) { - std::string error_msg = last_error_message.empty() ? "Model not initialized." : last_error_message; - CACTUS_LOG_ERROR("detect_language", error_msg); - handle_error_response(error_msg, response_buffer, buffer_size); - return -1; - } - if (!response_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("detect_language", "Invalid parameters: response_buffer or buffer_size"); - handle_error_response("Invalid parameters", response_buffer, buffer_size); - return -1; - } - if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { - CACTUS_LOG_ERROR("detect_language", "No audio input provided"); - handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); - return -1; - } - if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { - CACTUS_LOG_ERROR("detect_language", "Both audio_file_path and pcm_buffer provided"); - handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); - return -1; - } - if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { - CACTUS_LOG_ERROR("detect_language", "Invalid pcm_buffer_size: " << pcm_buffer_size); - handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); - return -1; - } - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - std::lock_guard lock(handle->model_mutex); - handle->should_stop = false; - - if (handle->model->get_config().model_type != cactus::engine::Config::ModelType::WHISPER) { - handle_error_response("Language detection currently requires a Whisper model", response_buffer, buffer_size); - return -1; - } - - InferenceOptions options = parse_inference_options_json(options_json ? options_json : ""); - - std::vector audio_buffer; - if (audio_file_path == nullptr) { - const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); - size_t num_samples = pcm_buffer_size / 2; - - std::vector waveform_fp32(num_samples); - for (size_t i = 0; i < num_samples; i++) { - waveform_fp32[i] = static_cast(pcm_samples[i]) / 32768.0f; - } - audio_buffer = resample_to_16k_fp32(waveform_fp32, WHISPER_SAMPLE_RATE); - } else { - AudioFP32 audio = load_wav(audio_file_path); - audio_buffer = resample_to_16k_fp32(audio.samples, audio.sample_rate); - } - - if (options.use_vad && handle->vad_model) { - auto* vad = static_cast(handle->vad_model.get()); - auto segments = vad->get_speech_timestamps(audio_buffer, {}); - - std::vector speech_audio; - for (const auto& segment : segments) { - speech_audio.insert( - speech_audio.end(), - audio_buffer.begin() + segment.start, - audio_buffer.begin() + std::min(segment.end, audio_buffer.size()) - ); - } - audio_buffer = std::move(speech_audio); - } - - if (audio_buffer.empty()) { - handle_error_response("No speech detected in audio input", response_buffer, buffer_size); - return -1; - } - - AudioProcessor ap; - auto cfg = get_whisper_spectrogram_config(); - const size_t mel_bins = std::max(1, static_cast(handle->model->get_config().num_mel_bins)); - init_whisper_mel_filters(ap, cfg, mel_bins); - std::vector mel = ap.compute_spectrogram(audio_buffer, cfg); - std::vector features = normalize_whisper_mel(mel, mel_bins, mel_bins > 80); - if (features.empty()) { - handle_error_response("Computed audio features are empty", response_buffer, buffer_size); - return -1; - } - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); - return -1; - } - - std::vector decode_tokens = tokenizer->encode("<|startoftranscript|>"); - if (decode_tokens.empty()) { - handle_error_response("Failed to encode Whisper start token", response_buffer, buffer_size); - return -1; - } - - handle->model->reset_cache(); - float entropy = 1.0f; - uint32_t token_id = 0; - std::string token_text; - std::string language; - - // Whisper may emit one or more control tokens before the language token. - // Decode a few initial tokens and select the first valid language token. - constexpr size_t kMaxLanguageProbeSteps = 4; - for (size_t step = 0; step < kMaxLanguageProbeSteps; ++step) { - float step_entropy = 1.0f; - const uint32_t step_token_id = handle->model->decode_with_audio( - decode_tokens, features, 0.0f, 0.0f, 0, "", &step_entropy, - 0.0f, 1.0f - ); - const std::string step_token_text = tokenizer->decode({step_token_id}); - const std::string step_language = extract_whisper_language_code(step_token_text); - - token_id = step_token_id; - token_text = step_token_text; - entropy = step_entropy; - - if (!step_language.empty()) { - language = step_language; - break; - } - - decode_tokens.push_back(step_token_id); - } - handle->model->reset_cache(); - - if (language.empty()) { - language = "unknown"; - } - - float confidence = 1.0f - entropy; - if (confidence < 0.0f) confidence = 0.0f; - if (confidence > 1.0f) confidence = 1.0f; - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"language\":\"" << escape_json_string(language) << "\","; - json << "\"language_token\":\"" << escape_json_string(token_text) << "\","; - json << "\"token_id\":" << token_id << ","; - json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; - json << "\"entropy\":" << std::fixed << std::setprecision(4) << entropy << ","; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - - std::string response = json.str(); - if (response.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, response.c_str()); - return static_cast(response.size()); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("detect_language", "Exception: " << e.what()); - handle_error_response(e.what(), response_buffer, buffer_size); - return -1; - } catch (...) { - CACTUS_LOG_ERROR("detect_language", "Unknown exception during language detection"); - handle_error_response("Unknown error in detect_language", response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus-transpile/ffi/cactus_utils.h b/cactus-transpile/ffi/cactus_utils.h deleted file mode 100644 index 925a7c99c..000000000 --- a/cactus-transpile/ffi/cactus_utils.h +++ /dev/null @@ -1,1850 +0,0 @@ -#ifndef CACTUS_UTILS_H -#define CACTUS_UTILS_H - -#include "../engine/engine.h" -#include "../models/model.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#elif defined(_WIN32) -#include -#include -#elif defined(__linux__) || defined(__ANDROID__) -#include -#endif - -inline size_t get_memory_footprint_bytes() { -#ifdef __APPLE__ - task_vm_info_data_t vm_info; - mach_msg_type_number_t count = TASK_VM_INFO_COUNT; - if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count) == KERN_SUCCESS) - return vm_info.phys_footprint; - -#elif defined(_WIN32) - PROCESS_MEMORY_COUNTERS_EX pmc; - if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) - return pmc.PrivateUsage; - -#elif defined(__linux__) || defined(__ANDROID__) - std::ifstream statm("/proc/self/statm"); - if (statm.is_open()) { - size_t size, resident; - statm >> size >> resident; - return resident * sysconf(_SC_PAGESIZE); - } -#endif - return 0; -} - -inline double get_ram_usage_mb() { - return get_memory_footprint_bytes() / (1024.0 * 1024.0); -} - -struct CactusModelHandle { - std::unique_ptr model; - std::unique_ptr vad_model; - std::atomic should_stop; - std::vector processed_tokens; - struct ProcessedImage { - std::string path; - long long last_modified_timestamp = 0; - - bool operator==(const ProcessedImage& other) const { - return path == other.path && last_modified_timestamp == other.last_modified_timestamp; - } - }; - - std::vector> processed_images; - std::mutex model_mutex; - std::string model_name; - std::unique_ptr corpus_index; - std::string corpus_dir; - size_t corpus_embedding_dim = 0; - std::vector> tool_embeddings; - std::vector tool_texts; - - CactusModelHandle() : should_stop(false) {} -}; - -extern std::string last_error_message; - -bool matches_stop_sequence(const std::vector& generated_tokens, - const std::vector>& stop_sequences); - -std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query); - -namespace cactus { -namespace audio { - -static constexpr size_t WHISPER_TARGET_FRAMES = 3000; -static constexpr int WHISPER_SAMPLE_RATE = 16000; - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_whisper_spectrogram_config() { - cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; - cfg.n_fft = 400; - cfg.frame_length = 400; - cfg.hop_length = 160; - cfg.power = 2.0f; - cfg.center = true; - cfg.pad_mode = "reflect"; - cfg.onesided = true; - cfg.dither = 0.0f; - cfg.mel_floor = 1e-10f; - cfg.log_mel = "log10"; - cfg.reference = 1.0f; - cfg.min_value = 1e-10f; - cfg.remove_dc_offset = true; - return cfg; -} - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_parakeet_spectrogram_config() { - cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; - cfg.n_fft = 512; - cfg.frame_length = 400; - cfg.hop_length = 160; - cfg.power = 2.0f; - cfg.center = true; - cfg.pad_mode = "constant"; - cfg.onesided = true; - cfg.dither = 0.0f; - cfg.mel_floor = 5.960464477539063e-08f; // 2^-24 guard value used by HF Parakeet. - cfg.log_mel = "log"; - cfg.reference = 1.0f; - cfg.min_value = 1e-10f; - cfg.remove_dc_offset = false; - cfg.hann_periodic = false; - return cfg; -} - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_htk_spectrogram_config() { - cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; - cfg.n_fft = 321; - cfg.frame_length = 320; - cfg.fft_override = 1024; - cfg.hop_length = 160; - cfg.power = 1.0f; - cfg.center = false; - cfg.pad_mode = "constant"; - cfg.onesided = true; - cfg.dither = 0.0f; - cfg.mel_floor = 0.001f; - cfg.log_mel = "log"; - cfg.reference = 1.0f; - cfg.min_value = 0.001f; - cfg.remove_dc_offset = false; - cfg.hann_periodic = true; - return cfg; -} - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_gemma4_audio_spectrogram_config( - const cactus::engine::Config& model_config) { - auto cfg = get_htk_spectrogram_config(); - cfg.fft_override = model_config.audio_fft_length; - cfg.mel_floor_additive = true; - return cfg; -} - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_wespeaker_spectrogram_config() { - cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; - cfg.n_fft = 512; - cfg.frame_length = 400; - cfg.hop_length = 160; - cfg.power = 2.0f; - cfg.center = false; - cfg.pad_mode = "constant"; - cfg.onesided = true; - cfg.dither = 0.0f; - cfg.mel_floor = 1.1754944e-38f; - cfg.log_mel = "log"; - cfg.reference = 1.0f; - cfg.min_value = 1.1754944e-38f; - cfg.remove_dc_offset = true; - cfg.preemphasis = 0.97f; - cfg.hann_periodic = false; - cfg.window_a0 = 0.54f; - return cfg; -} - -// Whisper v1/v2: 80 mel bins, HTK. Whisper v3: 128 mel bins, Slaney, 512-FFT, no DC removal. -inline void init_whisper_mel_filters(cactus::engine::AudioProcessor& ap, - cactus::engine::AudioProcessor::SpectrogramConfig& cfg, - size_t mel_bins) { - const size_t num_mel_filters = std::max(1, mel_bins); - const bool is_v3 = mel_bins > 80; - if (is_v3) { - cfg.fft_override = 512; - cfg.remove_dc_offset = false; - } - const size_t fft_len = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; - const size_t num_frequency_bins = fft_len / 2 + 1; - if (is_v3) { - ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, - WHISPER_SAMPLE_RATE, "slaney", "slaney"); - } else { - ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, - WHISPER_SAMPLE_RATE); - } -} - -// use_mel_floor_padding=true pads short audio with the normalized mel floor (required for v3). -inline std::vector normalize_whisper_mel(std::vector& mel, size_t n_mels, - bool use_mel_floor_padding = false) { - if (mel.empty() || n_mels == 0) return mel; - size_t n_frames = mel.size() / n_mels; - - float max_val = -std::numeric_limits::infinity(); - for (float v : mel) if (v > max_val) max_val = v; - - float min_allowed = max_val - 8.0f; - for (float& v : mel) { - if (v < min_allowed) v = min_allowed; - v = (v + 4.0f) * 0.25f; - } - - if (n_frames != WHISPER_TARGET_FRAMES) { - float pad_val = use_mel_floor_padding ? (min_allowed + 4.0f) * 0.25f : 0.0f; - std::vector fixed(n_mels * WHISPER_TARGET_FRAMES, pad_val); - size_t copy_frames = std::min(n_frames, WHISPER_TARGET_FRAMES); - for (size_t m = 0; m < n_mels; ++m) { - const float* src = &mel[m * n_frames]; - float* dst = &fixed[m * WHISPER_TARGET_FRAMES]; - std::copy(src, src + copy_frames, dst); - } - return fixed; - } - return std::move(mel); -} - -inline std::vector transpose_mel_to_frame_major(const std::vector& mel, - size_t num_mels, size_t num_frames) { - std::vector transposed(num_frames * num_mels); - for (size_t m = 0; m < num_mels; m++) { - for (size_t t = 0; t < num_frames; t++) { - transposed[t * num_mels + m] = mel[m * num_frames + t]; - } - } - return transposed; -} - -inline void apply_preemphasis(std::vector& waveform, float coefficient = 0.97f) { - if (waveform.size() < 2 || coefficient == 0.0f) { - return; - } - for (size_t i = waveform.size() - 1; i > 0; --i) { - waveform[i] -= coefficient * waveform[i - 1]; - } -} - -inline void normalize_parakeet_log_mel(std::vector& mel, size_t num_mels, float epsilon = 1e-5f) { - if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { - return; - } - const size_t num_frames = mel.size() / num_mels; - if (num_frames == 0) { - return; - } - - for (size_t m = 0; m < num_mels; ++m) { - const size_t base = m * num_frames; - float mean = 0.0f; - for (size_t t = 0; t < num_frames; ++t) { - mean += mel[base + t]; - } - mean /= static_cast(num_frames); - - float variance = 0.0f; - for (size_t t = 0; t < num_frames; ++t) { - const float d = mel[base + t] - mean; - variance += d * d; - } - const float denom = static_cast(std::max(1, num_frames - 1)); - const float inv_std = 1.0f / std::sqrt((variance / denom) + epsilon); - for (size_t t = 0; t < num_frames; ++t) { - mel[base + t] = (mel[base + t] - mean) * inv_std; - } - } -} - -inline void trim_mel_frames(std::vector& mel, size_t num_mels, size_t valid_frames) { - if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { - return; - } - size_t total_frames = mel.size() / num_mels; - if (valid_frames == 0 || valid_frames >= total_frames) { - return; - } - std::vector trimmed(num_mels * valid_frames); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &mel[m * total_frames]; - float* dst = &trimmed[m * valid_frames]; - std::copy(src, src + valid_frames, dst); - } - mel.swap(trimmed); -} - -struct AudioPreprocessResult { - std::vector features; - size_t num_frames = 0; - size_t num_soft_tokens = 0; -}; - -inline AudioPreprocessResult preprocess_audio_for_gemma4( - std::vector audio_samples, - const cactus::engine::Config& model_config -) { - AudioPreprocessResult result; - if (audio_samples.empty()) return result; - - size_t pad_amt = 320 - (audio_samples.size() % 320); - if (pad_amt < 320) - audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); - - size_t mel_bins = model_config.audio_input_feat_size; - auto cfg = get_gemma4_audio_spectrogram_config(model_config); - - size_t semicausal_pad = cfg.frame_length / 2; - audio_samples.insert(audio_samples.begin(), semicausal_pad, 0.0f); - - cactus::engine::AudioProcessor ap; - size_t fft_for_mel = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; - ap.init_mel_filters(fft_for_mel / 2 + 1, mel_bins, 0.0f, 8000.0f, 16000, - nullptr, "htk"); - std::vector mel = ap.compute_spectrogram(audio_samples, cfg); - - result.num_frames = mel.size() / mel_bins; - result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); - - size_t after_stage1 = (result.num_frames + 1) / 2; - result.num_soft_tokens = (after_stage1 + 1) / 2; - - return result; -} - -inline std::vector pcm_buffer_to_float_samples( - const uint8_t* pcm_buffer, size_t pcm_buffer_size -) { - const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); - size_t num_samples = pcm_buffer_size / 2; - std::vector waveform_fp32(num_samples); - constexpr float inv_32768 = 1.0f / 32768.0f; - for (size_t i = 0; i < num_samples; i++) - waveform_fp32[i] = static_cast(pcm_samples[i]) * inv_32768; - return waveform_fp32; -} - -} // namespace audio -} // namespace cactus - -namespace cactus { -namespace ffi { - -inline bool env_flag_enabled(const char* key) { - const char* value = std::getenv(key); - return value && value[0] != '\0' && !(value[0] == '0' && value[1] == '\0'); -} - -inline std::string generateUUID() { -#ifdef __APPLE__ - uuid_t uuid; - uuid_generate_random(uuid); - char uuid_str[37]; - uuid_unparse_lower(uuid, uuid_str); - return std::string(uuid_str); -#else - static std::random_device rd; - static std::mt19937 gen(rd()); - static std::uniform_int_distribution<> dis(0, 15); - static std::uniform_int_distribution<> dis2(8, 11); - - std::stringstream ss; - ss << std::hex; - for (int i = 0; i < 8; i++) ss << dis(gen); - ss << "-"; - for (int i = 0; i < 4; i++) ss << dis(gen); - ss << "-4"; - for (int i = 0; i < 3; i++) ss << dis(gen); - ss << "-"; - ss << dis2(gen); - for (int i = 0; i < 3; i++) ss << dis(gen); - ss << "-"; - for (int i = 0; i < 12; i++) ss << dis(gen); - return ss.str(); -#endif -} - -struct ToolFunction { - std::string name; - std::string description; - std::unordered_map parameters; -}; - -struct InferenceOptions { - float temperature = 0.0f; - float top_p = 0.0f; - float min_p = 0.15f; - float repetition_penalty = 1.1f; - float confidence_threshold = 0.7f; - size_t top_k = 0; - size_t max_tokens = 100; - size_t tool_rag_top_k = 2; - size_t cloud_timeout_ms = 15000; - std::vector stop_sequences; - bool force_tools = false; - bool include_stop_sequences = false; - bool use_vad = true; - bool telemetry_enabled = true; - bool auto_handoff = true; - bool handoff_with_images = true; - bool enable_thinking_if_supported = true; -}; - -} // namespace ffi -} // namespace cactus - -std::vector select_relevant_tools( - CactusModelHandle* handle, - const std::string& query, - const std::vector& all_tools, - size_t top_k); - -#include "gemma_tools.h" - -namespace cactus { -namespace ffi { - -inline std::string escape_json_string(const std::string& s) { - std::ostringstream o; - for (char c : s) { - if (c == '"') o << "\\\""; - else if (c == '\n') o << "\\n"; - else if (c == '\r') o << "\\r"; - else if (c == '\t') o << "\\t"; - else if (c == '\\') o << "\\\\"; - else o << c; - } - return o.str(); -} - - -inline std::string trim_string(const std::string& s) { - size_t start = 0; - while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; - size_t end = s.size(); - while (end > start && std::isspace(static_cast(s[end - 1]))) --end; - return s.substr(start, end - start); -} - -inline size_t find_matching_delimiter(const std::string& s, size_t pos, char open, char close) { - int depth = 1; - pos++; - while (pos < s.length() && depth > 0) { - if (s[pos] == open) depth++; - else if (s[pos] == close) depth--; - else if (s[pos] == '"') { - pos++; - while (pos < s.length() && s[pos] != '"') { - if (s[pos] == '\\') pos++; - pos++; - } - } - pos++; - } - return pos; -} - -inline std::string env_or_default(const char* key, const char* fallback) { - const char* v = std::getenv(key); - if (v && v[0] != '\0') return std::string(v); - return std::string(fallback); -} - -inline std::string json_string_field(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return {}; - - size_t i = pos + pattern.size(); - while (i < json.size() && std::isspace(static_cast(json[i]))) i++; - if (i >= json.size() || json[i] != '"') return {}; - ++i; - - std::string out; - out.reserve(128); - while (i < json.size()) { - char c = json[i++]; - if (c == '"') return out; - if (c == '\\' && i < json.size()) { - char e = json[i++]; - switch (e) { - case '"': out.push_back('"'); break; - case '\\': out.push_back('\\'); break; - case '/': out.push_back('/'); break; - case 'b': out.push_back('\b'); break; - case 'f': out.push_back('\f'); break; - case 'n': out.push_back('\n'); break; - case 'r': out.push_back('\r'); break; - case 't': out.push_back('\t'); break; - default: out.push_back(e); break; - } - continue; - } - out.push_back(c); - } - return {}; -} - -inline std::string json_array_field(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return "[]"; - size_t start = pos + pattern.size(); - while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; - if (start >= json.size() || json[start] != '[') return "[]"; - - int depth = 1; - size_t end = start + 1; - while (end < json.size() && depth > 0) { - if (json[end] == '[') depth++; - else if (json[end] == ']') depth--; - end++; - } - return json.substr(start, end - start); -} - -inline std::vector split_json_array(const std::string& array_json) { - std::vector out; - if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; - - size_t i = 1; - while (i + 1 < array_json.size()) { - while (i + 1 < array_json.size() && - (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) i++; - if (i + 1 >= array_json.size() || array_json[i] != '{') break; - - size_t start = i; - int depth = 0; - bool in_str = false; - bool esc = false; - for (; i < array_json.size(); ++i) { - char c = array_json[i]; - if (in_str) { - if (esc) esc = false; - else if (c == '\\') esc = true; - else if (c == '"') in_str = false; - continue; - } - if (c == '"') { in_str = true; continue; } - if (c == '{') depth++; - if (c == '}') { - depth--; - if (depth == 0) { - out.push_back(array_json.substr(start, i - start + 1)); - i++; - break; - } - } - } - } - return out; -} - -inline std::string serialize_tools_json(const std::vector& tools) { - if (tools.empty()) return ""; - std::ostringstream oss; - oss << "["; - for (size_t i = 0; i < tools.size(); ++i) { - if (i > 0) oss << ","; - oss << "{\"type\":\"function\",\"function\":{"; - oss << "\"name\":\"" << escape_json_string(tools[i].name) << "\","; - oss << "\"description\":\"" << escape_json_string(tools[i].description) << "\""; - auto it = tools[i].parameters.find("schema"); - if (it != tools[i].parameters.end()) { - oss << ",\"parameters\":" << it->second; - } - oss << "}}"; - } - oss << "]"; - return oss.str(); -} - -namespace json_sorted { - -inline void skip_ws(const std::string& s, size_t& p) { - while (p < s.size() && std::isspace(static_cast(s[p]))) p++; -} - -inline std::string parse_string(const std::string& s, size_t& p) { - std::string r = "\""; - p++; - while (p < s.size()) { - if (s[p] == '\\') { - r += s[p++]; - if (p < s.size()) r += s[p++]; - } else if (s[p] == '"') { - r += '"'; - p++; - return r; - } else { - r += s[p++]; - } - } - return r; -} - -inline std::string parse_value(const std::string& s, size_t& p); - -inline std::string parse_object(const std::string& s, size_t& p) { - p++; - std::map entries; - skip_ws(s, p); - while (p < s.size() && s[p] != '}') { - if (s[p] == ',') { p++; skip_ws(s, p); continue; } - std::string key = parse_string(s, p); - skip_ws(s, p); - if (p < s.size() && s[p] == ':') p++; - skip_ws(s, p); - std::string val = parse_value(s, p); - entries[key] = val; - skip_ws(s, p); - } - if (p < s.size()) p++; - std::string r = "{"; - bool first = true; - for (const auto& kv : entries) { - if (!first) r += ", "; - r += kv.first + ": " + kv.second; - first = false; - } - r += "}"; - return r; -} - -inline std::string parse_array(const std::string& s, size_t& p) { - p++; - std::vector items; - skip_ws(s, p); - while (p < s.size() && s[p] != ']') { - if (s[p] == ',') { p++; skip_ws(s, p); continue; } - items.push_back(parse_value(s, p)); - skip_ws(s, p); - } - if (p < s.size()) p++; - std::string r = "["; - for (size_t i = 0; i < items.size(); i++) { - if (i > 0) r += ", "; - r += items[i]; - } - r += "]"; - return r; -} - -inline std::string parse_value(const std::string& s, size_t& p) { - skip_ws(s, p); - if (p >= s.size()) return ""; - if (s[p] == '"') return parse_string(s, p); - if (s[p] == '{') return parse_object(s, p); - if (s[p] == '[') return parse_array(s, p); - size_t start = p; - while (p < s.size() && s[p] != ',' && s[p] != '}' && s[p] != ']' && !std::isspace(static_cast(s[p]))) p++; - return s.substr(start, p - start); -} - -inline std::string reformat(const std::string& json) { - size_t p = 0; - return parse_value(json, p); -} - -} // namespace json_sorted - -inline std::string serialize_tools_for_template(const std::vector& tools) { - if (tools.empty()) return ""; - std::string result; - for (const auto& tool : tools) { - std::map func_fields; - func_fields["\"description\""] = "\"" + escape_json_string(tool.description) + "\""; - func_fields["\"name\""] = "\"" + escape_json_string(tool.name) + "\""; - auto it = tool.parameters.find("schema"); - if (it != tool.parameters.end()) { - func_fields["\"parameters\""] = json_sorted::reformat(it->second); - } - std::string func_json = "{"; - bool first = true; - for (const auto& kv : func_fields) { - if (!first) func_json += ", "; - func_json += kv.first + ": " + kv.second; - first = false; - } - func_json += "}"; - result += "\n{\"function\": " + func_json + ", \"type\": \"function\"}"; - } - return result; -} - -inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) { - std::ostringstream json; - json << "{"; - json << "\"success\":false,"; - json << "\"error\":\"" << escape_json_string(error_message) << "\","; - json << "\"cloud_handoff\":false,"; - json << "\"response\":null,"; - json << "\"function_calls\":[],"; - json << "\"confidence\":0.0,"; - json << "\"time_to_first_token_ms\":0.0,"; - json << "\"total_time_ms\":0.0,"; - json << "\"prefill_tps\":0.0,"; - json << "\"decode_tps\":0.0,"; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; - json << "\"prefill_tokens\":0,"; - json << "\"decode_tokens\":0,"; - json << "\"total_tokens\":0"; - json << "}"; - std::string error_json = json.str(); - if (response_buffer && error_json.length() < buffer_size) { - std::strcpy(response_buffer, error_json.c_str()); - } -} - -inline std::vector parse_messages_json(const std::string& json, - std::vector& out_image_paths, - std::vector* out_audio_paths = nullptr) { - std::vector messages; - out_image_paths.clear(); - if (out_audio_paths) out_audio_paths->clear(); - - size_t pos = json.find('['); - if (pos == std::string::npos) { - throw std::runtime_error("Invalid JSON: expected array"); - } - - pos = json.find('{', pos); - while (pos != std::string::npos) { - cactus::engine::ChatMessage msg; - - size_t obj_start = pos; - int brace_count = 1; - size_t obj_end = obj_start + 1; - while (obj_end < json.length() && brace_count > 0) { - if (json[obj_end] == '{') brace_count++; - else if (json[obj_end] == '}') brace_count--; - obj_end++; - } - - size_t role_pos = json.find("\"role\"", pos); - if (role_pos == std::string::npos || role_pos >= obj_end) break; - - size_t role_start = json.find('"', role_pos + 6) + 1; - size_t role_end = json.find('"', role_start); - msg.role = json.substr(role_start, role_end - role_start); - - size_t content_pos = json.find("\"content\"", role_end); - if (content_pos != std::string::npos && content_pos < obj_end) { - size_t content_start = json.find('"', content_pos + 9) + 1; - size_t content_end = content_start; - - while (content_end < json.length()) { - content_end = json.find('"', content_end); - if (content_end == std::string::npos) break; - if (json[content_end - 1] != '\\') break; - content_end++; - } - - msg.content = json.substr(content_start, content_end - content_start); - - size_t escape_pos = 0; - while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { - msg.content.replace(escape_pos, 2, "\n"); - escape_pos += 1; - } - escape_pos = 0; - while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) { - msg.content.replace(escape_pos, 2, "\""); - escape_pos += 1; - } - } - - auto parse_path_array = [&](const char* key, std::vector& dest, - std::vector* out_paths) { - size_t key_pos = json.find(key, pos); - if (key_pos == std::string::npos || key_pos >= obj_end) return; - size_t array_start = json.find('[', key_pos); - if (array_start == std::string::npos || array_start >= obj_end) return; - size_t array_end = json.find(']', array_start); - if (array_end == std::string::npos || array_end >= obj_end) return; - size_t cur = array_start; - while (true) { - cur = json.find('"', cur + 1); - if (cur == std::string::npos || cur >= array_end) break; - size_t str_start = cur + 1; - size_t str_end = json.find('"', str_start); - if (str_end == std::string::npos || str_end > array_end) break; - std::string path = std::filesystem::absolute( - std::filesystem::path(json.substr(str_start, str_end - str_start))).string(); - dest.push_back(path); - if (out_paths) out_paths->push_back(path); - cur = str_end; - } - }; - - parse_path_array("\"images\"", msg.images, &out_image_paths); - parse_path_array("\"audio\"", msg.audio, out_audio_paths); - - if (msg.role == "tool") { - size_t name_pos = json.find("\"name\"", obj_start); - if (name_pos != std::string::npos && name_pos < obj_end) { - size_t name_quote = json.find('"', name_pos + 6); - if (name_quote != std::string::npos && name_quote < obj_end) { - size_t name_start = name_quote + 1; - size_t name_end = json.find('"', name_start); - if (name_end != std::string::npos && name_end < obj_end) { - msg.name = json.substr(name_start, name_end - name_start); - } - } - } - } - - size_t tool_calls_pos = json.find("\"tool_calls\"", obj_start); - if (tool_calls_pos != std::string::npos && tool_calls_pos < obj_end) { - size_t tool_calls_arr_start = json.find('[', tool_calls_pos); - if (tool_calls_arr_start != std::string::npos && tool_calls_arr_start < obj_end) { - size_t tool_calls_arr_end = find_matching_delimiter(json, tool_calls_arr_start, '[', ']'); - - size_t search_pos = tool_calls_arr_start; - while (true) { - size_t func_pos = json.find("\"function\"", search_pos); - if (func_pos == std::string::npos || func_pos >= tool_calls_arr_end) break; - - size_t func_obj_start = json.find('{', func_pos + 10); - if (func_obj_start == std::string::npos || func_obj_start >= tool_calls_arr_end) break; - - size_t func_obj_end = find_matching_delimiter(json, func_obj_start, '{', '}'); - - cactus::engine::ToolCallInfo tool_call; - - size_t fn_name_pos = json.find("\"name\"", func_obj_start); - if (fn_name_pos != std::string::npos && fn_name_pos < func_obj_end) { - size_t fn_name_quote = json.find('"', fn_name_pos + 6); - if (fn_name_quote != std::string::npos && fn_name_quote < func_obj_end) { - size_t fn_name_start = fn_name_quote + 1; - size_t fn_name_end = json.find('"', fn_name_start); - if (fn_name_end != std::string::npos && fn_name_end < func_obj_end) { - tool_call.name = json.substr(fn_name_start, fn_name_end - fn_name_start); - } - } - } - - size_t args_pos = json.find("\"arguments\"", func_obj_start); - if (args_pos != std::string::npos && args_pos < func_obj_end) { - size_t colon_pos = json.find(':', args_pos + 11); - if (colon_pos != std::string::npos && colon_pos < func_obj_end) { - size_t args_start = colon_pos + 1; - while (args_start < json.length() && std::isspace(static_cast(json[args_start]))) args_start++; - - if (args_start < func_obj_end && json[args_start] == '{') { - size_t args_end = find_matching_delimiter(json, args_start, '{', '}'); - tool_call.arguments = json.substr(args_start, args_end - args_start); - } else if (args_start < func_obj_end && json[args_start] == '"') { - size_t str_start = args_start + 1; - size_t str_end = str_start; - while (str_end < json.length() && json[str_end] != '"') { - if (json[str_end] == '\\') str_end++; - str_end++; - } - tool_call.arguments = json.substr(str_start, str_end - str_start); - } - } - } - - if (!tool_call.name.empty()) { - msg.tool_calls.push_back(tool_call); - } - search_pos = func_obj_end; - } - } - } - - messages.push_back(msg); - - pos = json.find('{', obj_end); - } - - return messages; -} - -inline std::vector parse_tools_json(const std::string& json) { - std::vector tools; - - if (json.empty()) return tools; - - size_t pos = json.find('['); - if (pos == std::string::npos) return tools; - - pos = json.find("\"function\"", pos); - while (pos != std::string::npos) { - ToolFunction tool; - - size_t name_pos = json.find("\"name\"", pos); - if (name_pos != std::string::npos) { - size_t name_start = json.find('"', name_pos + 6) + 1; - size_t name_end = json.find('"', name_start); - tool.name = json.substr(name_start, name_end - name_start); - } - - size_t desc_pos = json.find("\"description\"", pos); - if (desc_pos != std::string::npos) { - size_t desc_start = json.find('"', desc_pos + 13) + 1; - size_t desc_end = json.find('"', desc_start); - tool.description = json.substr(desc_start, desc_end - desc_start); - } - - size_t params_pos = json.find("\"parameters\"", pos); - if (params_pos != std::string::npos) { - size_t params_start = json.find('{', params_pos); - if (params_start != std::string::npos) { - int brace_count = 1; - size_t params_end = params_start + 1; - while (params_end < json.length() && brace_count > 0) { - if (json[params_end] == '{') brace_count++; - else if (json[params_end] == '}') brace_count--; - params_end++; - } - tool.parameters["schema"] = json.substr(params_start, params_end - params_start); - } - } - - tools.push_back(tool); - - pos = json.find("\"function\"", name_pos); - } - - return tools; -} - -inline bool try_parse_json_float(const std::string& json, const std::string& key, float& out_value) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return false; - - size_t start = pos + pattern.size(); - while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; - - size_t end = start; - while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; - - try { - out_value = std::stof(json.substr(start, end - start)); - return true; - } catch (...) { - return false; - } -} - -inline std::vector parse_json_string_array_field(const std::string& json, const std::string& key) { - std::vector out; - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return out; - - size_t start = pos + pattern.size(); - while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; - if (start >= json.size() || json[start] != '[') return out; - - int depth = 1; - bool in_string = false; - bool escaped = false; - size_t end = start + 1; - - while (end < json.size() && depth > 0) { - char c = json[end]; - if (in_string) { - if (escaped) escaped = false; - else if (c == '\\') escaped = true; - else if (c == '"') in_string = false; - } else { - if (c == '"') in_string = true; - else if (c == '[') depth++; - else if (c == ']') depth--; - } - ++end; - } - - if (depth != 0) return out; - const std::string array_json = json.substr(start, end - start); - if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; - - size_t i = 1; - while (i + 1 < array_json.size()) { - while (i + 1 < array_json.size() && - (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) { - ++i; - } - if (i + 1 >= array_json.size() || array_json[i] == ']') break; - if (array_json[i] != '"') break; - - ++i; - std::string value; - bool escaped = false; - while (i < array_json.size()) { - char c = array_json[i++]; - if (escaped) { - switch (c) { - case '"': value.push_back('"'); break; - case '\\': value.push_back('\\'); break; - case '/': value.push_back('/'); break; - case 'b': value.push_back('\b'); break; - case 'f': value.push_back('\f'); break; - case 'n': value.push_back('\n'); break; - case 'r': value.push_back('\r'); break; - case 't': value.push_back('\t'); break; - default: value.push_back(c); break; - } - escaped = false; - continue; - } - if (c == '\\') { - escaped = true; - continue; - } - if (c == '"') { - out.push_back(value); - break; - } - value.push_back(c); - } - } - - return out; -} - -inline void parse_custom_vocabulary_options(const std::string& json, - std::vector& custom_vocabulary, - float& vocabulary_boost) { - custom_vocabulary.clear(); - vocabulary_boost = 5.0f; - if (json.empty()) return; - - float parsed_boost = vocabulary_boost; - if (try_parse_json_float(json, "vocabulary_boost", parsed_boost)) { - vocabulary_boost = std::clamp(parsed_boost, 0.0f, 20.0f); - } - - custom_vocabulary = parse_json_string_array_field(json, "custom_vocabulary"); -} - -inline std::unordered_map build_token_bias_map(const std::vector>& tokenized_entries, - float vocabulary_boost) { - std::unordered_map vocab_bias; - const float clamped_boost = std::clamp(vocabulary_boost, 0.0f, 20.0f); - if (clamped_boost == 0.0f) return vocab_bias; - - for (const auto& token_ids : tokenized_entries) { - for (uint32_t token_id : token_ids) { - float& entry = vocab_bias[token_id]; - if (entry < clamped_boost) { - entry = clamped_boost; - } - } - } - - return vocab_bias; -} - -inline std::unordered_map build_custom_vocabulary_bias(cactus::engine::Tokenizer* tokenizer, - const std::vector& custom_vocabulary, - float vocabulary_boost) { - if (!tokenizer || custom_vocabulary.empty()) return {}; - std::vector> tokenized_entries; - tokenized_entries.reserve(custom_vocabulary.size()); - - for (const auto& word : custom_vocabulary) { - if (word.empty()) continue; - tokenized_entries.push_back(tokenizer->encode(word)); - } - - return build_token_bias_map(tokenized_entries, vocabulary_boost); -} - -inline void apply_custom_vocabulary_options(cactus::engine::Model* model, const std::string& json) { - if (!model) return; - - std::vector custom_vocabulary; - float vocabulary_boost = 5.0f; - parse_custom_vocabulary_options(json, custom_vocabulary, vocabulary_boost); - model->set_vocab_bias(build_custom_vocabulary_bias(model->get_tokenizer(), custom_vocabulary, vocabulary_boost)); -} - -inline size_t levenshtein_ci(const std::string& a, const std::string& b) { - const size_t m = a.size(), n = b.size(); - std::vector prev(n + 1), curr(n + 1); - for (size_t j = 0; j <= n; ++j) prev[j] = j; - for (size_t i = 1; i <= m; ++i) { - curr[0] = i; - for (size_t j = 1; j <= n; ++j) { - const bool match = std::tolower(static_cast(a[i - 1])) == - std::tolower(static_cast(b[j - 1])); - curr[j] = std::min({prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (match ? 0 : 1)}); - } - std::swap(prev, curr); - } - return prev[n]; -} - -inline std::string collapse_spaces(const std::string& s) { - std::string out; - out.reserve(s.size()); - for (char c : s) { - if (c != ' ') out += c; - } - return out; -} - -inline void apply_vocabulary_spelling_correction( - std::string& text, - const std::vector& custom_vocabulary) -{ - if (custom_vocabulary.empty() || text.empty()) return; - - struct VocabEntry { - const std::string* original; - std::string collapsed; - }; - std::vector vocab_entries; - vocab_entries.reserve(custom_vocabulary.size()); - for (const auto& v : custom_vocabulary) { - vocab_entries.push_back({&v, collapse_spaces(v)}); - } - - struct Token { std::string text; bool is_word; }; - std::vector tokens; - size_t pos = 0; - while (pos < text.size()) { - if (std::isalnum(static_cast(text[pos])) || - text[pos] == '\'' || text[pos] == '-') { - size_t start = pos; - while (pos < text.size() && (std::isalnum(static_cast(text[pos])) || - text[pos] == '\'' || text[pos] == '-')) { - ++pos; - } - tokens.push_back({text.substr(start, pos - start), true}); - } else { - size_t start = pos; - while (pos < text.size() && !std::isalnum(static_cast(text[pos])) && - text[pos] != '\'' && text[pos] != '-') { - ++pos; - } - tokens.push_back({text.substr(start, pos - start), false}); - } - } - - std::vector word_indices; - for (size_t i = 0; i < tokens.size(); ++i) { - if (tokens[i].is_word) word_indices.push_back(i); - } - - std::vector consumed(tokens.size(), false); - - auto strip_suffix = [](const std::string& word) -> std::pair { - if (word.size() >= 3 && word.substr(word.size() - 2) == "'s") { - return {word.substr(0, word.size() - 2), "'s"}; - } - if (word.size() >= 3 && word.substr(word.size() - 2) == "'t") { - return {word.substr(0, word.size() - 2), "'t"}; - } - if (word.size() >= 4 && word.back() == 's' && - word[word.size() - 2] != 's' && // avoid stripping from "boss", "class" - std::isalpha(static_cast(word[word.size() - 2]))) { - return {word.substr(0, word.size() - 1), "s"}; - } - return {word, ""}; - }; - - size_t wi = 0; - while (wi < word_indices.size()) { - size_t best_dist = std::numeric_limits::max(); - const std::string* best_match = nullptr; - size_t best_window = 0; - size_t best_first_token = 0; - size_t best_last_token = 0; - std::string best_suffix; - - for (size_t window = std::min(3, word_indices.size() - wi); window >= 1; --window) { - std::string window_collapsed; - const size_t first_tok = word_indices[wi]; - const size_t last_tok = word_indices[wi + window - 1]; - for (size_t w = 0; w < window; ++w) { - window_collapsed += tokens[word_indices[wi + w]].text; - } - - if (window == 1 && window_collapsed.size() < 3) break; - - auto [stem, suffix] = strip_suffix(window_collapsed); - const std::string* candidates[] = {&window_collapsed, &stem}; - const std::string suffixes[] = {"", suffix}; - const size_t num_candidates = suffix.empty() ? 1 : 2; - - for (size_t ci = 0; ci < num_candidates; ++ci) { - const std::string& candidate = *candidates[ci]; - if (candidate.empty()) continue; - - for (const auto& entry : vocab_entries) { - const size_t wlen = candidate.size(); - const size_t vlen = entry.collapsed.size(); - - const size_t len_diff = wlen > vlen ? wlen - vlen : vlen - wlen; - const size_t max_dist = std::max(1, std::min(wlen, vlen) / 3); - if (len_diff > max_dist) continue; - - const size_t dist = levenshtein_ci(candidate, entry.collapsed); - - // For single-edit corrections, require first char match to prevent - // false positives like "vortex" → "Cortex". - if (dist == 1 && window == 1) { - const bool first_char_match = - std::tolower(static_cast(candidate[0])) == - std::tolower(static_cast(entry.collapsed[0])); - if (!first_char_match) continue; - } - - if (dist <= max_dist && dist < best_dist) { - best_dist = dist; - best_match = entry.original; - best_window = window; - best_first_token = first_tok; - best_last_token = last_tok; - best_suffix = suffixes[ci]; - } - } - } - - if (best_dist == 0) break; - } - - // Allow dist==0 for multi-word merges where word boundaries changed. - const bool should_replace = best_match && - best_dist != std::numeric_limits::max() && - (best_dist > 0 || best_window > 1); - - if (should_replace) { - tokens[best_first_token].text = *best_match + best_suffix; - for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { - consumed[t] = true; - } - for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { - if (t > 0) consumed[t - 1] = consumed[t - 1] || !tokens[t - 1].is_word; - } - wi += best_window; - } else { - ++wi; - } - } - - std::string result; - result.reserve(text.size()); - for (size_t i = 0; i < tokens.size(); ++i) { - if (!consumed[i]) { - result += tokens[i].text; - } - } - - text = std::move(result); -} - -inline InferenceOptions parse_inference_options_json(const std::string& json) { - InferenceOptions options; - - if (json.empty()) return options; - - size_t pos = json.find("\"temperature\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.temperature = std::stof(json.substr(pos)); - } - - pos = json.find("\"top_p\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.top_p = std::stof(json.substr(pos)); - } - - float parsed_min_p = options.min_p; - if (try_parse_json_float(json, "min_p", parsed_min_p)) { - options.min_p = std::clamp(parsed_min_p, 0.0f, 1.0f); - } - - float parsed_rep_penalty = options.repetition_penalty; - if (try_parse_json_float(json, "repetition_penalty", parsed_rep_penalty)) { - if (std::isfinite(parsed_rep_penalty) && parsed_rep_penalty > 0.0f) { - options.repetition_penalty = parsed_rep_penalty; - } - } - - pos = json.find("\"top_k\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.top_k = std::stoul(json.substr(pos)); - } - - pos = json.find("\"max_tokens\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.max_tokens = std::stoul(json.substr(pos)); - } - - pos = json.find("\"force_tools\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.force_tools = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"tool_rag_top_k\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.tool_rag_top_k = std::stoul(json.substr(pos)); - } - - pos = json.find("\"confidence_threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.confidence_threshold = std::stof(json.substr(pos)); - } - - pos = json.find("\"include_stop_sequences\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.include_stop_sequences = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"use_vad\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.use_vad = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"telemetry_enabled\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.telemetry_enabled = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"auto_handoff\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.auto_handoff = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"cloud_timeout_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.cloud_timeout_ms = std::stoul(json.substr(pos)); - } - - pos = json.find("\"handoff_with_images\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.handoff_with_images = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"enable_thinking_if_supported\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"stop_sequences\""); - if (pos != std::string::npos) { - pos = json.find('[', pos); - if (pos != std::string::npos) { - size_t end_pos = json.find(']', pos); - size_t seq_pos = json.find('"', pos); - - while (seq_pos != std::string::npos && seq_pos < end_pos) { - size_t seq_start = seq_pos + 1; - size_t seq_end = json.find('"', seq_start); - if (seq_end != std::string::npos) { - options.stop_sequences.push_back(json.substr(seq_start, seq_end - seq_start)); - } - seq_pos = json.find('"', seq_end + 1); - } - } - } - - return options; -} - -static inline std::string trim_lfm2_slice(const std::string& value, size_t begin, size_t end) { - return trim_string(value.substr(begin, end - begin)); -} - -static inline void append_lfm2_call(const std::string& entry, - std::vector& function_calls) { - if (entry.empty()) return; - - std::string trimmed_entry = trim_lfm2_slice(entry, 0, entry.size()); - if (trimmed_entry.empty()) return; - - size_t paren_pos = trimmed_entry.find('('); - if (paren_pos == std::string::npos) return; - - std::string func_name = trim_lfm2_slice(trimmed_entry, 0, paren_pos); - std::string args_str = trim_lfm2_slice(trimmed_entry, paren_pos + 1, trimmed_entry.size()); - - if (!args_str.empty() && args_str.back() == ')') { - args_str.pop_back(); - args_str = trim_lfm2_slice(args_str, 0, args_str.size()); - } - - std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{"; - - size_t arg_pos = 0; - bool first_arg = true; - while (arg_pos < args_str.length()) { - while (arg_pos < args_str.length() && std::isspace(static_cast(args_str[arg_pos]))) { - arg_pos++; - } - - size_t eq_pos = args_str.find('=', arg_pos); - if (eq_pos == std::string::npos) break; - - std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos); - - size_t val_start = eq_pos + 1; - size_t val_end = val_start; - - if (val_start < args_str.length() && args_str[val_start] == '"') { - val_start++; - val_end = args_str.find('"', val_start); - if (val_end == std::string::npos) break; - } else { - val_end = args_str.find(',', val_start); - if (val_end == std::string::npos) val_end = args_str.length(); - } - - std::string arg_value = args_str.substr(val_start, val_end - val_start); - - if (!first_arg) json_call += ","; - json_call += "\"" + arg_name + "\":\"" + arg_value + "\""; - first_arg = false; - - arg_pos = args_str.find(',', val_end); - if (arg_pos != std::string::npos) { - arg_pos++; - } else { - break; - } - } - - json_call += "}}"; - function_calls.push_back(json_call); -} - -inline void parse_function_calls_from_response(const std::string& response_text, - std::string& regular_response, - std::vector& function_calls) { - regular_response = response_text; - function_calls.clear(); - - gemma::parse_function_calls(regular_response, function_calls); - - const std::string QWEN_TOOL_START = ""; - const std::string QWEN_TOOL_END = ""; - size_t qwen_start_pos = 0; - - while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) { - size_t content_start = qwen_start_pos + QWEN_TOOL_START.length(); - size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start); - - size_t erase_end; - std::string json_content; - - if (qwen_end_pos != std::string::npos) { - json_content = regular_response.substr(content_start, qwen_end_pos - content_start); - erase_end = qwen_end_pos + QWEN_TOOL_END.length(); - } else { - json_content = regular_response.substr(content_start); - erase_end = regular_response.length(); - } - - size_t first = json_content.find_first_not_of(" \t\n\r"); - size_t last = json_content.find_last_not_of(" \t\n\r"); - if (first != std::string::npos && last != std::string::npos) { - json_content = json_content.substr(first, last - first + 1); - } - - if (json_content.size() > 2 && json_content.find("\"name\"") != std::string::npos) { - // Unwrap array wrapper if present: [{"name":...}] -> {"name":...} - if (json_content[0] == '[') { - size_t obj_start = json_content.find('{'); - size_t obj_end = json_content.rfind('}'); - if (obj_start != std::string::npos && obj_end != std::string::npos && obj_end > obj_start) { - json_content = json_content.substr(obj_start, obj_end - obj_start + 1); - } - } - if (json_content[0] == '{') { - size_t depth = 0; - bool in_string = false; - bool escaped = false; - size_t end_pos = 0; - for (size_t c = 0; c < json_content.size(); c++) { - char ch = json_content[c]; - if (escaped) { escaped = false; continue; } - if (ch == '\\' && in_string) { escaped = true; continue; } - if (ch == '"') { in_string = !in_string; continue; } - if (!in_string) { - if (ch == '{') depth++; - else if (ch == '}' && --depth == 0) { end_pos = c + 1; break; } - } - } - if (end_pos > 0) { - function_calls.push_back(json_content.substr(0, end_pos)); - } - } - } - - regular_response.erase(qwen_start_pos, erase_end - qwen_start_pos); - } - - const std::string TOOL_CALL_START = "<|tool_call_start|>"; - const std::string TOOL_CALL_END = "<|tool_call_end|>"; - size_t lfm2_start_pos = 0; - - while ((lfm2_start_pos = regular_response.find(TOOL_CALL_START, lfm2_start_pos)) != std::string::npos) { - size_t content_start = lfm2_start_pos + TOOL_CALL_START.length(); - size_t tool_end_pos = regular_response.find(TOOL_CALL_END, content_start); - - if (tool_end_pos != std::string::npos) { - std::string tool_content = regular_response.substr(content_start, tool_end_pos - content_start); - std::string content = tool_content; - size_t trim_start = 0; - while (trim_start < content.size() && std::isspace(static_cast(content[trim_start]))) { - trim_start++; - } - - if (trim_start < content.size()) { - size_t trim_end = content.size() - 1; - while (trim_end > trim_start && std::isspace(static_cast(content[trim_end]))) { - trim_end--; - } - content = content.substr(trim_start, trim_end - trim_start + 1); - } else { - content.clear(); - } - - if (!content.empty() && content.front() == '[' && content.back() == ']') { - std::string inner = content.substr(1, content.size() - 2); - - size_t inner_first = inner.find_first_not_of(" \t\n\r"); - if (inner_first != std::string::npos && inner[inner_first] == '{') { - size_t pos = inner_first; - while (pos < inner.size()) { - if (inner[pos] == '{') { - int brace_depth = 1; - size_t obj_start = pos; - pos++; - while (pos < inner.size() && brace_depth > 0) { - if (inner[pos] == '{') brace_depth++; - else if (inner[pos] == '}') brace_depth--; - pos++; - } - if (brace_depth == 0) { - std::string json_obj = inner.substr(obj_start, pos - obj_start); - if (json_obj.find("\"name\"") != std::string::npos) { - function_calls.push_back(json_obj); - } - } - } else { - pos++; - } - } - } else { - size_t start = 0; - int paren_depth = 0; - - for (size_t i = 0; i < inner.size(); ++i) { - char c = inner[i]; - if (c == '(') { - paren_depth++; - } else if (c == ')' && paren_depth > 0) { - paren_depth--; - } else if (c == ',' && paren_depth == 0) { - append_lfm2_call(inner.substr(start, i - start), function_calls); - start = i + 1; - } - } - - if (start < inner.size()) { - append_lfm2_call(inner.substr(start), function_calls); - } - } - } else if (!content.empty()) { - append_lfm2_call(content, function_calls); - } - - regular_response.erase(lfm2_start_pos, tool_end_pos + TOOL_CALL_END.length() - lfm2_start_pos); - } else { - break; - } - } - - const char* FUNCTION_CALL_MARKER = "\"function_call\""; - size_t search_pos = 0; - const size_t text_len = regular_response.length(); - - while (search_pos < text_len) { - size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos); - if (marker_pos == std::string::npos) break; - - size_t json_start = regular_response.find('{', marker_pos); - if (json_start == std::string::npos) break; - - int brace_count = 1; - size_t json_end = json_start + 1; - while (json_end < text_len && brace_count > 0) { - char c = regular_response[json_end]; - brace_count += (c == '{') - (c == '}'); - json_end++; - } - - if (brace_count == 0) { - function_calls.push_back(regular_response.substr(json_start, json_end - json_start)); - regular_response = regular_response.substr(0, marker_pos); - size_t last_bracket = regular_response.rfind('{'); - if(last_bracket != std::string::npos) { - regular_response = regular_response.substr(0, last_bracket); - } - } - search_pos = json_end; - } -} - -inline std::vector> find_channel_token_ranges( - const std::vector& tokens, size_t offset, - uint32_t channel_open_id, uint32_t channel_close_id) { - std::vector> ranges; - size_t pos = 0; - while (pos < tokens.size()) { - if (tokens[pos] != channel_open_id) { - pos++; - continue; - } - - size_t block_start = pos; - pos++; - while (pos < tokens.size() && tokens[pos] != channel_close_id) { - pos++; - } - if (pos < tokens.size()) { - pos++; - } - ranges.push_back({offset + block_start, pos - block_start}); - } - return ranges; -} - -inline void strip_tag_blocks(std::string& text, std::string& extracted, - const std::string& open_tag, const std::string& close_tag) { - std::string result; - size_t pos = 0; - - size_t first_close = text.find(close_tag); - size_t first_open = text.find(open_tag); - if (first_close != std::string::npos && - (first_open == std::string::npos || first_close < first_open)) { - extracted += text.substr(0, first_close); - pos = first_close + close_tag.size(); - } - - while (pos < text.size()) { - size_t open_pos = text.find(open_tag, pos); - if (open_pos == std::string::npos) { - result += text.substr(pos); - break; - } - result += text.substr(pos, open_pos - pos); - size_t content_start = open_pos + open_tag.size(); - size_t close_pos = text.find(close_tag, content_start); - if (close_pos == std::string::npos) { - if (!extracted.empty()) extracted += "\n"; - extracted += text.substr(content_start); - break; - } - if (!extracted.empty()) extracted += "\n"; - extracted += text.substr(content_start, close_pos - content_start); - pos = close_pos + close_tag.size(); - } - text = result; -} - -inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { - thinking.clear(); - content = input; - - auto trim = [](std::string& s) { - size_t first = s.find_first_not_of(" \t\n\r"); - size_t last = s.find_last_not_of(" \t\n\r"); - if (first != std::string::npos && last != std::string::npos) - s = s.substr(first, last - first + 1); - else - s.clear(); - }; - - if (content.find("<|channel>") != std::string::npos || content.find("") != std::string::npos) { - strip_tag_blocks(content, thinking, "<|channel>", ""); - } else if (content.find("") != std::string::npos || content.find("") != std::string::npos) { - strip_tag_blocks(content, thinking, "", ""); - } else { - return; - } - - trim(thinking); - trim(content); -} - -struct TranscriptSegment { - float start; - float end; - std::string text; -}; - -inline std::string construct_response_json(const std::string& regular_response, - const std::vector& function_calls, - double time_to_first_token, - double total_time_ms, - double prefill_tps, - double decode_tps, - size_t prompt_tokens, - size_t completion_tokens, - float confidence = 0.0f, - bool cloud_handoff = false, - const std::string& thinking = "", - const std::vector& segments = {}) { - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; - json << "\"response\":\"" << escape_json_string(regular_response) << "\","; - if (!thinking.empty()) { - json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; - } - json << "\"function_calls\":["; - for (size_t i = 0; i < function_calls.size(); ++i) { - if (i > 0) json << ","; - json << function_calls[i]; - } - json << "],"; - json << "\"segments\":["; - for (size_t i = 0; i < segments.size(); ++i) { - if (i > 0) json << ","; - json << "{\"start\":" << std::fixed << std::setprecision(3) << segments[i].start - << ",\"end\":" << std::fixed << std::setprecision(3) << segments[i].end - << ",\"text\":\"" << escape_json_string(segments[i].text) << "\"}"; - } - json << "],"; - json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; - json << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << time_to_first_token << ","; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; - json << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; - json << "\"prefill_tokens\":" << prompt_tokens << ","; - json << "\"decode_tokens\":" << completion_tokens << ","; - json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); - json << "}"; - return json.str(); -} - -inline std::string serialize_function_calls(const std::vector& calls) { - if (calls.empty()) return "[]"; - std::ostringstream oss; - oss << "["; - for (size_t i = 0; i < calls.size(); ++i) { - if (i > 0) oss << ","; - oss << calls[i]; - } - oss << "]"; - return oss.str(); -} - -inline int validate_audio_params( - const char* component, - void* model, - char* response_buffer, size_t buffer_size, - const char* audio_file_path, - const uint8_t* pcm_buffer, size_t pcm_buffer_size) { - if (!model) { - std::string err = last_error_message.empty() ? "Model not initialized." : last_error_message; - CACTUS_LOG_ERROR(component, err); - handle_error_response(err, response_buffer, buffer_size); - return -1; - } - if (!response_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR(component, "Invalid parameters: response_buffer or buffer_size"); - handle_error_response("Invalid parameters", response_buffer, buffer_size); - return -1; - } - if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { - CACTUS_LOG_ERROR(component, "No audio input provided"); - handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); - return -1; - } - if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { - CACTUS_LOG_ERROR(component, "Both audio_file_path and pcm_buffer provided"); - handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); - return -1; - } - if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { - CACTUS_LOG_ERROR(component, "Invalid pcm_buffer_size"); - handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); - return -1; - } - return 0; -} - -inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buffer_size) { - const int16_t* samples = reinterpret_cast(pcm_buffer); - size_t n = pcm_buffer_size / 2; - std::vector out(n); - for (size_t i = 0; i < n; ++i) - out[i] = static_cast(samples[i]) / 32768.0f; - return out; -} - -} // namespace ffi -} // namespace cactus - -#ifdef __cplusplus -extern "C" { -#endif - -const char* cactus_get_last_error(); - -#ifdef __cplusplus -} -#endif - -#endif // CACTUS_UTILS_H diff --git a/cactus-transpile/ffi/cactus_vad.cpp b/cactus-transpile/ffi/cactus_vad.cpp deleted file mode 100644 index 10b2e99c3..000000000 --- a/cactus-transpile/ffi/cactus_vad.cpp +++ /dev/null @@ -1,176 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../models/model.h" -#include "../../libs/audio/wav.h" -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - - -static SileroVADModel::SpeechTimestampsOptions parse_speech_timestamps_options(const std::string& json) { - SileroVADModel::SpeechTimestampsOptions options; - - if (json.empty()) return options; - - size_t pos = json.find("\"threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.threshold = std::stof(json.substr(pos)); - } - - pos = json.find("\"neg_threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.neg_threshold = std::stof(json.substr(pos)); - } - - pos = json.find("\"min_speech_duration_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.min_speech_duration_ms = std::stoi(json.substr(pos)); - } - - pos = json.find("\"max_speech_duration_s\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.max_speech_duration_s = std::stof(json.substr(pos)); - } - - pos = json.find("\"min_silence_duration_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.min_silence_duration_ms = std::stoi(json.substr(pos)); - } - - pos = json.find("\"speech_pad_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.speech_pad_ms = std::stoi(json.substr(pos)); - } - - pos = json.find("\"window_size_samples\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.window_size_samples = std::stoi(json.substr(pos)); - } - - pos = json.find("\"min_silence_at_max_speech\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.min_silence_at_max_speech = std::stoi(json.substr(pos)); - } - - pos = json.find("\"use_max_poss_sil_at_max_speech\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - std::string value = json.substr(pos, 5); - options.use_max_poss_sil_at_max_speech = (value.find("true") != std::string::npos); - } - - pos = json.find("\"sampling_rate\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.sampling_rate = std::stoi(json.substr(pos)); - } - - return options; -} - - -extern "C" { - -int cactus_vad( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (validate_audio_params("vad", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) - return -1; - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - auto* vad_model = static_cast(handle->model.get()); - - SileroVADModel::SpeechTimestampsOptions options = parse_speech_timestamps_options(options_json ? options_json : ""); - - std::vector audio; - if (audio_file_path == nullptr) { - audio = pcm_to_float(pcm_buffer, pcm_buffer_size); - } else { - AudioFP32 wav_audio = load_wav(audio_file_path); - audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); - } - - if (audio.empty()) { - last_error_message = "Failed to load audio or audio is empty"; - CACTUS_LOG_ERROR("vad", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - auto segments = vad_model->get_speech_timestamps(audio, options); - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"segments\":["; - - for (size_t i = 0; i < segments.size(); ++i) { - if (i > 0) json << ","; - json << "{"; - json << "\"start\":" << segments[i].start << ","; - json << "\"end\":" << segments[i].end; - json << "}"; - } - - json << "],"; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - - std::string response = json.str(); - if (response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("vad", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, response.c_str()); - return static_cast(response.length()); - - } catch (const std::exception& e) { - last_error_message = "Exception during VAD processing: " + std::string(e.what()); - CACTUS_LOG_ERROR("vad", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } catch (...) { - last_error_message = "Unknown exception during VAD processing"; - CACTUS_LOG_ERROR("vad", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus-transpile/ffi/gemma_tools.h b/cactus-transpile/ffi/gemma_tools.h deleted file mode 100644 index f0f9fe26d..000000000 --- a/cactus-transpile/ffi/gemma_tools.h +++ /dev/null @@ -1,576 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace gemma { - -inline std::string to_upper(const std::string& s) { - std::string result = s; - for (auto& c : result) c = std::toupper(c); - return result; -} - -inline std::string escape(const std::string& s) { - return "" + s + ""; -} - -inline void skip_whitespace(const std::string& json, size_t& pos) { - while (pos < json.length() && std::isspace(json[pos])) pos++; -} - -inline std::string extract_json_string(const std::string& json, size_t& pos) { - std::string value; - while (pos < json.length() && json[pos] != '"') { - if (json[pos] == '\\' && pos + 1 < json.length()) { - pos++; - if (json[pos] == 'n') value += '\n'; - else if (json[pos] == 't') value += '\t'; - else if (json[pos] == 'r') value += '\r'; - else if (json[pos] == '"') value += '"'; - else if (json[pos] == '\\') value += '\\'; - else value += json[pos]; - } else { - value += json[pos]; - } - pos++; - } - if (pos < json.length()) pos++; - return value; -} - -std::string format_argument(const std::string& json, size_t& pos, bool escape_keys); -std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/); - -inline std::string format_argument(const std::string& json, size_t& pos, bool escape_keys = true) { - skip_whitespace(json, pos); - if (pos >= json.length()) return ""; - - char c = json[pos]; - - if (c == '"') { - pos++; - std::string value = extract_json_string(json, pos); - return escape(value); - } else if (c == '{') { - std::string result = "{"; - pos++; - bool first = true; - - while (pos < json.length()) { - skip_whitespace(json, pos); - if (pos >= json.length() || json[pos] == '}') { pos++; break; } - if (json[pos] == ',') { pos++; continue; } - - if (json[pos] != '"') break; - pos++; - std::string key = extract_json_string(json, pos); - - skip_whitespace(json, pos); - if (pos < json.length() && json[pos] == ':') pos++; - - std::string value = format_argument(json, pos, escape_keys); - - if (!first) result += ","; - first = false; - if (escape_keys) { - result += escape(key) + ":" + value; - } else { - result += key + ":" + value; - } - } - result += "}"; - return result; - } else if (c == '[') { - std::string result = "["; - pos++; - bool first = true; - - while (pos < json.length()) { - skip_whitespace(json, pos); - if (pos >= json.length() || json[pos] == ']') { pos++; break; } - if (json[pos] == ',') { pos++; continue; } - - std::string value = format_argument(json, pos, escape_keys); - - if (!first) result += ","; - first = false; - result += value; - } - result += "]"; - return result; - } else if (json.compare(pos, 4, "true") == 0) { - pos += 4; - return "true"; - } else if (json.compare(pos, 5, "false") == 0) { - pos += 5; - return "false"; - } else if (json.compare(pos, 4, "null") == 0) { - pos += 4; - return "null"; - } else { - size_t start = pos; - while (pos < json.length() && (std::isdigit(json[pos]) || json[pos] == '.' || - json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E')) { - pos++; - } - return json.substr(start, pos - start); - } -} - -inline std::map parse_json_object_raw(const std::string& json, size_t& pos) { - std::map result; - skip_whitespace(json, pos); - if (pos >= json.length() || json[pos] != '{') return result; - pos++; - - while (pos < json.length()) { - skip_whitespace(json, pos); - if (pos >= json.length() || json[pos] == '}') { pos++; break; } - if (json[pos] == ',') { pos++; continue; } - - if (json[pos] != '"') break; - pos++; - std::string key = extract_json_string(json, pos); - - skip_whitespace(json, pos); - if (pos < json.length() && json[pos] == ':') pos++; - skip_whitespace(json, pos); - - size_t value_start = pos; - if (json[pos] == '"') { - pos++; - while (pos < json.length() && json[pos] != '"') { - if (json[pos] == '\\') pos++; - pos++; - } - pos++; - } else if (json[pos] == '{') { - int depth = 1; - pos++; - while (pos < json.length() && depth > 0) { - if (json[pos] == '{') depth++; - else if (json[pos] == '}') depth--; - else if (json[pos] == '"') { - pos++; - while (pos < json.length() && json[pos] != '"') { - if (json[pos] == '\\') pos++; - pos++; - } - } - pos++; - } - } else if (json[pos] == '[') { - int depth = 1; - pos++; - while (pos < json.length() && depth > 0) { - if (json[pos] == '[') depth++; - else if (json[pos] == ']') depth--; - else if (json[pos] == '"') { - pos++; - while (pos < json.length() && json[pos] != '"') { - if (json[pos] == '\\') pos++; - pos++; - } - } - pos++; - } - } else { - while (pos < json.length() && json[pos] != ',' && json[pos] != '}') pos++; - } - result[key] = json.substr(value_start, pos - value_start); - } - return result; -} - -inline std::string get_json_string_value(const std::string& json, size_t pos) { - skip_whitespace(json, pos); - if (pos < json.length() && json[pos] == '"') { - pos++; - return extract_json_string(json, pos); - } - return ""; -} - -inline std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/) { - static const std::set standard_keys = {"description", "type", "properties", "required", "nullable"}; - - size_t pos = 0; - auto properties = parse_json_object_raw(properties_json, pos); - - std::string result; - bool first = true; - - for (const auto& [key, value_json] : properties) { - if (standard_keys.count(key)) continue; - - if (!first) result += ","; - first = false; - - size_t prop_pos = 0; - auto prop_obj = parse_json_object_raw(value_json, prop_pos); - - result += key + ":{"; - - if (prop_obj.count("description")) { - std::string desc = get_json_string_value(prop_obj["description"], 0); - result += "description:" + escape(desc); - } - - std::string type_val; - if (prop_obj.count("type")) { - type_val = get_json_string_value(prop_obj["type"], 0); - } - - if (to_upper(type_val) == "STRING") { - if (prop_obj.count("enum")) { - size_t enum_pos = 0; - std::string enum_formatted = format_argument(prop_obj["enum"], enum_pos, true); - result += ",enum:" + enum_formatted; - } - } else if (to_upper(type_val) == "OBJECT") { - if (prop_obj.count("properties")) { - std::string nested_required; - if (prop_obj.count("required")) { - nested_required = prop_obj["required"]; - } - result += ",properties:{" + format_parameters(prop_obj["properties"], nested_required) + "}"; - } - if (prop_obj.count("required")) { - std::string req_items; - size_t req_pos = 0; - skip_whitespace(prop_obj["required"], req_pos); - if (req_pos < prop_obj["required"].length() && prop_obj["required"][req_pos] == '[') { - req_pos++; - bool req_first = true; - while (req_pos < prop_obj["required"].length()) { - skip_whitespace(prop_obj["required"], req_pos); - if (prop_obj["required"][req_pos] == ']') break; - if (prop_obj["required"][req_pos] == ',') { req_pos++; continue; } - if (prop_obj["required"][req_pos] == '"') { - req_pos++; - std::string req_item = extract_json_string(prop_obj["required"], req_pos); - if (!req_first) req_items += ","; - req_first = false; - req_items += escape(req_item); - } - } - } - if (!req_items.empty()) { - result += ",required:[" + req_items + "]"; - } - } - } else if (to_upper(type_val) == "ARRAY") { - if (prop_obj.count("items")) { - result += ",items:{"; - size_t items_pos = 0; - auto items_obj = parse_json_object_raw(prop_obj["items"], items_pos); - bool items_first = true; - - for (const auto& [item_key, item_value] : items_obj) { - if (!items_first) result += ","; - items_first = false; - - if (item_key == "properties") { - std::string items_required; - if (items_obj.count("required")) { - items_required = items_obj["required"]; - } - result += "properties:{" + format_parameters(item_value, items_required) + "}"; - } else if (item_key == "required") { - result += "required:["; - size_t req_pos = 0; - skip_whitespace(item_value, req_pos); - if (req_pos < item_value.length() && item_value[req_pos] == '[') { - req_pos++; - bool req_first = true; - while (req_pos < item_value.length()) { - skip_whitespace(item_value, req_pos); - if (item_value[req_pos] == ']') break; - if (item_value[req_pos] == ',') { req_pos++; continue; } - if (item_value[req_pos] == '"') { - req_pos++; - std::string req_item = extract_json_string(item_value, req_pos); - if (!req_first) result += ","; - req_first = false; - result += escape(req_item); - } - } - } - result += "]"; - } else if (item_key == "type") { - std::string item_type = get_json_string_value(item_value, 0); - result += "type:" + escape(to_upper(item_type)); - } else { - size_t val_pos = 0; - result += item_key + ":" + format_argument(item_value, val_pos, true); - } - } - result += "}"; - } - } - - if (!type_val.empty()) { - result += ",type:" + escape(to_upper(type_val)); - } - - result += "}"; - } - - return result; -} - -inline std::string format_function_declaration(const std::string& name, - const std::string& description, - const std::string& params_json) { - std::string result = "declaration:" + name + "{"; - result += "description:" + escape(description); - - if (!params_json.empty()) { - result += ",parameters:{"; - - size_t pos = 0; - auto params = parse_json_object_raw(params_json, pos); - - if (params.count("properties")) { - std::string required_json; - if (params.count("required")) { - required_json = params["required"]; - } - result += "properties:{" + format_parameters(params["properties"], required_json) + "}"; - } - - if (params.count("required")) { - std::string req_items; - size_t req_pos = 0; - skip_whitespace(params["required"], req_pos); - if (req_pos < params["required"].length() && params["required"][req_pos] == '[') { - req_pos++; - bool first = true; - while (req_pos < params["required"].length()) { - skip_whitespace(params["required"], req_pos); - if (params["required"][req_pos] == ']') break; - if (params["required"][req_pos] == ',') { req_pos++; continue; } - if (params["required"][req_pos] == '"') { - req_pos++; - std::string item = extract_json_string(params["required"], req_pos); - if (!first) req_items += ","; - first = false; - req_items += escape(item); - } - } - } - if (!req_items.empty()) { - result += ",required:[" + req_items + "]"; - } - } - - if (params.count("type")) { - std::string type_val = get_json_string_value(params["type"], 0); - result += ",type:" + escape(to_upper(type_val)); - } - - result += "}"; - } - - result += "}"; - return result; -} - -template -inline std::string format_tools(const std::vector& tools, bool use_pipe_tags = false) { - if (tools.empty()) return ""; - - const char* decl_start = use_pipe_tags ? "<|tool>" : ""; - const char* decl_end = use_pipe_tags ? "" : ""; - - std::string result; - for (const auto& tool : tools) { - result += decl_start; - std::string params_json; - auto it = tool.parameters.find("schema"); - if (it != tool.parameters.end()) { - params_json = it->second; - } - - result += format_function_declaration(tool.name, tool.description, params_json); - result += decl_end; - } - return result; -} - - -inline size_t match_quote_tag(const std::string& s, size_t pos) { - if (s.compare(pos, 8, "") == 0) return 8; - if (s.compare(pos, 5, "<|\"|>") == 0) return 5; - return 0; -} - -inline size_t find_quote_tag(const std::string& s, size_t pos) { - size_t e = s.find("", pos); - size_t t = s.find("<|\"|>", pos); - if (e == std::string::npos) return t; - if (t == std::string::npos) return e; - return std::min(e, t); -} - -inline std::string unescape(const std::string& s) { - const std::string ESCAPE_TAG = ""; - std::string result = s; - size_t pos = 0; - while ((pos = result.find(ESCAPE_TAG, pos)) != std::string::npos) { - result.erase(pos, ESCAPE_TAG.length()); - } - return result; -} - -inline std::string args_to_json(const std::string& args_content) { - std::string result = "{"; - size_t pos = 0; - bool first = true; - - if (!args_content.empty() && args_content[0] == '{') pos = 1; - - while (pos < args_content.length()) { - while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; - if (pos >= args_content.length() || args_content[pos] == '}') break; - if (args_content[pos] == ',') { pos++; continue; } - - size_t key_start = pos; - while (pos < args_content.length() && args_content[pos] != ':') pos++; - std::string key = args_content.substr(key_start, pos - key_start); - if (pos < args_content.length()) pos++; - - std::string value; - while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; - - if (pos < args_content.length()) { - size_t qtag_len = match_quote_tag(args_content, pos); - if (qtag_len > 0) { - pos += qtag_len; - size_t val_end = find_quote_tag(args_content, pos); - if (val_end != std::string::npos) { - value = "\"" + args_content.substr(pos, val_end - pos) + "\""; - pos = val_end + match_quote_tag(args_content, val_end); - } - } else if (args_content[pos] == '{') { - int depth = 1; - size_t start = pos; - pos++; - while (pos < args_content.length() && depth > 0) { - if (args_content[pos] == '{') depth++; - else if (args_content[pos] == '}') depth--; - pos++; - } - value = args_to_json(args_content.substr(start, pos - start)); - } else if (args_content[pos] == '[') { - int depth = 1; - size_t start = pos; - pos++; - while (pos < args_content.length() && depth > 0) { - if (args_content[pos] == '[') depth++; - else if (args_content[pos] == ']') depth--; - pos++; - } - std::string arr_content = args_content.substr(start + 1, pos - start - 2); - value = "["; - size_t arr_pos = 0; - bool first_item = true; - while (arr_pos < arr_content.length()) { - while (arr_pos < arr_content.length() && (std::isspace(arr_content[arr_pos]) || arr_content[arr_pos] == ',')) arr_pos++; - if (arr_pos >= arr_content.length()) break; - - if (!first_item) value += ","; - first_item = false; - - size_t aq_len = match_quote_tag(arr_content, arr_pos); - if (aq_len > 0) { - arr_pos += aq_len; - size_t end = find_quote_tag(arr_content, arr_pos); - if (end != std::string::npos) { - value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; - arr_pos = end + match_quote_tag(arr_content, end); - } - } else { - size_t end = arr_content.find_first_of(",]", arr_pos); - if (end == std::string::npos) end = arr_content.length(); - value += arr_content.substr(arr_pos, end - arr_pos); - arr_pos = end; - } - } - value += "]"; - } else { - size_t val_start = pos; - while (pos < args_content.length() && args_content[pos] != ',' && args_content[pos] != '}') { - pos++; - } - value = args_content.substr(val_start, pos - val_start); - while (!value.empty() && std::isspace(value.back())) value.pop_back(); - } - } - - if (!first) result += ","; - first = false; - result += "\"" + key + "\":" + value; - } - - result += "}"; - return result; -} - -inline void parse_function_calls(std::string& response, std::vector& function_calls) { - - const std::string CALL_START = (response.find("<|tool_call>") != std::string::npos) - ? "<|tool_call>" : ""; - const std::string CALL_END = (CALL_START == "<|tool_call>") - ? "" : ""; - size_t pos = 0; - - while ((pos = response.find(CALL_START, pos)) != std::string::npos) { - size_t content_start = pos + CALL_START.length(); - size_t call_end_pos = response.find(CALL_END, content_start); - - size_t content_end = (call_end_pos != std::string::npos) ? call_end_pos : response.length(); - std::string call_content = response.substr(content_start, content_end - content_start); - - if (call_content.compare(0, 5, "call:") == 0) { - size_t brace_pos = call_content.find('{'); - - if (brace_pos == std::string::npos) { - size_t sep_pos = call_content.find_first_of(", ", 5); - if (sep_pos != std::string::npos) { - std::string func_name = call_content.substr(5, sep_pos - 5); - size_t args_start = sep_pos + 1; - while (args_start < call_content.length() && - (call_content[args_start] == ' ' || call_content[args_start] == ',')) { - args_start++; - } - std::string args_content = "{" + call_content.substr(args_start); - if (args_content.back() != '}') args_content += "}"; - - std::string args_json = args_to_json(args_content); - std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; - function_calls.push_back(json_call); - } - } else { - std::string func_name = call_content.substr(5, brace_pos - 5); - std::string args_content = call_content.substr(brace_pos); - if (args_content.back() != '}') args_content += "}"; - - std::string args_json = args_to_json(args_content); - std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; - function_calls.push_back(json_call); - } - } - - size_t erase_end = (call_end_pos != std::string::npos) ? - call_end_pos + CALL_END.length() : response.length(); - response.erase(pos, erase_end - pos); - } -} - -} // namespace gemma \ No newline at end of file diff --git a/cactus-transpile/graph/graph.h b/cactus-transpile/graph/graph.h deleted file mode 100644 index 4c387e582..000000000 --- a/cactus-transpile/graph/graph.h +++ /dev/null @@ -1,805 +0,0 @@ -#ifndef GRAPH_H -#define GRAPH_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { - -enum class LogLevel { - DEBUG = 0, - INFO = 1, - WARN = 2, - ERROR = 3, - NONE = 4 -}; - -class Logger { -public: - static Logger& instance() { - static Logger logger; - return logger; - } - - void set_level(LogLevel level) { min_level_ = level; } - LogLevel get_level() const { return min_level_; } - - - void set_callback(std::function cb) { - std::lock_guard lock(mutex_); - callback_ = cb; - } - - void log(LogLevel level, const std::string& component, const std::string& message) { - if (level < min_level_) return; - - std::lock_guard lock(mutex_); - - if (callback_) { - callback_(level, component, message); - } else { - std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl; - } - - if (level == LogLevel::ERROR) { - last_error_ = "[" + component + "] " + message; - } - } - - const std::string& last_error() const { return last_error_; } - void clear_error() { last_error_.clear(); } - -private: - Logger() : min_level_(LogLevel::WARN) {} - - static const char* level_string(LogLevel level) { - switch (level) { - case LogLevel::DEBUG: return "DEBUG"; - case LogLevel::INFO: return "INFO"; - case LogLevel::WARN: return "WARN"; - case LogLevel::ERROR: return "ERROR"; - default: return "?"; - } - } - - LogLevel min_level_; - std::mutex mutex_; - std::string last_error_; - std::function callback_; -}; - -} // namespace cactus - -#define CACTUS_LOG(level, component, msg) \ - do { \ - if (static_cast(level) >= static_cast(cactus::Logger::instance().get_level())) { \ - std::ostringstream _cactus_log_ss; \ - _cactus_log_ss << msg; \ - cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \ - } \ - } while(0) - -#define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg) -#define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg) -#define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) -#define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) - -namespace GraphFile { - class MappedFile; - struct SerializedGraph; -} - -enum class Precision { - INT8, - FP16, - FP32, - INT4 -}; - -enum class ComputeBackend { - CPU, - NPU -}; - -enum class Activation { - SILU, - GELU, - GELU_ERF, - RELU, - SIGMOID, - TANH -}; - -enum class OpType { - INPUT, PRECISION_CAST, - ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE, - ABS, POW, FLATTEN, VIEW, - MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, - BILINEAR_INTERPOLATION, - SUM, MEAN, VARIANCE, MIN, MAX, - RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, CONV1D_SAME_DEPTHWISE_K9, CONV1D_POINTWISE, CONV2D_K3S2P1, CONV2D_DEPTHWISE_K3S2P1, CONV2D_POINTWISE_1X1, GLU, BATCHNORM, - SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN, SCALAR_LOG, - RELU, SILU, GELU, GELU_ERF, SIGMOID, TANH, - SAMPLE, CONCAT, CAT, - SCATTER_TOPK, - TOPK, LAYERNORM, GROUPNORM, - MOE_LAYER, - INDEX, - PERSISTENT, - QUANTIZE_ACTIVATIONS, - LSTM_CELL, - GATED_DELTANET_DECODE, - GATED_DELTANET_PREFILL, - STFT, - ALTUP_PREDICT, - ALTUP_CORRECT, - GAUSSIAN_TOPK, - MAXPOOL1D, - BILSTM_SEQUENCE, - LEAKY_RELU, - CONV2D_K3S1P1, - STATS_POOL, - WEIGHTED_STATS_POOL, - NOT_EQUAL, - SCALAR_NOT_EQUAL, - SCALAR_FLOOR_DIVIDE, - MASKED_SELECT_PREFIX, - MASKED_SCATTER, - EXPAND, - CONV2D, - CUMSUM -}; - -struct PrecisionTraits { - static constexpr size_t size_of(Precision prec) { - switch (prec) { - case Precision::INT8: return 1; - case Precision::FP16: return 2; - case Precision::FP32: return 4; - case Precision::INT4: return 1; - } - return 1; - } - - static constexpr size_t packed_size_of(Precision prec, size_t count) { - switch (prec) { - case Precision::INT4: return (count + 1) / 2; - default: return count * size_of(prec); - } - } - - static size_t byte_offset_of(Precision prec, size_t element_offset) { - switch (prec) { - case Precision::INT4: - assert(element_offset % 32 == 0 && "INT4 byte offset must be group-aligned (multiple of 32)"); - return element_offset / 2; - default: return element_offset * size_of(prec); - } - } - - static constexpr bool is_integer(Precision prec) { - switch (prec) { - case Precision::INT8: return true; - case Precision::INT4: return true; - case Precision::FP16: return false; - case Precision::FP32: return false; - } - return true; - } - - static constexpr bool is_floating_point(Precision prec) { - switch (prec) { - case Precision::INT8: return false; - case Precision::INT4: return false; - case Precision::FP16: return true; - case Precision::FP32: return true; - } - return false; - } -}; - -namespace Quantization { - void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); - void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); - void fp16_to_fp32(const __fp16* src, float* dst, size_t count); - void fp32_to_fp16(const float* src, __fp16* dst, size_t count); - void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); - void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); -} - -struct TensorConfig { - Precision default_precision = Precision::INT8; - Precision compute_precision = Precision::INT8; - Precision output_precision = Precision::INT8; - bool auto_mixed_precision = false; - - static TensorConfig& global(); -}; - -struct BroadcastInfo { - std::vector output_shape; - bool needs_broadcasting; - - static BroadcastInfo compute(const std::vector& lhs, const std::vector& rhs); -}; - -class BufferPool; - -struct BufferDesc { - std::vector shape; - size_t total_size; - size_t byte_size; - std::unique_ptr data; - void* external_data; - char* pooled_data; - Precision precision; - - size_t group_size = 0; - size_t num_groups = 0; - void* scales_data = nullptr; - std::unique_ptr owned_scales; - - bool is_interleaved = false; - size_t original_N = 0; - - void* activation_scales_data = nullptr; - std::unique_ptr owned_activation_scales; - size_t num_rows_for_activation_scales = 0; - - BufferDesc(); - BufferDesc(const std::vector& s, Precision prec = Precision::INT8); - ~BufferDesc(); - - BufferDesc(BufferDesc&& other) noexcept; - BufferDesc& operator=(BufferDesc&& other) noexcept; - - BufferDesc(const BufferDesc&) = delete; - BufferDesc& operator=(const BufferDesc&) = delete; - - void* get_data(); - const void* get_data() const; - - template - T* data_as() { return static_cast(get_data()); } - - template - const T* data_as() const { return static_cast(get_data()); } - - const __fp16* scales_as_fp16() const { - return reinterpret_cast(scales_data); - } - - bool is_grouped_int8() const { - return precision == Precision::INT8 && group_size > 0; - } - - bool is_grouped_int4() const { - return precision == Precision::INT4 && group_size > 0; - } - - void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) { - group_size = gs; - num_groups = ng; - scales_data = scales_ptr; - } - - void set_interleaved(bool interleaved, size_t orig_n) { - is_interleaved = interleaved; - original_N = orig_n; - } - - bool has_activation_scales() const { - return activation_scales_data != nullptr && num_rows_for_activation_scales > 0; - } - const float* activation_scales_as_float() const { - return reinterpret_cast(activation_scales_data); - } - float* activation_scales_as_float() { - return reinterpret_cast(activation_scales_data); - } - void allocate_activation_scales(size_t num_rows) { - num_rows_for_activation_scales = num_rows; - owned_activation_scales = std::make_unique(num_rows * sizeof(float)); - activation_scales_data = owned_activation_scales.get(); - } - void set_activation_scales(void* scales_ptr, size_t num_rows) { - activation_scales_data = scales_ptr; - num_rows_for_activation_scales = num_rows; - } - - void allocate(); - void allocate_from_pool(BufferPool& pool); - void release_to_pool(BufferPool& pool); - void set_external(void* ptr); -}; - -struct OpParams { - float scalar = 0.0f; - float scale = 1.0f; - float theta = 10000.0f; - float epsilon = 1e-6f; - int axis = -1; - bool pretransposed_rhs = false; - size_t position_offset = 0; - size_t slice_start = 0; - size_t slice_length = 0; - size_t window_size = 0; - bool is_causal = true; - bool attention_mask_is_additive = false; - float logit_cap = 0.0f; - std::vector new_shape; - std::vector permutation; - Precision output_precision = Precision::INT8; - BroadcastInfo broadcast_info; - ComputeBackend backend = ComputeBackend::CPU; - - size_t dilation = 1; - size_t stride = 1; - size_t padding = 0; - float temperature = 1.0f; - float top_p = 1.0f; - float min_p = 0.15f; - float repetition_penalty = 1.1f; - size_t top_k = 0; - size_t random_seed = 0; - - size_t index_value = 0; - size_t num_classes = 0; - size_t num_groups = 0; - size_t dst_height = 0; - size_t dst_width = 0; - bool align_corners = true; - bool normalize_routing = false; - size_t num_experts = 0; - size_t num_experts_per_tok = 0; - bool moe_gated = true; - Activation activation = Activation::SILU; - - std::vector bias_values; - std::vector bias_indices; - - const int8_t* cached_keys_int8 = nullptr; - const int8_t* cached_values_int8 = nullptr; - const float* cached_k_scales = nullptr; - const float* cached_v_scales = nullptr; - size_t cache_seq_len = 0; - size_t num_kv_heads = 0; - size_t head_dim = 0; - size_t num_fft_bins = 0; - size_t chunk_size = 0; - size_t num_altup_inputs = 0; - size_t v_head_dim = 0; - size_t kernel_size = 0; -}; - -struct GraphNode { - size_t id; - OpType op_type; - std::vector input_ids; - BufferDesc output_buffer; - OpParams params; - - GraphNode(size_t node_id, OpType type); -}; - -using nodes_vector = std::vector>; -using node_index_map_t = std::unordered_map; - -inline const BufferDesc& get_input(const GraphNode& node, size_t idx, - const nodes_vector& nodes, - const node_index_map_t& node_index_map) { - return nodes[node_index_map.at(node.input_ids[idx])]->output_buffer; -} - -struct AxisDims { - size_t outer, axis_size, inner; - static AxisDims from_shape(const std::vector& shape, size_t axis) { - AxisDims d; - d.outer = 1; - for (size_t i = 0; i < axis; i++) d.outer *= shape[i]; - d.axis_size = shape[axis]; - d.inner = 1; - for (size_t i = axis + 1; i < shape.size(); i++) d.inner *= shape[i]; - return d; - } -}; - -template -void dispatch_binary_op(OpType op, const T* lhs, const T* rhs, T* output, size_t count); - -template -void dispatch_unary_op(OpType op, const T* input, T* output, size_t count, float param = 0.0f); - -void compute_node_optimized(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_cumsum_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_masked_select_prefix_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_masked_scatter_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_expand_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_conv2d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); - -void shrink_thread_local_buffers(); -class BufferPool { -public: - BufferPool() = default; - ~BufferPool() = default; - - BufferPool(const BufferPool&) = delete; - BufferPool& operator=(const BufferPool&) = delete; - BufferPool(BufferPool&&) noexcept = default; - BufferPool& operator=(BufferPool&&) noexcept = default; - - char* acquire(size_t byte_size); - void release(char* ptr, size_t byte_size); - void clear(); - - size_t active_bytes() const { return active_bytes_; } - size_t pool_bytes() const { return pool_bytes_; } - size_t peak_bytes() const { return peak_bytes_; } - -private: - std::unordered_map>> free_buffers_; - size_t active_bytes_ = 0; - size_t pool_bytes_ = 0; - size_t peak_bytes_ = 0; - - size_t round_up_size(size_t size) const; -}; - -namespace ValidationUtils { - void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name); - void validate_precision(Precision actual, Precision required, const std::string& op_name); - void validate_input_count(size_t actual, size_t required, const std::string& op_name); -} - - -class CactusGraph { -public: - CactusGraph(); - ~CactusGraph() = default; - - CactusGraph(const CactusGraph&) = delete; - CactusGraph& operator=(const CactusGraph&) = delete; - CactusGraph(CactusGraph&&) noexcept = default; - CactusGraph& operator=(CactusGraph&&) noexcept = default; - - struct DebugNodeEntry { - uint32_t layer_idx; - std::string name; - size_t node_id; - }; - - void save(const std::string& path); - static CactusGraph load(const std::string& path); - - size_t input(const std::vector& shape, Precision precision = Precision::INT8); - size_t precision_cast(size_t input, Precision target_precision); - size_t quantize_activations(size_t input); - - size_t add(size_t input1, size_t input2); - size_t add_clipped(size_t input1, size_t input2); - size_t subtract(size_t input1, size_t input2); - size_t multiply(size_t input1, size_t input2); - size_t divide(size_t input1, size_t input2); - size_t not_equal(size_t input1, size_t input2); - - size_t scalar_add(size_t input, float value); - size_t scalar_subtract(size_t input, float value); - size_t scalar_multiply(size_t input, float value); - size_t scalar_divide(size_t input, float value); - size_t scalar_floor_divide(size_t input, float value); - size_t scalar_not_equal(size_t input, float value); - size_t scalar_exp(size_t input); - size_t scalar_sqrt(size_t input); - size_t scalar_cos(size_t input); - size_t scalar_sin(size_t input); - size_t scalar_log(size_t input); - - size_t relu(size_t input); - size_t silu(size_t input); - size_t gelu(size_t input); - size_t gelu_erf(size_t input); - size_t sigmoid(size_t input); - size_t tanh(size_t input); - size_t glu(size_t input, int axis = -1); - - size_t abs(size_t input); - size_t pow(size_t input, float exponent); - size_t view(size_t input, const std::vector& new_shape); - size_t flatten(size_t input, int start_dim = 0, int end_dim = -1); - - size_t matmul( - size_t input1, - size_t input2, - bool pretransposed_rhs = false, - ComputeBackend backend = ComputeBackend::CPU, - Precision output_precision = Precision::INT8 - ); - size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU); - size_t transposeN(size_t input, const std::vector& permutation, ComputeBackend backend = ComputeBackend::CPU); - size_t reshape(size_t input, const std::vector& new_shape); - size_t expand(size_t input, const std::vector& new_shape); - size_t slice(size_t input, int axis, size_t start, size_t length); - size_t index(size_t input, size_t index_value, int dim); - size_t masked_select_prefix(size_t input, size_t mask); - size_t masked_scatter(size_t input, size_t mask, size_t source); - - size_t sum(size_t input, int axis); - size_t mean(size_t input, int axis); - size_t variance(size_t input, int axis); - size_t min(size_t input, int axis); - size_t max(size_t input, int axis); - size_t cumsum(size_t input, int axis); - - size_t gather(size_t embeddings, size_t indices, int axis = 0); - size_t mmap_embeddings(const std::string& filename); - size_t mmap_weights(const std::string& filename); - void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr); - void set_interleaved(size_t node_id, bool interleaved, size_t original_N); - - void release_weight_pages(size_t node_id); - void prefetch_weight_pages(size_t node_id); - void release_all_weight_pages(); - size_t embedding(const std::string& filename, size_t indices); - size_t embedding(size_t embedding_tensor, size_t indices); - size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners = true); - - size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f); - size_t layernorm(size_t input, size_t weight, float epsilon = 1e-5f); // No bias version - size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); - size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); - size_t topk(size_t input, size_t k); - size_t moe_layer(size_t hidden, - size_t routing_probs, - size_t topk_indices, - const std::vector& w1_weights, - const std::vector& w3_weights, - const std::vector& w2_weights, - size_t num_experts, - size_t num_experts_per_tok, - bool normalize_routing, - float epsilon, - float routed_scaling_factor, - Activation activation = Activation::SILU, - size_t per_expert_scale = 0); - size_t moe_layer(size_t hidden, - size_t routing_probs, - size_t topk_indices, - const std::vector& w1_weights, - const std::vector& w2_weights, - size_t num_experts, - size_t num_experts_per_tok, - bool normalize_routing, - float epsilon, - float routed_scaling_factor, - Activation activation); - size_t rms_norm(size_t input, size_t weight, float epsilon = 1e-5f); - size_t rope(size_t input, float theta, size_t position_offset = 0, ComputeBackend backend = ComputeBackend::CPU); - size_t rope_gptj(size_t input, float theta, size_t position_offset = 0, size_t rot_dim = 0, ComputeBackend backend = ComputeBackend::CPU); - size_t softmax(size_t input, int axis = -1); - size_t attention(size_t query, size_t key, size_t value, float scale, bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU); - size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU); - size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU); - size_t attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, - bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU, - bool additive_mask = false, size_t position_offset = 0, size_t window_size = 0, - float logit_cap = 0.0f); - size_t rel_pos_bias(size_t query, size_t relative_key, float scale); - - size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, - const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, - size_t window_size = 0, size_t v_head_dim = 0); - - size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); - size_t conv1d_k3(size_t input, size_t weight, size_t stride); - size_t conv1d_k7s3(size_t input, size_t weight, size_t bias); - size_t conv1d(size_t input, size_t weight, size_t stride); - size_t conv1d(size_t input, size_t weight, size_t bias, size_t stride); - size_t conv1d_same_depthwise_k9(size_t input, size_t weight); - size_t conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias); - size_t conv1d_pointwise(size_t input, size_t weight); - size_t conv1d_pointwise(size_t input, size_t weight, size_t bias); - size_t conv2d_k3s2p1(size_t input, size_t weight); - size_t conv2d_k3s2p1(size_t input, size_t weight, size_t bias); - size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight); - size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias); - size_t conv2d_pointwise_1x1(size_t input, size_t weight); - size_t conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias); - size_t conv2d(size_t input, size_t weight, size_t stride = 1, size_t padding = 0, size_t dilation = 1, size_t groups = 1); - size_t conv2d(size_t input, size_t weight, size_t bias, size_t stride, size_t padding, size_t dilation, size_t groups); - - size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh); - size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, - size_t initial_state, float scale = 0.0f); - size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, - size_t initial_state, size_t chunk_size = 64, float scale = 0.0f); - size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins); - - size_t altup_predict(size_t coefs, const size_t* streams, size_t num_streams); - size_t altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions); - - size_t gaussian_topk(size_t input, float ppf); - - size_t maxpool1d(size_t input, size_t kernel_size, size_t stride); - size_t leaky_relu(size_t input, float negative_slope = 0.01f); - size_t bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, - size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd); - size_t conv2d_k3s1p1(size_t input, size_t weight); - size_t conv2d_k3s1p1(size_t input, size_t weight, size_t bias); - size_t stats_pool(size_t input); - size_t weighted_stats_pool(size_t input, size_t weights); - - size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20, - const std::unordered_map& logit_bias = {}); - size_t sample_with_options(size_t logits, float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, const std::unordered_map& logit_bias = {}); - - size_t concat(size_t input1, size_t input2, int axis = 0); - size_t cat(const std::vector& inputs, int axis); - size_t scatter_topk(size_t indices, size_t values, size_t num_classes); - - void set_input(size_t node_id, const void* data, Precision precision); - void set_external_input(size_t node_id, void* data, Precision precision); - void* get_output(size_t node_id); - - void execute(const std::string& profile_file = ""); - void hard_reset(); - void soft_reset(); - void soft_reset_keep_pool(); - void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; } - - void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); - void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); - const std::vector& get_debug_nodes() const; - void clear_debug_nodes(); - - size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); - const BufferDesc& get_output_buffer(size_t node_id) const; - void allocate_buffers(); - size_t get_node_count() const; - - size_t persistent(size_t source_node); - bool is_populated(size_t persistent_node_id) const; - void invalidate_persistent(size_t persistent_node_id); - - std::vector> nodes_; - std::unordered_map node_index_map_; - -private: - static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); - size_t next_node_id_; - std::vector> mapped_files_; - std::unordered_map weight_cache_; - std::unordered_map node_to_mapped_file_; - std::vector debug_nodes_; - BufferPool buffer_pool_; - bool prefill_mode_ = false; - - std::unordered_set persistent_node_ids_; - std::unordered_set populated_node_ids_; -}; - - -namespace GraphFile { - struct LoadedNode { - size_t node_id; - std::vector shape; - Precision precision; - size_t byte_size; - }; - - struct GraphHeader { - uint32_t magic; - uint32_t version; - uint32_t node_count; - uint32_t flags = 0; - }; - - struct NodeEntry { - uint32_t index; // serialized node index 0..n-1 - OpType op_type; - std::vector inputs; - std::vector output_shape; - Precision precision; - OpParams params; - }; - - struct SerializedGraph { - GraphHeader header; - std::vector nodes; - std::vector graph_inputs; // IDs of serialized inputs - std::vector graph_outputs; // IDs of serialized outputs - }; - - SerializedGraph load_graph(const std::string& filename); - void save_graph(const CactusGraph& graph, const std::string& filename); - - void save_node(CactusGraph& graph, size_t node_id, const std::string& filename); - - class MappedFile { - public: - MappedFile(const std::string& filename); - ~MappedFile(); - - MappedFile(const MappedFile&) = delete; - MappedFile& operator=(const MappedFile&) = delete; - MappedFile(MappedFile&& other) noexcept; - MappedFile& operator=(MappedFile&& other) noexcept; - - const std::vector& shape() const; - Precision precision() const; - size_t byte_size() const; - - size_t group_size() const { return group_size_; } - size_t num_groups() const { return num_groups_; } - const void* scales_data() const; - - bool is_interleaved() const { return is_interleaved_; } - size_t original_N() const { return original_N_; } - - void* data(); - const void* data() const; - - template - const T* typed_data() const; - - void release_pages(); - void prefetch_pages(); - - private: - int fd_; - void* mapped_data_; - size_t file_size_, data_offset_; - std::vector shape_; - Precision precision_; - size_t byte_size_; - size_t group_size_ = 0; - size_t num_groups_ = 0; - size_t scales_offset_ = 0; - size_t scales_bytes_ = 0; - uint32_t alignment_ = 32; - - bool is_interleaved_ = false; - size_t original_N_ = 0; - - void parse_header(); - void apply_madvise_hints(); - }; -} - -#endif diff --git a/cactus-transpile/graph/graph_builder.cpp b/cactus-transpile/graph/graph_builder.cpp deleted file mode 100644 index 0710a1485..000000000 --- a/cactus-transpile/graph/graph_builder.cpp +++ /dev/null @@ -1,1879 +0,0 @@ -#include "graph.h" -#include -#include -#include -#include - -namespace { - -size_t conv2d_output_dim(size_t input, size_t kernel, size_t stride, size_t padding, size_t dilation) { - if (stride == 0) { - throw std::runtime_error("conv2d stride must be > 0"); - } - if (dilation == 0) { - throw std::runtime_error("conv2d dilation must be > 0"); - } - const size_t effective_kernel = dilation * (kernel - 1) + 1; - if (input + 2 * padding < effective_kernel) { - throw std::runtime_error("conv2d input spatial dimensions are too small for the requested kernel"); - } - return (input + 2 * padding - effective_kernel) / stride + 1; -} - -void validate_conv2d_weight_shape( - const BufferDesc& xin, - const BufferDesc& w, - size_t groups, - const char* op_name -) { - if (xin.shape.size() != 4) { - throw std::runtime_error(std::string(op_name) + " expects input [N, C_in, H, W]"); - } - if (w.shape.size() != 4) { - throw std::runtime_error(std::string(op_name) + " weight must be [C_out, C_in/groups, K_h, K_w]"); - } - if (groups == 0) { - throw std::runtime_error(std::string(op_name) + " groups must be > 0"); - } - - const size_t c_in = xin.shape[1]; - const size_t c_out = w.shape[0]; - const size_t c_in_per_group = w.shape[1]; - if ((c_in % groups) != 0) { - throw std::runtime_error(std::string(op_name) + " input channels must be divisible by groups"); - } - if ((c_out % groups) != 0) { - throw std::runtime_error(std::string(op_name) + " output channels must be divisible by groups"); - } - if (c_in_per_group * groups != c_in) { - throw std::runtime_error(std::string(op_name) + " weight must match [C_out, C_in/groups, K_h, K_w]"); - } - if (xin.shape[2] == 0 || xin.shape[3] == 0) { - throw std::runtime_error(std::string(op_name) + " input spatial dimensions must be > 0"); - } -} - -int normalize_reduction_axis(const BufferDesc& input_buffer, int axis, const char* op_name) { - if (input_buffer.shape.empty()) { - if (axis == 0 || axis == -1) { - return 0; - } - throw std::runtime_error(std::string(op_name) + " axis out of range for scalar input"); - } - - int actual_axis = axis; - if (actual_axis < 0) { - actual_axis += static_cast(input_buffer.shape.size()); - } - if (actual_axis < 0 || static_cast(actual_axis) >= input_buffer.shape.size()) { - throw std::runtime_error(std::string(op_name) + " axis out of range"); - } - return actual_axis; -} - -std::vector reduction_output_shape(const BufferDesc& input_buffer, int axis) { - std::vector output_shape = input_buffer.shape; - output_shape.erase(output_shape.begin() + axis); - if (output_shape.empty()) { - output_shape = {1}; - } - return output_shape; -} - -} // namespace - -size_t CactusGraph::input(const std::vector& shape, Precision precision) { - return add_node(OpType::INPUT, {}, shape, {.output_precision = precision}); -} - -size_t CactusGraph::add(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::ADD, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::add_clipped(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::ADD_CLIPPED, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::subtract(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::SUBTRACT, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::multiply(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::MULTIPLY, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::divide(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::DIVIDE, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::not_equal(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info, .output_precision = Precision::FP16}; - - return add_node(OpType::NOT_EQUAL, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::abs(size_t input) { - const auto& input_buffer = get_output_buffer(input); - OpParams params{.output_precision = input_buffer.precision}; - return add_node(OpType::ABS, {input}, input_buffer.shape, params); -} - -size_t CactusGraph::pow(size_t input, float exponent) { - const auto& input_buffer = get_output_buffer(input); - OpParams params{.scalar = exponent, .output_precision = input_buffer.precision}; - return add_node(OpType::POW, {input}, input_buffer.shape, params); -} - -size_t CactusGraph::view(size_t input, const std::vector& new_shape) { - const auto& input_buffer = get_output_buffer(input); - - size_t input_elements = 1; - for (size_t dim : input_buffer.shape) { - input_elements *= dim; - } - - size_t new_elements = 1; - for (size_t dim : new_shape) { - new_elements *= dim; - } - - if (input_elements != new_elements) { - throw std::runtime_error("View operation requires total number of elements to remain the same"); - } - - OpParams params{.new_shape = new_shape}; - return add_node(OpType::VIEW, {input}, new_shape, params); -} - -size_t CactusGraph::flatten(size_t input, int start_dim, int end_dim) { - const auto& input_buffer = get_output_buffer(input); - const auto& shape = input_buffer.shape; - size_t rank = shape.size(); - - if (start_dim < 0) start_dim += rank; - if (end_dim < 0) end_dim += rank; - - if (start_dim < 0 || static_cast(start_dim) >= rank || - end_dim < 0 || static_cast(end_dim) >= rank || - start_dim > end_dim) { - throw std::runtime_error("Invalid start_dim or end_dim for flatten operation"); - } - - std::vector output_shape; - - for (int i = 0; i < start_dim; ++i) { - output_shape.push_back(shape[i]); - } - - size_t flattened_dim = 1; - for (int i = start_dim; i <= end_dim; ++i) { - flattened_dim *= shape[i]; - } - output_shape.push_back(flattened_dim); - - for (size_t i = end_dim + 1; i < rank; ++i) { - output_shape.push_back(shape[i]); - } - - OpParams params{.new_shape = output_shape}; - return add_node(OpType::FLATTEN, {input}, output_shape, params); -} - -size_t CactusGraph::matmul( - size_t input1, - size_t input2, - bool pretransposed_rhs, - ComputeBackend backend, - Precision output_precision -) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - if (lhs_buffer.shape.size() != 2 || rhs_buffer.shape.size() != 2) { - throw std::invalid_argument("Matrix multiplication requires 2D tensors"); - } - - size_t M = lhs_buffer.shape[0]; - size_t K = lhs_buffer.shape[1]; - size_t rhs_K = pretransposed_rhs ? rhs_buffer.shape[1] : rhs_buffer.shape[0]; - - // For interleaved weights, use original_N (shape stores padded N) - size_t N; - if (rhs_buffer.is_interleaved && rhs_buffer.original_N > 0) { - N = rhs_buffer.original_N; - } else { - N = pretransposed_rhs ? rhs_buffer.shape[0] : rhs_buffer.shape[1]; - } - - if (K != rhs_K) { - std::cout << "Matrix dimensions incompatible for multiplication: " - << "lhs=[" << M << "," << K << "] rhs=[" << rhs_buffer.shape[0] << "," << rhs_buffer.shape[1] << "]" - << " pretransposed=" << pretransposed_rhs - << " (K=" << K << " != rhs_K=" << rhs_K << ")" << std::endl; - throw std::invalid_argument("Matrix dimensions incompatible for multiplication"); - } - - std::vector output_shape = {M, N}; - OpParams params{ - .pretransposed_rhs = pretransposed_rhs, - .output_precision = output_precision, - .backend = backend, - }; - return add_node(OpType::MATMUL, {input1, input2}, output_shape, params); -} - -size_t CactusGraph::transpose(size_t input, ComputeBackend backend) { - const auto& input_buffer = get_output_buffer(input); - std::vector output_shape = input_buffer.shape; - - if (output_shape.size() >= 2) { - std::swap(output_shape[output_shape.size()-2], output_shape[output_shape.size()-1]); - } - - std::vector permutation; - for (size_t i = 0; i < input_buffer.shape.size(); ++i) { - permutation.push_back(i); - } - if (permutation.size() >= 2) { - std::swap(permutation[permutation.size()-2], permutation[permutation.size()-1]); - } - - OpParams params{.permutation = permutation, .backend = backend}; - return add_node(OpType::TRANSPOSE, {input}, output_shape, params); -} - -size_t CactusGraph::transposeN(size_t input, const std::vector& permutation, ComputeBackend backend) { - const auto& input_buffer = get_output_buffer(input); - if (permutation.size() != input_buffer.shape.size()) { - throw std::runtime_error("transposeN permutation size must match tensor rank"); - } - std::vector output_shape(permutation.size()); - for (size_t i = 0; i < permutation.size(); ++i) { - size_t p = permutation[i]; - if (p >= input_buffer.shape.size()) { - throw std::runtime_error("transposeN permutation index out of range"); - } - output_shape[i] = input_buffer.shape[p]; - } - OpParams params{.permutation = permutation, .backend = backend}; - return add_node(OpType::TRANSPOSE, {input}, output_shape, params); -} - -size_t CactusGraph::reshape(size_t input, const std::vector& new_shape) { - OpParams params{.new_shape = new_shape}; - return add_node(OpType::RESHAPE, {input}, new_shape, params); -} - -size_t CactusGraph::expand(size_t input, const std::vector& new_shape) { - const auto& input_buffer = get_output_buffer(input); - const auto& input_shape = input_buffer.shape; - - if (new_shape.size() < input_shape.size()) { - throw std::invalid_argument("expand cannot reduce rank"); - } - - const size_t rank_delta = new_shape.size() - input_shape.size(); - for (size_t axis = 0; axis < new_shape.size(); ++axis) { - const size_t input_dim = axis < rank_delta ? 1 : input_shape[axis - rank_delta]; - const size_t output_dim = new_shape[axis]; - if (input_dim != 1 && input_dim != output_dim) { - throw std::invalid_argument("expand target shape is not broadcast-compatible with input"); - } - } - - OpParams params{}; - params.output_precision = input_buffer.precision; - return add_node(OpType::EXPAND, {input}, new_shape, params); -} - -size_t CactusGraph::index(size_t input, size_t index_value, int dim) { - const auto& input_buffer = get_output_buffer(input); - const auto& shape = input_buffer.shape; - - if (shape.empty()) { - throw std::invalid_argument("Cannot index a scalar tensor"); - } - - int actual_dim = dim; - if (actual_dim < 0) { - actual_dim += static_cast(shape.size()); - } - - if (actual_dim < 0 || static_cast(actual_dim) >= shape.size()) { - throw std::invalid_argument("Index dimension out of bounds"); - } - - if (index_value >= shape[actual_dim]) { - throw std::invalid_argument("Index value " + std::to_string(index_value) + - " out of bounds for dimension " + std::to_string(actual_dim) + - " with size " + std::to_string(shape[actual_dim])); - } - - std::vector output_shape; - for (size_t i = 0; i < shape.size(); ++i) { - if (static_cast(i) != actual_dim) { - output_shape.push_back(shape[i]); - } - } - - if (output_shape.empty()) { - output_shape = {1}; - } - - OpParams params{.axis = actual_dim, .output_precision = input_buffer.precision, .index_value = index_value}; - return add_node(OpType::INDEX, {input}, output_shape, params); -} - -size_t CactusGraph::masked_select_prefix(size_t input, size_t mask) { - const auto& input_buffer = get_output_buffer(input); - const auto& mask_buffer = get_output_buffer(mask); - - if (mask_buffer.shape.empty()) { - throw std::invalid_argument("masked_select_prefix requires a non-scalar mask"); - } - if (input_buffer.shape.size() < mask_buffer.shape.size()) { - throw std::invalid_argument("masked_select_prefix mask rank must not exceed input rank"); - } - for (size_t i = 0; i < mask_buffer.shape.size(); ++i) { - if (input_buffer.shape[i] != mask_buffer.shape[i]) { - throw std::invalid_argument("masked_select_prefix mask shape must match the input prefix shape"); - } - } - - std::vector output_shape; - output_shape.push_back(mask_buffer.total_size); - for (size_t i = mask_buffer.shape.size(); i < input_buffer.shape.size(); ++i) { - output_shape.push_back(input_buffer.shape[i]); - } - - OpParams params{}; - params.output_precision = input_buffer.precision; - return add_node(OpType::MASKED_SELECT_PREFIX, {input, mask}, output_shape, params); -} - -size_t CactusGraph::masked_scatter(size_t input, size_t mask, size_t source) { - const auto& input_buffer = get_output_buffer(input); - const auto& mask_buffer = get_output_buffer(mask); - const auto& source_buffer = get_output_buffer(source); - - if (input_buffer.shape != mask_buffer.shape) { - throw std::invalid_argument("masked_scatter mask shape must match input shape"); - } - if (input_buffer.precision != source_buffer.precision) { - throw std::invalid_argument("masked_scatter source precision must match input precision"); - } - - OpParams params{}; - params.output_precision = input_buffer.precision; - return add_node(OpType::MASKED_SCATTER, {input, mask, source}, input_buffer.shape, params); -} - -size_t CactusGraph::sum(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - const int actual_axis = normalize_reduction_axis(input_buffer, axis, "sum"); - const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); - OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; - return add_node(OpType::SUM, {input}, output_shape, params); -} - -size_t CactusGraph::mean(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - const int actual_axis = normalize_reduction_axis(input_buffer, axis, "mean"); - const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); - OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; - return add_node(OpType::MEAN, {input}, output_shape, params); -} - -size_t CactusGraph::variance(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - const int actual_axis = normalize_reduction_axis(input_buffer, axis, "variance"); - const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); - OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; - return add_node(OpType::VARIANCE, {input}, output_shape, params); -} - -size_t CactusGraph::min(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - const int actual_axis = normalize_reduction_axis(input_buffer, axis, "min"); - const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); - OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; - return add_node(OpType::MIN, {input}, output_shape, params); -} - -size_t CactusGraph::max(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - const int actual_axis = normalize_reduction_axis(input_buffer, axis, "max"); - const std::vector output_shape = reduction_output_shape(input_buffer, actual_axis); - OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; - return add_node(OpType::MAX, {input}, output_shape, params); -} - -size_t CactusGraph::cumsum(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - const int actual_axis = normalize_reduction_axis(input_buffer, axis, "cumsum"); - OpParams params{.axis = actual_axis, .output_precision = input_buffer.precision}; - return add_node(OpType::CUMSUM, {input}, input_buffer.shape, params); -} - -size_t CactusGraph::rms_norm(size_t input, size_t weight, float epsilon) { - OpParams params{.epsilon = epsilon}; - return add_node(OpType::RMS_NORM, {input, weight}, {}, params); -} - -size_t CactusGraph::rope(size_t input, float theta, size_t position_offset, ComputeBackend backend) { - OpParams params{.theta = theta, .position_offset = position_offset, .backend = backend}; - return add_node(OpType::ROPE, {input}, {}, params); -} - -size_t CactusGraph::softmax(size_t input, int axis) { - OpParams params{.axis = axis}; - return add_node(OpType::SOFTMAX, {input}, {}, params); -} - -size_t CactusGraph::topk(size_t input, size_t k) { - const auto& input_buffer = get_output_buffer(input); - - if (input_buffer.shape.empty()) { - throw std::runtime_error("TopK requires non-empty input tensor"); - } - - std::vector output_shape = {2, input_buffer.shape[0], k}; - OpParams params{.output_precision = Precision::FP32, .top_k = k}; - - return add_node(OpType::TOPK, {input}, output_shape, params); -} - -size_t CactusGraph::moe_layer(size_t hidden, - size_t routing_probs, - size_t topk_indices, - const std::vector& w1_weights, - const std::vector& w3_weights, - const std::vector& w2_weights, - size_t num_experts, - size_t num_experts_per_tok, - bool normalize_routing, - float epsilon, - float routed_scaling_factor, - Activation activation, - size_t per_expert_scale) { - const auto& hidden_buffer = get_output_buffer(hidden); - const auto& routing_buffer = get_output_buffer(routing_probs); - const auto& topk_buffer = get_output_buffer(topk_indices); - - if (hidden_buffer.shape.size() != 2) { - throw std::runtime_error("moe_layer expects [tokens, hidden_dim] for hidden"); - } - if (routing_buffer.shape.size() != 2 || topk_buffer.shape.size() != 2) { - throw std::runtime_error("moe_layer expects 2D routing_probs and topk_indices"); - } - if (routing_buffer.shape[0] != hidden_buffer.shape[0] || topk_buffer.shape[0] != hidden_buffer.shape[0]) { - throw std::runtime_error("moe_layer token dimension mismatch across inputs"); - } - if (w1_weights.size() != num_experts || w3_weights.size() != num_experts || w2_weights.size() != num_experts) { - throw std::runtime_error("moe_layer expects num_experts weight tensors for each of w1, w3, w2"); - } - - std::vector input_ids; - input_ids.reserve(3 + 3 * num_experts + 1); - input_ids.push_back(hidden); - input_ids.push_back(routing_probs); - input_ids.push_back(topk_indices); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w1_weights[i]); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w3_weights[i]); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w2_weights[i]); - if (per_expert_scale != 0) input_ids.push_back(per_expert_scale); - - OpParams params; - params.num_experts = num_experts; - params.num_experts_per_tok = num_experts_per_tok; - params.normalize_routing = normalize_routing; - params.epsilon = epsilon; - params.scalar = routed_scaling_factor; - params.output_precision = hidden_buffer.precision; - params.activation = activation; - params.moe_gated = true; - - return add_node(OpType::MOE_LAYER, input_ids, hidden_buffer.shape, params); -} - -size_t CactusGraph::moe_layer(size_t hidden, - size_t routing_probs, - size_t topk_indices, - const std::vector& w1_weights, - const std::vector& w2_weights, - size_t num_experts, - size_t num_experts_per_tok, - bool normalize_routing, - float epsilon, - float routed_scaling_factor, - Activation activation) { - const auto& hidden_buffer = get_output_buffer(hidden); - const auto& routing_buffer = get_output_buffer(routing_probs); - const auto& topk_buffer = get_output_buffer(topk_indices); - - if (hidden_buffer.shape.size() != 2) { - throw std::runtime_error("moe_layer expects [tokens, hidden_dim] for hidden"); - } - if (routing_buffer.shape.size() != 2 || topk_buffer.shape.size() != 2) { - throw std::runtime_error("moe_layer expects 2D routing_probs and topk_indices"); - } - if (routing_buffer.shape[0] != hidden_buffer.shape[0] || topk_buffer.shape[0] != hidden_buffer.shape[0]) { - throw std::runtime_error("moe_layer token dimension mismatch across inputs"); - } - if (w1_weights.size() != num_experts || w2_weights.size() != num_experts) { - throw std::runtime_error("moe_layer expects num_experts weight tensors for each of w1, w2"); - } - - std::vector input_ids; - input_ids.reserve(3 + 2 * num_experts); - input_ids.push_back(hidden); - input_ids.push_back(routing_probs); - input_ids.push_back(topk_indices); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w1_weights[i]); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w2_weights[i]); - - OpParams params; - params.num_experts = num_experts; - params.num_experts_per_tok = num_experts_per_tok; - params.normalize_routing = normalize_routing; - params.epsilon = epsilon; - params.scalar = routed_scaling_factor; - params.output_precision = hidden_buffer.precision; - params.moe_gated = false; - params.activation = activation; - - return add_node(OpType::MOE_LAYER, input_ids, hidden_buffer.shape, params); -} - -size_t CactusGraph::layernorm(size_t input, size_t weight, size_t bias, float epsilon) { - OpParams params{.epsilon = epsilon}; - return add_node(OpType::LAYERNORM, {input, weight, bias}, {}, params); -} - -size_t CactusGraph::layernorm(size_t input, size_t weight, float epsilon) { - OpParams params{.epsilon = epsilon}; - return add_node(OpType::LAYERNORM, {input, weight}, {}, params); -} - -size_t CactusGraph::groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups, float epsilon) { - OpParams params{.epsilon = epsilon, .num_groups = num_groups}; - return add_node(OpType::GROUPNORM, {input, weight, bias}, {}, params); -} - -size_t CactusGraph::batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis, float epsilon) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - const auto& rm = get_output_buffer(running_mean); - const auto& rv = get_output_buffer(running_var); - - if (xin.shape.empty()) { - throw std::runtime_error("batchnorm expects non-scalar input"); - } - - int actual_axis = axis; - if (actual_axis < 0) { - actual_axis += static_cast(xin.shape.size()); - } - if (actual_axis < 0 || static_cast(actual_axis) >= xin.shape.size()) { - throw std::runtime_error("batchnorm axis out of range"); - } - - const size_t C = xin.shape[static_cast(actual_axis)]; - if (w.total_size != C || b.total_size != C || rm.total_size != C || rv.total_size != C) { - throw std::runtime_error("batchnorm parameter size mismatch"); - } - - OpParams params{.epsilon = epsilon, .axis = actual_axis}; - params.output_precision = xin.precision; - return add_node(OpType::BATCHNORM, {input, weight, bias, running_mean, running_var}, xin.shape, params); -} - -size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, bool is_causal, ComputeBackend backend) { - OpParams params{.scale = scale, .is_causal = is_causal, .backend = backend}; - const auto& qs = get_output_buffer(query).shape; - const auto& vs = get_output_buffer(value).shape; - return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); -} - -size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend) { - OpParams params{.scale = scale, .position_offset = position_offset, .backend = backend}; - const auto& qs = get_output_buffer(query).shape; - const auto& vs = get_output_buffer(value).shape; - return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); -} - -size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend) { - OpParams params{.scale = scale, .position_offset = position_offset, .window_size = window_size, .backend = backend}; - const auto& qs = get_output_buffer(query).shape; - const auto& vs = get_output_buffer(value).shape; - return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); -} - -size_t CactusGraph::attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, - bool is_causal, ComputeBackend backend, bool additive_mask, - size_t position_offset, size_t window_size, float logit_cap) { - OpParams params{ - .scale = scale, - .position_offset = position_offset, - .window_size = window_size, - .is_causal = is_causal, - .backend = backend - }; - params.attention_mask_is_additive = additive_mask; - params.logit_cap = logit_cap; - const auto& qs = get_output_buffer(query).shape; - const auto& vs = get_output_buffer(value).shape; - return add_node(OpType::ATTENTION, {query, key, value, mask}, {qs[0], qs[1], qs[2], vs[3]}, params); -} - -size_t CactusGraph::rel_pos_bias(size_t query, size_t relative_key, float scale) { - const auto& q = get_output_buffer(query); - const auto& r = get_output_buffer(relative_key); - - if (q.shape.size() != 4) { - throw std::runtime_error("rel_pos_bias expects query [B, T, H, D]"); - } - if (r.shape.size() != 4) { - throw std::runtime_error("rel_pos_bias expects relative_key [B, R, H, D]"); - } - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t H = q.shape[2]; - const size_t D = q.shape[3]; - const size_t R = r.shape[1]; - - if (r.shape[0] != 1 && r.shape[0] != B) { - throw std::runtime_error("rel_pos_bias relative_key batch must be 1 or match query batch"); - } - if (r.shape[2] != H || r.shape[3] != D) { - throw std::runtime_error("rel_pos_bias expects matching [H, D] between query and relative_key"); - } - if (R < (2 * T - 1)) { - throw std::runtime_error("rel_pos_bias requires relative_key length >= 2*T-1"); - } - - OpParams params{.scale = scale, .output_precision = q.precision}; - return add_node(OpType::REL_POS_BIAS, {query, relative_key}, {B, H, T, T}, params); -} - -size_t CactusGraph::attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, - const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, - size_t window_size, size_t v_head_dim) { - OpParams params; - params.scale = scale; - params.position_offset = position_offset; - params.window_size = window_size; - params.cached_keys_int8 = cached_keys; - params.cached_values_int8 = cached_values; - params.cached_k_scales = k_scales; - params.cached_v_scales = v_scales; - params.cache_seq_len = cache_len; - params.num_kv_heads = num_kv_heads; - params.head_dim = head_dim; - params.v_head_dim = v_head_dim; - std::vector out_shape; - if (v_head_dim != 0 && v_head_dim != head_dim) { - const auto& q_buf = get_output_buffer(query); - out_shape = {q_buf.shape[0], q_buf.shape[1], q_buf.shape[2], v_head_dim}; - } - return add_node(OpType::ATTENTION_INT8_HYBRID, {query, key_new, value_new}, out_shape, params); -} - -size_t CactusGraph::conv1d_causal(size_t input, size_t weight, size_t, size_t dilation) { - OpParams params{.dilation = dilation}; - return add_node(OpType::CONV1D_CAUSAL, {input, weight}, {}, params); -} - -size_t CactusGraph::conv1d_k3(size_t input, size_t weight, size_t stride) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) throw std::runtime_error("conv1d_k3 expects N,C,L"); - if (w.shape.size() != 3) throw std::runtime_error("weight must be [C_out,C_in,3]"); - if (w.shape[1] != xin.shape[1]) throw std::runtime_error("C_in mismatch in conv1d_k3"); - if (w.shape[2] != 3) throw std::runtime_error("K=3 expected in conv1d_k3"); - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[2]; - const size_t C_out= w.shape[0]; - const size_t K = w.shape[2]; - - const size_t pad = 1; - const size_t L_out = (L + 2 * pad - K) / stride + 1; - - OpParams params{}; - params.stride = stride; - params.output_precision = xin.precision; - - std::vector out_shape{N, C_out, L_out}; - return add_node(OpType::CONV1D_K3, {input, weight}, out_shape, params); -} - -size_t CactusGraph::conv1d_k7s3(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 3) throw std::runtime_error("conv1d_k7s3 expects N,C,L"); - if (w.shape.size() != 3) throw std::runtime_error("weight must be [C_in, 7, C_out]"); - if (w.shape[0] != xin.shape[1]) throw std::runtime_error("C_in mismatch in conv1d_k7s3"); - if (w.shape[1] != 7) throw std::runtime_error("K=7 expected in conv1d_k7s3"); - - size_t C_out = w.shape[2]; - if (b.total_size != C_out) throw std::runtime_error("Bias size mismatch"); - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[2]; - const size_t K = 7; - const size_t stride = 3; - - const size_t L_out = (L < K) ? 0 : (L - K) / stride + 1; - - OpParams params{}; - params.stride = stride; - params.output_precision = xin.precision; - - std::vector out_shape{N, C_out, L_out}; - return add_node(OpType::CONV1D_K7S3, {input, weight, bias}, out_shape, params); -} - -size_t CactusGraph::conv1d(size_t input, size_t weight, size_t stride) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) throw std::runtime_error("conv1d expects N,C,L"); - if (w.shape.size() != 3) throw std::runtime_error("conv1d weight expects [C_out, C_in, K]"); - - size_t N = xin.shape[0]; - size_t C_out = w.shape[0]; - size_t L = xin.shape[2]; - size_t K = w.shape[2]; - size_t L_out = (L - K) / stride + 1; - - OpParams params{.stride = stride}; - return add_node(OpType::CONV1D, {input, weight}, {N, C_out, L_out}, params); -} - -size_t CactusGraph::conv1d(size_t input, size_t weight, size_t bias, size_t stride) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 3) throw std::runtime_error("conv1d expects N,C,L"); - if (w.shape.size() != 3) throw std::runtime_error("conv1d weight expects [C_out, C_in, K]"); - - size_t N = xin.shape[0]; - size_t C_out = w.shape[0]; - size_t L = xin.shape[2]; - size_t K = w.shape[2]; - size_t L_out = (L - K) / stride + 1; - if (b.total_size != C_out) throw std::runtime_error("conv1d bias size mismatch"); - - OpParams params{.output_precision = xin.precision, .stride = stride}; - return add_node(OpType::CONV1D, {input, weight, bias}, {N, C_out, L_out}, params); -} - -size_t CactusGraph::conv1d_same_depthwise_k9(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) { - throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); - } - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[1]; - const size_t C = xin.shape[2]; - - if (w.shape.size() == 2) { - if (w.shape[0] != C || w.shape[1] != 9) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); - } - } else if (w.shape.size() == 3) { - if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 9) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); - } - } else { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV1D_SAME_DEPTHWISE_K9, {input, weight}, {N, L, C}, params); -} - -size_t CactusGraph::conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 3) { - throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); - } - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[1]; - const size_t C = xin.shape[2]; - - if (w.shape.size() == 2) { - if (w.shape[0] != C || w.shape[1] != 9) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); - } - } else if (w.shape.size() == 3) { - if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 9) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); - } - } else { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); - } - - if (b.total_size != C) { - throw std::runtime_error("conv1d_same_depthwise_k9 bias size mismatch"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV1D_SAME_DEPTHWISE_K9, {input, weight, bias}, {N, L, C}, params); -} - -size_t CactusGraph::conv1d_pointwise(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) { - throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); - } - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[1]; - const size_t C_in = xin.shape[2]; - size_t C_out = 0; - - if (w.shape.size() == 2) { - C_out = w.shape[0]; - if (w.shape[1] != C_in) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); - } - } else if (w.shape.size() == 3) { - C_out = w.shape[0]; - if (w.shape[1] != C_in || w.shape[2] != 1) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); - } - } else { - throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV1D_POINTWISE, {input, weight}, {N, L, C_out}, params); -} - -size_t CactusGraph::conv1d_pointwise(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 3) { - throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); - } - - const size_t C_in = xin.shape[2]; - size_t C_out = 0; - - if (w.shape.size() == 2) { - C_out = w.shape[0]; - if (w.shape[1] != C_in) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); - } - } else if (w.shape.size() == 3) { - C_out = w.shape[0]; - if (w.shape[1] != C_in || w.shape[2] != 1) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); - } - } else { - throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); - } - - if (b.total_size != C_out) { - throw std::runtime_error("conv1d_pointwise bias size mismatch"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV1D_POINTWISE, {input, weight, bias}, {xin.shape[0], xin.shape[1], C_out}, params); -} - -size_t CactusGraph::conv2d_k3s2p1(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); - } - if (w.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 weight must be [C_out, C_in, 3, 3]"); - } - - const size_t N = xin.shape[0]; - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - const size_t C_out = w.shape[0]; - - if (w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_k3s2p1 weight must match [C_out, C_in, 3, 3]"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); - } - - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_K3S2P1, {input, weight}, {N, C_out, H_out, W_out}, params); -} - -size_t CactusGraph::conv2d_k3s2p1(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); - } - - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - const size_t C_out = w.shape[0]; - - if (w.shape.size() != 4 || w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_k3s2p1 weight must be [C_out, C_in, 3, 3]"); - } - if (b.total_size != C_out) { - throw std::runtime_error("conv2d_k3s2p1 bias size mismatch"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); - } - - const size_t N = xin.shape[0]; - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_K3S2P1, {input, weight, bias}, {N, C_out, H_out, W_out}, params); -} - -size_t CactusGraph::conv2d_depthwise_k3s2p1(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); - } - - const size_t N = xin.shape[0]; - const size_t C = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - - if (w.shape.size() == 3) { - if (w.shape[0] != C || w.shape[1] != 3 || w.shape[2] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); - } - } else if (w.shape.size() == 4) { - if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); - } - } else { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); - } - - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); - } - - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_DEPTHWISE_K3S2P1, {input, weight}, {N, C, H_out, W_out}, params); -} - -size_t CactusGraph::conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); - } - - const size_t C = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - - if (w.shape.size() == 3) { - if (w.shape[0] != C || w.shape[1] != 3 || w.shape[2] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); - } - } else if (w.shape.size() == 4) { - if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); - } - } else { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); - } - - if (b.total_size != C) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 bias size mismatch"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); - } - - const size_t N = xin.shape[0]; - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_DEPTHWISE_K3S2P1, {input, weight, bias}, {N, C, H_out, W_out}, params); -} - -size_t CactusGraph::conv2d_pointwise_1x1(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); - } - - const size_t N = xin.shape[0]; - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - size_t C_out = 0; - - if (w.shape.size() == 2) { - C_out = w.shape[0]; - if (w.shape[1] != C_in) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); - } - } else if (w.shape.size() == 4) { - C_out = w.shape[0]; - if (w.shape[1] != C_in || w.shape[2] != 1 || w.shape[3] != 1) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); - } - } else { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); - } - - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_POINTWISE_1X1, {input, weight}, {N, C_out, H, W}, params); -} - -size_t CactusGraph::conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); - } - - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - size_t C_out = 0; - - if (w.shape.size() == 2) { - C_out = w.shape[0]; - if (w.shape[1] != C_in) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); - } - } else if (w.shape.size() == 4) { - C_out = w.shape[0]; - if (w.shape[1] != C_in || w.shape[2] != 1 || w.shape[3] != 1) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); - } - } else { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); - } - - if (b.total_size != C_out) { - throw std::runtime_error("conv2d_pointwise_1x1 bias size mismatch"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_POINTWISE_1X1, {input, weight, bias}, {xin.shape[0], C_out, H, W}, params); -} - -size_t CactusGraph::conv2d(size_t input, size_t weight, size_t stride, size_t padding, size_t dilation, size_t groups) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - validate_conv2d_weight_shape(xin, w, groups, "conv2d"); - - const size_t n = xin.shape[0]; - const size_t c_out = w.shape[0]; - const size_t h_out = conv2d_output_dim(xin.shape[2], w.shape[2], stride, padding, dilation); - const size_t w_out = conv2d_output_dim(xin.shape[3], w.shape[3], stride, padding, dilation); - - OpParams params{}; - params.output_precision = xin.precision; - params.stride = stride; - params.padding = padding; - params.dilation = dilation; - params.num_groups = groups; - return add_node(OpType::CONV2D, {input, weight}, {n, c_out, h_out, w_out}, params); -} - -size_t CactusGraph::conv2d(size_t input, size_t weight, size_t bias, size_t stride, size_t padding, size_t dilation, size_t groups) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - validate_conv2d_weight_shape(xin, w, groups, "conv2d"); - - const size_t c_out = w.shape[0]; - if (b.total_size != c_out) { - throw std::runtime_error("conv2d bias size mismatch"); - } - - const size_t n = xin.shape[0]; - const size_t h_out = conv2d_output_dim(xin.shape[2], w.shape[2], stride, padding, dilation); - const size_t w_out = conv2d_output_dim(xin.shape[3], w.shape[3], stride, padding, dilation); - - OpParams params{}; - params.output_precision = xin.precision; - params.stride = stride; - params.padding = padding; - params.dilation = dilation; - params.num_groups = groups; - return add_node(OpType::CONV2D, {input, weight, bias}, {n, c_out, h_out, w_out}, params); -} - -size_t CactusGraph::lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh) { - const auto& h_buffer = get_output_buffer(h_prev); - std::vector output_shape = {h_buffer.shape[0], h_buffer.shape[1], 2}; - return add_node(OpType::LSTM_CELL, {input, h_prev, c_prev, weight_ih, weight_hh, bias_ih, bias_hh}, output_shape, {}); -} - -size_t CactusGraph::gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, - size_t initial_state, float scale) { - const auto& q = get_output_buffer(query); - const auto& k = get_output_buffer(key); - const auto& v = get_output_buffer(value); - const auto& g = get_output_buffer(gate_log); - const auto& b = get_output_buffer(beta); - const auto& s = get_output_buffer(initial_state); - - if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { - throw std::runtime_error("gated_deltanet_decode expects query/key/value rank 4 [B, T, H, D]"); - } - if (g.shape.size() != 3 || b.shape.size() != 3) { - throw std::runtime_error("gated_deltanet_decode expects gate_log/beta rank 3 [B, T, H]"); - } - if (s.shape.size() != 4) { - throw std::runtime_error("gated_deltanet_decode expects initial_state rank 4 [B, K, H, V]"); - } - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - const size_t Hv = v.shape[2]; - const size_t V = v.shape[3]; - if (T != 1) { - throw std::runtime_error("gated_deltanet_decode expects sequence length T=1"); - } - auto is_supported_precision = [](Precision p) { - return p == Precision::FP16 || p == Precision::FP32; - }; - if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || - !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || - !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { - throw std::runtime_error("gated_deltanet_decode requires FP16/FP32 inputs"); - } - - float op_scale = scale; - if (op_scale == 0.0f) { - op_scale = 1.0f / std::sqrt(static_cast(K)); - } - - OpParams params; - params.scale = op_scale; - params.num_kv_heads = Hq; - params.output_precision = Precision::FP16; - return add_node(OpType::GATED_DELTANET_DECODE, - {query, key, value, gate_log, beta, initial_state}, - {B, static_cast(1 + K), Hv, V}, - params); -} - -size_t CactusGraph::gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, - size_t initial_state, size_t chunk_size, float scale) { - const auto& q = get_output_buffer(query); - const auto& k = get_output_buffer(key); - const auto& v = get_output_buffer(value); - const auto& g = get_output_buffer(gate_log); - const auto& b = get_output_buffer(beta); - const auto& s = get_output_buffer(initial_state); - - if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { - throw std::runtime_error("gated_deltanet_prefill expects query/key/value rank 4 [B, T, H, D]"); - } - if (g.shape.size() != 3 || b.shape.size() != 3) { - throw std::runtime_error("gated_deltanet_prefill expects gate_log/beta rank 3 [B, T, H]"); - } - if (s.shape.size() != 4) { - throw std::runtime_error("gated_deltanet_prefill expects initial_state rank 4 [B, K, H, V]"); - } - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - const size_t Hv = v.shape[2]; - const size_t V = v.shape[3]; - auto is_supported_precision = [](Precision p) { - return p == Precision::FP16 || p == Precision::FP32; - }; - if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || - !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || - !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { - throw std::runtime_error("gated_deltanet_prefill requires FP16/FP32 inputs"); - } - - if (chunk_size == 0) { - chunk_size = 64; - } - - float op_scale = scale; - if (op_scale == 0.0f) { - op_scale = 1.0f / std::sqrt(static_cast(K)); - } - - OpParams params; - params.scale = op_scale; - params.chunk_size = chunk_size; - params.num_kv_heads = Hq; - params.output_precision = Precision::FP16; - return add_node(OpType::GATED_DELTANET_PREFILL, - {query, key, value, gate_log, beta, initial_state}, - {B, T + K, Hv, V}, - params); -} - -size_t CactusGraph::stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) throw std::runtime_error("stft expects N,C,L input"); - if (w.shape.size() != 3) throw std::runtime_error("stft weight expects [C_out, C_in, K]"); - - size_t N = xin.shape[0]; - size_t L = xin.shape[2]; - size_t K = w.shape[2]; - size_t L_out = (L - K) / stride + 1; - - OpParams params{}; - params.stride = stride; - params.num_fft_bins = num_fft_bins; - - return add_node(OpType::STFT, {input, weight}, {N, 2 * num_fft_bins, L_out}, params); -} - -size_t CactusGraph::concat(size_t input1, size_t input2, int axis) { - const auto& buffer1 = get_output_buffer(input1); - const auto& buffer2 = get_output_buffer(input2); - - if (buffer1.shape.size() != buffer2.shape.size()) { - throw std::runtime_error("Concat requires inputs with same number of dimensions"); - } - - std::vector output_shape = buffer1.shape; - size_t ndims = output_shape.size(); - - if (axis < 0) axis += ndims; - if (axis < 0 || static_cast(axis) >= ndims) { - throw std::runtime_error("Invalid axis for concat operation"); - } - - for (size_t i = 0; i < ndims; ++i) { - if (i != static_cast(axis) && buffer1.shape[i] != buffer2.shape[i]) { - throw std::runtime_error("Concat inputs must have same shape except on concat axis"); - } - } - - output_shape[axis] = buffer1.shape[axis] + buffer2.shape[axis]; - - OpParams params; - params.axis = axis; - return add_node(OpType::CONCAT, {input1, input2}, output_shape, params); -} - -size_t CactusGraph::cat(const std::vector& inputs, int axis) { - if (inputs.empty()) { - throw std::runtime_error("Cat requires at least one input"); - } - - const auto& first_buffer = get_output_buffer(inputs[0]); - std::vector output_shape = first_buffer.shape; - size_t ndims = output_shape.size(); - - if (axis < 0) axis += ndims; - if (axis < 0 || static_cast(axis) >= ndims) { - throw std::runtime_error("Invalid axis for cat operation"); - } - - for (size_t i = 1; i < inputs.size(); ++i) { - const auto& buffer = get_output_buffer(inputs[i]); - if (buffer.shape.size() != ndims) { - throw std::runtime_error("All inputs to cat must have same number of dimensions"); - } - for (size_t d = 0; d < ndims; ++d) { - if (d != static_cast(axis) && buffer.shape[d] != output_shape[d]) { - throw std::runtime_error("All inputs to cat must have same shape except on cat axis"); - } - } - output_shape[axis] += buffer.shape[axis]; - } - - OpParams params; - params.axis = axis; - return add_node(OpType::CAT, inputs, output_shape, params); -} - -size_t CactusGraph::scatter_topk(size_t indices, size_t values, size_t num_classes) { - const auto& indices_buffer = get_output_buffer(indices); - const auto& values_buffer = get_output_buffer(values); - - if (indices_buffer.shape != values_buffer.shape) { - throw std::runtime_error("ScatterTopK requires indices and values with identical shapes"); - } - if (indices_buffer.shape.size() != 2) { - throw std::runtime_error("ScatterTopK currently supports 2D tensors [batch, top_k]"); - } - if (indices_buffer.precision != Precision::FP32 || values_buffer.precision != Precision::FP32) { - throw std::runtime_error("ScatterTopK expects FP32 indices and values"); - } - - std::vector output_shape = {num_classes, indices_buffer.shape[0]}; - OpParams params{.output_precision = Precision::FP32, .num_classes = num_classes}; - return add_node(OpType::SCATTER_TOPK, {indices, values}, output_shape, params); -} - -size_t CactusGraph::sample(size_t logits, float temperature, float top_p, size_t top_k, - const std::unordered_map& logit_bias) { - return this->sample_with_options(logits, temperature, top_p, 0.15f, 1.1f, top_k, logit_bias); -} - -size_t CactusGraph::sample_with_options(size_t logits, float temperature, float top_p, - float min_p, float repetition_penalty, size_t top_k, - const std::unordered_map& logit_bias) { - const auto& logits_buffer = get_output_buffer(logits); - - if (logits_buffer.shape.empty()) { - throw std::runtime_error("Sample requires non-empty logits tensor"); - } - - OpParams params; - params.temperature = temperature; - params.top_p = top_p; - params.min_p = min_p; - params.repetition_penalty = repetition_penalty; - params.top_k = top_k; - params.random_seed = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - params.output_precision = Precision::FP32; - - if (!logit_bias.empty()) { - params.bias_indices.reserve(logit_bias.size()); - params.bias_values.reserve(logit_bias.size()); - for (const auto& [idx, val] : logit_bias) { - params.bias_indices.push_back(idx); - params.bias_values.push_back(val); - } - } - - std::vector output_shape = {1}; - return add_node(OpType::SAMPLE, {logits}, output_shape, params); -} - -size_t CactusGraph::scalar_add(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_ADD, {input}, {}, params); -} - -size_t CactusGraph::scalar_subtract(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_SUBTRACT, {input}, {}, params); -} - -size_t CactusGraph::scalar_multiply(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_MULTIPLY, {input}, {}, params); -} - -size_t CactusGraph::scalar_divide(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_DIVIDE, {input}, {}, params); -} - -size_t CactusGraph::scalar_floor_divide(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_FLOOR_DIVIDE, {input}, {}, params); -} - -size_t CactusGraph::scalar_not_equal(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = Precision::FP16; - return add_node(OpType::SCALAR_NOT_EQUAL, {input}, {}, params); -} - -size_t CactusGraph::scalar_exp(size_t input) { - return add_node(OpType::SCALAR_EXP, {input}, {}); -} - -size_t CactusGraph::scalar_sqrt(size_t input) { - return add_node(OpType::SCALAR_SQRT, {input}, {}); -} - -size_t CactusGraph::scalar_cos(size_t input) { - return add_node(OpType::SCALAR_COS, {input}, {}); -} - -size_t CactusGraph::scalar_sin(size_t input) { - return add_node(OpType::SCALAR_SIN, {input}, {}); -} - -size_t CactusGraph::scalar_log(size_t input) { - return add_node(OpType::SCALAR_LOG, {input}, {}); -} - -size_t CactusGraph::relu(size_t input) { - return add_node(OpType::RELU, {input}, {}); -} - -size_t CactusGraph::silu(size_t input) { - return add_node(OpType::SILU, {input}, {}); -} - -size_t CactusGraph::gelu(size_t input) { - return add_node(OpType::GELU, {input}, {}); -} - -size_t CactusGraph::gelu_erf(size_t input) { - return add_node(OpType::GELU_ERF, {input}, {}); -} - -size_t CactusGraph::sigmoid(size_t input) { - return add_node(OpType::SIGMOID, {input}, {}); -} - -size_t CactusGraph::tanh(size_t input) { - return add_node(OpType::TANH, {input}, {}); -} - -size_t CactusGraph::glu(size_t input, int axis) { - const auto& xin = get_output_buffer(input); - if (xin.shape.empty()) { - throw std::runtime_error("glu expects non-scalar input"); - } - - int actual_axis = axis; - if (actual_axis < 0) { - actual_axis += static_cast(xin.shape.size()); - } - if (actual_axis < 0 || static_cast(actual_axis) >= xin.shape.size()) { - throw std::runtime_error("glu axis out of range"); - } - - std::vector out_shape = xin.shape; - const size_t axis_size = out_shape[static_cast(actual_axis)]; - if ((axis_size % 2) != 0) { - throw std::runtime_error("glu split dimension must be even"); - } - out_shape[static_cast(actual_axis)] = axis_size / 2; - - OpParams params{}; - params.axis = actual_axis; - params.output_precision = xin.precision; - return add_node(OpType::GLU, {input}, out_shape, params); -} - -size_t CactusGraph::rope_gptj(size_t input, float theta, size_t position_offset, size_t rot_dim, ComputeBackend backend) { - OpParams params; - params.theta = theta; - params.position_offset = position_offset; - params.scalar = static_cast(rot_dim); - params.backend = backend; - return add_node(OpType::ROPE_GPTJ, {input}, {}, params); -} - -size_t CactusGraph::gather(size_t tensor, size_t indices, int axis) { - const auto& tensor_buffer = get_output_buffer(tensor); - const auto& idx_shape = get_output_buffer(indices).shape; - - if (tensor_buffer.shape.empty()) { - throw std::runtime_error("Cannot gather from scalar tensor"); - } - - int actual_axis = axis; - if (actual_axis < 0) { - actual_axis += static_cast(tensor_buffer.shape.size()); - } - if (actual_axis < 0 || actual_axis >= static_cast(tensor_buffer.shape.size())) { - throw std::runtime_error("Gather axis out of range"); - } - - std::vector output_shape; - if (idx_shape.size() == tensor_buffer.shape.size()) { - output_shape = idx_shape; - } else { - if (actual_axis != 0) { - throw std::runtime_error("Lower-rank gather only supports axis 0"); - } - output_shape = idx_shape; - for (size_t i = 1; i < tensor_buffer.shape.size(); i++) { - output_shape.push_back(tensor_buffer.shape[i]); - } - } - - OpParams params; - params.axis = actual_axis; - params.output_precision = tensor_buffer.precision; - - return add_node(OpType::GATHER, {tensor, indices}, output_shape, params); -} - -size_t CactusGraph::slice(size_t input, int axis, size_t start, size_t length) { - const auto& input_buffer = get_output_buffer(input); - if (input_buffer.shape.empty()) { - throw std::runtime_error("Cannot slice a scalar tensor"); - } - - size_t axis_index = static_cast(axis); - size_t axis_size = input_buffer.shape[axis_index]; - - if (start + length > axis_size) { - throw std::runtime_error("Slice range extends beyond axis size"); - } - - std::vector output_shape = input_buffer.shape; - output_shape[axis_index] = length; - - OpParams params; - params.axis = axis_index; - params.slice_start = start; - params.slice_length = length; - params.output_precision = input_buffer.precision; - - return add_node(OpType::SLICE, {input}, output_shape, params); -} - -size_t CactusGraph::embedding(size_t embedding_tensor, size_t indices) { - const auto& emb_buffer = get_output_buffer(embedding_tensor); - const auto& idx_shape = get_output_buffer(indices).shape; - - if (emb_buffer.shape.size() != 2) { - std::cerr << "Error: Embedding tensor " << embedding_tensor << " has invalid shape: ["; - for(auto d : emb_buffer.shape) std::cerr << d << ","; - std::cerr << "]. OpType=" << (int)nodes_[node_index_map_[embedding_tensor]]->op_type - << " ExtData=" << emb_buffer.external_data << std::endl; - throw std::runtime_error("Embedding tensor must be 2D [vocab_size, hidden_dim]"); - } - - std::vector output_shape = idx_shape; - output_shape.push_back(emb_buffer.shape[1]); - - OpParams params; - params.output_precision = Precision::FP16; - return add_node(OpType::EMBEDDING, {embedding_tensor, indices}, output_shape, params); -} - -size_t CactusGraph::bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners) { - const auto& pos_embeds_buffer = get_output_buffer(pos_embeds); - size_t embed_dim = pos_embeds_buffer.shape[1]; - std::vector output_shape = {dst_height * dst_width, embed_dim}; - - OpParams params; - params.dst_height = dst_height; - params.dst_width = dst_width; - params.align_corners = align_corners; - params.output_precision = Precision::FP16; - - return add_node(OpType::BILINEAR_INTERPOLATION, {pos_embeds}, output_shape, params); -} - -size_t CactusGraph::precision_cast(size_t input, Precision target_precision) { - OpParams params{}; - params.output_precision = target_precision; - return add_node(OpType::PRECISION_CAST, {input}, {}, params); -} - -size_t CactusGraph::quantize_activations(size_t input) { - const auto& input_buffer = get_output_buffer(input); - - if (input_buffer.precision != Precision::FP16) { - throw std::invalid_argument("quantize_activations requires FP16 input"); - } - - OpParams params{}; - params.output_precision = Precision::INT8; - return add_node(OpType::QUANTIZE_ACTIVATIONS, {input}, input_buffer.shape, params); -} - -size_t CactusGraph::add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params) { - auto node = std::make_unique(next_node_id_, op_type); - node->input_ids = inputs; - node->params = params; - - std::vector result_shape = output_shape; - if (result_shape.empty() && !inputs.empty()) { - result_shape = nodes_[node_index_map_[inputs[0]]]->output_buffer.shape; - } - - Precision result_precision = params.output_precision; - if (op_type == OpType::PRECISION_CAST) { - result_precision = params.output_precision; - } else if (result_precision == Precision::INT8 && !inputs.empty()) { - result_precision = nodes_[node_index_map_[inputs[0]]]->output_buffer.precision; - } - - node->output_buffer = BufferDesc(result_shape, result_precision); - - size_t node_id = next_node_id_++; - node_index_map_[node_id] = nodes_.size(); - nodes_.push_back(std::move(node)); - - return node_id; -} - -const BufferDesc& CactusGraph::get_output_buffer(size_t node_id) const { - return nodes_[node_index_map_.at(node_id)]->output_buffer; -} - -size_t CactusGraph::persistent(size_t source_node) { - const auto& source_buffer = get_output_buffer(source_node); - OpParams params; - params.output_precision = source_buffer.precision; - size_t node_id = add_node(OpType::PERSISTENT, {source_node}, source_buffer.shape, params); - persistent_node_ids_.insert(node_id); - return node_id; -} - -bool CactusGraph::is_populated(size_t persistent_node_id) const { - return populated_node_ids_.count(persistent_node_id) > 0; -} - -void CactusGraph::invalidate_persistent(size_t persistent_node_id) { - populated_node_ids_.erase(persistent_node_id); - persistent_node_ids_.erase(persistent_node_id); -} - -size_t CactusGraph::altup_predict(size_t coefs, const size_t* streams, size_t num_streams) { - const auto& stream0_buf = get_output_buffer(streams[0]); - - size_t seq_len = stream0_buf.shape[0]; - size_t hidden_dim = stream0_buf.shape[1]; - - std::vector input_ids = {coefs}; - for (size_t i = 0; i < num_streams; i++) - input_ids.push_back(streams[i]); - - OpParams params; - params.num_altup_inputs = num_streams; - return add_node(OpType::ALTUP_PREDICT, input_ids, {num_streams * seq_len, hidden_dim}, params); -} - -size_t CactusGraph::altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions) { - const auto& pred0_buf = get_output_buffer(predictions[0]); - - size_t seq_len = pred0_buf.shape[0]; - size_t hidden_dim = pred0_buf.shape[1]; - - std::vector input_ids = {coefs, innovation}; - for (size_t i = 0; i < num_predictions; i++) - input_ids.push_back(predictions[i]); - - OpParams params; - params.num_altup_inputs = num_predictions; - return add_node(OpType::ALTUP_CORRECT, input_ids, {num_predictions * seq_len, hidden_dim}, params); -} - -size_t CactusGraph::gaussian_topk(size_t input, float ppf) { - const auto& in_buf = get_output_buffer(input); - OpParams params; - params.scalar = ppf; - return add_node(OpType::GAUSSIAN_TOPK, {input}, in_buf.shape, params); -} - -size_t CactusGraph::leaky_relu(size_t input, float negative_slope) { - const auto& in_buf = get_output_buffer(input); - OpParams params; - params.scalar = negative_slope; - return add_node(OpType::LEAKY_RELU, {input}, in_buf.shape, params); -} - -size_t CactusGraph::bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, - size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd) { - const auto& in_buf = get_output_buffer(input); - const auto& w_ih = get_output_buffer(w_ih_fwd); - size_t batch = in_buf.shape[0]; - size_t seq_len = in_buf.shape[1]; - size_t hidden_size = w_ih.shape[0] / 4; - return add_node(OpType::BILSTM_SEQUENCE, - {input, w_ih_fwd, w_hh_fwd, b_ih_fwd, b_hh_fwd, w_ih_bwd, w_hh_bwd, b_ih_bwd, b_hh_bwd}, - {batch, seq_len, 2 * hidden_size}, {}); -} - -size_t CactusGraph::maxpool1d(size_t input, size_t kernel_size, size_t stride) { - const auto& in_buf = get_output_buffer(input); - size_t batch = in_buf.shape[0]; - size_t channels = in_buf.shape[1]; - size_t input_length = in_buf.shape[2]; - size_t output_length = (input_length - kernel_size) / stride + 1; - - OpParams params; - params.kernel_size = kernel_size; - params.stride = stride; - return add_node(OpType::MAXPOOL1D, {input}, {batch, channels, output_length}, params); -} - -size_t CactusGraph::conv2d_k3s1p1(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); - } - if (w.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 weight must be [C_out, C_in, 3, 3]"); - } - - const size_t N = xin.shape[0]; - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - const size_t C_out = w.shape[0]; - - if (w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_k3s1p1 weight must match [C_out, C_in, 3, 3]"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_K3S1P1, {input, weight}, {N, C_out, H, W}, params); -} - -size_t CactusGraph::conv2d_k3s1p1(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); - } - - const size_t N = xin.shape[0]; - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - - if (w.shape.size() != 4 || w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_k3s1p1 weight must be [C_out, C_in, 3, 3]"); - } - const size_t C_out = w.shape[0]; - if (b.total_size != C_out) { - throw std::runtime_error("conv2d_k3s1p1 bias size mismatch"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_K3S1P1, {input, weight, bias}, {N, C_out, H, W}, params); -} - -size_t CactusGraph::stats_pool(size_t input) { - const auto& xin = get_output_buffer(input); - size_t batch = xin.shape[0]; - size_t features = 1; - for (size_t i = 1; i < xin.shape.size() - 1; ++i) features *= xin.shape[i]; - return add_node(OpType::STATS_POOL, {input}, {batch, features * 2}); -} - -size_t CactusGraph::weighted_stats_pool(size_t input, size_t weights) { - const auto& xin = get_output_buffer(input); - size_t batch = xin.shape[0]; - size_t features = 1; - for (size_t i = 1; i < xin.shape.size() - 1; ++i) features *= xin.shape[i]; - return add_node(OpType::WEIGHTED_STATS_POOL, {input, weights}, {batch, features * 2}); -} diff --git a/cactus-transpile/graph/graph_core.cpp b/cactus-transpile/graph/graph_core.cpp deleted file mode 100644 index f418da72b..000000000 --- a/cactus-transpile/graph/graph_core.cpp +++ /dev/null @@ -1,265 +0,0 @@ -#include "graph.h" -#include -#include -#include - -size_t BufferPool::round_up_size(size_t size) const { - constexpr size_t ALIGNMENT = 64; - constexpr size_t MIN_BUCKET = 1024; - - if (size < MIN_BUCKET) return MIN_BUCKET; - - size_t aligned = (size + ALIGNMENT - 1) & ~(ALIGNMENT - 1); - - size_t bucket = MIN_BUCKET; - while (bucket < aligned) { - bucket *= 2; - } - return bucket; -} - -char* BufferPool::acquire(size_t byte_size) { - if (byte_size == 0) return nullptr; - - size_t bucket_size = round_up_size(byte_size); - - auto it = free_buffers_.find(bucket_size); - if (it != free_buffers_.end() && !it->second.empty()) { - auto buffer = std::move(it->second.back()); - it->second.pop_back(); - pool_bytes_ -= bucket_size; - active_bytes_ += bucket_size; - if (active_bytes_ > peak_bytes_) { - peak_bytes_ = active_bytes_; - } - return buffer.release(); - } - - active_bytes_ += bucket_size; - if (active_bytes_ > peak_bytes_) { - peak_bytes_ = active_bytes_; - } - return new char[bucket_size]; -} - -void BufferPool::release(char* ptr, size_t byte_size) { - if (!ptr || byte_size == 0) return; - - size_t bucket_size = round_up_size(byte_size); - active_bytes_ -= bucket_size; - pool_bytes_ += bucket_size; - - free_buffers_[bucket_size].push_back(std::unique_ptr(ptr)); -} - -void BufferPool::clear() { - free_buffers_.clear(); - pool_bytes_ = 0; -} - -BufferDesc::BufferDesc() - : total_size(0), byte_size(0), external_data(nullptr), pooled_data(nullptr), - precision(Precision::FP16) {} - -BufferDesc::BufferDesc(const std::vector& s, Precision prec) - : shape(s), external_data(nullptr), pooled_data(nullptr), precision(prec) { - total_size = 1; - for (size_t dim : shape) total_size *= dim; - byte_size = PrecisionTraits::packed_size_of(prec, total_size); -} - -BufferDesc::~BufferDesc() { - if (pooled_data) { - delete[] pooled_data; - pooled_data = nullptr; - } -} - -BufferDesc::BufferDesc(BufferDesc&& other) noexcept - : shape(std::move(other.shape)), - total_size(other.total_size), - byte_size(other.byte_size), - data(std::move(other.data)), - external_data(other.external_data), - pooled_data(other.pooled_data), - precision(other.precision), - group_size(other.group_size), - num_groups(other.num_groups), - scales_data(other.scales_data), - owned_scales(std::move(other.owned_scales)), - is_interleaved(other.is_interleaved), - original_N(other.original_N), - activation_scales_data(other.activation_scales_data), - owned_activation_scales(std::move(other.owned_activation_scales)), - num_rows_for_activation_scales(other.num_rows_for_activation_scales) { - other.total_size = 0; - other.byte_size = 0; - other.external_data = nullptr; - other.pooled_data = nullptr; - other.group_size = 0; - other.num_groups = 0; - other.scales_data = nullptr; - other.is_interleaved = false; - other.original_N = 0; - other.activation_scales_data = nullptr; - other.num_rows_for_activation_scales = 0; -} - -BufferDesc& BufferDesc::operator=(BufferDesc&& other) noexcept { - if (this != &other) { - if (pooled_data) { - delete[] pooled_data; - } - - shape = std::move(other.shape); - total_size = other.total_size; - byte_size = other.byte_size; - data = std::move(other.data); - external_data = other.external_data; - pooled_data = other.pooled_data; - precision = other.precision; - group_size = other.group_size; - num_groups = other.num_groups; - scales_data = other.scales_data; - owned_scales = std::move(other.owned_scales); - is_interleaved = other.is_interleaved; - original_N = other.original_N; - activation_scales_data = other.activation_scales_data; - owned_activation_scales = std::move(other.owned_activation_scales); - num_rows_for_activation_scales = other.num_rows_for_activation_scales; - - other.total_size = 0; - other.byte_size = 0; - other.external_data = nullptr; - other.pooled_data = nullptr; - other.group_size = 0; - other.num_groups = 0; - other.scales_data = nullptr; - other.is_interleaved = false; - other.original_N = 0; - other.activation_scales_data = nullptr; - other.num_rows_for_activation_scales = 0; - } - return *this; -} - -void* BufferDesc::get_data() { - if (external_data) return external_data; - if (pooled_data) return pooled_data; - return data.get(); -} - -const void* BufferDesc::get_data() const { - if (external_data) return external_data; - if (pooled_data) return pooled_data; - return data.get(); -} - -void BufferDesc::allocate() { - if (!data && !external_data && !pooled_data) { - data = std::make_unique(byte_size); - } -} - -void BufferDesc::allocate_from_pool(BufferPool& pool) { - if (!data && !external_data && !pooled_data && byte_size > 0) { - pooled_data = pool.acquire(byte_size); - } -} - -void BufferDesc::release_to_pool(BufferPool& pool) { - if (pooled_data && byte_size > 0) { - pool.release(pooled_data, byte_size); - pooled_data = nullptr; - } -} - -void BufferDesc::set_external(void* ptr) { - external_data = ptr; - data.reset(); - pooled_data = nullptr; -} - -// GraphNode implementation -GraphNode::GraphNode(size_t node_id, OpType type) : id(node_id), op_type(type) {} - -// TensorConfig implementation -TensorConfig& TensorConfig::global() { - static TensorConfig instance; - return instance; -} - -// BroadcastInfo implementation -BroadcastInfo BroadcastInfo::compute(const std::vector& lhs, const std::vector& rhs) { - BroadcastInfo info; - size_t max_dims = std::max(lhs.size(), rhs.size()); - info.output_shape.resize(max_dims); - - for (size_t i = 0; i < max_dims; ++i) { - size_t lhs_dim = i < lhs.size() ? lhs[lhs.size() - 1 - i] : 1; - size_t rhs_dim = i < rhs.size() ? rhs[rhs.size() - 1 - i] : 1; - - if (lhs_dim != rhs_dim && lhs_dim != 1 && rhs_dim != 1) { - throw std::invalid_argument("Shapes are not compatible for broadcasting"); - } - - info.output_shape[max_dims - 1 - i] = std::max(lhs_dim, rhs_dim); - } - - info.needs_broadcasting = (lhs != info.output_shape || rhs != info.output_shape); - return info; -} - -namespace ValidationUtils { - void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name) { - if (shape.size() != required_dims) { - throw std::runtime_error(op_name + " requires " + std::to_string(required_dims) + - "D tensor, got " + std::to_string(shape.size()) + "D tensor"); - } - } - - void validate_precision(Precision actual, Precision required, const std::string& op_name) { - if (actual != required) { - std::string actual_str = (actual == Precision::INT8) ? "INT8" : "FP32"; - std::string required_str = (required == Precision::INT8) ? "INT8" : "FP32"; - throw std::runtime_error(op_name + " requires " + required_str + " precision, got " + actual_str); - } - } - - void validate_input_count(size_t actual, size_t required, const std::string& op_name) { - if (actual < required) { - throw std::runtime_error(op_name + " requires " + std::to_string(required) + - " inputs, got " + std::to_string(actual) + " inputs"); - } - } -} - -CactusGraph::CactusGraph() : next_node_id_(1) {} - -size_t CactusGraph::get_node_count() const { - return nodes_.size(); -} - -void CactusGraph::register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) { - debug_nodes_.push_back({layer_idx, name, node_id}); -} - -void CactusGraph::capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) { - register_debug_node(layer_idx, name, node_id); -} - -const std::vector& CactusGraph::get_debug_nodes() const { - return debug_nodes_; -} - -void CactusGraph::clear_debug_nodes() { - debug_nodes_.clear(); -} - -void CactusGraph::allocate_buffers() { - for (auto& node : nodes_) { - if (node->op_type != OpType::INPUT) { - node->output_buffer.allocate(); - } - } -} diff --git a/cactus-transpile/graph/graph_execute.cpp b/cactus-transpile/graph/graph_execute.cpp deleted file mode 100644 index 985c43aae..000000000 --- a/cactus-transpile/graph/graph_execute.cpp +++ /dev/null @@ -1,948 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using ComputeFn = void(*)(GraphNode&, const nodes_vector&, const node_index_map_t&); - -#define DECLARE_COMPUTE(name) \ - extern void name(GraphNode&, const nodes_vector&, const node_index_map_t&) - -DECLARE_COMPUTE(compute_binary_op_node); -DECLARE_COMPUTE(compute_unary_op_node); -DECLARE_COMPUTE(compute_activation_node); -DECLARE_COMPUTE(compute_reduce_node); -DECLARE_COMPUTE(compute_cumsum_node); -DECLARE_COMPUTE(compute_reshape_node); -DECLARE_COMPUTE(compute_precision_cast_node); -DECLARE_COMPUTE(compute_matmul_node); -DECLARE_COMPUTE(compute_rms_norm_node); -DECLARE_COMPUTE(compute_rope_node); -DECLARE_COMPUTE(compute_softmax_node); -DECLARE_COMPUTE(compute_attention_node); -DECLARE_COMPUTE(compute_attention_int8_hybrid_node); -DECLARE_COMPUTE(compute_rel_pos_bias_node); -DECLARE_COMPUTE(compute_layernorm_node); -DECLARE_COMPUTE(compute_conv1d_causal_node); -DECLARE_COMPUTE(compute_conv1d_k3_node); -DECLARE_COMPUTE(compute_conv1d_k7s3_node); -DECLARE_COMPUTE(compute_conv1d_node); -DECLARE_COMPUTE(compute_conv1d_same_depthwise_k9_node); -DECLARE_COMPUTE(compute_conv1d_pointwise_node); -DECLARE_COMPUTE(compute_conv2d_k3s2p1_node); -DECLARE_COMPUTE(compute_conv2d_depthwise_k3s2p1_node); -DECLARE_COMPUTE(compute_conv2d_pointwise_1x1_node); -DECLARE_COMPUTE(compute_glu_node); -DECLARE_COMPUTE(compute_batchnorm_node); -DECLARE_COMPUTE(compute_groupnorm_node); -DECLARE_COMPUTE(compute_rope_gptj_node); -DECLARE_COMPUTE(compute_lstm_cell_node); -DECLARE_COMPUTE(compute_gated_deltanet_decode_node); -DECLARE_COMPUTE(compute_gated_deltanet_prefill_node); -DECLARE_COMPUTE(compute_stft_node); -DECLARE_COMPUTE(compute_altup_predict_node); -DECLARE_COMPUTE(compute_altup_correct_node); -DECLARE_COMPUTE(compute_gaussian_topk_node); -DECLARE_COMPUTE(compute_maxpool1d_node); -DECLARE_COMPUTE(compute_bilstm_sequence_node); -DECLARE_COMPUTE(compute_conv2d_k3s1p1_node); -DECLARE_COMPUTE(compute_conv2d_node); -DECLARE_COMPUTE(compute_stats_pool_node); -DECLARE_COMPUTE(compute_transpose_node); -DECLARE_COMPUTE(compute_gather_node); -DECLARE_COMPUTE(compute_slice_node); -DECLARE_COMPUTE(compute_embedding_node); -DECLARE_COMPUTE(compute_concat_node); -DECLARE_COMPUTE(compute_cat_node); -DECLARE_COMPUTE(compute_index_node); -DECLARE_COMPUTE(compute_bilinear_interpolation_node); -DECLARE_COMPUTE(compute_sample_node); -DECLARE_COMPUTE(compute_topk_node); -DECLARE_COMPUTE(compute_scatter_topk_node); -DECLARE_COMPUTE(compute_moe_layer_node); -DECLARE_COMPUTE(compute_persistent_node); -DECLARE_COMPUTE(compute_quantize_activations_node); -extern void shrink_thread_local_buffers(); -#undef DECLARE_COMPUTE - -static const std::unordered_map dispatch_table = { - {OpType::ADD, compute_binary_op_node}, - {OpType::ADD_CLIPPED, compute_binary_op_node}, - {OpType::SUBTRACT, compute_binary_op_node}, - {OpType::MULTIPLY, compute_binary_op_node}, - {OpType::DIVIDE, compute_binary_op_node}, - {OpType::NOT_EQUAL, compute_binary_op_node}, - {OpType::SCALAR_ADD, compute_unary_op_node}, - {OpType::SCALAR_SUBTRACT, compute_unary_op_node}, - {OpType::SCALAR_MULTIPLY, compute_unary_op_node}, - {OpType::SCALAR_DIVIDE, compute_unary_op_node}, - {OpType::SCALAR_NOT_EQUAL, compute_unary_op_node}, - {OpType::SCALAR_FLOOR_DIVIDE, compute_unary_op_node}, - {OpType::SCALAR_EXP, compute_unary_op_node}, - {OpType::SCALAR_SQRT, compute_unary_op_node}, - {OpType::SCALAR_COS, compute_unary_op_node}, - {OpType::SCALAR_SIN, compute_unary_op_node}, - {OpType::SCALAR_LOG, compute_unary_op_node}, - {OpType::ABS, compute_unary_op_node}, - {OpType::POW, compute_unary_op_node}, - {OpType::RELU, compute_activation_node}, - {OpType::SILU, compute_activation_node}, - {OpType::GELU, compute_activation_node}, - {OpType::GELU_ERF, compute_activation_node}, - {OpType::SIGMOID, compute_activation_node}, - {OpType::TANH, compute_activation_node}, - {OpType::LEAKY_RELU, compute_activation_node}, - {OpType::SUM, compute_reduce_node}, - {OpType::MEAN, compute_reduce_node}, - {OpType::VARIANCE, compute_reduce_node}, - {OpType::MIN, compute_reduce_node}, - {OpType::MAX, compute_reduce_node}, - {OpType::CUMSUM, compute_cumsum_node}, - {OpType::FLATTEN, compute_reshape_node}, - {OpType::VIEW, compute_reshape_node}, - {OpType::RESHAPE, compute_reshape_node}, - {OpType::PRECISION_CAST, compute_precision_cast_node}, - {OpType::MATMUL, compute_matmul_node}, - {OpType::RMS_NORM, compute_rms_norm_node}, - {OpType::LAYERNORM, compute_layernorm_node}, - {OpType::GROUPNORM, compute_groupnorm_node}, - {OpType::BATCHNORM, compute_batchnorm_node}, - {OpType::ROPE, compute_rope_node}, - {OpType::ROPE_GPTJ, compute_rope_gptj_node}, - {OpType::SOFTMAX, compute_softmax_node}, - {OpType::ATTENTION, compute_attention_node}, - {OpType::ATTENTION_INT8_HYBRID, compute_attention_int8_hybrid_node}, - {OpType::REL_POS_BIAS, compute_rel_pos_bias_node}, - {OpType::CONV1D_CAUSAL, compute_conv1d_causal_node}, - {OpType::CONV1D_K3, compute_conv1d_k3_node}, - {OpType::CONV1D_K7S3, compute_conv1d_k7s3_node}, - {OpType::CONV1D, compute_conv1d_node}, - {OpType::CONV1D_SAME_DEPTHWISE_K9, compute_conv1d_same_depthwise_k9_node}, - {OpType::CONV1D_POINTWISE, compute_conv1d_pointwise_node}, - {OpType::CONV2D_K3S2P1, compute_conv2d_k3s2p1_node}, - {OpType::CONV2D_DEPTHWISE_K3S2P1, compute_conv2d_depthwise_k3s2p1_node}, - {OpType::CONV2D_POINTWISE_1X1, compute_conv2d_pointwise_1x1_node}, - {OpType::CONV2D_K3S1P1, compute_conv2d_k3s1p1_node}, - {OpType::GLU, compute_glu_node}, - {OpType::TRANSPOSE, compute_transpose_node}, - {OpType::GATHER, compute_gather_node}, - {OpType::SLICE, compute_slice_node}, - {OpType::EMBEDDING, compute_embedding_node}, - {OpType::CONCAT, compute_concat_node}, - {OpType::CAT, compute_cat_node}, - {OpType::INDEX, compute_index_node}, - {OpType::MASKED_SELECT_PREFIX, compute_masked_select_prefix_node}, - {OpType::MASKED_SCATTER, compute_masked_scatter_node}, - {OpType::EXPAND, compute_expand_node}, - {OpType::CONV2D, compute_conv2d_node}, - {OpType::BILINEAR_INTERPOLATION, compute_bilinear_interpolation_node}, - {OpType::SAMPLE, compute_sample_node}, - {OpType::TOPK, compute_topk_node}, - {OpType::SCATTER_TOPK, compute_scatter_topk_node}, - {OpType::MOE_LAYER, compute_moe_layer_node}, - {OpType::PERSISTENT, compute_persistent_node}, - {OpType::QUANTIZE_ACTIVATIONS, compute_quantize_activations_node}, - {OpType::LSTM_CELL, compute_lstm_cell_node}, - {OpType::GATED_DELTANET_DECODE, compute_gated_deltanet_decode_node}, - {OpType::GATED_DELTANET_PREFILL, compute_gated_deltanet_prefill_node}, - {OpType::STFT, compute_stft_node}, - {OpType::ALTUP_PREDICT, compute_altup_predict_node}, - {OpType::ALTUP_CORRECT, compute_altup_correct_node}, - {OpType::GAUSSIAN_TOPK, compute_gaussian_topk_node}, - {OpType::MAXPOOL1D, compute_maxpool1d_node}, - {OpType::BILSTM_SEQUENCE, compute_bilstm_sequence_node}, - {OpType::STATS_POOL, compute_stats_pool_node}, - {OpType::WEIGHTED_STATS_POOL, compute_weighted_stats_pool_node}, -}; - -static const char* get_op_name(OpType op) { - switch (op) { - case OpType::INPUT: return "INPUT"; - case OpType::PRECISION_CAST: return "PRECISION_CAST"; - case OpType::ADD: return "ADD"; - case OpType::ADD_CLIPPED: return "ADD_CLIPPED"; - case OpType::SUBTRACT: return "SUBTRACT"; - case OpType::MULTIPLY: return "MULTIPLY"; - case OpType::DIVIDE: return "DIVIDE"; - case OpType::ABS: return "ABS"; - case OpType::POW: return "POW"; - case OpType::FLATTEN: return "FLATTEN"; - case OpType::VIEW: return "VIEW"; - case OpType::MATMUL: return "MATMUL"; - case OpType::TRANSPOSE: return "TRANSPOSE"; - case OpType::RESHAPE: return "RESHAPE"; - case OpType::SLICE: return "SLICE"; - case OpType::GATHER: return "GATHER"; - case OpType::EMBEDDING: return "EMBEDDING"; - case OpType::BILINEAR_INTERPOLATION: return "BILINEAR_INTERPOLATION"; - case OpType::SUM: return "SUM"; - case OpType::MEAN: return "MEAN"; - case OpType::VARIANCE: return "VARIANCE"; - case OpType::MIN: return "MIN"; - case OpType::MAX: return "MAX"; - case OpType::CUMSUM: return "CUMSUM"; - case OpType::RMS_NORM: return "RMS_NORM"; - case OpType::ROPE: return "ROPE"; - case OpType::ROPE_GPTJ: return "ROPE_GPTJ"; - case OpType::SOFTMAX: return "SOFTMAX"; - case OpType::ATTENTION: return "ATTENTION"; - case OpType::ATTENTION_INT8_HYBRID: return "ATTENTION_INT8_HYBRID"; - case OpType::REL_POS_BIAS: return "REL_POS_BIAS"; - case OpType::CONV1D_CAUSAL: return "CONV1D_CAUSAL"; - case OpType::CONV1D_K3: return "CONV1D_K3"; - case OpType::CONV1D_K7S3: return "CONV1D_K7S3"; - case OpType::CONV1D: return "CONV1D"; - case OpType::CONV1D_SAME_DEPTHWISE_K9: return "CONV1D_SAME_DEPTHWISE_K9"; - case OpType::CONV1D_POINTWISE: return "CONV1D_POINTWISE"; - case OpType::CONV2D_K3S2P1: return "CONV2D_K3S2P1"; - case OpType::CONV2D_DEPTHWISE_K3S2P1: return "CONV2D_DEPTHWISE_K3S2P1"; - case OpType::CONV2D_POINTWISE_1X1: return "CONV2D_POINTWISE_1X1"; - case OpType::GLU: return "GLU"; - case OpType::BATCHNORM: return "BATCHNORM"; - case OpType::SCALAR_ADD: return "SCALAR_ADD"; - case OpType::SCALAR_SUBTRACT: return "SCALAR_SUBTRACT"; - case OpType::SCALAR_MULTIPLY: return "SCALAR_MULTIPLY"; - case OpType::SCALAR_DIVIDE: return "SCALAR_DIVIDE"; - case OpType::SCALAR_EXP: return "SCALAR_EXP"; - case OpType::SCALAR_SQRT: return "SCALAR_SQRT"; - case OpType::SCALAR_COS: return "SCALAR_COS"; - case OpType::SCALAR_SIN: return "SCALAR_SIN"; - case OpType::SCALAR_LOG: return "SCALAR_LOG"; - case OpType::RELU: return "RELU"; - case OpType::SILU: return "SILU"; - case OpType::GELU: return "GELU"; - case OpType::GELU_ERF: return "GELU_ERF"; - case OpType::SIGMOID: return "SIGMOID"; - case OpType::TANH: return "TANH"; - case OpType::SAMPLE: return "SAMPLE"; - case OpType::CONCAT: return "CONCAT"; - case OpType::CAT: return "CAT"; - case OpType::SCATTER_TOPK: return "SCATTER_TOPK"; - case OpType::TOPK: return "TOPK"; - case OpType::LAYERNORM: return "LAYERNORM"; - case OpType::GROUPNORM: return "GROUPNORM"; - case OpType::MOE_LAYER: return "MOE_LAYER"; - case OpType::INDEX: return "INDEX"; - case OpType::PERSISTENT: return "PERSISTENT"; - case OpType::QUANTIZE_ACTIVATIONS: return "QUANTIZE_ACTIVATIONS"; - case OpType::LSTM_CELL: return "LSTM_CELL"; - case OpType::GATED_DELTANET_DECODE: return "GATED_DELTANET_DECODE"; - case OpType::GATED_DELTANET_PREFILL: return "GATED_DELTANET_PREFILL"; - case OpType::STFT: return "STFT"; - case OpType::ALTUP_PREDICT: return "ALTUP_PREDICT"; - case OpType::ALTUP_CORRECT: return "ALTUP_CORRECT"; - case OpType::GAUSSIAN_TOPK: return "GAUSSIAN_TOPK"; - case OpType::MAXPOOL1D: return "MAXPOOL1D"; - case OpType::BILSTM_SEQUENCE: return "BILSTM_SEQUENCE"; - case OpType::LEAKY_RELU: return "LEAKY_RELU"; - case OpType::CONV2D_K3S1P1: return "CONV2D_K3S1P1"; - case OpType::STATS_POOL: return "STATS_POOL"; - case OpType::WEIGHTED_STATS_POOL: return "WEIGHTED_STATS_POOL"; - case OpType::NOT_EQUAL: return "NOT_EQUAL"; - case OpType::SCALAR_NOT_EQUAL: return "SCALAR_NOT_EQUAL"; - case OpType::SCALAR_FLOOR_DIVIDE: return "SCALAR_FLOOR_DIVIDE"; - case OpType::MASKED_SELECT_PREFIX: return "MASKED_SELECT_PREFIX"; - case OpType::MASKED_SCATTER: return "MASKED_SCATTER"; - case OpType::EXPAND: return "EXPAND"; - case OpType::CONV2D: return "CONV2D"; - } - return "UNKNOWN_OP"; -} - -static std::string format_shape(const std::vector& shape) { - std::ostringstream oss; - oss << "["; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - oss << ","; - } - oss << shape[i]; - } - oss << "]"; - return oss.str(); -} - -static std::string format_node_debug(const GraphNode& node, const nodes_vector& nodes, const node_index_map_t& node_index_map) { - std::ostringstream oss; - oss << "node_id=" << node.id - << " op=" << get_op_name(node.op_type) - << " output_shape=" << format_shape(node.output_buffer.shape) - << " output_elems=" << node.output_buffer.total_size; - if (!node.input_ids.empty()) { - oss << " inputs=["; - for (size_t i = 0; i < node.input_ids.size(); ++i) { - if (i > 0) { - oss << "; "; - } - const size_t input_id = node.input_ids[i]; - oss << input_id; - auto it = node_index_map.find(input_id); - if (it != node_index_map.end()) { - const auto& input_node = nodes[it->second]; - oss << ":" << get_op_name(input_node->op_type) - << ":" << format_shape(input_node->output_buffer.shape) - << ":" << input_node->output_buffer.total_size; - } - } - oss << "]"; - } - return oss.str(); -} - -void compute_node_optimized(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& node_index_map) { - if (node.op_type == OpType::INPUT) return; - - auto it = dispatch_table.find(node.op_type); - if (it != dispatch_table.end()) { - it->second(node, nodes, node_index_map); - } else { - throw std::runtime_error("Unknown operation type: " + std::to_string(static_cast(node.op_type))); - } -} - -void CactusGraph::set_input(size_t node_id, const void* data, Precision) { - auto it = node_index_map_.find(node_id); - if (it == node_index_map_.end()) { - throw std::out_of_range("Unknown input node id: " + std::to_string(node_id)); - } - - auto& node = *nodes_[it->second]; - if (node.op_type != OpType::INPUT) { - throw std::invalid_argument("Can only set data on input nodes"); - } - - if (!node.output_buffer.data && !node.output_buffer.external_data) { - node.output_buffer.allocate(); - } - - std::memcpy(node.output_buffer.get_data(), data, node.output_buffer.byte_size); -} - -void CactusGraph::set_external_input(size_t node_id, void* data, Precision) { - auto it = node_index_map_.find(node_id); - if (it == node_index_map_.end()) { - throw std::out_of_range("Unknown input node id: " + std::to_string(node_id)); - } - - auto& node = *nodes_[it->second]; - if (node.op_type != OpType::INPUT) { - throw std::invalid_argument("Can only set data on input nodes"); - } - - node.output_buffer.set_external(data); -} - -void* CactusGraph::get_output(size_t node_id) { - auto it = node_index_map_.find(node_id); - if (it == node_index_map_.end()) { - throw std::out_of_range("Unknown output node id: " + std::to_string(node_id)); - } - - auto& buffer = nodes_[it->second]->output_buffer; - if (!buffer.get_data()) { - buffer.allocate(); - } - return buffer.get_data(); -} - -void CactusGraph::execute(const std::string& profile_file) { - std::vector last_use(nodes_.size(), 0); - for (size_t i = 0; i < nodes_.size(); ++i) { - for (size_t input_id : nodes_[i]->input_ids) { - auto it = node_index_map_.find(input_id); - if (it != node_index_map_.end()) { - last_use[it->second] = std::max(last_use[it->second], i); - } - } - } - - BufferPool& pool = buffer_pool_; - - auto get_env_int = [](const char* name, int fallback) -> int { - const char* val = std::getenv(name); - return val ? std::atoi(val) : fallback; - }; - - auto get_env_str = [](const char* name) -> std::string { - const char* val = std::getenv(name); - return val ? std::string(val) : std::string(); - }; - - bool capture_to_stdout = get_env_int("CACTUS_CAPTURE_STDOUT", 0) != 0; - std::string capture_file_path = get_env_str("CACTUS_CAPTURE_FILE"); - bool capture_requested = get_env_int("CACTUS_CAPTURE_ENABLE", 0) != 0; - std::string capture_dir = get_env_str("CACTUS_CAPTURE_DIR"); - bool trace_nodes = get_env_int("CACTUS_TRACE_NODES", 0) != 0; - - if (!capture_requested) { - capture_requested = capture_to_stdout || !capture_file_path.empty() || !capture_dir.empty(); - } else if (capture_file_path.empty() && !capture_to_stdout && capture_dir.empty()) { - capture_to_stdout = true; - } - - size_t capture_preview_count = static_cast(get_env_int("CACTUS_CAPTURE_PREVIEW_COUNT", 8)); - size_t capture_max_elements = static_cast(get_env_int("CACTUS_CAPTURE_MAX_ELEMENTS", 65536)); - - std::string env_profile = get_env_str("CACTUS_PROFILE_FILE"); - if (env_profile.empty()) env_profile = get_env_str("CACTUS_PROFILE"); - - std::string target_profile = profile_file; - if (target_profile.empty() && !env_profile.empty()) { - target_profile = env_profile; - } - - bool enable_profiling = !target_profile.empty(); - bool to_stdout = (target_profile == "stdout" || target_profile == "-"); - - std::ofstream profile_out; - std::ostream* out = &std::cout; - - if (enable_profiling && !to_stdout) { - profile_out.open(target_profile, std::ios::app); - if (profile_out.is_open()) { - out = &profile_out; - } - } - - auto total_start = std::chrono::high_resolution_clock::now(); - - if (enable_profiling) { - *out << "=== Graph Execution Profile ===" << std::endl; - *out << std::left << std::setw(24) << "Operation" - << std::setw(12) << "Time (ms)" - << std::setw(20) << "Output Shape" - << "Backend" << std::endl; - *out << std::string(72, '-') << std::endl; - } - - for (size_t node_idx = 0; node_idx < nodes_.size(); ++node_idx) { - auto& node = nodes_[node_idx]; - - if (node->op_type != OpType::INPUT) { - node->output_buffer.allocate_from_pool(pool); - } - - if (trace_nodes) { - std::cerr << "[graph:node_begin] idx=" << node_idx - << " " << format_node_debug(*node, nodes_, node_index_map_) - << std::endl; - } - - if (enable_profiling && node->op_type != OpType::INPUT) { - auto start = std::chrono::high_resolution_clock::now(); - - try { - compute_node_optimized(*node, nodes_, node_index_map_); - } catch (const std::exception& e) { - throw std::runtime_error( - std::string(e.what()) + " | " + format_node_debug(*node, nodes_, node_index_map_) - ); - } - - if (node->op_type == OpType::PERSISTENT) { - populated_node_ids_.insert(node->id); - } - - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - double ms = duration.count() / 1000.0; - - std::string shape_str = "["; - for (size_t i = 0; i < node->output_buffer.shape.size(); ++i) { - if (i > 0) shape_str += ","; - shape_str += std::to_string(node->output_buffer.shape[i]); - } - shape_str += "]"; - - std::string values_str = ""; - if (node->output_buffer.get_data()) { - size_t num_values = std::min(size_t(5), node->output_buffer.total_size); - values_str = " values=["; - - if (node->output_buffer.precision == Precision::FP32) { - if (node->op_type == OpType::SAMPLE) { - uint32_t* uint32_data = reinterpret_cast(node->output_buffer.get_data()); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) values_str += ","; - values_str += std::to_string(uint32_data[i]); - } - } else { - float* float_data = reinterpret_cast(node->output_buffer.get_data()); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) values_str += ","; - values_str += std::to_string(float_data[i]).substr(0, 6); - } - } - } else if (node->output_buffer.precision == Precision::FP16) { - __fp16* fp16_data = reinterpret_cast<__fp16*>(node->output_buffer.get_data()); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) values_str += ","; - values_str += std::to_string(static_cast(fp16_data[i])).substr(0, 6); - } - } else if (node->output_buffer.precision == Precision::INT8) { - int8_t* int8_data = reinterpret_cast(node->output_buffer.get_data()); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) values_str += ","; - values_str += std::to_string(static_cast(int8_data[i])); - } - } - - if (node->output_buffer.total_size > 5) { - values_str += ",..."; - } - values_str += "]"; - } - - std::string weights_str = ""; - if ((node->op_type == OpType::RMS_NORM || node->op_type == OpType::MATMUL || - node->op_type == OpType::GATHER || node->op_type == OpType::EMBEDDING || - node->op_type == OpType::ATTENTION || node->op_type == OpType::CONCAT) && - node->input_ids.size() >= 2) { - const auto& weight_node = nodes_[node_index_map_.at(node->input_ids[1])]; - if (weight_node->output_buffer.get_data()) { - size_t num_values = std::min(size_t(5), weight_node->output_buffer.total_size); - weights_str = " weights=["; - - if (weight_node->output_buffer.precision == Precision::FP32) { - const float* float_data = weight_node->output_buffer.data_as(); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) weights_str += ","; - weights_str += std::to_string(float_data[i]).substr(0, 6); - } - } else if (weight_node->output_buffer.precision == Precision::FP16) { - const __fp16* fp16_data = weight_node->output_buffer.data_as<__fp16>(); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) weights_str += ","; - weights_str += std::to_string(static_cast(fp16_data[i])).substr(0, 6); - } - } else if (weight_node->output_buffer.precision == Precision::INT8) { - const int8_t* int8_data = weight_node->output_buffer.data_as(); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) weights_str += ","; - weights_str += std::to_string(static_cast(int8_data[i])); - } - } else if (weight_node->output_buffer.precision == Precision::INT4) { - const uint8_t* packed = weight_node->output_buffer.data_as(); - int8x16_t high, low; - unpack_int4_as_int8x16x2(packed, high, low); - int8_t low_lanes[16], high_lanes[16]; - vst1q_s8(low_lanes, low); - vst1q_s8(high_lanes, high); - - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) weights_str += ","; - int8_t val = (i < 16) ? low_lanes[i] : high_lanes[i - 16]; - weights_str += std::to_string(static_cast(val)); - } - } - - if (weight_node->output_buffer.total_size > 5) { - weights_str += ",..."; - } - weights_str += "]"; - } - } - - *out << std::left << std::setw(24) << get_op_name(node->op_type) - << std::setw(12) << std::fixed << std::setprecision(3) << ms - << std::setw(20) << shape_str - << values_str << weights_str << std::endl; - } else { - try { - compute_node_optimized(*node, nodes_, node_index_map_); - } catch (const std::exception& e) { - throw std::runtime_error( - std::string(e.what()) + " | " + format_node_debug(*node, nodes_, node_index_map_) - ); - } - - if (node->op_type == OpType::PERSISTENT) { - populated_node_ids_.insert(node->id); - } - } - - if (trace_nodes) { - std::cerr << "[graph:node_end] idx=" << node_idx - << " node_id=" << node->id - << " op=" << get_op_name(node->op_type) - << std::endl; - } - } - - if (trace_nodes) { - std::cerr << "[graph:after_node_loop]" << std::endl; - } - - std::unique_ptr capture_file_stream; - std::vector capture_outputs; - - if (trace_nodes) { - std::cerr << "[graph:after_capture_decl]" << std::endl; - } - - if (capture_requested) { - if (capture_to_stdout) { - capture_outputs.push_back(&std::cout); - } - - if (!capture_file_path.empty()) { - std::filesystem::path capture_path(capture_file_path); - if (capture_path.has_parent_path()) { - std::error_code ec; - std::filesystem::create_directories(capture_path.parent_path(), ec); - } - - auto stream_ptr = std::make_unique(capture_path, std::ios::out | std::ios::app); - if (stream_ptr->is_open()) { - capture_outputs.push_back(stream_ptr.get()); - capture_file_stream = std::move(stream_ptr); - } else { - std::cerr << "Failed to open capture file: " << capture_path << std::endl; - } - } - - if (!capture_dir.empty()) { - std::filesystem::path dir_path(capture_dir); - std::error_code ec; - std::filesystem::create_directories(dir_path, ec); - } - - if (capture_outputs.empty() && capture_dir.empty()) { - capture_requested = false; - } - } - - if (capture_requested) { - auto precision_to_string = [](Precision p) -> const char* { - switch (p) { - case Precision::FP32: return "FP32"; - case Precision::FP16: return "FP16"; - case Precision::INT8: return "INT8"; - default: return "UNKNOWN"; - } - }; - - auto format_double = [](double value) { - std::ostringstream oss; - oss << std::fixed << std::setprecision(6) << value; - return oss.str(); - }; - - auto now = std::chrono::system_clock::now(); - std::time_t now_time = std::chrono::system_clock::to_time_t(now); - std::tm time_info{}; -#if defined(_WIN32) - localtime_s(&time_info, &now_time); -#else - localtime_r(&now_time, &time_info); -#endif - - auto write_header = [&](std::ostream& stream) { - stream << "=== Graph Debug Capture ===" << std::endl; - stream << "Timestamp: " << std::put_time(&time_info, "%Y-%m-%d %H:%M:%S") << std::endl; - stream << "Captured nodes: " << debug_nodes_.size() << std::endl; - stream << std::string(60, '-') << std::endl; - }; - - auto write_separator = [](std::ostream& stream) { - stream << std::string(60, '-') << std::endl; - }; - - if (debug_nodes_.empty()) { - for (auto* stream : capture_outputs) { - write_header(*stream); - *stream << "No debug nodes registered on this graph." << std::endl; - write_separator(*stream); - stream->flush(); - } - } else { - for (auto* stream : capture_outputs) { - write_header(*stream); - } - - for (const auto& entry : debug_nodes_) { - auto node_it = node_index_map_.find(entry.node_id); - const GraphNode* node_ptr = nullptr; - if (node_it != node_index_map_.end()) { - node_ptr = nodes_[node_it->second].get(); - } - - if (!node_ptr) { - for (auto* stream : capture_outputs) { - *stream << "Layer " << entry.layer_idx << " - " << entry.name - << " (node " << entry.node_id << ")" << std::endl; - *stream << " Data: " << std::endl; - write_separator(*stream); - } - continue; - } - - const BufferDesc& buffer = node_ptr->output_buffer; - const void* data_ptr = buffer.get_data(); - size_t total_size = buffer.total_size; - - std::ostringstream shape_ss; - shape_ss << "["; - for (size_t i = 0; i < buffer.shape.size(); ++i) { - if (i > 0) { - shape_ss << ","; - } - shape_ss << buffer.shape[i]; - } - shape_ss << "]"; - std::string shape_str = shape_ss.str(); - - bool has_data = data_ptr != nullptr && total_size > 0; - size_t elements_to_process = total_size; - bool truncated = false; - if (has_data && elements_to_process > capture_max_elements && capture_max_elements > 0) { - elements_to_process = capture_max_elements; - truncated = true; - } - - std::vector preview_values; - if (capture_preview_count > 0) { - preview_values.reserve(std::min(capture_preview_count, elements_to_process)); - } - - double min_val = std::numeric_limits::infinity(); - double max_val = -std::numeric_limits::infinity(); - long double sum = 0.0L; - long double sum_sq = 0.0L; - - if (has_data && elements_to_process > 0) { - auto accumulate = [&](float value, size_t index) { - double v = static_cast(value); - min_val = std::min(min_val, v); - max_val = std::max(max_val, v); - sum += static_cast(value); - sum_sq += static_cast(value) * static_cast(value); - if (capture_preview_count > 0 && index < capture_preview_count) { - preview_values.push_back(value); - } - }; - - if (buffer.precision == Precision::FP32) { - const float* typed = static_cast(data_ptr); - for (size_t i = 0; i < elements_to_process; ++i) { - accumulate(typed[i], i); - } - } else if (buffer.precision == Precision::FP16) { - const __fp16* typed = reinterpret_cast(data_ptr); - for (size_t i = 0; i < elements_to_process; ++i) { - accumulate(static_cast(typed[i]), i); - } - } else if (buffer.precision == Precision::INT8) { - const int8_t* typed = reinterpret_cast(data_ptr); - for (size_t i = 0; i < elements_to_process; ++i) { - accumulate(static_cast(typed[i]), i); - } - } else if (buffer.precision == Precision::INT4) { - assert(elements_to_process % 32 == 0 && "INT4 precision capture requires element count to be multiple of 32"); - const uint8_t* packed_ptr = reinterpret_cast(data_ptr); - for (size_t i = 0; i < elements_to_process; i+=32) { - int8x16_t high, low; - unpack_int4_as_int8x16x2(packed_ptr + i / 2, high, low); - int8_t high_lanes[16], low_lanes[16]; - vst1q_s8(high_lanes, high); - vst1q_s8(low_lanes, low); - - for (size_t j = 0; j < 16; ++j) { - accumulate(static_cast(low_lanes[j]), i + j); - } - for (size_t j = 0; j < 16; ++j) { - accumulate(static_cast(high_lanes[j]), i + 16 + j); - } - } - } else { - has_data = false; - } - } else { - has_data = false; - } - - if (!capture_dir.empty() && has_data) { - std::string safe_name = entry.name; - std::string filename = capture_dir + "/" + safe_name + ".bin"; - std::ofstream bin_file(filename, std::ios::binary); - if (bin_file.is_open()) { - size_t bytes_to_write = buffer.byte_size; - if (truncated) { - bytes_to_write = PrecisionTraits::packed_size_of(buffer.precision, elements_to_process); - } - bin_file.write(reinterpret_cast(data_ptr), bytes_to_write); - } - } - - size_t processed_count = has_data ? elements_to_process : 0; - long double mean_ld = processed_count > 0 ? sum / processed_count : 0.0L; - long double variance_ld = processed_count > 0 ? (sum_sq / processed_count) - (mean_ld * mean_ld) : 0.0L; - if (variance_ld < 0.0L) { - variance_ld = 0.0L; - } - double mean_val = static_cast(mean_ld); - double stddev_val = processed_count > 0 ? std::sqrt(static_cast(variance_ld)) : 0.0; - - std::ostringstream preview_ss; - if (capture_preview_count > 0 && !preview_values.empty()) { - preview_ss << "["; - for (size_t i = 0; i < preview_values.size(); ++i) { - if (i > 0) { - preview_ss << ", "; - } - preview_ss << format_double(static_cast(preview_values[i])); - } - if (processed_count > preview_values.size()) { - if (!preview_values.empty()) { - preview_ss << ", ..."; - } else { - preview_ss << "..."; - } - } - preview_ss << "]"; - } - - for (auto* stream : capture_outputs) { - *stream << "Layer " << entry.layer_idx << " - " << entry.name - << " (node " << entry.node_id << ")" << std::endl; - *stream << " Shape: " << shape_str << " Precision: " << precision_to_string(buffer.precision) << std::endl; - if (!has_data) { - *stream << " Data: " << std::endl; - } else { - *stream << " Stats: min=" << format_double(min_val) - << " max=" << format_double(max_val) - << " mean=" << format_double(mean_val) - << " std=" << format_double(stddev_val) << std::endl; - if (truncated || processed_count < total_size) { - *stream << " Note: stats computed on first " << processed_count - << " of " << total_size << " values" << std::endl; - } - if (capture_preview_count > 0 && !preview_values.empty()) { - *stream << " Preview: " << preview_ss.str() << std::endl; - } - } - write_separator(*stream); - } - } - - for (auto* stream : capture_outputs) { - stream->flush(); - } - } - } - - if (enable_profiling) { - auto total_end = std::chrono::high_resolution_clock::now(); - auto total_duration = std::chrono::duration_cast(total_end - total_start); - double total_ms = total_duration.count() / 1000.0; - - *out << std::string(72, '-') << std::endl; - *out << "Total execution time: " << std::fixed << std::setprecision(3) << total_ms << " ms" << std::endl; - *out << "================================" << std::endl; - - if (profile_out.is_open()) { - profile_out.close(); - } - } - - if (trace_nodes) { - std::cerr << "[graph:execute_end]" << std::endl; - } -} - -void CactusGraph::hard_reset() { - nodes_.clear(); - node_index_map_.clear(); - mapped_files_.clear(); - weight_cache_.clear(); - next_node_id_ = 1; - debug_nodes_.clear(); - buffer_pool_.clear(); -} - -void CactusGraph::soft_reset() { - std::set cached_node_ids; - for (const auto& cache_entry : weight_cache_) { - cached_node_ids.insert(cache_entry.second); - } - - for (size_t pid : persistent_node_ids_) { - cached_node_ids.insert(pid); - } - - size_t max_preserved_id = 0; - for (const auto& node : nodes_) { - if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || - cached_node_ids.count(node->id)) { - max_preserved_id = std::max(max_preserved_id, node->id); - } - } - - auto preserved_nodes = std::move(nodes_); - auto preserved_index_map = std::move(node_index_map_); - - nodes_.clear(); - node_index_map_.clear(); - - for (auto& node : preserved_nodes) { - if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || - cached_node_ids.count(node->id)) { - size_t index = nodes_.size(); - node_index_map_[node->id] = index; - nodes_.push_back(std::move(node)); - } - } - - next_node_id_ = max_preserved_id + 1; - debug_nodes_.clear(); - if (!prefill_mode_) { - buffer_pool_.clear(); - shrink_thread_local_buffers(); - } -} - -void CactusGraph::soft_reset_keep_pool() { - std::set cached_node_ids; - for (const auto& cache_entry : weight_cache_) { - cached_node_ids.insert(cache_entry.second); - } - - for (size_t pid : persistent_node_ids_) { - cached_node_ids.insert(pid); - } - - size_t max_preserved_id = 0; - for (const auto& node : nodes_) { - if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || - cached_node_ids.count(node->id)) { - max_preserved_id = std::max(max_preserved_id, node->id); - } - } - - auto preserved_nodes = std::move(nodes_); - - nodes_.clear(); - node_index_map_.clear(); - - for (auto& node : preserved_nodes) { - if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || - cached_node_ids.count(node->id)) { - size_t index = nodes_.size(); - node_index_map_[node->id] = index; - nodes_.push_back(std::move(node)); - } - } - - next_node_id_ = max_preserved_id + 1; - debug_nodes_.clear(); -} diff --git a/cactus-transpile/graph/graph_io.cpp b/cactus-transpile/graph/graph_io.cpp deleted file mode 100644 index 418258c3c..000000000 --- a/cactus-transpile/graph/graph_io.cpp +++ /dev/null @@ -1,809 +0,0 @@ -#include "graph.h" -#include "graph_param_io.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { - constexpr uint32_t fourcc(char a, char b, char c, char d) { - return static_cast(static_cast(a)) | - (static_cast(static_cast(b)) << 8) | - (static_cast(static_cast(c)) << 16) | - (static_cast(static_cast(d)) << 24); - } - - constexpr uint32_t CACTUS_MAGIC = 0x54434143; - constexpr uint32_t CACTUS_GRAPH_MAGIC = fourcc('C', 'G', 'R', 'F'); - constexpr uint32_t FLAG_HAS_SCALES = 1 << 0; - constexpr uint32_t FLAG_INTERLEAVED = 1 << 3; - constexpr size_t HEADER_SIZE = 84; - - inline size_t align_offset(size_t offset, size_t alignment) { - size_t remainder = offset % alignment; - if (remainder == 0) return offset; - return offset + (alignment - remainder); - } - - inline void write_u32(std::ostream& out, uint32_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); - } - - inline void write_u64(std::ostream& out, uint64_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); - } - - void write_size_vector(std::ostream& out, const std::vector& values) { - uint32_t size = static_cast(values.size()); - write_u32(out, size); - for (size_t v : values) { - write_u64(out, static_cast(v)); - } - } - - void write_u32_vector(std::ostream& out, const std::vector& values) { - uint32_t size = static_cast(values.size()); - write_u32(out, size); - for (uint32_t v : values) { - write_u32(out, v); - } - } - - std::vector compute_leaf_outputs(const GraphFile::SerializedGraph& sg) { - std::vector referenced(sg.nodes.size(), false); - - for (const auto& node : sg.nodes) { - for (uint32_t input_idx : node.inputs) { - if (input_idx >= referenced.size()) { - throw std::runtime_error("Graph save failed: input index out of range"); - } - referenced[input_idx] = true; - } - } - - std::vector outputs; - outputs.reserve(sg.nodes.size()); - - for (uint32_t i = 0; i < sg.nodes.size(); ++i) { - if (!referenced[i]) { - outputs.push_back(i); - } - } - - return outputs; - } - - void write_serialized_graph(std::ostream& out, const GraphFile::SerializedGraph& sg) { - write_u32(out, sg.header.magic); - write_u32(out, sg.header.version); - write_u32(out, sg.header.node_count); - write_u32(out, sg.header.flags); - - write_u32_vector(out, sg.graph_inputs); - write_u32_vector(out, sg.graph_outputs); - - for (const auto& node : sg.nodes) { - write_u32(out, node.index); - write_u32(out, static_cast(node.op_type)); - write_u32_vector(out, node.inputs); - write_size_vector(out, node.output_shape); - write_u32(out, static_cast(node.precision)); - GraphParamIO::write_op_params(out, node.op_type, node.params); - } - - if (!out) { - throw std::runtime_error("Error writing serialized graph"); - } - } - - // read helpers - uint32_t read_u32(std::istream& in) { - uint32_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) { - throw std::runtime_error("Unexpected EOF while reading uint32"); - } - return v; - } - - uint64_t read_u64(std::istream& in) { - uint64_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) { - throw std::runtime_error("Unexpected EOF while reading uint64"); - } - return v; - } - - std::vector read_u32_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) { - values.push_back(read_u32(in)); - } - return values; - } - - std::vector read_size_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) { - values.push_back(static_cast(read_u64(in))); - } - return values; - } - - bool is_binary_broadcast_op(OpType op_type) { - switch (op_type) { - case OpType::ADD: - case OpType::ADD_CLIPPED: - case OpType::SUBTRACT: - case OpType::MULTIPLY: - case OpType::DIVIDE: - case OpType::NOT_EQUAL: - return true; - default: - return false; - } - } - - void populate_derived_params(CactusGraph& graph, - const GraphFile::NodeEntry& node_entry, - const std::vector& runtime_inputs, - OpParams& params) { - if (is_binary_broadcast_op(node_entry.op_type)) { - if (runtime_inputs.size() != 2) { - throw std::runtime_error("Graph file corrupted: binary op missing inputs"); - } - const auto& lhs = graph.get_output_buffer(runtime_inputs[0]); - const auto& rhs = graph.get_output_buffer(runtime_inputs[1]); - params.broadcast_info = BroadcastInfo::compute(lhs.shape, rhs.shape); - } - } - - GraphFile::GraphHeader read_graph_header(std::istream& in) { - GraphFile::GraphHeader header; - header.magic = read_u32(in); - header.version = read_u32(in); - header.node_count = read_u32(in); - header.flags = read_u32(in); - - if (header.magic != CACTUS_GRAPH_MAGIC) { - throw std::runtime_error("Invalid graph file: bad magic"); - } - if (header.version != 4) { - throw std::runtime_error("Unsupported graph file version: " + - std::to_string(header.version)); - } - - return header; - } - - GraphFile::NodeEntry read_node_entry(std::istream& in) { - GraphFile::NodeEntry node; - node.index = read_u32(in); - uint32_t op_type_val = read_u32(in); - if (op_type_val > static_cast(OpType::CUMSUM)) { - throw std::runtime_error("Graph file corrupted: invalid op type"); - } - node.op_type = static_cast(op_type_val); - node.inputs = read_u32_vector(in); - node.output_shape = read_size_vector(in); - uint32_t precision_val = read_u32(in); - if (precision_val > static_cast(Precision::INT4)) { - throw std::runtime_error("Graph file corrupted: invalid precision"); - } - node.precision = static_cast(precision_val); - GraphParamIO::read_op_params(in, node.op_type, node.params); - return node; - } - - -} // namespace - -void CactusGraph::save(const std::string& path) { - GraphFile::save_graph(*this, path); -} - -CactusGraph CactusGraph::from_serialized(const GraphFile::SerializedGraph& sg) { - CactusGraph graph; - std::vector runtime_ids; - runtime_ids.reserve(sg.nodes.size()); - - if (sg.nodes.size() != sg.header.node_count) { - throw std::runtime_error("Graph file corrupted: node count mismatch"); - } - - for (size_t i = 0; i < sg.nodes.size(); ++i) { - const auto& node_entry = sg.nodes[i]; - - if (node_entry.index != i) { - throw std::runtime_error("Graph file corrupted: node indices must be dense and ordered"); - } - std::vector runtime_inputs; - runtime_inputs.reserve(node_entry.inputs.size()); - - for (uint32_t serialized_input_idx : node_entry.inputs) { - if (serialized_input_idx >= runtime_ids.size()) { - throw std::runtime_error( - "Graph file corrupted: input refers to a node that has not been reconstructed yet" - ); - } - runtime_inputs.push_back(runtime_ids[serialized_input_idx]); - } - - size_t new_node_id = 0; - - if (node_entry.op_type == OpType::INPUT) { - new_node_id = graph.input(node_entry.output_shape, node_entry.precision); - } - else { - OpParams params = node_entry.params; - params.output_precision = node_entry.precision; - populate_derived_params(graph, node_entry, runtime_inputs, params); - new_node_id = graph.add_node(node_entry.op_type, runtime_inputs, node_entry.output_shape, params); - - if (node_entry.op_type == OpType::PERSISTENT) { - graph.persistent_node_ids_.insert(new_node_id); - } - } - runtime_ids.push_back(new_node_id); - } - return graph; -} - -CactusGraph CactusGraph::load(const std::string& path) { - GraphFile::SerializedGraph sg = GraphFile::load_graph(path); - return from_serialized(sg); -} - -size_t CactusGraph::mmap_embeddings(const std::string& filename) { - auto mapped_file = std::make_unique(filename); - - const auto& shape = mapped_file->shape(); - if (shape.size() != 2) { - throw std::runtime_error("Memory-mapped embeddings must be 2D [vocab_size, embedding_dim]"); - } - - Precision precision = mapped_file->precision(); - - size_t node_id = input(shape, precision); - set_external_input(node_id, const_cast(mapped_file->data()), precision); - - if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { - set_grouped_scales(node_id, mapped_file->group_size(), mapped_file->num_groups(), - const_cast(mapped_file->scales_data())); - - if (mapped_file->is_interleaved()) { - auto& buffer = nodes_[node_index_map_.at(node_id)]->output_buffer; - buffer.set_interleaved(true, mapped_file->original_N()); - } - } - - size_t file_idx = mapped_files_.size(); - mapped_files_.push_back(std::move(mapped_file)); - node_to_mapped_file_[node_id] = file_idx; - weight_cache_[filename] = node_id; - return node_id; -} - -size_t CactusGraph::mmap_weights(const std::string& filename) { - auto it = weight_cache_.find(filename); - if (it != weight_cache_.end()) { - return it->second; - } - - auto mapped_file = std::make_unique(filename); - - const auto& shape = mapped_file->shape(); - Precision precision = mapped_file->precision(); - - size_t node_id = input(shape, precision); - set_external_input(node_id, const_cast(mapped_file->data()), precision); - - if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { - set_grouped_scales(node_id, mapped_file->group_size(), mapped_file->num_groups(), - const_cast(mapped_file->scales_data())); - - if (mapped_file->is_interleaved()) { - auto& buffer = nodes_[node_index_map_.at(node_id)]->output_buffer; - buffer.set_interleaved(true, mapped_file->original_N()); - } - } - - size_t file_idx = mapped_files_.size(); - mapped_files_.push_back(std::move(mapped_file)); - node_to_mapped_file_[node_id] = file_idx; - weight_cache_[filename] = node_id; - return node_id; -} - -void CactusGraph::release_weight_pages(size_t node_id) { - auto it = node_to_mapped_file_.find(node_id); - if (it != node_to_mapped_file_.end() && it->second < mapped_files_.size()) { - mapped_files_[it->second]->release_pages(); - } -} - -void CactusGraph::prefetch_weight_pages(size_t node_id) { - auto it = node_to_mapped_file_.find(node_id); - if (it != node_to_mapped_file_.end() && it->second < mapped_files_.size()) { - mapped_files_[it->second]->prefetch_pages(); - } -} - -void CactusGraph::release_all_weight_pages() { - for (auto& mf : mapped_files_) { - if (mf) mf->release_pages(); - } -} - -void CactusGraph::set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr) { - auto it = node_index_map_.find(node_id); - if (it != node_index_map_.end()) { - nodes_[it->second]->output_buffer.set_grouped_scales(group_size, num_groups, scales_ptr); - } -} - -void CactusGraph::set_interleaved(size_t node_id, bool interleaved, size_t original_N) { - auto it = node_index_map_.find(node_id); - if (it != node_index_map_.end()) { - nodes_[it->second]->output_buffer.set_interleaved(interleaved, original_N); - } -} - -size_t CactusGraph::embedding(const std::string& filename, size_t indices) { - auto mapped_file = std::make_unique(filename); - - const auto& shape = mapped_file->shape(); - if (shape.size() != 2) { - throw std::runtime_error("Embedding file must contain 2D tensor [vocab_size, hidden_dim]"); - } - - Precision precision = mapped_file->precision(); - size_t embeddings_node = input(shape, precision); - set_external_input(embeddings_node, const_cast(mapped_file->data()), precision); - - if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { - set_grouped_scales(embeddings_node, mapped_file->group_size(), mapped_file->num_groups(), - const_cast(mapped_file->scales_data())); - - if (mapped_file->is_interleaved()) { - auto& buffer = nodes_[node_index_map_.at(embeddings_node)]->output_buffer; - buffer.set_interleaved(true, mapped_file->original_N()); - } - } - - mapped_files_.push_back(std::move(mapped_file)); - - const auto& idx_shape = get_output_buffer(indices).shape; - std::vector output_shape = idx_shape; - output_shape.push_back(shape[1]); - - OpParams params; - params.output_precision = Precision::FP16; - return add_node(OpType::EMBEDDING, {embeddings_node, indices}, output_shape, params); -} - - -namespace GraphFile { - -void save_graph(const CactusGraph& graph, - const std::string& filename) { - - std::ofstream out(filename, std::ios::binary); - if (!out) { - throw std::runtime_error("Cannot open file for writing: " + filename); - } - - std::unordered_map id_to_index; - id_to_index.reserve(graph.nodes_.size()); - - for (uint32_t i = 0; i < graph.nodes_.size(); ++i) { - id_to_index[graph.nodes_[i]->id] = i; - } - - SerializedGraph sg; - sg.header.magic = CACTUS_GRAPH_MAGIC; - sg.header.version = 4; - sg.header.node_count = static_cast(graph.nodes_.size()); - sg.header.flags = 0; - - sg.nodes.reserve(graph.nodes_.size()); - - for (uint32_t i = 0; i < graph.nodes_.size(); ++i) { - const auto& node = graph.nodes_[i]; - - NodeEntry entry; - entry.index = i; - entry.op_type = node->op_type; - entry.output_shape = node->output_buffer.shape; - entry.precision = node->output_buffer.precision; - entry.params = node->params; - - entry.inputs.reserve(node->input_ids.size()); - for (size_t input_id : node->input_ids) { - auto it = id_to_index.find(input_id); - if (it == id_to_index.end()) { - throw std::runtime_error("Graph save failed: missing input id mapping"); - } - entry.inputs.push_back(it->second); - } - - if (node->op_type == OpType::INPUT) { - sg.graph_inputs.push_back(entry.index); - } - - sg.nodes.push_back(std::move(entry)); - } - - sg.graph_outputs = compute_leaf_outputs(sg); - - write_serialized_graph(out, sg); - - if (!out) { - throw std::runtime_error("Error writing graph data to file: " + filename); - } -} - -SerializedGraph load_graph(const std::string& filename) { - std::ifstream in(filename, std::ios::binary); - if (!in) { - throw std::runtime_error("Cannot open file for reading: " + filename); - } - SerializedGraph sg; - sg.header = read_graph_header(in); - sg.graph_inputs = read_u32_vector(in); - sg.graph_outputs = read_u32_vector(in); - sg.nodes.reserve(sg.header.node_count); - for (uint32_t i = 0; i < sg.header.node_count; ++i) { - sg.nodes.push_back(read_node_entry(in)); - } - - if (sg.nodes.size() != sg.header.node_count) { - throw std::runtime_error("Graph file corrupted: node count mismatch"); - } - - for (uint32_t i = 0; i < sg.nodes.size(); ++i) { - const auto& node = sg.nodes[i]; - - if (node.index != i) { - throw std::runtime_error("Graph file corrupted: node indices must be dense and ordered"); - } - - for (uint32_t input_idx : node.inputs) { - if (input_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: input index out of range"); - } - } - } - - for (uint32_t input_idx : sg.graph_inputs) { - if (input_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: graph input index out of range"); - } - } - - for (uint32_t output_idx : sg.graph_outputs) { - if (output_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: graph output index out of range"); - } - } - - return sg; -} - -void save_node(CactusGraph& graph, size_t node_id, const std::string& filename) { - graph.execute(); - void* data = graph.get_output(node_id); - - const auto& buffer = graph.get_output_buffer(node_id); - const auto& shape = buffer.shape; - Precision precision = buffer.precision; - - std::ofstream file(filename, std::ios::binary); - if (!file) { - throw std::runtime_error("Cannot open file for writing: " + filename); - } - - size_t total_elements = 1; - for (size_t dim : shape) { - total_elements *= dim; - } - - size_t byte_size = PrecisionTraits::packed_size_of(precision, total_elements); - - bool has_scales = PrecisionTraits::is_integer(precision) && buffer.group_size > 0 && buffer.scales_data; - size_t N = shape.size() >= 1 ? shape[0] : 1; - size_t scales_bytes = has_scales ? (N * buffer.num_groups * sizeof(__fp16)) : 0; - - uint32_t ndim = static_cast(shape.size()); - uint32_t flags = has_scales ? FLAG_HAS_SCALES : 0; - if (buffer.is_interleaved) { - flags |= FLAG_INTERLEAVED; - } - uint32_t alignment = 32; - - uint32_t magic = CACTUS_MAGIC; - file.write(reinterpret_cast(&magic), sizeof(magic)); - file.write(reinterpret_cast(&flags), sizeof(flags)); - file.write(reinterpret_cast(&alignment), sizeof(alignment)); - file.write(reinterpret_cast(&ndim), sizeof(ndim)); - - for (uint32_t i = 0; i < 4; i++) { - uint64_t dim_val = (i < shape.size()) ? static_cast(shape[i]) : 0; - file.write(reinterpret_cast(&dim_val), sizeof(dim_val)); - } - - uint32_t prec_val = static_cast(precision); - file.write(reinterpret_cast(&prec_val), sizeof(prec_val)); - - uint64_t data_bytes = static_cast(byte_size); - uint64_t scales_bytes_val = static_cast(scales_bytes); - file.write(reinterpret_cast(&data_bytes), sizeof(data_bytes)); - file.write(reinterpret_cast(&scales_bytes_val), sizeof(scales_bytes_val)); - - uint32_t group_size = has_scales ? static_cast(buffer.group_size) : 0; - uint32_t num_groups = has_scales ? static_cast(buffer.num_groups) : 0; - file.write(reinterpret_cast(&group_size), sizeof(group_size)); - file.write(reinterpret_cast(&num_groups), sizeof(num_groups)); - - uint64_t original_N = buffer.is_interleaved ? buffer.original_N : N; - file.write(reinterpret_cast(&original_N), sizeof(original_N)); - - size_t header_end = HEADER_SIZE; - size_t aligned_header = align_offset(header_end, alignment); - size_t header_padding = aligned_header - header_end; - for (size_t i = 0; i < header_padding; i++) { - char zero = 0; - file.write(&zero, 1); - } - - if (has_scales) { - file.write(static_cast(buffer.scales_data), scales_bytes); - - size_t scales_end = aligned_header + scales_bytes; - size_t data_start = align_offset(scales_end, alignment); - size_t scales_padding = data_start - scales_end; - for (size_t i = 0; i < scales_padding; i++) { - char zero = 0; - file.write(&zero, 1); - } - } - - file.write(static_cast(data), byte_size); - - if (!file) { - throw std::runtime_error("Error writing node data to file: " + filename); - } -} - -// MappedFile implementation - -MappedFile::MappedFile(const std::string& filename) - : fd_(-1), mapped_data_(nullptr), file_size_(0), data_offset_(0) { - fd_ = open(filename.c_str(), O_RDONLY); - if (fd_ == -1) { - throw std::runtime_error("Cannot open file for mapping: " + filename); - } - - struct stat st; - if (fstat(fd_, &st) == -1) { - close(fd_); - throw std::runtime_error("Cannot get file size: " + filename); - } - file_size_ = static_cast(st.st_size); - - mapped_data_ = mmap(nullptr, file_size_, PROT_READ, MAP_SHARED, fd_, 0); - if (mapped_data_ == MAP_FAILED) { - close(fd_); - throw std::runtime_error("Cannot map file: " + filename); - } - - close(fd_); - fd_ = -1; - - parse_header(); - apply_madvise_hints(); -} - -MappedFile::~MappedFile() { - if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { - madvise(mapped_data_, file_size_, MADV_DONTNEED); - munmap(mapped_data_, file_size_); - mapped_data_ = nullptr; - } - if (fd_ != -1) { - close(fd_); - fd_ = -1; - } -} - -MappedFile::MappedFile(MappedFile&& other) noexcept - : fd_(other.fd_), mapped_data_(other.mapped_data_), file_size_(other.file_size_), - data_offset_(other.data_offset_), shape_(std::move(other.shape_)), - precision_(other.precision_), byte_size_(other.byte_size_), - group_size_(other.group_size_), num_groups_(other.num_groups_), - scales_offset_(other.scales_offset_), scales_bytes_(other.scales_bytes_), - alignment_(other.alignment_), - is_interleaved_(other.is_interleaved_), - original_N_(other.original_N_) { - other.fd_ = -1; - other.mapped_data_ = nullptr; - other.file_size_ = 0; - other.is_interleaved_ = false; - other.original_N_ = 0; -} - -MappedFile& MappedFile::operator=(MappedFile&& other) noexcept { - if (this != &other) { - if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { - munmap(mapped_data_, file_size_); - } - if (fd_ != -1) { - close(fd_); - } - - fd_ = other.fd_; - mapped_data_ = other.mapped_data_; - file_size_ = other.file_size_; - data_offset_ = other.data_offset_; - shape_ = std::move(other.shape_); - precision_ = other.precision_; - byte_size_ = other.byte_size_; - group_size_ = other.group_size_; - num_groups_ = other.num_groups_; - scales_offset_ = other.scales_offset_; - scales_bytes_ = other.scales_bytes_; - alignment_ = other.alignment_; - is_interleaved_ = other.is_interleaved_; - original_N_ = other.original_N_; - other.fd_ = -1; - other.mapped_data_ = nullptr; - other.file_size_ = 0; - other.is_interleaved_ = false; - other.original_N_ = 0; - } - return *this; -} - -const std::vector& MappedFile::shape() const { - return shape_; -} - -Precision MappedFile::precision() const { - return precision_; -} - -size_t MappedFile::byte_size() const { - return byte_size_; -} - -const void* MappedFile::scales_data() const { - return static_cast(mapped_data_) + scales_offset_; -} - -void* MappedFile::data() { - return static_cast(mapped_data_) + data_offset_; -} - -const void* MappedFile::data() const { - return static_cast(mapped_data_) + data_offset_; -} - -template -const T* MappedFile::typed_data() const { - return static_cast(data()); -} - -void MappedFile::parse_header() { - if (file_size_ < HEADER_SIZE) { - throw std::runtime_error("File too small: insufficient data for header"); - } - - const char* ptr = static_cast(mapped_data_); - size_t offset = 0; - - uint32_t magic = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - if (magic != CACTUS_MAGIC) { - throw std::runtime_error("Invalid tensor file: missing CACT magic number"); - } - - uint32_t flags = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - is_interleaved_ = (flags & FLAG_INTERLEAVED) != 0; - - alignment_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - if (alignment_ == 0) alignment_ = 1; - - uint32_t ndim = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - - shape_.clear(); - for (uint32_t i = 0; i < 4; i++) { - uint64_t dim_val = *reinterpret_cast(ptr + offset); - offset += sizeof(uint64_t); - if (i < ndim && dim_val > 0) { - shape_.push_back(static_cast(dim_val)); - } - } - - uint32_t prec_val = *reinterpret_cast(ptr + offset); - precision_ = static_cast(prec_val); - offset += sizeof(uint32_t); - - byte_size_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint64_t); - - scales_bytes_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint64_t); - - group_size_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - - num_groups_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - - original_N_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint64_t); - - size_t aligned_header = align_offset(HEADER_SIZE, alignment_); - - if (scales_bytes_ > 0) { - scales_offset_ = aligned_header; - size_t scales_end = scales_offset_ + scales_bytes_; - data_offset_ = align_offset(scales_end, alignment_); - } else { - scales_offset_ = 0; - data_offset_ = aligned_header; - } - - if (data_offset_ + byte_size_ > file_size_) { - throw std::runtime_error("File corrupted: data extends beyond file size"); - } - -} - -void MappedFile::apply_madvise_hints() { - if (scales_bytes_ > 0 && scales_offset_ > 0) { - madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_WILLNEED); - } - - madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_SEQUENTIAL); - - if (byte_size_ > 1024 * 1024) { - madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_WILLNEED); - } -} - -void MappedFile::release_pages() { - if (mapped_data_ == nullptr || mapped_data_ == MAP_FAILED) return; - - if (scales_bytes_ > 0 && scales_offset_ > 0) { - madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_DONTNEED); - } - madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_DONTNEED); -} - -void MappedFile::prefetch_pages() { - if (mapped_data_ == nullptr || mapped_data_ == MAP_FAILED) return; - - if (scales_bytes_ > 0 && scales_offset_ > 0) { - madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_WILLNEED); - } - madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_WILLNEED); -} - -template const int8_t* MappedFile::typed_data() const; -template const float* MappedFile::typed_data() const; -template const uint16_t* MappedFile::typed_data() const; -template const uint8_t* MappedFile::typed_data() const; - -} // namespace GraphFile diff --git a/cactus-transpile/graph/graph_ops_math.cpp b/cactus-transpile/graph/graph_ops_math.cpp deleted file mode 100644 index ee49bfd56..000000000 --- a/cactus-transpile/graph/graph_ops_math.cpp +++ /dev/null @@ -1,737 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel.h" -#include -#include -#include - -namespace Quantization { - void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale) { - cactus_int8_to_fp32(src, dst, count, scale); - } - - void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale) { - cactus_fp32_to_int8(src, dst, count, scale); - } - - void fp16_to_fp32(const __fp16* src, float* dst, size_t count) { - cactus_fp16_to_fp32(src, dst, count); - } - - void fp32_to_fp16(const float* src, __fp16* dst, size_t count) { - cactus_fp32_to_fp16(src, dst, count); - } - - void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale) { - cactus_int8_to_fp16(src, dst, count, scale); - } - - void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale) { - cactus_fp16_to_int8(src, dst, count, scale); - } -} - -static std::vector compute_strides(const std::vector& shape, const std::vector& target_shape) { - std::vector strides(target_shape.size()); - - size_t shape_offset = target_shape.size() - shape.size(); - - for (size_t i = 0; i < target_shape.size(); ++i) { - if (i < shape_offset) { - strides[i] = 0; - } else { - size_t dim_idx = i - shape_offset; - if (shape[dim_idx] == 1) { - strides[i] = 0; - } else { - strides[i] = 1; - for (size_t j = dim_idx + 1; j < shape.size(); ++j) { - strides[i] *= shape[j]; - } - } - } - } - - return strides; -} - -void dispatch_binary_op_f16(OpType op, const __fp16* lhs, const __fp16* rhs, __fp16* output, size_t count) { - switch (op) { - case OpType::ADD: - cactus_add_f16(lhs, rhs, output, count); - break; - case OpType::ADD_CLIPPED: - cactus_add_f16_clipped(lhs, rhs, output, count); - break; - case OpType::SUBTRACT: - cactus_subtract_f16(lhs, rhs, output, count); - break; - case OpType::MULTIPLY: - cactus_multiply_f16(lhs, rhs, output, count); - break; - case OpType::DIVIDE: - cactus_divide_f16(lhs, rhs, output, count); - break; - case OpType::NOT_EQUAL: - cactus_not_equal_f16(lhs, rhs, output, count); - break; - default: - break; - } -} - -static float dispatch_binary_op_scalar_fp32(OpType op, float lhs, float rhs) { - switch (op) { - case OpType::ADD: - return lhs + rhs; - case OpType::ADD_CLIPPED: { - constexpr float FP16_MAX = 65500.0f; - float value = lhs + rhs; - return std::fmin(std::fmax(value, -FP16_MAX), FP16_MAX); - } - case OpType::SUBTRACT: - return lhs - rhs; - case OpType::MULTIPLY: - return lhs * rhs; - case OpType::DIVIDE: - return lhs / rhs; - case OpType::NOT_EQUAL: - return lhs != rhs ? 1.0f : 0.0f; - default: - throw std::runtime_error("unsupported FP32 binary op"); - } -} - -static void dispatch_binary_op_fp32(OpType op, const float* lhs, const float* rhs, float* output, size_t count) { - for (size_t i = 0; i < count; ++i) { - output[i] = dispatch_binary_op_scalar_fp32(op, lhs[i], rhs[i]); - } -} - -static void dispatch_binary_op_fp32_to_fp16( - OpType op, - const float* lhs, - const float* rhs, - __fp16* output, - size_t count -) { - for (size_t i = 0; i < count; ++i) { - output[i] = static_cast<__fp16>(dispatch_binary_op_scalar_fp32(op, lhs[i], rhs[i])); - } -} - -static void dispatch_binary_op_f16_to_fp32( - OpType op, - const __fp16* lhs, - const __fp16* rhs, - float* output, - size_t count -) { - for (size_t i = 0; i < count; ++i) { - output[i] = dispatch_binary_op_scalar_fp32( - op, - static_cast(lhs[i]), - static_cast(rhs[i]) - ); - } -} - -static void dispatch_binary_op_broadcast_fp32( - OpType op, - const float* lhs, - const float* rhs, - float* output, - const std::vector& lhs_strides, - const std::vector& rhs_strides, - const std::vector& output_shape -) { - size_t output_size = 1; - for (size_t dim : output_shape) { - output_size *= dim; - } - - for (size_t linear_index = 0; linear_index < output_size; ++linear_index) { - size_t remainder = linear_index; - size_t lhs_index = 0; - size_t rhs_index = 0; - for (size_t dim = output_shape.size(); dim-- > 0;) { - const size_t coord = remainder % output_shape[dim]; - remainder /= output_shape[dim]; - lhs_index += coord * lhs_strides[dim]; - rhs_index += coord * rhs_strides[dim]; - } - output[linear_index] = dispatch_binary_op_scalar_fp32(op, lhs[lhs_index], rhs[rhs_index]); - } -} - -static void dispatch_binary_op_broadcast_fp32_to_fp16( - OpType op, - const float* lhs, - const float* rhs, - __fp16* output, - const std::vector& lhs_strides, - const std::vector& rhs_strides, - const std::vector& output_shape -) { - size_t output_size = 1; - for (size_t dim : output_shape) { - output_size *= dim; - } - - for (size_t linear_index = 0; linear_index < output_size; ++linear_index) { - size_t remainder = linear_index; - size_t lhs_index = 0; - size_t rhs_index = 0; - for (size_t dim = output_shape.size(); dim-- > 0;) { - const size_t coord = remainder % output_shape[dim]; - remainder /= output_shape[dim]; - lhs_index += coord * lhs_strides[dim]; - rhs_index += coord * rhs_strides[dim]; - } - output[linear_index] = static_cast<__fp16>( - dispatch_binary_op_scalar_fp32(op, lhs[lhs_index], rhs[rhs_index]) - ); - } -} - -static void dispatch_binary_op_broadcast_f16_to_fp32( - OpType op, - const __fp16* lhs, - const __fp16* rhs, - float* output, - const std::vector& lhs_strides, - const std::vector& rhs_strides, - const std::vector& output_shape -) { - size_t output_size = 1; - for (size_t dim : output_shape) { - output_size *= dim; - } - - for (size_t linear_index = 0; linear_index < output_size; ++linear_index) { - size_t remainder = linear_index; - size_t lhs_index = 0; - size_t rhs_index = 0; - for (size_t dim = output_shape.size(); dim-- > 0;) { - const size_t coord = remainder % output_shape[dim]; - remainder /= output_shape[dim]; - lhs_index += coord * lhs_strides[dim]; - rhs_index += coord * rhs_strides[dim]; - } - output[linear_index] = dispatch_binary_op_scalar_fp32( - op, - static_cast(lhs[lhs_index]), - static_cast(rhs[rhs_index]) - ); - } -} - -void dispatch_unary_op_f16(OpType op, const __fp16* input, __fp16* output, size_t count, float param) { - ScalarOpType scalar_op; - switch (op) { - case OpType::SCALAR_ADD: scalar_op = ScalarOpType::ADD; break; - case OpType::SCALAR_SUBTRACT: scalar_op = ScalarOpType::SUBTRACT; break; - case OpType::SCALAR_MULTIPLY: scalar_op = ScalarOpType::MULTIPLY; break; - case OpType::SCALAR_DIVIDE: scalar_op = ScalarOpType::DIVIDE; break; - case OpType::SCALAR_NOT_EQUAL: scalar_op = ScalarOpType::NOT_EQUAL; break; - case OpType::SCALAR_EXP: scalar_op = ScalarOpType::EXP; break; - case OpType::SCALAR_SQRT: scalar_op = ScalarOpType::SQRT; break; - case OpType::SCALAR_COS: scalar_op = ScalarOpType::COS; break; - case OpType::SCALAR_SIN: scalar_op = ScalarOpType::SIN; break; - case OpType::SCALAR_LOG: scalar_op = ScalarOpType::LOG; break; - case OpType::SCALAR_FLOOR_DIVIDE: scalar_op = ScalarOpType::FLOOR_DIVIDE; break; - case OpType::ABS: scalar_op = ScalarOpType::ABS; break; - case OpType::POW: scalar_op = ScalarOpType::POW; break; - default: return; - } - - cactus_scalar_op_f16(input, output, count, param, scalar_op); -} - -static float dispatch_unary_op_scalar_fp32(OpType op, float input, float param) { - switch (op) { - case OpType::SCALAR_ADD: - return input + param; - case OpType::SCALAR_SUBTRACT: - return input - param; - case OpType::SCALAR_MULTIPLY: - return input * param; - case OpType::SCALAR_DIVIDE: - return input / param; - case OpType::SCALAR_NOT_EQUAL: - return input != param ? 1.0f : 0.0f; - case OpType::SCALAR_EXP: - return std::exp(input); - case OpType::SCALAR_SQRT: - return std::sqrt(input); - case OpType::SCALAR_COS: - return std::cos(input); - case OpType::SCALAR_SIN: - return std::sin(input); - case OpType::SCALAR_LOG: - return std::log(input); - case OpType::SCALAR_FLOOR_DIVIDE: - return std::floor(input / param); - case OpType::ABS: - return std::fabs(input); - case OpType::POW: - return std::pow(input, param); - default: - throw std::runtime_error("unsupported FP32 unary op"); - } -} - -static void dispatch_unary_op_fp32(OpType op, const float* input, float* output, size_t count, float param) { - for (size_t i = 0; i < count; ++i) { - output[i] = dispatch_unary_op_scalar_fp32(op, input[i], param); - } -} - -static void dispatch_unary_op_fp32_to_fp16( - OpType op, - const float* input, - __fp16* output, - size_t count, - float param -) { - for (size_t i = 0; i < count; ++i) { - output[i] = static_cast<__fp16>(dispatch_unary_op_scalar_fp32(op, input[i], param)); - } -} - -static void dispatch_unary_op_f16_to_fp32( - OpType op, - const __fp16* input, - float* output, - size_t count, - float param -) { - for (size_t i = 0; i < count; ++i) { - output[i] = dispatch_unary_op_scalar_fp32(op, static_cast(input[i]), param); - } -} - -void compute_binary_op_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& lhs = get_input(node, 0, nodes, node_index_map); - const auto& rhs = get_input(node, 1, nodes, node_index_map); - const Precision output_precision = node.output_buffer.precision; - - if (lhs.precision != rhs.precision) { - throw std::runtime_error("Binary operations require matching input precision"); - } - - if (lhs.precision != Precision::FP16 && lhs.precision != Precision::FP32) { - throw std::runtime_error( - "Binary operations only support FP16/FP32 precision (got " + - std::to_string(static_cast(lhs.precision)) + ")" - ); - } - - if (node.params.broadcast_info.needs_broadcasting) { - std::vector lhs_strides = compute_strides(lhs.shape, node.params.broadcast_info.output_shape); - std::vector rhs_strides = compute_strides(rhs.shape, node.params.broadcast_info.output_shape); - - if (lhs.precision == Precision::FP16) { - if (output_precision == Precision::FP16) { - switch (node.op_type) { - case OpType::ADD: - case OpType::ADD_CLIPPED: - cactus_add_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - case OpType::SUBTRACT: - cactus_subtract_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - case OpType::MULTIPLY: - cactus_multiply_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - case OpType::DIVIDE: - cactus_divide_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - case OpType::NOT_EQUAL: - cactus_not_equal_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - default: - break; - } - } else if (output_precision == Precision::FP32) { - dispatch_binary_op_broadcast_f16_to_fp32( - node.op_type, - lhs.data_as<__fp16>(), - rhs.data_as<__fp16>(), - node.output_buffer.data_as(), - lhs_strides, - rhs_strides, - node.params.broadcast_info.output_shape - ); - } else { - throw std::runtime_error("Binary broadcast FP16 inputs only support FP16/FP32 outputs"); - } - } else { - if (output_precision == Precision::FP32) { - dispatch_binary_op_broadcast_fp32( - node.op_type, - lhs.data_as(), - rhs.data_as(), - node.output_buffer.data_as(), - lhs_strides, - rhs_strides, - node.params.broadcast_info.output_shape - ); - } else if (output_precision == Precision::FP16) { - dispatch_binary_op_broadcast_fp32_to_fp16( - node.op_type, - lhs.data_as(), - rhs.data_as(), - node.output_buffer.data_as<__fp16>(), - lhs_strides, - rhs_strides, - node.params.broadcast_info.output_shape - ); - } else { - throw std::runtime_error("Binary broadcast FP32 inputs only support FP16/FP32 outputs"); - } - } - } else { - if (lhs.precision == Precision::FP16) { - if (output_precision == Precision::FP16) { - dispatch_binary_op_f16(node.op_type, lhs.data_as<__fp16>(), - rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - } else if (output_precision == Precision::FP32) { - dispatch_binary_op_f16_to_fp32( - node.op_type, - lhs.data_as<__fp16>(), - rhs.data_as<__fp16>(), - node.output_buffer.data_as(), - node.output_buffer.total_size - ); - } else { - throw std::runtime_error("Binary FP16 inputs only support FP16/FP32 outputs"); - } - } else { - if (output_precision == Precision::FP32) { - dispatch_binary_op_fp32(node.op_type, lhs.data_as(), - rhs.data_as(), node.output_buffer.data_as(), - node.output_buffer.total_size); - } else if (output_precision == Precision::FP16) { - dispatch_binary_op_fp32_to_fp16( - node.op_type, - lhs.data_as(), - rhs.data_as(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size - ); - } else { - throw std::runtime_error("Binary FP32 inputs only support FP16/FP32 outputs"); - } - } - } -} - -void compute_unary_op_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const Precision output_precision = node.output_buffer.precision; - - if (input.precision != Precision::FP16 && input.precision != Precision::FP32) { - throw std::runtime_error("Scalar operations only support FP16/FP32 precision"); - } - - if (input.precision == Precision::FP16) { - if (output_precision == Precision::FP16) { - dispatch_unary_op_f16(node.op_type, input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size, node.params.scalar); - } else if (output_precision == Precision::FP32) { - dispatch_unary_op_f16_to_fp32( - node.op_type, - input.data_as<__fp16>(), - node.output_buffer.data_as(), - node.output_buffer.total_size, - node.params.scalar - ); - } else { - throw std::runtime_error("Scalar FP16 inputs only support FP16/FP32 outputs"); - } - } else { - if (output_precision == Precision::FP32) { - dispatch_unary_op_fp32(node.op_type, input.data_as(), - node.output_buffer.data_as(), - node.output_buffer.total_size, node.params.scalar); - } else if (output_precision == Precision::FP16) { - dispatch_unary_op_fp32_to_fp16( - node.op_type, - input.data_as(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size, - node.params.scalar - ); - } else { - throw std::runtime_error("Scalar FP32 inputs only support FP16/FP32 outputs"); - } - } -} - -void compute_activation_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - - if (input.precision != Precision::FP16) { - throw std::runtime_error("Activation operations only support FP16 precision"); - } - - switch (node.op_type) { - case OpType::RELU: - cactus_relu_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::SILU: - cactus_silu_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::GELU: - cactus_gelu_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::GELU_ERF: - cactus_gelu_f16_erf(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::SIGMOID: - cactus_sigmoid_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::TANH: - cactus_tanh_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::LEAKY_RELU: - cactus_leaky_relu_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size, node.params.scalar); - break; - default: - break; - } -} - -void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - int axis = node.params.axis; - - if (input_buffer.precision != Precision::FP16 && input_buffer.precision != Precision::FP32) { - throw std::runtime_error("Reduction operations only support FP16 and FP32 precision"); - } - - if (axis == -1) { - if (input_buffer.precision == Precision::FP32) { - switch (node.op_type) { - case OpType::SUM: { - double result = cactus_sum_all_f32(input_buffer.data_as(), input_buffer.total_size); - node.output_buffer.data_as()[0] = static_cast(result); - break; - } - case OpType::MEAN: { - double result = cactus_mean_all_f32(input_buffer.data_as(), input_buffer.total_size); - node.output_buffer.data_as()[0] = static_cast(result); - break; - } - case OpType::VARIANCE: { - double result = cactus_variance_all_f32(input_buffer.data_as(), input_buffer.total_size); - node.output_buffer.data_as()[0] = static_cast(result); - break; - } - case OpType::MIN: - node.output_buffer.data_as()[0] = cactus_min_all_f32(input_buffer.data_as(), input_buffer.total_size); - break; - case OpType::MAX: - node.output_buffer.data_as()[0] = cactus_max_all_f32(input_buffer.data_as(), input_buffer.total_size); - break; - default: - break; - } - } else { - switch (node.op_type) { - case OpType::SUM: { - double result = cactus_sum_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); - break; - } - case OpType::MEAN: { - double result = cactus_mean_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); - break; - } - case OpType::VARIANCE: { - double result = cactus_variance_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); - break; - } - case OpType::MIN: { - __fp16 result = cactus_min_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = result; - break; - } - case OpType::MAX: { - __fp16 result = cactus_max_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = result; - break; - } - default: - break; - } - } - } else { - auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); - - if (input_buffer.precision == Precision::FP32) { - switch (node.op_type) { - case OpType::SUM: - cactus_sum_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MEAN: - cactus_mean_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::VARIANCE: - cactus_variance_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MIN: - cactus_min_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MAX: - cactus_max_axis_f32(input_buffer.data_as(), node.output_buffer.data_as(), - dims.outer, dims.axis_size, dims.inner); - break; - default: - break; - } - } else { - switch (node.op_type) { - case OpType::SUM: - cactus_sum_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MEAN: - cactus_mean_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::VARIANCE: - cactus_variance_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MIN: - cactus_min_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MAX: - cactus_max_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - default: - break; - } - } - } -} - -void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - - size_t input_total_elements = input_buffer.total_size; - size_t output_total_elements = node.output_buffer.total_size; - - if (input_total_elements != output_total_elements) { - throw std::runtime_error("Reshape operation: input elements (" + std::to_string(input_total_elements) + - ") must match output elements (" + std::to_string(output_total_elements) + ")"); - } - - std::memcpy(node.output_buffer.get_data(), input_buffer.get_data(), input_buffer.byte_size); -} - -template -static void dequant_grouped_int8(const int8_t* src, OutT* dst, const __fp16* scales, - const std::vector& shape, size_t group_size) { - if (shape.size() == 2) { - size_t N = shape[0], K = shape[1]; - size_t num_groups = K / group_size; - for (size_t row = 0; row < N; ++row) { - for (size_t col = 0; col < K; ++col) { - size_t idx = row * K + col; - float scale = static_cast(scales[row * num_groups + col / group_size]); - dst[idx] = static_cast(static_cast(src[idx]) * scale); - } - } - } else if (shape.size() == 1) { - size_t K = shape[0]; - for (size_t col = 0; col < K; ++col) { - float scale = static_cast(scales[col / group_size]); - dst[col] = static_cast(static_cast(src[col]) * scale); - } - } -} - -void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buf = get_input(node, 0, nodes, node_index_map); - - if (input_buf.precision == node.output_buffer.precision) { - std::memcpy(node.output_buffer.get_data(), input_buf.get_data(), input_buf.byte_size); - return; - } - - size_t count = input_buf.total_size; - - if (input_buf.precision == Precision::INT8 && node.output_buffer.precision == Precision::FP32) { - if (input_buf.is_grouped_int8()) { - dequant_grouped_int8(input_buf.data_as(), node.output_buffer.data_as(), - input_buf.scales_as_fp16(), input_buf.shape, input_buf.group_size); - } else { - Quantization::int8_to_fp32(input_buf.data_as(), node.output_buffer.data_as(), count, 1.0f); - } - } else if (input_buf.precision == Precision::FP32 && node.output_buffer.precision == Precision::INT8) { - Quantization::fp32_to_int8(input_buf.data_as(), node.output_buffer.data_as(), count, 1.0f); - } else if (input_buf.precision == Precision::FP16 && node.output_buffer.precision == Precision::FP32) { - Quantization::fp16_to_fp32(input_buf.data_as<__fp16>(), node.output_buffer.data_as(), count); - } else if (input_buf.precision == Precision::FP32 && node.output_buffer.precision == Precision::FP16) { - Quantization::fp32_to_fp16(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), count); - } else if (input_buf.precision == Precision::INT8 && node.output_buffer.precision == Precision::FP16) { - if (input_buf.is_grouped_int8()) { - dequant_grouped_int8<__fp16>(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), - input_buf.scales_as_fp16(), input_buf.shape, input_buf.group_size); - } else { - Quantization::int8_to_fp16(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), count, 1.0f); - } - } else if (input_buf.precision == Precision::FP16 && node.output_buffer.precision == Precision::INT8) { - Quantization::fp16_to_int8(input_buf.data_as<__fp16>(), node.output_buffer.data_as(), count, 1.0f); - } else { - throw std::runtime_error("Unsupported precision conversion from " + - std::to_string(static_cast(input_buf.precision)) + - " to " + std::to_string(static_cast(node.output_buffer.precision))); - } -} diff --git a/cactus-transpile/graph/graph_ops_nn.cpp b/cactus-transpile/graph/graph_ops_nn.cpp deleted file mode 100644 index 21799bc52..000000000 --- a/cactus-transpile/graph/graph_ops_nn.cpp +++ /dev/null @@ -1,2754 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel.h" -#include "../kernel/kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - thread_local std::vector<__fp16> transpose_buffer_fp16; - thread_local std::vector transpose_buffer_fp32; - thread_local std::vector quant_activation_buffer; - thread_local std::vector quant_scales_buffer; - - thread_local const __fp16* cached_quant_src = nullptr; - thread_local size_t cached_quant_M = 0; - thread_local size_t cached_quant_K = 0; - - void ensure_transpose_buffer_fp16(size_t required_size) { - if (transpose_buffer_fp16.size() < required_size) { - transpose_buffer_fp16.resize(required_size); - } - } - - void ensure_transpose_buffer_fp32(size_t required_size) { - if (transpose_buffer_fp32.size() < required_size) { - transpose_buffer_fp32.resize(required_size); - } - } - - void ensure_quant_buffers(size_t M, size_t K) { - size_t required_data = M * K; - if (quant_activation_buffer.size() < required_data) { - quant_activation_buffer.resize(required_data); - } - if (quant_scales_buffer.size() < M) { - quant_scales_buffer.resize(M); - } - } - - void matmul_fp16_inputs_to_fp32( - const __fp16* lhs, - const __fp16* rhs_transposed, - float* output, - size_t M, - size_t K, - size_t N - ) { - CactusThreading::parallel_for( - M, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_row, size_t end_row) { - for (size_t m = start_row; m < end_row; ++m) { - const __fp16* lhs_row = lhs + (m * K); - float* out_row = output + (m * N); - for (size_t n = 0; n < N; ++n) { - const __fp16* rhs_row = rhs_transposed + (n * K); - float acc = 0.0f; - for (size_t k = 0; k < K; ++k) { - acc += static_cast(lhs_row[k]) * static_cast(rhs_row[k]); - } - out_row[n] = acc; - } - } - } - ); - } - - void matmul_fp16_inputs_to_fp16( - const __fp16* lhs, - const __fp16* rhs_transposed, - __fp16* output, - size_t M, - size_t K, - size_t N - ) { - CactusThreading::parallel_for( - M, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_row, size_t end_row) { - for (size_t m = start_row; m < end_row; ++m) { - const __fp16* lhs_row = lhs + (m * K); - __fp16* out_row = output + (m * N); - for (size_t n = 0; n < N; ++n) { - const __fp16* rhs_row = rhs_transposed + (n * K); - float acc = 0.0f; - for (size_t k = 0; k < K; ++k) { - acc += static_cast(lhs_row[k]) * static_cast(rhs_row[k]); - } - out_row[n] = static_cast<__fp16>(acc); - } - } - } - ); - } - - void matmul_fp32_inputs_to_fp32( - const float* lhs, - const float* rhs_transposed, - float* output, - size_t M, - size_t K, - size_t N - ) { - CactusThreading::parallel_for( - M, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_row, size_t end_row) { - for (size_t m = start_row; m < end_row; ++m) { - const float* lhs_row = lhs + (m * K); - float* out_row = output + (m * N); - for (size_t n = 0; n < N; ++n) { - const float* rhs_row = rhs_transposed + (n * K); - float acc = 0.0f; - for (size_t k = 0; k < K; ++k) { - acc += lhs_row[k] * rhs_row[k]; - } - out_row[n] = acc; - } - } - } - ); - } - - void matmul_fp32_inputs_to_fp16( - const float* lhs, - const float* rhs_transposed, - __fp16* output, - size_t M, - size_t K, - size_t N - ) { - CactusThreading::parallel_for( - M, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_row, size_t end_row) { - for (size_t m = start_row; m < end_row; ++m) { - const float* lhs_row = lhs + (m * K); - __fp16* out_row = output + (m * N); - for (size_t n = 0; n < N; ++n) { - const float* rhs_row = rhs_transposed + (n * K); - float acc = 0.0f; - for (size_t k = 0; k < K; ++k) { - acc += lhs_row[k] * rhs_row[k]; - } - out_row[n] = static_cast<__fp16>(acc); - } - } - } - ); - } - - void quantize_activations_fp16_to_int8(const __fp16* src, int8_t* dst, float* scales, size_t M, size_t K) { - if (src == cached_quant_src && M == cached_quant_M && K == cached_quant_K) { - return; - } - - constexpr size_t PARALLEL_THRESHOLD = 16; - - if (M >= PARALLEL_THRESHOLD) { - CactusThreading::parallel_for(M, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scales, K](size_t m_start, size_t m_end) { - for (size_t m = m_start; m < m_end; m++) { - float max_abs = cactus_fp16_max_abs(src + m * K, K); - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - scales[m] = scale; - cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); - } - }); - } else { - for (size_t m = 0; m < M; m++) { - float max_abs = cactus_fp16_max_abs(src + m * K, K); - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - scales[m] = scale; - cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); - } - } - - cached_quant_src = src; - cached_quant_M = M; - cached_quant_K = K; - } - - const __fp16* as_fp16_ptr(const BufferDesc& buffer, std::vector<__fp16>& scratch) { - if (buffer.precision == Precision::FP16) { - return buffer.data_as<__fp16>(); - } - if (buffer.precision == Precision::FP32) { - scratch.resize(buffer.total_size); - cactus_fp32_to_fp16(buffer.data_as(), scratch.data(), buffer.total_size); - return scratch.data(); - } - throw std::runtime_error("GATED_DELTANET unsupported precision (expected FP16/FP32)"); - } - - const float* as_fp32_ptr(const BufferDesc& buffer, std::vector& scratch) { - if (buffer.precision == Precision::FP32) { - return buffer.data_as(); - } - if (buffer.precision == Precision::FP16) { - scratch.resize(buffer.total_size); - cactus_fp16_to_fp32(buffer.data_as<__fp16>(), scratch.data(), buffer.total_size); - return scratch.data(); - } - throw std::runtime_error("matmul unsupported precision (expected FP16/FP32)"); - } - - void transpose_2d_fp32(const float* source, float* destination, size_t num_rows, size_t num_cols) { - CactusThreading::parallel_for( - num_rows, - CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_row, size_t end_row) { - for (size_t row = start_row; row < end_row; ++row) { - for (size_t col = 0; col < num_cols; ++col) { - destination[col * num_rows + row] = source[row * num_cols + col]; - } - } - } - ); - } - - void validate_gated_deltanet_inputs( - const BufferDesc& q, - const BufferDesc& k, - const BufferDesc& v, - const BufferDesc& g, - const BufferDesc& b, - const BufferDesc& s) { - auto is_supported_precision = [](Precision p) { - return p == Precision::FP16 || p == Precision::FP32; - }; - if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || - !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || - !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { - throw std::runtime_error("GATED_DELTANET requires FP16/FP32 inputs"); - } - - if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { - throw std::runtime_error("GATED_DELTANET expects query/key/value rank 4 [B, T, H, D]"); - } - if (g.shape.size() != 3 || b.shape.size() != 3) { - throw std::runtime_error("GATED_DELTANET expects gate_log/beta rank 3 [B, T, H]"); - } - if (s.shape.size() != 4) { - throw std::runtime_error("GATED_DELTANET expects state rank 4 [B, K, H, V]"); - } - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - - if (k.shape[0] != B || k.shape[1] != T || k.shape[2] != Hq || k.shape[3] != K) { - throw std::runtime_error("GATED_DELTANET query/key shape mismatch"); - } - if (v.shape[0] != B || v.shape[1] != T) { - throw std::runtime_error("GATED_DELTANET value shape mismatch"); - } - const size_t Hv = v.shape[2]; - if (g.shape[0] != B || g.shape[1] != T || g.shape[2] != Hv || - b.shape[0] != B || b.shape[1] != T || b.shape[2] != Hv) { - throw std::runtime_error("GATED_DELTANET gate_log/beta shape mismatch"); - } - if (Hq == 0 || Hv == 0 || (Hv % Hq) != 0) { - throw std::runtime_error("GATED_DELTANET expects value heads divisible by q/k heads"); - } - const size_t V = v.shape[3]; - if (s.shape[0] != B || s.shape[1] != K || s.shape[2] != Hv || s.shape[3] != V) { - throw std::runtime_error("GATED_DELTANET state shape mismatch"); - } - } - - -} - -void shrink_thread_local_buffers() { - std::vector<__fp16>().swap(transpose_buffer_fp16); - std::vector().swap(quant_activation_buffer); - std::vector().swap(quant_scales_buffer); - cached_quant_src = nullptr; - cached_quant_M = 0; - cached_quant_K = 0; -} - -void compute_quantize_activations_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& shape = input_buffer.shape; - - if (input_buffer.precision != Precision::FP16) { - throw std::runtime_error("QUANTIZE_ACTIVATIONS requires FP16 input"); - } - - if (shape.size() < 2) { - throw std::runtime_error("QUANTIZE_ACTIVATIONS requires at least 2D tensor"); - } - - size_t K = shape.back(); - size_t M = 1; - for (size_t i = 0; i < shape.size() - 1; ++i) { - M *= shape[i]; - } - - if (!node.output_buffer.has_activation_scales() || - node.output_buffer.num_rows_for_activation_scales != M) { - node.output_buffer.allocate_activation_scales(M); - } - - const __fp16* src = input_buffer.data_as<__fp16>(); - int8_t* dst = node.output_buffer.data_as(); - float* scales = node.output_buffer.activation_scales_as_float(); - - constexpr size_t PARALLEL_THRESHOLD = 16; - - if (M >= PARALLEL_THRESHOLD) { - CactusThreading::parallel_for(M, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scales, K](size_t m_start, size_t m_end) { - for (size_t m = m_start; m < m_end; m++) { - float max_abs = cactus_fp16_max_abs(src + m * K, K); - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - scales[m] = scale; - cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); - } - }); - } else { - for (size_t m = 0; m < M; m++) { - float max_abs = cactus_fp16_max_abs(src + m * K, K); - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - scales[m] = scale; - cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); - } - } -} - -void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& lhs_buffer = get_input(node, 0, nodes, node_index_map); - const auto& rhs_buffer = get_input(node, 1, nodes, node_index_map); - const auto& lhs_shape = lhs_buffer.shape; - const auto& rhs_shape = rhs_buffer.shape; - - size_t M = lhs_shape[lhs_shape.size() - 2]; - size_t K = lhs_shape[lhs_shape.size() - 1]; - size_t N; - if (rhs_buffer.is_interleaved && rhs_buffer.original_N > 0) { - N = rhs_buffer.original_N; - } else { - N = node.params.pretransposed_rhs ? - rhs_shape[rhs_shape.size() - 2] : rhs_shape[rhs_shape.size() - 1]; - } - - bool pretransposed_rhs = node.params.pretransposed_rhs; - - ComputeBackend backend = node.params.backend; - - if (backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU matrix multiplication not yet implemented"); - } - - const bool lhs_is_prequantized_int8 = (lhs_buffer.precision == Precision::INT8 && - lhs_buffer.has_activation_scales()); - - if (PrecisionTraits::is_integer(rhs_buffer.precision) && rhs_buffer.group_size > 0) { - const int8_t* rhs = rhs_buffer.data_as(); - const __fp16* rhs_scales = rhs_buffer.scales_as_fp16(); - if (node.output_buffer.precision != Precision::FP16) { - throw std::runtime_error("Quantized matmul currently only supports FP16 outputs"); - } - __fp16* output = node.output_buffer.data_as<__fp16>(); - - if (!pretransposed_rhs) { - throw std::runtime_error("Group-wise quantized matmul requires pretransposed weights"); - } - - const int8_t* lhs_int8; - const float* lhs_scales; - - if (lhs_is_prequantized_int8) { - lhs_int8 = lhs_buffer.data_as(); - lhs_scales = lhs_buffer.activation_scales_as_float(); - } else if (lhs_buffer.precision == Precision::FP16) { - ensure_quant_buffers(M, K); - cached_quant_src = nullptr; - quantize_activations_fp16_to_int8(lhs_buffer.data_as<__fp16>(), quant_activation_buffer.data(), - quant_scales_buffer.data(), M, K); - lhs_int8 = quant_activation_buffer.data(); - lhs_scales = quant_scales_buffer.data(); - } else { - throw std::runtime_error("Quantized matmul requires INT8 (pre-quantized) or FP16 activations"); - } - - cactus_matmul_integer(rhs_buffer.precision, - lhs_int8, lhs_scales, - rhs, rhs_scales, output, - M, K, N, rhs_buffer.group_size); - } else { - const bool fp16_fast_path = - lhs_buffer.precision == Precision::FP16 && rhs_buffer.precision == Precision::FP16; - if (fp16_fast_path) { - const __fp16* lhs = lhs_buffer.data_as<__fp16>(); - const __fp16* rhs = rhs_buffer.data_as<__fp16>(); - - if (pretransposed_rhs) { - if (node.output_buffer.precision == Precision::FP16) { - matmul_fp16_inputs_to_fp16(lhs, rhs, node.output_buffer.data_as<__fp16>(), M, K, N); - } else if (node.output_buffer.precision == Precision::FP32) { - matmul_fp16_inputs_to_fp32(lhs, rhs, node.output_buffer.data_as(), M, K, N); - } else { - throw std::runtime_error("FP16 matmul only supports FP16/FP32 outputs"); - } - } else { - size_t transpose_size = rhs_shape[0] * rhs_shape[1]; - ensure_transpose_buffer_fp16(transpose_size); - - cactus_transpose_2d_f16(rhs, transpose_buffer_fp16.data(), - rhs_shape[0], rhs_shape[1], 0, rhs_shape[0]); - if (node.output_buffer.precision == Precision::FP16) { - matmul_fp16_inputs_to_fp16(lhs, transpose_buffer_fp16.data(), node.output_buffer.data_as<__fp16>(), M, K, N); - } else if (node.output_buffer.precision == Precision::FP32) { - matmul_fp16_inputs_to_fp32(lhs, transpose_buffer_fp16.data(), node.output_buffer.data_as(), M, K, N); - } else { - throw std::runtime_error("FP16 matmul only supports FP16/FP32 outputs"); - } - } - } else { - std::vector lhs_fp32_scratch; - std::vector rhs_fp32_scratch; - const float* lhs = as_fp32_ptr(lhs_buffer, lhs_fp32_scratch); - const float* rhs = as_fp32_ptr(rhs_buffer, rhs_fp32_scratch); - const float* rhs_transposed = rhs; - - if (!pretransposed_rhs) { - size_t transpose_size = rhs_shape[0] * rhs_shape[1]; - ensure_transpose_buffer_fp32(transpose_size); - transpose_2d_fp32(rhs, transpose_buffer_fp32.data(), rhs_shape[0], rhs_shape[1]); - rhs_transposed = transpose_buffer_fp32.data(); - } - - if (node.output_buffer.precision == Precision::FP32) { - matmul_fp32_inputs_to_fp32(lhs, rhs_transposed, node.output_buffer.data_as(), M, K, N); - } else if (node.output_buffer.precision == Precision::FP16) { - matmul_fp32_inputs_to_fp16(lhs, rhs_transposed, node.output_buffer.data_as<__fp16>(), M, K, N); - } else { - throw std::runtime_error("FP32 matmul only supports FP16/FP32 outputs"); - } - } - } -} - -namespace { - thread_local std::vector<__fp16> moe_compact_hidden_buf; - thread_local std::vector<__fp16> moe_gate_buf; - thread_local std::vector<__fp16> moe_up_buf; - thread_local std::vector<__fp16> moe_expert_out_buf; - thread_local std::vector moe_lhs_q_buf; - thread_local std::vector moe_lhs_scales_buf; - thread_local std::vector moe_expert_offsets_buf; - thread_local std::vector moe_expert_tokens_buf; - thread_local std::vector moe_routing_denom_buf; - - void ensure_moe_buffers(size_t max_tokens, size_t hidden_dim, size_t intermediate_dim, - size_t num_experts, size_t top_k) { - size_t hidden_size = max_tokens * hidden_dim; - size_t inter_size = max_tokens * intermediate_dim; - if (moe_compact_hidden_buf.size() < hidden_size) moe_compact_hidden_buf.resize(hidden_size); - if (moe_gate_buf.size() < inter_size) moe_gate_buf.resize(inter_size); - if (moe_up_buf.size() < inter_size) moe_up_buf.resize(inter_size); - if (moe_expert_out_buf.size() < hidden_size) moe_expert_out_buf.resize(hidden_size); - size_t max_k = std::max(hidden_dim, intermediate_dim); - size_t quant_size = max_tokens * max_k; - if (moe_lhs_q_buf.size() < quant_size) moe_lhs_q_buf.resize(quant_size); - if (moe_lhs_scales_buf.size() < max_tokens) moe_lhs_scales_buf.resize(max_tokens); - size_t total_assignments = max_tokens * top_k; - if (moe_expert_offsets_buf.size() < num_experts + 1) moe_expert_offsets_buf.resize(num_experts + 1); - if (moe_expert_tokens_buf.size() < total_assignments) moe_expert_tokens_buf.resize(total_assignments); - if (moe_routing_denom_buf.size() < max_tokens) moe_routing_denom_buf.resize(max_tokens); - } - - void moe_matmul(const __fp16* lhs, - size_t M, - size_t K, - const BufferDesc& rhs_buffer, - __fp16* output, - size_t N, - bool lhs_prequantized = false) { - if (rhs_buffer.precision == Precision::FP16) { - cactus_matmul_f16(lhs, rhs_buffer.data_as<__fp16>(), output, M, K, N); - return; - } - - if (PrecisionTraits::is_integer(rhs_buffer.precision) && rhs_buffer.group_size > 0) { - int8_t* lhs_q = moe_lhs_q_buf.data(); - float* lhs_scales = moe_lhs_scales_buf.data(); - if (!lhs_prequantized) { - for (size_t row = 0; row < M; ++row) { - float scale = cactus_fp16_max_abs(lhs + row * K, K) / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - lhs_scales[row] = scale; - cactus_fp16_to_int8(lhs + row * K, lhs_q + row * K, K, scale); - } - } - cactus_matmul_integer(rhs_buffer.precision, - lhs_q, lhs_scales, - rhs_buffer.data_as(), - rhs_buffer.scales_as_fp16(), - output, M, K, N, rhs_buffer.group_size); - return; - } - - throw std::runtime_error("moe_layer only supports FP16 or grouped INT4/INT8 expert weights"); - } -} - -void compute_moe_layer_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const size_t num_experts = node.params.num_experts; - const size_t top_k = node.params.num_experts_per_tok; - const bool normalize_routing = node.params.normalize_routing; - const float eps = node.params.epsilon; - const float routed_scaling_factor = node.params.scalar; - const bool gated = node.params.moe_gated; - const Activation activation = node.params.activation; - const size_t base_inputs = gated ? (3 + 3 * num_experts) : (3 + 2 * num_experts); - bool has_per_expert_scale = node.input_ids.size() == base_inputs + 1; - if (node.input_ids.size() != base_inputs && node.input_ids.size() != base_inputs + 1) { - throw std::runtime_error("moe_layer expects " + std::to_string(base_inputs) + " or " + std::to_string(base_inputs + 1) + " inputs, got " + std::to_string(node.input_ids.size())); - } - - const auto& hidden_buffer = get_input(node, 0, nodes, node_index_map); - const auto& routing_buffer = get_input(node, 1, nodes, node_index_map); - const auto& topk_idx_buffer = get_input(node, 2, nodes, node_index_map); - - if (hidden_buffer.precision != Precision::FP16 || node.output_buffer.precision != Precision::FP16) { - throw std::runtime_error("moe_layer expects FP16 hidden/output"); - } - if (topk_idx_buffer.precision != Precision::FP32) { - throw std::runtime_error("moe_layer expects FP32 topk indices"); - } - - const __fp16* expert_scales_fp16 = nullptr; - if (has_per_expert_scale) { - const auto& scale_buffer = get_input(node, base_inputs, nodes, node_index_map); - if (scale_buffer.precision != Precision::FP16) { - throw std::runtime_error("moe_layer expects FP16 per_expert_scale"); - } - expert_scales_fp16 = scale_buffer.data_as<__fp16>(); - } - - const size_t token_count = hidden_buffer.shape[0]; - const size_t hidden_dim = hidden_buffer.shape[1]; - const size_t total_num_experts = routing_buffer.shape[1]; - - const auto& w1_0_buffer = get_input(node, 3, nodes, node_index_map); - const size_t expert_intermediate_dim = w1_0_buffer.shape[0]; - - const auto* hidden = hidden_buffer.data_as<__fp16>(); - auto* output = node.output_buffer.data_as<__fp16>(); - const auto* topk_idx = topk_idx_buffer.data_as(); - const auto* routing_fp16 = routing_buffer.precision == Precision::FP16 ? routing_buffer.data_as<__fp16>() : nullptr; - const auto* routing_fp32 = routing_buffer.precision == Precision::FP32 ? routing_buffer.data_as() : nullptr; - - auto routing_prob = [&](size_t tok, size_t exp) -> float { - const size_t offset = tok * total_num_experts + exp; - if (routing_fp16) return static_cast(routing_fp16[offset]); - return routing_fp32[offset]; - }; - - ensure_moe_buffers(token_count, hidden_dim, expert_intermediate_dim, num_experts, top_k); - - size_t* expert_offsets = moe_expert_offsets_buf.data(); - size_t* expert_tokens_flat = moe_expert_tokens_buf.data(); - - std::memset(expert_offsets, 0, (num_experts + 1) * sizeof(size_t)); - for (size_t tok = 0; tok < token_count; ++tok) { - for (size_t k = 0; k < top_k; ++k) { - float raw_idx = topk_idx[tok * top_k + k]; - if (!std::isfinite(raw_idx)) { - throw std::runtime_error("moe_layer got non-finite expert index"); - } - size_t idx = static_cast(raw_idx + 0.5f); - if (idx >= num_experts) { - throw std::runtime_error("moe_layer got expert index out of range"); - } - expert_offsets[idx + 1]++; - } - } - - for (size_t e = 0; e < num_experts; ++e) { - expert_offsets[e + 1] += expert_offsets[e]; - } - - thread_local std::vector moe_write_cursors; - if (moe_write_cursors.size() < num_experts) moe_write_cursors.resize(num_experts); - std::memcpy(moe_write_cursors.data(), expert_offsets, num_experts * sizeof(size_t)); - - for (size_t tok = 0; tok < token_count; ++tok) { - for (size_t k = 0; k < top_k; ++k) { - size_t idx = static_cast(topk_idx[tok * top_k + k] + 0.5f); - expert_tokens_flat[moe_write_cursors[idx]++] = tok; - } - } - - float* routing_denom = moe_routing_denom_buf.data(); - if (normalize_routing) { - for (size_t tok = 0; tok < token_count; ++tok) { - float sum_probs = 0.0f; - for (size_t k = 0; k < top_k; ++k) { - size_t idx = static_cast(topk_idx[tok * top_k + k] + 0.5f); - sum_probs += routing_prob(tok, idx); - } - routing_denom[tok] = sum_probs + eps; - } - } - - std::memset(output, 0, token_count * hidden_dim * sizeof(__fp16)); - - for (size_t expert_idx = 0; expert_idx < num_experts; ++expert_idx) { - const size_t start = expert_offsets[expert_idx]; - const size_t end = expert_offsets[expert_idx + 1]; - if (start == end) continue; - - const size_t selected_count = end - start; - const size_t* selected_tokens = expert_tokens_flat + start; - - const auto& w1_buffer = get_input(node, 3 + expert_idx, nodes, node_index_map); - const auto& w2_buffer = gated - ? get_input(node, 3 + 2 * num_experts + expert_idx, nodes, node_index_map) - : get_input(node, 3 + num_experts + expert_idx, nodes, node_index_map); - - __fp16* compact_hidden = moe_compact_hidden_buf.data(); - for (size_t i = 0; i < selected_count; ++i) { - std::memcpy(compact_hidden + i * hidden_dim, - hidden + selected_tokens[i] * hidden_dim, - hidden_dim * sizeof(__fp16)); - } - - __fp16* gate = moe_gate_buf.data(); - __fp16* up = moe_up_buf.data(); - __fp16* expert_out = moe_expert_out_buf.data(); - - moe_matmul(compact_hidden, selected_count, hidden_dim, w1_buffer, gate, expert_intermediate_dim); - const bool w1_was_int8 = w1_buffer.is_grouped_int8(); - - switch (activation) { - case Activation::GELU: - cactus_gelu_f16(gate, gate, selected_count * expert_intermediate_dim); - break; - case Activation::GELU_ERF: - cactus_gelu_f16_erf(gate, gate, selected_count * expert_intermediate_dim); - break; - case Activation::RELU: - cactus_relu_f16(gate, gate, selected_count * expert_intermediate_dim); - break; - case Activation::SILU: - default: - cactus_silu_f16(gate, gate, selected_count * expert_intermediate_dim); - break; - } - - if (gated) { - const auto& w3_buffer = get_input(node, 3 + num_experts + expert_idx, nodes, node_index_map); - moe_matmul(compact_hidden, selected_count, hidden_dim, w3_buffer, up, expert_intermediate_dim, w1_was_int8); - cactus_multiply_f16(gate, up, gate, selected_count * expert_intermediate_dim); - } - - moe_matmul(gate, selected_count, expert_intermediate_dim, w2_buffer, expert_out, hidden_dim); - - for (size_t i = 0; i < selected_count; ++i) { - const size_t tok = selected_tokens[i]; - float expert_prob = routing_prob(tok, expert_idx); - if (expert_prob <= 0.0f) continue; - - float route_weight = expert_prob; - if (normalize_routing) { - route_weight = expert_prob / routing_denom[tok]; - } - route_weight *= routed_scaling_factor; - if (expert_scales_fp16) { - route_weight *= static_cast(expert_scales_fp16[expert_idx]); - } - - auto* out_row = output + tok * hidden_dim; - const auto* expert_row = expert_out + i * hidden_dim; - cactus_add_scaled_f16(out_row, expert_row, out_row, hidden_dim, route_weight); - } - } -} - -void compute_rms_norm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& weight_buffer = get_input(node, 1, nodes, node_index_map); - - if (input_buffer.shape.size() != 2) { - throw std::runtime_error("RMS normalization requires 2D input tensor [batch_size, dims], got " + - std::to_string(input_buffer.shape.size()) + "D tensor"); - } - - size_t batch_size = input_buffer.shape[0]; - size_t dims = input_buffer.shape[1]; - - if (input_buffer.precision != Precision::FP16) { - throw std::runtime_error("RMS normalization only supports FP16 precision"); - } - - cactus_rms_norm_f16(input_buffer.data_as<__fp16>(), weight_buffer.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), batch_size, dims, node.params.epsilon); -} - -void compute_rope_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU RoPE operation not yet implemented"); - } - - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& shape = input_buffer.shape; - - if (shape.size() < 4) { - throw std::runtime_error("RoPE operation requires 4D tensor with shape [batch, seq_len, num_heads, head_dim], got " + - std::to_string(shape.size()) + "D tensor"); - } - - if (input_buffer.precision != Precision::FP16 || node.output_buffer.precision != Precision::FP16) { - throw std::runtime_error("RoPE operation only supports FP16 precision"); - } - - size_t batch_size = shape[0]; - size_t seq_len = shape[1]; - size_t num_heads = shape[2]; - size_t head_dim = shape[3]; - - cactus_rope_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, num_heads, head_dim, node.params.position_offset, node.params.theta); -} - -void compute_softmax_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& shape = input_buffer.shape; - - if (shape.size() < 2) { - throw std::runtime_error("Softmax operation requires at least 2D tensor, got " + - std::to_string(shape.size()) + "D tensor"); - } - - if (input_buffer.precision != Precision::FP16) { - throw std::runtime_error("Softmax operation only supports FP16 precision"); - } - - size_t batch_size = 1; - for (size_t i = 0; i < shape.size() - 1; i++) { - batch_size *= shape[i]; - } - size_t vocab_size = shape[shape.size() - 1]; - - cactus_softmax_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - batch_size, 1, vocab_size); -} - -void compute_rel_pos_bias_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.input_ids.size() != 2) { - throw std::runtime_error("REL_POS_BIAS requires 2 inputs (query, relative_key)"); - } - - const auto& q_buffer = get_input(node, 0, nodes, node_index_map); - const auto& r_buffer = get_input(node, 1, nodes, node_index_map); - auto& y_buffer = node.output_buffer; - - if (q_buffer.shape.size() != 4) { - throw std::runtime_error("REL_POS_BIAS query must be [B, T, H, D]"); - } - if (r_buffer.shape.size() != 4) { - throw std::runtime_error("REL_POS_BIAS relative_key must be [B, R, H, D]"); - } - if (q_buffer.precision != Precision::FP16 || r_buffer.precision != Precision::FP16) { - throw std::runtime_error("REL_POS_BIAS currently only supports FP16 tensors"); - } - - const size_t B = q_buffer.shape[0]; - const size_t T = q_buffer.shape[1]; - const size_t H = q_buffer.shape[2]; - const size_t D = q_buffer.shape[3]; - const size_t Rb = r_buffer.shape[0]; - const size_t R = r_buffer.shape[1]; - - if (Rb != 1 && Rb != B) { - throw std::runtime_error("REL_POS_BIAS relative_key batch must be 1 or match query batch"); - } - if (r_buffer.shape[2] != H || r_buffer.shape[3] != D) { - throw std::runtime_error("REL_POS_BIAS expects matching [H, D] between query and relative_key"); - } - if (R < (2 * T - 1)) { - throw std::runtime_error("REL_POS_BIAS requires relative_key length >= 2*T-1"); - } - - const __fp16* q = q_buffer.data_as<__fp16>(); - const __fp16* r = r_buffer.data_as<__fp16>(); - __fp16* y = y_buffer.data_as<__fp16>(); - - const float scale = node.params.scale; - - const size_t q_batch_stride = T * H * D; - const size_t r_batch_stride = R * H * D; - const size_t y_batch_stride = H * T * T; - const size_t q_head_stride = D; - const size_t r_head_stride = D; - const size_t q_time_stride = H * D; - const size_t r_time_stride = H * D; - - CactusThreading::parallel_for(B * H * T, CactusThreading::Thresholds::ATTENTION, - [&](size_t start_idx, size_t end_idx) { - for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { - const size_t b = work_idx / (H * T); - const size_t rem = work_idx % (H * T); - const size_t h = rem / T; - const size_t t = rem % T; - - const size_t rb = (Rb == 1) ? 0 : b; - const __fp16* q_vec = q + b * q_batch_stride + t * q_time_stride + h * q_head_stride; - const __fp16* r_base = r + rb * r_batch_stride + h * r_head_stride; - __fp16* y_row = y + b * y_batch_stride + h * (T * T) + t * T; - - for (size_t j = 0; j < T; ++j) { - const size_t rel_idx = (T - 1) - t + j; - const __fp16* r_vec = r_base + rel_idx * r_time_stride; - - float acc = 0.0f; - for (size_t d = 0; d < D; ++d) { - acc += static_cast(q_vec[d]) * static_cast(r_vec[d]); - } - y_row[j] = static_cast<__fp16>(acc * scale); - } - } - }); -} - -void compute_attention_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU attention operation not yet implemented"); - } - - if (node.input_ids.size() < 3 || node.input_ids.size() > 4) { - throw std::runtime_error("Attention operation requires 3 or 4 inputs (query, key, value[, mask]), got " + - std::to_string(node.input_ids.size()) + " inputs"); - } - - const auto& query_buffer = get_input(node, 0, nodes, node_index_map); - const auto& key_buffer = get_input(node, 1, nodes, node_index_map); - const auto& value_buffer = get_input(node, 2, nodes, node_index_map); - const BufferDesc* mask_buffer = nullptr; - if (node.input_ids.size() == 4) { - mask_buffer = &get_input(node, 3, nodes, node_index_map); - } - const auto& q_shape = query_buffer.shape; - const auto& k_shape = key_buffer.shape; - - if (q_shape.size() < 4) { - throw std::runtime_error("Attention operation requires 4D tensors [batch, seq_len, num_heads, head_dim], got " + - std::to_string(q_shape.size()) + "D tensor"); - } - - if (query_buffer.precision != Precision::FP16) { - throw std::runtime_error("Attention operation only supports FP16 precision"); - } - - size_t batch_size = q_shape[0]; - size_t seq_len = q_shape[1]; - size_t num_q_heads = q_shape[2]; - size_t head_dim = q_shape[3]; - size_t num_kv_heads = k_shape[2]; - size_t kv_seq_len = key_buffer.shape[1]; - size_t v_head_dim = value_buffer.shape[3]; - bool mask_per_head = false; - const __fp16* mask_ptr = nullptr; - - if (mask_buffer) { - if (mask_buffer->precision != Precision::FP16) { - throw std::runtime_error("Attention mask tensor must be FP16"); - } - - if (mask_buffer->shape.size() == 3) { - if (mask_buffer->shape[0] != batch_size || - mask_buffer->shape[1] != seq_len || - mask_buffer->shape[2] != kv_seq_len) { - throw std::runtime_error("Attention mask [B, T, S] shape mismatch"); - } - mask_per_head = false; - } else if (mask_buffer->shape.size() == 4) { - if (mask_buffer->shape[0] != batch_size || - mask_buffer->shape[2] != seq_len || - mask_buffer->shape[3] != kv_seq_len) { - throw std::runtime_error("Attention mask [B, H, T, S] shape mismatch"); - } - if (mask_buffer->shape[1] == 1) { - mask_per_head = false; - } else if (mask_buffer->shape[1] == num_q_heads) { - mask_per_head = true; - } else { - throw std::runtime_error("Attention mask [B, H, T, S] head dimension mismatch"); - } - } else { - throw std::runtime_error("Attention mask must be rank 3 or 4"); - } - - mask_ptr = mask_buffer->data_as<__fp16>(); - } - - cactus_attention_f16(query_buffer.data_as<__fp16>(), key_buffer.data_as<__fp16>(), - value_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, kv_seq_len, num_q_heads, num_kv_heads, head_dim, node.params.scale, mask_ptr, - node.params.position_offset, node.params.window_size, node.params.is_causal, - node.params.attention_mask_is_additive, mask_per_head, v_head_dim, node.params.logit_cap); -} - -void compute_attention_int8_hybrid_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& query_buffer = get_input(node, 0, nodes, node_index_map); - const auto& key_new_buffer = get_input(node, 1, nodes, node_index_map); - const auto& value_new_buffer = get_input(node, 2, nodes, node_index_map); - const auto& q_shape = query_buffer.shape; - - if (q_shape.size() < 4) { - throw std::runtime_error("ATTENTION_INT8_HYBRID requires 4D query tensor"); - } - - size_t batch_size = q_shape[0]; - size_t seq_len = q_shape[1]; - size_t num_q_heads = q_shape[2]; - size_t head_dim = node.params.head_dim; - size_t v_head_dim = node.params.v_head_dim; - size_t num_kv_heads = node.params.num_kv_heads; - size_t cache_len = node.params.cache_seq_len; - size_t new_len = key_new_buffer.shape[1]; - - cactus_attention_hybrid_int8_fp16( - query_buffer.data_as<__fp16>(), - node.params.cached_keys_int8, - node.params.cached_values_int8, - node.params.cached_k_scales, - node.params.cached_v_scales, - key_new_buffer.data_as<__fp16>(), - value_new_buffer.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, cache_len, new_len, - num_q_heads, num_kv_heads, head_dim, - node.params.scale, node.params.position_offset, true, - node.params.window_size, KV_QUANT_GROUP_SIZE, v_head_dim - ); -} - -void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& weight_buffer = get_input(node, 1, nodes, node_index_map); - bool has_bias = node.input_ids.size() > 2; - float epsilon = node.params.epsilon; - - if (input_buffer.shape.empty()) { - throw std::runtime_error("LayerNorm requires non-empty input tensor"); - } - - size_t feature_size = input_buffer.shape.back(); - size_t batch_size = input_buffer.total_size / feature_size; - - if (weight_buffer.total_size != feature_size) { - throw std::runtime_error("LayerNorm weight size mismatch with input feature dimension"); - } - - using BufferDesc = std::remove_reference_t; - const BufferDesc* bias_buffer_ptr = nullptr; - if (has_bias) { - const auto& bias_buffer = get_input(node, 2, nodes, node_index_map); - if (bias_buffer.total_size != feature_size) { - throw std::runtime_error("LayerNorm bias size mismatch with input feature dimension"); - } - bias_buffer_ptr = &bias_buffer; - } - - if (input_buffer.precision == Precision::FP16 && - weight_buffer.precision == Precision::FP16 && - node.output_buffer.precision == Precision::FP16 && - (!has_bias || bias_buffer_ptr->precision == Precision::FP16)) { - cactus_layer_norm_f16( - input_buffer.data_as<__fp16>(), - weight_buffer.data_as<__fp16>(), - has_bias ? bias_buffer_ptr->data_as<__fp16>() : nullptr, - node.output_buffer.data_as<__fp16>(), - batch_size, - feature_size, - epsilon); - return; - } - - std::vector input_float(input_buffer.total_size); - std::vector weight_float(feature_size); - std::vector bias_float(feature_size, 0.0f); - - if (input_buffer.precision == Precision::INT8) { - throw std::runtime_error("LayerNorm currently does not support INT8 input"); - } else if (input_buffer.precision == Precision::FP16) { - const __fp16* input_fp16 = input_buffer.data_as<__fp16>(); - for (size_t i = 0; i < input_buffer.total_size; ++i) { - input_float[i] = static_cast(input_fp16[i]); - } - } else { - std::memcpy(input_float.data(), input_buffer.data_as(), input_buffer.total_size * sizeof(float)); - } - - if (weight_buffer.precision == Precision::INT8) { - throw std::runtime_error("LayerNorm currently does not support INT8 weight"); - } else if (weight_buffer.precision == Precision::FP16) { - const __fp16* weight_fp16 = weight_buffer.data_as<__fp16>(); - for (size_t i = 0; i < feature_size; ++i) { - weight_float[i] = static_cast(weight_fp16[i]); - } - } else { - std::memcpy(weight_float.data(), weight_buffer.data_as(), feature_size * sizeof(float)); - } - - if (has_bias) { - const auto& bias_buffer = *bias_buffer_ptr; - if (bias_buffer.precision == Precision::INT8) { - throw std::runtime_error("LayerNorm currently does not support INT8 bias"); - } else if (bias_buffer.precision == Precision::FP16) { - const __fp16* bias_fp16 = bias_buffer.data_as<__fp16>(); - for (size_t i = 0; i < feature_size; ++i) { - bias_float[i] = static_cast(bias_fp16[i]); - } - } else { - std::memcpy(bias_float.data(), bias_buffer.data_as(), feature_size * sizeof(float)); - } - } - - std::vector output_float(input_buffer.total_size); - for (size_t b = 0; b < batch_size; ++b) { - const float* input_row = input_float.data() + b * feature_size; - float* output_row = output_float.data() + b * feature_size; - - float mean = 0.0f; - for (size_t i = 0; i < feature_size; ++i) { - mean += input_row[i]; - } - mean /= feature_size; - - float variance = 0.0f; - for (size_t i = 0; i < feature_size; ++i) { - float diff = input_row[i] - mean; - variance += diff * diff; - } - variance /= feature_size; - - float std_inv = 1.0f / std::sqrt(variance + epsilon); - for (size_t i = 0; i < feature_size; ++i) { - output_row[i] = (input_row[i] - mean) * std_inv * weight_float[i] + bias_float[i]; - } - } - - if (node.output_buffer.precision == Precision::INT8) { - throw std::runtime_error("LayerNorm currently does not support INT8 output"); - } else if (node.output_buffer.precision == Precision::FP16) { - __fp16* output_fp16 = node.output_buffer.data_as<__fp16>(); - for (size_t i = 0; i < input_buffer.total_size; ++i) { - output_fp16[i] = static_cast<__fp16>(output_float[i]); - } - } else { - std::memcpy(node.output_buffer.data_as(), output_float.data(), input_buffer.total_size * sizeof(float)); - } -} - -void compute_conv1d_causal_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU causal convolution operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) { - throw std::runtime_error("Causal conv requires 3D input [batch, seq_len, in_channels]"); - } - if (W.shape.size() != 3) { - throw std::runtime_error("Weight must be 3D"); - } - - const size_t N = X.shape[0]; - const size_t L = X.shape[1]; - const size_t C_in = X.shape[2]; - const size_t W0 = W.shape[0]; - const size_t W1 = W.shape[1]; - const size_t W2 = W.shape[2]; - const size_t dil = node.params.dilation; - if (dil < 1) throw std::runtime_error("dilation must be >= 1"); - - size_t M = 1; - size_t C_out = 0; - const bool standard_layout = (W1 == 1); - const bool transposed_layout = (W2 == 1); - if ((!standard_layout && !transposed_layout) || (W0 % C_in != 0)) { - throw std::runtime_error("Only depthwise causal convolution is supported currently"); - } - const size_t K = standard_layout ? W2 : W1; - M = W0 / C_in; - C_out = C_in * M; - - Y.shape = { N, L, C_out }; - Y.precision = X.precision; - - auto transpose_depthwise_weights_fp16 = [&](const __fp16* src) { - std::vector<__fp16> transposed(W0 * K); - for (size_t oc = 0; oc < W0; ++oc) { - for (size_t k = 0; k < K; ++k) { - transposed[oc * K + k] = src[(oc * W1 + k) * W2]; - } - } - return transposed; - }; - - if (W.precision == Precision::INT8) { - const size_t W_size = W0 * W1 * W2; - const int8_t* W_int8 = W.data_as(); - - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t K_total = W1 * K; - const size_t group_size = W.group_size; - const size_t num_groups = K_total / group_size; - - for (size_t row = 0; row < W0; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - if (transposed_layout && !standard_layout) { - auto fixed = transpose_depthwise_weights_fp16(W_fp16.data()); - cactus_conv1d_causal_depthwise_f16( - X.data_as<__fp16>(), fixed.data(), Y.data_as<__fp16>(), - N, L, C_in, K, dil); - } else { - cactus_conv1d_causal_depthwise_f16( - X.data_as<__fp16>(), W_fp16.data(), Y.data_as<__fp16>(), - N, L, C_in, K, dil); - } - } else if (W.precision == Precision::FP16) { - if (transposed_layout && !standard_layout) { - auto fixed = transpose_depthwise_weights_fp16(W.data_as<__fp16>()); - cactus_conv1d_causal_depthwise_f16( - X.data_as<__fp16>(), fixed.data(), Y.data_as<__fp16>(), - N, L, C_in, K, dil); - } else { - cactus_conv1d_causal_depthwise_f16( - X.data_as<__fp16>(), W.data_as<__fp16>(), Y.data_as<__fp16>(), - N, L, C_in, K, dil); - } - } else { - throw std::runtime_error("Depthwise causal conv supports INT8/FP16 weights"); - } -} - -void compute_conv1d_k3_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU causal convolution operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) - throw std::runtime_error("Conv requires 3D input [N, C_in, L]!"); - - if (W.shape.size() != 3) - throw std::runtime_error("Weight must be [C_out, C_in, 3]!"); - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t L = X.shape[2]; - - const size_t C_out = W.shape[0]; - const size_t K = W.shape[2]; - const size_t stride = node.params.stride; - - if (K != 3) - throw std::runtime_error("Conv1d_k3 only supports K=3!"); - - size_t L_out = ((L - 1) / stride) + 1; - Y.shape = { N, C_out, L_out }; - Y.precision = X.precision; - - if (X.precision != Precision::FP16) { - throw std::runtime_error("Conv1d_k3 only supports FP16 activations"); - } - - if (W.precision == Precision::INT8) { - const size_t W_size = C_out * C_in * K; - const int8_t* W_int8 = W.data_as(); - - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t K_total = C_in * K; - const size_t group_size = W.group_size; - const size_t num_groups = K_total / group_size; - - for (size_t row = 0; row < C_out; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv1d_f16_k3( - X.data_as<__fp16>(), - W_fp16.data(), - Y.data_as<__fp16>(), - N, L, C_in, C_out, stride - ); - } else if (W.precision == Precision::FP16) { - cactus_conv1d_f16_k3( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - Y.data_as<__fp16>(), - N, L, C_in, C_out, stride - ); - } else { - throw std::runtime_error("Conv1d_k3 only supports FP16 and INT8 weights"); - } -} - -void compute_conv1d_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) { - throw std::runtime_error("conv1d expects input [N, C_in, L]"); - } - if (W.shape.size() != 3) { - throw std::runtime_error("conv1d weight must be [C_out, C_in, K]"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t L = X.shape[2]; - const size_t C_out = W.shape[0]; - const size_t K = W.shape[2]; - const size_t stride = node.params.stride; - - if (W.shape[1] != C_in) { - throw std::runtime_error("conv1d weight C_in mismatch"); - } - - if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { - throw std::runtime_error("Conv1d only supports FP16"); - } - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv1d bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv1d bias only supports FP16/FP32"); - } - } - - cactus_conv1d_f16(X.data_as<__fp16>(), W.data_as<__fp16>(), bias_ptr, - Y.data_as<__fp16>(), N, L, C_in, C_out, K, stride); -} - -void compute_conv1d_same_depthwise_k9_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv1d_same_depthwise_k9 operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) { - throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv1d_same_depthwise_k9 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t L = X.shape[1]; - const size_t C = X.shape[2]; - const size_t K = 9; - - if (W.shape.size() == 2) { - if (W.shape[0] != C || W.shape[1] != K) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); - } - } else if (W.shape.size() == 3) { - if (W.shape[0] != C || W.shape[1] != 1 || W.shape[2] != K) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); - } - } else { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); - } - - Y.shape = {N, L, C}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C) { - throw std::runtime_error("conv1d_same_depthwise_k9 bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv1d_same_depthwise_k9 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - cactus_conv1d_same_depthwise_f16_k9( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t W_size = C * K; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t K_total = K; - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv1d_same_depthwise_k9 requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv1d_same_depthwise_f16_k9( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C - ); - return; - } - - throw std::runtime_error("conv1d_same_depthwise_k9 only supports FP16/INT8 weights"); -} - -void compute_conv2d_k3s2p1_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv2d_k3s2p1 operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d_k3s2p1 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t H = X.shape[2]; - const size_t W_in = X.shape[3]; - const size_t C_out = Y.shape[1]; - - if (H == 0 || W_in == 0) { - throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); - } - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d_k3s2p1 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - if (W.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 FP16 weight must be [C_out, C_in, 3, 3]"); - } - cactus_conv2d_f16_k3s2p1_nchw( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t K_total = C_in * 9; - const size_t W_size = C_out * K_total; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv2d_k3s2p1 requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C_out; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv2d_f16_k3s2p1_nchw( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out - ); - return; - } - - throw std::runtime_error("conv2d_k3s2p1 only supports FP16/INT8 weights"); -} - -void compute_conv2d_depthwise_k3s2p1_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv2d_depthwise_k3s2p1 operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t C = X.shape[1]; - const size_t H = X.shape[2]; - const size_t W_in = X.shape[3]; - if (H == 0 || W_in == 0) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); - } - - if (W.shape.size() == 3) { - if (W.shape[0] != C || W.shape[1] != 3 || W.shape[2] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); - } - } else if (W.shape.size() == 4) { - if (W.shape[0] != C || W.shape[1] != 1 || W.shape[2] != 3 || W.shape[3] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); - } - } else { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); - } - - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W_in - 1) / 2 + 1; - Y.shape = {N, C, H_out, W_out}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d_depthwise_k3s2p1 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - cactus_conv2d_depthwise_f16_k3s2p1_nchw( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, C, H, W_in - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t K_total = 9; - const size_t W_size = C * K_total; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv2d_depthwise_k3s2p1 requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv2d_depthwise_f16_k3s2p1_nchw( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, C, H, W_in - ); - return; - } - - throw std::runtime_error("conv2d_depthwise_k3s2p1 only supports FP16/INT8 weights"); -} - -void compute_conv2d_pointwise_1x1_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv2d_pointwise_1x1 operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d_pointwise_1x1 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t H = X.shape[2]; - const size_t W_in = X.shape[3]; - if (H == 0 || W_in == 0) { - throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); - } - - size_t C_out = 0; - if (W.shape.size() == 2) { - C_out = W.shape[0]; - if (W.shape[1] != C_in) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); - } - } else if (W.shape.size() == 4) { - C_out = W.shape[0]; - if (W.shape[1] != C_in || W.shape[2] != 1 || W.shape[3] != 1) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); - } - } else { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); - } - - Y.shape = {N, C_out, H, W_in}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv2d_pointwise_1x1 bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d_pointwise_1x1 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - cactus_conv2d_pointwise_f16_1x1_nchw_gemm( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t K_total = C_in; - const size_t W_size = C_out * K_total; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv2d_pointwise_1x1 requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C_out; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv2d_pointwise_f16_1x1_nchw_gemm( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out - ); - return; - } - - throw std::runtime_error("conv2d_pointwise_1x1 only supports FP16/INT8 weights"); -} - -void compute_conv1d_pointwise_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv1d_pointwise operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) { - throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv1d_pointwise only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t L = X.shape[1]; - const size_t C_in = X.shape[2]; - - size_t C_out = 0; - if (W.shape.size() == 2) { - C_out = W.shape[0]; - if (W.shape[1] != C_in) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); - } - } else if (W.shape.size() == 3) { - C_out = W.shape[0]; - if (W.shape[1] != C_in || W.shape[2] != 1) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); - } - } else { - throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); - } - - Y.shape = {N, L, C_out}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv1d_pointwise bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv1d_pointwise bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - cactus_conv1d_pointwise_f16_gemm( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C_in, C_out - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t K_total = C_in; - const size_t W_size = C_out * K_total; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv1d_pointwise requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C_out; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv1d_pointwise_f16_gemm( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C_in, C_out - ); - return; - } - - throw std::runtime_error("conv1d_pointwise only supports FP16/INT8 weights"); -} - -void compute_glu_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - auto& Y = node.output_buffer; - - if (X.shape.empty()) { - throw std::runtime_error("GLU expects non-scalar input"); - } - - int axis = node.params.axis; - if (axis < 0) axis += static_cast(X.shape.size()); - if (axis < 0 || static_cast(axis) >= X.shape.size()) { - throw std::runtime_error("GLU axis out of range"); - } - - const size_t axis_size = X.shape[static_cast(axis)]; - if ((axis_size % 2) != 0) { - throw std::runtime_error("GLU split dimension must be even"); - } - const size_t split = axis_size / 2; - - size_t outer = 1; - for (int i = 0; i < axis; ++i) { - outer *= X.shape[static_cast(i)]; - } - size_t inner = 1; - for (size_t i = static_cast(axis) + 1; i < X.shape.size(); ++i) { - inner *= X.shape[i]; - } - - std::vector out_shape = X.shape; - out_shape[static_cast(axis)] = split; - Y.shape = out_shape; - Y.precision = X.precision; - - if (X.precision == Precision::FP16) { - cactus_glu_f16(X.data_as<__fp16>(), Y.data_as<__fp16>(), outer, split, inner); - return; - } - - if (X.precision == Precision::FP32) { - cactus_glu_f32(X.data_as(), Y.data_as(), outer, split, inner); - return; - } - - throw std::runtime_error("GLU only supports FP16/FP32"); -} - -void compute_batchnorm_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.input_ids.size() != 5) { - throw std::runtime_error("BatchNorm expects 5 inputs: input, weight, bias, running_mean, running_var"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const auto& B = get_input(node, 2, nodes, node_index_map); - const auto& RM = get_input(node, 3, nodes, node_index_map); - const auto& RV = get_input(node, 4, nodes, node_index_map); - auto& Y = node.output_buffer; - - if (X.shape.empty()) { - throw std::runtime_error("BatchNorm expects non-scalar input"); - } - - int axis = node.params.axis; - if (axis < 0) axis += static_cast(X.shape.size()); - if (axis < 0 || static_cast(axis) >= X.shape.size()) { - throw std::runtime_error("BatchNorm axis out of range"); - } - - const size_t C = X.shape[static_cast(axis)]; - if (W.total_size != C || B.total_size != C || RM.total_size != C || RV.total_size != C) { - throw std::runtime_error("BatchNorm parameter size mismatch"); - } - - auto load_1d_float = [C](const BufferDesc& buf, const char* name) -> std::vector { - if (buf.total_size != C) { - throw std::runtime_error(std::string("BatchNorm parameter size mismatch for ") + name); - } - std::vector out(C); - if (buf.precision == Precision::FP16) { - const __fp16* p = buf.data_as<__fp16>(); - for (size_t i = 0; i < C; ++i) out[i] = static_cast(p[i]); - } else if (buf.precision == Precision::FP32) { - std::memcpy(out.data(), buf.data_as(), C * sizeof(float)); - } else { - throw std::runtime_error(std::string("BatchNorm parameter ") + name + " must be FP16 or FP32"); - } - return out; - }; - - const std::vector gamma = load_1d_float(W, "weight"); - const std::vector beta = load_1d_float(B, "bias"); - const std::vector mean = load_1d_float(RM, "running_mean"); - const std::vector var = load_1d_float(RV, "running_var"); - - size_t outer = 1; - for (int i = 0; i < axis; ++i) { - outer *= X.shape[static_cast(i)]; - } - size_t inner = 1; - for (size_t i = static_cast(axis) + 1; i < X.shape.size(); ++i) { - inner *= X.shape[i]; - } - - Y.shape = X.shape; - Y.precision = X.precision; - - if (X.precision == Precision::FP16) { - cactus_batchnorm_f16( - X.data_as<__fp16>(), - gamma.data(), - beta.data(), - mean.data(), - var.data(), - Y.data_as<__fp16>(), - outer, - C, - inner, - node.params.epsilon - ); - return; - } - - if (X.precision == Precision::FP32) { - cactus_batchnorm_f32( - X.data_as(), - gamma.data(), - beta.data(), - mean.data(), - var.data(), - Y.data_as(), - outer, - C, - inner, - node.params.epsilon - ); - return; - } - - throw std::runtime_error("BatchNorm only supports FP16/FP32 activations"); -} - -void compute_stft_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - auto& Y = node.output_buffer; - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t L = X.shape[2]; - const size_t C_out = W.shape[0]; - const size_t K = W.shape[2]; - const size_t stride = node.params.stride; - const size_t num_fft_bins = node.params.num_fft_bins; - - if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { - throw std::runtime_error("stft only supports FP16"); - } - - cactus_stft_f16(X.data_as<__fp16>(), W.data_as<__fp16>(), - Y.data_as<__fp16>(), N, L, C_in, C_out, K, stride, num_fft_bins); -} - -void compute_conv1d_k7s3_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - - auto& Y = node.output_buffer; - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t L = X.shape[2]; - - if (W.shape.size() != 3) throw std::runtime_error("Weight must be 3D"); - const size_t C_in_W = W.shape[0]; - const size_t K = W.shape[1]; - const size_t C_out = W.shape[2]; - const size_t stride = node.params.stride; - - if (C_in != C_in_W) throw std::runtime_error("Channel mismatch in conv1d_k7s3"); - if (K != 7 || stride != 3) throw std::runtime_error("conv1d_k7s3 requires K=7, stride=3"); - - if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { - throw std::runtime_error("Conv1d specialized only supports FP16"); - } - - size_t L_out = (L < 7) ? 0 : (L - 7) / 3 + 1; - Y.shape = {N, C_out, L_out}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv1d_k7s3 bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv1d_k7s3 bias only supports FP16/FP32"); - } - } - - cactus_conv1d_f16_k7s3_oc8( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C_in, C_out - ); -} - -void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.input_ids.size() != 6) { - throw std::runtime_error("GATED_DELTANET_DECODE expects 6 inputs"); - } - - const auto& q = get_input(node, 0, nodes, node_index_map); - const auto& k = get_input(node, 1, nodes, node_index_map); - const auto& v = get_input(node, 2, nodes, node_index_map); - const auto& g = get_input(node, 3, nodes, node_index_map); - const auto& b = get_input(node, 4, nodes, node_index_map); - const auto& s = get_input(node, 5, nodes, node_index_map); - - validate_gated_deltanet_inputs(q, k, v, g, b, s); - if (q.shape[1] != 1) { - throw std::runtime_error("GATED_DELTANET_DECODE expects T=1"); - } - - const size_t B = q.shape[0]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - const size_t Hv = v.shape[2]; - const size_t V = v.shape[3]; - const size_t qk_heads_from_params = node.params.num_kv_heads; - if (qk_heads_from_params != 0 && qk_heads_from_params != Hq) { - throw std::runtime_error("GATED_DELTANET_DECODE num_qk_heads param mismatch"); - } - - std::vector<__fp16> q_cast; - std::vector<__fp16> k_cast; - std::vector<__fp16> v_cast; - std::vector<__fp16> g_cast; - std::vector<__fp16> b_cast; - std::vector<__fp16> s_cast; - const __fp16* q_data = as_fp16_ptr(q, q_cast); - const __fp16* k_data = as_fp16_ptr(k, k_cast); - const __fp16* v_data = as_fp16_ptr(v, v_cast); - const __fp16* g_data = as_fp16_ptr(g, g_cast); - const __fp16* b_data = as_fp16_ptr(b, b_cast); - const __fp16* s_data = as_fp16_ptr(s, s_cast); - __fp16* out = node.output_buffer.data_as<__fp16>(); - - cactus_gated_deltanet_decode_f16( - q_data, k_data, v_data, g_data, b_data, s_data, out, - B, Hq, Hv, K, V, node.params.scale); -} - -void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.input_ids.size() != 6) { - throw std::runtime_error("GATED_DELTANET_PREFILL expects 6 inputs"); - } - - const auto& q = get_input(node, 0, nodes, node_index_map); - const auto& k = get_input(node, 1, nodes, node_index_map); - const auto& v = get_input(node, 2, nodes, node_index_map); - const auto& g = get_input(node, 3, nodes, node_index_map); - const auto& b = get_input(node, 4, nodes, node_index_map); - const auto& s = get_input(node, 5, nodes, node_index_map); - - validate_gated_deltanet_inputs(q, k, v, g, b, s); - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - const size_t Hv = v.shape[2]; - const size_t V = v.shape[3]; - const size_t qk_heads_from_params = node.params.num_kv_heads; - if (qk_heads_from_params != 0 && qk_heads_from_params != Hq) { - throw std::runtime_error("GATED_DELTANET_PREFILL num_qk_heads param mismatch"); - } - - std::vector<__fp16> q_cast; - std::vector<__fp16> k_cast; - std::vector<__fp16> v_cast; - std::vector<__fp16> g_cast; - std::vector<__fp16> b_cast; - std::vector<__fp16> s_cast; - const __fp16* q_data = as_fp16_ptr(q, q_cast); - const __fp16* k_data = as_fp16_ptr(k, k_cast); - const __fp16* v_data = as_fp16_ptr(v, v_cast); - const __fp16* g_data = as_fp16_ptr(g, g_cast); - const __fp16* b_data = as_fp16_ptr(b, b_cast); - const __fp16* s_data = as_fp16_ptr(s, s_cast); - __fp16* out = node.output_buffer.data_as<__fp16>(); - - cactus_gated_deltanet_prefill_f16( - q_data, k_data, v_data, g_data, b_data, s_data, out, - B, T, Hq, Hv, K, V, node.params.chunk_size, node.params.scale); -} - -void compute_rope_gptj_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& shape = input_buffer.shape; - - size_t batch_size = shape[0]; - size_t seq_len = shape[1]; - size_t num_heads = shape[2]; - size_t head_dim = shape[3]; - size_t rot_dim = static_cast(node.params.scalar); - - cactus_gpt_j_rope_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, num_heads, head_dim, rot_dim, - node.params.position_offset, node.params.theta); -} - -void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const auto& weight = get_input(node, 1, nodes, node_index_map); - const auto& bias = get_input(node, 2, nodes, node_index_map); - float epsilon = node.params.epsilon; - - size_t batch_size = input.shape[0]; - size_t channels = input.shape[1]; - size_t spatial_size = 1; - for (size_t i = 2; i < input.shape.size(); ++i) spatial_size *= input.shape[i]; - - size_t num_groups = node.params.num_groups; - if (num_groups == 0) num_groups = 32; - - if (channels % num_groups != 0) { - throw std::runtime_error("GroupNorm: channels must be divisible by num_groups"); - } - - size_t channels_per_group = channels / num_groups; - - const __fp16* src = input.data_as<__fp16>(); - const __fp16* w = weight.data_as<__fp16>(); - const __fp16* b = bias.data_as<__fp16>(); - __fp16* dst = node.output_buffer.data_as<__fp16>(); - - for (size_t n = 0; n < batch_size; ++n) { - for (size_t g = 0; g < num_groups; ++g) { - float sum = 0.0f, sum_sq = 0.0f; - size_t count = 0; - - for (size_t c = 0; c < channels_per_group; ++c) { - size_t ch = g * channels_per_group + c; - for (size_t s = 0; s < spatial_size; ++s) { - size_t idx = n * channels * spatial_size + ch * spatial_size + s; - float val = static_cast(src[idx]); - sum += val; - sum_sq += val * val; - count++; - } - } - - float mean = sum / count; - float var = (sum_sq / count) - (mean * mean); - float inv_std = 1.0f / std::sqrt(var + epsilon); - - for (size_t c = 0; c < channels_per_group; ++c) { - size_t ch = g * channels_per_group + c; - float wt = static_cast(w[ch]); - float bi = static_cast(b[ch]); - - for (size_t s = 0; s < spatial_size; ++s) { - size_t idx = n * channels * spatial_size + ch * spatial_size + s; - float val = static_cast(src[idx]); - dst[idx] = static_cast<__fp16>((val - mean) * inv_std * wt + bi); - } - } - } - } -} - -void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& h_prev_buffer = get_input(node, 1, nodes, node_index_map); - const auto& c_prev_buffer = get_input(node, 2, nodes, node_index_map); - const auto& weight_ih_buffer = get_input(node, 3, nodes, node_index_map); - const auto& weight_hh_buffer = get_input(node, 4, nodes, node_index_map); - const auto& bias_ih_buffer = get_input(node, 5, nodes, node_index_map); - const auto& bias_hh_buffer = get_input(node, 6, nodes, node_index_map); - - if (input_buffer.precision != Precision::FP16 || h_prev_buffer.precision != Precision::FP16 || - c_prev_buffer.precision != Precision::FP16 || weight_ih_buffer.precision != Precision::FP16 || - weight_hh_buffer.precision != Precision::FP16 || bias_ih_buffer.precision != Precision::FP16 || - bias_hh_buffer.precision != Precision::FP16) { - throw std::runtime_error("LSTM cell requires all inputs to be FP16"); - } - - if (input_buffer.shape.size() != 2 || h_prev_buffer.shape.size() != 2 || c_prev_buffer.shape.size() != 2) { - throw std::runtime_error("LSTM cell input/state shapes must be 2D [batch, features]"); - } - - const size_t batch_size = input_buffer.shape[0]; - const size_t input_size = input_buffer.shape[1]; - const size_t hidden_size = h_prev_buffer.shape[1]; - - const __fp16* x_input = input_buffer.data_as<__fp16>(); - const __fp16* h_prev = h_prev_buffer.data_as<__fp16>(); - const __fp16* c_prev = c_prev_buffer.data_as<__fp16>(); - const __fp16* weight_ih = weight_ih_buffer.data_as<__fp16>(); - const __fp16* weight_hh = weight_hh_buffer.data_as<__fp16>(); - const __fp16* bias_ih = bias_ih_buffer.data_as<__fp16>(); - const __fp16* bias_hh = bias_hh_buffer.data_as<__fp16>(); - - node.output_buffer.shape = {batch_size, hidden_size, 2}; - node.output_buffer.total_size = batch_size * hidden_size * 2; - node.output_buffer.precision = Precision::FP16; - node.output_buffer.allocate(); - - std::vector<__fp16> h_new_temp(batch_size * hidden_size); - std::vector<__fp16> c_new_temp(batch_size * hidden_size); - - cactus_lstm_cell_f16( - x_input, h_prev, c_prev, - weight_ih, weight_hh, - bias_ih, bias_hh, - h_new_temp.data(), c_new_temp.data(), - batch_size, input_size, hidden_size - ); - - __fp16* output = node.output_buffer.data_as<__fp16>(); - for (size_t b = 0; b < batch_size; ++b) { - for (size_t i = 0; i < hidden_size; ++i) { - const size_t idx = b * hidden_size + i; - output[b * hidden_size * 2 + i * 2] = h_new_temp[idx]; - output[b * hidden_size * 2 + i * 2 + 1] = c_new_temp[idx]; - } - } -} - -void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - size_t n = node.params.num_altup_inputs; - const auto& coefs_buf = get_input(node, 0, nodes, node_index_map); - - std::vector stream_ptrs(n); - for (size_t i = 0; i < n; i++) { - stream_ptrs[i] = get_input(node, 1 + i, nodes, node_index_map).data_as<__fp16>(); - } - - const auto& stream0_buf = get_input(node, 1, nodes, node_index_map); - size_t seq_len = stream0_buf.shape[0]; - size_t hidden_dim = stream0_buf.shape[1]; - - cactus_altup_predict_f16( - coefs_buf.data_as<__fp16>(), - stream_ptrs.data(), - node.output_buffer.data_as<__fp16>(), - n, seq_len, hidden_dim); -} - -void compute_gaussian_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buf = get_input(node, 0, nodes, node_index_map); - const __fp16* input = input_buf.data_as<__fp16>(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - - size_t rows = input_buf.shape[0]; - size_t cols = input_buf.shape[1]; - float ppf = node.params.scalar; - - cactus_gaussian_topk_f16(input, output, rows, cols, ppf); -} - -void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - size_t n = node.params.num_altup_inputs; - const auto& coefs_buf = get_input(node, 0, nodes, node_index_map); - const auto& innov_buf = get_input(node, 1, nodes, node_index_map); - - std::vector pred_ptrs(n); - for (size_t i = 0; i < n; i++) { - pred_ptrs[i] = get_input(node, 2 + i, nodes, node_index_map).data_as<__fp16>(); - } - - size_t seq_len = innov_buf.shape[0]; - size_t hidden_dim = innov_buf.shape[1]; - - cactus_altup_correct_f16( - coefs_buf.data_as<__fp16>(), - innov_buf.data_as<__fp16>(), - pred_ptrs.data(), - node.output_buffer.data_as<__fp16>(), - n, seq_len, hidden_dim); -} - -void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const auto& w_ih_fwd = get_input(node, 1, nodes, node_index_map); - const auto& w_hh_fwd = get_input(node, 2, nodes, node_index_map); - const auto& b_ih_fwd = get_input(node, 3, nodes, node_index_map); - const auto& b_hh_fwd = get_input(node, 4, nodes, node_index_map); - const auto& w_ih_bwd = get_input(node, 5, nodes, node_index_map); - const auto& w_hh_bwd = get_input(node, 6, nodes, node_index_map); - const auto& b_ih_bwd = get_input(node, 7, nodes, node_index_map); - const auto& b_hh_bwd = get_input(node, 8, nodes, node_index_map); - - size_t batch_size = input.shape[0]; - size_t seq_len = input.shape[1]; - size_t input_size = input.shape[2]; - size_t hidden_size = w_ih_fwd.shape[0] / 4; - - cactus_bilstm_sequence_f16( - input.data_as<__fp16>(), - w_ih_fwd.data_as<__fp16>(), w_hh_fwd.data_as<__fp16>(), - b_ih_fwd.data_as<__fp16>(), b_hh_fwd.data_as<__fp16>(), - w_ih_bwd.data_as<__fp16>(), w_hh_bwd.data_as<__fp16>(), - b_ih_bwd.data_as<__fp16>(), b_hh_bwd.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, input_size, hidden_size); -} - -void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - - size_t batch_size = input.shape[0]; - size_t channels = input.shape[1]; - size_t input_length = input.shape[2]; - size_t kernel_size = node.params.kernel_size; - size_t stride = node.params.stride; - - cactus_maxpool1d_f16( - input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, channels, input_length, - kernel_size, stride); -} - -void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d_k3s1p1 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t H = X.shape[2]; - const size_t W_in = X.shape[3]; - const size_t C_out = Y.shape[1]; - - if (H == 0 || W_in == 0) { - throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); - } - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d_k3s1p1 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - if (W.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 FP16 weight must be [C_out, C_in, 3, 3]"); - } - cactus_conv2d_f16_k3s1p1_nchw( - X.data_as<__fp16>(), W.data_as<__fp16>(), bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out); - return; - } - - throw std::runtime_error("conv2d_k3s1p1 only supports FP16 weights"); -} - -void compute_conv2d_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv2d operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d expects input [N, C_in, H, W]"); - } - if (W.shape.size() != 4) { - throw std::runtime_error("conv2d weight must be [C_out, C_in/groups, K_h, K_w]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d only supports FP16 activations"); - } - - const size_t stride = node.params.stride; - const size_t padding = node.params.padding; - const size_t dilation = node.params.dilation; - const size_t groups = node.params.num_groups; - if (stride == 0) { - throw std::runtime_error("conv2d stride must be > 0"); - } - if (dilation == 0) { - throw std::runtime_error("conv2d dilation must be > 0"); - } - if (groups == 0) { - throw std::runtime_error("conv2d groups must be > 0"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t H_in = X.shape[2]; - const size_t W_in = X.shape[3]; - const size_t C_out = W.shape[0]; - const size_t C_in_per_group = W.shape[1]; - const size_t K_h = W.shape[2]; - const size_t K_w = W.shape[3]; - - if ((C_in % groups) != 0) { - throw std::runtime_error("conv2d input channels must be divisible by groups"); - } - if ((C_out % groups) != 0) { - throw std::runtime_error("conv2d output channels must be divisible by groups"); - } - if (C_in_per_group * groups != C_in) { - throw std::runtime_error("conv2d weight must match [C_out, C_in/groups, K_h, K_w]"); - } - - const size_t effective_kh = dilation * (K_h - 1) + 1; - const size_t effective_kw = dilation * (K_w - 1) + 1; - if (H_in + 2 * padding < effective_kh || W_in + 2 * padding < effective_kw) { - throw std::runtime_error("conv2d input spatial dimensions are too small for the requested kernel"); - } - - const size_t H_out_expected = (H_in + 2 * padding - effective_kh) / stride + 1; - const size_t W_out_expected = (W_in + 2 * padding - effective_kw) / stride + 1; - if (Y.shape.size() != 4 || Y.shape[0] != N || Y.shape[1] != C_out || - Y.shape[2] != H_out_expected || Y.shape[3] != W_out_expected) { - throw std::runtime_error("conv2d output buffer shape mismatch"); - } - Y.precision = Precision::FP16; - - const __fp16* x_ptr = X.data_as<__fp16>(); - __fp16* y_ptr = Y.data_as<__fp16>(); - - auto finite_check_enabled = []() -> bool { - static const bool enabled = []() { - const char* env = std::getenv("CACTUS_CHECK_FINITE"); - return env != nullptr && env[0] != '\0' && env[0] != '0'; - }(); - return enabled; - }; - - auto throw_nonfinite_fp16 = [&](const char* label, const __fp16* data, size_t count) { - if (!finite_check_enabled() || data == nullptr) { - return; - } - for (size_t i = 0; i < count; ++i) { - const float value = static_cast(data[i]); - if (!std::isfinite(value)) { - throw std::runtime_error( - std::string("conv2d detected non-finite ") + label + - " at flat index " + std::to_string(i) + - " value=" + std::to_string(value) - ); - } - } - }; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv2d bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d bias only supports FP16/FP32"); - } - } - - const size_t kernel_elems_per_out = C_in_per_group * K_h * K_w; - const __fp16* w_ptr = nullptr; - std::vector<__fp16> w_fp16; - if (W.precision == Precision::FP16) { - w_ptr = W.data_as<__fp16>(); - } else if (W.precision == Precision::INT8) { - const int8_t* w_int8 = W.data_as(); - w_fp16.resize(C_out * kernel_elems_per_out); - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (kernel_elems_per_out % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv2d requires valid per-group scales"); - } - const size_t num_qgroups = kernel_elems_per_out / group_size; - for (size_t oc = 0; oc < C_out; ++oc) { - for (size_t col = 0; col < kernel_elems_per_out; ++col) { - const size_t idx = oc * kernel_elems_per_out + col; - const size_t qgroup = col / group_size; - const float scale = static_cast(scales[oc * num_qgroups + qgroup]); - w_fp16[idx] = static_cast<__fp16>(w_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < w_fp16.size(); ++i) { - w_fp16[i] = static_cast<__fp16>(w_int8[i]); - } - } - w_ptr = w_fp16.data(); - } else { - throw std::runtime_error("conv2d only supports FP16/INT8 weights"); - } - - const size_t C_out_per_group = C_out / groups; - const size_t C_in_group = C_in / groups; - const size_t H_out = Y.shape[2]; - const size_t W_out = Y.shape[3]; - const size_t total_work = N * C_out; - - throw_nonfinite_fp16("input activation", x_ptr, X.total_size); - throw_nonfinite_fp16("weight", w_ptr, C_out * kernel_elems_per_out); - if (bias_ptr != nullptr) { - throw_nonfinite_fp16("bias", bias_ptr, C_out); - } - - CactusThreading::parallel_for( - total_work, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t work_start, size_t work_end) { - for (size_t work_idx = work_start; work_idx < work_end; ++work_idx) { - const size_t n = work_idx / C_out; - const size_t oc = work_idx % C_out; - const size_t group = oc / C_out_per_group; - const size_t ic_base = group * C_in_group; - const size_t w_oc_base = oc * kernel_elems_per_out; - - for (size_t oh = 0; oh < H_out; ++oh) { - for (size_t ow = 0; ow < W_out; ++ow) { - float acc = bias_ptr ? static_cast(bias_ptr[oc]) : 0.0f; - const size_t ih_origin = oh * stride; - const size_t iw_origin = ow * stride; - - for (size_t ic_local = 0; ic_local < C_in_group; ++ic_local) { - const size_t ic = ic_base + ic_local; - const size_t w_ic_base = w_oc_base + ic_local * K_h * K_w; - - for (size_t kh = 0; kh < K_h; ++kh) { - const long long ih = static_cast(ih_origin) + - static_cast(kh * dilation) - - static_cast(padding); - if (ih < 0 || ih >= static_cast(H_in)) { - continue; - } - - for (size_t kw = 0; kw < K_w; ++kw) { - const long long iw = static_cast(iw_origin) + - static_cast(kw * dilation) - - static_cast(padding); - if (iw < 0 || iw >= static_cast(W_in)) { - continue; - } - - const size_t x_idx = ((n * C_in + ic) * H_in + static_cast(ih)) * W_in + static_cast(iw); - const size_t w_idx = w_ic_base + kh * K_w + kw; - acc += static_cast(x_ptr[x_idx]) * static_cast(w_ptr[w_idx]); - } - } - } - - const size_t y_idx = ((n * C_out + oc) * H_out + oh) * W_out + ow; - y_ptr[y_idx] = static_cast<__fp16>(acc); - } - } - } - } - ); - - throw_nonfinite_fp16("output activation", y_ptr, Y.total_size); -} - -void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const __fp16* src = input.data_as<__fp16>(); - __fp16* dst = node.output_buffer.data_as<__fp16>(); - - size_t batch = input.shape[0]; - size_t total_per_batch = input.total_size / batch; - size_t T = input.shape.back(); - size_t features = total_per_batch / T; - - for (size_t b = 0; b < batch; ++b) { - const __fp16* batch_src = src + b * total_per_batch; - __fp16* batch_dst = dst + b * features * 2; - - for (size_t f = 0; f < features; ++f) { - float sum = 0.0f, sum_sq = 0.0f; - for (size_t t = 0; t < T; ++t) { - float v = static_cast(batch_src[f * T + t]); - sum += v; - sum_sq += v * v; - } - float mean = sum / static_cast(T); - float var = T > 1 ? (sum_sq - static_cast(T) * mean * mean) / static_cast(T - 1) : 0.0f; - float std_val = sqrtf(fmaxf(var, 0.0f)); - batch_dst[f] = static_cast<__fp16>(mean); - batch_dst[features + f] = static_cast<__fp16>(std_val); - } - } -} - -void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const auto& weight_buf = get_input(node, 1, nodes, node_index_map); - const __fp16* src = input.data_as<__fp16>(); - const float* weights = weight_buf.data_as(); - __fp16* dst = node.output_buffer.data_as<__fp16>(); - - size_t batch = input.shape[0]; - size_t total_per_batch = input.total_size / batch; - size_t T = input.shape.back(); - size_t features = total_per_batch / T; - - constexpr float eps = 1e-8f; - - for (size_t b = 0; b < batch; ++b) { - const __fp16* batch_src = src + b * total_per_batch; - const float* batch_w = weights + b * T; - __fp16* batch_dst = dst + b * features * 2; - - float v1 = 0.0f, v2 = 0.0f; - for (size_t t = 0; t < T; ++t) { - float w = batch_w[t]; - v1 += w; - v2 += w * w; - } - float v1_safe = v1 + eps; - float var_denom = v1_safe - v2 / v1_safe + eps; - - for (size_t f = 0; f < features; ++f) { - float wsum = 0.0f; - for (size_t t = 0; t < T; ++t) { - wsum += static_cast(batch_src[f * T + t]) * batch_w[t]; - } - float mean = wsum / v1_safe; - - float wvar = 0.0f; - for (size_t t = 0; t < T; ++t) { - float dx = static_cast(batch_src[f * T + t]) - mean; - wvar += batch_w[t] * dx * dx; - } - float std_val = sqrtf(fmaxf(wvar / var_denom, 0.0f)); - - batch_dst[f] = static_cast<__fp16>(mean); - batch_dst[features + f] = static_cast<__fp16>(std_val); - } - } -} diff --git a/cactus-transpile/graph/graph_ops_sample.cpp b/cactus-transpile/graph/graph_ops_sample.cpp deleted file mode 100644 index 26f6b621f..000000000 --- a/cactus-transpile/graph/graph_ops_sample.cpp +++ /dev/null @@ -1,134 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include - -void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& logits_buffer = get_input(node, 0, nodes, node_index_map); - - float temperature = node.params.temperature; - float top_p = node.params.top_p; - float min_p = node.params.min_p; - float repetition_penalty = node.params.repetition_penalty; - size_t top_k = node.params.top_k; - size_t random_seed = node.params.random_seed; - - const float* bias_values = node.params.bias_values.empty() ? nullptr : node.params.bias_values.data(); - const uint32_t* bias_indices = node.params.bias_indices.empty() ? nullptr : node.params.bias_indices.data(); - size_t bias_count = node.params.bias_values.size(); - - if (logits_buffer.shape.size() != 2) { - throw std::runtime_error("Sample expects 2D logits tensor [seq_len, vocab_size]"); - } - - size_t seq_len = logits_buffer.shape[0]; - size_t vocab_size = logits_buffer.shape[1]; - size_t last_token_offset = (seq_len - 1) * vocab_size; - - if (logits_buffer.precision == Precision::FP16) { - const __fp16* logits_fp16 = logits_buffer.data_as<__fp16>(); - cactus_sample_f16_ex(logits_fp16 + last_token_offset, node.output_buffer.data_as(), - vocab_size, temperature, top_p, min_p, repetition_penalty, top_k, random_seed, - bias_values, bias_indices, bias_count); - } else { - const float* logits_fp32 = logits_buffer.data_as(); - cactus_sample_f32_ex(logits_fp32 + last_token_offset, node.output_buffer.data_as(), - vocab_size, temperature, top_p, min_p, repetition_penalty, top_k, random_seed, - bias_values, bias_indices, bias_count); - } -} - -void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - if (input_buffer.shape.size() != 2) { - throw std::runtime_error("TopK currently only supports 2D tensors [batch, features]"); - } - - size_t k = node.params.top_k; - size_t batch_size = input_buffer.shape[0]; - size_t feature_size = input_buffer.shape[1]; - size_t block_size = batch_size * k; - - std::vector input_float(input_buffer.total_size); - if (input_buffer.precision == Precision::INT8) { - throw std::runtime_error("TopK currently does not support INT8 input"); - } else if (input_buffer.precision == Precision::FP16) { - const __fp16* input_fp16 = input_buffer.data_as<__fp16>(); - for (size_t i = 0; i < input_buffer.total_size; ++i) { - input_float[i] = static_cast(input_fp16[i]); - } - } else { - const float* input_fp32 = input_buffer.data_as(); - std::memcpy(input_float.data(), input_fp32, input_buffer.total_size * sizeof(float)); - } - - float* output = node.output_buffer.data_as(); - - for (size_t b = 0; b < batch_size; ++b) { - const float* row = input_float.data() + b * feature_size; - - std::vector> indexed_values(feature_size); - for (size_t i = 0; i < feature_size; ++i) { - indexed_values[i] = {i, row[i]}; - } - - std::partial_sort(indexed_values.begin(), - indexed_values.begin() + k, - indexed_values.end(), - [](const auto& a, const auto& b) { return a.second > b.second; }); - - float* idx_out_row = output + b * k; - float* val_out_row = output + block_size + b * k; - for (size_t i = 0; i < k; ++i) { - idx_out_row[i] = static_cast(indexed_values[i].first); - val_out_row[i] = indexed_values[i].second; - } - } -} - -void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& indices_buffer = get_input(node, 0, nodes, node_index_map); - const auto& values_buffer = get_input(node, 1, nodes, node_index_map); - - if (indices_buffer.shape != values_buffer.shape) { - throw std::runtime_error("ScatterTopK requires indices and values with identical shapes"); - } - if (indices_buffer.shape.size() != 2) { - throw std::runtime_error("ScatterTopK currently supports 2D tensors"); - } - - size_t batch_size = indices_buffer.shape[0]; - size_t top_k = indices_buffer.shape[1]; - size_t num_classes = node.params.num_classes; - - if (num_classes == 0) { - throw std::runtime_error("ScatterTopK requires num_classes > 0"); - } - - float* output = node.output_buffer.data_as(); - std::fill(output, output + num_classes * batch_size, 0.0f); - - if (indices_buffer.precision != Precision::FP32 || values_buffer.precision != Precision::FP32) { - throw std::runtime_error("ScatterTopK currently expects FP32 inputs"); - } - - const float* indices_data = indices_buffer.data_as(); - const float* values_data = values_buffer.data_as(); - - for (size_t b = 0; b < batch_size; ++b) { - for (size_t k = 0; k < top_k; ++k) { - float raw_index = indices_data[b * top_k + k]; - if (!std::isfinite(raw_index)) { - throw std::runtime_error("ScatterTopK index is not finite"); - } - size_t expert_index = static_cast(raw_index + 0.5f); - if (expert_index >= num_classes) { - throw std::runtime_error("ScatterTopK index out of range"); - } - float weight = values_data[b * top_k + k]; - output[expert_index * batch_size + b] = weight; - } - } -} diff --git a/cactus-transpile/graph/graph_ops_tensor.cpp b/cactus-transpile/graph/graph_ops_tensor.cpp deleted file mode 100644 index c852ad480..000000000 --- a/cactus-transpile/graph/graph_ops_tensor.cpp +++ /dev/null @@ -1,813 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel.h" -#include "../kernel/kernel_utils.h" -#include -#include -#include - -template -static void transpose_scalar_nd( - const T* source, - T* destination, - const std::vector& shape, - const std::vector& permutation -) { - const size_t ndim = shape.size(); - std::vector output_shape(ndim, 0); - for (size_t axis = 0; axis < ndim; ++axis) { - output_shape[axis] = shape[permutation[axis]]; - } - - std::vector input_strides(ndim, 1); - for (size_t axis = ndim; axis-- > 1;) { - input_strides[axis - 1] = input_strides[axis] * shape[axis]; - } - - std::vector output_strides(ndim, 1); - for (size_t axis = ndim; axis-- > 1;) { - output_strides[axis - 1] = output_strides[axis] * output_shape[axis]; - } - - size_t total = 1; - for (size_t dim : output_shape) { - total *= dim; - } - - for (size_t out_idx = 0; out_idx < total; ++out_idx) { - size_t remaining = out_idx; - size_t src_idx = 0; - for (size_t axis = 0; axis < ndim; ++axis) { - const size_t coord = remaining / output_strides[axis]; - remaining %= output_strides[axis]; - src_idx += coord * input_strides[permutation[axis]]; - } - destination[out_idx] = source[src_idx]; - } -} - -void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU transpose operation not yet implemented"); - } - - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& permutation = node.params.permutation; - if (input_buffer.precision == Precision::FP16) { - const __fp16* input = input_buffer.data_as<__fp16>(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - cactus_transpose_f16( - input, - output, - input_buffer.shape.data(), - permutation.data(), - permutation.size(), - 0, - input_buffer.total_size - ); - return; - } - - if (input_buffer.precision == Precision::FP32) { - const float* input = input_buffer.data_as(); - float* output = node.output_buffer.data_as(); - transpose_scalar_nd(input, output, input_buffer.shape, permutation); - return; - } - - throw std::runtime_error("Transpose only supports FP16 and FP32 precision"); -} - -void compute_gather_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& tensor_buffer = get_input(node, 0, nodes, node_index_map); - const auto& indices_buffer = get_input(node, 1, nodes, node_index_map); - const int axis = node.params.axis; - - if (indices_buffer.shape.size() == tensor_buffer.shape.size()) { - if (tensor_buffer.precision == Precision::INT4) { - throw std::runtime_error("Generic gather does not support INT4 tensors"); - } - - const size_t rank = tensor_buffer.shape.size(); - if (rank == 0) { - throw std::runtime_error("Cannot gather from scalar tensor"); - } - - std::vector input_strides(rank, 1); - std::vector output_strides(rank, 1); - for (size_t i = rank; i-- > 1;) { - input_strides[i - 1] = input_strides[i] * tensor_buffer.shape[i]; - output_strides[i - 1] = output_strides[i] * indices_buffer.shape[i]; - } - - auto read_index = [&](size_t flat_index) -> size_t { - if (indices_buffer.precision == Precision::INT8) { - const int8_t* indices = indices_buffer.data_as(); - return static_cast(indices[flat_index]); - } - if (indices_buffer.precision == Precision::FP16) { - const __fp16* indices = indices_buffer.data_as<__fp16>(); - return static_cast(indices[flat_index]); - } - const float* indices = indices_buffer.data_as(); - return static_cast(indices[flat_index]); - }; - - const size_t total = node.output_buffer.total_size; - const size_t bytes_per_scalar = PrecisionTraits::packed_size_of(tensor_buffer.precision, 1); - const char* input_data = static_cast(tensor_buffer.get_data()); - char* output_data = static_cast(node.output_buffer.get_data()); - - for (size_t flat_output = 0; flat_output < total; ++flat_output) { - size_t remainder = flat_output; - size_t input_offset = 0; - - const size_t gathered_index = read_index(flat_output); - if (gathered_index >= tensor_buffer.shape[static_cast(axis)]) { - throw std::runtime_error( - "Gather index " + std::to_string(gathered_index) + - " out of bounds for dimension " + std::to_string(tensor_buffer.shape[static_cast(axis)]) - ); - } - - remainder = flat_output; - for (size_t dim = 0; dim < rank; ++dim) { - const size_t coord = remainder / output_strides[dim]; - remainder %= output_strides[dim]; - const size_t input_coord = (static_cast(dim) == axis) ? gathered_index : coord; - input_offset += input_coord * input_strides[dim]; - } - - std::memcpy( - output_data + flat_output * bytes_per_scalar, - input_data + PrecisionTraits::byte_offset_of(tensor_buffer.precision, input_offset), - bytes_per_scalar - ); - } - return; - } - - size_t first_dim = tensor_buffer.shape[0]; - size_t element_size = 1; - for (size_t i = 1; i < tensor_buffer.shape.size(); i++) { - element_size *= tensor_buffer.shape[i]; - } - - size_t num_indices = indices_buffer.total_size; - size_t bytes_per_element = PrecisionTraits::packed_size_of(tensor_buffer.precision, element_size); - - if (PrecisionTraits::is_integer(tensor_buffer.precision)) { - const char* tensor_data = static_cast(tensor_buffer.get_data()); - char* output = static_cast(node.output_buffer.get_data()); - Precision prec = tensor_buffer.precision; - - const bool is_grouped = tensor_buffer.group_size > 0; - __fp16* gathered_scales = nullptr; - const __fp16* src_scales = nullptr; - size_t num_groups = 0; - - if (is_grouped) { - num_groups = tensor_buffer.num_groups; - src_scales = tensor_buffer.scales_as_fp16(); - size_t scales_bytes = num_indices * num_groups * sizeof(__fp16); - node.output_buffer.owned_scales = std::make_unique(scales_bytes); - gathered_scales = reinterpret_cast<__fp16*>(node.output_buffer.owned_scales.get()); - } - - const int8_t* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + PrecisionTraits::byte_offset_of(prec, i * element_size), - tensor_data + PrecisionTraits::byte_offset_of(prec, idx * element_size), - bytes_per_element); - if (is_grouped) { - for (size_t g = 0; g < num_groups; g++) { - gathered_scales[i * num_groups + g] = src_scales[idx * num_groups + g]; - } - } - } - - if (is_grouped) { - node.output_buffer.group_size = tensor_buffer.group_size; - node.output_buffer.num_groups = num_groups; - node.output_buffer.scales_data = gathered_scales; - } - } else if (tensor_buffer.precision == Precision::FP16) { - const __fp16* tensor_data = tensor_buffer.data_as<__fp16>(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - - if (indices_buffer.precision == Precision::INT8) { - const int8_t* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); - } - } else { - const float* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); - } - } - } else { - const float* tensor_data = tensor_buffer.data_as(); - float* output = node.output_buffer.data_as(); - - if (indices_buffer.precision == Precision::INT8) { - const int8_t* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); - } - } else { - const float* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); - } - } - } -} - -void compute_slice_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - auto* input_node = nodes[node_index_map.at(node.input_ids[0])].get(); - auto& input_buffer = input_node->output_buffer; - - const size_t axis_index = static_cast(node.params.axis); - - const size_t axis_size = input_buffer.shape[axis_index]; - const size_t slice_start = node.params.slice_start; - size_t slice_length = node.params.slice_length; - - if (slice_length == 0) { - slice_length = axis_size - slice_start; - } - - if (axis_index == 0) { - auto dims0 = AxisDims::from_shape(input_buffer.shape, 0); - - auto* base_ptr = static_cast(input_buffer.get_data()); - if (!base_ptr) { - throw std::runtime_error("Slice input buffer is not available"); - } - - const size_t byte_offset = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims0.inner); - - node.output_buffer.set_external(base_ptr + byte_offset); - node.output_buffer.precision = input_buffer.precision; - - if (input_buffer.is_grouped_int8()) { - size_t num_groups = input_buffer.num_groups; - size_t scales_bytes = slice_length * num_groups * sizeof(__fp16); - node.output_buffer.owned_scales = std::make_unique(scales_bytes); - __fp16* sliced_scales = reinterpret_cast<__fp16*>(node.output_buffer.owned_scales.get()); - const __fp16* input_scales = input_buffer.scales_as_fp16(); - - for (size_t i = 0; i < slice_length; i++) { - for (size_t g = 0; g < num_groups; g++) { - sliced_scales[i * num_groups + g] = input_scales[(slice_start + i) * num_groups + g]; - } - } - - node.output_buffer.group_size = input_buffer.group_size; - node.output_buffer.num_groups = num_groups; - node.output_buffer.scales_data = sliced_scales; - } - return; - } - - const char* input_ptr = static_cast(input_buffer.get_data()); - if (!input_ptr) { - throw std::runtime_error("Slice input buffer is not available"); - } - - auto dims = AxisDims::from_shape(input_buffer.shape, axis_index); - - node.output_buffer.external_data = nullptr; - node.output_buffer.allocate(); - node.output_buffer.precision = input_buffer.precision; - - auto* output_ptr = static_cast(node.output_buffer.get_data()); - if (!output_ptr) { - throw std::runtime_error("Slice output buffer could not be allocated"); - } - - const size_t copy_block_elements = slice_length * dims.inner; - const size_t axis_stride_elements = axis_size * dims.inner; - const size_t copy_block_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, copy_block_elements); - const size_t axis_stride_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, axis_stride_elements); - - for (size_t outer = 0; outer < dims.outer; ++outer) { - const char* src = input_ptr + outer * axis_stride_bytes + PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims.inner); - char* dst = output_ptr + outer * copy_block_bytes; - std::memcpy(dst, src, copy_block_bytes); - } -} - -void compute_embedding_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& embeddings_buffer = get_input(node, 0, nodes, node_index_map); - const auto& indices_buffer = get_input(node, 1, nodes, node_index_map); - - size_t hidden_dim = embeddings_buffer.shape[1]; - size_t num_indices = indices_buffer.total_size; - size_t vocab_size = embeddings_buffer.original_N > 0 - ? embeddings_buffer.original_N - : embeddings_buffer.shape[0]; - - std::vector indices_float; - const float* indices_ptr; - if (indices_buffer.precision == Precision::FP32) { - indices_ptr = indices_buffer.data_as(); - } else { - indices_float.resize(num_indices); - const int8_t* int_indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - indices_float[i] = static_cast(int_indices[i]); - } - indices_ptr = indices_float.data(); - } - - __fp16* output = node.output_buffer.data_as<__fp16>(); - - Precision emb_prec = embeddings_buffer.precision; - if (PrecisionTraits::is_integer(emb_prec) && embeddings_buffer.group_size > 0) { - const int8_t* embeddings = embeddings_buffer.data_as(); - const __fp16* scales = embeddings_buffer.scales_as_fp16(); - size_t group_size = embeddings_buffer.group_size; - size_t num_groups = embeddings_buffer.num_groups; - - static const uint8_t gather_indices[4][16] = { - {0, 1, 2, 3, 16, 17, 18, 19, 32, 33, 34, 35, 48, 49, 50, 51}, // lane 0 - {4, 5, 6, 7, 20, 21, 22, 23, 36, 37, 38, 39, 52, 53, 54, 55}, // lane 1 - {8, 9, 10, 11, 24, 25, 26, 27, 40, 41, 42, 43, 56, 57, 58, 59}, // lane 2 - {12, 13, 14, 15, 28, 29, 30, 31, 44, 45, 46, 47, 60, 61, 62, 63} // lane 3 - }; - - auto load_table = [emb_prec](const int8_t* base) -> int8x16x4_t { - if (emb_prec == Precision::INT4) { - const uint8_t* ubase = reinterpret_cast(base); - int8x16_t low_a, high_a, low_b, high_b; - unpack_int4_as_int8x16x2(ubase, high_a, low_a); - unpack_int4_as_int8x16x2(ubase + 16, high_b, low_b); - return {low_a, high_a, low_b, high_b}; - } - return vld1q_s8_x4(base); - }; - - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices_ptr[i]); - if (idx >= vocab_size) { - throw std::runtime_error("Embedding index out of bounds: " + std::to_string(idx) + " >= " + std::to_string(vocab_size)); - } - - size_t block = idx / 4; - size_t lane = idx % 4; - __fp16* out_row = output + i * hidden_dim; - - uint8x16_t indices_vec = vld1q_u8(gather_indices[lane]); - - for (size_t g = 0; g < num_groups; g++) { - float scale = (float)scales[(block * num_groups + g) * 4 + lane]; - float32x4_t scale_vec = vdupq_n_f32(scale); - - size_t k_start = g * group_size; - size_t k_end = std::min(k_start + group_size, hidden_dim); - - const int8_t* group_base = embeddings + PrecisionTraits::byte_offset_of(emb_prec, (block * hidden_dim + k_start) * 4); - - size_t k = k_start; - for (; k + 16 <= k_end; k += 16) { - const int8_t* chunk_base = group_base + PrecisionTraits::byte_offset_of(emb_prec, (k - k_start) * 4); - int8x16x4_t table = load_table(chunk_base); - - int8x16_t values = vqtbl4q_s8(table, indices_vec); - - int16x8_t lo16 = vmovl_s8(vget_low_s8(values)); - int16x8_t hi16 = vmovl_s8(vget_high_s8(values)); - - float32x4_t f0 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(lo16))), scale_vec); - float32x4_t f1 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(lo16))), scale_vec); - float32x4_t f2 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(hi16))), scale_vec); - float32x4_t f3 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(hi16))), scale_vec); - - vst1q_f16(out_row + k, vcombine_f16(vcvt_f16_f32(f0), vcvt_f16_f32(f1))); - vst1q_f16(out_row + k + 8, vcombine_f16(vcvt_f16_f32(f2), vcvt_f16_f32(f3))); - } - - if (k < k_end) - assert (embeddings_buffer.precision != Precision::INT4 && "grouped INT4 embeddings must have hidden_dim that is a multiple of 32"); - for (; k < k_end; k++) { - size_t k_group = k / 4; - size_t k_within = k % 4; - int8_t val = embeddings[(block * (hidden_dim / 4) + k_group) * 16 + lane * 4 + k_within]; - out_row[k] = static_cast<__fp16>(val * scale); - } - } - } - } else if (embeddings_buffer.precision == Precision::FP16) { - const __fp16* embeddings = embeddings_buffer.data_as<__fp16>(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices_ptr[i]); - if (idx >= vocab_size) { - throw std::runtime_error("Embedding index out of bounds: " + std::to_string(idx) + " >= " + std::to_string(vocab_size)); - } - std::memcpy(output + i * hidden_dim, embeddings + idx * hidden_dim, hidden_dim * sizeof(__fp16)); - } - } else { - throw std::runtime_error("Embedding requires interleaved grouped INT4/INT8 or FP16"); - } -} - -static size_t product_range(const std::vector& shape, size_t start, size_t end) { - size_t product = 1; - for (size_t i = start; i < end; ++i) { - product *= shape[i]; - } - return product; -} - -static void concat_same_precision_tensors( - const std::vector& inputs, - BufferDesc& output, - int axis -) { - if (inputs.empty()) { - throw std::runtime_error("concat requires at least one input tensor"); - } - if (axis < 0) { - throw std::runtime_error("concat does not support negative axis"); - } - - const auto& first = *inputs.front(); - if (first.precision == Precision::INT4 || first.is_grouped_int8()) { - throw std::runtime_error("concat does not support packed/grouped integer tensors"); - } - - const size_t rank = first.shape.size(); - if (static_cast(axis) >= rank) { - throw std::runtime_error("concat axis out of range"); - } - - const size_t element_bytes = PrecisionTraits::size_of(first.precision); - const size_t outer = product_range(first.shape, 0, static_cast(axis)); - const size_t inner = product_range(first.shape, static_cast(axis) + 1, rank); - const size_t output_axis = output.shape[static_cast(axis)]; - char* output_ptr = static_cast(output.get_data()); - - for (size_t outer_idx = 0; outer_idx < outer; ++outer_idx) { - size_t output_axis_offset = 0; - for (const BufferDesc* input : inputs) { - if (input->precision != first.precision) { - throw std::runtime_error("concat requires all input precisions to match"); - } - if (input->shape.size() != rank) { - throw std::runtime_error("concat requires all input ranks to match"); - } - if (input->precision == Precision::INT4 || input->is_grouped_int8()) { - throw std::runtime_error("concat does not support packed/grouped integer tensors"); - } - - const size_t axis_size = input->shape[static_cast(axis)]; - const size_t chunk_elements = axis_size * inner; - const size_t output_base = ((outer_idx * output_axis) + output_axis_offset) * inner * element_bytes; - const size_t input_base = outer_idx * chunk_elements * element_bytes; - std::memcpy( - output_ptr + output_base, - static_cast(input->get_data()) + input_base, - chunk_elements * element_bytes - ); - output_axis_offset += axis_size; - } - } -} - -void compute_concat_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input1_buffer = get_input(node, 0, nodes, node_index_map); - const auto& input2_buffer = get_input(node, 1, nodes, node_index_map); - std::vector inputs = {&input1_buffer, &input2_buffer}; - concat_same_precision_tensors(inputs, node.output_buffer, node.params.axis); -} - -void compute_cat_node( - GraphNode& node, - const std::vector>& nodes, - const std::unordered_map& node_index_map -) { - if (node.params.axis < 0) { - throw std::runtime_error("Cat operation does not support negative axis"); - } - if (node.input_ids.size() < 2) { - throw std::runtime_error("Cat operation requires at least 2 input tensors"); - } - - std::vector inputs; - inputs.reserve(node.input_ids.size()); - - for (size_t i = 0; i < node.input_ids.size(); i++) { - const auto& buffer = get_input(node, i, nodes, node_index_map); - inputs.push_back(&buffer); - } - concat_same_precision_tensors(inputs, node.output_buffer, node.params.axis); -} - -void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& input_shape = input_buffer.shape; - - int dim = node.params.axis; - size_t index_value = node.params.index_value; - - const char* input_data = static_cast(input_buffer.get_data()); - char* output_data = static_cast(node.output_buffer.get_data()); - - if (dim == 0) { - size_t slice_size = input_buffer.total_size / input_shape[0]; - size_t offset_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, index_value * slice_size); - node.output_buffer.set_external(const_cast(input_data) + offset_bytes); - return; - } - - std::vector input_strides(input_shape.size()); - input_strides[input_shape.size() - 1] = 1; - for (int i = static_cast(input_shape.size()) - 2; i >= 0; --i) { - input_strides[i] = input_strides[i + 1] * input_shape[i + 1]; - } - - size_t slice_size = input_strides[dim]; - size_t outer_size = input_buffer.total_size / input_strides[dim - 1]; - size_t dim_stride = input_strides[dim]; - size_t block_size = dim_stride * input_shape[dim]; - - size_t output_idx = 0; - for (size_t outer_idx = 0; outer_idx < outer_size; ++outer_idx) { - size_t input_base = outer_idx * block_size + index_value * dim_stride; - - char* output_offset_bytes = output_data + PrecisionTraits::byte_offset_of(input_buffer.precision, output_idx); - const char* input_offset_bytes = input_data + PrecisionTraits::byte_offset_of(input_buffer.precision, input_base); - size_t length = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_size); - std::memcpy(output_offset_bytes, input_offset_bytes, length); - - output_idx += slice_size; - } -} - -static bool mask_element_is_true(const BufferDesc& mask_buffer, size_t index) { - switch (mask_buffer.precision) { - case Precision::FP16: - return static_cast(mask_buffer.data_as<__fp16>()[index]) != 0.0f; - case Precision::FP32: - return mask_buffer.data_as()[index] != 0.0f; - case Precision::INT8: - return mask_buffer.data_as()[index] != 0; - default: - throw std::runtime_error("Unsupported mask precision for masked select/scatter"); - } -} - -void compute_masked_select_prefix_node( - GraphNode& node, - const std::vector>& nodes, - const std::unordered_map& node_index_map -) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& mask_buffer = get_input(node, 1, nodes, node_index_map); - - if (input_buffer.precision == Precision::INT4 || input_buffer.is_grouped_int8()) { - throw std::runtime_error("masked_select_prefix does not support packed/grouped integer tensors"); - } - if (mask_buffer.shape.empty()) { - throw std::runtime_error("masked_select_prefix requires non-scalar mask"); - } - if (input_buffer.shape.size() < mask_buffer.shape.size()) { - throw std::runtime_error("masked_select_prefix mask rank must not exceed input rank"); - } - for (size_t i = 0; i < mask_buffer.shape.size(); ++i) { - if (input_buffer.shape[i] != mask_buffer.shape[i]) { - throw std::runtime_error("masked_select_prefix mask shape must match the input prefix shape"); - } - } - - const size_t prefix_elems = mask_buffer.total_size; - const size_t trailing_elems = input_buffer.total_size / prefix_elems; - const size_t block_bytes = trailing_elems * PrecisionTraits::size_of(input_buffer.precision); - const char* input_ptr = static_cast(input_buffer.get_data()); - char* output_ptr = static_cast(node.output_buffer.get_data()); - std::memset(output_ptr, 0, node.output_buffer.byte_size); - - size_t packed_row = 0; - for (size_t prefix_idx = 0; prefix_idx < prefix_elems; ++prefix_idx) { - if (!mask_element_is_true(mask_buffer, prefix_idx)) { - continue; - } - std::memcpy( - output_ptr + (packed_row * block_bytes), - input_ptr + (prefix_idx * block_bytes), - block_bytes - ); - packed_row++; - } -} - -void compute_masked_scatter_node( - GraphNode& node, - const std::vector>& nodes, - const std::unordered_map& node_index_map -) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& mask_buffer = get_input(node, 1, nodes, node_index_map); - const auto& source_buffer = get_input(node, 2, nodes, node_index_map); - - if (input_buffer.precision == Precision::INT4 || source_buffer.precision == Precision::INT4) { - throw std::runtime_error("masked_scatter does not support INT4 tensors"); - } - if (input_buffer.shape != mask_buffer.shape) { - throw std::runtime_error("masked_scatter mask shape must match input shape"); - } - if (input_buffer.precision != source_buffer.precision) { - throw std::runtime_error("masked_scatter source precision must match input precision"); - } - - const size_t element_bytes = PrecisionTraits::size_of(input_buffer.precision); - const char* input_ptr = static_cast(input_buffer.get_data()); - const char* source_ptr = static_cast(source_buffer.get_data()); - char* output_ptr = static_cast(node.output_buffer.get_data()); - std::memcpy(output_ptr, input_ptr, node.output_buffer.byte_size); - - size_t source_index = 0; - for (size_t element_idx = 0; element_idx < mask_buffer.total_size; ++element_idx) { - if (!mask_element_is_true(mask_buffer, element_idx)) { - continue; - } - if (source_index >= source_buffer.total_size) { - throw std::runtime_error("masked_scatter source does not contain enough elements"); - } - std::memcpy( - output_ptr + (element_idx * element_bytes), - source_ptr + (source_index * element_bytes), - element_bytes - ); - source_index++; - } -} - -void compute_expand_node( - GraphNode& node, - const std::vector>& nodes, - const std::unordered_map& node_index_map -) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& input_shape = input_buffer.shape; - const auto& output_shape = node.output_buffer.shape; - - if (input_buffer.precision == Precision::INT4 || input_buffer.is_grouped_int8()) { - throw std::runtime_error("expand does not support packed/grouped integer tensors"); - } - if (output_shape.size() < input_shape.size()) { - throw std::runtime_error("expand cannot reduce rank"); - } - - const size_t rank_delta = output_shape.size() - input_shape.size(); - std::vector padded_input_shape(output_shape.size(), 1); - for (size_t axis = 0; axis < input_shape.size(); ++axis) { - padded_input_shape[rank_delta + axis] = input_shape[axis]; - } - for (size_t axis = 0; axis < output_shape.size(); ++axis) { - if (padded_input_shape[axis] != 1 && padded_input_shape[axis] != output_shape[axis]) { - throw std::runtime_error("expand target shape is not broadcast-compatible with input"); - } - } - - std::vector input_strides(output_shape.size(), 1); - std::vector output_strides(output_shape.size(), 1); - for (size_t axis = output_shape.size(); axis-- > 1;) { - input_strides[axis - 1] = input_strides[axis] * padded_input_shape[axis]; - output_strides[axis - 1] = output_strides[axis] * output_shape[axis]; - } - - const size_t element_bytes = PrecisionTraits::size_of(input_buffer.precision); - const char* input_ptr = static_cast(input_buffer.get_data()); - char* output_ptr = static_cast(node.output_buffer.get_data()); - for (size_t output_index = 0; output_index < node.output_buffer.total_size; ++output_index) { - size_t remainder = output_index; - size_t input_index = 0; - for (size_t axis = 0; axis < output_shape.size(); ++axis) { - const size_t coord = remainder / output_strides[axis]; - remainder %= output_strides[axis]; - const size_t input_coord = padded_input_shape[axis] == 1 ? 0 : coord; - input_index += input_coord * input_strides[axis]; - } - std::memcpy( - output_ptr + (output_index * element_bytes), - input_ptr + (input_index * element_bytes), - element_bytes - ); - } -} - -void compute_cumsum_node( - GraphNode& node, - const std::vector>& nodes, - const std::unordered_map& node_index_map -) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - - if (input_buffer.precision != Precision::FP16 && input_buffer.precision != Precision::FP32) { - throw std::runtime_error("cumsum only supports FP16 and FP32 precision"); - } - - const int axis = node.params.axis; - if (axis < 0 || static_cast(axis) >= input_buffer.shape.size()) { - throw std::runtime_error("cumsum axis out of range"); - } - - auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); - if (input_buffer.precision == Precision::FP32) { - cactus_cumsum_axis_f32( - input_buffer.data_as(), - node.output_buffer.data_as(), - dims.outer, - dims.axis_size, - dims.inner - ); - return; - } - - cactus_cumsum_axis_f16( - input_buffer.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - dims.outer, - dims.axis_size, - dims.inner - ); -} - -void compute_bilinear_interpolation_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& pos_embeds_buffer = get_input(node, 0, nodes, node_index_map); - - size_t total_pos_embeds = pos_embeds_buffer.shape[0]; - size_t embed_dim = pos_embeds_buffer.shape[1]; - - size_t src_height = static_cast(std::sqrt(total_pos_embeds)); - size_t src_width = src_height; - - size_t dst_height = node.params.dst_height; - size_t dst_width = node.params.dst_width; - bool align_corners = node.params.align_corners; - - __fp16* output = node.output_buffer.data_as<__fp16>(); - - if (pos_embeds_buffer.precision == Precision::FP16) { - const __fp16* input = pos_embeds_buffer.data_as<__fp16>(); - cactus_bilinear_interpolation_f16(input, output, src_height, src_width, embed_dim, - dst_height, dst_width, align_corners); - } - else if (pos_embeds_buffer.precision == Precision::INT8) { - std::vector<__fp16> input_fp16(total_pos_embeds * embed_dim); - cactus_int8_to_fp16(pos_embeds_buffer.data_as(), input_fp16.data(), - total_pos_embeds * embed_dim); - cactus_bilinear_interpolation_f16(input_fp16.data(), output, src_height, src_width, embed_dim, - dst_height, dst_width, align_corners); - } - else { - throw std::runtime_error("BILINEAR_INTERPOLATION only supports INT8 and FP16 input precision"); - } -} - -void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.input_ids.empty()) { - return; - } - - auto it = node_index_map.find(node.input_ids[0]); - - if (it != node_index_map.end()) { - const auto& input_buffer = nodes[it->second]->output_buffer; - - if (!node.output_buffer.get_data()) { - node.output_buffer.allocate(); - } - - std::memcpy(node.output_buffer.get_data(), - input_buffer.get_data(), - input_buffer.byte_size); - } else { - if (node.output_buffer.get_data()) { - return; - } - throw std::runtime_error("PERSISTENT node input not found and not populated - this should not happen"); - } -} diff --git a/cactus-transpile/graph/graph_param_io.cpp b/cactus-transpile/graph/graph_param_io.cpp deleted file mode 100644 index 64af5ca19..000000000 --- a/cactus-transpile/graph/graph_param_io.cpp +++ /dev/null @@ -1,394 +0,0 @@ -#include "graph_param_io.h" - -#include -#include -#include -#include - -namespace { - -void write_u32(std::ostream& out, uint32_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); -} - -void write_u64(std::ostream& out, uint64_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); -} - -void write_i32(std::ostream& out, int32_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); -} - -void write_f32(std::ostream& out, float v) { - out.write(reinterpret_cast(&v), sizeof(v)); -} - -uint32_t read_u32(std::istream& in) { - uint32_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) throw std::runtime_error("Unexpected EOF while reading uint32"); - return v; -} - -uint64_t read_u64(std::istream& in) { - uint64_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) throw std::runtime_error("Unexpected EOF while reading uint64"); - return v; -} - -int32_t read_i32(std::istream& in) { - int32_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) throw std::runtime_error("Unexpected EOF while reading int32"); - return v; -} - -float read_f32(std::istream& in) { - float v = 0.0f; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) throw std::runtime_error("Unexpected EOF while reading float"); - return v; -} - -void write_size_vector(std::ostream& out, const std::vector& values) { - write_u32(out, static_cast(values.size())); - for (size_t v : values) write_u64(out, static_cast(v)); -} - -std::vector read_size_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) values.push_back(static_cast(read_u64(in))); - return values; -} - -void write_u32_vector(std::ostream& out, const std::vector& values) { - write_u32(out, static_cast(values.size())); - for (uint32_t v : values) write_u32(out, v); -} - -std::vector read_u32_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) values.push_back(read_u32(in)); - return values; -} - -void write_f32_vector(std::ostream& out, const std::vector& values) { - write_u32(out, static_cast(values.size())); - for (float v : values) write_f32(out, v); -} - -std::vector read_f32_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) values.push_back(read_f32(in)); - return values; -} - -enum class ParamField : uint32_t { - Scalar = 1, - Axis, - NewShape, - PretransposedRhs, - Backend, - SliceStart, - SliceLength, - Epsilon, - NumGroups, - IndexValue, - Permutation, - Scale, - Theta, - PositionOffset, - WindowSize, - IsCausal, - AttentionMaskIsAdditive, - LogitCap, - TopK, - Temperature, - TopP, - MinP, - RepetitionPenalty, - RandomSeed, - BiasIndices, - BiasValues, - NormalizeRouting, - NumExperts, - NumExpertsPerTok, - MoeGated, - Activation, - NumKvHeads, - ChunkSize, - DstHeight, - DstWidth, - AlignCorners, - NumClasses, - NumAltupInputs, - KernelSize, - Stride, - Dilation, - NumFftBins, - CacheSeqLen, - HeadDim, - VHeadDim, - CachedKeysInt8Ptr, - CachedValuesInt8Ptr, - CachedKScalesPtr, - CachedVScalesPtr, - Padding, -}; - -enum class FieldPersistence { - Persistent, - RuntimeOnly, - Derived, -}; - -struct FieldSpec { - ParamField field; - FieldPersistence persistence; -}; - -using Schema = std::vector; - -const Schema& op_schema(OpType op_type) { - static const Schema kEmpty{}; - static const std::unordered_map kSchemas = { - {OpType::POW, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_ADD, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_SUBTRACT, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_MULTIPLY, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_DIVIDE, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_NOT_EQUAL, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_FLOOR_DIVIDE, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SOFTMAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::SUM, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::MEAN, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::VARIANCE, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::MIN, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::MAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::CUMSUM, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::INDEX, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::IndexValue, FieldPersistence::Persistent}}}, - {OpType::GATHER, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::CONCAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::CAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::VIEW, {{ParamField::NewShape, FieldPersistence::Persistent}}}, - {OpType::RESHAPE, {{ParamField::NewShape, FieldPersistence::Persistent}}}, - {OpType::FLATTEN, {{ParamField::NewShape, FieldPersistence::Persistent}}}, - {OpType::MATMUL, {{ParamField::PretransposedRhs, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::TRANSPOSE, {{ParamField::Permutation, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::SLICE, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::SliceStart, FieldPersistence::Persistent}, {ParamField::SliceLength, FieldPersistence::Persistent}}}, - {OpType::RMS_NORM, {{ParamField::Epsilon, FieldPersistence::Persistent}}}, - {OpType::LAYERNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}}}, - {OpType::GROUPNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::NumGroups, FieldPersistence::Persistent}}}, - {OpType::BATCHNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::ROPE, {{ParamField::Theta, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::ROPE_GPTJ, {{ParamField::Theta, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::Scalar, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::TOPK, {{ParamField::TopK, FieldPersistence::Persistent}}}, - {OpType::ATTENTION, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::WindowSize, FieldPersistence::Persistent}, {ParamField::IsCausal, FieldPersistence::Persistent}, {ParamField::AttentionMaskIsAdditive, FieldPersistence::Persistent}, {ParamField::LogitCap, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::REL_POS_BIAS, {{ParamField::Scale, FieldPersistence::Persistent}}}, - {OpType::ATTENTION_INT8_HYBRID, { - {ParamField::Scale, FieldPersistence::Persistent}, - {ParamField::PositionOffset, FieldPersistence::Persistent}, - {ParamField::WindowSize, FieldPersistence::Persistent}, - {ParamField::NumKvHeads, FieldPersistence::Persistent}, - {ParamField::CacheSeqLen, FieldPersistence::Persistent}, - {ParamField::HeadDim, FieldPersistence::Persistent}, - {ParamField::VHeadDim, FieldPersistence::Persistent}, - {ParamField::CachedKeysInt8Ptr, FieldPersistence::RuntimeOnly}, - {ParamField::CachedValuesInt8Ptr, FieldPersistence::RuntimeOnly}, - {ParamField::CachedKScalesPtr, FieldPersistence::RuntimeOnly}, - {ParamField::CachedVScalesPtr, FieldPersistence::RuntimeOnly}, - }}, - {OpType::MOE_LAYER, {{ParamField::Scalar, FieldPersistence::Persistent}, {ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::NormalizeRouting, FieldPersistence::Persistent}, {ParamField::NumExperts, FieldPersistence::Persistent}, {ParamField::NumExpertsPerTok, FieldPersistence::Persistent}, {ParamField::MoeGated, FieldPersistence::Persistent}, {ParamField::Activation, FieldPersistence::Persistent}}}, - {OpType::GATED_DELTANET_DECODE, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::NumKvHeads, FieldPersistence::Persistent}}}, - {OpType::GATED_DELTANET_PREFILL, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::NumKvHeads, FieldPersistence::Persistent}, {ParamField::ChunkSize, FieldPersistence::Persistent}}}, - {OpType::STFT, {{ParamField::Stride, FieldPersistence::Persistent}, {ParamField::NumFftBins, FieldPersistence::Persistent}}}, - {OpType::BILINEAR_INTERPOLATION, {{ParamField::DstHeight, FieldPersistence::Persistent}, {ParamField::DstWidth, FieldPersistence::Persistent}, {ParamField::AlignCorners, FieldPersistence::Persistent}}}, - {OpType::SCATTER_TOPK, {{ParamField::NumClasses, FieldPersistence::Persistent}}}, - {OpType::ALTUP_PREDICT, {{ParamField::NumAltupInputs, FieldPersistence::Persistent}}}, - {OpType::ALTUP_CORRECT, {{ParamField::NumAltupInputs, FieldPersistence::Persistent}}}, - {OpType::MAXPOOL1D, {{ParamField::KernelSize, FieldPersistence::Persistent}, {ParamField::Stride, FieldPersistence::Persistent}}}, - {OpType::CONV1D_CAUSAL, {{ParamField::Dilation, FieldPersistence::Persistent}}}, - {OpType::CONV1D_K3, {{ParamField::Stride, FieldPersistence::Persistent}}}, - {OpType::CONV1D_K7S3, {{ParamField::Stride, FieldPersistence::Persistent}}}, - {OpType::CONV1D, {{ParamField::Stride, FieldPersistence::Persistent}}}, - {OpType::GLU, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::CONV2D, { - {ParamField::Stride, FieldPersistence::Persistent}, - {ParamField::Padding, FieldPersistence::Persistent}, - {ParamField::Dilation, FieldPersistence::Persistent}, - {ParamField::NumGroups, FieldPersistence::Persistent}, - }}, - {OpType::SAMPLE, {{ParamField::Temperature, FieldPersistence::Persistent}, {ParamField::TopP, FieldPersistence::Persistent}, {ParamField::MinP, FieldPersistence::Persistent}, {ParamField::RepetitionPenalty, FieldPersistence::Persistent}, {ParamField::TopK, FieldPersistence::Persistent}, {ParamField::RandomSeed, FieldPersistence::Persistent}, {ParamField::BiasIndices, FieldPersistence::Persistent}, {ParamField::BiasValues, FieldPersistence::Persistent}}}, - }; - - auto it = kSchemas.find(op_type); - return it == kSchemas.end() ? kEmpty : it->second; -} - -void write_field(std::ostream& out, ParamField field, const OpParams& params) { - switch (field) { - case ParamField::Scalar: write_f32(out, params.scalar); break; - case ParamField::Axis: write_i32(out, static_cast(params.axis)); break; - case ParamField::NewShape: write_size_vector(out, params.new_shape); break; - case ParamField::PretransposedRhs: write_u32(out, params.pretransposed_rhs ? 1u : 0u); break; - case ParamField::Backend: write_u32(out, static_cast(params.backend)); break; - case ParamField::SliceStart: write_u64(out, static_cast(params.slice_start)); break; - case ParamField::SliceLength: write_u64(out, static_cast(params.slice_length)); break; - case ParamField::Epsilon: write_f32(out, params.epsilon); break; - case ParamField::NumGroups: write_u64(out, static_cast(params.num_groups)); break; - case ParamField::IndexValue: write_u64(out, static_cast(params.index_value)); break; - case ParamField::Permutation: write_size_vector(out, params.permutation); break; - case ParamField::Scale: write_f32(out, params.scale); break; - case ParamField::Theta: write_f32(out, params.theta); break; - case ParamField::PositionOffset: write_u64(out, static_cast(params.position_offset)); break; - case ParamField::WindowSize: write_u64(out, static_cast(params.window_size)); break; - case ParamField::IsCausal: write_u32(out, params.is_causal ? 1u : 0u); break; - case ParamField::AttentionMaskIsAdditive: write_u32(out, params.attention_mask_is_additive ? 1u : 0u); break; - case ParamField::LogitCap: write_f32(out, params.logit_cap); break; - case ParamField::TopK: write_u64(out, static_cast(params.top_k)); break; - case ParamField::Temperature: write_f32(out, params.temperature); break; - case ParamField::TopP: write_f32(out, params.top_p); break; - case ParamField::MinP: write_f32(out, params.min_p); break; - case ParamField::RepetitionPenalty: write_f32(out, params.repetition_penalty); break; - case ParamField::RandomSeed: write_u64(out, static_cast(params.random_seed)); break; - case ParamField::BiasIndices: write_u32_vector(out, params.bias_indices); break; - case ParamField::BiasValues: write_f32_vector(out, params.bias_values); break; - case ParamField::NormalizeRouting: write_u32(out, params.normalize_routing ? 1u : 0u); break; - case ParamField::NumExperts: write_u64(out, static_cast(params.num_experts)); break; - case ParamField::NumExpertsPerTok: write_u64(out, static_cast(params.num_experts_per_tok)); break; - case ParamField::MoeGated: write_u32(out, params.moe_gated ? 1u : 0u); break; - case ParamField::Activation: write_u32(out, static_cast(params.activation)); break; - case ParamField::NumKvHeads: write_u64(out, static_cast(params.num_kv_heads)); break; - case ParamField::ChunkSize: write_u64(out, static_cast(params.chunk_size)); break; - case ParamField::DstHeight: write_u64(out, static_cast(params.dst_height)); break; - case ParamField::DstWidth: write_u64(out, static_cast(params.dst_width)); break; - case ParamField::AlignCorners: write_u32(out, params.align_corners ? 1u : 0u); break; - case ParamField::NumClasses: write_u64(out, static_cast(params.num_classes)); break; - case ParamField::NumAltupInputs: write_u64(out, static_cast(params.num_altup_inputs)); break; - case ParamField::KernelSize: write_u64(out, static_cast(params.kernel_size)); break; - case ParamField::Stride: write_u64(out, static_cast(params.stride)); break; - case ParamField::Dilation: write_u64(out, static_cast(params.dilation)); break; - case ParamField::NumFftBins: write_u64(out, static_cast(params.num_fft_bins)); break; - case ParamField::CacheSeqLen: write_u64(out, static_cast(params.cache_seq_len)); break; - case ParamField::HeadDim: write_u64(out, static_cast(params.head_dim)); break; - case ParamField::VHeadDim: write_u64(out, static_cast(params.v_head_dim)); break; - case ParamField::CachedKeysInt8Ptr: - case ParamField::CachedValuesInt8Ptr: - case ParamField::CachedKScalesPtr: - case ParamField::CachedVScalesPtr: - throw std::runtime_error("Attempted to serialize runtime-only pointer field"); - case ParamField::Padding: write_u64(out, static_cast(params.padding)); break; - } -} - -void read_field(std::istream& in, ParamField field, OpParams& params) { - switch (field) { - case ParamField::Scalar: params.scalar = read_f32(in); break; - case ParamField::Axis: params.axis = static_cast(read_i32(in)); break; - case ParamField::NewShape: params.new_shape = read_size_vector(in); break; - case ParamField::PretransposedRhs: params.pretransposed_rhs = (read_u32(in) != 0); break; - case ParamField::Backend: { - uint32_t backend_val = read_u32(in); - if (backend_val > static_cast(ComputeBackend::NPU)) { - throw std::runtime_error("Graph file corrupted: invalid backend"); - } - params.backend = static_cast(backend_val); - break; - } - case ParamField::SliceStart: params.slice_start = static_cast(read_u64(in)); break; - case ParamField::SliceLength: params.slice_length = static_cast(read_u64(in)); break; - case ParamField::Epsilon: params.epsilon = read_f32(in); break; - case ParamField::NumGroups: params.num_groups = static_cast(read_u64(in)); break; - case ParamField::IndexValue: params.index_value = static_cast(read_u64(in)); break; - case ParamField::Permutation: params.permutation = read_size_vector(in); break; - case ParamField::Scale: params.scale = read_f32(in); break; - case ParamField::Theta: params.theta = read_f32(in); break; - case ParamField::PositionOffset: params.position_offset = static_cast(read_u64(in)); break; - case ParamField::WindowSize: params.window_size = static_cast(read_u64(in)); break; - case ParamField::IsCausal: params.is_causal = (read_u32(in) != 0); break; - case ParamField::AttentionMaskIsAdditive: params.attention_mask_is_additive = (read_u32(in) != 0); break; - case ParamField::LogitCap: params.logit_cap = read_f32(in); break; - case ParamField::TopK: params.top_k = static_cast(read_u64(in)); break; - case ParamField::Temperature: params.temperature = read_f32(in); break; - case ParamField::TopP: params.top_p = read_f32(in); break; - case ParamField::MinP: params.min_p = read_f32(in); break; - case ParamField::RepetitionPenalty: params.repetition_penalty = read_f32(in); break; - case ParamField::RandomSeed: params.random_seed = static_cast(read_u64(in)); break; - case ParamField::BiasIndices: params.bias_indices = read_u32_vector(in); break; - case ParamField::BiasValues: params.bias_values = read_f32_vector(in); break; - case ParamField::NormalizeRouting: params.normalize_routing = (read_u32(in) != 0); break; - case ParamField::NumExperts: params.num_experts = static_cast(read_u64(in)); break; - case ParamField::NumExpertsPerTok: params.num_experts_per_tok = static_cast(read_u64(in)); break; - case ParamField::MoeGated: params.moe_gated = (read_u32(in) != 0); break; - case ParamField::Activation: params.activation = static_cast(read_u32(in)); break; - case ParamField::NumKvHeads: params.num_kv_heads = static_cast(read_u64(in)); break; - case ParamField::ChunkSize: params.chunk_size = static_cast(read_u64(in)); break; - case ParamField::DstHeight: params.dst_height = static_cast(read_u64(in)); break; - case ParamField::DstWidth: params.dst_width = static_cast(read_u64(in)); break; - case ParamField::AlignCorners: params.align_corners = (read_u32(in) != 0); break; - case ParamField::NumClasses: params.num_classes = static_cast(read_u64(in)); break; - case ParamField::NumAltupInputs: params.num_altup_inputs = static_cast(read_u64(in)); break; - case ParamField::KernelSize: params.kernel_size = static_cast(read_u64(in)); break; - case ParamField::Stride: params.stride = static_cast(read_u64(in)); break; - case ParamField::Dilation: params.dilation = static_cast(read_u64(in)); break; - case ParamField::NumFftBins: params.num_fft_bins = static_cast(read_u64(in)); break; - case ParamField::CacheSeqLen: params.cache_seq_len = static_cast(read_u64(in)); break; - case ParamField::HeadDim: params.head_dim = static_cast(read_u64(in)); break; - case ParamField::VHeadDim: params.v_head_dim = static_cast(read_u64(in)); break; - case ParamField::CachedKeysInt8Ptr: - case ParamField::CachedValuesInt8Ptr: - case ParamField::CachedKScalesPtr: - case ParamField::CachedVScalesPtr: - throw std::runtime_error("Graph file corrupted: runtime-only field serialized"); - case ParamField::Padding: params.padding = static_cast(read_u64(in)); break; - } -} - -} // namespace - -namespace GraphParamIO { - -void write_op_params(std::ostream& out, OpType op_type, const OpParams& params) { - const auto& schema = op_schema(op_type); - std::vector fields; - fields.reserve(schema.size()); - for (const auto& spec : schema) { - if (spec.persistence == FieldPersistence::Persistent) { - fields.push_back(spec.field); - } - } - - write_u32(out, static_cast(fields.size())); - for (ParamField field : fields) { - write_u32(out, static_cast(field)); - write_field(out, field, params); - } -} - -void read_op_params(std::istream& in, OpType op_type, OpParams& params) { - uint32_t field_count = read_u32(in); - const auto& schema = op_schema(op_type); - std::unordered_set allowed; - for (const auto& spec : schema) { - if (spec.persistence == FieldPersistence::Persistent) { - allowed.insert(static_cast(spec.field)); - } - } - - for (uint32_t i = 0; i < field_count; ++i) { - uint32_t raw_field = read_u32(in); - if (allowed.count(raw_field) == 0) { - throw std::runtime_error("Graph file corrupted: field not allowed for op"); - } - read_field(in, static_cast(raw_field), params); - } -} - -} // namespace GraphParamIO diff --git a/cactus-transpile/graph/graph_param_io.h b/cactus-transpile/graph/graph_param_io.h deleted file mode 100644 index 0362ea8c5..000000000 --- a/cactus-transpile/graph/graph_param_io.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "graph.h" -#include - -namespace GraphParamIO { - -void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); -void read_op_params(std::istream& in, OpType op_type, OpParams& params); - -} // namespace GraphParamIO diff --git a/cactus-transpile/kernel/kernel.h b/cactus-transpile/kernel/kernel.h deleted file mode 100644 index 8141ce43c..000000000 --- a/cactus-transpile/kernel/kernel.h +++ /dev/null @@ -1,488 +0,0 @@ -#ifndef KERNEL_H -#define KERNEL_H - -#include -#include - -enum class Precision; - -enum class ScalarOpType { - ADD, - SUBTRACT, - MULTIPLY, - DIVIDE, - NOT_EQUAL, - ABS, - EXP, - POW, - SQRT, - COS, - SIN, - LOG, - FLOOR_DIVIDE -}; - -constexpr size_t KV_QUANT_GROUP_SIZE = 32; - -void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale); -void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_not_equal_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); - -void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); -void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); -void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); -void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); -void cactus_not_equal_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); - -void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type); - -void cactus_gemv_int8(const int8_t* A, float A_scale, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t K, size_t N, size_t group_size); - -void cactus_gemm_int8(const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_matmul_int8(const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_gemv_int8_i8mm(const int8_t* A, float A_scale, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t K, size_t N, size_t group_size); - -void cactus_gemm_int8_i8mm(const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_gemv_int4(const int8_t* A, float A_scale, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t K, size_t N, size_t group_size); - -void cactus_gemm_int4(const int8_t* A, const float* A_scales, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_matmul_int4(const int8_t* A, const float* A_scales, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_matmul_integer(Precision precision, - const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c, - size_t M, size_t K, size_t N); - -void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, - size_t num_rows, size_t num_cols, size_t start_row, size_t end_row); -void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, - const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx); - -double cactus_sum_all_f16(const __fp16* data, size_t num_elements); -void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); -double cactus_sum_all_f32(const float* data, size_t num_elements); -void cactus_sum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); - -double cactus_mean_all_f16(const __fp16* data, size_t num_elements); -void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); -double cactus_mean_all_f32(const float* data, size_t num_elements); -void cactus_mean_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); - -double cactus_variance_all_f16(const __fp16* data, size_t num_elements); -void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); -double cactus_variance_all_f32(const float* data, size_t num_elements); -void cactus_variance_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); - -__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements); -void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); -float cactus_min_all_f32(const float* data, size_t num_elements); -void cactus_min_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); - -__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements); -void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); -float cactus_max_all_f32(const float* data, size_t num_elements); -void cactus_max_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); -void cactus_cumsum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); -void cactus_cumsum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size); - -void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output, - size_t batch_size, size_t dims, float eps); - -void cactus_layer_norm_f16(const __fp16* input, const __fp16* weight, const __fp16* bias, - __fp16* output, size_t batch_size, size_t dims, float eps); - -void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, - size_t num_heads, size_t head_dim, size_t start_pos, float theta); - -void cactus_gpt_j_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, - size_t num_heads, size_t head_dim, size_t rot_dim, size_t start_pos, float theta); - -void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, - size_t seq_len, size_t vocab_size); - -void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope); - -void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_glu_f16( - const __fp16* input, - __fp16* output, - size_t outer_size, - size_t split_size, - size_t inner_size -); - -void cactus_glu_f32( - const float* input, - float* output, - size_t outer_size, - size_t split_size, - size_t inner_size -); - -void cactus_batchnorm_f16( - const __fp16* input, - const float* weight, - const float* bias, - const float* running_mean, - const float* running_var, - __fp16* output, - size_t outer_size, - size_t channels, - size_t inner_size, - float epsilon -); - -void cactus_batchnorm_f32( - const float* input, - const float* weight, - const float* bias, - const float* running_mean, - const float* running_var, - float* output, - size_t outer_size, - size_t channels, - size_t inner_size, - float epsilon -); - -void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output, - size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, - size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, - bool is_causal = true, bool mask_is_additive = false, bool mask_per_head = false, - size_t v_head_dim = 0, float logit_cap = 0.0f); - -void cactus_attention_hybrid_int8_fp16( - const __fp16* queries, - const int8_t* keys_cached, - const int8_t* values_cached, - const float* k_scales, - const float* v_scales, - const __fp16* keys_new, - const __fp16* values_new, - __fp16* output, - size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len, - size_t num_q_heads, size_t num_kv_heads, size_t head_dim, - float scale, size_t position_offset = 0, bool is_causal = true, size_t window_size = 0, - size_t group_size = KV_QUANT_GROUP_SIZE, size_t v_head_dim = 0); - -void cactus_gated_deltanet_decode_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - float scale); - -void cactus_gated_deltanet_prefill_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t T, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - size_t requested_chunk_size, - float scale); - -void cactus_conv1d_causal_depthwise_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, - size_t L, - size_t C, - size_t K, - size_t dilation); - -void cactus_conv1d_f16_k3( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out, - size_t stride -); - -void cactus_conv1d_f16( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out, - size_t K, - size_t stride -); - -void cactus_stft_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, size_t stride, - size_t num_fft_bins -); - -void cactus_conv1d_f16_k7s3_oc8( - const __fp16* input, - const __fp16* Wpack, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out -); - -void cactus_conv1d_same_depthwise_f16_k9( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C -); - -void cactus_conv2d_f16_k3s1p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, - size_t H, - size_t W, - size_t C_out -); - -void cactus_conv2d_f16_k3s2p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, - size_t H, - size_t W, - size_t C_out -); - -void cactus_conv2d_depthwise_f16_k3s2p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C, - size_t H, - size_t W -); - -void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, - size_t H, - size_t W, - size_t C_out -); - -void cactus_conv1d_pointwise_f16_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out -); - -void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, - size_t dst_height, size_t dst_width, bool align_corners = true); - -void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, size_t top_k, size_t random_seed, - const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, - size_t bias_count = 0); -void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, size_t top_k, size_t random_seed, - const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, - size_t bias_count = 0); - -void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, size_t random_seed, - const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, - size_t bias_count = 0); -void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, size_t random_seed, - const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, - size_t bias_count = 0); - -void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, - const size_t* shape1, const size_t* shape2, const size_t* output_shape, - size_t ndims, int axis); -void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, - const size_t* output_shape, size_t num_inputs, size_t rank, int axis); - -void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); -void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); -void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count); -void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count); -void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); -void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); -float cactus_fp16_max_abs(const __fp16* src, size_t count); - -void cactus_quantize_kv_fp16_to_int8( - const __fp16* src, - int8_t* dst, - float* scales, - size_t seq_len, size_t kv_heads, size_t head_dim, - size_t group_size = KV_QUANT_GROUP_SIZE); - -inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) { - size_t num_groups = (head_dim + group_size - 1) / group_size; - return seq_len * kv_heads * num_groups; -} - -void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count); - -void cactus_gaussian_topk_f16( - const __fp16* input, - __fp16* output, - size_t rows, - size_t cols, - float ppf); - -void cactus_altup_predict_f16( - const __fp16* coefs, - const __fp16* const* streams, - __fp16* output, - size_t n, - size_t seq_len, - size_t hidden_dim); - -void cactus_altup_correct_f16( - const __fp16* coefs, - const __fp16* innovation, - const __fp16* const* predictions, - __fp16* output, - size_t n, - size_t seq_len, - size_t hidden_dim); - -void cactus_lstm_cell_f16( - const __fp16* x_input, - const __fp16* h_prev, - const __fp16* c_prev, - const __fp16* weight_ih, - const __fp16* weight_hh, - const __fp16* bias_ih, - const __fp16* bias_hh, - __fp16* h_new, - __fp16* c_new, - size_t batch_size, - size_t input_size, - size_t hidden_size -); - -void cactus_bilstm_sequence_f16( - const __fp16* input, - const __fp16* weight_ih_fwd, - const __fp16* weight_hh_fwd, - const __fp16* bias_ih_fwd, - const __fp16* bias_hh_fwd, - const __fp16* weight_ih_bwd, - const __fp16* weight_hh_bwd, - const __fp16* bias_ih_bwd, - const __fp16* bias_hh_bwd, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t input_size, - size_t hidden_size -); - -void cactus_maxpool1d_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t channels, - size_t input_length, - size_t kernel_size, - size_t stride -); - -#endif diff --git a/cactus-transpile/kernel/kernel_altup.cpp b/cactus-transpile/kernel/kernel_altup.cpp deleted file mode 100644 index f15bf9030..000000000 --- a/cactus-transpile/kernel/kernel_altup.cpp +++ /dev/null @@ -1,180 +0,0 @@ -#include "kernel.h" -#include -#include -#include -#include - -void cactus_gaussian_topk_f16( - const __fp16* input, - __fp16* output, - size_t rows, - size_t cols, - float ppf -) { - static constexpr float MAX_SAFE_ABS = 240.0f; - - for (size_t r = 0; r < rows; r++) { - const __fp16* row_in = input + r * cols; - __fp16* row_out = output + r * cols; - - // Pass 1: compute mean and max absolute deviation - float32x4_t sum_vec = vdupq_n_f32(0.0f); - size_t d = 0; - for (; d + 8 <= cols; d += 8) { - float16x8_t v = vld1q_f16(row_in + d); - sum_vec = vaddq_f32(sum_vec, vcvt_f32_f16(vget_low_f16(v))); - sum_vec = vaddq_f32(sum_vec, vcvt_f32_f16(vget_high_f16(v))); - } - float sum = vaddvq_f32(sum_vec); - for (; d < cols; d++) { - sum += static_cast(row_in[d]); - } - float mu = sum / static_cast(cols); - - float32x4_t mu_vec = vdupq_n_f32(mu); - float32x4_t max_abs_vec = vdupq_n_f32(0.0f); - d = 0; - for (; d + 8 <= cols; d += 8) { - float16x8_t v = vld1q_f16(row_in + d); - float32x4_t lo = vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), mu_vec); - float32x4_t hi = vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), mu_vec); - max_abs_vec = vmaxq_f32(max_abs_vec, vabsq_f32(lo)); - max_abs_vec = vmaxq_f32(max_abs_vec, vabsq_f32(hi)); - } - float max_abs = vmaxvq_f32(max_abs_vec); - for (; d < cols; d++) { - float diff = static_cast(row_in[d]) - mu; - float a = diff < 0.0f ? -diff : diff; - if (a > max_abs) max_abs = a; - } - - // Safe scaling: scale = max(max_abs / 240, 1) - float scale = max_abs * (1.0f / MAX_SAFE_ABS); - if (scale < 1.0f) scale = 1.0f; - float inv_scale = 1.0f / scale; - - // Pass 2: compute variance of scaled deviations - float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); - float32x4_t var_sum_vec = vdupq_n_f32(0.0f); - d = 0; - for (; d + 8 <= cols; d += 8) { - float16x8_t v = vld1q_f16(row_in + d); - float32x4_t lo = vmulq_f32(vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), mu_vec), inv_scale_vec); - float32x4_t hi = vmulq_f32(vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), mu_vec), inv_scale_vec); - var_sum_vec = vfmaq_f32(var_sum_vec, lo, lo); - var_sum_vec = vfmaq_f32(var_sum_vec, hi, hi); - } - float var_sum = vaddvq_f32(var_sum_vec); - for (; d < cols; d++) { - float ds = (static_cast(row_in[d]) - mu) * inv_scale; - var_sum += ds * ds; - } - float variance = var_sum / static_cast(cols); - float sigma = sqrtf(variance) * scale; - - // cutoff = mu + ppf * sigma, output = relu(input - cutoff) - float cutoff = mu + ppf * sigma; - float32x4_t cutoff_vec = vdupq_n_f32(cutoff); - float32x4_t zero_vec = vdupq_n_f32(0.0f); - d = 0; - for (; d + 8 <= cols; d += 8) { - float16x8_t v = vld1q_f16(row_in + d); - float32x4_t lo = vmaxq_f32(vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), cutoff_vec), zero_vec); - float32x4_t hi = vmaxq_f32(vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), cutoff_vec), zero_vec); - vst1q_f16(row_out + d, vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi))); - } - for (; d < cols; d++) { - float val = static_cast(row_in[d]) - cutoff; - row_out[d] = static_cast<__fp16>(val > 0.0f ? val : 0.0f); - } - } -} - -void cactus_altup_predict_f16( - const __fp16* coefs, - const __fp16* const* streams, - __fp16* output, - size_t n, - size_t seq_len, - size_t hidden_dim -) { - if (n > 8) throw std::runtime_error("cactus_altup_predict_f16 expects n <= 8"); - const size_t coef_stride = n * n; - - for (size_t i = 0; i < n; i++) { - for (size_t s = 0; s < seq_len; s++) { - const __fp16* coef_row = coefs + s * coef_stride + i * n; - __fp16* out_row = output + (i * seq_len + s) * hidden_dim; - const __fp16* src_i = streams[i] + s * hidden_dim; - - float c[8]; - for (size_t j = 0; j < n; j++) { - c[j] = static_cast(coef_row[j]); - } - - size_t d = 0; - for (; d + 8 <= hidden_dim; d += 8) { - float32x4_t acc_lo = vcvt_f32_f16(vget_low_f16(vld1q_f16(src_i + d))); - float32x4_t acc_hi = vcvt_f32_f16(vget_high_f16(vld1q_f16(src_i + d))); - - for (size_t j = 0; j < n; j++) { - float16x8_t sj = vld1q_f16(streams[j] + s * hidden_dim + d); - float32x4_t cj = vdupq_n_f32(c[j]); - acc_lo = vfmaq_f32(acc_lo, vcvt_f32_f16(vget_low_f16(sj)), cj); - acc_hi = vfmaq_f32(acc_hi, vcvt_f32_f16(vget_high_f16(sj)), cj); - } - - vst1q_f16(out_row + d, vcombine_f16(vcvt_f16_f32(acc_lo), vcvt_f16_f32(acc_hi))); - } - for (; d < hidden_dim; d++) { - float acc = static_cast(src_i[d]); - for (size_t j = 0; j < n; j++) { - acc += c[j] * static_cast(streams[j][s * hidden_dim + d]); - } - out_row[d] = static_cast<__fp16>(acc); - } - } - } -} - -void cactus_altup_correct_f16( - const __fp16* coefs, - const __fp16* innovation, - const __fp16* const* predictions, - __fp16* output, - size_t n, - size_t seq_len, - size_t hidden_dim -) { - for (size_t i = 0; i < n; i++) { - for (size_t s = 0; s < seq_len; s++) { - float ci = static_cast(coefs[s * n + i]); - float32x4_t ci_vec = vdupq_n_f32(ci); - - const __fp16* pred_row = predictions[i] + s * hidden_dim; - const __fp16* innov_row = innovation + s * hidden_dim; - __fp16* out_row = output + (i * seq_len + s) * hidden_dim; - - size_t d = 0; - for (; d + 8 <= hidden_dim; d += 8) { - float16x8_t pred = vld1q_f16(pred_row + d); - float16x8_t innov = vld1q_f16(innov_row + d); - - float32x4_t p_lo = vcvt_f32_f16(vget_low_f16(pred)); - float32x4_t p_hi = vcvt_f32_f16(vget_high_f16(pred)); - float32x4_t i_lo = vcvt_f32_f16(vget_low_f16(innov)); - float32x4_t i_hi = vcvt_f32_f16(vget_high_f16(innov)); - - float32x4_t out_lo = vfmaq_f32(p_lo, i_lo, ci_vec); - float32x4_t out_hi = vfmaq_f32(p_hi, i_hi, ci_vec); - - vst1q_f16(out_row + d, vcombine_f16(vcvt_f16_f32(out_lo), vcvt_f16_f32(out_hi))); - } - for (; d < hidden_dim; d++) { - out_row[d] = static_cast<__fp16>( - static_cast(pred_row[d]) + ci * static_cast(innov_row[d]) - ); - } - } - } -} diff --git a/cactus-transpile/kernel/kernel_attention.cpp b/cactus-transpile/kernel/kernel_attention.cpp deleted file mode 100644 index 1ff483802..000000000 --- a/cactus-transpile/kernel/kernel_attention.cpp +++ /dev/null @@ -1,1365 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { - -inline size_t sliding_window_kv_start(size_t absolute_q_pos, size_t window_size) { - if (window_size == 0) { - return 0; - } - const size_t visible_tokens = absolute_q_pos + 1; - if (visible_tokens <= window_size) { - return 0; - } - return visible_tokens - window_size; -} - -inline bool sliding_window_masks_kv(size_t absolute_q_pos, size_t kv_pos, size_t window_size) { - return window_size > 0 && kv_pos < absolute_q_pos && (absolute_q_pos - kv_pos) >= window_size; -} - -} // namespace - -#ifdef __APPLE__ -#include -#endif - -#ifdef __APPLE__ -static void cactus_attention_f16_accelerate( - const __fp16* queries, - const __fp16* keys, - const __fp16* values, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t kv_seq_len, - size_t num_q_heads, - size_t num_kv_heads, - size_t head_dim, - size_t v_head_dim, - float scale, - size_t position_offset, - bool is_causal -) { - constexpr size_t BLOCK_SIZE = 64; - - const size_t group_size = num_q_heads / num_kv_heads; - const size_t q_batch_stride = seq_len * num_q_heads * head_dim; - const size_t k_batch_stride = kv_seq_len * num_kv_heads * head_dim; - const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; - const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; - const size_t q_seq_stride = num_q_heads * head_dim; - const size_t k_seq_stride = num_kv_heads * head_dim; - const size_t v_seq_stride = num_kv_heads * v_head_dim; - const size_t o_seq_stride = num_q_heads * v_head_dim; - - static constexpr CactusThreading::ParallelConfig ATTENTION_BATCHED{1, 1}; - CactusThreading::parallel_for(batch_size * num_q_heads, ATTENTION_BATCHED, - [&](size_t start, size_t end) { - - std::vector Q_f32(seq_len * head_dim); - std::vector K_f32(BLOCK_SIZE * head_dim); - std::vector V_f32(BLOCK_SIZE * v_head_dim); - std::vector scores(seq_len * BLOCK_SIZE); - std::vector acc(seq_len * v_head_dim); - std::vector row_max(seq_len); - std::vector row_sum(seq_len); - - for (size_t work = start; work < end; ++work) { - const size_t batch = work / num_q_heads; - const size_t q_head = work % num_q_heads; - const size_t kv_head = q_head / group_size; - - for (size_t q = 0; q < seq_len; ++q) { - const __fp16* q_src = queries + batch*q_batch_stride + q*q_seq_stride + q_head*head_dim; - float* q_dst = Q_f32.data() + q * head_dim; - for (size_t d = 0; d < head_dim; d += 8) { - float16x8_t v = vld1q_f16(q_src + d); - vst1q_f32(q_dst + d, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(q_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); - } - } - - std::fill(row_max.begin(), row_max.begin() + seq_len, -INFINITY); - std::fill(row_sum.begin(), row_sum.begin() + seq_len, 0.0f); - memset(acc.data(), 0, seq_len * v_head_dim * sizeof(float)); - - for (size_t kv0 = 0; kv0 < kv_seq_len; kv0 += BLOCK_SIZE) { - const size_t block_len = std::min(BLOCK_SIZE, kv_seq_len - kv0); - - size_t q_start = 0; - size_t active_rows = seq_len; - if (is_causal) { - if (kv0 > position_offset) { - q_start = kv0 - position_offset; - } - if (q_start >= seq_len) continue; - active_rows = seq_len - q_start; - } - - for (size_t i = 0; i < block_len; ++i) { - const __fp16* k_src = keys + batch*k_batch_stride + (kv0+i)*k_seq_stride + kv_head*head_dim; - float* k_dst = K_f32.data() + i * head_dim; - for (size_t d = 0; d < head_dim; d += 8) { - float16x8_t v = vld1q_f16(k_src + d); - vst1q_f32(k_dst + d, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(k_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - (int)active_rows, (int)block_len, (int)head_dim, - scale, - Q_f32.data() + q_start * head_dim, (int)head_dim, - K_f32.data(), (int)head_dim, - 0.0f, - scores.data(), (int)block_len); - - for (size_t r = 0; r < active_rows; ++r) { - const size_t q_pos = q_start + r; - const size_t abs_q = position_offset + q_pos; - float* s_row = scores.data() + r * block_len; - - size_t valid_len = block_len; - if (is_causal) { - if (abs_q < kv0) { - memset(s_row, 0, block_len * sizeof(float)); - continue; - } - valid_len = std::min(block_len, abs_q - kv0 + 1); - } - - float32x4_t vmax = vdupq_n_f32(-INFINITY); - size_t j = 0; - for (; j + 4 <= valid_len; j += 4) { - vmax = vmaxq_f32(vmax, vld1q_f32(s_row + j)); - } - float block_max = vmaxvq_f32(vmax); - for (; j < valid_len; ++j) { - block_max = std::max(block_max, s_row[j]); - } - - float prev_max = row_max[q_pos]; - float new_max = std::max(prev_max, block_max); - float scale_old = expf(prev_max - new_max); - - if (prev_max != -INFINITY) { - float* acc_row = acc.data() + q_pos * v_head_dim; - float32x4_t sv = vdupq_n_f32(scale_old); - for (size_t d = 0; d < v_head_dim; d += 4) { - float32x4_t a = vld1q_f32(acc_row + d); - vst1q_f32(acc_row + d, vmulq_f32(a, sv)); - } - } - row_sum[q_pos] = row_sum[q_pos] * scale_old; - row_max[q_pos] = new_max; - - float32x4_t vsum = vdupq_n_f32(0.0f); - float32x4_t vnmax = vdupq_n_f32(new_max); - float32x4_t log2e = vdupq_n_f32(1.442695f); - j = 0; - for (; j + 4 <= valid_len; j += 4) { - float32x4_t x = vmulq_f32(vsubq_f32(vld1q_f32(s_row + j), vnmax), log2e); - float32x4_t x_floor = vrndmq_f32(x); - int32x4_t xi = vcvtq_s32_f32(x_floor); - float32x4_t xf = vsubq_f32(x, x_floor); - float32x4_t t = vfmaq_n_f32(vdupq_n_f32(0.2246932f), xf, 0.0789673f); - t = vfmaq_f32(vdupq_n_f32(0.6963248f), t, xf); - float32x4_t y = vfmaq_f32(vdupq_n_f32(0.9999003f), t, xf); - xi = vshlq_n_s32(vaddq_s32(xi, vdupq_n_s32(127)), 23); - y = vmulq_f32(y, vreinterpretq_f32_s32(xi)); - uint32x4_t underflow = vcltq_f32(x, vdupq_n_f32(-126.0f)); - y = vbslq_f32(underflow, vdupq_n_f32(0.0f), y); - vst1q_f32(s_row + j, y); - vsum = vaddq_f32(vsum, y); - } - float block_sum = vaddvq_f32(vsum); - for (; j < valid_len; ++j) { - s_row[j] = expf(s_row[j] - new_max); - block_sum += s_row[j]; - } - if (valid_len < block_len) { - memset(s_row + valid_len, 0, (block_len - valid_len) * sizeof(float)); - } - row_sum[q_pos] += block_sum; - } - - for (size_t i = 0; i < block_len; ++i) { - const __fp16* v_src = values + batch*v_batch_stride + (kv0+i)*v_seq_stride + kv_head*v_head_dim; - float* v_dst = V_f32.data() + i * v_head_dim; - for (size_t d = 0; d < v_head_dim; d += 8) { - float16x8_t v = vld1q_f16(v_src + d); - vst1q_f32(v_dst + d, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(v_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - (int)active_rows, (int)v_head_dim, (int)block_len, - 1.0f, - scores.data(), (int)block_len, - V_f32.data(), (int)v_head_dim, - 1.0f, - acc.data() + q_start * v_head_dim, (int)v_head_dim); - } - - for (size_t q = 0; q < seq_len; ++q) { - __fp16* o = output + batch*o_batch_stride + q*o_seq_stride + q_head*v_head_dim; - float sum = row_sum[q]; - if (sum == 0.0f) { - memset(o, 0, v_head_dim * sizeof(__fp16)); - continue; - } - float inv = 1.0f / sum; - float32x4_t invv = vdupq_n_f32(inv); - float* acc_row = acc.data() + q * v_head_dim; - for (size_t d = 0; d < v_head_dim; d += 8) { - float32x4_t a0 = vmulq_f32(vld1q_f32(acc_row + d), invv); - float32x4_t a1 = vmulq_f32(vld1q_f32(acc_row + d + 4), invv); - vst1q_f16(o + d, vcombine_f16(vcvt_f16_f32(a0), vcvt_f16_f32(a1))); - } - } - } - }); -} -#endif - -static inline void cactus_attention_f16_fast( - const __fp16* queries, - const __fp16* keys, - const __fp16* values, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t kv_seq_len, - size_t num_q_heads, - size_t num_kv_heads, - size_t head_dim, - float scale, - size_t position_offset, - bool is_causal, - size_t window_size, - size_t v_head_dim -) { - constexpr size_t BLOCK_SIZE = 32; - const size_t qk_nblocks = head_dim / 8; - const size_t v_nblocks = v_head_dim / 8; - -#ifdef __APPLE__ - if (seq_len >= 64 && window_size == 0) { - cactus_attention_f16_accelerate( - queries, keys, values, output, - batch_size, seq_len, kv_seq_len, - num_q_heads, num_kv_heads, head_dim, v_head_dim, - scale, position_offset, is_causal - ); - return; - } -#endif - - const size_t group_size = num_q_heads / num_kv_heads; - const size_t q_batch_stride = seq_len * num_q_heads * head_dim; - const size_t kv_batch_stride = kv_seq_len * num_kv_heads * head_dim; - const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; - const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; - const size_t q_seq_stride = num_q_heads * head_dim; - const size_t kv_seq_stride = num_kv_heads * head_dim; - const size_t v_seq_stride = num_kv_heads * v_head_dim; - const size_t o_seq_stride = num_q_heads * v_head_dim; - - CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, - [&](size_t start, size_t end) { - - float block_scores[BLOCK_SIZE]; - std::vector acc_lo(v_nblocks), acc_hi(v_nblocks); - - for (size_t work = start; work < end; ++work) { - const size_t batch = work / (num_q_heads * seq_len); - const size_t rem = work % (num_q_heads * seq_len); - const size_t q_head = rem / seq_len; - const size_t q_pos = rem % seq_len; - const size_t kv_head = q_head / group_size; - - const __fp16* q = queries + batch*q_batch_stride + q_pos*q_seq_stride + q_head*head_dim; - __fp16* o = output + batch*o_batch_stride + q_pos*o_seq_stride + q_head*v_head_dim; - - for (size_t i = 0; i < v_nblocks; i++) { - acc_lo[i] = vdupq_n_f32(0.f); - acc_hi[i] = vdupq_n_f32(0.f); - } - - float running_max = -INFINITY; - float running_sum = 0.f; - - const size_t abs_q = position_offset + q_pos; - size_t kv_end = is_causal ? std::min(kv_seq_len, abs_q + 1) : kv_seq_len; - size_t kv_start = sliding_window_kv_start(abs_q, window_size); - - for (size_t kv0 = kv_start; kv0 < kv_end; kv0 += BLOCK_SIZE) { - const size_t kv1 = std::min(kv0 + BLOCK_SIZE, kv_end); - float block_max = -INFINITY; - - for (size_t i = kv0; i < kv1; i++) { - float32x4_t s0 = vdupq_n_f32(0.f); - float32x4_t s1 = vdupq_n_f32(0.f); - - const __fp16* k = keys + batch*kv_batch_stride + i*kv_seq_stride + kv_head*head_dim; - - for (size_t d = 0; d < qk_nblocks; d++) { - float16x8_t qv = vld1q_f16(q + d*8); - float16x8_t kv = vld1q_f16(k + d*8); - - float32x4_t ql = vcvt_f32_f16(vget_low_f16(qv)); - float32x4_t qh = vcvt_f32_f16(vget_high_f16(qv)); - float32x4_t kl = vcvt_f32_f16(vget_low_f16(kv)); - float32x4_t kh = vcvt_f32_f16(vget_high_f16(kv)); - - s0 = vfmaq_f32(s0, ql, kl); - s1 = vfmaq_f32(s1, qh, kh); - } - - float score = vaddvq_f32(vaddq_f32(s0, s1)) * scale; - block_scores[i - kv0] = score; - block_max = std::max(block_max, score); - } - - float current_block_scale = 1.0f; - if (block_max > running_max) { - float scale_correction = expf(running_max - block_max); - running_sum *= scale_correction; - - for (size_t d = 0; d < v_nblocks; d++) { - acc_lo[d] = vmulq_n_f32(acc_lo[d], scale_correction); - acc_hi[d] = vmulq_n_f32(acc_hi[d], scale_correction); - } - running_max = block_max; - } else { - current_block_scale = expf(block_max - running_max); - } - - float block_sum = 0.f; - for (size_t i = 0; i < kv1 - kv0; i++) { - block_scores[i] = expf(block_scores[i] - block_max); - block_sum += block_scores[i]; - } - - for (size_t i = 0; i < kv1 - kv0; i++) { - const float attn_weight = block_scores[i] * current_block_scale; - if (attn_weight == 0.f) continue; - - const __fp16* v = values + batch*v_batch_stride + (kv0+i)*v_seq_stride + kv_head*v_head_dim; - float32x4_t wv = vdupq_n_f32(attn_weight); - - for (size_t d = 0; d < v_nblocks; d++) { - float16x8_t vv = vld1q_f16(v + d*8); - acc_lo[d] = vfmaq_f32(acc_lo[d], vcvt_f32_f16(vget_low_f16(vv)), wv); - acc_hi[d] = vfmaq_f32(acc_hi[d], vcvt_f32_f16(vget_high_f16(vv)), wv); - } - } - - running_sum += block_sum * current_block_scale; - } - - if (running_sum == 0.f) { - memset(o, 0, v_head_dim * sizeof(__fp16)); - continue; - } - - float inv = 1.f / running_sum; - float32x4_t invv = vdupq_n_f32(inv); - - for (size_t d = 0; d < v_nblocks; d++) { - float16x8_t out = vcombine_f16( - vcvt_f16_f32(vmulq_f32(acc_lo[d], invv)), - vcvt_f16_f32(vmulq_f32(acc_hi[d], invv)) - ); - vst1q_f16(o + d*8, out); - } - } - }); -} - -void cactus_attention_f16( - const __fp16* queries, - const __fp16* keys, - const __fp16* values, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t kv_seq_len, - size_t num_q_heads, - size_t num_kv_heads, - size_t head_dim, - float scale, - const __fp16* mask, - size_t position_offset, - size_t window_size, - bool is_causal, - bool mask_is_additive, - bool mask_per_head, - size_t v_head_dim, - float logit_cap -) { - if (v_head_dim == 0) v_head_dim = head_dim; - if (scale == 0.0f) { - scale = 1.0f / sqrtf(static_cast(head_dim)); - } - - if (mask == nullptr && head_dim % 8 == 0 && v_head_dim % 8 == 0 && logit_cap == 0.0f) { - cactus_attention_f16_fast( - queries, keys, values, output, - batch_size, seq_len, kv_seq_len, - num_q_heads, num_kv_heads, head_dim, - scale, position_offset, is_causal, window_size, v_head_dim - ); - return; - } - - constexpr size_t VECTOR_WIDTH = 8; - constexpr size_t BLOCK_SIZE = 32; - const size_t head_dim_aligned = (head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; - const size_t v_head_dim_aligned = (v_head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; - - const size_t group_size = num_q_heads / num_kv_heads; - - const size_t q_batch_stride = seq_len * num_q_heads * head_dim; - const size_t kv_batch_stride = kv_seq_len * num_kv_heads * head_dim; - const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; - const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; - const size_t q_seq_stride = num_q_heads * head_dim; - const size_t kv_seq_stride = num_kv_heads * head_dim; - const size_t v_seq_stride = num_kv_heads * v_head_dim; - const size_t o_seq_stride = num_q_heads * v_head_dim; - const size_t mask_batch_stride = mask - ? (mask_per_head ? (num_q_heads * seq_len * kv_seq_len) : (seq_len * kv_seq_len)) - : 0; - - CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, - [=](size_t start_idx, size_t end_idx) { - std::vector block_scores(BLOCK_SIZE); - std::vector output_accum_low(v_head_dim_aligned / VECTOR_WIDTH * 2); - std::vector output_accum_high(v_head_dim_aligned / VECTOR_WIDTH * 2); - - const size_t v_tail_dims = v_head_dim - v_head_dim_aligned; - std::vector output_accum_tail(v_tail_dims, 0.0f); - - const float NEG_INF = -std::numeric_limits::infinity(); - const size_t used_vec_blocks = v_head_dim_aligned / VECTOR_WIDTH; - - for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { - const size_t batch_idx = work_idx / (num_q_heads * seq_len); - const size_t remainder = work_idx % (num_q_heads * seq_len); - const size_t q_head_idx = remainder / seq_len; - const size_t q_pos = remainder % seq_len; - - const size_t kv_head_idx = q_head_idx / group_size; - - const __fp16* Q_base = queries + batch_idx * q_batch_stride; - const __fp16* K_base = keys + batch_idx * kv_batch_stride; - const __fp16* V_base = values + batch_idx * v_batch_stride; - __fp16* O_base = output + batch_idx * o_batch_stride; - const __fp16* M = mask ? (mask + batch_idx * mask_batch_stride) : nullptr; - const __fp16* q_vec = Q_base + q_pos * q_seq_stride + q_head_idx * head_dim; - __fp16* o_vec = O_base + q_pos * o_seq_stride + q_head_idx * v_head_dim; - - float running_max = -std::numeric_limits::infinity(); - float running_sum = 0.0f; - - for (size_t i = 0; i < output_accum_low.size(); ++i) { - output_accum_low[i] = vdupq_n_f32(0.0f); - output_accum_high[i] = vdupq_n_f32(0.0f); - } - for (size_t i = 0; i < v_tail_dims; ++i) { - output_accum_tail[i] = 0.0f; - } - - const bool is_decode = (q_pos == seq_len - 1) && seq_len > 1; - const size_t absolute_q_pos = position_offset + q_pos; - - size_t kv_start = 0; - size_t kv_end = kv_seq_len; - - if (window_size > 0 && window_size < kv_seq_len) { - kv_start = sliding_window_kv_start(absolute_q_pos, window_size); - if (is_causal) { - kv_end = std::min(kv_end, absolute_q_pos + 1); - } - } else if (is_causal) { - kv_end = std::min(kv_end, absolute_q_pos + 1); - } - - for (size_t kv_block_start = kv_start; kv_block_start < kv_end; kv_block_start += BLOCK_SIZE) { - const size_t kv_block_end = std::min(kv_block_start + BLOCK_SIZE, kv_end); - const size_t block_size = kv_block_end - kv_block_start; - - float block_max = -std::numeric_limits::infinity(); - - if (!is_decode && is_causal && kv_block_start > absolute_q_pos) { - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - block_scores[kv_idx] = NEG_INF; - } - continue; - } - - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - const size_t kv_pos = kv_block_start + kv_idx; - - if (!is_decode && is_causal && kv_pos > absolute_q_pos) { - block_scores[kv_idx] = NEG_INF; - continue; - } - - const __fp16* k_vec = K_base + kv_pos * kv_seq_stride + kv_head_idx * head_dim; - - if (kv_idx + 1 < block_size) { - const __fp16* next_k_vec = K_base + (kv_pos + 1) * kv_seq_stride + kv_head_idx * head_dim; - __builtin_prefetch(next_k_vec, 0, 1); - } - - float32x4_t score_accum_low = vdupq_n_f32(0.0f); - float32x4_t score_accum_high = vdupq_n_f32(0.0f); - - for (size_t dim_block = 0; dim_block < head_dim_aligned; dim_block += VECTOR_WIDTH) { - float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); - float16x8_t k_vec_f16 = vld1q_f16(&k_vec[dim_block]); - - float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); - float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); - float32x4_t k_low = vcvt_f32_f16(vget_low_f16(k_vec_f16)); - float32x4_t k_high = vcvt_f32_f16(vget_high_f16(k_vec_f16)); - - score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); - score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); - } - - float score = vaddvq_f32(vaddq_f32(score_accum_low, score_accum_high)); - - for (size_t dim = head_dim_aligned; dim < head_dim; ++dim) { - score += static_cast(q_vec[dim]) * static_cast(k_vec[dim]); - } - - score *= scale; - - size_t absolute_q_pos = position_offset + q_pos; - - if (is_causal && kv_pos > absolute_q_pos) { - score = NEG_INF; - } - else if (sliding_window_masks_kv(absolute_q_pos, kv_pos, window_size)) { - score = NEG_INF; - } - else if (M) { - const size_t mask_index = mask_per_head - ? ((q_head_idx * seq_len + q_pos) * kv_seq_len + kv_pos) - : (q_pos * kv_seq_len + kv_pos); - const float mask_value = static_cast(M[mask_index]); - if (mask_is_additive) { - if (!std::isfinite(mask_value)) { - score = NEG_INF; - } else { - score += mask_value; - } - } else if (mask_value == 0.0f) { - score = NEG_INF; - } - } - - if (logit_cap > 0.0f && std::isfinite(score)) { - score = logit_cap * tanhf(score / logit_cap); - } - - block_scores[kv_idx] = score; - block_max = std::max(block_max, score); - } - - float current_block_scale = 1.0f; - - if (block_max > NEG_INF) { - if (block_max > running_max) { - float scale_correction = expf(running_max - block_max); - running_sum *= scale_correction; - - for (size_t i = 0; i < used_vec_blocks; ++i) { - output_accum_low[i] = vmulq_n_f32(output_accum_low[i], scale_correction); - output_accum_high[i] = vmulq_n_f32(output_accum_high[i], scale_correction); - } - for (size_t i = 0; i < v_tail_dims; ++i) { - output_accum_tail[i] *= scale_correction; - } - running_max = block_max; - } else { - current_block_scale = expf(block_max - running_max); - } - } - - float block_sum = 0.0f; - const size_t vec_size = (block_size / 4) * 4; - - for (size_t kv_idx = 0; kv_idx < vec_size; kv_idx += 4) { - float32x4_t scores = vld1q_f32(&block_scores[kv_idx]); - uint32x4_t inf_mask = vceqq_f32(scores, vdupq_n_f32(NEG_INF)); - - float32x4_t x = vsubq_f32(scores, vdupq_n_f32(block_max)); - x = vmulq_n_f32(x, 1.442695f); - float32x4_t x_floor = vrndmq_f32(x); - int32x4_t xi = vcvtq_s32_f32(x_floor); - float32x4_t xf = vsubq_f32(x, x_floor); - - float32x4_t t = vfmaq_n_f32(vdupq_n_f32(0.2246932f), xf, 0.0789673f); - t = vfmaq_f32(vdupq_n_f32(0.6963248f), t, xf); - float32x4_t y = vfmaq_f32(vdupq_n_f32(0.9999003f), t, xf); - - xi = vaddq_s32(xi, vdupq_n_s32(127)); - xi = vshlq_n_s32(xi, 23); - y = vmulq_f32(y, vreinterpretq_f32_s32(xi)); - - uint32x4_t underflow_mask = vcltq_f32(x, vdupq_n_f32(-126.0f)); - uint32x4_t zero_mask = vorrq_u32(inf_mask, underflow_mask); - y = vbslq_f32(zero_mask, vdupq_n_f32(0.0f), y); - - vst1q_f32(&block_scores[kv_idx], y); - block_sum += vaddvq_f32(y); - } - - for (size_t kv_idx = vec_size; kv_idx < block_size; ++kv_idx) { - if (block_scores[kv_idx] != NEG_INF) { - block_scores[kv_idx] = expf(block_scores[kv_idx] - block_max); - block_sum += block_scores[kv_idx]; - } else { - block_scores[kv_idx] = 0.0f; - } - } - - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - const float attn_weight = block_scores[kv_idx] * current_block_scale; - if (attn_weight == 0.0f) continue; - - const size_t kv_pos = kv_block_start + kv_idx; - const __fp16* v_vec = V_base + kv_pos * v_seq_stride + kv_head_idx * v_head_dim; - - const float32x4_t weight_vec = vdupq_n_f32(attn_weight); - - for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { - float16x8_t v_vec_f16 = vld1q_f16(&v_vec[dim_block]); - float32x4_t v_low = vcvt_f32_f16(vget_low_f16(v_vec_f16)); - float32x4_t v_high = vcvt_f32_f16(vget_high_f16(v_vec_f16)); - - size_t idx = dim_block / VECTOR_WIDTH; - output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); - output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); - } - - for (size_t dim = v_head_dim_aligned; dim < v_head_dim; ++dim) { - float val = attn_weight * static_cast(v_vec[dim]); - output_accum_tail[dim - v_head_dim_aligned] += val; - } - } - - running_sum += block_sum * current_block_scale; - } - - if (running_sum > 0.0f) { - const float inv_sum = 1.0f / running_sum; - const float32x4_t inv_sum_vec = vdupq_n_f32(inv_sum); - - for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { - size_t idx = dim_block / VECTOR_WIDTH; - float32x4_t final_low = vmulq_f32(output_accum_low[idx], inv_sum_vec); - float32x4_t final_high = vmulq_f32(output_accum_high[idx], inv_sum_vec); - - float16x4_t low_f16 = vcvt_f16_f32(final_low); - float16x4_t high_f16 = vcvt_f16_f32(final_high); - float16x8_t combined = vcombine_f16(low_f16, high_f16); - - vst1q_f16(&o_vec[dim_block], combined); - } - - for (size_t dim = v_head_dim_aligned; dim < v_head_dim; ++dim) { - o_vec[dim] = static_cast<__fp16>(output_accum_tail[dim - v_head_dim_aligned] * inv_sum); - } - } else { - for (size_t dim = 0; dim < v_head_dim; ++dim) { - o_vec[dim] = static_cast<__fp16>(0.0f); - } - } - } - }); -} - -void cactus_attention_hybrid_int8_fp16( - const __fp16* queries, - const int8_t* keys_cached, - const int8_t* values_cached, - const float* k_scales, - const float* v_scales, - const __fp16* keys_new, - const __fp16* values_new, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t cache_len, - size_t new_len, - size_t num_q_heads, - size_t num_kv_heads, - size_t head_dim, - float scale, - size_t position_offset, - bool is_causal, - size_t window_size, - size_t quant_group_size, - size_t v_head_dim -) { - if (v_head_dim == 0) v_head_dim = head_dim; - if (scale == 0.0f) { - scale = 1.0f / sqrtf(static_cast(head_dim)); - } - - const size_t kv_seq_len = cache_len + new_len; - - constexpr size_t VECTOR_WIDTH = 8; - constexpr size_t BLOCK_SIZE = 32; - const size_t head_dim_aligned = (head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; - const size_t v_head_dim_aligned = (v_head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; - - const size_t gqa_group_size = num_q_heads / num_kv_heads; - const size_t num_quant_groups_k = (head_dim + quant_group_size - 1) / quant_group_size; - const size_t num_quant_groups_v = (v_head_dim + quant_group_size - 1) / quant_group_size; - - const size_t q_batch_stride = seq_len * num_q_heads * head_dim; - const size_t k_cached_batch_stride = cache_len * num_kv_heads * head_dim; - const size_t v_cached_batch_stride = cache_len * num_kv_heads * v_head_dim; - const size_t k_new_batch_stride = new_len * num_kv_heads * head_dim; - const size_t v_new_batch_stride = new_len * num_kv_heads * v_head_dim; - const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; - const size_t q_seq_stride = num_q_heads * head_dim; - const size_t k_seq_stride = num_kv_heads * head_dim; - const size_t v_seq_stride = num_kv_heads * v_head_dim; - const size_t o_seq_stride = num_q_heads * v_head_dim; - - CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, - [=](size_t start_idx, size_t end_idx) { - std::vector block_scores(BLOCK_SIZE); - std::vector output_accum_low(v_head_dim_aligned / VECTOR_WIDTH * 2); - std::vector output_accum_high(v_head_dim_aligned / VECTOR_WIDTH * 2); - - for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { - const size_t batch_idx = work_idx / (num_q_heads * seq_len); - const size_t remainder = work_idx % (num_q_heads * seq_len); - const size_t q_head_idx = remainder / seq_len; - const size_t q_pos = remainder % seq_len; - - const size_t kv_head_idx = q_head_idx / gqa_group_size; - - const __fp16* Q_base = queries + batch_idx * q_batch_stride; - const int8_t* K_cached_base = keys_cached + batch_idx * k_cached_batch_stride; - const int8_t* V_cached_base = values_cached + batch_idx * v_cached_batch_stride; - const __fp16* K_new_base = keys_new + batch_idx * k_new_batch_stride; - const __fp16* V_new_base = values_new + batch_idx * v_new_batch_stride; - __fp16* O_base = output + batch_idx * o_batch_stride; - - const __fp16* q_vec = Q_base + q_pos * q_seq_stride + q_head_idx * head_dim; - __fp16* o_vec = O_base + q_pos * o_seq_stride + q_head_idx * v_head_dim; - - float running_max = -std::numeric_limits::infinity(); - float running_sum = 0.0f; - - for (size_t i = 0; i < output_accum_low.size(); ++i) { - output_accum_low[i] = vdupq_n_f32(0.0f); - output_accum_high[i] = vdupq_n_f32(0.0f); - } - - const size_t absolute_q_pos = position_offset + q_pos; - size_t kv_end = is_causal ? std::min(kv_seq_len, cache_len + q_pos + 1) : kv_seq_len; - - size_t kv_start = sliding_window_kv_start(absolute_q_pos, window_size); - - constexpr size_t SINK_SIZE = 4; - const size_t cache_abs_offset = (position_offset >= cache_len) ? (position_offset - cache_len) : 0; - - const size_t kv_block_start0 = (window_size > 0 && kv_start > 0) ? 0 - : (kv_start / BLOCK_SIZE) * BLOCK_SIZE; - - for (size_t kv_block_start = kv_block_start0; kv_block_start < kv_end; kv_block_start += BLOCK_SIZE) { - const size_t kv_block_end = std::min(kv_block_start + BLOCK_SIZE, kv_end); - const size_t block_size = kv_block_end - kv_block_start; - - float block_max = -std::numeric_limits::infinity(); - - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - const size_t kv_pos = kv_block_start + kv_idx; - - bool window_masked = false; - if (window_size > 0 && kv_start > 0) { - if (kv_pos < cache_len) { - if (cache_abs_offset == 0 || kv_pos >= SINK_SIZE) { - window_masked = (cache_abs_offset + kv_pos < kv_start); - } - } else { - window_masked = (kv_pos + cache_abs_offset < kv_start); - } - } - - if ((is_causal && kv_pos > absolute_q_pos) || window_masked) { - block_scores[kv_idx] = -std::numeric_limits::infinity(); - continue; - } - - float32x4_t score_accum_low = vdupq_n_f32(0.0f); - float32x4_t score_accum_high = vdupq_n_f32(0.0f); - - if (kv_pos < cache_len) { - const int8_t* k_vec = K_cached_base + kv_pos * k_seq_stride + kv_head_idx * head_dim; - const float* k_scale_base = k_scales + (kv_pos * num_kv_heads + kv_head_idx) * num_quant_groups_k; - - for (size_t quant_group = 0; quant_group < num_quant_groups_k; quant_group++) { - const size_t dim_base = quant_group * quant_group_size; - const float k_scale = k_scale_base[quant_group]; - const float32x4_t k_scale_vec = vdupq_n_f32(k_scale); - - #pragma unroll - for (size_t i = 0; i < 4; i++) { - const size_t dim_block = dim_base + i * VECTOR_WIDTH; - if (dim_block >= head_dim_aligned) break; - - float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); - float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); - float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); - - int8x8_t k_vec_i8 = vld1_s8(&k_vec[dim_block]); - int16x8_t k_vec_i16 = vmovl_s8(k_vec_i8); - float32x4_t k_low = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(k_vec_i16))), k_scale_vec); - float32x4_t k_high = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(k_vec_i16))), k_scale_vec); - - score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); - score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); - } - } - } else { - const size_t new_pos = kv_pos - cache_len; - const __fp16* k_vec = K_new_base + new_pos * k_seq_stride + kv_head_idx * head_dim; - - for (size_t dim_block = 0; dim_block < head_dim_aligned; dim_block += VECTOR_WIDTH) { - float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); - float16x8_t k_vec_f16 = vld1q_f16(&k_vec[dim_block]); - - float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); - float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); - float32x4_t k_low = vcvt_f32_f16(vget_low_f16(k_vec_f16)); - float32x4_t k_high = vcvt_f32_f16(vget_high_f16(k_vec_f16)); - - score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); - score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); - } - } - - float score = vaddvq_f32(vaddq_f32(score_accum_low, score_accum_high)) * scale; - block_scores[kv_idx] = score; - block_max = std::max(block_max, score); - } - - float current_block_scale = 1.0f; - if (block_max > -std::numeric_limits::infinity()) { - if (block_max > running_max) { - float scale_correction = expf(running_max - block_max); - running_sum *= scale_correction; - - for (size_t i = 0; i < output_accum_low.size() / 2; ++i) { - output_accum_low[i] = vmulq_n_f32(output_accum_low[i], scale_correction); - output_accum_high[i] = vmulq_n_f32(output_accum_high[i], scale_correction); - } - running_max = block_max; - } else { - current_block_scale = expf(block_max - running_max); - } - } - - float block_sum = 0.0f; - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - if (block_scores[kv_idx] != -std::numeric_limits::infinity()) { - block_scores[kv_idx] = expf(block_scores[kv_idx] - block_max); - block_sum += block_scores[kv_idx]; - } else { - block_scores[kv_idx] = 0.0f; - } - } - - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - const float attn_weight = block_scores[kv_idx] * current_block_scale; - if (attn_weight == 0.0f) continue; - - const size_t kv_pos = kv_block_start + kv_idx; - const float32x4_t weight_vec = vdupq_n_f32(attn_weight); - - if (kv_pos < cache_len) { - const int8_t* v_vec = V_cached_base + kv_pos * v_seq_stride + kv_head_idx * v_head_dim; - const float* v_scale_base = v_scales + (kv_pos * num_kv_heads + kv_head_idx) * num_quant_groups_v; - - for (size_t quant_group = 0; quant_group < num_quant_groups_v; quant_group++) { - const size_t dim_base = quant_group * quant_group_size; - const float v_scale = v_scale_base[quant_group]; - const float32x4_t v_scale_vec = vdupq_n_f32(v_scale); - - #pragma unroll - for (size_t i = 0; i < 4; i++) { - const size_t dim_block = dim_base + i * VECTOR_WIDTH; - if (dim_block >= v_head_dim_aligned) break; - - int8x8_t v_vec_i8 = vld1_s8(&v_vec[dim_block]); - int16x8_t v_vec_i16 = vmovl_s8(v_vec_i8); - float32x4_t v_low = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_vec_i16))), v_scale_vec); - float32x4_t v_high = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_vec_i16))), v_scale_vec); - - size_t idx = dim_block / VECTOR_WIDTH; - output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); - output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); - } - } - } else { - const size_t new_pos = kv_pos - cache_len; - const __fp16* v_vec = V_new_base + new_pos * v_seq_stride + kv_head_idx * v_head_dim; - - for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { - float16x8_t v_vec_f16 = vld1q_f16(&v_vec[dim_block]); - float32x4_t v_low = vcvt_f32_f16(vget_low_f16(v_vec_f16)); - float32x4_t v_high = vcvt_f32_f16(vget_high_f16(v_vec_f16)); - - size_t idx = dim_block / VECTOR_WIDTH; - output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); - output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); - } - } - } - - running_sum += block_sum * current_block_scale; - } - - if (running_sum > 0.0f) { - const float inv_sum = 1.0f / running_sum; - const float32x4_t inv_sum_vec = vdupq_n_f32(inv_sum); - - for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { - size_t idx = dim_block / VECTOR_WIDTH; - float32x4_t final_low = vmulq_f32(output_accum_low[idx], inv_sum_vec); - float32x4_t final_high = vmulq_f32(output_accum_high[idx], inv_sum_vec); - - float16x4_t low_f16 = vcvt_f16_f32(final_low); - float16x4_t high_f16 = vcvt_f16_f32(final_high); - float16x8_t combined = vcombine_f16(low_f16, high_f16); - - vst1q_f16(&o_vec[dim_block], combined); - } - } else { - for (size_t dim = 0; dim < v_head_dim; ++dim) { - o_vec[dim] = static_cast<__fp16>(0.0f); - } - } - } - }); -} - -void cactus_rms_norm_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t batch_size, - size_t dims, - float eps -) { - constexpr size_t SIMD_WIDTH = 8; - constexpr size_t UNROLL_FACTOR = 2; - constexpr size_t TILE_SIZE = SIMD_WIDTH * UNROLL_FACTOR; - - for (size_t b = 0; b < batch_size; ++b) { - const __fp16* input_row = input + b * dims; - __fp16* output_row = output + b * dims; - - float32x4_t sum_squares_vec[UNROLL_FACTOR * 2]; - for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { - sum_squares_vec[u] = vdupq_n_f32(0.0f); - } - - size_t i = 0; - const size_t tile_end = (dims >= TILE_SIZE) ? dims - TILE_SIZE + 1 : 0; - - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); - sum_squares_vec[u * 2] = vfmaq_f32(sum_squares_vec[u * 2], input_low, input_low); - sum_squares_vec[u * 2 + 1] = vfmaq_f32(sum_squares_vec[u * 2 + 1], input_high, input_high); - } - } - - const size_t simd_end = (dims >= SIMD_WIDTH) ? dims - SIMD_WIDTH + 1 : 0; - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); - sum_squares_vec[0] = vfmaq_f32(sum_squares_vec[0], input_low, input_low); - sum_squares_vec[1] = vfmaq_f32(sum_squares_vec[1], input_high, input_high); - } - - float32x4_t total_sum = sum_squares_vec[0]; - for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { - total_sum = vaddq_f32(total_sum, sum_squares_vec[u]); - } - float sum_squares = vaddvq_f32(total_sum); - - for (; i < dims; ++i) { - float val = static_cast(input_row[i]); - sum_squares += val * val; - } - - float rms = sqrtf(sum_squares / static_cast(dims) + eps); - float inv_rms = 1.0f / rms; - float16x8_t inv_rms_vec = vdupq_n_f16(static_cast<__fp16>(inv_rms)); - - i = 0; - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); - float16x8_t norm_vec = vmulq_f16(vmulq_f16(input_vec, inv_rms_vec), weight_vec); - vst1q_f16(&output_row[i + u * SIMD_WIDTH], norm_vec); - } - } - - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float16x8_t weight_vec = vld1q_f16(&weight[i]); - float16x8_t norm_vec = vmulq_f16(vmulq_f16(input_vec, inv_rms_vec), weight_vec); - vst1q_f16(&output_row[i], norm_vec); - } - - for (; i < dims; ++i) { - output_row[i] = static_cast<__fp16>(static_cast(input_row[i]) * inv_rms * static_cast(weight[i])); - } - } -} - -void cactus_layer_norm_f16( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t batch_size, - size_t dims, - float eps -) { - constexpr size_t SIMD_WIDTH = 8; - constexpr size_t UNROLL_FACTOR = 3; - constexpr size_t TILE_SIZE = SIMD_WIDTH * UNROLL_FACTOR; - - const size_t tile_end = (dims >= TILE_SIZE) ? dims - TILE_SIZE + 1 : 0; - const size_t simd_end = (dims >= SIMD_WIDTH) ? dims - SIMD_WIDTH + 1 : 0; - - for (size_t b = 0; b < batch_size; ++b) { - const __fp16* input_row = input + b * dims; - __fp16* output_row = output + b * dims; - - float32x4_t sum_input_vec[UNROLL_FACTOR * 2]; - float32x4_t sum_squares_vec[UNROLL_FACTOR * 2]; - for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { - sum_input_vec[u] = vdupq_n_f32(0.0f); - sum_squares_vec[u] = vdupq_n_f32(0.0f); - } - - size_t i = 0; - - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); - - sum_input_vec[u * 2] = vaddq_f32(sum_input_vec[u * 2], input_low); - sum_input_vec[u * 2 + 1] = vaddq_f32(sum_input_vec[u * 2 + 1], input_high); - - sum_squares_vec[u * 2] = vfmaq_f32(sum_squares_vec[u * 2], input_low, input_low); - sum_squares_vec[u * 2 + 1] = vfmaq_f32(sum_squares_vec[u * 2 + 1], input_high, input_high); - } - } - - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); - sum_input_vec[0] = vaddq_f32(sum_input_vec[0], input_low); - sum_input_vec[1] = vaddq_f32(sum_input_vec[1], input_high); - sum_squares_vec[0] = vfmaq_f32(sum_squares_vec[0], input_low, input_low); - sum_squares_vec[1] = vfmaq_f32(sum_squares_vec[1], input_high, input_high); - } - - float32x4_t total_sum_inputs = sum_input_vec[0]; - float32x4_t total_sum_squares = sum_squares_vec[0]; - for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { - total_sum_inputs = vaddq_f32(total_sum_inputs, sum_input_vec[u]); - total_sum_squares = vaddq_f32(total_sum_squares, sum_squares_vec[u]); - } - - float sum_inputs = vaddvq_f32(total_sum_inputs); - float sum_squares = vaddvq_f32(total_sum_squares); - for (; i < dims; ++i) { - float val = static_cast(input_row[i]); - sum_inputs += val; - sum_squares += val * val; - } - - float mean = sum_inputs / static_cast(dims); - float mean_squares = sum_squares / static_cast(dims); - float variance = mean_squares - mean * mean; - if (variance < 0.0f) variance = 0.0f; - float inv_std = 1.0f / sqrtf(variance + eps); - - float16x8_t mean_vec = vdupq_n_f16(static_cast<__fp16>(mean)); - float16x8_t inv_std_vec = vdupq_n_f16(static_cast<__fp16>(inv_std)); - - i = 0; - if (bias) { - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); - float16x8_t bias_vec = vld1q_f16(&bias[i + u * SIMD_WIDTH]); - float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); - out_vec = vaddq_f16(out_vec, bias_vec); - vst1q_f16(&output_row[i + u * SIMD_WIDTH], out_vec); - } - } - - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float16x8_t weight_vec = vld1q_f16(&weight[i]); - float16x8_t bias_vec = vld1q_f16(&bias[i]); - float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); - out_vec = vaddq_f16(out_vec, bias_vec); - vst1q_f16(&output_row[i], out_vec); - } - - for (; i < dims; ++i) { - output_row[i] = static_cast<__fp16>((static_cast(input_row[i]) - mean) * inv_std * static_cast(weight[i]) + static_cast(bias[i])); - } - } else { - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); - float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); - vst1q_f16(&output_row[i + u * SIMD_WIDTH], out_vec); - } - } - - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float16x8_t weight_vec = vld1q_f16(&weight[i]); - float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); - vst1q_f16(&output_row[i], out_vec); - } - - for (; i < dims; ++i) { - output_row[i] = static_cast<__fp16>((static_cast(input_row[i]) - mean) * inv_std * static_cast(weight[i])); - } - } - } -} - -namespace CactusRoPEF16 { - -struct RoPECacheF16 { - std::vector<__fp16> cos_table; - std::vector<__fp16> sin_table; - size_t max_seq_len; - size_t head_dim; - float theta; - bool initialized; - - RoPECacheF16() : max_seq_len(0), head_dim(0), theta(0.0f), initialized(false) {} -}; - -static thread_local RoPECacheF16 rope_cache_f16; - -void precompute_rope_tables_f16(size_t seq_len, size_t head_dim, float theta) { - if (rope_cache_f16.initialized && - rope_cache_f16.max_seq_len >= seq_len && - rope_cache_f16.head_dim == head_dim && - rope_cache_f16.theta == theta) { - return; - } - - const size_t half_dim = head_dim / 2; - const size_t table_size = seq_len * half_dim; - - rope_cache_f16.cos_table.resize(table_size); - rope_cache_f16.sin_table.resize(table_size); - - for (size_t pos = 0; pos < seq_len; ++pos) { - const float pos_float = static_cast(pos); - for (size_t i = 0; i < half_dim; ++i) { - const float freq = 1.0f / powf(theta, (2.0f * i) / head_dim); - const float angle = pos_float * freq; - - const size_t idx = pos * half_dim + i; - rope_cache_f16.cos_table[idx] = static_cast<__fp16>(cosf(angle)); - rope_cache_f16.sin_table[idx] = static_cast<__fp16>(sinf(angle)); - } - } - - rope_cache_f16.max_seq_len = seq_len; - rope_cache_f16.head_dim = head_dim; - rope_cache_f16.theta = theta; - rope_cache_f16.initialized = true; -} - -} - -void cactus_rope_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t num_heads, - size_t head_dim, - size_t start_pos, - float theta -) { - const size_t half_dim = head_dim / 2; - - CactusRoPEF16::precompute_rope_tables_f16(seq_len + start_pos, head_dim, theta); - - const __fp16* cos_cache = CactusRoPEF16::rope_cache_f16.cos_table.data() + start_pos * half_dim; - const __fp16* sin_cache = CactusRoPEF16::rope_cache_f16.sin_table.data() + start_pos * half_dim; - - CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - for (size_t idx = start_idx; idx < end_idx; ++idx) { - const size_t batch_idx = idx / seq_len; - const size_t seq_idx = idx % seq_len; - - for (size_t head_idx = 0; head_idx < num_heads; ++head_idx) { - const size_t offset = ((batch_idx * seq_len + seq_idx) * num_heads + head_idx) * head_dim; - const __fp16* input_ptr = input + offset; - __fp16* output_ptr = output + offset; - - const __fp16* cos_ptr = cos_cache + seq_idx * half_dim; - const __fp16* sin_ptr = sin_cache + seq_idx * half_dim; - - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_half_dim = (half_dim / SIMD_WIDTH) * SIMD_WIDTH; - - for (size_t i = 0; i < vectorized_half_dim; i += SIMD_WIDTH) { - float16x8_t cos_vec = vld1q_f16(&cos_ptr[i]); - float16x8_t sin_vec = vld1q_f16(&sin_ptr[i]); - - float16x8_t x_first_half = vld1q_f16(&input_ptr[i]); - float16x8_t x_second_half = vld1q_f16(&input_ptr[i + half_dim]); - - float16x8_t first_result = vfmsq_f16(vmulq_f16(x_first_half, cos_vec), x_second_half, sin_vec); - float16x8_t second_result = vfmaq_f16(vmulq_f16(x_second_half, cos_vec), x_first_half, sin_vec); - - vst1q_f16(&output_ptr[i], first_result); - vst1q_f16(&output_ptr[i + half_dim], second_result); - } - - for (size_t i = vectorized_half_dim; i < half_dim; ++i) { - const __fp16 cos_val = cos_ptr[i]; - const __fp16 sin_val = sin_ptr[i]; - - const __fp16 x_first_half = input_ptr[i]; - const __fp16 x_second_half = input_ptr[i + half_dim]; - - output_ptr[i] = x_first_half * cos_val - x_second_half * sin_val; - - output_ptr[i + half_dim] = x_second_half * cos_val + x_first_half * sin_val; - } - } - } - }); -} - -void cactus_gpt_j_rope_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t num_heads, - size_t head_dim, - size_t rot_dim, - size_t start_pos, - float theta -) { - const size_t half_rot_dim = rot_dim / 2; - - CactusRoPEF16::precompute_rope_tables_f16(seq_len + start_pos, rot_dim, theta); - - const __fp16* cos_cache = CactusRoPEF16::rope_cache_f16.cos_table.data() + start_pos * half_rot_dim; - const __fp16* sin_cache = CactusRoPEF16::rope_cache_f16.sin_table.data() + start_pos * half_rot_dim; - - CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - for (size_t idx = start_idx; idx < end_idx; ++idx) { - const size_t batch_idx = idx / seq_len; - const size_t seq_idx = idx % seq_len; - - for (size_t head_idx = 0; head_idx < num_heads; ++head_idx) { - const size_t offset = ((batch_idx * seq_len + seq_idx) * num_heads + head_idx) * head_dim; - const __fp16* input_ptr = input + offset; - __fp16* output_ptr = output + offset; - - const __fp16* cos_ptr = cos_cache + seq_idx * half_rot_dim; - const __fp16* sin_ptr = sin_cache + seq_idx * half_rot_dim; - - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_half_rot_dim = (half_rot_dim / SIMD_WIDTH) * SIMD_WIDTH; - - for (size_t i = 0; i < vectorized_half_rot_dim; i += SIMD_WIDTH) { - float16x8_t cos_vec = vld1q_f16(&cos_ptr[i]); - float16x8_t sin_vec = vld1q_f16(&sin_ptr[i]); - - float16x8x2_t x_vec = vld2q_f16(&input_ptr[2*i]); - float16x8_t x_first_half = x_vec.val[0]; - float16x8_t x_second_half = x_vec.val[1]; - - float16x8_t first_result = vfmsq_f16(vmulq_f16(x_first_half, cos_vec), x_second_half, sin_vec); - float16x8_t second_result = vfmaq_f16(vmulq_f16(x_second_half, cos_vec), x_first_half, sin_vec); - - float16x8x2_t t; - t.val[0] = first_result; - t.val[1] = second_result; - vst2q_f16(&output_ptr[2*i], t); - } - - for (size_t i = vectorized_half_rot_dim; i < half_rot_dim; ++i) { - const __fp16 cos_val = cos_ptr[i]; - const __fp16 sin_val = sin_ptr[i]; - - const __fp16 x_first_half = input_ptr[2*i]; - const __fp16 x_second_half = input_ptr[2*i + 1]; - - output_ptr[2*i] = x_first_half * cos_val - x_second_half * sin_val; - - output_ptr[2*i + 1] = x_second_half * cos_val + x_first_half * sin_val; - } - - constexpr size_t TAIL_SIMD_WIDTH = 8; - size_t copy_idx = rot_dim; - const size_t copy_end_vec = (head_dim / TAIL_SIMD_WIDTH) * TAIL_SIMD_WIDTH; - - for (; copy_idx + TAIL_SIMD_WIDTH <= copy_end_vec; copy_idx += TAIL_SIMD_WIDTH) { - float16x8_t v = vld1q_f16(&input_ptr[copy_idx]); - vst1q_f16(&output_ptr[copy_idx], v); - } - for (; copy_idx < head_dim; ++copy_idx) { - output_ptr[copy_idx] = input_ptr[copy_idx]; - } - } - } - }); -} diff --git a/cactus-transpile/kernel/kernel_bilstm.cpp b/cactus-transpile/kernel/kernel_bilstm.cpp deleted file mode 100644 index 2b4ea2321..000000000 --- a/cactus-transpile/kernel/kernel_bilstm.cpp +++ /dev/null @@ -1,370 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#endif - -static void apply_lstm_gates_f32( - const float* __restrict gates, - float* __restrict c, - float* __restrict h, - size_t hidden_size -) { - const float32x4_t one = vdupq_n_f32(1.0f); - size_t i = 0; - - for (; i + 4 <= hidden_size; i += 4) { - float32x4_t i_raw = vld1q_f32(gates + i); - float32x4_t f_raw = vld1q_f32(gates + hidden_size + i); - float32x4_t g_raw = vld1q_f32(gates + 2 * hidden_size + i); - float32x4_t o_raw = vld1q_f32(gates + 3 * hidden_size + i); - - float32x4_t i_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_raw)))); - float32x4_t f_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_raw)))); - float32x4_t g_act = fast_tanh_f32x4(g_raw); - float32x4_t o_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_raw)))); - - float32x4_t c_prev = vld1q_f32(c + i); - float32x4_t c_new = vfmaq_f32(vmulq_f32(f_act, c_prev), i_act, g_act); - vst1q_f32(c + i, c_new); - - vst1q_f32(h + i, vmulq_f32(o_act, fast_tanh_f32x4(c_new))); - } - - for (; i < hidden_size; ++i) { - float ig = 1.0f / (1.0f + expf(-gates[i])); - float fg = 1.0f / (1.0f + expf(-gates[hidden_size + i])); - float gg = tanhf(gates[2 * hidden_size + i]); - float og = 1.0f / (1.0f + expf(-gates[3 * hidden_size + i])); - float cv = fg * c[i] + ig * gg; - c[i] = cv; - h[i] = og * tanhf(cv); - } -} - -#ifndef __APPLE__ -static void apply_lstm_gates_f16( - const __fp16* __restrict gates, - __fp16* __restrict c, - __fp16* __restrict h, - size_t hidden_size -) { - constexpr size_t W = 8; - const size_t simd_end = (hidden_size / W) * W; - const float32x4_t one = vdupq_n_f32(1.0f); - - for (size_t i = 0; i < simd_end; i += W) { - float16x8_t i_raw = vld1q_f16(gates + i); - float16x8_t f_raw = vld1q_f16(gates + hidden_size + i); - float16x8_t g_raw = vld1q_f16(gates + 2 * hidden_size + i); - float16x8_t o_raw = vld1q_f16(gates + 3 * hidden_size + i); - - float32x4_t il = vcvt_f32_f16(vget_low_f16(i_raw)); - float32x4_t ih = vcvt_f32_f16(vget_high_f16(i_raw)); - float16x8_t i_act = vcombine_f16( - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(il))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(ih)))))); - - float32x4_t fl = vcvt_f32_f16(vget_low_f16(f_raw)); - float32x4_t fh = vcvt_f32_f16(vget_high_f16(f_raw)); - float16x8_t f_act = vcombine_f16( - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(fl))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(fh)))))); - - float32x4_t gl = vcvt_f32_f16(vget_low_f16(g_raw)); - float32x4_t gh = vcvt_f32_f16(vget_high_f16(g_raw)); - float16x8_t g_act = vcombine_f16( - vcvt_f16_f32(fast_tanh_f32x4(gl)), - vcvt_f16_f32(fast_tanh_f32x4(gh))); - - float32x4_t ol = vcvt_f32_f16(vget_low_f16(o_raw)); - float32x4_t oh = vcvt_f32_f16(vget_high_f16(o_raw)); - float16x8_t o_act = vcombine_f16( - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(ol))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(oh)))))); - - float16x8_t c_prev = vld1q_f16(c + i); - float16x8_t c_new = vfmaq_f16(vmulq_f16(f_act, c_prev), i_act, g_act); - vst1q_f16(c + i, c_new); - - float32x4_t cl = vcvt_f32_f16(vget_low_f16(c_new)); - float32x4_t ch = vcvt_f32_f16(vget_high_f16(c_new)); - float16x8_t c_tanh = vcombine_f16( - vcvt_f16_f32(fast_tanh_f32x4(cl)), - vcvt_f16_f32(fast_tanh_f32x4(ch))); - vst1q_f16(h + i, vmulq_f16(o_act, c_tanh)); - } - - for (size_t i = simd_end; i < hidden_size; ++i) { - float ig = 1.0f / (1.0f + expf(-static_cast(gates[i]))); - float fg = 1.0f / (1.0f + expf(-static_cast(gates[hidden_size + i]))); - float gg = tanhf(static_cast(gates[2 * hidden_size + i])); - float og = 1.0f / (1.0f + expf(-static_cast(gates[3 * hidden_size + i]))); - float cv = fg * static_cast(c[i]) + ig * gg; - c[i] = static_cast<__fp16>(cv); - h[i] = static_cast<__fp16>(og * tanhf(cv)); - } -} - -static inline float hsum_f16x8_f32(float16x8_t v) { - float16x4_t lo = vget_low_f16(v); - float16x4_t hi = vget_high_f16(v); - float16x4_t s4 = vadd_f16(lo, hi); - float16x4_t s2 = vadd_f16(s4, vext_f16(s4, s4, 2)); - float16x4_t s1 = vadd_f16(s2, vext_f16(s2, s2, 1)); - return static_cast(vget_lane_f16(s1, 0)); -} - -static void lstm_gemv_f16_neon( - const __fp16* __restrict x, - const __fp16* __restrict W, - const __fp16* __restrict bias, - __fp16* __restrict out, - size_t K, size_t N -) { - const size_t K16 = (K / 16) * 16; - const size_t K8 = (K / 8) * 8; - size_t n = 0; - for (; n + 4 <= N; n += 4) { - const __fp16* w0 = W + n * K; - const __fp16* w1 = W + (n + 1) * K; - const __fp16* w2 = W + (n + 2) * K; - const __fp16* w3 = W + (n + 3) * K; - float16x8_t a0 = vdupq_n_f16(0); - float16x8_t a1 = vdupq_n_f16(0); - float16x8_t a2 = vdupq_n_f16(0); - float16x8_t a3 = vdupq_n_f16(0); - for (size_t k = 0; k < K16; k += 16) { - float16x8_t xlo = vld1q_f16(x + k); - float16x8_t xhi = vld1q_f16(x + k + 8); - a0 = vfmaq_f16(a0, xlo, vld1q_f16(w0 + k)); - a0 = vfmaq_f16(a0, xhi, vld1q_f16(w0 + k + 8)); - a1 = vfmaq_f16(a1, xlo, vld1q_f16(w1 + k)); - a1 = vfmaq_f16(a1, xhi, vld1q_f16(w1 + k + 8)); - a2 = vfmaq_f16(a2, xlo, vld1q_f16(w2 + k)); - a2 = vfmaq_f16(a2, xhi, vld1q_f16(w2 + k + 8)); - a3 = vfmaq_f16(a3, xlo, vld1q_f16(w3 + k)); - a3 = vfmaq_f16(a3, xhi, vld1q_f16(w3 + k + 8)); - } - for (size_t k = K16; k < K8; k += 8) { - float16x8_t xv = vld1q_f16(x + k); - a0 = vfmaq_f16(a0, xv, vld1q_f16(w0 + k)); - a1 = vfmaq_f16(a1, xv, vld1q_f16(w1 + k)); - a2 = vfmaq_f16(a2, xv, vld1q_f16(w2 + k)); - a3 = vfmaq_f16(a3, xv, vld1q_f16(w3 + k)); - } - float s0 = hsum_f16x8_f32(a0); - float s1 = hsum_f16x8_f32(a1); - float s2 = hsum_f16x8_f32(a2); - float s3 = hsum_f16x8_f32(a3); - for (size_t k = K8; k < K; ++k) { - float xv = static_cast(x[k]); - s0 += xv * static_cast(w0[k]); - s1 += xv * static_cast(w1[k]); - s2 += xv * static_cast(w2[k]); - s3 += xv * static_cast(w3[k]); - } - out[n] = static_cast<__fp16>(s0 + static_cast(bias[n])); - out[n + 1] = static_cast<__fp16>(s1 + static_cast(bias[n + 1])); - out[n + 2] = static_cast<__fp16>(s2 + static_cast(bias[n + 2])); - out[n + 3] = static_cast<__fp16>(s3 + static_cast(bias[n + 3])); - } - for (; n < N; ++n) { - const __fp16* w = W + n * K; - float16x8_t acc = vdupq_n_f16(0); - for (size_t k = 0; k < K8; k += 8) - acc = vfmaq_f16(acc, vld1q_f16(x + k), vld1q_f16(w + k)); - float s = hsum_f16x8_f32(acc); - for (size_t k = K8; k < K; ++k) - s += static_cast(x[k]) * static_cast(w[k]); - out[n] = static_cast<__fp16>(s + static_cast(bias[n])); - } -} -#endif - -static inline void fp16_to_fp32_neon(const __fp16* src, float* dst, size_t n) { - size_t i = 0; - for (; i + 8 <= n; i += 8) { - float16x8_t v = vld1q_f16(src + i); - vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(v))); - } - for (; i < n; ++i) - dst[i] = static_cast(src[i]); -} - -static inline void fp32_to_fp16_neon(const float* src, __fp16* dst, size_t n) { - size_t i = 0; - for (; i + 8 <= n; i += 8) { - float32x4_t lo = vld1q_f32(src + i); - float32x4_t hi = vld1q_f32(src + i + 4); - vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi))); - } - for (; i < n; ++i) - dst[i] = static_cast<__fp16>(src[i]); -} - -void cactus_bilstm_sequence_f16( - const __fp16* input, - const __fp16* weight_ih_fwd, - const __fp16* weight_hh_fwd, - const __fp16* bias_ih_fwd, - const __fp16* bias_hh_fwd, - const __fp16* weight_ih_bwd, - const __fp16* weight_hh_bwd, - const __fp16* bias_ih_bwd, - const __fp16* bias_hh_bwd, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t input_size, - size_t hidden_size -) { - const size_t gate_size = 4 * hidden_size; - const size_t combined_K = input_size + hidden_size; - const size_t output_size = 2 * hidden_size; - - std::vector bias_fwd_f32(gate_size); - std::vector bias_bwd_f32(gate_size); - for (size_t g = 0; g < gate_size; ++g) { - bias_fwd_f32[g] = static_cast(bias_ih_fwd[g]) + static_cast(bias_hh_fwd[g]); - bias_bwd_f32[g] = static_cast(bias_ih_bwd[g]) + static_cast(bias_hh_bwd[g]); - } - -#ifdef __APPLE__ - std::vector W_fwd_f32(gate_size * combined_K); - std::vector W_bwd_f32(gate_size * combined_K); - for (size_t g = 0; g < gate_size; ++g) { - fp16_to_fp32_neon(weight_ih_fwd + g * input_size, - W_fwd_f32.data() + g * combined_K, input_size); - fp16_to_fp32_neon(weight_hh_fwd + g * hidden_size, - W_fwd_f32.data() + g * combined_K + input_size, hidden_size); - fp16_to_fp32_neon(weight_ih_bwd + g * input_size, - W_bwd_f32.data() + g * combined_K, input_size); - fp16_to_fp32_neon(weight_hh_bwd + g * hidden_size, - W_bwd_f32.data() + g * combined_K + input_size, hidden_size); - } - - for (size_t b = 0; b < batch_size; ++b) { - const __fp16* batch_in = input + b * seq_len * input_size; - __fp16* batch_out = output + b * seq_len * output_size; - - auto& pool = CactusThreading::get_thread_pool(); - auto bwd_future = pool.enqueue([&, batch_in, batch_out]() { - std::vector xh(combined_K); - std::vector gates(gate_size); - std::vector h(hidden_size, 0.0f); - std::vector c(hidden_size, 0.0f); - - for (size_t idx = 0; idx < seq_len; ++idx) { - const size_t t = seq_len - 1 - idx; - fp16_to_fp32_neon(batch_in + t * input_size, xh.data(), input_size); - memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(float)); - - memcpy(gates.data(), bias_bwd_f32.data(), gate_size * sizeof(float)); - cblas_sgemv(CblasRowMajor, CblasNoTrans, - static_cast(gate_size), static_cast(combined_K), - 1.0f, W_bwd_f32.data(), static_cast(combined_K), - xh.data(), 1, 1.0f, gates.data(), 1); - - apply_lstm_gates_f32(gates.data(), c.data(), h.data(), hidden_size); - - fp32_to_fp16_neon(h.data(), batch_out + t * output_size + hidden_size, hidden_size); - } - }); - - { - std::vector xh(combined_K); - std::vector gates(gate_size); - std::vector h(hidden_size, 0.0f); - std::vector c(hidden_size, 0.0f); - - for (size_t t = 0; t < seq_len; ++t) { - fp16_to_fp32_neon(batch_in + t * input_size, xh.data(), input_size); - memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(float)); - - memcpy(gates.data(), bias_fwd_f32.data(), gate_size * sizeof(float)); - cblas_sgemv(CblasRowMajor, CblasNoTrans, - static_cast(gate_size), static_cast(combined_K), - 1.0f, W_fwd_f32.data(), static_cast(combined_K), - xh.data(), 1, 1.0f, gates.data(), 1); - - apply_lstm_gates_f32(gates.data(), c.data(), h.data(), hidden_size); - - fp32_to_fp16_neon(h.data(), batch_out + t * output_size, hidden_size); - } - } - - bwd_future.get(); - } - -#else - std::vector<__fp16> W_fwd_f16(gate_size * combined_K); - std::vector<__fp16> W_bwd_f16(gate_size * combined_K); - for (size_t g = 0; g < gate_size; ++g) { - memcpy(W_fwd_f16.data() + g * combined_K, - weight_ih_fwd + g * input_size, input_size * sizeof(__fp16)); - memcpy(W_fwd_f16.data() + g * combined_K + input_size, - weight_hh_fwd + g * hidden_size, hidden_size * sizeof(__fp16)); - memcpy(W_bwd_f16.data() + g * combined_K, - weight_ih_bwd + g * input_size, input_size * sizeof(__fp16)); - memcpy(W_bwd_f16.data() + g * combined_K + input_size, - weight_hh_bwd + g * hidden_size, hidden_size * sizeof(__fp16)); - } - - std::vector<__fp16> bias_fwd_f16(gate_size); - std::vector<__fp16> bias_bwd_f16(gate_size); - for (size_t g = 0; g < gate_size; ++g) { - bias_fwd_f16[g] = static_cast<__fp16>(bias_fwd_f32[g]); - bias_bwd_f16[g] = static_cast<__fp16>(bias_bwd_f32[g]); - } - - for (size_t b = 0; b < batch_size; ++b) { - const __fp16* batch_in = input + b * seq_len * input_size; - __fp16* batch_out = output + b * seq_len * output_size; - - auto& pool = CactusThreading::get_thread_pool(); - auto bwd_future = pool.enqueue([&, batch_in, batch_out]() { - std::vector<__fp16> xh(combined_K); - std::vector<__fp16> gates(gate_size); - std::vector<__fp16> h(hidden_size, static_cast<__fp16>(0)); - std::vector<__fp16> c(hidden_size, static_cast<__fp16>(0)); - - for (size_t idx = 0; idx < seq_len; ++idx) { - const size_t t = seq_len - 1 - idx; - memcpy(xh.data(), batch_in + t * input_size, input_size * sizeof(__fp16)); - memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(__fp16)); - lstm_gemv_f16_neon(xh.data(), W_bwd_f16.data(), bias_bwd_f16.data(), - gates.data(), combined_K, gate_size); - apply_lstm_gates_f16(gates.data(), c.data(), h.data(), hidden_size); - memcpy(batch_out + t * output_size + hidden_size, - h.data(), hidden_size * sizeof(__fp16)); - } - }); - - { - std::vector<__fp16> xh(combined_K); - std::vector<__fp16> gates(gate_size); - std::vector<__fp16> h(hidden_size, static_cast<__fp16>(0)); - std::vector<__fp16> c(hidden_size, static_cast<__fp16>(0)); - - for (size_t t = 0; t < seq_len; ++t) { - memcpy(xh.data(), batch_in + t * input_size, input_size * sizeof(__fp16)); - memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(__fp16)); - lstm_gemv_f16_neon(xh.data(), W_fwd_f16.data(), bias_fwd_f16.data(), - gates.data(), combined_K, gate_size); - apply_lstm_gates_f16(gates.data(), c.data(), h.data(), hidden_size); - memcpy(batch_out + t * output_size, - h.data(), hidden_size * sizeof(__fp16)); - } - } - - bwd_future.get(); - } -#endif -} diff --git a/cactus-transpile/kernel/kernel_blas.cpp b/cactus-transpile/kernel/kernel_blas.cpp deleted file mode 100644 index fba63880b..000000000 --- a/cactus-transpile/kernel/kernel_blas.cpp +++ /dev/null @@ -1,639 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include -#include - -static inline size_t compute_linear_index(const size_t* coords, const size_t* strides, size_t ndim) { - size_t index = 0; - for (size_t i = 0; i < ndim; ++i) { - index += coords[i] * strides[i]; - } - return index; -} - -static inline void increment_coords(size_t* coords, const size_t* shape, size_t ndim) { - for (int i = ndim - 1; i >= 0; --i) { - coords[i]++; - if (coords[i] < shape[i]) { - break; - } - coords[i] = 0; - } -} - -enum class BroadcastOp { ADD, SUB, MUL, DIV, NE }; - -template -static inline float16x8_t broadcast_op_vec(float16x8_t a, float16x8_t b) { - if constexpr (Op == BroadcastOp::ADD) return vaddq_f16(a, b); - else if constexpr (Op == BroadcastOp::SUB) return vsubq_f16(a, b); - else if constexpr (Op == BroadcastOp::MUL) return vmulq_f16(a, b); - else if constexpr (Op == BroadcastOp::DIV) return vdivq_f16(a, b); - else return vreinterpretq_f16_u16(vdupq_n_u16(0)); -} - -template -static inline __fp16 broadcast_op_scalar(__fp16 a, __fp16 b) { - if constexpr (Op == BroadcastOp::ADD) return a + b; - else if constexpr (Op == BroadcastOp::SUB) return a - b; - else if constexpr (Op == BroadcastOp::MUL) return a * b; - else if constexpr (Op == BroadcastOp::DIV) return a / b; - else return static_cast<__fp16>(a != b ? 1.0f : 0.0f); -} - -template -static void broadcast_op_optimized(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - size_t total_elements = 1; - for (size_t i = 0; i < ndim; ++i) { - total_elements *= output_shape[i]; - } - - if (total_elements == 0) return; - - size_t inner_size = output_shape[ndim - 1]; - bool a_inner_contiguous = (a_strides[ndim - 1] == 1) || (a_strides[ndim - 1] == 0); - bool b_inner_contiguous = (b_strides[ndim - 1] == 1) || (b_strides[ndim - 1] == 0); - bool a_inner_broadcast = (a_strides[ndim - 1] == 0); - bool b_inner_broadcast = (b_strides[ndim - 1] == 0); - - size_t outer_size = total_elements / inner_size; - - if constexpr (Op == BroadcastOp::NE) { - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_outer, size_t end_outer) { - std::vector coords(ndim, 0); - - size_t tmp = start_outer; - for (int i = ndim - 2; i >= 0; --i) { - coords[i] = tmp % output_shape[i]; - tmp /= output_shape[i]; - } - - for (size_t outer_idx = start_outer; outer_idx < end_outer; ++outer_idx) { - size_t a_base = 0, b_base = 0; - for (size_t i = 0; i < ndim - 1; ++i) { - a_base += coords[i] * a_strides[i]; - b_base += coords[i] * b_strides[i]; - } - - __fp16* out_ptr = output + outer_idx * inner_size; - for (size_t i = 0; i < inner_size; ++i) { - const size_t a_idx = a_base + (a_inner_broadcast ? 0 : i); - const size_t b_idx = b_base + (b_inner_broadcast ? 0 : i); - out_ptr[i] = static_cast<__fp16>(a[a_idx] != b[b_idx] ? 1.0f : 0.0f); - } - - increment_coords(coords.data(), output_shape, ndim); - } - }); - return; - } - - if (a_inner_contiguous && b_inner_contiguous && inner_size >= 8) { - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_outer, size_t end_outer) { - std::vector coords(ndim, 0); - - size_t tmp = start_outer; - for (int i = ndim - 2; i >= 0; --i) { - coords[i] = tmp % output_shape[i]; - tmp /= output_shape[i]; - } - - for (size_t outer_idx = start_outer; outer_idx < end_outer; ++outer_idx) { - size_t a_base = 0, b_base = 0; - for (size_t i = 0; i < ndim - 1; ++i) { - a_base += coords[i] * a_strides[i]; - b_base += coords[i] * b_strides[i]; - } - - __fp16* out_ptr = output + outer_idx * inner_size; - const size_t vec_end = (inner_size / 8) * 8; - - const bool use_stream = total_elements >= STREAMING_STORE_THRESHOLD; - - if (a_inner_broadcast && b_inner_broadcast) { - __fp16 result = broadcast_op_scalar(a[a_base], b[b_base]); - float16x8_t result_vec = vdupq_n_f16(result); - if (use_stream) { - for (size_t i = 0; i < vec_end; i += 8) { - stream_store_f16x8(out_ptr + i, result_vec); - } - } else { - for (size_t i = 0; i < vec_end; i += 8) { - vst1q_f16(out_ptr + i, result_vec); - } - } - for (size_t i = vec_end; i < inner_size; ++i) { - out_ptr[i] = result; - } - } else if (a_inner_broadcast) { - float16x8_t a_vec = vdupq_n_f16(a[a_base]); - const __fp16* b_ptr = b + b_base; - if (use_stream) { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t b_vec = vld1q_f16(b_ptr + i); - stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } else { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t b_vec = vld1q_f16(b_ptr + i); - vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } - for (size_t i = vec_end; i < inner_size; ++i) { - out_ptr[i] = broadcast_op_scalar(a[a_base], b_ptr[i]); - } - } else if (b_inner_broadcast) { - const __fp16* a_ptr = a + a_base; - float16x8_t b_vec = vdupq_n_f16(b[b_base]); - if (use_stream) { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t a_vec = vld1q_f16(a_ptr + i); - stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } else { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t a_vec = vld1q_f16(a_ptr + i); - vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } - for (size_t i = vec_end; i < inner_size; ++i) { - out_ptr[i] = broadcast_op_scalar(a_ptr[i], b[b_base]); - } - } else { - const __fp16* a_ptr = a + a_base; - const __fp16* b_ptr = b + b_base; - if (use_stream) { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t a_vec = vld1q_f16(a_ptr + i); - float16x8_t b_vec = vld1q_f16(b_ptr + i); - stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } else { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t a_vec = vld1q_f16(a_ptr + i); - float16x8_t b_vec = vld1q_f16(b_ptr + i); - vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } - for (size_t i = vec_end; i < inner_size; ++i) { - out_ptr[i] = broadcast_op_scalar(a_ptr[i], b_ptr[i]); - } - } - - for (int i = ndim - 2; i >= 0; --i) { - coords[i]++; - if (coords[i] < output_shape[i]) break; - coords[i] = 0; - } - } - }); - } else { - CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - std::vector coords(ndim); - - size_t tmp = start_idx; - for (int i = ndim - 1; i >= 0; --i) { - coords[i] = tmp % output_shape[i]; - tmp /= output_shape[i]; - } - - for (size_t linear_idx = start_idx; linear_idx < end_idx; ++linear_idx) { - size_t a_idx = compute_linear_index(coords.data(), a_strides, ndim); - size_t b_idx = compute_linear_index(coords.data(), b_strides, ndim); - - output[linear_idx] = broadcast_op_scalar(a[a_idx], b[b_idx]); - - increment_coords(coords.data(), output_shape, ndim); - } - }); - } -} - -void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - constexpr size_t UNROLL = 4; - const size_t unrolled_end = start_idx + ((end_idx - start_idx) / (SIMD_WIDTH * UNROLL)) * (SIMD_WIDTH * UNROLL); - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - if (use_streaming) { - for (size_t i = start_idx; i < unrolled_end; i += SIMD_WIDTH * UNROLL) { - __builtin_prefetch(&a[i + 256], 0, 0); - __builtin_prefetch(&b[i + 256], 0, 0); - float16x8_t v0 = vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i])); - float16x8_t v1 = vaddq_f16(vld1q_f16(&a[i + 8]), vld1q_f16(&b[i + 8])); - float16x8_t v2 = vaddq_f16(vld1q_f16(&a[i + 16]), vld1q_f16(&b[i + 16])); - float16x8_t v3 = vaddq_f16(vld1q_f16(&a[i + 24]), vld1q_f16(&b[i + 24])); - stream_store_f16x8(&output[i], v0); - stream_store_f16x8(&output[i + 8], v1); - stream_store_f16x8(&output[i + 16], v2); - stream_store_f16x8(&output[i + 24], v3); - } - for (size_t i = unrolled_end; i < vectorized_end; i += SIMD_WIDTH) { - stream_store_f16x8(&output[i], vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i]))); - } - } else { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - vst1q_f16(&output[i], vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i]))); - } - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - output[i] = a[i] + b[i]; - } - }); -} - -void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - constexpr float FP16_MAX = 65500.0f; - const float32x4_t max_val = vdupq_n_f32(FP16_MAX); - const float32x4_t min_val = vdupq_n_f32(-FP16_MAX); - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - - float32x4_t a_low = vcvt_f32_f16(vget_low_f16(a_vec)); - float32x4_t a_high = vcvt_f32_f16(vget_high_f16(a_vec)); - float32x4_t b_low = vcvt_f32_f16(vget_low_f16(b_vec)); - float32x4_t b_high = vcvt_f32_f16(vget_high_f16(b_vec)); - - float32x4_t result_low = vaddq_f32(a_low, b_low); - float32x4_t result_high = vaddq_f32(a_high, b_high); - - result_low = vminq_f32(vmaxq_f32(result_low, min_val), max_val); - result_high = vminq_f32(vmaxq_f32(result_high, min_val), max_val); - - float16x4_t result_low_f16 = vcvt_f16_f32(result_low); - float16x4_t result_high_f16 = vcvt_f16_f32(result_high); - float16x8_t result_vec = vcombine_f16(result_low_f16, result_high_f16); - - vst1q_f16(&output[i], result_vec); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float result = static_cast(a[i]) + static_cast(b[i]); - result = std::fmin(std::fmax(result, -FP16_MAX), FP16_MAX); - output[i] = static_cast<__fp16>(result); - } - }); -} - -void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - if (use_streaming) { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - stream_store_f16x8(&output[i], vsubq_f16(a_vec, b_vec)); - } - } else { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - vst1q_f16(&output[i], vsubq_f16(a_vec, b_vec)); - } - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - output[i] = a[i] - b[i]; - } - }); -} - -void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - if (use_streaming) { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - stream_store_f16x8(&output[i], vmulq_f16(a_vec, b_vec)); - } - } else { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - vst1q_f16(&output[i], vmulq_f16(a_vec, b_vec)); - } - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - output[i] = a[i] * b[i]; - } - }); -} - -void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale) { - constexpr size_t SIMD_WIDTH = 8; - const float32x4_t vscale = vdupq_n_f32(scale); - const size_t vec_end = (num_elements / SIMD_WIDTH) * SIMD_WIDTH; - - for (size_t i = 0; i < vec_end; i += SIMD_WIDTH) { - float16x8_t base_vec = vld1q_f16(base + i); - float16x8_t src_vec = vld1q_f16(src + i); - - float32x4_t base_lo = vcvt_f32_f16(vget_low_f16(base_vec)); - float32x4_t base_hi = vcvt_f32_f16(vget_high_f16(base_vec)); - float32x4_t src_lo = vcvt_f32_f16(vget_low_f16(src_vec)); - float32x4_t src_hi = vcvt_f32_f16(vget_high_f16(src_vec)); - - float32x4_t result_lo = vfmaq_f32(base_lo, src_lo, vscale); - float32x4_t result_hi = vfmaq_f32(base_hi, src_hi, vscale); - - vst1q_f16(output + i, vcombine_f16(vcvt_f16_f32(result_lo), vcvt_f16_f32(result_hi))); - } - for (size_t i = vec_end; i < num_elements; ++i) { - output[i] = static_cast<__fp16>(static_cast(base[i]) - + static_cast(src[i]) * scale); - } -} - -void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - if (use_streaming) { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - stream_store_f16x8(&output[i], vdivq_f16(a_vec, b_vec)); - } - } else { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - vst1q_f16(&output[i], vdivq_f16(a_vec, b_vec)); - } - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - output[i] = a[i] / b[i]; - } - }); -} - -void cactus_not_equal_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - for (size_t i = start_idx; i < end_idx; ++i) { - output[i] = static_cast<__fp16>(a[i] != b[i] ? 1.0f : 0.0f); - } - }); -} - -void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_not_equal_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, - const size_t* shape1, const size_t* shape2, const size_t* output_shape, - size_t ndims, int axis) { - if (axis < 0) axis += ndims; - - size_t outer_size = 1; - for (size_t i = 0; i < static_cast(axis); ++i) { - outer_size *= output_shape[i]; - } - - size_t inner_size = 1; - for (size_t i = axis + 1; i < ndims; ++i) { - inner_size *= output_shape[i]; - } - - size_t axis_size1 = shape1[axis]; - size_t axis_size2 = shape2[axis]; - - size_t input1_stride = axis_size1 * inner_size; - size_t input2_stride = axis_size2 * inner_size; - size_t output_stride = (axis_size1 + axis_size2) * inner_size; - - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start, size_t end) { - for (size_t outer = start; outer < end; ++outer) { - const __fp16* in1_ptr = input1 + outer * input1_stride; - const __fp16* in2_ptr = input2 + outer * input2_stride; - __fp16* out_ptr = output + outer * output_stride; - - size_t copy_size1 = axis_size1 * inner_size; - std::memcpy(out_ptr, in1_ptr, copy_size1 * sizeof(__fp16)); - - size_t copy_size2 = axis_size2 * inner_size; - std::memcpy(out_ptr + copy_size1, in2_ptr, copy_size2 * sizeof(__fp16)); - } - }); -} - -void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, - const size_t* output_shape, size_t num_inputs, size_t rank, int axis) { - if (axis < 0) axis += rank; - - size_t outer_size = 1; - for (size_t i = 0; i < static_cast(axis); ++i) { - outer_size *= output_shape[i]; - } - - size_t inner_size = 1; - for (size_t i = axis + 1; i < rank; ++i) { - inner_size *= output_shape[i]; - } - - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start, size_t end) { - for (size_t outer = start; outer < end; ++outer) { - __fp16* out_ptr = output + outer * inner_size * output_shape[axis]; - size_t offset = 0; - - for (size_t input_idx = 0; input_idx < num_inputs; ++input_idx) { - const __fp16* in_ptr = inputs[input_idx] + outer * inner_size * input_shapes[input_idx][axis]; - size_t copy_size = input_shapes[input_idx][axis] * inner_size; - std::memcpy(out_ptr + offset, in_ptr, copy_size * sizeof(__fp16)); - offset += copy_size; - } - } - }); -} - -void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, size_t num_rows, size_t num_cols, size_t start_row, size_t end_row) { - constexpr size_t TILE_SIZE = 32; - constexpr size_t VECTOR_WIDTH = 8; - - for (size_t row_tile_start = start_row; row_tile_start < end_row; row_tile_start += TILE_SIZE) { - const size_t row_tile_end = std::min(row_tile_start + TILE_SIZE, end_row); - - for (size_t col_tile_start = 0; col_tile_start < num_cols; col_tile_start += TILE_SIZE) { - const size_t col_tile_end = std::min(col_tile_start + TILE_SIZE, num_cols); - - for (size_t row_block = row_tile_start; row_block < row_tile_end; row_block += VECTOR_WIDTH) { - const size_t row_block_end = std::min(row_block + VECTOR_WIDTH, row_tile_end); - - for (size_t col_block = col_tile_start; col_block < col_tile_end; col_block += VECTOR_WIDTH) { - const size_t col_block_end = std::min(col_block + VECTOR_WIDTH, col_tile_end); - - if (row_block_end - row_block >= 8 && col_block_end - col_block >= 8) { - float16x8_t rows[8]; - for (int i = 0; i < 8; i++) { - if (row_block + i < row_block_end) { - rows[i] = vld1q_f16(&source[(row_block + i) * num_cols + col_block]); - } else { - rows[i] = vdupq_n_f16(0.0f); - } - } - - float16x8x2_t r01 = vtrnq_f16(rows[0], rows[1]); - float16x8x2_t r23 = vtrnq_f16(rows[2], rows[3]); - float16x8x2_t r45 = vtrnq_f16(rows[4], rows[5]); - float16x8x2_t r67 = vtrnq_f16(rows[6], rows[7]); - - float32x4x2_t r0123_lo = vtrnq_f32( - vreinterpretq_f32_f16(r01.val[0]), - vreinterpretq_f32_f16(r23.val[0])); - float32x4x2_t r0123_hi = vtrnq_f32( - vreinterpretq_f32_f16(r01.val[1]), - vreinterpretq_f32_f16(r23.val[1])); - float32x4x2_t r4567_lo = vtrnq_f32( - vreinterpretq_f32_f16(r45.val[0]), - vreinterpretq_f32_f16(r67.val[0])); - float32x4x2_t r4567_hi = vtrnq_f32( - vreinterpretq_f32_f16(r45.val[1]), - vreinterpretq_f32_f16(r67.val[1])); - - float16x8_t col0 = vreinterpretq_f16_f32(vcombine_f32( - vget_low_f32(r0123_lo.val[0]), vget_low_f32(r4567_lo.val[0]))); - float16x8_t col1 = vreinterpretq_f16_f32(vcombine_f32( - vget_low_f32(r0123_hi.val[0]), vget_low_f32(r4567_hi.val[0]))); - float16x8_t col2 = vreinterpretq_f16_f32(vcombine_f32( - vget_low_f32(r0123_lo.val[1]), vget_low_f32(r4567_lo.val[1]))); - float16x8_t col3 = vreinterpretq_f16_f32(vcombine_f32( - vget_low_f32(r0123_hi.val[1]), vget_low_f32(r4567_hi.val[1]))); - float16x8_t col4 = vreinterpretq_f16_f32(vcombine_f32( - vget_high_f32(r0123_lo.val[0]), vget_high_f32(r4567_lo.val[0]))); - float16x8_t col5 = vreinterpretq_f16_f32(vcombine_f32( - vget_high_f32(r0123_hi.val[0]), vget_high_f32(r4567_hi.val[0]))); - float16x8_t col6 = vreinterpretq_f16_f32(vcombine_f32( - vget_high_f32(r0123_lo.val[1]), vget_high_f32(r4567_lo.val[1]))); - float16x8_t col7 = vreinterpretq_f16_f32(vcombine_f32( - vget_high_f32(r0123_hi.val[1]), vget_high_f32(r4567_hi.val[1]))); - - float16x8_t cols[8] = {col0, col1, col2, col3, col4, col5, col6, col7}; - for (int c = 0; c < 8 && col_block + c < col_block_end; c++) { - if (col_block + c < num_cols) { - if (row_block_end - row_block >= 8) { - vst1q_f16(&destination[(col_block + c) * num_rows + row_block], cols[c]); - } else { - __fp16 temp[8]; - vst1q_f16(temp, cols[c]); - for (size_t i = 0; i < row_block_end - row_block; ++i) { - destination[(col_block + c) * num_rows + row_block + i] = temp[i]; - } - } - } - } - } else { - for (size_t row = row_block; row < row_block_end; row++) { - for (size_t col = col_block; col < col_block_end; col++) { - destination[col * num_rows + row] = source[row * num_cols + col]; - } - } - } - } - } - } - } -} - -void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx) { - if (ndim == 2 && permutation[0] == 1 && permutation[1] == 0) { - size_t num_rows = shape[0]; - size_t num_cols = shape[1]; - - constexpr size_t THRESHOLD = 8192; - constexpr size_t TILE_ROWS = 32; - if (num_rows * num_cols >= THRESHOLD) { - const size_t num_row_blocks = (num_rows + TILE_ROWS - 1) / TILE_ROWS; - - CactusThreading::parallel_for(num_row_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [=](size_t start_block, size_t end_block) { - for (size_t block_idx = start_block; block_idx < end_block; ++block_idx) { - size_t start_row = block_idx * TILE_ROWS; - size_t end_row = std::min(start_row + TILE_ROWS, num_rows); - - cactus_transpose_2d_f16(source, destination, num_rows, num_cols, start_row, end_row); - } - }); - } else { - cactus_transpose_2d_f16(source, destination, num_rows, num_cols, 0, num_rows); - } - } else { - for (size_t idx = start_idx; idx < end_idx; ++idx) { - size_t src_idx = 0; - size_t tmp_idx = idx; - - for (size_t i = 0; i < ndim; ++i) { - size_t coord = tmp_idx % shape[permutation[ndim - 1 - i]]; - tmp_idx /= shape[permutation[ndim - 1 - i]]; - - size_t stride = 1; - for (size_t j = permutation[ndim - 1 - i] + 1; j < ndim; ++j) { - stride *= shape[j]; - } - src_idx += coord * stride; - } - - destination[idx] = source[src_idx]; - } - } -} diff --git a/cactus-transpile/kernel/kernel_conv.cpp b/cactus-transpile/kernel/kernel_conv.cpp deleted file mode 100644 index c1d6b4f52..000000000 --- a/cactus-transpile/kernel/kernel_conv.cpp +++ /dev/null @@ -1,1663 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#endif - -constexpr size_t T_TILE_F16 = 2; -constexpr size_t ACCELERATE_K_THRESHOLD = 32; -constexpr size_t ACCELERATE_L_THRESHOLD = 128; - -namespace { - -void fp16_rowmajor_gemm_rhs_transposed_to_fp32( - const __fp16* lhs, - const __fp16* rhs, - float* output, - size_t M, - size_t K, - size_t N -) { - if (M == 0 || K == 0 || N == 0) { - return; - } - -#ifdef __APPLE__ - std::vector lhs_f32(M * K); - std::vector rhs_f32(N * K); - for (size_t i = 0; i < lhs_f32.size(); ++i) { - lhs_f32[i] = static_cast(lhs[i]); - } - for (size_t i = 0; i < rhs_f32.size(); ++i) { - rhs_f32[i] = static_cast(rhs[i]); - } - - cblas_sgemm( - CblasRowMajor, - CblasNoTrans, - CblasTrans, - static_cast(M), - static_cast(N), - static_cast(K), - 1.0f, - lhs_f32.data(), - static_cast(K), - rhs_f32.data(), - static_cast(K), - 0.0f, - output, - static_cast(N) - ); - return; -#endif - - CactusThreading::parallel_for( - M, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t row_start, size_t row_end) { - for (size_t m = row_start; m < row_end; ++m) { - const __fp16* lhs_row = lhs + m * K; - float* out_row = output + m * N; - for (size_t n = 0; n < N; ++n) { - const __fp16* rhs_row = rhs + n * K; - float acc = 0.0f; - for (size_t k = 0; k < K; ++k) { - acc += static_cast(lhs_row[k]) * static_cast(rhs_row[k]); - } - out_row[n] = acc; - } - } - } - ); -} - -} // namespace - -#ifdef __APPLE__ -static void conv1d_causal_depthwise_f16_accelerate( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, size_t C, size_t K, size_t dilation) -{ - const size_t in_bs = L * C; - const size_t out_bs = L * C; - CactusThreading::parallel_for_2d(N, C, CactusThreading::Thresholds::ATTENTION, [&](size_t n, size_t c) { - const __fp16* Xb = input + n * in_bs; - __fp16* Yb = output + n * out_bs; - const __fp16* Wc = weight + c * K; - - if (dilation == 1) { - - size_t i = 0; - for(; i + 7 < L; i += 8){ - float16x8_t acc = vdupq_n_f16(0.0f); - - for (size_t k = 0; k < K; ++k) { - float16x8_t input_vec = vld1q_f16(&Xb[(i + k) * C + c]); - float16x8_t weight_vec = vdupq_n_f16(Wc[k]); - acc = vfmaq_f16(acc, input_vec, weight_vec); - } - vst1q_f16(&Yb[i * C + c], acc); - } - - for (; i < L; ++i) { - float acc = 0.0f; - for (size_t k = 0; k < K; ++k) { - acc += float(Xb[(i + k) * C + c]) * float(Wc[k]); - } - Yb[i * C + c] = (__fp16)acc; - } - } else { - - for (size_t i = 0; i < L; ++i) { - float acc = 0.0f; - - for (size_t k = 0; k < K; ++k) { - size_t idx = i + k * dilation; - acc += float(Xb[idx * C + c]) * float(Wc[k]); - } - - Yb[i * C + c] = (__fp16)acc; - } - } - }); -} -#endif - -void cactus_conv1d_causal_depthwise_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, size_t C, size_t K, size_t dilation) -{ -#ifdef __APPLE__ - if (K >= ACCELERATE_K_THRESHOLD && L >= ACCELERATE_L_THRESHOLD) { - conv1d_causal_depthwise_f16_accelerate(input, weight, output, N, L, C, K, dilation); - return; - } -#endif - - const size_t in_bs = L * C; - const size_t out_bs = L * C; - - CactusThreading::parallel_for_2d(N, C, CactusThreading::Thresholds::ATTENTION, [&](size_t n, size_t c) { - const __fp16* Xb = input + n * in_bs; - __fp16* Yb = output + n * out_bs; - - std::vector wrev(K); - const __fp16* Wc = weight + c * K; - for (size_t k = 0; k < K; ++k) wrev[k] = (float)Wc[K - 1 - k]; - - for (size_t t0 = 0; t0 < L; t0 += T_TILE_F16) { - const size_t t1 = std::min(t0 + 1, L - 1); - - float32x4_t vacc0 = vdupq_n_f32(0.f); - float32x4_t vacc1 = vdupq_n_f32(0.f); - - size_t k = 0; - for (; k + 8 <= K; k += 8) { - - float x0_0=0, x1_0=0, x2_0=0, x3_0=0; - float x0_1=0, x1_1=0, x2_1=0, x3_1=0; - { - ptrdiff_t a0=(ptrdiff_t)t0-(ptrdiff_t)((k+0)*dilation); - ptrdiff_t a1=(ptrdiff_t)t0-(ptrdiff_t)((k+1)*dilation); - ptrdiff_t a2=(ptrdiff_t)t0-(ptrdiff_t)((k+2)*dilation); - ptrdiff_t a3=(ptrdiff_t)t0-(ptrdiff_t)((k+3)*dilation); - if (a0>=0) x0_0 = (float)Xb[(size_t)a0*C + c]; - if (a1>=0) x1_0 = (float)Xb[(size_t)a1*C + c]; - if (a2>=0) x2_0 = (float)Xb[(size_t)a2*C + c]; - if (a3>=0) x3_0 = (float)Xb[(size_t)a3*C + c]; - - ptrdiff_t b0=(ptrdiff_t)t1-(ptrdiff_t)((k+0)*dilation); - ptrdiff_t b1=(ptrdiff_t)t1-(ptrdiff_t)((k+1)*dilation); - ptrdiff_t b2=(ptrdiff_t)t1-(ptrdiff_t)((k+2)*dilation); - ptrdiff_t b3=(ptrdiff_t)t1-(ptrdiff_t)((k+3)*dilation); - if (b0>=0) x0_1 = (float)Xb[(size_t)b0*C + c]; - if (b1>=0) x1_1 = (float)Xb[(size_t)b1*C + c]; - if (b2>=0) x2_1 = (float)Xb[(size_t)b2*C + c]; - if (b3>=0) x3_1 = (float)Xb[(size_t)b3*C + c]; - } - float32x4_t xv0 = {x0_0,x1_0,x2_0,x3_0}; - float32x4_t yv0 = {x0_1,x1_1,x2_1,x3_1}; - float32x4_t wv0 = {wrev[k+0],wrev[k+1],wrev[k+2],wrev[k+3]}; - vacc0 = vfmaq_f32(vacc0, xv0, wv0); - vacc1 = vfmaq_f32(vacc1, yv0, wv0); - - float a0_0=0, a1_0=0, a2_0=0, a3_0=0; - float a0_1=0, a1_1=0, a2_1=0, a3_1=0; - { - ptrdiff_t a0i=(ptrdiff_t)t0-(ptrdiff_t)((k+4)*dilation); - ptrdiff_t a1i=(ptrdiff_t)t0-(ptrdiff_t)((k+5)*dilation); - ptrdiff_t a2i=(ptrdiff_t)t0-(ptrdiff_t)((k+6)*dilation); - ptrdiff_t a3i=(ptrdiff_t)t0-(ptrdiff_t)((k+7)*dilation); - if (a0i>=0) a0_0 = (float)Xb[(size_t)a0i*C + c]; - if (a1i>=0) a1_0 = (float)Xb[(size_t)a1i*C + c]; - if (a2i>=0) a2_0 = (float)Xb[(size_t)a2i*C + c]; - if (a3i>=0) a3_0 = (float)Xb[(size_t)a3i*C + c]; - - ptrdiff_t b0i=(ptrdiff_t)t1-(ptrdiff_t)((k+4)*dilation); - ptrdiff_t b1i=(ptrdiff_t)t1-(ptrdiff_t)((k+5)*dilation); - ptrdiff_t b2i=(ptrdiff_t)t1-(ptrdiff_t)((k+6)*dilation); - ptrdiff_t b3i=(ptrdiff_t)t1-(ptrdiff_t)((k+7)*dilation); - if (b0i>=0) a0_1 = (float)Xb[(size_t)b0i*C + c]; - if (b1i>=0) a1_1 = (float)Xb[(size_t)b1i*C + c]; - if (b2i>=0) a2_1 = (float)Xb[(size_t)b2i*C + c]; - if (b3i>=0) a3_1 = (float)Xb[(size_t)b3i*C + c]; - } - float32x4_t xv1 = {a0_0,a1_0,a2_0,a3_0}; - float32x4_t yv1 = {a0_1,a1_1,a2_1,a3_1}; - float32x4_t wv1 = {wrev[k+4],wrev[k+5],wrev[k+6],wrev[k+7]}; - vacc0 = vfmaq_f32(vacc0, xv1, wv1); - vacc1 = vfmaq_f32(vacc1, yv1, wv1); - } - - float acc0 = vaddvq_f32(vacc0); - float acc1 = vaddvq_f32(vacc1); - - for (; k < K; ++k) { - ptrdiff_t a=(ptrdiff_t)t0-(ptrdiff_t)(k*dilation); - if (a>=0) acc0 += wrev[k] * (float)Xb[(size_t)a*C + c]; - ptrdiff_t b=(ptrdiff_t)t1-(ptrdiff_t)(k*dilation); - if (b>=0) acc1 += wrev[k] * (float)Xb[(size_t)b*C + c]; - } - - Yb[t0*C + c] = (__fp16)acc0; - if (t0 + 1 < L) Yb[t1*C + c] = (__fp16)acc1; - } - }); -} - -void cactus_conv1d_f16_k3( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t stride -){ - const size_t out_len = ((L - 1) / stride) + 1; - const size_t in_bs = C_in * L; - const size_t out_bs = C_out * out_len; - - const size_t total_compute = N * C_out * out_len * C_in * 3; - CactusThreading::ParallelConfig config = (total_compute < 100000) - ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d(N, C_out, config, [&](size_t n, size_t oc) { - const __fp16* Xb = input + n * in_bs; - __fp16* Yoc = output + n * out_bs + oc * out_len; - const __fp16* Woc = weight + oc * (C_in * 3); - - for (size_t out_idx = 0; out_idx < out_len; out_idx += 2) { - const size_t out_t0 = out_idx; - const bool have_t1 = (out_idx + 1) < out_len; - const size_t out_t1 = have_t1 ? (out_idx + 1) : out_idx; - - const size_t t0 = out_t0 * stride; - const size_t t1 = have_t1 ? (out_t1 * stride) : t0; - - float32x4_t acc0 = vdupq_n_f32(0.f); - float32x4_t acc1 = vdupq_n_f32(0.f); - - size_t ic = 0; - for (; ic + 16 <= C_in; ic += 16) { - for (size_t u = 0; u < 16; ++u) { - const __fp16* Xc = Xb + (ic + u) * L; - const __fp16* Wc = Woc + (ic + u) * 3; - - const float16x8_t wv = { - Wc[0], Wc[1], Wc[2], (__fp16)0, - Wc[0], Wc[1], Wc[2], (__fp16)0 - }; - - const ptrdiff_t tm0 = (ptrdiff_t)t0 - 1; - const ptrdiff_t tp0 = (ptrdiff_t)t0 + 1; - const ptrdiff_t tm1 = (ptrdiff_t)t1 - 1; - const ptrdiff_t tp1 = (ptrdiff_t)t1 + 1; - - const __fp16 x0m = (tm0 >= 0) ? Xc[tm0] : (__fp16)0; - const __fp16 x00 = Xc[t0]; - const __fp16 x0p = (tp0 < (ptrdiff_t)L) ? Xc[tp0] : (__fp16)0; - - __fp16 x1m = 0, x10 = 0, x1p = 0; - if (have_t1) { - x1m = (tm1 >= 0) ? Xc[tm1] : (__fp16)0; - x10 = Xc[t1]; - x1p = (tp1 < (ptrdiff_t)L) ? Xc[tp1] : (__fp16)0; - } - - const float16x8_t xv = { - x0m, x00, x0p, (__fp16)0, - x1m, x10, x1p, (__fp16)0 - }; - - const float16x4_t xv0_h = vget_low_f16(xv); - const float16x4_t wv0_h = vget_low_f16(wv); - acc0 = vfmaq_f32(acc0, vcvt_f32_f16(xv0_h), vcvt_f32_f16(wv0_h)); - - if (have_t1) { - const float16x4_t xv1_h = vget_high_f16(xv); - const float16x4_t wv1_h = vget_high_f16(wv); - acc1 = vfmaq_f32(acc1, vcvt_f32_f16(xv1_h), vcvt_f32_f16(wv1_h)); - } - } - } - - for (; ic < C_in; ++ic) { - const __fp16* Xc = Xb + ic * L; - const __fp16* Wc = Woc + ic * 3; - - const float16x8_t wv = { - Wc[0], Wc[1], Wc[2], (__fp16)0, - Wc[0], Wc[1], Wc[2], (__fp16)0 - }; - - const ptrdiff_t tm0 = (ptrdiff_t)t0 - 1; - const ptrdiff_t tp0 = (ptrdiff_t)t0 + 1; - const ptrdiff_t tm1 = (ptrdiff_t)t1 - 1; - const ptrdiff_t tp1 = (ptrdiff_t)t1 + 1; - - const __fp16 x0m = (tm0 >= 0) ? Xc[tm0] : (__fp16)0; - const __fp16 x00 = Xc[t0]; - const __fp16 x0p = (tp0 < (ptrdiff_t)L) ? Xc[tp0] : (__fp16)0; - - __fp16 x1m = 0, x10 = 0, x1p = 0; - if (have_t1) { - x1m = (tm1 >= 0) ? Xc[tm1] : (__fp16)0; - x10 = Xc[t1]; - x1p = (tp1 < (ptrdiff_t)L) ? Xc[tp1] : (__fp16)0; - } - - const float16x8_t xv = { - x0m, x00, x0p, (__fp16)0, - x1m, x10, x1p, (__fp16)0 - }; - - const float16x4_t xv0_h = vget_low_f16(xv); - const float16x4_t wv0_h = vget_low_f16(wv); - acc0 = vfmaq_f32(acc0, vcvt_f32_f16(xv0_h), vcvt_f32_f16(wv0_h)); - - if (have_t1) { - const float16x4_t xv1_h = vget_high_f16(xv); - const float16x4_t wv1_h = vget_high_f16(wv); - acc1 = vfmaq_f32(acc1, vcvt_f32_f16(xv1_h), vcvt_f32_f16(wv1_h)); - } - } - - float32x2_t s0 = vadd_f32(vget_low_f32(acc0), vget_high_f32(acc0)); - float sum0 = vget_lane_f32(s0, 0) + vget_lane_f32(s0, 1); - Yoc[out_t0] = (__fp16)sum0; - - if (have_t1) { - float32x2_t s1 = vadd_f32(vget_low_f32(acc1), vget_high_f32(acc1)); - float sum1 = vget_lane_f32(s1, 0) + vget_lane_f32(s1, 1); - Yoc[out_t1] = (__fp16)sum1; - } - } - }); -} - -#ifdef __APPLE__ -static void conv1d_f16_accelerate( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, - size_t stride -){ - const size_t out_len = ((L - K) / stride) + 1; - const size_t in_bs = C_in * L; - const size_t out_bs = C_out * out_len; - - const size_t total_compute = N * C_out * out_len * C_in * K; - CactusThreading::ParallelConfig config = (total_compute < 100000) - ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d( - N, C_out, config, - [&](size_t n, size_t oc) { - - const __fp16* Xb = input + n * in_bs; - __fp16* Yoc = output + n * out_bs + oc * out_len; - const __fp16* Woc = weight + oc * (C_in * K); - const float b = bias ? (float)bias[oc] : 0.f; - - std::vector out_f32(out_len, b); - std::vector input_f32(L); - std::vector weight_f32(K); - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xb + ic * L; - const __fp16* Wc = Woc + ic * K; - - for (size_t i = 0; i < L; ++i) input_f32[i] = (float)Xc[i]; - for (size_t k = 0; k < K; ++k) weight_f32[k] = (float)Wc[k]; - - if (stride == 1) { - std::vector conv_out(out_len); - vDSP_conv(input_f32.data(), 1, weight_f32.data(), 1, - conv_out.data(), 1, out_len, K); - vDSP_vadd(out_f32.data(), 1, conv_out.data(), 1, - out_f32.data(), 1, out_len); - } else { - std::vector full_conv(L - K + 1); - vDSP_conv(input_f32.data(), 1, weight_f32.data(), 1, - full_conv.data(), 1, L - K + 1, K); - for (size_t out_t = 0; out_t < out_len; ++out_t) { - out_f32[out_t] += full_conv[out_t * stride]; - } - } - } - - for (size_t out_t = 0; out_t < out_len; ++out_t) { - Yoc[out_t] = (__fp16)out_f32[out_t]; - } - }); -} -#endif - -static void conv1d_f16_neon( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, - size_t stride -){ - const size_t out_len = ((L - K) / stride) + 1; - const size_t in_bs = C_in * L; - const size_t out_bs = C_out * out_len; - - const size_t total_compute = N * C_out * out_len * C_in * K; - CactusThreading::ParallelConfig config = (total_compute < 100000) - ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d( - N, C_out, config, - [&](size_t n, size_t oc) { - - const __fp16* Xb = input + n * in_bs; - __fp16* Yoc = output + n * out_bs + oc * out_len; - const __fp16* Woc = weight + oc * (C_in * K); - const float b = bias ? (float)bias[oc] : 0.f; - - for (size_t out_t = 0; out_t < out_len; ++out_t) { - const size_t t = out_t * stride; - float sum = b; - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xb + ic * L + t; - const __fp16* Wc = Woc + ic * K; - - float32x4_t acc0 = vdupq_n_f32(0.f); - float32x4_t acc1 = vdupq_n_f32(0.f); - - size_t k = 0; - - for (; k + 8 <= K; k += 8) { - const float16x8_t xv = vld1q_f16(Xc + k); - const float16x8_t wv = vld1q_f16(Wc + k); - - acc0 = vfmaq_f32(acc0, - vcvt_f32_f16(vget_low_f16(xv)), - vcvt_f32_f16(vget_low_f16(wv))); - acc1 = vfmaq_f32(acc1, - vcvt_f32_f16(vget_high_f16(xv)), - vcvt_f32_f16(vget_high_f16(wv))); - } - - float32x2_t s1 = vadd_f32(vget_low_f32(acc0), vget_high_f32(acc0)); - sum += vget_lane_f32(s1, 0) + vget_lane_f32(s1, 1); - - float32x2_t s2 = vadd_f32(vget_low_f32(acc1), vget_high_f32(acc1)); - sum += vget_lane_f32(s2, 0) + vget_lane_f32(s2, 1); - - for (; k < K; ++k) { - sum += (float)Xc[k] * (float)Wc[k]; - } - } - - Yoc[out_t] = (__fp16)sum; - } - }); -} - -#ifdef __APPLE__ -static void conv1d_f16_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, - size_t stride -) { - const size_t out_len = (L - K) / stride + 1; - const size_t col_K = C_in * K; - - std::vector W_f32(C_out * col_K); - for (size_t i = 0; i < C_out * col_K; ++i) - W_f32[i] = static_cast(weight[i]); - - std::vector bias_f32; - if (bias) { - bias_f32.resize(C_out); - for (size_t i = 0; i < C_out; ++i) - bias_f32[i] = static_cast(bias[i]); - } - - std::vector col(col_K * out_len); - std::vector Y_f32(C_out * out_len); - - for (size_t n = 0; n < N; ++n) { - const __fp16* Xn = input + n * C_in * L; - __fp16* Yn = output + n * C_out * out_len; - - if (stride == 1) { - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xn + ic * L; - for (size_t k = 0; k < K; ++k) { - float* dst = col.data() + (ic * K + k) * out_len; - const __fp16* src = Xc + k; - size_t t = 0; - for (; t + 8 <= out_len; t += 8) { - float16x8_t v = vld1q_f16(src + t); - vst1q_f32(dst + t, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(dst + t + 4, vcvt_f32_f16(vget_high_f16(v))); - } - for (; t < out_len; ++t) - dst[t] = static_cast(src[t]); - } - } - } else { - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xn + ic * L; - for (size_t k = 0; k < K; ++k) { - float* dst = col.data() + (ic * K + k) * out_len; - for (size_t t = 0; t < out_len; ++t) - dst[t] = static_cast(Xc[t * stride + k]); - } - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - static_cast(C_out), static_cast(out_len), static_cast(col_K), - 1.0f, W_f32.data(), static_cast(col_K), - col.data(), static_cast(out_len), - 0.0f, Y_f32.data(), static_cast(out_len)); - - if (bias) { - for (size_t oc = 0; oc < C_out; ++oc) { - float b = bias_f32[oc]; - const float* src = Y_f32.data() + oc * out_len; - __fp16* dst = Yn + oc * out_len; - size_t t = 0; - for (; t + 8 <= out_len; t += 8) { - float32x4_t v0 = vaddq_f32(vld1q_f32(src + t), vdupq_n_f32(b)); - float32x4_t v1 = vaddq_f32(vld1q_f32(src + t + 4), vdupq_n_f32(b)); - vst1q_f16(dst + t, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - } - for (; t < out_len; ++t) - dst[t] = static_cast<__fp16>(src[t] + b); - } - } else { - for (size_t oc = 0; oc < C_out; ++oc) { - const float* src = Y_f32.data() + oc * out_len; - __fp16* dst = Yn + oc * out_len; - size_t t = 0; - for (; t + 8 <= out_len; t += 8) { - float32x4_t v0 = vld1q_f32(src + t); - float32x4_t v1 = vld1q_f32(src + t + 4); - vst1q_f16(dst + t, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - } - for (; t < out_len; ++t) - dst[t] = static_cast<__fp16>(src[t]); - } - } - } -} -#endif - -void cactus_conv1d_f16( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, - size_t stride -){ -#ifdef __APPLE__ - if (stride > 1 || K < ACCELERATE_K_THRESHOLD) { - conv1d_f16_gemm(input, weight, bias, output, N, L, C_in, C_out, K, stride); - return; - } - if (K >= ACCELERATE_K_THRESHOLD && L >= ACCELERATE_L_THRESHOLD) { - conv1d_f16_accelerate(input, weight, bias, output, N, L, C_in, C_out, K, stride); - return; - } -#endif - conv1d_f16_neon(input, weight, bias, output, N, L, C_in, C_out, K, stride); -} - -inline void conv1d_k7s3_oc8_t4( - const __fp16* Xb, - const __fp16* Wpack, - const __fp16* bias, - __fp16* Yb, - size_t L, - size_t out_len, - size_t C_in, - size_t C_out, - size_t out_t, - size_t oc0 -){ - float32x4_t acc[4][2]; - float32x4_t b0 = vdupq_n_f32(0.f); - float32x4_t b1 = vdupq_n_f32(0.f); - if (bias) { - float16x8_t bv = vld1q_f16(bias + oc0); - b0 = vcvt_f32_f16(vget_low_f16(bv)); - b1 = vcvt_f32_f16(vget_high_f16(bv)); - } - - for (int j = 0; j < 4; ++j) { - acc[j][0] = b0; - acc[j][1] = b1; - } - - const size_t t_base = out_t * 3; - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Wic = Wpack + (ic * 7) * C_out + oc0; - const __fp16* Xic = Xb + ic * L + t_base; - - for (int k = 0; k < 7; ++k) { - float16x8_t w_half = vld1q_f16(Wic + k * C_out); - float32x4_t w0 = vcvt_f32_f16(vget_low_f16(w_half)); - float32x4_t w1 = vcvt_f32_f16(vget_high_f16(w_half)); - for (int j = 0; j < 4; ++j) { - float x_val = (float)Xic[j * 3 + k]; - float32x4_t xv = vdupq_n_f32(x_val); - - acc[j][0] = vfmaq_f32(acc[j][0], w0, xv); - acc[j][1] = vfmaq_f32(acc[j][1], w1, xv); - } - } - } - - float tmp0[4], tmp1[4]; - for(int j=0; j<4; ++j) { - vst1q_f32(tmp0, acc[j][0]); - vst1q_f32(tmp1, acc[j][1]); - - for(int i=0; i<4; ++i) { - Yb[(oc0 + i) * out_len + out_t + j] = (__fp16)tmp0[i]; - } - for(int i=0; i<4; ++i) { - Yb[(oc0 + 4 + i) * out_len + out_t + j] = (__fp16)tmp1[i]; - } - } -} - -inline void conv1d_k7s3_oc8_scalar( - const __fp16* Xb, - const __fp16* Wpack, - const __fp16* bias, - __fp16* Yb, - size_t L, - size_t out_len, - size_t C_in, - size_t C_out, - size_t out_t, - size_t oc0 -){ - float32x4_t acc0 = vdupq_n_f32(0.f); - float32x4_t acc1 = vdupq_n_f32(0.f); - - if (bias) { - float16x8_t bv = vld1q_f16(bias + oc0); - acc0 = vcvt_f32_f16(vget_low_f16(bv)); - acc1 = vcvt_f32_f16(vget_high_f16(bv)); - } - - const size_t t_base = out_t * 3; - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Wic = Wpack + (ic * 7) * C_out + oc0; - const __fp16* Xic = Xb + ic * L + t_base; - for (int k = 0; k < 7; ++k) { - float x_val = (float)Xic[k]; - float32x4_t xv = vdupq_n_f32(x_val); - - float16x8_t w_half = vld1q_f16(Wic + k * C_out); - acc0 = vfmaq_f32(acc0, vcvt_f32_f16(vget_low_f16(w_half)), xv); - acc1 = vfmaq_f32(acc1, vcvt_f32_f16(vget_high_f16(w_half)), xv); - } - } - - float tmp[8]; - vst1q_f32(tmp, acc0); - vst1q_f32(tmp+4, acc1); - - for(int i=0; i<8; ++i) { - if (oc0 + i < C_out) { - Yb[(oc0 + i) * out_len + out_t] = (__fp16)tmp[i]; - } - } -} - -void cactus_conv1d_f16_k7s3_oc8( - const __fp16* input, - const __fp16* Wpack, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out) -{ - if (L < 7) return; - size_t out_len = (L - 7) / 3 + 1; - size_t num_oc_blocks = (C_out + 7) / 8; - - CactusThreading::parallel_for_2d(N, num_oc_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, [=](size_t n, size_t ob) { - size_t oc0 = ob * 8; - const __fp16* Xb = input + n * (C_in * L); - __fp16* Yb = output + n * (C_out * out_len); - - size_t out_t = 0; - for (; out_t + 4 <= out_len; out_t += 4) { - conv1d_k7s3_oc8_t4(Xb, Wpack, bias, Yb, L, out_len, C_in, C_out, out_t, oc0); - } - - for (; out_t < out_len; ++out_t) { - conv1d_k7s3_oc8_scalar(Xb, Wpack, bias, Yb, L, out_len, C_in, C_out, out_t, oc0); - } - }); -} - - - -void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, - size_t dst_height, size_t dst_width, bool align_corners) -{ - float scale_h, scale_w; - if (align_corners) { - scale_h = (src_height > 1 && dst_height > 1) - ? static_cast(src_height - 1) / static_cast(dst_height - 1) - : 0.0f; - scale_w = (src_width > 1 && dst_width > 1) - ? static_cast(src_width - 1) / static_cast(dst_width - 1) - : 0.0f; - } else { - scale_h = (dst_height > 0) - ? static_cast(src_height) / static_cast(dst_height) - : 0.0f; - scale_w = (dst_width > 0) - ? static_cast(src_width) / static_cast(dst_width) - : 0.0f; - } - - for (size_t dst_y = 0; dst_y < dst_height; ++dst_y) { - for (size_t dst_x = 0; dst_x < dst_width; ++dst_x) { - float src_y_float, src_x_float; - if (align_corners) { - src_y_float = static_cast(dst_y) * scale_h; - src_x_float = static_cast(dst_x) * scale_w; - } else { - src_y_float = (static_cast(dst_y) + 0.5f) * scale_h - 0.5f; - src_x_float = (static_cast(dst_x) + 0.5f) * scale_w - 0.5f; - if (src_y_float < 0.0f) src_y_float = 0.0f; - if (src_x_float < 0.0f) src_x_float = 0.0f; - const float max_y = static_cast(src_height) - 1.0f; - const float max_x = static_cast(src_width) - 1.0f; - if (src_y_float > max_y) src_y_float = max_y; - if (src_x_float > max_x) src_x_float = max_x; - } - - int y0 = static_cast(std::floor(src_y_float)); - int x0 = static_cast(std::floor(src_x_float)); - - int y1 = ((y0 + 1) < static_cast(src_height)) ? (y0 + 1) : (static_cast(src_height) - 1); - int x1 = ((x0 + 1) < static_cast(src_width)) ? (x0 + 1) : (static_cast(src_width) - 1); - - float dy = src_y_float - y0; - float dx = src_x_float - x0; - - float w00 = (1.0f - dx) * (1.0f - dy); - float w01 = dx * (1.0f - dy); - float w10 = (1.0f - dx) * dy; - float w11 = dx * dy; - - size_t idx00 = (y0 * static_cast(src_width) + x0) * static_cast(embed_dim); - size_t idx01 = (y0 * static_cast(src_width) + x1) * static_cast(embed_dim); - size_t idx10 = (y1 * static_cast(src_width) + x0) * static_cast(embed_dim); - size_t idx11 = (y1 * static_cast(src_width) + x1) * static_cast(embed_dim); - - size_t out_idx = (dst_y * dst_width + dst_x) * embed_dim; - - size_t d = 0; - for(; d + 8 <= embed_dim; d += 8){ - float16x8_t v00 = vld1q_f16(input + idx00 + d); - float16x8_t v01 = vld1q_f16(input + idx01 + d); - float16x8_t v10 = vld1q_f16(input + idx10 + d); - float16x8_t v11 = vld1q_f16(input + idx11 + d); - - float16x8_t result = vfmaq_f16( - vfmaq_f16( - vfmaq_f16(vdupq_n_f16(0.0f), v00, vdupq_n_f16(w00)), - v01, vdupq_n_f16(w01)), - v10, vdupq_n_f16(w10)); - result = vfmaq_f16(result, v11, vdupq_n_f16(w11)); - - vst1q_f16(output + out_idx + d, result); - } - - for (; d < embed_dim; ++d) { - output[out_idx + d] = - input[idx00 + d] * w00 + - input[idx01 + d] * w01 + - input[idx10 + d] * w10 + - input[idx11 + d] * w11; - } - } - } -} - -void cactus_stft_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t /*C_out*/, - size_t K, size_t stride, - size_t num_fft_bins -) { - const size_t out_len = ((L - K) / stride) + 1; - const size_t in_bs = C_in * L; - const size_t out_bs = 2 * num_fft_bins * out_len; - - for (size_t n = 0; n < N; ++n) { - const __fp16* Xb = input + n * in_bs; - - for (size_t bin = 0; bin < num_fft_bins; ++bin) { - const __fp16* Wr = weight + bin * (C_in * K); - const __fp16* Wi = weight + (bin + num_fft_bins) * (C_in * K); - - for (size_t out_t = 0; out_t < out_len; ++out_t) { - const size_t t = out_t * stride; - float sum_real = 0.0f; - float sum_imag = 0.0f; - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xb + ic * L + t; - const __fp16* Wrc = Wr + ic * K; - const __fp16* Wic = Wi + ic * K; - - float32x4_t acc_r0 = vdupq_n_f32(0.f); - float32x4_t acc_r1 = vdupq_n_f32(0.f); - float32x4_t acc_i0 = vdupq_n_f32(0.f); - float32x4_t acc_i1 = vdupq_n_f32(0.f); - - size_t k = 0; - for (; k + 8 <= K; k += 8) { - const float16x8_t xv = vld1q_f16(Xc + k); - const float16x8_t wr = vld1q_f16(Wrc + k); - const float16x8_t wi = vld1q_f16(Wic + k); - - acc_r0 = vfmaq_f32(acc_r0, vcvt_f32_f16(vget_low_f16(xv)), vcvt_f32_f16(vget_low_f16(wr))); - acc_r1 = vfmaq_f32(acc_r1, vcvt_f32_f16(vget_high_f16(xv)), vcvt_f32_f16(vget_high_f16(wr))); - acc_i0 = vfmaq_f32(acc_i0, vcvt_f32_f16(vget_low_f16(xv)), vcvt_f32_f16(vget_low_f16(wi))); - acc_i1 = vfmaq_f32(acc_i1, vcvt_f32_f16(vget_high_f16(xv)), vcvt_f32_f16(vget_high_f16(wi))); - } - - sum_real += vaddvq_f32(acc_r0) + vaddvq_f32(acc_r1); - sum_imag += vaddvq_f32(acc_i0) + vaddvq_f32(acc_i1); - - for (; k < K; ++k) { - float x = (float)Xc[k]; - sum_real += x * (float)Wrc[k]; - sum_imag += x * (float)Wic[k]; - } - } - - output[n * out_bs + bin * out_len + out_t] = (__fp16)sum_real; - output[n * out_bs + (bin + num_fft_bins) * out_len + out_t] = (__fp16)sum_imag; - } - } - } -} - -void cactus_conv1d_same_depthwise_f16_k9( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, size_t C) -{ - constexpr int K = 9; - - const size_t bs = L * C; - - CactusThreading::parallel_for_2d( - N, C, CactusThreading::Thresholds::ATTENTION, - [&](size_t n, size_t c) { - - const __fp16* Xb = input + n * bs; - __fp16* Yb = output + n * bs; - - const __fp16* Wc = weight + c * K; - const float b = bias ? (float)bias[c] : 0.0f; - - const float w0 = (float)Wc[0]; - const float w1 = (float)Wc[1]; - const float w2 = (float)Wc[2]; - const float w3 = (float)Wc[3]; - const float w4 = (float)Wc[4]; - const float w5 = (float)Wc[5]; - const float w6 = (float)Wc[6]; - const float w7 = (float)Wc[7]; - const float w8 = (float)Wc[8]; - - const float32x4_t wv0 = {w0, w1, w2, w3}; - const float32x4_t wv1 = {w4, w5, w6, w7}; - - for (size_t t0 = 0; t0 < L; t0 += T_TILE_F16) { - const bool have_t1 = (t0 + 1 < L); - const size_t t1 = have_t1 ? (t0 + 1) : t0; - - float x0_0=0, x1_0=0, x2_0=0, x3_0=0; - float x4_0=0, x5_0=0, x6_0=0, x7_0=0; - float x8_0=0; - - { - const ptrdiff_t i0 = (ptrdiff_t)t0 - 4; - const ptrdiff_t i1 = (ptrdiff_t)t0 - 3; - const ptrdiff_t i2 = (ptrdiff_t)t0 - 2; - const ptrdiff_t i3 = (ptrdiff_t)t0 - 1; - if (i0 >= 0 && i0 < (ptrdiff_t)L) x0_0 = (float)Xb[(size_t)i0 * C + c]; - if (i1 >= 0 && i1 < (ptrdiff_t)L) x1_0 = (float)Xb[(size_t)i1 * C + c]; - if (i2 >= 0 && i2 < (ptrdiff_t)L) x2_0 = (float)Xb[(size_t)i2 * C + c]; - if (i3 >= 0 && i3 < (ptrdiff_t)L) x3_0 = (float)Xb[(size_t)i3 * C + c]; - - const ptrdiff_t j4 = (ptrdiff_t)t0 + 0; - const ptrdiff_t j5 = (ptrdiff_t)t0 + 1; - const ptrdiff_t j6 = (ptrdiff_t)t0 + 2; - const ptrdiff_t j7 = (ptrdiff_t)t0 + 3; - const ptrdiff_t j8 = (ptrdiff_t)t0 + 4; - if (j4 >= 0 && j4 < (ptrdiff_t)L) x4_0 = (float)Xb[(size_t)j4 * C + c]; - if (j5 >= 0 && j5 < (ptrdiff_t)L) x5_0 = (float)Xb[(size_t)j5 * C + c]; - if (j6 >= 0 && j6 < (ptrdiff_t)L) x6_0 = (float)Xb[(size_t)j6 * C + c]; - if (j7 >= 0 && j7 < (ptrdiff_t)L) x7_0 = (float)Xb[(size_t)j7 * C + c]; - if (j8 >= 0 && j8 < (ptrdiff_t)L) x8_0 = (float)Xb[(size_t)j8 * C + c]; - } - - float32x4_t acc0v = vdupq_n_f32(0.f); - const float32x4_t xv0_0 = {x0_0, x1_0, x2_0, x3_0}; - const float32x4_t xv1_0 = {x4_0, x5_0, x6_0, x7_0}; - acc0v = vfmaq_f32(acc0v, xv0_0, wv0); - acc0v = vfmaq_f32(acc0v, xv1_0, wv1); - float acc0 = vaddvq_f32(acc0v) + (w8 * x8_0) + b; - - float acc1 = 0.f; - if (have_t1) { - float x0_1=0, x1_1=0, x2_1=0, x3_1=0; - float x4_1=0, x5_1=0, x6_1=0, x7_1=0; - float x8_1=0; - - { - const ptrdiff_t i0 = (ptrdiff_t)t1 - 4; - const ptrdiff_t i1 = (ptrdiff_t)t1 - 3; - const ptrdiff_t i2 = (ptrdiff_t)t1 - 2; - const ptrdiff_t i3 = (ptrdiff_t)t1 - 1; - if (i0 >= 0 && i0 < (ptrdiff_t)L) x0_1 = (float)Xb[(size_t)i0 * C + c]; - if (i1 >= 0 && i1 < (ptrdiff_t)L) x1_1 = (float)Xb[(size_t)i1 * C + c]; - if (i2 >= 0 && i2 < (ptrdiff_t)L) x2_1 = (float)Xb[(size_t)i2 * C + c]; - if (i3 >= 0 && i3 < (ptrdiff_t)L) x3_1 = (float)Xb[(size_t)i3 * C + c]; - - const ptrdiff_t j4 = (ptrdiff_t)t1 + 0; - const ptrdiff_t j5 = (ptrdiff_t)t1 + 1; - const ptrdiff_t j6 = (ptrdiff_t)t1 + 2; - const ptrdiff_t j7 = (ptrdiff_t)t1 + 3; - const ptrdiff_t j8 = (ptrdiff_t)t1 + 4; - if (j4 >= 0 && j4 < (ptrdiff_t)L) x4_1 = (float)Xb[(size_t)j4 * C + c]; - if (j5 >= 0 && j5 < (ptrdiff_t)L) x5_1 = (float)Xb[(size_t)j5 * C + c]; - if (j6 >= 0 && j6 < (ptrdiff_t)L) x6_1 = (float)Xb[(size_t)j6 * C + c]; - if (j7 >= 0 && j7 < (ptrdiff_t)L) x7_1 = (float)Xb[(size_t)j7 * C + c]; - if (j8 >= 0 && j8 < (ptrdiff_t)L) x8_1 = (float)Xb[(size_t)j8 * C + c]; - } - - float32x4_t acc1v = vdupq_n_f32(0.f); - const float32x4_t xv0_1 = {x0_1, x1_1, x2_1, x3_1}; - const float32x4_t xv1_1 = {x4_1, x5_1, x6_1, x7_1}; - acc1v = vfmaq_f32(acc1v, xv0_1, wv0); - acc1v = vfmaq_f32(acc1v, xv1_1, wv1); - acc1 = vaddvq_f32(acc1v) + (w8 * x8_1) + b; - } - - Yb[t0 * C + c] = (__fp16)acc0; - if (have_t1) Yb[t1 * C + c] = (__fp16)acc1; - } - }); -} - - -void cactus_conv2d_f16_k3s2p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, size_t H, size_t W, - size_t C_out -){ - if (H + 2 < 3 || W + 2 < 3) return; - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - -#ifdef __APPLE__ - { - const size_t spatial = H_out * W_out; - const size_t col_K = C_in * 9; - std::vector W_f32(C_out * col_K); - for (size_t oc = 0; oc < C_out; ++oc) - for (size_t ic = 0; ic < C_in; ++ic) - for (size_t kh = 0; kh < 3; ++kh) - for (size_t kw = 0; kw < 3; ++kw) - W_f32[oc * col_K + ic * 9 + kh * 3 + kw] = - static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); - - std::vector bias_f32(C_out, 0.0f); - if (bias) - for (size_t i = 0; i < C_out; ++i) - bias_f32[i] = static_cast(bias[i]); - - std::vector col(col_K * spatial); - std::vector Y_f32(C_out * spatial); - - for (size_t n = 0; n < N; ++n) { - const __fp16* Xn = input + n * C_in * H * W; - __fp16* Yn = output + n * C_out * spatial; - - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - for (size_t kw = 0; kw < 3; ++kw) { - float* dst = col.data() + (ic * 9 + kh * 3 + kw) * spatial; - for (size_t oh = 0; oh < H_out; ++oh) { - ptrdiff_t ih = static_cast(oh) * 2 + static_cast(kh) - 1; - float* dst_row = dst + oh * W_out; - if (ih < 0 || ih >= static_cast(H)) { - memset(dst_row, 0, W_out * sizeof(float)); - continue; - } - const __fp16* src_row = Xn + ic * H * W + static_cast(ih) * W; - const ptrdiff_t iw_offset = static_cast(kw) - 1; - for (size_t ow = 0; ow < W_out; ++ow) { - ptrdiff_t iw = static_cast(ow) * 2 + iw_offset; - dst_row[ow] = (iw < 0 || iw >= static_cast(W)) - ? 0.0f : static_cast(src_row[iw]); - } - } - } - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - static_cast(C_out), static_cast(spatial), static_cast(col_K), - 1.0f, W_f32.data(), static_cast(col_K), - col.data(), static_cast(spatial), - 0.0f, Y_f32.data(), static_cast(spatial)); - - for (size_t oc = 0; oc < C_out; ++oc) { - const float b = bias_f32[oc]; - const float* src = Y_f32.data() + oc * spatial; - __fp16* dst = Yn + oc * spatial; - size_t i = 0; - for (; i + 8 <= spatial; i += 8) { - float32x4_t v0 = vaddq_f32(vld1q_f32(src + i), vdupq_n_f32(b)); - float32x4_t v1 = vaddq_f32(vld1q_f32(src + i + 4), vdupq_n_f32(b)); - vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - } - for (; i < spatial; ++i) - dst[i] = static_cast<__fp16>(src[i] + b); - } - } - return; - } -#endif - - auto in_idx = [&](size_t n, size_t ic, size_t ih, size_t iw) -> size_t { - return ((n * C_in + ic) * H + ih) * W + iw; - }; - auto w_idx = [&](size_t oc, size_t ic, size_t kh, size_t kw) -> size_t { - return (((oc * C_in + ic) * 3 + kh) * 3 + kw); - }; - auto out_idx = [&](size_t n, size_t oc, size_t oh, size_t ow) -> size_t { - return ((n * C_out + oc) * H_out + oh) * W_out + ow; - }; - - constexpr size_t OW_VEC = 4; - - const size_t total_compute = N * C_out * H_out * W_out * C_in * 9; - CactusThreading::ParallelConfig cfg = - (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d(N, C_out, cfg, [&](size_t n, size_t oc) { - const float b0 = bias ? (float)bias[oc] : 0.0f; - - for (size_t oh = 0; oh < H_out; ++oh) { - const ptrdiff_t ih0 = (ptrdiff_t)oh * 2 - 1; - - const bool h_interior = (ih0 >= 0) && ((size_t)(ih0 + 2) < H); - - for (size_t ow = 0; ow < W_out; ) { - const size_t rem = W_out - ow; - - const bool do_vec = (rem >= OW_VEC); - const bool w_interior_vec = - do_vec && - (ow >= 1) && - (((ow + (OW_VEC - 1)) * 2 + 1) < W); - - float32x4_t vacc = vdupq_n_f32(b0); - - if (h_interior && w_interior_vec) { - for (size_t ic = 0; ic < C_in; ++ic) { - - const __fp16* row0 = input + in_idx(n, ic, (size_t)(ih0 + 0), 0); - const __fp16* row1 = input + in_idx(n, ic, (size_t)(ih0 + 1), 0); - const __fp16* row2 = input + in_idx(n, ic, (size_t)(ih0 + 2), 0); - const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; - { - const float w00 = (float)weight[w_idx(oc, ic, 0, 0)]; - const float w01 = (float)weight[w_idx(oc, ic, 0, 1)]; - const float w02 = (float)weight[w_idx(oc, ic, 0, 2)]; - - float x0 = (float)row0[(size_t)(iw_base0 + 0)]; - float x1 = (float)row0[(size_t)(iw_base0 + 2)]; - float x2 = (float)row0[(size_t)(iw_base0 + 4)]; - float x3 = (float)row0[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w00)); - - x0 = (float)row0[(size_t)(iw_base0 + 1)]; - x1 = (float)row0[(size_t)(iw_base0 + 3)]; - x2 = (float)row0[(size_t)(iw_base0 + 5)]; - x3 = (float)row0[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w01)); - - x0 = (float)row0[(size_t)(iw_base0 + 2)]; - x1 = (float)row0[(size_t)(iw_base0 + 4)]; - x2 = (float)row0[(size_t)(iw_base0 + 6)]; - x3 = (float)row0[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w02)); - } - - { - const float w10 = (float)weight[w_idx(oc, ic, 1, 0)]; - const float w11 = (float)weight[w_idx(oc, ic, 1, 1)]; - const float w12 = (float)weight[w_idx(oc, ic, 1, 2)]; - const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; - - float x0 = (float)row1[(size_t)(iw_base0 + 0)]; - float x1 = (float)row1[(size_t)(iw_base0 + 2)]; - float x2 = (float)row1[(size_t)(iw_base0 + 4)]; - float x3 = (float)row1[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w10)); - - x0 = (float)row1[(size_t)(iw_base0 + 1)]; - x1 = (float)row1[(size_t)(iw_base0 + 3)]; - x2 = (float)row1[(size_t)(iw_base0 + 5)]; - x3 = (float)row1[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w11)); - - x0 = (float)row1[(size_t)(iw_base0 + 2)]; - x1 = (float)row1[(size_t)(iw_base0 + 4)]; - x2 = (float)row1[(size_t)(iw_base0 + 6)]; - x3 = (float)row1[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w12)); - } - - { - const float w20 = (float)weight[w_idx(oc, ic, 2, 0)]; - const float w21 = (float)weight[w_idx(oc, ic, 2, 1)]; - const float w22 = (float)weight[w_idx(oc, ic, 2, 2)]; - const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; - - float x0 = (float)row2[(size_t)(iw_base0 + 0)]; - float x1 = (float)row2[(size_t)(iw_base0 + 2)]; - float x2 = (float)row2[(size_t)(iw_base0 + 4)]; - float x3 = (float)row2[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w20)); - - x0 = (float)row2[(size_t)(iw_base0 + 1)]; - x1 = (float)row2[(size_t)(iw_base0 + 3)]; - x2 = (float)row2[(size_t)(iw_base0 + 5)]; - x3 = (float)row2[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w21)); - - x0 = (float)row2[(size_t)(iw_base0 + 2)]; - x1 = (float)row2[(size_t)(iw_base0 + 4)]; - x2 = (float)row2[(size_t)(iw_base0 + 6)]; - x3 = (float)row2[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w22)); - } - } - - __fp16* Y = output + out_idx(n, oc, oh, ow); - const float16x4_t yv = vcvt_f16_f32(vacc); - vst1_f16(Y, yv); - - ow += OW_VEC; - continue; - } - - const size_t tile = do_vec ? OW_VEC : 1; - float acc_s[OW_VEC] = {b0, b0, b0, b0}; - - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - const ptrdiff_t ih = ih0 + (ptrdiff_t)kh; - if (ih < 0 || ih >= (ptrdiff_t)H) continue; - - const __fp16* row = input + in_idx(n, ic, (size_t)ih, 0); - - for (size_t kw = 0; kw < 3; ++kw) { - const float wv = (float)weight[w_idx(oc, ic, kh, kw)]; - for (size_t t = 0; t < tile; ++t) { - const size_t ow_t = ow + t; - const ptrdiff_t iw = (ptrdiff_t)ow_t * 2 - 1 + (ptrdiff_t)kw; - if (iw < 0 || iw >= (ptrdiff_t)W) continue; - acc_s[t] += (float)row[(size_t)iw] * wv; - } - } - } - } - - __fp16* Y = output + out_idx(n, oc, oh, ow); - for (size_t t = 0; t < tile; ++t) { - Y[t] = (__fp16)acc_s[t]; - } - ow += tile; - } - } - }); -} - -void cactus_conv2d_depthwise_f16_k3s2p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C, - size_t H, - size_t W -){ - if (H + 2 < 3 || W + 2 < 3) return; - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - - auto in_idx = [&](size_t n, size_t c, size_t ih, size_t iw) -> size_t { - return ((n * C + c) * H + ih) * W + iw; - }; - auto out_idx = [&](size_t n, size_t c, size_t oh, size_t ow) -> size_t { - return ((n * C + c) * H_out + oh) * W_out + ow; - }; - - constexpr size_t OW_VEC = 4; - const size_t total_compute = N * C * H_out * W_out * 9; - CactusThreading::ParallelConfig cfg = - (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d(N, C, cfg, [&](size_t n, size_t c) { - const __fp16* Wc = weight + c * 9; - const float w00 = (float)Wc[0], w01 = (float)Wc[1], w02 = (float)Wc[2]; - const float w10 = (float)Wc[3], w11 = (float)Wc[4], w12 = (float)Wc[5]; - const float w20 = (float)Wc[6], w21 = (float)Wc[7], w22 = (float)Wc[8]; - const float b0 = bias ? (float)bias[c] : 0.0f; - - for (size_t oh = 0; oh < H_out; ++oh) { - const ptrdiff_t ih0 = (ptrdiff_t)oh * 2 - 1; - const bool h_interior = (ih0 >= 0) && ((size_t)(ih0 + 2) < H); - - for (size_t ow = 0; ow < W_out; ) { - const size_t rem = W_out - ow; - const bool do_vec = (rem >= OW_VEC); - const bool w_interior_vec = - do_vec && - (ow >= 1) && - (((ow + (OW_VEC - 1)) * 2 + 1) < W); - - if (h_interior && w_interior_vec) { - const __fp16* row0 = input + in_idx(n, c, (size_t)(ih0 + 0), 0); - const __fp16* row1 = input + in_idx(n, c, (size_t)(ih0 + 1), 0); - const __fp16* row2 = input + in_idx(n, c, (size_t)(ih0 + 2), 0); - const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; - - float32x4_t vacc = vdupq_n_f32(b0); - float x0, x1, x2, x3; - - x0 = (float)row0[(size_t)(iw_base0 + 0)]; - x1 = (float)row0[(size_t)(iw_base0 + 2)]; - x2 = (float)row0[(size_t)(iw_base0 + 4)]; - x3 = (float)row0[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w00)); - - x0 = (float)row0[(size_t)(iw_base0 + 1)]; - x1 = (float)row0[(size_t)(iw_base0 + 3)]; - x2 = (float)row0[(size_t)(iw_base0 + 5)]; - x3 = (float)row0[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w01)); - - x0 = (float)row0[(size_t)(iw_base0 + 2)]; - x1 = (float)row0[(size_t)(iw_base0 + 4)]; - x2 = (float)row0[(size_t)(iw_base0 + 6)]; - x3 = (float)row0[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w02)); - - x0 = (float)row1[(size_t)(iw_base0 + 0)]; - x1 = (float)row1[(size_t)(iw_base0 + 2)]; - x2 = (float)row1[(size_t)(iw_base0 + 4)]; - x3 = (float)row1[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w10)); - - x0 = (float)row1[(size_t)(iw_base0 + 1)]; - x1 = (float)row1[(size_t)(iw_base0 + 3)]; - x2 = (float)row1[(size_t)(iw_base0 + 5)]; - x3 = (float)row1[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w11)); - - x0 = (float)row1[(size_t)(iw_base0 + 2)]; - x1 = (float)row1[(size_t)(iw_base0 + 4)]; - x2 = (float)row1[(size_t)(iw_base0 + 6)]; - x3 = (float)row1[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w12)); - - x0 = (float)row2[(size_t)(iw_base0 + 0)]; - x1 = (float)row2[(size_t)(iw_base0 + 2)]; - x2 = (float)row2[(size_t)(iw_base0 + 4)]; - x3 = (float)row2[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w20)); - - x0 = (float)row2[(size_t)(iw_base0 + 1)]; - x1 = (float)row2[(size_t)(iw_base0 + 3)]; - x2 = (float)row2[(size_t)(iw_base0 + 5)]; - x3 = (float)row2[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w21)); - - x0 = (float)row2[(size_t)(iw_base0 + 2)]; - x1 = (float)row2[(size_t)(iw_base0 + 4)]; - x2 = (float)row2[(size_t)(iw_base0 + 6)]; - x3 = (float)row2[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w22)); - - __fp16* Y = output + out_idx(n, c, oh, ow); - vst1_f16(Y, vcvt_f16_f32(vacc)); - ow += OW_VEC; - continue; - } - - const size_t tile = do_vec ? OW_VEC : 1; - float acc_s[OW_VEC] = {b0, b0, b0, b0}; - for (size_t t = 0; t < tile; ++t) { - const size_t ow_t = ow + t; - for (size_t kh = 0; kh < 3; ++kh) { - const ptrdiff_t ih = ih0 + (ptrdiff_t)kh; - if (ih < 0 || ih >= (ptrdiff_t)H) continue; - - const __fp16* row = input + in_idx(n, c, (size_t)ih, 0); - for (size_t kw = 0; kw < 3; ++kw) { - const ptrdiff_t iw = (ptrdiff_t)ow_t * 2 - 1 + (ptrdiff_t)kw; - if (iw < 0 || iw >= (ptrdiff_t)W) continue; - const float wv = (float)Wc[kh * 3 + kw]; - acc_s[t] += (float)row[(size_t)iw] * wv; - } - } - } - - __fp16* Y = output + out_idx(n, c, oh, ow); - for (size_t t = 0; t < tile; ++t) { - Y[t] = (__fp16)acc_s[t]; - } - ow += tile; - } - } - }); -} - -void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, - size_t H, - size_t W, - size_t C_out -){ - const size_t HW = H * W; - if (HW == 0 || N == 0) return; - - CactusThreading::parallel_for( - N, - CactusThreading::Thresholds::ATTENTION, - [&](size_t n_start, size_t n_end) { - std::vector<__fp16> packed_in(HW * C_in); - std::vector packed_out(HW * C_out); - - for (size_t n = n_start; n < n_end; ++n) { - const __fp16* Xn = input + n * C_in * HW; - __fp16* Yn = output + n * C_out * HW; - - for (size_t hw = 0; hw < HW; ++hw) { - const size_t h = hw / W; - const size_t w = hw - h * W; - __fp16* row = packed_in.data() + hw * C_in; - for (size_t ic = 0; ic < C_in; ++ic) { - row[ic] = Xn[(ic * H + h) * W + w]; - } - } - - fp16_rowmajor_gemm_rhs_transposed_to_fp32( - packed_in.data(), - weight, - packed_out.data(), - HW, - C_in, - C_out - ); - - for (size_t hw = 0; hw < HW; ++hw) { - const size_t h = hw / W; - const size_t w = hw - h * W; - const float* row = packed_out.data() + hw * C_out; - for (size_t oc = 0; oc < C_out; ++oc) { - float outv = row[oc]; - if (bias) outv += (float)bias[oc]; - Yn[(oc * H + h) * W + w] = (__fp16)outv; - } - } - } - }); -} - -void cactus_conv1d_pointwise_f16_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out -){ - const size_t M = N * L; - if (M == 0) return; - - std::vector output_f32(M * C_out); - fp16_rowmajor_gemm_rhs_transposed_to_fp32( - input, - weight, - output_f32.data(), - M, - C_in, - C_out - ); - - CactusThreading::parallel_for( - M, - CactusThreading::Thresholds::ATTENTION, - [&](size_t row_start, size_t row_end) { - for (size_t m = row_start; m < row_end; ++m) { - const float* src_row = output_f32.data() + m * C_out; - __fp16* dst_row = output + m * C_out; - for (size_t oc = 0; oc < C_out; ++oc) { - float value = src_row[oc]; - if (bias) { - value += static_cast(bias[oc]); - } - dst_row[oc] = static_cast<__fp16>(value); - } - } - } - ); -} - -void cactus_conv2d_f16_k3s1p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, size_t H, size_t W, - size_t C_out -) { - const size_t H_out = H; - const size_t W_out = W; - -#ifdef __APPLE__ - const size_t col_K = C_in * 9; - std::vector W_f32(C_out * col_K); - for (size_t oc = 0; oc < C_out; ++oc) { - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - for (size_t kw = 0; kw < 3; ++kw) { - W_f32[oc * col_K + ic * 9 + kh * 3 + kw] = - static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); - } - } - } - } - - std::vector bias_f32(C_out, 0.0f); - if (bias) { - for (size_t i = 0; i < C_out; ++i) - bias_f32[i] = static_cast(bias[i]); - } - - std::vector col(col_K * H_out * W_out); - std::vector Y_f32(C_out * H_out * W_out); - - for (size_t n = 0; n < N; ++n) { - const __fp16* Xn = input + n * C_in * H * W; - __fp16* Yn = output + n * C_out * H_out * W_out; - - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - for (size_t kw = 0; kw < 3; ++kw) { - float* dst = col.data() + (ic * 9 + kh * 3 + kw) * H_out * W_out; - for (size_t oh = 0; oh < H_out; ++oh) { - ptrdiff_t ih = static_cast(oh) + static_cast(kh) - 1; - float* dst_row = dst + oh * W_out; - if (ih < 0 || ih >= static_cast(H)) { - memset(dst_row, 0, W_out * sizeof(float)); - continue; - } - const __fp16* src_row = Xn + ic * H * W + static_cast(ih) * W; - const ptrdiff_t iw_offset = static_cast(kw) - 1; - size_t ow_start = 0, ow_end = W_out; - if (iw_offset < 0) { dst_row[0] = 0.0f; ow_start = 1; } - if (iw_offset > 0) { dst_row[W_out - 1] = 0.0f; ow_end = W_out - 1; } - size_t ow = ow_start; - for (; ow + 8 <= ow_end; ow += 8) { - float16x8_t v = vld1q_f16(src_row + ow + iw_offset); - vst1q_f32(dst_row + ow, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(dst_row + ow + 4, vcvt_f32_f16(vget_high_f16(v))); - } - for (; ow < ow_end; ++ow) - dst_row[ow] = static_cast(src_row[ow + iw_offset]); - } - } - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - static_cast(C_out), static_cast(H_out * W_out), static_cast(col_K), - 1.0f, W_f32.data(), static_cast(col_K), - col.data(), static_cast(H_out * W_out), - 0.0f, Y_f32.data(), static_cast(H_out * W_out)); - - for (size_t oc = 0; oc < C_out; ++oc) { - float b = bias_f32[oc]; - const float* src = Y_f32.data() + oc * H_out * W_out; - __fp16* dst = Yn + oc * H_out * W_out; - size_t i = 0; - for (; i + 8 <= H_out * W_out; i += 8) { - float32x4_t v0 = vaddq_f32(vld1q_f32(src + i), vdupq_n_f32(b)); - float32x4_t v1 = vaddq_f32(vld1q_f32(src + i + 4), vdupq_n_f32(b)); - vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - } - for (; i < H_out * W_out; ++i) - dst[i] = static_cast<__fp16>(src[i] + b); - } - } - -#else - const size_t total_compute = N * C_out * H_out * W_out * C_in * 9; - CactusThreading::ParallelConfig cfg = - (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d(N, C_out, cfg, [&](size_t n, size_t oc) { - const float b0 = bias ? static_cast(bias[oc]) : 0.0f; - for (size_t oh = 0; oh < H_out; ++oh) { - for (size_t ow = 0; ow < W_out; ++ow) { - float acc = b0; - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - for (size_t kw = 0; kw < 3; ++kw) { - const ptrdiff_t ih = static_cast(oh) + kh - 1; - const ptrdiff_t iw = static_cast(ow) + kw - 1; - if (ih >= 0 && ih < static_cast(H) && - iw >= 0 && iw < static_cast(W)) { - acc += static_cast(input[((n * C_in + ic) * H + ih) * W + iw]) * - static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); - } - } - } - } - output[((n * C_out + oc) * H_out + oh) * W_out + ow] = static_cast<__fp16>(acc); - } - } - }); -#endif -} - -void cactus_maxpool1d_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t channels, - size_t input_length, - size_t kernel_size, - size_t stride -) { - const size_t output_length = (input_length - kernel_size) / stride + 1; - - CactusThreading::parallel_for(batch_size * channels, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start, size_t end) { - for (size_t bc = start; bc < end; ++bc) { - const size_t b = bc / channels; - const size_t c = bc % channels; - - const __fp16* src = input + b * channels * input_length + c * input_length; - __fp16* dst = output + b * channels * output_length + c * output_length; - - for (size_t i = 0; i < output_length; ++i) { - const size_t in_start = i * stride; - float max_val = -std::numeric_limits::infinity(); - for (size_t k = 0; k < kernel_size; ++k) { - float val = static_cast(src[in_start + k]); - if (val > max_val) max_val = val; - } - dst[i] = static_cast<__fp16>(max_val); - } - } - }); -} diff --git a/cactus-transpile/kernel/kernel_deltanet.cpp b/cactus-transpile/kernel/kernel_deltanet.cpp deleted file mode 100644 index 4972bc49b..000000000 --- a/cactus-transpile/kernel/kernel_deltanet.cpp +++ /dev/null @@ -1,677 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" - -#include -#include -#include -#include -#include - -namespace { - -inline float safe_exp_gate(float gate_log) { - const float clamped = std::max(-20.0f, std::min(6.0f, gate_log)); - return std::exp(clamped); -} - -inline void fold_state_scale(std::vector& state, float factor) { - const float32x4_t f4 = vdupq_n_f32(factor); - size_t i = 0; - for (; i + 8 <= state.size(); i += 8) { - vst1q_f32(&state[i], vmulq_f32(vld1q_f32(&state[i]), f4)); - vst1q_f32(&state[i + 4], vmulq_f32(vld1q_f32(&state[i + 4]), f4)); - } - for (; i < state.size(); ++i) state[i] *= factor; -} - -struct GatedDeltaChunkScratch { - std::vector state; - std::vector m_chunk; - std::vector q_chunk; - std::vector k_chunk; - std::vector v_chunk; - std::vector g_chunk; - std::vector beta_chunk; - std::vector n0_proj; - std::vector delta_chunk; - std::vector gram; - std::vector p_prev; - std::vector p_curr; - std::vector inv_p_curr; - std::vector coeff; - std::vector out_acc; - - void ensure(size_t k_dim, size_t v_dim, size_t c_max) { - const size_t kv = k_dim * v_dim; - if (state.size() < kv) state.resize(kv); - if (m_chunk.size() < kv) m_chunk.resize(kv); - if (q_chunk.size() < c_max * k_dim) q_chunk.resize(c_max * k_dim); - if (k_chunk.size() < c_max * k_dim) k_chunk.resize(c_max * k_dim); - if (v_chunk.size() < c_max * v_dim) v_chunk.resize(c_max * v_dim); - if (g_chunk.size() < c_max) g_chunk.resize(c_max); - if (beta_chunk.size() < c_max) beta_chunk.resize(c_max); - if (n0_proj.size() < c_max * v_dim) n0_proj.resize(c_max * v_dim); - if (delta_chunk.size() < c_max * v_dim) delta_chunk.resize(c_max * v_dim); - if (gram.size() < c_max * c_max) gram.resize(c_max * c_max); - if (p_prev.size() < c_max) p_prev.resize(c_max); - if (p_curr.size() < c_max) p_curr.resize(c_max); - if (inv_p_curr.size() < c_max) inv_p_curr.resize(c_max); - if (coeff.size() < c_max * c_max) coeff.resize(c_max * c_max); - if (out_acc.size() < v_dim) out_acc.resize(v_dim); - } -}; - -thread_local GatedDeltaChunkScratch g_gated_deltanet_chunk_scratch; - -inline size_t tuned_gated_deltanet_chunk_size(size_t requested_chunk, size_t k_dim, size_t v_dim) { - size_t chunk = requested_chunk == 0 ? 64 : requested_chunk; - - const char* env_chunk = std::getenv("CACTUS_GATED_DELTANET_CHUNK_SIZE"); - if (env_chunk != nullptr) { - long parsed = std::strtol(env_chunk, nullptr, 10); - if (parsed > 1) { - chunk = static_cast(parsed); - } - } else { - (void)k_dim; - (void)v_dim; - chunk = std::min(chunk, 16); - } - - return std::max(2, chunk); -} - -void gated_deltanet_step( - const __fp16* q_ptr, - const __fp16* k_ptr, - const __fp16* v_ptr, - float gate_log, - float beta, - float scale, - size_t k_dim, - size_t v_dim, - std::vector& state, - float& state_scale, - std::vector& proj, - std::vector& delta, - __fp16* out_ptr) { - const float gate_log_safe = std::isfinite(gate_log) ? gate_log : -20.0f; - const float beta_safe = std::isfinite(beta) ? std::min(1.0f, std::max(0.0f, beta)) : 0.0f; - - if (proj.size() != v_dim) { - proj.assign(v_dim, 0.0f); - } - if (delta.size() != v_dim) { - delta.assign(v_dim, 0.0f); - } - - const float prev_scale = state_scale; - std::fill(proj.begin(), proj.end(), 0.0f); - - for (size_t kd = 0; kd < k_dim; ++kd) { - if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 0, 3); - const float k_val = static_cast(k_ptr[kd]); - const float* state_row = state.data() + kd * v_dim; - size_t vd = 0; - const float32x4_t k4 = vdupq_n_f32(k_val); - for (; vd + 4 <= v_dim; vd += 4) { - float32x4_t p = vld1q_f32(proj.data() + vd); - float32x4_t s = vld1q_f32(state_row + vd); - p = vfmaq_f32(p, s, k4); - vst1q_f32(proj.data() + vd, p); - } - for (; vd < v_dim; ++vd) { - proj[vd] += state_row[vd] * k_val; - } - } - { - size_t vd = 0; - const float32x4_t scale4 = vdupq_n_f32(prev_scale); - for (; vd + 4 <= v_dim; vd += 4) { - float32x4_t p = vld1q_f32(proj.data() + vd); - p = vmulq_f32(p, scale4); - vst1q_f32(proj.data() + vd, p); - } - for (; vd < v_dim; ++vd) { - proj[vd] *= prev_scale; - } - } - - for (size_t vd = 0; vd < v_dim; ++vd) { - const float v_val = static_cast(v_ptr[vd]); - delta[vd] = (v_val - proj[vd]) * beta_safe; - } - - state_scale = prev_scale * safe_exp_gate(gate_log_safe); - if (!std::isfinite(state_scale) || - std::fabs(state_scale) < 1e-8f || - std::fabs(state_scale) > 1e8f) { - fold_state_scale(state, state_scale); - state_scale = 1.0f; - } - const float inv_scale = 1.0f / state_scale; - - for (size_t kd = 0; kd < k_dim; ++kd) { - if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 1, 3); - const float k_scaled = static_cast(k_ptr[kd]) * inv_scale; - float* state_row = state.data() + kd * v_dim; - size_t vd = 0; - const float32x4_t k4 = vdupq_n_f32(k_scaled); - for (; vd + 4 <= v_dim; vd += 4) { - float32x4_t s = vld1q_f32(state_row + vd); - float32x4_t d = vld1q_f32(delta.data() + vd); - s = vfmaq_f32(s, d, k4); - vst1q_f32(state_row + vd, s); - } - for (; vd < v_dim; ++vd) { - state_row[vd] += k_scaled * delta[vd]; - } - for (size_t vd = 0; vd < v_dim; ++vd) { - if (!std::isfinite(state_row[vd])) { - state_row[vd] = 0.0f; - } - } - } - - std::fill(proj.begin(), proj.end(), 0.0f); - for (size_t kd = 0; kd < k_dim; ++kd) { - if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 0, 3); - const float q_val = static_cast(q_ptr[kd]); - const float* state_row = state.data() + kd * v_dim; - size_t vd = 0; - const float32x4_t q4 = vdupq_n_f32(q_val); - for (; vd + 4 <= v_dim; vd += 4) { - float32x4_t o = vld1q_f32(proj.data() + vd); - float32x4_t s = vld1q_f32(state_row + vd); - o = vfmaq_f32(o, s, q4); - vst1q_f32(proj.data() + vd, o); - } - for (; vd < v_dim; ++vd) { - proj[vd] += state_row[vd] * q_val; - } - } - - const float out_scale = state_scale * scale; - for (size_t vd = 0; vd < v_dim; ++vd) { - float acc = proj[vd]; - if (!std::isfinite(acc)) { - acc = 0.0f; - } - out_ptr[vd] = static_cast<__fp16>(acc * out_scale); - } -} - -void gated_deltanet_prefill_old_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t T, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - float scale) { - const size_t out_seq = T + K; - const size_t qk_repeat = Hv / Hq; - - CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t bh_start, size_t bh_end) { - std::vector state(K * V); - std::vector proj(V); - std::vector delta(V); - - for (size_t bh = bh_start; bh < bh_end; ++bh) { - const size_t batch = bh / Hv; - const size_t v_head = bh % Hv; - const size_t qk_head = v_head / qk_repeat; - - for (size_t kd = 0; kd < K; ++kd) { - for (size_t vd = 0; vd < V; ++vd) { - const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V + vd); - state[kd * V + vd] = static_cast(s_data[s_idx]); - } - } - - float state_scale = 1.0f; - for (size_t t = 0; t < T; ++t) { - const size_t qkv_k_base = (((batch * T + t) * Hq + qk_head) * K); - const size_t qkv_v_base = (((batch * T + t) * Hv + v_head) * V); - const size_t gb_idx = ((batch * T + t) * Hv + v_head); - - gated_deltanet_step( - q_data + qkv_k_base, - k_data + qkv_k_base, - v_data + qkv_v_base, - static_cast(g_data[gb_idx]), - static_cast(b_data[gb_idx]), - scale, - K, - V, - state, - state_scale, - proj, - delta, - out + (((batch * out_seq + t) * Hv + v_head) * V)); - } - - fold_state_scale(state, state_scale); - for (size_t kd = 0; kd < K; ++kd) { - for (size_t vd = 0; vd < V; ++vd) { - const size_t out_idx = (((batch * out_seq + (T + kd)) * Hv + v_head) * V + vd); - out[out_idx] = static_cast<__fp16>(state[kd * V + vd]); - } - } - } - }); -} - -void gated_deltanet_prefill_chunked_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t T, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - size_t chunk_size, - float scale) { - const size_t out_seq = T + K; - const size_t qk_repeat = Hv / Hq; - constexpr float kMinAbs = 1e-12f; - - CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t bh_start, size_t bh_end) { - auto& ws = g_gated_deltanet_chunk_scratch; - ws.ensure(K, V, chunk_size); - - float* state = ws.state.data(); - float* m_chunk = ws.m_chunk.data(); - float* q_chunk = ws.q_chunk.data(); - float* k_chunk = ws.k_chunk.data(); - float* v_chunk = ws.v_chunk.data(); - float* g_chunk = ws.g_chunk.data(); - float* beta_chunk = ws.beta_chunk.data(); - float* n0_proj = ws.n0_proj.data(); - float* delta_chunk = ws.delta_chunk.data(); - float* gram = ws.gram.data(); - float* p_prev = ws.p_prev.data(); - float* p_curr = ws.p_curr.data(); - float* inv_p_curr = ws.inv_p_curr.data(); - float* coeff = ws.coeff.data(); - float* out_acc = ws.out_acc.data(); - - for (size_t bh = bh_start; bh < bh_end; ++bh) { - const size_t batch = bh / Hv; - const size_t v_head = bh % Hv; - const size_t qk_head = v_head / qk_repeat; - - for (size_t kd = 0; kd < K; ++kd) { - const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V); - cactus_fp16_to_fp32(s_data + s_idx, state + kd * V, V); - } - - for (size_t t0 = 0; t0 < T; t0 += chunk_size) { - const size_t C = std::min(chunk_size, T - t0); - const size_t CV = C * V; - const size_t CC = C * C; - - std::fill(n0_proj, n0_proj + CV, 0.0f); - std::fill(delta_chunk, delta_chunk + CV, 0.0f); - std::fill(gram, gram + CC, 0.0f); - std::fill(coeff, coeff + CC, 0.0f); - std::fill(m_chunk, m_chunk + (K * V), 0.0f); - - // Fused fp16→fp32 conversion + n0_proj: k_chunk stays in L1 - for (size_t ct = 0; ct < C; ++ct) { - const size_t t = t0 + ct; - const size_t qkv_k_base = (((batch * T + t) * Hq + qk_head) * K); - const size_t qkv_v_base = (((batch * T + t) * Hv + v_head) * V); - const size_t gb_idx = ((batch * T + t) * Hv + v_head); - - // Inline fp16→fp32 for q - { - const __fp16* src = q_data + qkv_k_base; - float* dst = q_chunk + ct * K; - size_t i = 0; - for (; i + 8 <= K; i += 8) { - float16x8_t in = vld1q_f16(src + i); - vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); - vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); - } - for (; i < K; ++i) dst[i] = static_cast(src[i]); - } - - // Inline fp16→fp32 for k - { - const __fp16* src = k_data + qkv_k_base; - float* dst = k_chunk + ct * K; - size_t i = 0; - for (; i + 8 <= K; i += 8) { - float16x8_t in = vld1q_f16(src + i); - vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); - vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); - } - for (; i < K; ++i) dst[i] = static_cast(src[i]); - } - - // Inline fp16→fp32 for v - { - const __fp16* src = v_data + qkv_v_base; - float* dst = v_chunk + ct * V; - size_t i = 0; - for (; i + 8 <= V; i += 8) { - float16x8_t in = vld1q_f16(src + i); - vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); - vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); - } - for (; i < V; ++i) dst[i] = static_cast(src[i]); - } - - const float gate_log = static_cast(g_data[gb_idx]); - const float beta = static_cast(b_data[gb_idx]); - g_chunk[ct] = safe_exp_gate(std::isfinite(gate_log) ? gate_log : -20.0f); - beta_chunk[ct] = std::isfinite(beta) ? std::min(1.0f, std::max(0.0f, beta)) : 0.0f; - - // n0_proj fused here so k_chunk[ct] is still in L1 - const float* k_row = k_chunk + ct * K; - float* p_row = n0_proj + ct * V; - for (size_t kd = 0; kd < K; ++kd) { - const float kval = k_row[kd]; - const float* s_row = state + kd * V; - size_t vd = 0; - const float32x4_t kv4 = vdupq_n_f32(kval); - for (; vd + 8 <= V; vd += 8) { - float32x4_t p4 = vld1q_f32(p_row + vd); - const float32x4_t slo = vld1q_f32(s_row + vd); - const float32x4_t shi = vld1q_f32(s_row + vd + 4); - p4 = vfmaq_f32(p4, slo, kv4); - vst1q_f32(p_row + vd, p4); - p4 = vld1q_f32(p_row + vd + 4); - p4 = vfmaq_f32(p4, shi, kv4); - vst1q_f32(p_row + vd + 4, p4); - } - for (; vd < V; ++vd) { - p_row[vd] += kval * s_row[vd]; - } - } - } - - for (size_t t = 0; t < C; ++t) { - const float* kt = k_chunk + t * K; - for (size_t j = 0; j < t; ++j) { - const float* kj = k_chunk + j * K; - float dot = 0.0f; - size_t kd = 0; - float32x4_t acc4 = vdupq_n_f32(0.0f); - for (; kd + 8 <= K; kd += 8) { - const float32x4_t al = vld1q_f32(kt + kd); - const float32x4_t ah = vld1q_f32(kt + kd + 4); - const float32x4_t bl = vld1q_f32(kj + kd); - const float32x4_t bh = vld1q_f32(kj + kd + 4); - acc4 = vfmaq_f32(acc4, al, bl); - acc4 = vfmaq_f32(acc4, ah, bh); - } - dot += vaddvq_f32(acc4); - for (; kd < K; ++kd) { - dot += kt[kd] * kj[kd]; - } - gram[t * C + j] = dot; - } - } - - float running_p = 1.0f; - for (size_t t = 0; t < C; ++t) { - p_prev[t] = running_p; - running_p *= g_chunk[t]; - p_curr[t] = running_p; - inv_p_curr[t] = (std::fabs(running_p) > kMinAbs) ? (1.0f / running_p) : 0.0f; - } - - for (size_t t = 0; t < C; ++t) { - const float pp = p_prev[t]; - float* coeff_row = coeff + t * C; - for (size_t j = 0; j < t; ++j) { - const float inv_pc = inv_p_curr[j]; - coeff_row[j] = inv_pc == 0.0f ? 0.0f : (pp * inv_pc * gram[t * C + j]); - } - } - - for (size_t t = 0; t < C; ++t) { - const float pp = p_prev[t]; - const float bt = beta_chunk[t]; - const float* n0_row = n0_proj + t * V; - const float* v_row = v_chunk + t * V; - float* d_row = delta_chunk + t * V; - - size_t vd = 0; - const float32x4_t pp4 = vdupq_n_f32(pp); - const float32x4_t bt4 = vdupq_n_f32(bt); - for (; vd + 8 <= V; vd += 8) { - float32x4_t acc_lo = vmulq_f32(vld1q_f32(n0_row + vd), pp4); - float32x4_t acc_hi = vmulq_f32(vld1q_f32(n0_row + vd + 4), pp4); - for (size_t j = 0; j < t; ++j) { - const float c = coeff[t * C + j]; - const float* dj = delta_chunk + j * V + vd; - acc_lo = vmlaq_n_f32(acc_lo, vld1q_f32(dj), c); - acc_hi = vmlaq_n_f32(acc_hi, vld1q_f32(dj + 4), c); - } - vst1q_f32(d_row + vd, vmulq_f32(vsubq_f32(vld1q_f32(v_row + vd), acc_lo), bt4)); - vst1q_f32(d_row + vd + 4, vmulq_f32(vsubq_f32(vld1q_f32(v_row + vd + 4), acc_hi), bt4)); - } - for (; vd < V; ++vd) { - float acc = pp * n0_row[vd]; - for (size_t j = 0; j < t; ++j) { - acc += coeff[t * C + j] * delta_chunk[j * V + vd]; - } - d_row[vd] = bt * (v_row[vd] - acc); - } - } - - running_p = 1.0f; - for (size_t t = 0; t < C; ++t) { - const float gt = g_chunk[t]; - const float* k_row = k_chunk + t * K; - const float* q_row = q_chunk + t * K; - const float* d_row = delta_chunk + t * V; - - running_p *= gt; - const float p_run = running_p; - - for (size_t kd = 0; kd < K; ++kd) { - const float kval = k_row[kd]; - float* m_row = m_chunk + kd * V; - size_t vd = 0; - const float32x4_t gt4 = vdupq_n_f32(gt); - const float32x4_t kv4 = vdupq_n_f32(kval); - for (; vd + 8 <= V; vd += 8) { - float32x4_t ml = vld1q_f32(m_row + vd); - float32x4_t mh = vld1q_f32(m_row + vd + 4); - const float32x4_t dl = vld1q_f32(d_row + vd); - const float32x4_t dh = vld1q_f32(d_row + vd + 4); - ml = vmulq_f32(ml, gt4); - mh = vmulq_f32(mh, gt4); - ml = vfmaq_f32(ml, dl, kv4); - mh = vfmaq_f32(mh, dh, kv4); - vst1q_f32(m_row + vd, ml); - vst1q_f32(m_row + vd + 4, mh); - } - for (; vd < V; ++vd) { - m_row[vd] = gt * m_row[vd] + kval * d_row[vd]; - } - } - - const size_t out_base = (((batch * out_seq + (t0 + t)) * Hv + v_head) * V); - std::fill(out_acc, out_acc + V, 0.0f); - for (size_t kd = 0; kd < K; ++kd) { - const float qval = q_row[kd]; - const float* s_row = state + kd * V; - const float* m_row = m_chunk + kd * V; - size_t vd = 0; - const float32x4_t q4 = vdupq_n_f32(qval); - const float32x4_t p4 = vdupq_n_f32(p_run); - for (; vd + 8 <= V; vd += 8) { - float32x4_t acc4 = vld1q_f32(out_acc + vd); - float32x4_t acc4h = vld1q_f32(out_acc + vd + 4); - const float32x4_t sl = vld1q_f32(s_row + vd); - const float32x4_t sh = vld1q_f32(s_row + vd + 4); - const float32x4_t ml = vld1q_f32(m_row + vd); - const float32x4_t mh = vld1q_f32(m_row + vd + 4); - const float32x4_t nl = vfmaq_f32(ml, sl, p4); - const float32x4_t nh = vfmaq_f32(mh, sh, p4); - acc4 = vfmaq_f32(acc4, nl, q4); - acc4h = vfmaq_f32(acc4h, nh, q4); - vst1q_f32(out_acc + vd, acc4); - vst1q_f32(out_acc + vd + 4, acc4h); - } - for (; vd < V; ++vd) { - out_acc[vd] += qval * (p_run * s_row[vd] + m_row[vd]); - } - } - for (size_t vd = 0; vd < V; ++vd) { - out[out_base + vd] = static_cast<__fp16>(out_acc[vd] * scale); - } - } - - const float p_end = running_p; - for (size_t kd = 0; kd < K; ++kd) { - float* s_row = state + kd * V; - const float* m_row = m_chunk + kd * V; - size_t vd = 0; - const float32x4_t p4 = vdupq_n_f32(p_end); - for (; vd + 8 <= V; vd += 8) { - float32x4_t sl = vld1q_f32(s_row + vd); - float32x4_t sh = vld1q_f32(s_row + vd + 4); - const float32x4_t ml = vld1q_f32(m_row + vd); - const float32x4_t mh = vld1q_f32(m_row + vd + 4); - sl = vfmaq_f32(ml, sl, p4); - sh = vfmaq_f32(mh, sh, p4); - vst1q_f32(s_row + vd, sl); - vst1q_f32(s_row + vd + 4, sh); - } - for (; vd < V; ++vd) { - s_row[vd] = p_end * s_row[vd] + m_row[vd]; - } - } - } - - for (size_t kd = 0; kd < K; ++kd) { - const size_t out_idx = (((batch * out_seq + (T + kd)) * Hv + v_head) * V); - cactus_fp32_to_fp16(state + kd * V, out + out_idx, V); - } - } - }); -} - -} // namespace - -void cactus_gated_deltanet_decode_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - float scale) { - const size_t qk_repeat = Hv / Hq; - const size_t out_seq = 1 + K; - - CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t bh_start, size_t bh_end) { - std::vector state(K * V); - std::vector proj(V); - std::vector delta(V); - - for (size_t bh = bh_start; bh < bh_end; ++bh) { - const size_t batch = bh / Hv; - const size_t v_head = bh % Hv; - const size_t qk_head = v_head / qk_repeat; - - for (size_t kd = 0; kd < K; ++kd) { - for (size_t vd = 0; vd < V; ++vd) { - const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V + vd); - state[kd * V + vd] = static_cast(s_data[s_idx]); - } - } - - float state_scale = 1.0f; - const size_t qk_base = ((batch * Hq + qk_head) * K); - const size_t v_base = ((batch * Hv + v_head) * V); - const float gate_log = static_cast(g_data[batch * Hv + v_head]); - const float beta = static_cast(b_data[batch * Hv + v_head]); - - gated_deltanet_step( - q_data + qk_base, - k_data + qk_base, - v_data + v_base, - gate_log, - beta, - scale, - K, - V, - state, - state_scale, - proj, - delta, - out + (((batch * out_seq) * Hv + v_head) * V)); - - fold_state_scale(state, state_scale); - for (size_t kd = 0; kd < K; ++kd) { - for (size_t vd = 0; vd < V; ++vd) { - const size_t out_idx = (((batch * out_seq + (1 + kd)) * Hv + v_head) * V + vd); - out[out_idx] = static_cast<__fp16>(state[kd * V + vd]); - } - } - } - }); -} - -void cactus_gated_deltanet_prefill_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t T, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - size_t requested_chunk_size, - float scale) { - const char* force_old = std::getenv("CACTUS_GATED_DELTANET_PREFILL_OLD"); - if (force_old != nullptr && std::atoi(force_old) != 0) { - gated_deltanet_prefill_old_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, - B, T, Hq, Hv, K, V, scale); - return; - } - - if (requested_chunk_size <= 1) { - gated_deltanet_prefill_old_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, - B, T, Hq, Hv, K, V, scale); - return; - } - - const size_t chunk_size = tuned_gated_deltanet_chunk_size(requested_chunk_size, K, V); - gated_deltanet_prefill_chunked_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, - B, T, Hq, Hv, K, V, chunk_size, scale); -} diff --git a/cactus-transpile/kernel/kernel_i8mm.cpp b/cactus-transpile/kernel/kernel_i8mm.cpp deleted file mode 100644 index c8c976173..000000000 --- a/cactus-transpile/kernel/kernel_i8mm.cpp +++ /dev/null @@ -1,189 +0,0 @@ -#include - -#ifndef __ARM_FEATURE_MATMUL_INT8 -#error "kernel_i8mm.cpp must be compiled with I8MM enabled (e.g. -march=armv8.2-a+fp16+simd+dotprod+i8mm)" -#endif - -#include "kernel.h" -#include "kernel_utils.h" -#include -#include - -void cactus_gemv_int8_i8mm( - const int8_t* A, - const float A_scale, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t K, size_t N, - size_t group_size -) { - if (K == 0 || N == 0) return; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + 3) / 4; - - auto process_blocks = [=](size_t block_start, size_t block_end) { - for (size_t n_block = block_start; n_block < block_end; ++n_block) { - const size_t n_start = n_block * 4; - const size_t actual_n = std::min(size_t(4), N - n_start); - - float32x4_t running_sum = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const int8_t* b_ptr = B + (n_block * K + k_base) * 4; - - __builtin_prefetch(b_ptr + group_size * 8, 0, 3); - - int32x4_t acc_01 = vdupq_n_s32(0); - int32x4_t acc_23 = vdupq_n_s32(0); - - for (size_t k_off = 0; k_off < group_size; k_off += 8) { - int32x4_t chunk0 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4)); - int32x4_t chunk1 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4 + 16)); - - int8x16_t b01 = vreinterpretq_s8_s32(vzip1q_s32(chunk0, chunk1)); - int8x16_t b23 = vreinterpretq_s8_s32(vzip2q_s32(chunk0, chunk1)); - - int8x8_t a8 = vld1_s8(a_ptr + k_off); - int8x16_t a_dup = vcombine_s8(a8, a8); - - acc_01 = vmmlaq_s32(acc_01, a_dup, b01); - acc_23 = vmmlaq_s32(acc_23, a_dup, b23); - } - - float32x2_t r01 = vcvt_f32_s32(vget_low_s32(acc_01)); - float32x2_t r23 = vcvt_f32_s32(vget_low_s32(acc_23)); - float32x4_t group_result = vcombine_f32(r01, r23); - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float32x4_t scales = vcvt_f32_f16(vld1_f16(scale_ptr)); - running_sum = vmlaq_f32(running_sum, group_result, scales); - } - - float32x4_t result = vmulq_n_f32(running_sum, A_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - - if (actual_n == 4) { - vst1_f16(C + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - }; - - auto& pool = CactusThreading::get_thread_pool(); - size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); - num_threads = std::min(num_threads, N_blocks); - - if (num_threads <= 1) { - process_blocks(0, N_blocks); - } else { - pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); - pool.wait_all(); - } -} - -void cactus_gemm_int8_i8mm( - const int8_t* A, - const float* A_scales, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t M, size_t K, size_t N, - size_t group_size -) { - if (M == 0 || K == 0 || N == 0) return; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + 3) / 4; - const size_t M_pairs = (M + 1) / 2; - const size_t total_tiles = M_pairs * N_blocks; - - CactusThreading::parallel_gemm_tiles(M, total_tiles, - [=](size_t tile_start, size_t tile_end) { - for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { - const size_t m_pair = tile_idx / N_blocks; - const size_t n_block = tile_idx % N_blocks; - const size_t m0 = m_pair * 2; - const size_t m1 = std::min(m0 + 1, M - 1); - const size_t n_start = n_block * 4; - const size_t actual_n = std::min(size_t(4), N - n_start); - - const int8_t* a_row0 = A + m0 * K; - const int8_t* a_row1 = A + m1 * K; - - float32x4_t sum_row0 = vdupq_n_f32(0.0f); - float32x4_t sum_row1 = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* b_ptr = B + (n_block * K + k_base) * 4; - - __builtin_prefetch(b_ptr + group_size * 8, 0, 3); - - int32x4_t acc_01 = vdupq_n_s32(0); - int32x4_t acc_23 = vdupq_n_s32(0); - - for (size_t k_off = 0; k_off < group_size; k_off += 8) { - int32x4_t chunk0 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4)); - int32x4_t chunk1 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4 + 16)); - - int8x16_t b01 = vreinterpretq_s8_s32(vzip1q_s32(chunk0, chunk1)); - int8x16_t b23 = vreinterpretq_s8_s32(vzip2q_s32(chunk0, chunk1)); - - int8x8_t a0_8 = vld1_s8(a_row0 + k_base + k_off); - int8x8_t a1_8 = vld1_s8(a_row1 + k_base + k_off); - int8x16_t a_2rows = vcombine_s8(a0_8, a1_8); - - acc_01 = vmmlaq_s32(acc_01, a_2rows, b01); - acc_23 = vmmlaq_s32(acc_23, a_2rows, b23); - } - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float32x4_t scales = vcvt_f32_f16(vld1_f16(scale_ptr)); - - float32x2_t r0_01 = vcvt_f32_s32(vget_low_s32(acc_01)); - float32x2_t r0_23 = vcvt_f32_s32(vget_low_s32(acc_23)); - float32x4_t group_row0 = vcombine_f32(r0_01, r0_23); - sum_row0 = vmlaq_f32(sum_row0, group_row0, scales); - - float32x2_t r1_01 = vcvt_f32_s32(vget_high_s32(acc_01)); - float32x2_t r1_23 = vcvt_f32_s32(vget_high_s32(acc_23)); - float32x4_t group_row1 = vcombine_f32(r1_01, r1_23); - sum_row1 = vmlaq_f32(sum_row1, group_row1, scales); - } - - { - float32x4_t result = vmulq_n_f32(sum_row0, A_scales[m0]); - float16x4_t result_f16 = vcvt_f16_f32(result); - if (actual_n == 4) { - vst1_f16(C + m0 * N + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[m0 * N + n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - - if (m0 + 1 < M) { - float32x4_t result = vmulq_n_f32(sum_row1, A_scales[m1]); - float16x4_t result_f16 = vcvt_f16_f32(result); - if (actual_n == 4) { - vst1_f16(C + m1 * N + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[m1 * N + n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - } - }); -} diff --git a/cactus-transpile/kernel/kernel_lstm.cpp b/cactus-transpile/kernel/kernel_lstm.cpp deleted file mode 100644 index 84d0077a1..000000000 --- a/cactus-transpile/kernel/kernel_lstm.cpp +++ /dev/null @@ -1,117 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include - -void cactus_lstm_cell_f16( - const __fp16* x_input, - const __fp16* h_prev, - const __fp16* c_prev, - const __fp16* weight_ih, - const __fp16* weight_hh, - const __fp16* bias_ih, - const __fp16* bias_hh, - __fp16* h_new, - __fp16* c_new, - size_t batch_size, - size_t input_size, - size_t hidden_size -) { - constexpr size_t SIMD_WIDTH = 8; - const size_t gate_size = 4 * hidden_size; - - std::vector<__fp16> gates_ih(batch_size * gate_size); - std::vector<__fp16> gates_hh(batch_size * gate_size); - - cactus_matmul_f16(x_input, weight_ih, gates_ih.data(), batch_size, input_size, gate_size); - cactus_matmul_f16(h_prev, weight_hh, gates_hh.data(), batch_size, hidden_size, gate_size); - - const size_t simd_end = (hidden_size / SIMD_WIDTH) * SIMD_WIDTH; - - CactusThreading::parallel_for(batch_size, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t batch_start, size_t batch_end) { - const float32x4_t one = vdupq_n_f32(1.0f); - - for (size_t b = batch_start; b < batch_end; ++b) { - const size_t gate_offset = b * gate_size; - const size_t hidden_offset = b * hidden_size; - - for (size_t h = 0; h < simd_end; h += SIMD_WIDTH) { - float16x8_t i_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + h]), - vld1q_f16(&gates_hh[gate_offset + h])), - vaddq_f16(vld1q_f16(&bias_ih[h]), - vld1q_f16(&bias_hh[h]))); - - float16x8_t f_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + hidden_size + h]), - vld1q_f16(&gates_hh[gate_offset + hidden_size + h])), - vaddq_f16(vld1q_f16(&bias_ih[hidden_size + h]), - vld1q_f16(&bias_hh[hidden_size + h]))); - - float16x8_t g_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + 2 * hidden_size + h]), - vld1q_f16(&gates_hh[gate_offset + 2 * hidden_size + h])), - vaddq_f16(vld1q_f16(&bias_ih[2 * hidden_size + h]), - vld1q_f16(&bias_hh[2 * hidden_size + h]))); - - float16x8_t o_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + 3 * hidden_size + h]), - vld1q_f16(&gates_hh[gate_offset + 3 * hidden_size + h])), - vaddq_f16(vld1q_f16(&bias_ih[3 * hidden_size + h]), - vld1q_f16(&bias_hh[3 * hidden_size + h]))); - - float32x4_t i_low = vcvt_f32_f16(vget_low_f16(i_gate)); - float32x4_t i_high = vcvt_f32_f16(vget_high_f16(i_gate)); - float16x8_t i_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_low))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_high)))))); - - float32x4_t f_low = vcvt_f32_f16(vget_low_f16(f_gate)); - float32x4_t f_high = vcvt_f32_f16(vget_high_f16(f_gate)); - float16x8_t f_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_low))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_high)))))); - - float32x4_t g_low = vcvt_f32_f16(vget_low_f16(g_gate)); - float32x4_t g_high = vcvt_f32_f16(vget_high_f16(g_gate)); - float16x8_t g_act = vcombine_f16(vcvt_f16_f32(fast_tanh_f32x4(g_low)), - vcvt_f16_f32(fast_tanh_f32x4(g_high))); - - float32x4_t o_low = vcvt_f32_f16(vget_low_f16(o_gate)); - float32x4_t o_high = vcvt_f32_f16(vget_high_f16(o_gate)); - float16x8_t o_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_low))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_high)))))); - - float16x8_t c_prev_vec = vld1q_f16(&c_prev[hidden_offset + h]); - float16x8_t c_update = vfmaq_f16(vmulq_f16(f_act, c_prev_vec), i_act, g_act); - vst1q_f16(&c_new[hidden_offset + h], c_update); - - float32x4_t c_low = vcvt_f32_f16(vget_low_f16(c_update)); - float32x4_t c_high = vcvt_f32_f16(vget_high_f16(c_update)); - float16x8_t c_tanh = vcombine_f16(vcvt_f16_f32(fast_tanh_f32x4(c_low)), - vcvt_f16_f32(fast_tanh_f32x4(c_high))); - vst1q_f16(&h_new[hidden_offset + h], vmulq_f16(o_act, c_tanh)); - } - - for (size_t h = simd_end; h < hidden_size; ++h) { - float i_gate_val = static_cast(gates_ih[gate_offset + h] + - gates_hh[gate_offset + h] + - bias_ih[h] + bias_hh[h]); - float f_gate_val = static_cast(gates_ih[gate_offset + hidden_size + h] + - gates_hh[gate_offset + hidden_size + h] + - bias_ih[hidden_size + h] + bias_hh[hidden_size + h]); - float g_gate_val = static_cast(gates_ih[gate_offset + 2 * hidden_size + h] + - gates_hh[gate_offset + 2 * hidden_size + h] + - bias_ih[2 * hidden_size + h] + bias_hh[2 * hidden_size + h]); - float o_gate_val = static_cast(gates_ih[gate_offset + 3 * hidden_size + h] + - gates_hh[gate_offset + 3 * hidden_size + h] + - bias_ih[3 * hidden_size + h] + bias_hh[3 * hidden_size + h]); - - float i_act = 1.0f / (1.0f + expf(-i_gate_val)); - float f_act = 1.0f / (1.0f + expf(-f_gate_val)); - float g_act = tanhf(g_gate_val); - float o_act = 1.0f / (1.0f + expf(-o_gate_val)); - - float c_val = f_act * static_cast(c_prev[hidden_offset + h]) + i_act * g_act; - c_new[hidden_offset + h] = static_cast<__fp16>(c_val); - h_new[hidden_offset + h] = static_cast<__fp16>(o_act * tanhf(c_val)); - } - } - }); -} diff --git a/cactus-transpile/kernel/kernel_matmul.cpp b/cactus-transpile/kernel/kernel_matmul.cpp deleted file mode 100644 index 75ada0d99..000000000 --- a/cactus-transpile/kernel/kernel_matmul.cpp +++ /dev/null @@ -1,787 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include "../graph/graph.h" -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -constexpr size_t ACCELERATE_M_THRESHOLD = 4; -constexpr size_t ACCELERATE_K_THRESHOLD = 256; -#endif - -// Do NOT Remove: Uncomment for testing on various paths -// ----- -// TEMPORARY: Force fallback path for testing on DOTPROD devices -// #undef __ARM_FEATURE_DOTPROD - -#if defined(__ARM_FEATURE_DOTPROD) - #define CACTUS_DOTQ_LANE(acc, b, a, lane) vdotq_laneq_s32(acc, b, a, lane) -#else - static inline int32x4_t cactus_dotq_with_pattern(int32x4_t acc, int8x16_t b, int8x8_t a_pattern) { - int8x8_t b_lo = vget_low_s8(b); - int8x8_t b_hi = vget_high_s8(b); - - int16x8_t prod_lo = vmull_s8(b_lo, a_pattern); - int16x8_t prod_hi = vmull_s8(b_hi, a_pattern); - - int32x4_t sum_lo = vpaddlq_s16(prod_lo); - int32x4_t sum_hi = vpaddlq_s16(prod_hi); - - int32x2_t final_lo = vpadd_s32(vget_low_s32(sum_lo), vget_high_s32(sum_lo)); - int32x2_t final_hi = vpadd_s32(vget_low_s32(sum_hi), vget_high_s32(sum_hi)); - - return vaddq_s32(acc, vcombine_s32(final_lo, final_hi)); - } - - static inline int32x4_t cactus_dotq_lane0(int32x4_t acc, int8x16_t b, int8x16_t a) { - int8x8_t a_lo = vget_low_s8(a); - int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_lo), 0)); - return cactus_dotq_with_pattern(acc, b, a_pattern); - } - - static inline int32x4_t cactus_dotq_lane1(int32x4_t acc, int8x16_t b, int8x16_t a) { - int8x8_t a_lo = vget_low_s8(a); - int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_lo), 1)); - return cactus_dotq_with_pattern(acc, b, a_pattern); - } - - static inline int32x4_t cactus_dotq_lane2(int32x4_t acc, int8x16_t b, int8x16_t a) { - int8x8_t a_hi = vget_high_s8(a); - int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_hi), 0)); - return cactus_dotq_with_pattern(acc, b, a_pattern); - } - - static inline int32x4_t cactus_dotq_lane3(int32x4_t acc, int8x16_t b, int8x16_t a) { - int8x8_t a_hi = vget_high_s8(a); - int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_hi), 1)); - return cactus_dotq_with_pattern(acc, b, a_pattern); - } - - #define CACTUS_DOTQ_LANE(acc, b, a, lane) cactus_dotq_lane##lane(acc, b, a) -#endif - -static inline __fp16 hsum_f16x8(float16x8_t v) { - float16x4_t lo = vget_low_f16(v); - float16x4_t hi = vget_high_f16(v); - float16x4_t sum4 = vadd_f16(lo, hi); - float16x4_t sum2 = vadd_f16(sum4, vext_f16(sum4, sum4, 2)); - float16x4_t sum1 = vadd_f16(sum2, vext_f16(sum2, sum2, 1)); - return vget_lane_f16(sum1, 0); -} - -static void cactus_matmul_f16_worker( - const __fp16* a, - const __fp16* b_transposed, - __fp16* c, - size_t /*M*/, - size_t K, - size_t N, - size_t start_row, - size_t end_row -) { - constexpr size_t TILE_M = 4; - constexpr size_t TILE_N = 4; - const size_t K16 = (K / 16) * 16; - const size_t K8 = (K / 8) * 8; - - for (size_t row_block = start_row; row_block < end_row; row_block += TILE_M) { - const size_t m_end = std::min(row_block + TILE_M, end_row); - - for (size_t col_block = 0; col_block < N; col_block += TILE_N) { - const size_t n_end = std::min(col_block + TILE_N, N); - - float16x8_t acc[TILE_M][TILE_N]; - for (size_t m = 0; m < TILE_M; ++m) - for (size_t n = 0; n < TILE_N; ++n) - acc[m][n] = vdupq_n_f16(0); - - for (size_t k = 0; k < K16; k += 16) { - float16x8_t a0_lo = (row_block < m_end) ? vld1q_f16(a + row_block * K + k) : vdupq_n_f16(0); - float16x8_t a0_hi = (row_block < m_end) ? vld1q_f16(a + row_block * K + k + 8) : vdupq_n_f16(0); - float16x8_t a1_lo = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k) : vdupq_n_f16(0); - float16x8_t a1_hi = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k + 8) : vdupq_n_f16(0); - float16x8_t a2_lo = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k) : vdupq_n_f16(0); - float16x8_t a2_hi = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k + 8) : vdupq_n_f16(0); - float16x8_t a3_lo = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k) : vdupq_n_f16(0); - float16x8_t a3_hi = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k + 8) : vdupq_n_f16(0); - - for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { - float16x8_t b_lo = vld1q_f16(b_transposed + (col_block + ni) * K + k); - float16x8_t b_hi = vld1q_f16(b_transposed + (col_block + ni) * K + k + 8); - - acc[0][ni] = vfmaq_f16(acc[0][ni], a0_lo, b_lo); - acc[0][ni] = vfmaq_f16(acc[0][ni], a0_hi, b_hi); - acc[1][ni] = vfmaq_f16(acc[1][ni], a1_lo, b_lo); - acc[1][ni] = vfmaq_f16(acc[1][ni], a1_hi, b_hi); - acc[2][ni] = vfmaq_f16(acc[2][ni], a2_lo, b_lo); - acc[2][ni] = vfmaq_f16(acc[2][ni], a2_hi, b_hi); - acc[3][ni] = vfmaq_f16(acc[3][ni], a3_lo, b_lo); - acc[3][ni] = vfmaq_f16(acc[3][ni], a3_hi, b_hi); - } - } - - for (size_t k = K16; k < K8; k += 8) { - float16x8_t a0_v = (row_block < m_end) ? vld1q_f16(a + row_block * K + k) : vdupq_n_f16(0); - float16x8_t a1_v = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k) : vdupq_n_f16(0); - float16x8_t a2_v = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k) : vdupq_n_f16(0); - float16x8_t a3_v = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k) : vdupq_n_f16(0); - - for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { - float16x8_t b_v = vld1q_f16(b_transposed + (col_block + ni) * K + k); - acc[0][ni] = vfmaq_f16(acc[0][ni], a0_v, b_v); - acc[1][ni] = vfmaq_f16(acc[1][ni], a1_v, b_v); - acc[2][ni] = vfmaq_f16(acc[2][ni], a2_v, b_v); - acc[3][ni] = vfmaq_f16(acc[3][ni], a3_v, b_v); - } - } - - for (size_t k = K8; k < K; ++k) { - for (size_t mi = 0; mi < TILE_M && row_block + mi < m_end; ++mi) { - __fp16 av = a[(row_block + mi) * K + k]; - for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { - __fp16 bv = b_transposed[(col_block + ni) * K + k]; - acc[mi][ni] = vsetq_lane_f16(vgetq_lane_f16(acc[mi][ni], 0) + av * bv, acc[mi][ni], 0); - } - } - } - - for (size_t mi = 0; mi < TILE_M && row_block + mi < m_end; ++mi) { - for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { - c[(row_block + mi) * N + col_block + ni] = hsum_f16x8(acc[mi][ni]); - } - } - } - } -} - -#if defined(CACTUS_COMPILE_SME2) -constexpr size_t SME2_M_THRESHOLD = 4; - -void cactus_matmul_f16_sme2_caller( - const __fp16* a, - const __fp16* b_transposed, - __fp16* c, - size_t M, - size_t K, - size_t N -); -#endif - -void cactus_matmul_f16( - const __fp16* a, - const __fp16* b_transposed, - __fp16* c, - size_t M, - size_t K, - size_t N -) { - -#if defined(CACTUS_COMPILE_SME2) - if (cpu_has_sme2() && M >= SME2_M_THRESHOLD) { - cactus_matmul_f16_sme2_caller( - a, b_transposed, c, - M, K, N - ); - return; - } -#endif - -#ifdef __APPLE__ - if (K >= ACCELERATE_K_THRESHOLD && M >= ACCELERATE_M_THRESHOLD) { - const size_t a_len = M * K; - const size_t b_len = N * K; - const size_t c_len = M * N; - - std::vector A_f32(a_len); - std::vector BT_f32(b_len); - std::vector C_f32(c_len); - - for (size_t i = 0; i < a_len; i++) A_f32[i] = (float)a[i]; - for (size_t i = 0; i < b_len; i++) BT_f32[i] = (float)b_transposed[i]; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - (int)M, (int)N, (int)K, - 1.0f, A_f32.data(), (int)K, - BT_f32.data(), (int)K, - 0.0f, C_f32.data(), (int)N); - - for (size_t i = 0; i < c_len; i++) { - float v = C_f32[i]; - if (v > 65504.f) v = 65504.f; - else if (v < -65504.f) v = -65504.f; - c[i] = (__fp16)v; - } - return; - } -#endif - - constexpr size_t TILE_M = 4; - const size_t num_row_blocks = (M + TILE_M - 1) / TILE_M; - - CactusThreading::parallel_for(num_row_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [=](size_t start_block, size_t end_block) { - for (size_t block_idx = start_block; block_idx < end_block; ++block_idx) { - size_t start_row = block_idx * TILE_M; - size_t end_row = std::min(start_row + TILE_M, M); - - cactus_matmul_f16_worker( - a, b_transposed, c, - M, K, N, - start_row, end_row - ); - - } - }); -} - - -void cactus_gemv_int8( - const int8_t* A, - const float A_scale, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t K, size_t N, - size_t group_size -) { - if (K == 0 || N == 0) return; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + 3) / 4; - - auto process_blocks = [=](size_t block_start, size_t block_end) { - for (size_t n_block = block_start; n_block < block_end; ++n_block) { - const size_t n_start = n_block * 4; - const size_t actual_n = std::min(size_t(4), N - n_start); - - float32x4_t running_sum = vdupq_n_f32(0.0f); - - size_t g = 0; - for (; g + 1 < num_groups; g += 2) { - const size_t k_base0 = g * group_size; - const size_t k_base1 = (g + 1) * group_size; - - const int8_t* a_ptr0 = A + k_base0; - const int8_t* a_ptr1 = A + k_base1; - const int8_t* b_base0 = B + (n_block * K + k_base0) * 4; - const int8_t* b_base1 = B + (n_block * K + k_base1) * 4; - - __builtin_prefetch(b_base0 + group_size * 8, 0, 3); - - int32x4_t acc0 = vdupq_n_s32(0); - int32x4_t acc1 = vdupq_n_s32(0); - - { - int8x16_t a_vec = vld1q_s8(a_ptr0); - int8x16_t b0 = vld1q_s8(b_base0); - int8x16_t b1 = vld1q_s8(b_base0 + 16); - int8x16_t b2 = vld1q_s8(b_base0 + 32); - int8x16_t b3 = vld1q_s8(b_base0 + 48); - - acc0 = CACTUS_DOTQ_LANE(acc0, b0, a_vec, 0); - acc0 = CACTUS_DOTQ_LANE(acc0, b1, a_vec, 1); - acc0 = CACTUS_DOTQ_LANE(acc0, b2, a_vec, 2); - acc0 = CACTUS_DOTQ_LANE(acc0, b3, a_vec, 3); - - a_vec = vld1q_s8(a_ptr0 + 16); - b0 = vld1q_s8(b_base0 + 64); - b1 = vld1q_s8(b_base0 + 80); - b2 = vld1q_s8(b_base0 + 96); - b3 = vld1q_s8(b_base0 + 112); - - acc0 = CACTUS_DOTQ_LANE(acc0, b0, a_vec, 0); - acc0 = CACTUS_DOTQ_LANE(acc0, b1, a_vec, 1); - acc0 = CACTUS_DOTQ_LANE(acc0, b2, a_vec, 2); - acc0 = CACTUS_DOTQ_LANE(acc0, b3, a_vec, 3); - } - - { - int8x16_t a_vec = vld1q_s8(a_ptr1); - int8x16_t b0 = vld1q_s8(b_base1); - int8x16_t b1 = vld1q_s8(b_base1 + 16); - int8x16_t b2 = vld1q_s8(b_base1 + 32); - int8x16_t b3 = vld1q_s8(b_base1 + 48); - - acc1 = CACTUS_DOTQ_LANE(acc1, b0, a_vec, 0); - acc1 = CACTUS_DOTQ_LANE(acc1, b1, a_vec, 1); - acc1 = CACTUS_DOTQ_LANE(acc1, b2, a_vec, 2); - acc1 = CACTUS_DOTQ_LANE(acc1, b3, a_vec, 3); - - a_vec = vld1q_s8(a_ptr1 + 16); - b0 = vld1q_s8(b_base1 + 64); - b1 = vld1q_s8(b_base1 + 80); - b2 = vld1q_s8(b_base1 + 96); - b3 = vld1q_s8(b_base1 + 112); - - acc1 = CACTUS_DOTQ_LANE(acc1, b0, a_vec, 0); - acc1 = CACTUS_DOTQ_LANE(acc1, b1, a_vec, 1); - acc1 = CACTUS_DOTQ_LANE(acc1, b2, a_vec, 2); - acc1 = CACTUS_DOTQ_LANE(acc1, b3, a_vec, 3); - } - - const __fp16* scale_ptr0 = B_scales + (n_block * num_groups + g) * 4; - const __fp16* scale_ptr1 = B_scales + (n_block * num_groups + g + 1) * 4; - - float16x4_t scales0_f16 = vld1_f16(scale_ptr0); - float16x4_t scales1_f16 = vld1_f16(scale_ptr1); - float32x4_t scales0 = vcvt_f32_f16(scales0_f16); - float32x4_t scales1 = vcvt_f32_f16(scales1_f16); - - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc0), scales0); - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc1), scales1); - } - - for (; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const int8_t* b_base = B + (n_block * K + k_base) * 4; - - int32x4_t acc = vdupq_n_s32(0); - - int8x16_t a_vec = vld1q_s8(a_ptr); - int8x16_t b0 = vld1q_s8(b_base); - int8x16_t b1 = vld1q_s8(b_base + 16); - int8x16_t b2 = vld1q_s8(b_base + 32); - int8x16_t b3 = vld1q_s8(b_base + 48); - - acc = CACTUS_DOTQ_LANE(acc, b0, a_vec, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_vec, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_vec, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_vec, 3); - - a_vec = vld1q_s8(a_ptr + 16); - b0 = vld1q_s8(b_base + 64); - b1 = vld1q_s8(b_base + 80); - b2 = vld1q_s8(b_base + 96); - b3 = vld1q_s8(b_base + 112); - - acc = CACTUS_DOTQ_LANE(acc, b0, a_vec, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_vec, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_vec, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_vec, 3); - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float16x4_t scales_f16 = vld1_f16(scale_ptr); - float32x4_t scales = vcvt_f32_f16(scales_f16); - - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc), scales); - } - - float32x4_t result = vmulq_n_f32(running_sum, A_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - - if (actual_n == 4) { - vst1_f16(C + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - }; - - auto& pool = CactusThreading::get_thread_pool(); - size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); - num_threads = std::min(num_threads, N_blocks); - - if (num_threads <= 1) { - process_blocks(0, N_blocks); - } else { - pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); - pool.wait_all(); - } -} - -void cactus_gemm_int8( - const int8_t* A, - const float* A_scales, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t M, size_t K, size_t N, - size_t group_size -) { - if (M == 0 || K == 0 || N == 0) return; - - constexpr size_t TILE_M = 8; - constexpr size_t TILE_N = 4; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + TILE_N - 1) / TILE_N; - const size_t num_row_tiles = (M + TILE_M - 1) / TILE_M; - const size_t total_tiles = num_row_tiles * N_blocks; - - CactusThreading::parallel_gemm_tiles(M, total_tiles, - [=](size_t tile_start, size_t tile_end) { - for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { - const size_t tile_row = tile_idx / N_blocks; - const size_t n_block = tile_idx % N_blocks; - const size_t m_start = tile_row * TILE_M; - const size_t m_end = std::min(m_start + TILE_M, M); - const size_t n_start = n_block * TILE_N; - const size_t n_end = std::min(n_start + TILE_N, N); - const size_t actual_m = m_end - m_start; - const size_t actual_n = n_end - n_start; - - const int8_t* a_rows[TILE_M]; - for (size_t mi = 0; mi < TILE_M; mi++) { - size_t row = m_start + (mi < actual_m ? mi : actual_m - 1); - a_rows[mi] = A + row * K; - } - - float32x4_t running_sum[TILE_M]; - for (size_t mi = 0; mi < TILE_M; mi++) { - running_sum[mi] = vdupq_n_f32(0.0f); - } - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* b_base = B + (n_block * K + k_base) * 4; - - __builtin_prefetch(b_base + group_size * 4, 0, 3); - - int8x16_t b00 = vld1q_s8(b_base); - int8x16_t b01 = vld1q_s8(b_base + 16); - int8x16_t b02 = vld1q_s8(b_base + 32); - int8x16_t b03 = vld1q_s8(b_base + 48); - - int8x16_t b10 = vld1q_s8(b_base + 64); - int8x16_t b11 = vld1q_s8(b_base + 80); - int8x16_t b12 = vld1q_s8(b_base + 96); - int8x16_t b13 = vld1q_s8(b_base + 112); - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float16x4_t scales_f16 = vld1_f16(scale_ptr); - float32x4_t scales = vcvt_f32_f16(scales_f16); - - #define CACTUS_GEMM_ROW(ROW) do { \ - const int8_t* a_ptr_##ROW = a_rows[ROW] + k_base; \ - int32x4_t acc_##ROW = vdupq_n_s32(0); \ - int8x16_t a_lo_##ROW = vld1q_s8(a_ptr_##ROW); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b00, a_lo_##ROW, 0); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b01, a_lo_##ROW, 1); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b02, a_lo_##ROW, 2); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b03, a_lo_##ROW, 3); \ - int8x16_t a_hi_##ROW = vld1q_s8(a_ptr_##ROW + 16); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b10, a_hi_##ROW, 0); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b11, a_hi_##ROW, 1); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b12, a_hi_##ROW, 2); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b13, a_hi_##ROW, 3); \ - running_sum[ROW] = vmlaq_f32(running_sum[ROW], vcvtq_f32_s32(acc_##ROW), scales); \ - } while(0) - - CACTUS_GEMM_ROW(0); - CACTUS_GEMM_ROW(1); - CACTUS_GEMM_ROW(2); - CACTUS_GEMM_ROW(3); - CACTUS_GEMM_ROW(4); - CACTUS_GEMM_ROW(5); - CACTUS_GEMM_ROW(6); - CACTUS_GEMM_ROW(7); - #undef CACTUS_GEMM_ROW - } - - for (size_t mi = 0; mi < actual_m; mi++) { - const float a_scale = A_scales[m_start + mi]; - float32x4_t result = vmulq_n_f32(running_sum[mi], a_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - - if (actual_n == 4) { - vst1_f16(C + (m_start + mi) * N + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[(m_start + mi) * N + n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - } - }); -} - -void cactus_matmul_int8( - const int8_t* A, - const float* A_scales, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t M, size_t K, size_t N, - size_t group_size -) { - if (M == 0 || K == 0 || N == 0) return; - -#if defined(CACTUS_COMPILE_I8MM) - if (cpu_has_i8mm()) { - if (M == 1) { - cactus_gemv_int8_i8mm(A, A_scales[0], B, B_scales, C, K, N, group_size); - } else { - cactus_gemm_int8_i8mm(A, A_scales, B, B_scales, C, M, K, N, group_size); - } - return; - } -#endif - - if (M == 1) { - cactus_gemv_int8(A, A_scales[0], B, B_scales, C, K, N, group_size); - } else { - cactus_gemm_int8(A, A_scales, B, B_scales, C, M, K, N, group_size); - } -} - -void cactus_gemv_int4( - const int8_t* A, - const float A_scale, - const int8_t* B_packed_raw, - const __fp16* B_scales, - __fp16* C, - size_t K, size_t N, - size_t group_size -) { - const uint8_t* B_packed = reinterpret_cast(B_packed_raw); - if (K == 0 || N == 0) return; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + 3) / 4; - - auto process_blocks = [=](size_t block_start, size_t block_end) { - size_t n_block = block_start; - - for (; n_block + 1 < block_end; n_block += 2) { - float32x4_t sum_a = vdupq_n_f32(0.0f); - float32x4_t sum_b = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const uint8_t* ba = B_packed + (n_block * K + k_base) * 2; - const uint8_t* bb = B_packed + ((n_block + 1) * K + k_base) * 2; - - int32x4_t acc_a = vdupq_n_s32(0); - int32x4_t acc_b = vdupq_n_s32(0); - - int8x16_t a_lo = vld1q_s8(a_ptr); - int8x16_t a_hi = vld1q_s8(a_ptr + 16); - - { - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(ba, b1, b0); - unpack_int4_as_int8x16x2(ba + 16, b3, b2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_lo, 0); - acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_lo, 1); - acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_lo, 2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_lo, 3); - unpack_int4_as_int8x16x2(ba + 32, b1, b0); - unpack_int4_as_int8x16x2(ba + 48, b3, b2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_hi, 0); - acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_hi, 1); - acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_hi, 2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_hi, 3); - } - { - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(bb, b1, b0); - unpack_int4_as_int8x16x2(bb + 16, b3, b2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_lo, 0); - acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_lo, 1); - acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_lo, 2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_lo, 3); - unpack_int4_as_int8x16x2(bb + 32, b1, b0); - unpack_int4_as_int8x16x2(bb + 48, b3, b2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_hi, 0); - acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_hi, 1); - acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_hi, 2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_hi, 3); - } - - const __fp16* spa = B_scales + (n_block * num_groups + g) * 4; - const __fp16* spb = B_scales + ((n_block + 1) * num_groups + g) * 4; - float32x4_t sa = vcvt_f32_f16(vld1_f16(spa)); - float32x4_t sb = vcvt_f32_f16(vld1_f16(spb)); - sum_a = vmlaq_f32(sum_a, vcvtq_f32_s32(acc_a), sa); - sum_b = vmlaq_f32(sum_b, vcvtq_f32_s32(acc_b), sb); - } - - vst1_f16(C + n_block * 4, vcvt_f16_f32(vmulq_n_f32(sum_a, A_scale))); - vst1_f16(C + (n_block + 1) * 4, vcvt_f16_f32(vmulq_n_f32(sum_b, A_scale))); - } - - for (; n_block < block_end; ++n_block) { - const size_t n_start = n_block * 4; - const size_t actual_n = std::min(size_t(4), N - n_start); - float32x4_t running_sum = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const uint8_t* b_base = B_packed + (n_block * K + k_base) * 2; - - int32x4_t acc = vdupq_n_s32(0); - int8x16_t a_lo = vld1q_s8(a_ptr); - int8x16_t a_hi = vld1q_s8(a_ptr + 16); - - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(b_base, b1, b0); - unpack_int4_as_int8x16x2(b_base + 16, b3, b2); - acc = CACTUS_DOTQ_LANE(acc, b0, a_lo, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_lo, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_lo, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_lo, 3); - unpack_int4_as_int8x16x2(b_base + 32, b1, b0); - unpack_int4_as_int8x16x2(b_base + 48, b3, b2); - acc = CACTUS_DOTQ_LANE(acc, b0, a_hi, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_hi, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_hi, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_hi, 3); - - float32x4_t scales = vcvt_f32_f16(vld1_f16(B_scales + (n_block * num_groups + g) * 4)); - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc), scales); - } - - float32x4_t result = vmulq_n_f32(running_sum, A_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - if (actual_n == 4) { - vst1_f16(C + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - }; - - auto& pool = CactusThreading::get_thread_pool(); - size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); - num_threads = std::min(num_threads, N_blocks); - - if (num_threads <= 1) { - process_blocks(0, N_blocks); - } else { - pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); - pool.wait_all(); - } -} - - -void cactus_gemm_int4( - const int8_t* A, - const float* A_scales, - const int8_t* B_packed_raw, - const __fp16* B_scales, - __fp16* C, - size_t M, size_t K, size_t N, - size_t group_size -) { - if (M == 0 || K == 0 || N == 0) return; - - const uint8_t* B_packed = reinterpret_cast(B_packed_raw); - - constexpr size_t TILE_M = 4; - constexpr size_t TILE_N = 4; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + TILE_N - 1) / TILE_N; - const size_t num_row_tiles = (M + TILE_M - 1) / TILE_M; - const size_t total_tiles = num_row_tiles * N_blocks; - - CactusThreading::parallel_gemm_tiles(M, total_tiles, - [=](size_t tile_start, size_t tile_end) { - for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { - const size_t tile_row = tile_idx / N_blocks; - const size_t n_block = tile_idx % N_blocks; - const size_t m_start = tile_row * TILE_M; - const size_t m_end = std::min(m_start + TILE_M, M); - const size_t n_start = n_block * TILE_N; - const size_t n_end = std::min(n_start + TILE_N, N); - const size_t actual_m = m_end - m_start; - const size_t actual_n = n_end - n_start; - - float32x4_t running_sum[TILE_M] = { - vdupq_n_f32(0.0f), vdupq_n_f32(0.0f), - vdupq_n_f32(0.0f), vdupq_n_f32(0.0f) - }; - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const uint8_t* b_base = B_packed + (n_block * K + k_base) * 2; - - __builtin_prefetch(b_base + group_size * 2, 0, 3); - - int8x16_t b00, b01, b02, b03; - int8x16_t b10, b11, b12, b13; - - unpack_int4_as_int8x16x2(b_base, b01, b00); - unpack_int4_as_int8x16x2(b_base + 16, b03, b02); - unpack_int4_as_int8x16x2(b_base + 32, b11, b10); - unpack_int4_as_int8x16x2(b_base + 48, b13, b12); - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float16x4_t scales_f16 = vld1_f16(scale_ptr); - float32x4_t scales = vcvt_f32_f16(scales_f16); - - for (size_t mi = 0; mi < actual_m; mi++) { - const int8_t* a_ptr = A + (m_start + mi) * K + k_base; - - int32x4_t acc = vdupq_n_s32(0); - - int8x16_t a_vec = vld1q_s8(a_ptr); - acc = CACTUS_DOTQ_LANE(acc, b00, a_vec, 0); - acc = CACTUS_DOTQ_LANE(acc, b01, a_vec, 1); - acc = CACTUS_DOTQ_LANE(acc, b02, a_vec, 2); - acc = CACTUS_DOTQ_LANE(acc, b03, a_vec, 3); - - a_vec = vld1q_s8(a_ptr + 16); - acc = CACTUS_DOTQ_LANE(acc, b10, a_vec, 0); - acc = CACTUS_DOTQ_LANE(acc, b11, a_vec, 1); - acc = CACTUS_DOTQ_LANE(acc, b12, a_vec, 2); - acc = CACTUS_DOTQ_LANE(acc, b13, a_vec, 3); - - running_sum[mi] = vmlaq_f32(running_sum[mi], vcvtq_f32_s32(acc), scales); - } - } - - for (size_t mi = 0; mi < actual_m; mi++) { - const float a_scale = A_scales[m_start + mi]; - float32x4_t result = vmulq_n_f32(running_sum[mi], a_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - - if (actual_n == 4) { - vst1_f16(C + (m_start + mi) * N + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[(m_start + mi) * N + n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - } - }); -} - -void cactus_matmul_int4(const int8_t* A, const float* A_scales, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size) { - if (M == 0 || K == 0 || N == 0) return; - - if (M == 1) { - cactus_gemv_int4(A, A_scales[0], B_packed, B_scales, C, K, N, group_size); - } else { - cactus_gemm_int4(A, A_scales, B_packed, B_scales, C, M, K, N, group_size); - } -} - -void cactus_matmul_integer(Precision precision, - const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size) { - if (precision == Precision::INT4) { - cactus_matmul_int4(A, A_scales, B, B_scales, C, M, K, N, group_size); - } else { - cactus_matmul_int8(A, A_scales, B, B_scales, C, M, K, N, group_size); - } -} diff --git a/cactus-transpile/kernel/kernel_nn.cpp b/cactus-transpile/kernel/kernel_nn.cpp deleted file mode 100644 index 7dd96ddc0..000000000 --- a/cactus-transpile/kernel/kernel_nn.cpp +++ /dev/null @@ -1,874 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include - -void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements) { - for (size_t i = 0; i < num_elements; ++i) { - __fp16 x = input[i]; - output[i] = x > static_cast<__fp16>(0) ? x : static_cast<__fp16>(0); - } -} - -void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope) { - for (size_t i = 0; i < num_elements; ++i) { - __fp16 x = input[i]; - output[i] = x > static_cast<__fp16>(0) ? x : static_cast<__fp16>(static_cast(x) * negative_slope); - } -} - -void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - for (size_t i = start_idx; i < end_idx; ++i) { - float x_f32 = static_cast(input[i]); - float sigmoid = 1.0f / (1.0f + expf(-x_f32)); - output[i] = static_cast<__fp16>(x_f32 * sigmoid); - } - }); -} - -void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements) { - const float sqrt_2_over_pi = 0.7978845608028654f; - const float coeff = 0.044715f; - - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - const float32x4_t half = vdupq_n_f32(0.5f); - const float32x4_t one = vdupq_n_f32(1.0f); - const float32x4_t sqrt_2_pi_vec = vdupq_n_f32(sqrt_2_over_pi); - const float32x4_t coeff_vec = vdupq_n_f32(coeff); - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t x_f16 = vld1q_f16(&input[i]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_f16)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_f16)); - - float32x4_t x_cubed_low = vmulq_f32(vmulq_f32(x_low, x_low), x_low); - float32x4_t x_cubed_high = vmulq_f32(vmulq_f32(x_high, x_high), x_high); - - float32x4_t inner_low = vfmaq_f32(x_low, coeff_vec, x_cubed_low); - float32x4_t inner_high = vfmaq_f32(x_high, coeff_vec, x_cubed_high); - inner_low = vmulq_f32(sqrt_2_pi_vec, inner_low); - inner_high = vmulq_f32(sqrt_2_pi_vec, inner_high); - - float32x4_t tanh_low = fast_tanh_f32x4(inner_low); - float32x4_t tanh_high = fast_tanh_f32x4(inner_high); - - float32x4_t gelu_low = vmulq_f32(vmulq_f32(half, x_low), vaddq_f32(one, tanh_low)); - float32x4_t gelu_high = vmulq_f32(vmulq_f32(half, x_high), vaddq_f32(one, tanh_high)); - - float16x8_t gelu_f16 = vcombine_f16(vcvt_f16_f32(gelu_low), vcvt_f16_f32(gelu_high)); - vst1q_f16(&output[i], gelu_f16); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float x = static_cast(input[i]); - float inner = sqrt_2_over_pi * (x + coeff * x * x * x); - float gelu = 0.5f * x * (1.0f + tanhf(inner)); - output[i] = static_cast<__fp16>(gelu); - } - }); -} - -void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements) -{ - const float inv_sqrt2 = 0.70710678118654752440f; - - CactusThreading::parallel_for( - num_elements, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - - constexpr size_t SIMD = 8; - size_t vec_end = start_idx + ((end_idx - start_idx) / SIMD) * SIMD; - - for (size_t i = start_idx; i < vec_end; i += SIMD) { - - float16x8_t xh = vld1q_f16(&input[i]); - - float32x4_t x0 = vcvt_f32_f16(vget_low_f16(xh)); - float32x4_t x1 = vcvt_f32_f16(vget_high_f16(xh)); - - float32x4_t arg0 = vmulq_n_f32(x0, inv_sqrt2); - float32x4_t arg1 = vmulq_n_f32(x1, inv_sqrt2); - - float arg0_s[4], arg1_s[4]; - float erf0_s[4], erf1_s[4]; - vst1q_f32(arg0_s, arg0); - vst1q_f32(arg1_s, arg1); - - for (int j = 0; j < 4; j++) { - erf0_s[j] = erff(arg0_s[j]); - erf1_s[j] = erff(arg1_s[j]); - } - - float32x4_t erf0 = vld1q_f32(erf0_s); - float32x4_t erf1 = vld1q_f32(erf1_s); - - float32x4_t t0 = vaddq_f32(vdupq_n_f32(1.0f), erf0); - float32x4_t t1 = vaddq_f32(vdupq_n_f32(1.0f), erf1); - - float32x4_t gelu0 = vmulq_f32(vmulq_n_f32(x0, 0.5f), t0); - float32x4_t gelu1 = vmulq_f32(vmulq_n_f32(x1, 0.5f), t1); - - float16x8_t gelu_h = vcombine_f16( - vcvt_f16_f32(gelu0), - vcvt_f16_f32(gelu1) - ); - - vst1q_f16(&output[i], gelu_h); - } - - for (size_t i = vec_end; i < end_idx; i++) { - float x = (float)input[i]; - float arg = x * inv_sqrt2; - float res = 0.5f * x * (1.0f + erff(arg)); - output[i] = (__fp16)res; - } - } - ); -} - -void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - const float32x4_t one_f32 = vdupq_n_f32(1.0f); - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t x = vld1q_f16(&input[i]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x)); - - float32x4_t exp_low = fast_exp_f32x4(vnegq_f32(x_low)); - float32x4_t exp_high = fast_exp_f32x4(vnegq_f32(x_high)); - - float32x4_t sigmoid_low = vdivq_f32(one_f32, vaddq_f32(one_f32, exp_low)); - float32x4_t sigmoid_high = vdivq_f32(one_f32, vaddq_f32(one_f32, exp_high)); - - float16x8_t sigmoid = vcombine_f16(vcvt_f16_f32(sigmoid_low), vcvt_f16_f32(sigmoid_high)); - vst1q_f16(&output[i], sigmoid); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float x_f32 = static_cast(input[i]); - float sigmoid = 1.0f / (1.0f + expf(-x_f32)); - output[i] = static_cast<__fp16>(sigmoid); - } - }); -} - -void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t x_f16 = vld1q_f16(&input[i]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_f16)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_f16)); - - float32x4_t tanh_low = fast_tanh_f32x4(x_low); - float32x4_t tanh_high = fast_tanh_f32x4(x_high); - - float16x8_t tanh_f16 = vcombine_f16(vcvt_f16_f32(tanh_low), vcvt_f16_f32(tanh_high)); - vst1q_f16(&output[i], tanh_f16); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float x = static_cast(input[i]); - output[i] = static_cast<__fp16>(std::tanh(x)); - } - }); -} - -void cactus_glu_f16( - const __fp16* input, - __fp16* output, - size_t outer_size, - size_t split_size, - size_t inner_size -) { - if (outer_size == 0 || split_size == 0 || inner_size == 0) return; - - const size_t axis_size = split_size * 2; - const size_t rows = outer_size * split_size; - - CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t row_start, size_t row_end) { - for (size_t row = row_start; row < row_end; ++row) { - const size_t outer_idx = row / split_size; - const size_t split_idx = row - outer_idx * split_size; - const size_t in_base = outer_idx * axis_size * inner_size; - const size_t out_base = outer_idx * split_size * inner_size; - - const __fp16* a = input + in_base + split_idx * inner_size; - const __fp16* b = input + in_base + (split_idx + split_size) * inner_size; - __fp16* y = output + out_base + split_idx * inner_size; - - for (size_t i = 0; i < inner_size; ++i) { - const float gate_input = static_cast(b[i]); - const float gate = gate_input >= 0.0f - ? 1.0f / (1.0f + expf(-gate_input)) - : expf(gate_input) / (1.0f + expf(gate_input)); - y[i] = static_cast<__fp16>(static_cast(a[i]) * gate); - } - } - }); -} - -void cactus_glu_f32( - const float* input, - float* output, - size_t outer_size, - size_t split_size, - size_t inner_size -) { - if (outer_size == 0 || split_size == 0 || inner_size == 0) return; - - const size_t axis_size = split_size * 2; - const size_t rows = outer_size * split_size; - - CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t row_start, size_t row_end) { - for (size_t row = row_start; row < row_end; ++row) { - const size_t outer_idx = row / split_size; - const size_t split_idx = row - outer_idx * split_size; - const size_t in_base = outer_idx * axis_size * inner_size; - const size_t out_base = outer_idx * split_size * inner_size; - - const float* a = input + in_base + split_idx * inner_size; - const float* b = input + in_base + (split_idx + split_size) * inner_size; - float* y = output + out_base + split_idx * inner_size; - - for (size_t i = 0; i < inner_size; ++i) { - const float gate_input = b[i]; - const float gate = gate_input >= 0.0f - ? 1.0f / (1.0f + expf(-gate_input)) - : expf(gate_input) / (1.0f + expf(gate_input)); - y[i] = a[i] * gate; - } - } - }); -} - -void cactus_batchnorm_f16( - const __fp16* input, - const float* weight, - const float* bias, - const float* running_mean, - const float* running_var, - __fp16* output, - size_t outer_size, - size_t channels, - size_t inner_size, - float epsilon -) { - if (outer_size == 0 || channels == 0 || inner_size == 0) return; - - std::vector scale(channels); - std::vector shift(channels); - for (size_t c = 0; c < channels; ++c) { - const float inv_std = 1.0f / std::sqrt(running_var[c] + epsilon); - scale[c] = weight[c] * inv_std; - shift[c] = bias[c] - running_mean[c] * scale[c]; - } - - const size_t rows = outer_size * channels; - CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t row_start, size_t row_end) { - for (size_t row = row_start; row < row_end; ++row) { - const size_t c = row % channels; - const float32x4_t scv = vdupq_n_f32(scale[c]); - const float32x4_t shv = vdupq_n_f32(shift[c]); - - const __fp16* x = input + row * inner_size; - __fp16* y = output + row * inner_size; - - size_t i = 0; - for (; i + 8 <= inner_size; i += 8) { - const float16x8_t xv = vld1q_f16(x + i); - float32x4_t lo = vcvt_f32_f16(vget_low_f16(xv)); - float32x4_t hi = vcvt_f32_f16(vget_high_f16(xv)); - lo = vfmaq_f32(shv, lo, scv); - hi = vfmaq_f32(shv, hi, scv); - const float16x8_t yv = vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); - vst1q_f16(y + i, yv); - } - - for (; i < inner_size; ++i) { - y[i] = static_cast<__fp16>(static_cast(x[i]) * scale[c] + shift[c]); - } - } - }); -} - -void cactus_batchnorm_f32( - const float* input, - const float* weight, - const float* bias, - const float* running_mean, - const float* running_var, - float* output, - size_t outer_size, - size_t channels, - size_t inner_size, - float epsilon -) { - if (outer_size == 0 || channels == 0 || inner_size == 0) return; - - std::vector scale(channels); - std::vector shift(channels); - for (size_t c = 0; c < channels; ++c) { - const float inv_std = 1.0f / std::sqrt(running_var[c] + epsilon); - scale[c] = weight[c] * inv_std; - shift[c] = bias[c] - running_mean[c] * scale[c]; - } - - const size_t rows = outer_size * channels; - CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t row_start, size_t row_end) { - for (size_t row = row_start; row < row_end; ++row) { - const size_t c = row % channels; - const float32x4_t scv = vdupq_n_f32(scale[c]); - const float32x4_t shv = vdupq_n_f32(shift[c]); - - const float* x = input + row * inner_size; - float* y = output + row * inner_size; - - size_t i = 0; - for (; i + 4 <= inner_size; i += 4) { - const float32x4_t xv = vld1q_f32(x + i); - vst1q_f32(y + i, vfmaq_f32(shv, xv, scv)); - } - - for (; i < inner_size; ++i) { - y[i] = x[i] * scale[c] + shift[c]; - } - } - }); -} - -void kernel_softmax_f16_single(const __fp16* input, __fp16* output, size_t vocab_size) { - - constexpr size_t SIMD_WIDTH = 8; - constexpr size_t UNROLL_FACTOR = 4; - constexpr size_t VECTORIZED_WIDTH = SIMD_WIDTH * UNROLL_FACTOR; - const size_t vocab_vectorized = (vocab_size / VECTORIZED_WIDTH) * VECTORIZED_WIDTH; - - float32x4_t max_vec[UNROLL_FACTOR * 2]; - for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { - max_vec[u] = vdupq_n_f32(-std::numeric_limits::infinity()); - } - - for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t x_vec_f16 = vld1q_f16(&input[i + u * SIMD_WIDTH]); - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_vec_f16)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_vec_f16)); - max_vec[u * 2] = vmaxq_f32(max_vec[u * 2], x_low); - max_vec[u * 2 + 1] = vmaxq_f32(max_vec[u * 2 + 1], x_high); - } - } - - float32x4_t final_max = max_vec[0]; - for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { - final_max = vmaxq_f32(final_max, max_vec[u]); - } - - float max_val = vmaxvq_f32(final_max); - for (size_t i = vocab_vectorized; i < vocab_size; ++i) { - max_val = std::max(max_val, static_cast(input[i])); - } - - const float32x4_t max_broadcast = vdupq_n_f32(max_val); - - float32x4_t sum_vec[UNROLL_FACTOR * 2]; - for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { - sum_vec[u] = vdupq_n_f32(0.0f); - } - - for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t x_vec_f16 = vld1q_f16(&input[i + u * SIMD_WIDTH]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_vec_f16)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_vec_f16)); - - float32x4_t exp_low = fast_exp_f32x4(vsubq_f32(x_low, max_broadcast)); - float32x4_t exp_high = fast_exp_f32x4(vsubq_f32(x_high, max_broadcast)); - - float16x8_t exp_f16 = vcombine_f16(vcvt_f16_f32(exp_low), vcvt_f16_f32(exp_high)); - vst1q_f16(&output[i + u * SIMD_WIDTH], exp_f16); - - sum_vec[u * 2] = vaddq_f32(sum_vec[u * 2], exp_low); - sum_vec[u * 2 + 1] = vaddq_f32(sum_vec[u * 2 + 1], exp_high); - } - } - - float32x4_t final_sum = sum_vec[0]; - for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { - final_sum = vaddq_f32(final_sum, sum_vec[u]); - } - - float sum = vaddvq_f32(final_sum); - for (size_t i = vocab_vectorized; i < vocab_size; ++i) { - float exp_val = expf(static_cast(input[i]) - max_val); - output[i] = static_cast<__fp16>(exp_val); - sum += exp_val; - } - - const float inv_sum = 1.0f / sum; - const float16x8_t inv_sum_vec_f16 = vdupq_n_f16(static_cast<__fp16>(inv_sum)); - - for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t exp_vec = vld1q_f16(&output[i + u * SIMD_WIDTH]); - float16x8_t result = vmulq_f16(exp_vec, inv_sum_vec_f16); - vst1q_f16(&output[i + u * SIMD_WIDTH], result); - } - } - - for (size_t i = vocab_vectorized; i < vocab_size; ++i) { - output[i] = static_cast<__fp16>(static_cast(output[i]) * inv_sum); - } -} - -void cactus_softmax_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t vocab_size -) { - CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - for (size_t idx = start_idx; idx < end_idx; ++idx) { - const size_t offset = idx * vocab_size; - kernel_softmax_f16_single(input + offset, output + offset, vocab_size); - } - }); -} - -void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, size_t top_k, size_t random_seed, - const float* bias_values, const uint32_t* bias_indices, - size_t bias_count) { - cactus_sample_f32_ex(logits, output, vocab_size, - temperature, top_p, 0.15f, 1.1f, - top_k, random_seed, - bias_values, bias_indices, bias_count); -} - -void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, size_t random_seed, - const float* bias_values, const uint32_t* bias_indices, - size_t bias_count) { - if (vocab_size == 0) { - output[0] = 0; - return; - } - - const bool has_bias = bias_values && bias_indices && bias_count > 0; - - std::vector filtered_logits(vocab_size); - std::memcpy(filtered_logits.data(), logits, vocab_size * sizeof(float)); - - if (has_bias) { - for (size_t i = 0; i < bias_count; ++i) { - uint32_t idx = bias_indices[i]; - if (idx < vocab_size) { - filtered_logits[idx] += bias_values[i]; - } - } - } - - if (temperature == 0.0f) { - auto it = std::max_element(filtered_logits.begin(), filtered_logits.end()); - output[0] = static_cast(std::distance(filtered_logits.begin(), it)); - return; - } - - if (temperature > 0) { - for (size_t i = 0; i < vocab_size; ++i) { - filtered_logits[i] /= temperature; - } - } - - (void)repetition_penalty; - - if (top_k > 0) { - std::vector> logit_pairs; - logit_pairs.reserve(vocab_size); - for (size_t i = 0; i < vocab_size; ++i) { - logit_pairs.emplace_back(filtered_logits[i], i); - } - std::sort(logit_pairs.begin(), logit_pairs.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - if (top_k < vocab_size) { - float kth_value = logit_pairs[top_k - 1].first; - for (size_t i = 0; i < vocab_size; ++i) { - if (filtered_logits[i] < kth_value) { - filtered_logits[i] = -std::numeric_limits::infinity(); - } - } - } - } - - if (min_p > 0.0f) { - float max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); - if (!std::isinf(max_logit)) { - std::vector temp_probs(vocab_size); - float sum = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - if (!std::isinf(filtered_logits[i])) { - temp_probs[i] = std::exp(filtered_logits[i] - max_logit); - sum += temp_probs[i]; - } else { - temp_probs[i] = 0.0f; - } - } - - if (sum > 0.0f) { - for (size_t i = 0; i < vocab_size; ++i) { - temp_probs[i] /= sum; - } - - float max_prob = *std::max_element(temp_probs.begin(), temp_probs.end()); - float threshold = max_prob * min_p; - - for (size_t i = 0; i < vocab_size; ++i) { - if (temp_probs[i] < threshold) { - filtered_logits[i] = -std::numeric_limits::infinity(); - } - } - } - } - } - - if (top_p > 0.0f && top_p < 1.0f) { - std::vector> sorted_logits; - sorted_logits.reserve(vocab_size); - for (size_t i = 0; i < vocab_size; ++i) { - if (!std::isinf(filtered_logits[i])) { - sorted_logits.emplace_back(filtered_logits[i], i); - } - } - std::sort(sorted_logits.begin(), sorted_logits.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - float max_logit = sorted_logits.empty() ? 0.0f : sorted_logits[0].first; - std::vector temp_probs; - temp_probs.reserve(sorted_logits.size()); - float sum = 0.0f; - for (const auto& pair : sorted_logits) { - float prob = std::exp(pair.first - max_logit); - temp_probs.push_back(prob); - sum += prob; - } - - for (float& prob : temp_probs) { - prob /= sum; - } - - float cumulative_prob = 0.0f; - std::vector indices_to_remove(sorted_logits.size(), false); - for (size_t i = 0; i < sorted_logits.size(); ++i) { - cumulative_prob += temp_probs[i]; - if (cumulative_prob > top_p) { - indices_to_remove[i] = true; - } - } - - if (!indices_to_remove.empty()) { - for (size_t i = 1; i < indices_to_remove.size(); ++i) { - indices_to_remove[i] = indices_to_remove[i-1] || indices_to_remove[i]; - } - indices_to_remove[0] = false; - } - - for (size_t i = 0; i < sorted_logits.size(); ++i) { - if (indices_to_remove[i]) { - filtered_logits[sorted_logits[i].second] = -std::numeric_limits::infinity(); - } - } - } - - float max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); - if (std::isinf(max_logit)) { - output[0] = 0; - return; - } - - std::vector probs(vocab_size); - float sum = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - if (std::isinf(filtered_logits[i])) { - probs[i] = 0.0f; - } else { - probs[i] = std::exp(filtered_logits[i] - max_logit); - sum += probs[i]; - } - } - - if (sum == 0.0f) { - output[0] = 0; - return; - } - - for (size_t i = 0; i < vocab_size; ++i) { - probs[i] /= sum; - } - - uint32_t actual_seed = (random_seed == 0) ? std::random_device{}() : random_seed; - std::mt19937 gen(actual_seed); - std::uniform_real_distribution dist(0.0f, 1.0f); - float sample = dist(gen); - - float cumulative = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - cumulative += probs[i]; - if (cumulative >= sample) { - output[0] = static_cast(i); - return; - } - } - - for (size_t i = vocab_size; i > 0; --i) { - if (probs[i-1] > 0.0f) { - output[0] = static_cast(i-1); - return; - } - } - - output[0] = 0; -} - -void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, size_t top_k, size_t random_seed, - const float* bias_values, const uint32_t* bias_indices, - size_t bias_count) { - cactus_sample_f16_ex(logits, output, vocab_size, - temperature, top_p, 0.15f, 1.1f, - top_k, random_seed, - bias_values, bias_indices, bias_count); -} - -void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, size_t random_seed, - const float* bias_values, const uint32_t* bias_indices, - size_t bias_count) { - if (vocab_size == 0) { - output[0] = 0; - return; - } - - const bool has_bias = bias_values && bias_indices && bias_count > 0; - - std::vector<__fp16> filtered_logits(vocab_size); - std::memcpy(filtered_logits.data(), logits, vocab_size * sizeof(__fp16)); - - if (has_bias) { - for (size_t i = 0; i < bias_count; ++i) { - uint32_t idx = bias_indices[i]; - if (idx < vocab_size) { - filtered_logits[idx] = static_cast<__fp16>(static_cast(filtered_logits[idx]) + bias_values[i]); - } - } - } - - if (temperature == 0.0f) { - auto it = std::max_element(filtered_logits.begin(), filtered_logits.end()); - output[0] = static_cast(std::distance(filtered_logits.begin(), it)); - return; - } - - if (temperature > 0) { - __fp16 inv_temp = static_cast<__fp16>(1.0f / temperature); - float16x8_t inv_temp_vec = vdupq_n_f16(inv_temp); - size_t i = 0; - for (; i + 8 <= vocab_size; i += 8) { - float16x8_t logits_vec = vld1q_f16(&filtered_logits[i]); - float16x8_t scaled = vmulq_f16(logits_vec, inv_temp_vec); - vst1q_f16(&filtered_logits[i], scaled); - } - for (; i < vocab_size; ++i) { - filtered_logits[i] = filtered_logits[i] * inv_temp; - } - } - - (void)repetition_penalty; - - if (top_k > 0) { - std::vector> logit_pairs; - logit_pairs.reserve(vocab_size); - for (size_t i = 0; i < vocab_size; ++i) { - logit_pairs.emplace_back(filtered_logits[i], i); - } - std::partial_sort(logit_pairs.begin(), - logit_pairs.begin() + std::min(top_k, vocab_size), - logit_pairs.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - if (top_k < vocab_size) { - __fp16 kth_value = logit_pairs[top_k - 1].first; - __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); - for (size_t i = 0; i < vocab_size; ++i) { - if (filtered_logits[i] < kth_value) { - filtered_logits[i] = neg_inf; - } - } - } - } - - if (min_p > 0.0f) { - float max_logit = -std::numeric_limits::infinity(); - for (size_t i = 0; i < vocab_size; ++i) { - max_logit = std::max(max_logit, static_cast(filtered_logits[i])); - } - if (!std::isinf(max_logit)) { - std::vector temp_probs(vocab_size); - float sum = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - float v = static_cast(filtered_logits[i]); - if (!std::isinf(v)) { - temp_probs[i] = std::exp(v - max_logit); - sum += temp_probs[i]; - } else { - temp_probs[i] = 0.0f; - } - } - if (sum > 0.0f) { - for (float& p : temp_probs) p /= sum; - float max_prob = *std::max_element(temp_probs.begin(), temp_probs.end()); - float threshold = max_prob * min_p; - __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); - for (size_t i = 0; i < vocab_size; ++i) { - if (temp_probs[i] < threshold) { - filtered_logits[i] = neg_inf; - } - } - } - } - } - - if (top_p > 0.0f && top_p < 1.0f) { - std::vector> sorted_logits; - sorted_logits.reserve(vocab_size); - __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); - for (size_t i = 0; i < vocab_size; ++i) { - if (filtered_logits[i] != neg_inf) { - sorted_logits.emplace_back(filtered_logits[i], i); - } - } - std::sort(sorted_logits.begin(), sorted_logits.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - __fp16 max_logit = sorted_logits.empty() ? static_cast<__fp16>(0.0f) : sorted_logits[0].first; - std::vector temp_probs; - temp_probs.reserve(sorted_logits.size()); - float sum = 0.0f; - for (const auto& pair : sorted_logits) { - float prob = std::exp(static_cast(pair.first - max_logit)); - temp_probs.push_back(prob); - sum += prob; - } - - for (float& prob : temp_probs) { - prob /= sum; - } - - float cumulative_prob = 0.0f; - std::vector indices_to_remove(sorted_logits.size(), false); - bool threshold_reached = false; - for (size_t i = 0; i < sorted_logits.size(); ++i) { - cumulative_prob += temp_probs[i]; - if (cumulative_prob > top_p && i > 0) { - threshold_reached = true; - } - if (threshold_reached) { - indices_to_remove[i] = true; - } - } - - if (!indices_to_remove.empty()) { - for (size_t i = 1; i < indices_to_remove.size(); ++i) { - indices_to_remove[i] = indices_to_remove[i-1] || indices_to_remove[i]; - } - indices_to_remove[0] = false; - } - - for (size_t i = 0; i < sorted_logits.size(); ++i) { - if (indices_to_remove[i]) { - filtered_logits[sorted_logits[i].second] = neg_inf; - } - } - } - - __fp16 max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); - __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); - if (max_logit == neg_inf) { - output[0] = 0; - return; - } - - std::vector probs(vocab_size); - float sum = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - if (filtered_logits[i] == neg_inf) { - probs[i] = 0.0f; - } else { - probs[i] = std::exp(static_cast(filtered_logits[i] - max_logit)); - sum += probs[i]; - } - } - - if (sum == 0.0f) { - output[0] = 0; - return; - } - - for (size_t i = 0; i < vocab_size; ++i) { - probs[i] /= sum; - } - - uint32_t actual_seed = (random_seed == 0) ? std::random_device{}() : random_seed; - std::mt19937 gen(actual_seed); - std::uniform_real_distribution dist(0.0f, 1.0f); - float sample = dist(gen); - - float cumulative = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - cumulative += probs[i]; - if (cumulative >= sample) { - output[0] = static_cast(i); - return; - } - } - - for (size_t i = vocab_size; i > 0; --i) { - if (probs[i-1] > 0.0f) { - output[0] = static_cast(i-1); - return; - } - } - - output[0] = 0; -} diff --git a/cactus-transpile/kernel/kernel_quants.cpp b/cactus-transpile/kernel/kernel_quants.cpp deleted file mode 100644 index d3553603b..000000000 --- a/cactus-transpile/kernel/kernel_quants.cpp +++ /dev/null @@ -1,312 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include - -void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale) { - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scale](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 16) * 16; - float32x4_t scale_vec = vdupq_n_f32(scale); - - for (size_t i = start; i < simd_end; i += 16) { - int8x16_t input = vld1q_s8(&src[i]); - - int16x8_t low = vmovl_s8(vget_low_s8(input)); - int16x8_t high = vmovl_s8(vget_high_s8(input)); - - int32x4_t low_low = vmovl_s16(vget_low_s16(low)); - int32x4_t low_high = vmovl_s16(vget_high_s16(low)); - int32x4_t high_low = vmovl_s16(vget_low_s16(high)); - int32x4_t high_high = vmovl_s16(vget_high_s16(high)); - - float32x4_t f_low_low = vcvtq_f32_s32(low_low); - float32x4_t f_low_high = vcvtq_f32_s32(low_high); - float32x4_t f_high_low = vcvtq_f32_s32(high_low); - float32x4_t f_high_high = vcvtq_f32_s32(high_high); - - vst1q_f32(&dst[i], vmulq_f32(f_low_low, scale_vec)); - vst1q_f32(&dst[i + 4], vmulq_f32(f_low_high, scale_vec)); - vst1q_f32(&dst[i + 8], vmulq_f32(f_high_low, scale_vec)); - vst1q_f32(&dst[i + 12], vmulq_f32(f_high_high, scale_vec)); - } - - for (size_t i = simd_end; i < end; ++i) { - dst[i] = static_cast(src[i]) * scale; - } - }); -} - -void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale) { - const float inv_scale = 1.0f / scale; - - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, inv_scale](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 16) * 16; - float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); - float32x4_t min_vec = vdupq_n_f32(-128.0f); - float32x4_t max_vec = vdupq_n_f32(127.0f); - - for (size_t i = start; i < simd_end; i += 16) { - float32x4_t input_0 = vld1q_f32(&src[i]); - float32x4_t input_1 = vld1q_f32(&src[i + 4]); - float32x4_t input_2 = vld1q_f32(&src[i + 8]); - float32x4_t input_3 = vld1q_f32(&src[i + 12]); - - float32x4_t scaled_0 = vmulq_f32(input_0, inv_scale_vec); - float32x4_t scaled_1 = vmulq_f32(input_1, inv_scale_vec); - float32x4_t scaled_2 = vmulq_f32(input_2, inv_scale_vec); - float32x4_t scaled_3 = vmulq_f32(input_3, inv_scale_vec); - - scaled_0 = vmaxq_f32(vminq_f32(scaled_0, max_vec), min_vec); - scaled_1 = vmaxq_f32(vminq_f32(scaled_1, max_vec), min_vec); - scaled_2 = vmaxq_f32(vminq_f32(scaled_2, max_vec), min_vec); - scaled_3 = vmaxq_f32(vminq_f32(scaled_3, max_vec), min_vec); - - int32x4_t int_0 = vcvtnq_s32_f32(scaled_0); - int32x4_t int_1 = vcvtnq_s32_f32(scaled_1); - int32x4_t int_2 = vcvtnq_s32_f32(scaled_2); - int32x4_t int_3 = vcvtnq_s32_f32(scaled_3); - - int16x8_t int16_low = vcombine_s16(vqmovn_s32(int_0), vqmovn_s32(int_1)); - int16x8_t int16_high = vcombine_s16(vqmovn_s32(int_2), vqmovn_s32(int_3)); - - int8x16_t result = vcombine_s8(vqmovn_s16(int16_low), vqmovn_s16(int16_high)); - vst1q_s8(&dst[i], result); - } - - for (size_t i = simd_end; i < end; ++i) { - float quantized = src[i] * inv_scale; - dst[i] = static_cast(std::round(std::max(-128.0f, std::min(127.0f, quantized)))); - } - }); -} - -void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count) { - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 8) * 8; - - for (size_t i = start; i < simd_end; i += 8) { - float16x8_t input = vld1q_f16(&src[i]); - - float32x4_t output_low = vcvt_f32_f16(vget_low_f16(input)); - float32x4_t output_high = vcvt_f32_f16(vget_high_f16(input)); - - vst1q_f32(&dst[i], output_low); - vst1q_f32(&dst[i + 4], output_high); - } - - for (size_t i = simd_end; i < end; ++i) { - dst[i] = static_cast(src[i]); - } - }); -} - -void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count) { - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 8) * 8; - - for (size_t i = start; i < simd_end; i += 8) { - float32x4_t input_low = vld1q_f32(&src[i]); - float32x4_t input_high = vld1q_f32(&src[i + 4]); - - float16x4_t output_low = vcvt_f16_f32(input_low); - float16x4_t output_high = vcvt_f16_f32(input_high); - - float16x8_t output = vcombine_f16(output_low, output_high); - vst1q_f16(&dst[i], output); - } - - for (size_t i = simd_end; i < end; ++i) { - dst[i] = static_cast<__fp16>(src[i]); - } - }); -} - -void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale) { - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scale](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 8) * 8; - float32x4_t scale_vec = vdupq_n_f32(scale); - - for (size_t i = start; i < simd_end; i += 8) { - int8x8_t input = vld1_s8(&src[i]); - - int16x8_t int16 = vmovl_s8(input); - int32x4_t int32_low = vmovl_s16(vget_low_s16(int16)); - int32x4_t int32_high = vmovl_s16(vget_high_s16(int16)); - - float32x4_t float_low = vcvtq_f32_s32(int32_low); - float32x4_t float_high = vcvtq_f32_s32(int32_high); - - float_low = vmulq_f32(float_low, scale_vec); - float_high = vmulq_f32(float_high, scale_vec); - - float16x8_t output = vcombine_f16(vcvt_f16_f32(float_low), vcvt_f16_f32(float_high)); - vst1q_f16(&dst[i], output); - } - - for (size_t i = simd_end; i < end; ++i) { - dst[i] = static_cast<__fp16>(static_cast(src[i]) * scale); - } - }); -} - -void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale) { - const float inv_scale = 1.0f / scale; - - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, inv_scale](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 8) * 8; - float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); - float32x4_t min_vec = vdupq_n_f32(-128.0f); - float32x4_t max_vec = vdupq_n_f32(127.0f); - - for (size_t i = start; i < simd_end; i += 8) { - float16x8_t input = vld1q_f16(&src[i]); - - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input)); - - float32x4_t scaled_low = vmulq_f32(input_low, inv_scale_vec); - float32x4_t scaled_high = vmulq_f32(input_high, inv_scale_vec); - - scaled_low = vmaxq_f32(vminq_f32(scaled_low, max_vec), min_vec); - scaled_high = vmaxq_f32(vminq_f32(scaled_high, max_vec), min_vec); - - int32x4_t int_low = vcvtnq_s32_f32(scaled_low); - int32x4_t int_high = vcvtnq_s32_f32(scaled_high); - - int16x8_t int16_combined = vcombine_s16(vqmovn_s32(int_low), vqmovn_s32(int_high)); - int8x8_t result = vqmovn_s16(int16_combined); - - vst1_s8(&dst[i], result); - } - - for (size_t i = simd_end; i < end; ++i) { - float quantized = static_cast(src[i]) * inv_scale; - dst[i] = static_cast(std::round(std::max(-128.0f, std::min(127.0f, quantized)))); - } - }); -} - -float cactus_fp16_max_abs(const __fp16* src, size_t count) { - float32x4_t abs_max_vec = vdupq_n_f32(0.0f); - const size_t simd_end = (count / 8) * 8; - - for (size_t i = 0; i < simd_end; i += 8) { - float16x8_t input = vld1q_f16(&src[i]); - - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input)); - - float32x4_t abs_low = vabsq_f32(input_low); - float32x4_t abs_high = vabsq_f32(input_high); - - abs_max_vec = vmaxq_f32(abs_max_vec, abs_low); - abs_max_vec = vmaxq_f32(abs_max_vec, abs_high); - } - - float max_abs = vmaxvq_f32(abs_max_vec); - - for (size_t i = simd_end; i < count; ++i) { - float abs_val = std::abs(static_cast(src[i])); - max_abs = std::max(max_abs, abs_val); - } - - return max_abs; -} - -static inline float quantize_group_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count) { - - float32x4_t max_vec = vdupq_n_f32(0.0f); - size_t k = 0; - - for (; k + 8 <= count; k += 8) { - float16x8_t vals = vld1q_f16(src + k); - max_vec = vmaxq_f32(max_vec, vabsq_f32(vcvt_f32_f16(vget_low_f16(vals)))); - max_vec = vmaxq_f32(max_vec, vabsq_f32(vcvt_f32_f16(vget_high_f16(vals)))); - } - - float max_abs = vmaxvq_f32(max_vec); - for (; k < count; k++) { - float val = std::abs(static_cast(src[k])); - if (val > max_abs) max_abs = val; - } - - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - float inv_scale = 1.0f / scale; - float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); - - k = 0; - for (; k + 8 <= count; k += 8) { - float16x8_t vals = vld1q_f16(src + k); - int32x4_t i0 = vcvtnq_s32_f32(vmulq_f32(vcvt_f32_f16(vget_low_f16(vals)), inv_scale_vec)); - int32x4_t i1 = vcvtnq_s32_f32(vmulq_f32(vcvt_f32_f16(vget_high_f16(vals)), inv_scale_vec)); - int16x4_t s0 = vqmovn_s32(i0); - int16x4_t s1 = vqmovn_s32(i1); - int8x8_t result = vqmovn_s16(vcombine_s16(s0, s1)); - vst1_s8(dst + k, result); - } - - for (; k < count; k++) { - float val = static_cast(src[k]) * inv_scale; - int32_t q = static_cast(roundf(val)); - q = std::max(-128, std::min(127, q)); - dst[k] = static_cast(q); - } - - return scale; -} - -void cactus_quantize_kv_fp16_to_int8( - const __fp16* src, - int8_t* dst, - float* scales, - size_t seq_len, size_t kv_heads, size_t head_dim, - size_t group_size -) { - const size_t num_groups = (head_dim + group_size - 1) / group_size; - - CactusThreading::parallel_for(seq_len * kv_heads, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scales, head_dim, group_size, num_groups](size_t start, size_t end) { - for (size_t idx = start; idx < end; idx++) { - const __fp16* head_src = src + idx * head_dim; - int8_t* head_dst = dst + idx * head_dim; - float* head_scales = scales + idx * num_groups; - - for (size_t g = 0; g < num_groups; g++) { - size_t group_start = g * group_size; - size_t group_count = std::min(group_size, head_dim - group_start); - head_scales[g] = quantize_group_fp16_to_int8( - head_src + group_start, - head_dst + group_start, - group_count - ); - } - } - }); -} - -void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count) { - assert (unpacked_count % 32 == 0 && "Unpacked count must be a multiple of 32 for int4 unpacking"); - const size_t num_groups = unpacked_count / 32; - - CactusThreading::parallel_for(num_groups, CactusThreading::Thresholds::ELEMENT_WISE, - [packed, unpacked](size_t start, size_t end) { - for (size_t group = start; group < end; ++group) { - const uint8_t* src = &packed[group * 16]; - int8_t* dst = &unpacked[group * 32]; - - int8x16_t high_signed, low_signed; - unpack_int4_as_int8x16x2(src, high_signed, low_signed); - vst1q_s8(dst, low_signed); - vst1q_s8(dst + 16, high_signed); - } - }); -} diff --git a/cactus-transpile/kernel/kernel_reduce.cpp b/cactus-transpile/kernel/kernel_reduce.cpp deleted file mode 100644 index 5cea15cbf..000000000 --- a/cactus-transpile/kernel/kernel_reduce.cpp +++ /dev/null @@ -1,430 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include - -template -static void axis_reduce_f32_impl(const __fp16* input, __fp16* output, - size_t outer_size, size_t axis_size, size_t inner_size, - FinalizeFn finalize) { - CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - constexpr size_t W = SIMD_F16_WIDTH; - const size_t vec_axis = simd_align(axis_size); - - float32x4_t sum_lo = vdupq_n_f32(0.0f); - float32x4_t sum_hi = vdupq_n_f32(0.0f); - - for (size_t a = 0; a < vec_axis; a += W) { - __fp16 values[W]; - for (size_t j = 0; j < W; j++) { - values[j] = input[outer * axis_size * inner_size + (a + j) * inner_size + inner]; - } - float32x4_t lo, hi; - f16x8_split_f32(vld1q_f16(values), lo, hi); - sum_lo = vaddq_f32(sum_lo, lo); - sum_hi = vaddq_f32(sum_hi, hi); - } - - float total = vaddvq_f32(vaddq_f32(sum_lo, sum_hi)); - - for (size_t a = vec_axis; a < axis_size; a++) { - total += static_cast(input[outer * axis_size * inner_size + a * inner_size + inner]); - } - - output[outer * inner_size + inner] = static_cast<__fp16>(finalize(total, axis_size)); - }); -} - -template -static void axis_reduce_f32_impl(const float* input, float* output, - size_t outer_size, size_t axis_size, size_t inner_size, - FinalizeFn finalize) { - CactusThreading::parallel_for_2d( - outer_size, - inner_size, - CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - float total = 0.0f; - for (size_t a = 0; a < axis_size; ++a) { - total += input[outer * axis_size * inner_size + a * inner_size + inner]; - } - output[outer * inner_size + inner] = finalize(total, axis_size); - } - ); -} - -template -static void axis_reduce_f16_impl(const __fp16* input, __fp16* output, - size_t outer_size, size_t axis_size, size_t inner_size, - float16x8_t init_vec, __fp16 init_scalar, - SimdReduceOp simd_reduce, ScalarReduceOp scalar_reduce) { - CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - constexpr size_t W = SIMD_F16_WIDTH; - const size_t vec_axis = simd_align(axis_size); - float16x8_t acc = init_vec; - - for (size_t a = 0; a < vec_axis; a += W) { - __fp16 values[W]; - for (size_t j = 0; j < W; j++) { - values[j] = input[outer * axis_size * inner_size + (a + j) * inner_size + inner]; - } - acc = simd_reduce(acc, vld1q_f16(values)); - } - - __fp16 result = init_scalar; - __fp16 arr[8]; - vst1q_f16(arr, acc); - for (int j = 0; j < 8; j++) { - result = scalar_reduce(result, arr[j]); - } - - for (size_t a = vec_axis; a < axis_size; a++) { - result = scalar_reduce(result, input[outer * axis_size * inner_size + a * inner_size + inner]); - } - - output[outer * inner_size + inner] = result; - }); -} - -template -static void axis_reduce_f32_impl(const float* input, float* output, - size_t outer_size, size_t axis_size, size_t inner_size, - float init_scalar, - ScalarReduceOp scalar_reduce) { - CactusThreading::parallel_for_2d( - outer_size, - inner_size, - CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - float result = init_scalar; - for (size_t a = 0; a < axis_size; ++a) { - result = scalar_reduce(result, input[outer * axis_size * inner_size + a * inner_size + inner]); - } - output[outer * inner_size + inner] = result; - } - ); -} - -double cactus_sum_all_f16(const __fp16* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> double { - const size_t vec_end = start + simd_align(end - start); - float32x4_t sum_lo = vdupq_n_f32(0.0f); - float32x4_t sum_hi = vdupq_n_f32(0.0f); - - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - float32x4_t lo, hi; - f16x8_split_f32(vld1q_f16(&data[i]), lo, hi); - sum_lo = vaddq_f32(sum_lo, lo); - sum_hi = vaddq_f32(sum_hi, hi); - } - - double s = static_cast(vaddvq_f32(vaddq_f32(sum_lo, sum_hi))); - for (size_t i = vec_end; i < end; ++i) s += static_cast(data[i]); - return s; - }, - 0.0, [](double a, double b) { return a + b; } - ); -} - -void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, - [](float total, size_t) { return total; }); -} - -double cactus_mean_all_f16(const __fp16* data, size_t num_elements) { - return cactus_sum_all_f16(data, num_elements) / static_cast(num_elements); -} - -void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, - [](float total, size_t axis_size) { return total / static_cast(axis_size); }); -} - -double cactus_sum_all_f32(const float* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, - CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> double { - double sum = 0.0; - for (size_t i = start; i < end; ++i) { - sum += static_cast(data[i]); - } - return sum; - }, - 0.0, - [](double a, double b) { return a + b; } - ); -} - -void cactus_sum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, - [](float total, size_t) { return total; }); -} - -double cactus_mean_all_f32(const float* data, size_t num_elements) { - return cactus_sum_all_f32(data, num_elements) / static_cast(num_elements); -} - -void cactus_mean_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, - [](float total, size_t axis_size) { return total / static_cast(axis_size); }); -} - -struct VarianceState { - double sum; - double sum_sq; - VarianceState() : sum(0.0), sum_sq(0.0) {} - VarianceState(double s, double sq) : sum(s), sum_sq(sq) {} -}; - -double cactus_variance_all_f16(const __fp16* data, size_t num_elements) { - VarianceState result = CactusThreading::parallel_reduce( - num_elements, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> VarianceState { - const size_t vec_end = start + simd_align(end - start); - - float32x4_t sum_lo = vdupq_n_f32(0.0f), sum_hi = vdupq_n_f32(0.0f); - float32x4_t sq_lo = vdupq_n_f32(0.0f), sq_hi = vdupq_n_f32(0.0f); - - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - float32x4_t lo, hi; - f16x8_split_f32(vld1q_f16(&data[i]), lo, hi); - sum_lo = vaddq_f32(sum_lo, lo); - sum_hi = vaddq_f32(sum_hi, hi); - sq_lo = vfmaq_f32(sq_lo, lo, lo); - sq_hi = vfmaq_f32(sq_hi, hi, hi); - } - - double sum = static_cast(vaddvq_f32(vaddq_f32(sum_lo, sum_hi))); - double sum_sq = static_cast(vaddvq_f32(vaddq_f32(sq_lo, sq_hi))); - - for (size_t i = vec_end; i < end; ++i) { - double x = static_cast(data[i]); - sum += x; - sum_sq += x * x; - } - return VarianceState(sum, sum_sq); - }, - VarianceState(), - [](const VarianceState& a, const VarianceState& b) { - return VarianceState(a.sum + b.sum, a.sum_sq + b.sum_sq); - } - ); - - double mean = result.sum / static_cast(num_elements); - double mean_sq = result.sum_sq / static_cast(num_elements); - return mean_sq - mean * mean; -} - -void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - float sum = 0.0f, sum_sq = 0.0f; - for (size_t a = 0; a < axis_size; a++) { - float x = static_cast(input[outer * axis_size * inner_size + a * inner_size + inner]); - sum += x; - sum_sq += x * x; - } - float mean = sum / static_cast(axis_size); - float mean_sq = sum_sq / static_cast(axis_size); - output[outer * inner_size + inner] = static_cast<__fp16>(mean_sq - mean * mean); - }); -} - -double cactus_variance_all_f32(const float* data, size_t num_elements) { - VarianceState result = CactusThreading::parallel_reduce( - num_elements, - CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> VarianceState { - double sum = 0.0; - double sum_sq = 0.0; - for (size_t i = start; i < end; ++i) { - double x = static_cast(data[i]); - sum += x; - sum_sq += x * x; - } - return VarianceState(sum, sum_sq); - }, - VarianceState(), - [](const VarianceState& a, const VarianceState& b) { - return VarianceState(a.sum + b.sum, a.sum_sq + b.sum_sq); - } - ); - - double mean = result.sum / static_cast(num_elements); - double mean_sq = result.sum_sq / static_cast(num_elements); - return mean_sq - mean * mean; -} - -void cactus_variance_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { - CactusThreading::parallel_for_2d( - outer_size, - inner_size, - CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - float sum = 0.0f; - float sum_sq = 0.0f; - for (size_t a = 0; a < axis_size; ++a) { - float x = input[outer * axis_size * inner_size + a * inner_size + inner]; - sum += x; - sum_sq += x * x; - } - float mean = sum / static_cast(axis_size); - float mean_sq = sum_sq / static_cast(axis_size); - output[outer * inner_size + inner] = mean_sq - mean * mean; - } - ); -} - -__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> __fp16 { - const size_t vec_end = start + simd_align(end - start); - float16x8_t acc = vdupq_n_f16(static_cast<__fp16>(65504.0f)); - - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - acc = vminq_f16(acc, vld1q_f16(&data[i])); - } - - __fp16 result = static_cast<__fp16>(65504.0f); - __fp16 arr[8]; - vst1q_f16(arr, acc); - for (int j = 0; j < 8; j++) result = std::min(result, arr[j]); - for (size_t i = vec_end; i < end; ++i) result = std::min(result, data[i]); - return result; - }, - static_cast<__fp16>(65504.0f), - [](__fp16 a, __fp16 b) { return std::min(a, b); } - ); -} - -void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f16_impl(input, output, outer_size, axis_size, inner_size, - vdupq_n_f16(static_cast<__fp16>(65504.0f)), static_cast<__fp16>(65504.0f), - [](float16x8_t a, float16x8_t b) { return vminq_f16(a, b); }, - [](__fp16 a, __fp16 b) { return std::min(a, b); }); -} - -float cactus_min_all_f32(const float* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, - CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> float { - float result = std::numeric_limits::infinity(); - for (size_t i = start; i < end; ++i) { - result = std::min(result, data[i]); - } - return result; - }, - std::numeric_limits::infinity(), - [](float a, float b) { return std::min(a, b); } - ); -} - -void cactus_min_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f32_impl( - input, - output, - outer_size, - axis_size, - inner_size, - std::numeric_limits::infinity(), - [](float a, float b) { return std::min(a, b); } - ); -} - -__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> __fp16 { - const size_t vec_end = start + simd_align(end - start); - float16x8_t acc = vdupq_n_f16(static_cast<__fp16>(-65504.0f)); - - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - acc = vmaxq_f16(acc, vld1q_f16(&data[i])); - } - - __fp16 result = static_cast<__fp16>(-65504.0f); - __fp16 arr[8]; - vst1q_f16(arr, acc); - for (int j = 0; j < 8; j++) result = std::max(result, arr[j]); - for (size_t i = vec_end; i < end; ++i) result = std::max(result, data[i]); - return result; - }, - static_cast<__fp16>(-65504.0f), - [](__fp16 a, __fp16 b) { return std::max(a, b); } - ); -} - -void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f16_impl(input, output, outer_size, axis_size, inner_size, - vdupq_n_f16(static_cast<__fp16>(-65504.0f)), static_cast<__fp16>(-65504.0f), - [](float16x8_t a, float16x8_t b) { return vmaxq_f16(a, b); }, - [](__fp16 a, __fp16 b) { return std::max(a, b); }); -} - -float cactus_max_all_f32(const float* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, - CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> float { - float result = -std::numeric_limits::infinity(); - for (size_t i = start; i < end; ++i) { - result = std::max(result, data[i]); - } - return result; - }, - -std::numeric_limits::infinity(), - [](float a, float b) { return std::max(a, b); } - ); -} - -void cactus_max_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f32_impl( - input, - output, - outer_size, - axis_size, - inner_size, - -std::numeric_limits::infinity(), - [](float a, float b) { return std::max(a, b); } - ); -} - -void cactus_cumsum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - CactusThreading::parallel_for_2d( - outer_size, - inner_size, - CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - float running = 0.0f; - for (size_t a = 0; a < axis_size; ++a) { - const size_t offset = outer * axis_size * inner_size + a * inner_size + inner; - running += static_cast(input[offset]); - output[offset] = static_cast<__fp16>(running); - } - } - ); -} - -void cactus_cumsum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size) { - CactusThreading::parallel_for_2d( - outer_size, - inner_size, - CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - float running = 0.0f; - for (size_t a = 0; a < axis_size; ++a) { - const size_t offset = outer * axis_size * inner_size + a * inner_size + inner; - running += input[offset]; - output[offset] = running; - } - } - ); -} diff --git a/cactus-transpile/kernel/kernel_scalar.cpp b/cactus-transpile/kernel/kernel_scalar.cpp deleted file mode 100644 index 15e9f6dea..000000000 --- a/cactus-transpile/kernel/kernel_scalar.cpp +++ /dev/null @@ -1,119 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include - -void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type) { - const __fp16 scalar_f16 = static_cast<__fp16>(scalar_value); - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - - switch (op_type) { - case ScalarOpType::ADD: { - const float16x8_t sv = vdupq_n_f16(scalar_f16); - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [sv](float16x8_t v) { return vaddq_f16(v, sv); }, - [scalar_f16](__fp16 v) { return static_cast<__fp16>(v + scalar_f16); }); - break; - } - case ScalarOpType::SUBTRACT: { - const float16x8_t sv = vdupq_n_f16(scalar_f16); - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [sv](float16x8_t v) { return vsubq_f16(v, sv); }, - [scalar_f16](__fp16 v) { return static_cast<__fp16>(v - scalar_f16); }); - break; - } - case ScalarOpType::MULTIPLY: { - const float16x8_t sv = vdupq_n_f16(scalar_f16); - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [sv](float16x8_t v) { return vmulq_f16(v, sv); }, - [scalar_f16](__fp16 v) { return static_cast<__fp16>(v * scalar_f16); }); - break; - } - case ScalarOpType::DIVIDE: { - const float16x8_t sv = vdupq_n_f16(scalar_f16); - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [sv](float16x8_t v) { return vdivq_f16(v, sv); }, - [scalar_f16](__fp16 v) { return static_cast<__fp16>(v / scalar_f16); }); - break; - } - case ScalarOpType::NOT_EQUAL: - CactusThreading::parallel_for(num_elements, - CactusThreading::Thresholds::SCALAR_BASIC, - [&](size_t start, size_t end) { - for (size_t i = start; i < end; ++i) { - output[i] = static_cast<__fp16>( - static_cast(input[i]) != scalar_value ? 1.0f : 0.0f); - } - }); - break; - case ScalarOpType::ABS: - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [](float16x8_t v) { return vabsq_f16(v); }, - [](__fp16 v) { return static_cast<__fp16>(std::abs(static_cast(v))); }); - break; - - case ScalarOpType::EXP: - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [](float16x8_t v) { return apply_f32_op_on_f16x8(v, fast_exp_f32x4); }, - [](__fp16 v) { return static_cast<__fp16>(std::exp(static_cast(v))); }, 2); - break; - - case ScalarOpType::SQRT: - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [](float16x8_t v) { return apply_f32_op_on_f16x8(v, vsqrtq_f32); }, - [](__fp16 v) { return static_cast<__fp16>(std::sqrt(static_cast(v))); }, 2); - break; - - case ScalarOpType::POW: - CactusThreading::parallel_for(num_elements, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start, size_t end) { - for (size_t i = start; i < end; ++i) { - output[i] = static_cast<__fp16>( - std::pow(static_cast(input[i]), static_cast(scalar_f16))); - } - }); - break; - - case ScalarOpType::COS: - case ScalarOpType::SIN: - CactusThreading::parallel_for(num_elements, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start, size_t end) { - for (size_t i = start; i < end; ++i) { - float val = static_cast(input[i]); - float result = (op_type == ScalarOpType::COS) ? std::cos(val) : std::sin(val); - output[i] = static_cast<__fp16>(result); - } - }); - break; - - case ScalarOpType::LOG: - CactusThreading::parallel_for(num_elements, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start, size_t end) { - for (size_t i = start; i < end; ++i) { - output[i] = static_cast<__fp16>(std::log(static_cast(input[i]))); - } - }); - break; - - case ScalarOpType::FLOOR_DIVIDE: - CactusThreading::parallel_for(num_elements, - CactusThreading::Thresholds::SCALAR_BASIC, - [&](size_t start, size_t end) { - for (size_t i = start; i < end; ++i) { - output[i] = static_cast<__fp16>( - std::floor(static_cast(input[i]) / scalar_value)); - } - }); - break; - } -} diff --git a/cactus-transpile/kernel/kernel_sme2.cpp b/cactus-transpile/kernel/kernel_sme2.cpp deleted file mode 100644 index 9ce5b410b..000000000 --- a/cactus-transpile/kernel/kernel_sme2.cpp +++ /dev/null @@ -1,616 +0,0 @@ -#include -#include -#ifndef __ARM_FEATURE_SME2 -#error "kernel_sme2.cpp must be compiled with SME2 enabled (e.g. -march=armv9.2-a+sme2)" -#endif - -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include - -#if defined(__clang__) -#define CACTUS_UNROLL4 _Pragma("clang loop unroll_count(4)") -#else -#define CACTUS_UNROLL4 -#endif - -static inline void cactus_pack_a_f16_row_block( - const __fp16* __restrict a, - __fp16* __restrict a_packed, - size_t K, - size_t rb, - size_t row0, - size_t active_r, - size_t tile_rows, - size_t tile_pairs, - size_t k_pairs -) { - const size_t block_stride = k_pairs * tile_pairs; - const bool even_k = ((K & 1u) == 0); - for (size_t kp = 0; kp < k_pairs; ++kp) { - const size_t k0 = kp * 2; - const size_t k1 = k0 + 1; - __fp16* dst = a_packed + rb * block_stride + kp * tile_pairs; - const __fp16* src_col = a + row0 * K + k0; - - if (even_k) { - CACTUS_UNROLL4 - for (size_t r = 0; r < active_r; ++r) { - dst[2 * r] = src_col[0]; - dst[2 * r + 1] = src_col[1]; - src_col += K; - } - } else { - const bool has_k1 = (k1 < K); - CACTUS_UNROLL4 - for (size_t r = 0; r < active_r; ++r) { - dst[2 * r] = src_col[0]; - dst[2 * r + 1] = has_k1 ? src_col[1] : static_cast<__fp16>(0); - src_col += K; - } - } - CACTUS_UNROLL4 - for (size_t r = active_r; r < tile_rows; ++r) { - dst[2 * r] = static_cast<__fp16>(0); - dst[2 * r + 1] = static_cast<__fp16>(0); - } - } -} - -static void cactus_pack_b_f16_from_bt( - const __fp16* __restrict b_transposed, - __fp16* __restrict b_packed, - size_t K, - size_t N, - size_t tile_cols, - size_t tile_pairs -) { - const size_t k_pairs = (K + 1) / 2; - const size_t col_blocks = (N + tile_cols - 1) / tile_cols; - const size_t full_col_blocks = N / tile_cols; - const size_t cb4_tiles = (full_col_blocks / 4) * 4; - const size_t cb2_tiles = ((full_col_blocks - cb4_tiles) / 2) * 2; - const size_t cb1_tiles = col_blocks - cb4_tiles - cb2_tiles; - const size_t cb4_groups = cb4_tiles / 4; - const size_t cb2_groups = cb2_tiles / 2; - - const size_t off_cb4 = 0; - const size_t off_cb2 = off_cb4 + cb4_groups * k_pairs * 4 * tile_pairs; - const size_t off_cb1 = off_cb2 + cb2_groups * k_pairs * 2 * tile_pairs; - - const bool even_k = ((K & 1u) == 0); - - CactusThreading::parallel_for(cb4_groups * k_pairs, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [=](size_t start, size_t end) { - for (size_t idx = start; idx < end; ++idx) { - const size_t g4 = idx / k_pairs; - const size_t kp = idx % k_pairs; - const size_t col0 = g4 * 4 * tile_cols; - - const size_t k0 = kp * 2; - const size_t k1 = k0 + 1; - __fp16* dst = b_packed + off_cb4 + (g4 * k_pairs + kp) * (4 * tile_pairs); - - for (size_t t = 0; t < 4; ++t) { - __fp16* dst_t = dst + t * tile_pairs; - const size_t col_t = col0 + t * tile_cols; - const __fp16* src_col = b_transposed + col_t * K + k0; - if (even_k) { - CACTUS_UNROLL4 - for (size_t c = 0; c < tile_cols; ++c) { - dst_t[2 * c] = src_col[0]; - dst_t[2 * c + 1] = src_col[1]; - src_col += K; - } - } else { - const bool has_k1 = (k1 < K); - CACTUS_UNROLL4 - for (size_t c = 0; c < tile_cols; ++c) { - dst_t[2 * c] = src_col[0]; - dst_t[2 * c + 1] = has_k1 ? src_col[1] : static_cast<__fp16>(0); - src_col += K; - } - } - } - } - }); - - CactusThreading::parallel_for(cb2_groups * k_pairs, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [=](size_t start, size_t end) { - for (size_t idx = start; idx < end; ++idx) { - const size_t g2 = idx / k_pairs; - const size_t kp = idx % k_pairs; - const size_t col0 = cb4_tiles * tile_cols + g2 * 2 * tile_cols; - - const size_t k0 = kp * 2; - const size_t k1 = k0 + 1; - __fp16* dst = b_packed + off_cb2 + (g2 * k_pairs + kp) * (2 * tile_pairs); - - for (size_t t = 0; t < 2; ++t) { - __fp16* dst_t = dst + t * tile_pairs; - const size_t col_t = col0 + t * tile_cols; - const __fp16* src_col = b_transposed + col_t * K + k0; - if (even_k) { - CACTUS_UNROLL4 - for (size_t c = 0; c < tile_cols; ++c) { - dst_t[2 * c] = src_col[0]; - dst_t[2 * c + 1] = src_col[1]; - src_col += K; - } - } else { - const bool has_k1 = (k1 < K); - CACTUS_UNROLL4 - for (size_t c = 0; c < tile_cols; ++c) { - dst_t[2 * c] = src_col[0]; - dst_t[2 * c + 1] = has_k1 ? src_col[1] : static_cast<__fp16>(0); - src_col += K; - } - } - } - } - }); - - CactusThreading::parallel_for(cb1_tiles * k_pairs, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [=](size_t start, size_t end) { - for (size_t idx = start; idx < end; ++idx) { - const size_t g1 = idx / k_pairs; - const size_t kp = idx % k_pairs; - const size_t cb = cb4_tiles + cb2_tiles + g1; - const size_t col0 = cb * tile_cols; - const size_t active_c = (col0 < N) ? std::min(tile_cols, N - col0) : 0; - - const size_t k0 = kp * 2; - const size_t k1 = k0 + 1; - __fp16* dst = b_packed + off_cb1 + (g1 * k_pairs + kp) * tile_pairs; - const __fp16* src_col = b_transposed + col0 * K + k0; - if (even_k) { - CACTUS_UNROLL4 - for (size_t c = 0; c < active_c; ++c) { - dst[2 * c] = src_col[0]; - dst[2 * c + 1] = src_col[1]; - src_col += K; - } - } else { - const bool has_k1 = (k1 < K); - CACTUS_UNROLL4 - for (size_t c = 0; c < active_c; ++c) { - dst[2 * c] = src_col[0]; - dst[2 * c + 1] = has_k1 ? src_col[1] : static_cast<__fp16>(0); - src_col += K; - } - } - CACTUS_UNROLL4 - for (size_t c = active_c; c < tile_cols; ++c) { - dst[2 * c] = static_cast<__fp16>(0); - dst[2 * c + 1] = static_cast<__fp16>(0); - } - } - }); -} - -static void cactus_matmul_f16_sme2_worker( - const __fp16* a_packed, - const __fp16* b_packed, - __fp16* c, - size_t M, - size_t K, - size_t N, - size_t start_row, - size_t end_row, - size_t tile_rows, - size_t tile_pairs -) __arm_streaming __arm_inout("za") { - if (start_row >= end_row) return; - (void)M; - - const size_t k_pairs = (K + 1) / 2; - const size_t col_blocks = (N + tile_rows - 1) / tile_rows; - const size_t full_col_blocks = N / tile_rows; - const size_t cb4_tiles = (full_col_blocks / 4) * 4; - const size_t cb2_tiles = ((full_col_blocks - cb4_tiles) / 2) * 2; - const size_t cb4_groups = cb4_tiles / 4; - const size_t cb2_groups = cb2_tiles / 2; - const size_t cb1_off = cb4_groups * k_pairs * 4 * tile_pairs + cb2_groups * k_pairs * 2 * tile_pairs; - const size_t a_row_block_stride = k_pairs * tile_pairs; - - const svcount_t pNh_full_c = svptrue_c16(); - const svbool_t pNh_full = svptrue_b16(); - const svbool_t pN32_full = svwhilelt_b32(static_cast(0), static_cast(tile_rows)); - const svcount_t pOut16_full_c = svwhilelt_c16(static_cast(0), static_cast(tile_rows), 2); - const svfloat32_t z0_f32 = svdup_n_f32(0.0f); -#define CACTUS_STORE_FULL_TILE_ROW(dst_idx, out32_vec) \ - do { \ - svfloat16_t out16_local = svcvt_f16_f32_z(pN32_full, (out32_vec)); \ - out16_local = svuzp1_f16(out16_local, out16_local); \ - svst1_f16_x2(pOut16_full_c, &c[(dst_idx)], svcreate2(out16_local, out16_local)); \ - } while (0) - - for (size_t row = start_row; row < end_row; row += tile_rows) { - const size_t rb = row / tile_rows; - const size_t active_r = std::min(tile_rows, end_row - row); - const bool full_r = (active_r == tile_rows); - const svbool_t pMh = full_r - ? svptrue_b16() - : svwhilelt_b16(static_cast(0), static_cast(active_r * 2)); - - size_t cb = 0; - - for (; cb < cb4_tiles; cb += 4) { - svzero_za(); - const size_t g4 = cb / 4; - const __fp16* b_g4_base = b_packed + g4 * k_pairs * 4 * tile_pairs; - - size_t kp = 0; - for (; kp + 3 < k_pairs; kp += 4) { - const __fp16* a_ptr0 = a_packed + rb * a_row_block_stride + kp * tile_pairs; - const __fp16* b_ptr0 = b_g4_base + kp * (4 * tile_pairs); - const __fp16* a_ptr1 = a_ptr0 + tile_pairs; - const __fp16* b_ptr1 = b_ptr0 + 4 * tile_pairs; - const __fp16* a_ptr2 = a_ptr1 + tile_pairs; - const __fp16* b_ptr2 = b_ptr1 + 4 * tile_pairs; - const __fp16* a_ptr3 = a_ptr2 + tile_pairs; - const __fp16* b_ptr3 = b_ptr2 + 4 * tile_pairs; - - const svfloat16_t zA0 = svld1(pMh, a_ptr0); - const svfloat16x4_t zB40 = svld1_f16_x4(pNh_full_c, b_ptr0); - svmopa_za32_f16_m(0, pMh, pNh_full, zA0, svget4(zB40, 0)); - svmopa_za32_f16_m(1, pMh, pNh_full, zA0, svget4(zB40, 1)); - svmopa_za32_f16_m(2, pMh, pNh_full, zA0, svget4(zB40, 2)); - svmopa_za32_f16_m(3, pMh, pNh_full, zA0, svget4(zB40, 3)); - - const svfloat16_t zA1 = svld1(pMh, a_ptr1); - const svfloat16x4_t zB41 = svld1_f16_x4(pNh_full_c, b_ptr1); - svmopa_za32_f16_m(0, pMh, pNh_full, zA1, svget4(zB41, 0)); - svmopa_za32_f16_m(1, pMh, pNh_full, zA1, svget4(zB41, 1)); - svmopa_za32_f16_m(2, pMh, pNh_full, zA1, svget4(zB41, 2)); - svmopa_za32_f16_m(3, pMh, pNh_full, zA1, svget4(zB41, 3)); - - const svfloat16_t zA2 = svld1(pMh, a_ptr2); - const svfloat16x4_t zB42 = svld1_f16_x4(pNh_full_c, b_ptr2); - svmopa_za32_f16_m(0, pMh, pNh_full, zA2, svget4(zB42, 0)); - svmopa_za32_f16_m(1, pMh, pNh_full, zA2, svget4(zB42, 1)); - svmopa_za32_f16_m(2, pMh, pNh_full, zA2, svget4(zB42, 2)); - svmopa_za32_f16_m(3, pMh, pNh_full, zA2, svget4(zB42, 3)); - - const svfloat16_t zA3 = svld1(pMh, a_ptr3); - const svfloat16x4_t zB43 = svld1_f16_x4(pNh_full_c, b_ptr3); - svmopa_za32_f16_m(0, pMh, pNh_full, zA3, svget4(zB43, 0)); - svmopa_za32_f16_m(1, pMh, pNh_full, zA3, svget4(zB43, 1)); - svmopa_za32_f16_m(2, pMh, pNh_full, zA3, svget4(zB43, 2)); - svmopa_za32_f16_m(3, pMh, pNh_full, zA3, svget4(zB43, 3)); - } - - for (; kp < k_pairs; ++kp) { - const __fp16* a_ptr = a_packed + rb * a_row_block_stride + kp * tile_pairs; - const __fp16* b_ptr = b_g4_base + kp * (4 * tile_pairs); - - const svfloat16_t zA = svld1(pMh, a_ptr); - const svfloat16x4_t zB4 = svld1_f16_x4(pNh_full_c, b_ptr); - svmopa_za32_f16_m(0, pMh, pNh_full, zA, svget4(zB4, 0)); - svmopa_za32_f16_m(1, pMh, pNh_full, zA, svget4(zB4, 1)); - svmopa_za32_f16_m(2, pMh, pNh_full, zA, svget4(zB4, 2)); - svmopa_za32_f16_m(3, pMh, pNh_full, zA, svget4(zB4, 3)); - } - - const size_t col = cb * tile_rows; - if (full_r) { - for (size_t trow = 0; trow < tile_rows; trow += 4) { - const svfloat32x4_t zT0 = svread_hor_za32_f32_vg4(0, static_cast(trow)); - const svfloat32x4_t zT1 = svread_hor_za32_f32_vg4(1, static_cast(trow)); - const svfloat32x4_t zT2 = svread_hor_za32_f32_vg4(2, static_cast(trow)); - const svfloat32x4_t zT3 = svread_hor_za32_f32_vg4(3, static_cast(trow)); - const size_t dst0 = (row + trow + 0) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst0, svget4(zT0, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget4(zT1, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + 2 * tile_rows, svget4(zT2, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + 3 * tile_rows, svget4(zT3, 0)); - const size_t dst1 = (row + trow + 1) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst1, svget4(zT0, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget4(zT1, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + 2 * tile_rows, svget4(zT2, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + 3 * tile_rows, svget4(zT3, 1)); - const size_t dst2 = (row + trow + 2) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst2, svget4(zT0, 2)); - CACTUS_STORE_FULL_TILE_ROW(dst2 + tile_rows, svget4(zT1, 2)); - CACTUS_STORE_FULL_TILE_ROW(dst2 + 2 * tile_rows, svget4(zT2, 2)); - CACTUS_STORE_FULL_TILE_ROW(dst2 + 3 * tile_rows, svget4(zT3, 2)); - const size_t dst3 = (row + trow + 3) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst3, svget4(zT0, 3)); - CACTUS_STORE_FULL_TILE_ROW(dst3 + tile_rows, svget4(zT1, 3)); - CACTUS_STORE_FULL_TILE_ROW(dst3 + 2 * tile_rows, svget4(zT2, 3)); - CACTUS_STORE_FULL_TILE_ROW(dst3 + 3 * tile_rows, svget4(zT3, 3)); - } - } else { - size_t trow = 0; - for (; trow + 3 < active_r; trow += 4) { - const svfloat32x4_t zT0 = svread_hor_za32_f32_vg4(0, static_cast(trow)); - const svfloat32x4_t zT1 = svread_hor_za32_f32_vg4(1, static_cast(trow)); - const svfloat32x4_t zT2 = svread_hor_za32_f32_vg4(2, static_cast(trow)); - const svfloat32x4_t zT3 = svread_hor_za32_f32_vg4(3, static_cast(trow)); - const size_t dst0 = (row + trow + 0) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst0, svget4(zT0, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget4(zT1, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + 2 * tile_rows, svget4(zT2, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + 3 * tile_rows, svget4(zT3, 0)); - const size_t dst1 = (row + trow + 1) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst1, svget4(zT0, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget4(zT1, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + 2 * tile_rows, svget4(zT2, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + 3 * tile_rows, svget4(zT3, 1)); - const size_t dst2 = (row + trow + 2) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst2, svget4(zT0, 2)); - CACTUS_STORE_FULL_TILE_ROW(dst2 + tile_rows, svget4(zT1, 2)); - CACTUS_STORE_FULL_TILE_ROW(dst2 + 2 * tile_rows, svget4(zT2, 2)); - CACTUS_STORE_FULL_TILE_ROW(dst2 + 3 * tile_rows, svget4(zT3, 2)); - const size_t dst3 = (row + trow + 3) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst3, svget4(zT0, 3)); - CACTUS_STORE_FULL_TILE_ROW(dst3 + tile_rows, svget4(zT1, 3)); - CACTUS_STORE_FULL_TILE_ROW(dst3 + 2 * tile_rows, svget4(zT2, 3)); - CACTUS_STORE_FULL_TILE_ROW(dst3 + 3 * tile_rows, svget4(zT3, 3)); - } - for (; trow + 1 < active_r; trow += 2) { - const svfloat32x2_t zT0 = svread_hor_za32_f32_vg2(0, static_cast(trow)); - const svfloat32x2_t zT1 = svread_hor_za32_f32_vg2(1, static_cast(trow)); - const svfloat32x2_t zT2 = svread_hor_za32_f32_vg2(2, static_cast(trow)); - const svfloat32x2_t zT3 = svread_hor_za32_f32_vg2(3, static_cast(trow)); - const size_t dst0 = (row + trow + 0) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst0, svget2(zT0, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget2(zT1, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + 2 * tile_rows, svget2(zT2, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + 3 * tile_rows, svget2(zT3, 0)); - const size_t dst1 = (row + trow + 1) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst1, svget2(zT0, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget2(zT1, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + 2 * tile_rows, svget2(zT2, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + 3 * tile_rows, svget2(zT3, 1)); - } - for (; trow < active_r; ++trow) { - const size_t dst = (row + trow) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst, svread_hor_za32_f32_m(z0_f32, pN32_full, 0, static_cast(trow))); - CACTUS_STORE_FULL_TILE_ROW(dst + tile_rows, svread_hor_za32_f32_m(z0_f32, pN32_full, 1, static_cast(trow))); - CACTUS_STORE_FULL_TILE_ROW(dst + 2 * tile_rows, svread_hor_za32_f32_m(z0_f32, pN32_full, 2, static_cast(trow))); - CACTUS_STORE_FULL_TILE_ROW(dst + 3 * tile_rows, svread_hor_za32_f32_m(z0_f32, pN32_full, 3, static_cast(trow))); - } - } - } - - for (; cb < cb4_tiles + cb2_tiles; cb += 2) { - svzero_za(); - const size_t g2 = (cb - cb4_tiles) / 2; - const __fp16* b_g2_base = b_packed + cb4_groups * k_pairs * 4 * tile_pairs + g2 * k_pairs * 2 * tile_pairs; - - size_t kp = 0; - for (; kp < k_pairs; ++kp) { - const __fp16* a_ptr = a_packed + rb * a_row_block_stride + kp * tile_pairs; - const __fp16* b_ptr = b_g2_base + kp * (2 * tile_pairs); - - const svfloat16_t zA = svld1(pMh, a_ptr); - const svfloat16x2_t zB2 = svld1_f16_x2(pNh_full_c, b_ptr); - svmopa_za32_f16_m(0, pMh, pNh_full, zA, svget2(zB2, 0)); - svmopa_za32_f16_m(1, pMh, pNh_full, zA, svget2(zB2, 1)); - } - - const size_t col = cb * tile_rows; - if (full_r) { - for (size_t trow = 0; trow < tile_rows; trow += 4) { - const svfloat32x4_t zT0 = svread_hor_za32_f32_vg4(0, static_cast(trow)); - const svfloat32x4_t zT1 = svread_hor_za32_f32_vg4(1, static_cast(trow)); - const size_t dst0 = (row + trow + 0) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst0, svget4(zT0, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget4(zT1, 0)); - const size_t dst1 = (row + trow + 1) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst1, svget4(zT0, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget4(zT1, 1)); - const size_t dst2 = (row + trow + 2) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst2, svget4(zT0, 2)); - CACTUS_STORE_FULL_TILE_ROW(dst2 + tile_rows, svget4(zT1, 2)); - const size_t dst3 = (row + trow + 3) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst3, svget4(zT0, 3)); - CACTUS_STORE_FULL_TILE_ROW(dst3 + tile_rows, svget4(zT1, 3)); - } - } else { - size_t trow = 0; - for (; trow + 3 < active_r; trow += 4) { - const svfloat32x4_t zT0 = svread_hor_za32_f32_vg4(0, static_cast(trow)); - const svfloat32x4_t zT1 = svread_hor_za32_f32_vg4(1, static_cast(trow)); - const size_t dst0 = (row + trow + 0) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst0, svget4(zT0, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget4(zT1, 0)); - const size_t dst1 = (row + trow + 1) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst1, svget4(zT0, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget4(zT1, 1)); - const size_t dst2 = (row + trow + 2) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst2, svget4(zT0, 2)); - CACTUS_STORE_FULL_TILE_ROW(dst2 + tile_rows, svget4(zT1, 2)); - const size_t dst3 = (row + trow + 3) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst3, svget4(zT0, 3)); - CACTUS_STORE_FULL_TILE_ROW(dst3 + tile_rows, svget4(zT1, 3)); - } - for (; trow + 1 < active_r; trow += 2) { - const svfloat32x2_t zT0 = svread_hor_za32_f32_vg2(0, static_cast(trow)); - const svfloat32x2_t zT1 = svread_hor_za32_f32_vg2(1, static_cast(trow)); - const size_t dst0 = (row + trow + 0) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst0, svget2(zT0, 0)); - CACTUS_STORE_FULL_TILE_ROW(dst0 + tile_rows, svget2(zT1, 0)); - const size_t dst1 = (row + trow + 1) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst1, svget2(zT0, 1)); - CACTUS_STORE_FULL_TILE_ROW(dst1 + tile_rows, svget2(zT1, 1)); - } - for (; trow < active_r; ++trow) { - const size_t dst = (row + trow) * N + col; - CACTUS_STORE_FULL_TILE_ROW(dst, svread_hor_za32_f32_m(z0_f32, pN32_full, 0, static_cast(trow))); - CACTUS_STORE_FULL_TILE_ROW(dst + tile_rows, svread_hor_za32_f32_m(z0_f32, pN32_full, 1, static_cast(trow))); - } - } - } - - for (; cb < col_blocks; ++cb) { - const size_t col = cb * tile_rows; - const size_t active_c = std::min(tile_rows, N - col); - const svbool_t pNh = svwhilelt_b16(static_cast(0), static_cast(active_c * 2)); - const svbool_t pN32 = svwhilelt_b32(static_cast(0), static_cast(active_c)); - const svbool_t pOut16 = svwhilelt_b16(static_cast(0), static_cast(active_c)); - const size_t g1 = cb - cb4_tiles - cb2_tiles; - const __fp16* b_g1_base = b_packed + cb1_off + g1 * k_pairs * tile_pairs; - - svzero_za(); - - size_t kp = 0; - for (; kp + 1 < k_pairs; kp += 2) { - const __fp16* a_ptr0 = a_packed + rb * a_row_block_stride + kp * tile_pairs; - const __fp16* b_ptr0 = b_g1_base + kp * tile_pairs; - const __fp16* a_ptr1 = a_ptr0 + tile_pairs; - const __fp16* b_ptr1 = b_ptr0 + tile_pairs; - - const svfloat16_t zA0 = svld1(pMh, a_ptr0); - const svfloat16_t zB0 = svld1(pNh, b_ptr0); - svmopa_za32_f16_m(0, pMh, pNh, zA0, zB0); - - const svfloat16_t zA1 = svld1(pMh, a_ptr1); - const svfloat16_t zB1 = svld1(pNh, b_ptr1); - svmopa_za32_f16_m(0, pMh, pNh, zA1, zB1); - } - - for (; kp < k_pairs; ++kp) { - const __fp16* a_ptr = a_packed + rb * a_row_block_stride + kp * tile_pairs; - const __fp16* b_ptr = b_g1_base + kp * tile_pairs; - - const svfloat16_t zA = svld1(pMh, a_ptr); - const svfloat16_t zB = svld1(pNh, b_ptr); - svmopa_za32_f16_m(0, pMh, pNh, zA, zB); - } - - for (size_t trow = 0; trow < active_r; ++trow) { - svfloat32_t out32 = svread_hor_za32_f32_m(z0_f32, pN32, 0, static_cast(trow)); - svfloat16_t out16 = svcvt_f16_f32_z(pN32, out32); - out16 = svuzp1_f16(out16, out16); - svst1(pOut16, &c[(row + trow) * N + col], out16); - } - } - } -#undef CACTUS_STORE_FULL_TILE_ROW -} - -__arm_new("za") __arm_locally_streaming -static void cactus_matmul_f16_sme2_thread_entry( - const __fp16* a, - __fp16* a_packed, - const __fp16* b_packed, - __fp16* c, - size_t M, - size_t K, - size_t N, - size_t row_block_size, - size_t start_block, - size_t end_block -) { - const size_t tile_rows = svcntsw(); - const size_t tile_pairs = svcnth(); - const size_t k_pairs = (K + 1) / 2; - - for (size_t block_idx = start_block; block_idx < end_block; ++block_idx) { - const size_t start_row = block_idx * row_block_size; - const size_t end_row = std::min(start_row + row_block_size, M); - - for (size_t row = start_row; row < end_row; row += tile_rows) { - const size_t rb = row / tile_rows; - const size_t active_r = std::min(tile_rows, end_row - row); - cactus_pack_a_f16_row_block( - a, - a_packed, - K, - rb, - row, - active_r, - tile_rows, - tile_pairs, - k_pairs - ); - } - - cactus_matmul_f16_sme2_worker( - a_packed, - b_packed, - c, - M, - K, - N, - start_row, - end_row, - tile_rows, - tile_pairs - ); - } -} - -__arm_new("za") __arm_locally_streaming -void cactus_matmul_f16_sme2_caller( - const __fp16* a, - const __fp16* b_transposed, - __fp16* c, - size_t M, - size_t K, - size_t N -) { - const size_t tile_rows = svcntsw(); - const size_t tile_pairs = svcnth(); - constexpr size_t SME2_TILES_PER_THREAD = 3; - - const size_t row_blocks = (M + tile_rows - 1) / tile_rows; - const size_t k_pairs = (K + 1) / 2; - const size_t col_blocks = (N + tile_rows - 1) / tile_rows; - - std::unique_ptr<__fp16[]> a_packed(new __fp16[row_blocks * k_pairs * tile_pairs]); - std::unique_ptr<__fp16[]> b_packed(new __fp16[k_pairs * col_blocks * tile_pairs]); - __fp16* a_packed_ptr = a_packed.get(); - const __fp16* b_packed_ptr = b_packed.get(); - - cactus_pack_b_f16_from_bt(b_transposed, b_packed.get(), K, N, tile_rows, tile_pairs); - - const size_t row_block_size = SME2_TILES_PER_THREAD * tile_rows; - const size_t num_row_blocks = (M + row_block_size - 1) / row_block_size; - - auto& pool = CactusThreading::get_thread_pool(); - const size_t num_workers = std::min(pool.num_workers(), num_row_blocks); - if (num_workers <= 1) { - cactus_matmul_f16_sme2_thread_entry( - a, - a_packed_ptr, - b_packed_ptr, - c, - M, - K, - N, - row_block_size, - 0, - num_row_blocks - ); - return; - } - - std::atomic next_block{0}; - pool.enqueue_n_threads(num_workers, num_workers, [&](size_t, size_t) { - while (true) { - const size_t block_idx = next_block.fetch_add(1, std::memory_order_relaxed); - if (block_idx >= num_row_blocks) break; - cactus_matmul_f16_sme2_thread_entry( - a, - a_packed_ptr, - b_packed_ptr, - c, - M, - K, - N, - row_block_size, - block_idx, - block_idx + 1 - ); - } - }); - pool.wait_all(); -} - -#undef CACTUS_UNROLL4 diff --git a/cactus-transpile/kernel/kernel_utils.h b/cactus-transpile/kernel/kernel_utils.h deleted file mode 100644 index 916d9bf46..000000000 --- a/cactus-transpile/kernel/kernel_utils.h +++ /dev/null @@ -1,724 +0,0 @@ -#ifndef KERNEL_UTILS_H -#define KERNEL_UTILS_H - -#include -#if defined(__APPLE__) -#include -#include -#endif -#if defined(__ANDROID__) -#include -#include -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -constexpr size_t NEON_VECTOR_SIZE = 16; -constexpr size_t STREAMING_STORE_THRESHOLD = 32768; - -inline void stream_store_f16x8(__fp16* dst, float16x8_t val) { -#if defined(__aarch64__) - float16x4_t lo = vget_low_f16(val); - float16x4_t hi = vget_high_f16(val); - __asm__ __volatile__( - "stnp %d0, %d1, [%2]" - : - : "w"(lo), "w"(hi), "r"(dst) - : "memory" - ); -#else - vst1q_f16(dst, val); -#endif -} - -inline bool cpu_has_i8mm() { -#if defined(__aarch64__) - static std::once_flag once; - static bool has = false; - - std::call_once(once, []() { -#if defined(__APPLE__) - int ret = 0; - size_t size = sizeof(ret); - if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &ret, &size, nullptr, 0) == 0) { - has = (ret == 1); - } -#elif defined(__ANDROID__) - unsigned long hwcap2 = getauxval(AT_HWCAP2); - #ifndef HWCAP2_I8MM - #define HWCAP2_I8MM (1 << 13) - #endif - has = (hwcap2 & HWCAP2_I8MM) != 0; -#endif - }); - - return has; -#else - return false; -#endif -} - -inline bool cpu_has_sme2() { -#if defined(__aarch64__) - static std::once_flag once; - static bool has = false; - - std::call_once(once, []() { - -#if defined(__APPLE__) - int ret = 0; - size_t size = sizeof(ret); - if (sysctlbyname("hw.optional.arm.FEAT_SME2", &ret, &size, nullptr, 0) == 0) { - has = ret == 1; - } - -#elif defined(__ANDROID__) - unsigned long hwcap2 = getauxval(AT_HWCAP2); -#ifdef HWCAP2_SME2 - has = (hwcap2 & HWCAP2_SME2) != 0; -#endif - -#endif - }); - - return has; -#else - return false; -#endif -} - -inline float32x4_t fast_exp_f32x4(float32x4_t x) { - const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); - const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); - - const float32x4_t c0 = vdupq_n_f32(1.0f); - const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); - const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f); - const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f); - const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); - - x = vmaxq_f32(x, vdupq_n_f32(-87.0f)); - x = vminq_f32(x, vdupq_n_f32(87.0f)); - - float32x4_t z = vmulq_f32(x, log2e); - - int32x4_t zi = vcvtq_s32_f32(z); - float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi)); - - uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f)); - zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1))); - zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f))))); - - float32x4_t zf_ln2 = vmulq_f32(zf, ln2); - float32x4_t p = c4; - p = vfmaq_f32(c3, p, zf_ln2); - p = vfmaq_f32(c2, p, zf_ln2); - p = vfmaq_f32(c1, p, zf_ln2); - p = vfmaq_f32(c0, p, zf_ln2); - - int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23); - float32x4_t scale = vreinterpretq_f32_s32(exp_bits); - - return vmulq_f32(p, scale); -} - -// Cephes-style 13/6 rational tanh approximation (same coefficients as Eigen). -// Constants are stored as static splatted arrays so the compiler emits a single -// pc-relative `ldr q` per load. -alignas(16) inline constexpr float kFastTanhAlpha[7][4] = { - { 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f }, - { 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f }, - { 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f }, - { 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f }, - {-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f }, - { 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f }, - {-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f }, -}; -alignas(16) inline constexpr float kFastTanhBeta[4][4] = { - { 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f }, - { 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f }, - { 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f }, - { 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f }, -}; -alignas(16) inline constexpr float kFastTanhClampHi[4] = { 9.0f, 9.0f, 9.0f, 9.0f }; -alignas(16) inline constexpr float kFastTanhClampLo[4] = {-9.0f,-9.0f,-9.0f,-9.0f }; - -inline float32x4_t fast_tanh_f32x4(float32x4_t x) { - x = vmaxq_f32(vld1q_f32(kFastTanhClampLo), vminq_f32(vld1q_f32(kFastTanhClampHi), x)); - float32x4_t x2 = vmulq_f32(x, x); - float32x4_t p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[5]), vld1q_f32(kFastTanhAlpha[6]), x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[4]), p, x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[3]), p, x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[2]), p, x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[1]), p, x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[0]), p, x2); - p = vmulq_f32(p, x); - float32x4_t q = vfmaq_f32(vld1q_f32(kFastTanhBeta[2]), vld1q_f32(kFastTanhBeta[3]), x2); - q = vfmaq_f32(vld1q_f32(kFastTanhBeta[1]), q, x2); - q = vfmaq_f32(vld1q_f32(kFastTanhBeta[0]), q, x2); - return vdivq_f32(p, q); -} - -constexpr size_t SIMD_F16_WIDTH = 8; - -inline size_t simd_align(size_t count, size_t width = SIMD_F16_WIDTH) { - return (count / width) * width; -} - -inline void f16x8_split_f32(float16x8_t v, float32x4_t& lo, float32x4_t& hi) { - lo = vcvt_f32_f16(vget_low_f16(v)); - hi = vcvt_f32_f16(vget_high_f16(v)); -} - -inline float16x8_t f32_merge_f16(float32x4_t lo, float32x4_t hi) { - return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); -} - -inline float32x4_t fast_sigmoid_f32x4(float32x4_t x) { - const float32x4_t one = vdupq_n_f32(1.0f); - return vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(x)))); -} - -template -inline float16x8_t apply_f32_op_on_f16x8(float16x8_t v, F32x4Op op) { - float32x4_t lo, hi; - f16x8_split_f32(v, lo, hi); - return f32_merge_f16(op(lo), op(hi)); -} - -inline void unpack_int4_as_int8x16x2(const uint8_t* ptr, int8x16_t& high_decoded, int8x16_t& low_decoded) { - int8x16_t packed = vreinterpretq_s8_u8(vld1q_u8(ptr)); - high_decoded = vshrq_n_s8(packed, 4); - low_decoded = vshrq_n_s8(vshlq_n_s8(packed, 4), 4); -} - -namespace CactusThreading { - -#if defined(__ANDROID__) - struct CoreTopology { - std::vector performance_cores; - std::vector all_cores; - - static CoreTopology& get() { - static CoreTopology topo = detect(); - return topo; - } - - private: - static int read_sysfs_int(const char* path) { - std::ifstream f(path); - if (!f.is_open()) return -1; - int val = -1; - f >> val; - return val; - } - - static CoreTopology detect() { - CoreTopology topo; - constexpr int MAX_CPUS = 16; - std::vector> core_caps; - - for (int i = 0; i < MAX_CPUS; ++i) { - char path[128]; - - snprintf(path, sizeof(path), - "/sys/devices/system/cpu/cpu%d/cpu_capacity", i); - int cap = read_sysfs_int(path); - if (cap > 0) { - core_caps.push_back({i, cap}); - topo.all_cores.push_back(i); - continue; - } - - snprintf(path, sizeof(path), - "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); - int freq = read_sysfs_int(path); - if (freq > 0) { - core_caps.push_back({i, freq}); - topo.all_cores.push_back(i); - } - } - - if (core_caps.empty()) return topo; - - int max_cap = 0; - for (auto& [id, cap] : core_caps) { - max_cap = std::max(max_cap, cap); - } - - int threshold = static_cast(max_cap * 0.70); - for (auto& [id, cap] : core_caps) { - if (cap >= threshold) { - topo.performance_cores.push_back(id); - } - } - - return topo; - } - }; - - inline bool pin_current_thread_to_cores(const std::vector& cores) { - if (cores.empty()) return false; - cpu_set_t mask; - CPU_ZERO(&mask); - for (int core : cores) { - CPU_SET(core, &mask); - } - return sched_setaffinity(0, sizeof(mask), &mask) == 0; - } -#endif - - class ThreadPool { - private: - static constexpr size_t MAX_WORKERS = 16; - - std::vector workers; - std::deque> tasks; - - std::mutex mutex; - std::condition_variable work_available; - std::condition_variable work_done; - - bool stop{false}; - std::atomic pending_tasks{0}; - size_t num_workers_; - - void worker_thread() { - while (true) { - std::function task; - { - std::unique_lock lock(mutex); - work_available.wait(lock, [this] { - return stop || !tasks.empty(); - }); - - if (stop && tasks.empty()) { - return; - } - - task = std::move(tasks.front()); - tasks.pop_front(); - } - - task(); - - if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { - std::lock_guard lock(mutex); - work_done.notify_one(); - } - } - } - - public: - explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) - : stop(false), pending_tasks(0) { - num_workers_ = std::min(num_threads, MAX_WORKERS); - if (num_workers_ == 0) num_workers_ = 1; - -#if defined(__ANDROID__) - auto& topo = CoreTopology::get(); - if (!topo.performance_cores.empty()) { - num_workers_ = std::min(num_workers_, topo.performance_cores.size()); - } -#endif - - workers.reserve(num_workers_); - for (size_t i = 0; i < num_workers_; ++i) { - workers.emplace_back([this]() { -#if defined(__ANDROID__) - auto& perf = CoreTopology::get().performance_cores; - if (!perf.empty()) { - pin_current_thread_to_cores(perf); - } -#endif - worker_thread(); - }); - } - } - - ~ThreadPool() { - { - std::lock_guard lock(mutex); - stop = true; - } - work_available.notify_all(); - for (auto& worker : workers) { - if (worker.joinable()) { - worker.join(); - } - } - } - - template - auto enqueue(F&& f) -> std::future { - using return_type = decltype(f()); - - auto task = std::make_shared>( - std::forward(f) - ); - - std::future res = task->get_future(); - - { - std::lock_guard lock(mutex); - pending_tasks.fetch_add(1, std::memory_order_relaxed); - tasks.emplace_back([task](){ (*task)(); }); - } - work_available.notify_one(); - - return res; - } - - template - void enqueue_batch(size_t total_work, F task_func) { - if (total_work == 0) return; - - const size_t num_tasks = std::min(num_workers_, total_work); - const size_t per_worker = total_work / num_tasks; - const size_t remainder = total_work % num_tasks; - - { - std::lock_guard lock(mutex); - pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed); - - for (size_t w = 0; w < num_tasks; ++w) { - size_t start = w * per_worker + std::min(w, remainder); - size_t end = start + per_worker + (w < remainder ? 1 : 0); - tasks.emplace_back([=]() { task_func(start, end); }); - } - } - work_available.notify_all(); - } - - void wait_all() { - std::unique_lock lock(mutex); - work_done.wait(lock, [this] { - return pending_tasks.load(std::memory_order_acquire) == 0; - }); - } - - template - void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) { - if (total_work == 0 || num_threads == 0) return; - - num_threads = std::min(num_threads, std::min(num_workers_, total_work)); - const size_t per_thread = total_work / num_threads; - const size_t remainder = total_work % num_threads; - - { - std::lock_guard lock(mutex); - pending_tasks.fetch_add(num_threads, std::memory_order_relaxed); - - for (size_t t = 0; t < num_threads; ++t) { - size_t start = t * per_thread + std::min(t, remainder); - size_t end = start + per_thread + (t < remainder ? 1 : 0); - tasks.emplace_back([=]() { task_func(start, end); }); - } - } - work_available.notify_all(); - } - - size_t num_workers() const { return num_workers_; } - }; - - inline size_t get_configured_thread_count() { - const char* raw = std::getenv("CACTUS_NUM_THREADS"); - if (!raw || !*raw) { - return std::thread::hardware_concurrency(); - } - char* end = nullptr; - long parsed = std::strtol(raw, &end, 10); - if (end == raw || parsed <= 0) { - return std::thread::hardware_concurrency(); - } - return static_cast(parsed); - } - - inline ThreadPool& get_thread_pool() { - static ThreadPool pool(get_configured_thread_count()); - return pool; - } - - struct ParallelConfig { - size_t min_work_gate; - size_t work_per_thread; - - constexpr ParallelConfig(size_t gate, size_t per_thread) - : min_work_gate(gate), work_per_thread(per_thread) {} - }; - - inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) { - if (total_work < config.min_work_gate) return 1; - - size_t pool_size = get_thread_pool().num_workers(); - size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread; - return std::min(pool_size, std::max(static_cast(1), num_threads)); - } - - struct Thresholds { - #if defined(__ANDROID__) - static constexpr ParallelConfig ATTENTION{64, 32}; - static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; - static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; - static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; - static constexpr ParallelConfig SCALAR_BASIC{30000, 15000}; - static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000}; - #else // Apple - static constexpr ParallelConfig ATTENTION{32, 16}; - static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; - static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; - static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; - static constexpr ParallelConfig SCALAR_BASIC{5000, 2500}; - static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250}; - #endif - }; - - struct GemmThreading { - #if defined(__ANDROID__) - static size_t get_num_threads(size_t M, size_t pool_size) { - if (M <= 1) return 1; - return pool_size; - } - static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) { - return 1; - } - #elif defined(__APPLE__) && TARGET_OS_IPHONE - static constexpr size_t GEMV_MIN_N_BLOCKS = 512; - static size_t get_num_threads(size_t M, size_t pool_size) { - if (M <= 1) return std::min(pool_size, static_cast(2)); - return pool_size; - } - static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { - if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; - return std::min(pool_size, static_cast(3)); - } - #else - static constexpr size_t GEMV_MIN_N_BLOCKS = 256; - static size_t get_num_threads(size_t M, size_t pool_size) { - if (M <= 1) return std::min(pool_size, static_cast(4)); - return pool_size; - } - static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { - if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; - if (N_blocks < 512) return std::min(pool_size, static_cast(2)); - return std::min(pool_size, static_cast(5)); - } - #endif - }; - - inline size_t& get_gemm_thread_override() { - static size_t override_threads = 0; - return override_threads; - } - - inline void set_gemm_threads(size_t num_threads) { - get_gemm_thread_override() = num_threads; - } - - inline void reset_gemm_threads() { - get_gemm_thread_override() = 0; - } - - class TaskHandle { - private: - std::vector> futures_; - bool auto_wait_; - - public: - TaskHandle(bool auto_wait = true) : auto_wait_(auto_wait) {} - - ~TaskHandle() { - if (auto_wait_) { - wait(); - } - } - - TaskHandle(TaskHandle&&) = default; - TaskHandle& operator=(TaskHandle&&) = default; - TaskHandle(const TaskHandle&) = delete; - TaskHandle& operator=(const TaskHandle&) = delete; - - void add_future(std::future&& f) { - futures_.push_back(std::move(f)); - } - - void wait() { - for (auto& f : futures_) { - if (f.valid()) { - f.wait(); - } - } - futures_.clear(); - } - - bool is_ready() const { - for (const auto& f : futures_) { - if (f.valid() && f.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { - return false; - } - } - return true; - } - - size_t task_count() const { return futures_.size(); } - }; - - template - TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) { - const size_t num_threads = get_optimal_thread_count(total_work, config); - TaskHandle handle(!wait); - - if (num_threads == 1) { - if (wait) { - work_func(0, total_work); - return handle; - } - auto& pool = get_thread_pool(); - handle.add_future(pool.enqueue([work_func, total_work]() { - work_func(0, total_work); - })); - return handle; - } - - auto& pool = get_thread_pool(); - const size_t work_per_thread = total_work / num_threads; - - for (size_t t = 0; t < num_threads; ++t) { - handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() { - const size_t start_idx = t * work_per_thread; - const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; - work_func(start_idx, end_idx); - })); - } - - if (wait) { - handle.wait(); - } - return handle; - } - - template - void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) { - const size_t total_work = outer_size * inner_size; - parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) { - for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { - const size_t outer = work_idx / inner_size; - const size_t inner = work_idx % inner_size; - work_func(outer, inner); - } - }); - } - - template - ResultType parallel_reduce(size_t total_work, ParallelConfig config, - WorkFunc work_func, ResultType init_value, CombineFunc combine_func) { - const size_t num_threads = get_optimal_thread_count(total_work, config); - - if (num_threads == 1) { - return work_func(0, total_work); - } - - auto& pool = get_thread_pool(); - std::vector> futures; - std::vector partial_results(num_threads, init_value); - const size_t work_per_thread = total_work / num_threads; - - for (size_t t = 0; t < num_threads; ++t) { - futures.push_back(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() -> ResultType { - const size_t start_idx = t * work_per_thread; - const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; - return work_func(start_idx, end_idx); - })); - } - - ResultType result = init_value; - for (auto& future : futures) { - result = combine_func(result, future.get()); - } - return result; - } - - template - void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) { - auto& pool = get_thread_pool(); - - size_t override = get_gemm_thread_override(); - size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers()); - num_threads = std::min(num_threads, total_tiles); - - if (num_threads <= 1) { - work_func(0, total_tiles); - return; - } - - pool.enqueue_n_threads(total_tiles, num_threads, work_func); - pool.wait_all(); - } - -} - -template -void elementwise_op_f16(const __fp16* input, __fp16* output, size_t num_elements, - bool use_streaming, CactusThreading::ParallelConfig config, - SimdOp simd_op, ScalarOp scalar_op, size_t unroll = 4) { - CactusThreading::parallel_for(num_elements, config, - [&](size_t start, size_t end) { - const size_t n = end - start; - const size_t vec_end = start + simd_align(n); - - if (use_streaming && unroll >= 4) { - const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 4); - for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 4) { - __builtin_prefetch(&input[i + 256], 0, 0); - float16x8_t v0 = simd_op(vld1q_f16(&input[i])); - float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); - float16x8_t v2 = simd_op(vld1q_f16(&input[i + 16])); - float16x8_t v3 = simd_op(vld1q_f16(&input[i + 24])); - stream_store_f16x8(&output[i], v0); - stream_store_f16x8(&output[i + 8], v1); - stream_store_f16x8(&output[i + 16], v2); - stream_store_f16x8(&output[i + 24], v3); - } - for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { - stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); - } - } else if (use_streaming && unroll >= 2) { - const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 2); - for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 2) { - __builtin_prefetch(&input[i + 128], 0, 0); - float16x8_t v0 = simd_op(vld1q_f16(&input[i])); - float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); - stream_store_f16x8(&output[i], v0); - stream_store_f16x8(&output[i + 8], v1); - } - for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { - stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); - } - } else { - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - vst1q_f16(&output[i], simd_op(vld1q_f16(&input[i]))); - } - } - for (size_t i = vec_end; i < end; ++i) { - output[i] = scalar_op(input[i]); - } - }); -} - -#endif // KERNEL_UTILS_H diff --git a/cactus-transpile/models/gemma4/model_gemma4.cpp b/cactus-transpile/models/gemma4/model_gemma4.cpp deleted file mode 100644 index c3f9c5b70..000000000 --- a/cactus-transpile/models/gemma4/model_gemma4.cpp +++ /dev/null @@ -1,594 +0,0 @@ -#include "model_gemma4.h" -#include "../../graph/graph.h" -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Gemma4Model::Gemma4Model() : Model() {} - -Gemma4Model::Gemma4Model(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -bool Gemma4Model::is_global_layer(uint32_t idx) const { - return config_.layer_types[idx] == "global"; -} - -std::vector Gemma4Model::get_kv_layer_dims() const { - uint32_t n = config_.num_layers; - - std::vector dims(n); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) { - dims[i] = 0; - } else if (is_global_layer(i)) { - dims[i] = config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2; - } else { - dims[i] = config_.attention_head_dim; - } - } - return dims; -} - -std::vector Gemma4Model::get_kv_layer_heads() const { - uint32_t n = config_.num_layers; - - std::vector heads(n); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) { - heads[i] = 0; - } else if (is_global_layer(i) && config_.num_global_kv_heads > 0) { - heads[i] = config_.num_global_kv_heads; - } else { - heads[i] = config_.attention_kv_heads; - } - } - return heads; -} - -void Gemma4Model::compact_kv_cache() { - uint32_t n = config_.num_layers; - - std::vector target_windows(n, 0); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) continue; - if (!is_global_layer(i)) - target_windows[i] = config_.sliding_window; - } - kv_cache_.compact_to_windows(target_windows); -} - -void Gemma4Model::post_init() { - uint32_t n = config_.num_layers; - - kv_cache_.set_window_size(0, 0); - - kv_share_map_.resize(n, -1); - shared_k_nodes_.resize(n, 0); - shared_v_nodes_.resize(n, 0); - - for (uint32_t i = first_shared_layer_; i < n; i++) { - bool is_global = is_global_layer(i); - for (int j = static_cast(first_shared_layer_) - 1; j >= 0; j--) { - if (is_global_layer(j) == is_global) { - kv_share_map_[i] = j; - break; - } - } - } -} - -void Gemma4Model::load_weights_to_graph(CactusGraph* gb) { - uint32_t n = config_.num_layers; - uint32_t num_shared = config_.num_kv_shared_layers; - first_shared_layer_ = (n > num_shared) ? n - num_shared : n; - - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - bool has_pli = config_.hidden_size_per_layer_input > 0; - if (has_pli) { - weight_nodes_.embed_tokens_per_layer = gb->mmap_embeddings(model_folder_path_ + "/embed_tokens_per_layer.weights"); - weight_nodes_.per_layer_model_proj = gb->mmap_weights(model_folder_path_ + "/per_layer_model_proj.weights"); - weight_nodes_.per_layer_proj_norm = gb->mmap_weights(model_folder_path_ + "/per_layer_proj_norm.weights"); - } - - bool has_moe = config_.enable_moe_block; - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - bool is_shared = (i >= first_shared_layer_); - - layer.attn_q_weight = gb->mmap_weights(prefix + "attn_q.weights"); - layer.attn_k_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k.weights"); - bool k_eq_v = config_.attention_k_eq_v && is_global_layer(i); - layer.attn_v_weight = is_shared ? 0 : (k_eq_v ? layer.attn_k_weight : gb->mmap_weights(prefix + "attn_v.weights")); - layer.attn_output_weight = gb->mmap_weights(prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(prefix + "post_attn_norm.weights"); - layer.pre_feedforward_layernorm_weight = gb->mmap_weights(prefix + "pre_ffn_norm.weights"); - layer.post_feedforward_layernorm_weight = gb->mmap_weights(prefix + "post_ffn_norm.weights"); - if (has_pli) { - layer.per_layer_gate = gb->mmap_weights(prefix + "per_layer_gate.weights"); - layer.per_layer_proj = gb->mmap_weights(prefix + "per_layer_proj.weights"); - layer.post_per_layer_norm = gb->mmap_weights(prefix + "post_per_layer_norm.weights"); - } - layer.layer_scalar = std::filesystem::exists(prefix + "layer_scalar.weights") ? gb->mmap_weights(prefix + "layer_scalar.weights") : 0; - - if (has_moe) { - uint32_t num_experts = config_.num_experts; - uint32_t hidden = config_.hidden_dim; - uint32_t expert_dim = config_.expert_intermediate_size > 0 ? config_.expert_intermediate_size : config_.ffn_intermediate_dim; - layer.moe_w1_experts.resize(num_experts); - layer.moe_w3_experts.resize(num_experts); - layer.moe_w2_experts.resize(num_experts); - - auto w1_packed = gb->mmap_weights(prefix + "moe_gate_proj.weights"); - auto w3_packed = gb->mmap_weights(prefix + "moe_up_proj.weights"); - auto w2_packed = gb->mmap_weights(prefix + "moe_down_proj.weights"); - - const auto& w1_buf = gb->get_output_buffer(w1_packed); - const auto& w3_buf = gb->get_output_buffer(w3_packed); - const auto& w2_buf = gb->get_output_buffer(w2_packed); - - auto setup_experts = [&](const BufferDesc& buf, - std::vector& expert_nodes, - size_t out_dim, size_t in_dim) { - auto* base = static_cast(const_cast(buf.get_data())); - Precision prec = buf.precision; - bool is_quantized = PrecisionTraits::is_integer(prec) && buf.group_size > 0; - size_t K = in_dim; - if (is_quantized && K % buf.group_size != 0) - K = ((K + buf.group_size - 1) / buf.group_size) * buf.group_size; - size_t expert_data_bytes = PrecisionTraits::packed_size_of(prec, out_dim * K); - size_t groups_per_expert = is_quantized ? (out_dim * K + buf.group_size - 1) / buf.group_size : 0; - size_t expert_scales_bytes = groups_per_expert * sizeof(__fp16); - - char* scales_base = is_quantized ? static_cast(const_cast(buf.scales_data)) : nullptr; - - for (uint32_t e = 0; e < num_experts; e++) { - expert_nodes[e] = gb->input({out_dim, K}, prec); - gb->set_external_input(expert_nodes[e], base + e * expert_data_bytes, prec); - if (is_quantized) { - gb->set_grouped_scales(expert_nodes[e], buf.group_size, groups_per_expert, - scales_base + e * expert_scales_bytes); - if (buf.is_interleaved) - gb->set_interleaved(expert_nodes[e], true, out_dim); - } - } - }; - - setup_experts(w1_buf, layer.moe_w1_experts, expert_dim, hidden); - setup_experts(w3_buf, layer.moe_w3_experts, expert_dim, hidden); - setup_experts(w2_buf, layer.moe_w2_experts, hidden, expert_dim); - - layer.router_proj = gb->mmap_weights(prefix + "router_proj.weights"); - layer.router_scale = gb->mmap_weights(prefix + "router_scale.weights"); - layer.moe_per_expert_scale = gb->mmap_weights(prefix + "moe_per_expert_scale.weights"); - layer.post_ffn_norm_1 = gb->mmap_weights(prefix + "post_ffn_norm_1.weights"); - layer.pre_ffn_norm_2 = gb->mmap_weights(prefix + "pre_ffn_norm_2.weights"); - layer.post_ffn_norm_2 = gb->mmap_weights(prefix + "post_ffn_norm_2.weights"); - } - } - - size_t sliding_head_dim = config_.attention_head_dim; - size_t global_head_dim = config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2; - v_norm_ones_weight_.assign(std::max(sliding_head_dim, global_head_dim), static_cast<__fp16>(1.0f)); - v_norm_ones_node_ = gb->input({sliding_head_dim}, Precision::FP16); - gb->set_external_input(v_norm_ones_node_, v_norm_ones_weight_.data(), Precision::FP16); - v_norm_ones_global_node_ = gb->input({global_head_dim}, Precision::FP16); - gb->set_external_input(v_norm_ones_global_node_, v_norm_ones_weight_.data(), Precision::FP16); - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - if (std::filesystem::exists(npu_prefill_path)) { - if (!load_npu_prefill(npu_prefill_path) || !has_npu_prefill()) { - CACTUS_LOG_WARN("npu", "[gemma4] found model.mlpackage but failed to enable NPU prefill; using CPU prefill"); - } - } else { - CACTUS_LOG_WARN("npu", "[gemma4] model.mlpackage not found; using CPU prefill"); - } - } else { - CACTUS_LOG_WARN("npu", "[gemma4] NPU backend unavailable on this device; using CPU prefill"); - } -} - -size_t Gemma4Model::build_per_layer_input(CactusGraph* gb, size_t hidden, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto gate = gb->gelu(gb->matmul(hidden, layer.per_layer_gate, true, backend)); - auto pli_slice = gb->slice(pli_combined, 1, layer_idx * pli_dim, pli_dim); - auto gated = gb->multiply(gate, pli_slice); - auto pli_proj = gb->matmul(gated, layer.per_layer_proj, true, backend); - auto pli_normed = gb->rms_norm(pli_proj, layer.post_per_layer_norm, config_.layer_norm_eps); - - return gb->add(hidden, pli_normed); -} - -size_t Gemma4Model::apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, - float rope_freq, size_t position_offset) { - if (rot_dim < head_dim) { - float adjusted_theta = std::pow(rope_freq, static_cast(rot_dim) / static_cast(head_dim)); - auto rot_part = gb->slice(tensor, 3, 0, rot_dim); - auto pass_part = gb->slice(tensor, 3, rot_dim, head_dim - rot_dim); - return gb->concat(gb->rope(rot_part, adjusted_theta, position_offset), pass_part, 3); - } - return gb->rope(tensor, rope_freq, position_offset); -} - -size_t Gemma4Model::build_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t seq_len = gb->get_output_buffer(input).shape[0]; - int share_src = (layer_idx < kv_share_map_.size()) ? kv_share_map_[layer_idx] : -1; - - bool is_global = is_global_layer(layer_idx); - - size_t head_dim = is_global ? (config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2) : config_.attention_head_dim; - size_t num_heads = config_.attention_heads; - size_t kv_heads = is_global && config_.num_global_kv_heads > 0 ? config_.num_global_kv_heads : config_.attention_kv_heads; - float rope_freq = is_global ? config_.rope_theta : config_.rope_local_base_freq; - size_t window = is_global ? 0 : config_.sliding_window; - size_t rot_dim = static_cast(head_dim * (is_global ? config_.global_partial_rotary_factor : 1.0f)); - - auto q = gb->matmul(input, layer.attn_q_weight, true, backend); - q = gb->reshape(q, {seq_len * num_heads, head_dim}); - q = gb->rms_norm(q, layer.attn_q_norm_weight, config_.layer_norm_eps); - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - - size_t q4 = apply_partial_rope(gb, q, head_dim, rot_dim, rope_freq, position_offset); - - size_t k4, v4; - if (share_src >= 0 && shared_k_nodes_[share_src] != 0) { - k4 = shared_k_nodes_[share_src]; - v4 = shared_v_nodes_[share_src]; - } else { - auto k = gb->matmul(input, layer.attn_k_weight, true, backend); - k = gb->reshape(k, {seq_len * kv_heads, head_dim}); - k = gb->rms_norm(k, layer.attn_k_norm_weight, config_.layer_norm_eps); - k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); - - k4 = apply_partial_rope(gb, k, head_dim, rot_dim, rope_freq, position_offset); - - auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); - size_t v_ones = is_global ? v_norm_ones_global_node_ : v_norm_ones_node_; - auto v = gb->rms_norm(gb->reshape(v_proj, {seq_len * kv_heads, head_dim}), v_ones, config_.layer_norm_eps); - v4 = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); - - shared_k_nodes_[layer_idx] = k4; - shared_v_nodes_[layer_idx] = v4; - } - - if (use_cache && share_src < 0) { - cache_k_output_nodes_[layer_idx] = k4; - cache_v_output_nodes_[layer_idx] = v4; - } - - size_t cache_src = (share_src >= 0) ? static_cast(share_src) : layer_idx; - size_t attn; - if (use_cache && !kv_cache_.is_empty()) { - attn = gb->attention_int8_hybrid(q4, k4, v4, attention_scale_, position_offset, - kv_cache_.get_keys_int8(cache_src), kv_cache_.get_values_int8(cache_src), - kv_cache_.get_key_scales(cache_src), kv_cache_.get_value_scales(cache_src), - kv_cache_.current_seq_len, kv_heads, head_dim, window); - } else { - attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); - } - - return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), layer.attn_output_weight, true, backend); -} - -size_t Gemma4Model::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto gate = gb->gelu(gb->matmul(input, layer.ffn_gate_weight, true, backend)); - auto up = gb->matmul(input, layer.ffn_up_weight, true, backend); - return gb->matmul(gb->multiply(gate, up), layer.ffn_down_weight, true, backend); -} - -size_t Gemma4Model::build_moe(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto router_logits = gb->matmul(input, layer.router_proj, true, backend); - auto topk_result = gb->topk(router_logits, config_.num_experts_per_tok); - auto topk_indices = gb->index(topk_result, 0, 0); - auto routing_probs = gb->softmax(router_logits); - - return gb->moe_layer(input, routing_probs, topk_indices, - layer.moe_w1_experts, layer.moe_w3_experts, layer.moe_w2_experts, - config_.num_experts, config_.num_experts_per_tok, - true, 1e-6f, 1.0f, Activation::GELU, - layer.moe_per_expert_scale); -} - -size_t Gemma4Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto normed = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_raw = build_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto residual = gb->add(hidden, attn); - - if (config_.enable_moe_block) { - auto h1 = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - h1 = build_mlp(gb, h1, layer_idx, backend); - h1 = gb->rms_norm(h1, layer.post_ffn_norm_1, config_.layer_norm_eps); - - auto h2 = gb->multiply(residual, layer.router_scale); - h2 = gb->rms_norm(h2, layer.pre_ffn_norm_2, config_.layer_norm_eps); - h2 = build_moe(gb, h2, layer_idx, backend); - h2 = gb->rms_norm(h2, layer.post_ffn_norm_2, config_.layer_norm_eps); - - auto combined = gb->rms_norm(gb->add(h1, h2), layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - return gb->add(residual, combined); - } - - auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - auto mlp_raw = build_mlp(gb, pre_mlp, layer_idx, backend); - auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - - return gb->add(residual, mlp); -} - -size_t Gemma4Model::apply_transformer_layer(CactusGraph* gb, size_t hidden, size_t pli, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t pos_offset) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, pos_offset); - if (config_.hidden_size_per_layer_input > 0) - hidden = build_per_layer_input(gb, hidden, pli, layer_idx, backend); - if (weight_nodes_.layers[layer_idx].layer_scalar != 0) - hidden = gb->multiply(hidden, weight_nodes_.layers[layer_idx].layer_scalar); - return hidden; -} - -size_t Gemma4Model::build_pli_combined(CactusGraph* gb, size_t hidden, size_t pli_embed, - size_t seq_len, ComputeBackend backend) { - uint32_t num_layers = config_.num_layers; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto pli_proj = gb->scalar_multiply(gb->matmul(hidden, weight_nodes_.per_layer_model_proj, true, backend), - 1.0f / std::sqrt(static_cast(config_.hidden_dim))); - pli_proj = gb->reshape(pli_proj, {seq_len * num_layers, pli_dim}); - pli_proj = gb->rms_norm(pli_proj, weight_nodes_.per_layer_proj_norm, config_.layer_norm_eps); - pli_proj = gb->reshape(pli_proj, {seq_len, num_layers * pli_dim}); - return gb->scalar_multiply(gb->add(pli_proj, pli_embed), 1.0f / std::sqrt(2.0f)); -} - -std::pair Gemma4Model::build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input) { - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - token_input = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - pli_input = gb->input({seq_len}, Precision::FP32); - auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), - std::sqrt(static_cast(pli_dim))); - auto pli_combined = build_pli_combined(gb, hidden, pli_embed, seq_len, backend); - - return {hidden, pli_combined}; -} - -void Gemma4Model::set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens) { - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - gb->set_input(pli_input, input_data.data(), Precision::FP32); -} - -size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend, bool use_cache) { - return forward_from_embeddings(gb, hidden, hidden, pli_tokens, seq_len, backend, use_cache); -} - -size_t Gemma4Model::build_pli_combined_from_tokens(CactusGraph* gb, size_t hidden, - const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend) { - if (config_.hidden_size_per_layer_input == 0) - return 0; - - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto pli_input = gb->input({seq_len}, Precision::FP32); - auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), - std::sqrt(static_cast(pli_dim))); - auto pli_combined = build_pli_combined(gb, hidden, pli_embed, seq_len, backend); - - std::vector pli_data(pli_tokens.size()); - for (size_t i = 0; i < pli_tokens.size(); i++) - pli_data[i] = static_cast(pli_tokens[i]); - gb->set_input(pli_input, pli_data.data(), Precision::FP32); - - return pli_combined; -} - -size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, size_t pli_hidden_source, - const std::vector& pli_tokens, size_t seq_len, - ComputeBackend backend, bool use_cache) { - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - if (config_.hidden_size_per_layer_input == 0) { - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - auto pli_combined = build_pli_combined_from_tokens(gb, pli_hidden_source, pli_tokens, seq_len, backend); - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli_combined, i, backend, use_cache, pos_offset); - - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - -size_t Gemma4Model::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (tokens.empty()) - throw std::runtime_error("Token sequence cannot be empty"); - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - if (config_.hidden_size_per_layer_input == 0) { - auto token_input = gb->input({tokens.size()}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - size_t token_input, pli_input; - auto hidden_pli = build_preamble_and_embed(gb, tokens.size(), backend, token_input, pli_input); - size_t hidden = hidden_pli.first, pli = hidden_pli.second; - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); - - set_token_inputs(gb, token_input, pli_input, tokens); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - -size_t Gemma4Model::forward_split(const std::vector& tokens, bool use_cache) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - size_t seq_len = tokens.size(); - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - if (config_.hidden_size_per_layer_input == 0) { - auto token_input = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - for (uint32_t i = 0; i < first_shared_layer_; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - - hidden = gb->index(hidden, seq_len - 1, 0); - hidden = gb->reshape(hidden, {1, config_.hidden_dim}); - - size_t shared_pos_offset = pos_offset + seq_len - 1; - for (uint32_t i = first_shared_layer_; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, shared_pos_offset); - - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - size_t token_input, pli_input; - auto hidden_pli = build_preamble_and_embed(gb, seq_len, backend, token_input, pli_input); - size_t hidden = hidden_pli.first, pli = hidden_pli.second; - - for (uint32_t i = 0; i < first_shared_layer_; i++) - hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); - - hidden = gb->index(hidden, seq_len - 1, 0); - hidden = gb->reshape(hidden, {1, config_.hidden_dim}); - auto pli_last = gb->index(pli, seq_len - 1, 0); - pli_last = gb->reshape(pli_last, {1, config_.num_layers * config_.hidden_size_per_layer_input}); - - size_t shared_pos_offset = pos_offset + seq_len - 1; - for (uint32_t i = first_shared_layer_; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli_last, i, backend, use_cache, shared_pos_offset); - - set_token_inputs(gb, token_input, pli_input, tokens); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - - -void Gemma4Model::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (tokens.empty()) - return; - - if (has_npu_prefill()) { - size_t npu_chunk_size = static_cast(npu_prefill_->get_chunk_size()); - if (tokens.size() > npu_chunk_size) { - Model::prefill(tokens, chunk_size, profile_file); - return; - } - } - - static constexpr size_t SPLIT_PREFILL_MIN_TOKENS = 32; - bool use_split = config_.num_kv_shared_layers > 0 - && tokens.size() >= SPLIT_PREFILL_MIN_TOKENS - && !std::getenv("CACTUS_DISABLE_SPLIT_PREFILL"); - - if (!use_split) { - Model::prefill(tokens, chunk_size, profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - auto process_chunk = [&](const std::vector& chunk) { - forward_split(chunk, true); - gb->execute(profile_file); - update_kv_cache(gb, chunk.size()); - }; - - if (tokens.size() <= chunk_size) { - process_chunk(tokens); - return; - } - - size_t num_full_chunks = (tokens.size() - 1) / chunk_size; - for (size_t i = 0; i < num_full_chunks; ++i) { - size_t start = i * chunk_size; - std::vector chunk(tokens.begin() + start, tokens.begin() + start + chunk_size); - if (i == 1) - gb->set_prefill_mode(true); - process_chunk(chunk); - } - - gb->set_prefill_mode(false); - size_t final_start = num_full_chunks * chunk_size; - process_chunk(std::vector(tokens.begin() + final_start, tokens.end())); -} - -} -} diff --git a/cactus-transpile/models/gemma4/model_gemma4.h b/cactus-transpile/models/gemma4/model_gemma4.h deleted file mode 100644 index 2e061917e..000000000 --- a/cactus-transpile/models/gemma4/model_gemma4.h +++ /dev/null @@ -1,404 +0,0 @@ -#pragma once - -#include "../model.h" - -bool test_gemma4_vision(bool expect_npu); -bool test_gemma4_audio(bool expect_npu); - -namespace cactus { -namespace engine { - -class Gemma4Model : public Model { - friend class Gemma4MmModel; -public: - Gemma4Model(); - explicit Gemma4Model(const Config& config); - ~Gemma4Model() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_moe(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - std::vector get_kv_layer_dims() const override; - std::vector get_kv_layer_heads() const override; - void compact_kv_cache() override; - - size_t forward_from_embeddings(CactusGraph* gb, size_t hidden, const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend, bool use_cache); - size_t forward_from_embeddings(CactusGraph* gb, size_t hidden, size_t pli_hidden_source, - const std::vector& pli_tokens, size_t seq_len, - ComputeBackend backend, bool use_cache); - size_t build_pli_combined_from_tokens(CactusGraph* gb, size_t hidden, - const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend); - -private: - size_t forward_split(const std::vector& tokens, bool use_cache); - - std::pair build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input); - - void set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens); - - size_t build_pli_combined(CactusGraph* gb, size_t hidden, size_t pli_embed, - size_t seq_len, ComputeBackend backend); - - size_t build_per_layer_input(CactusGraph* gb, size_t hidden, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend) const; - - bool is_global_layer(uint32_t idx) const; - size_t apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, - float rope_freq, size_t position_offset); - size_t apply_transformer_layer(CactusGraph* gb, size_t hidden, size_t pli, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t pos_offset); - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - size_t embed_tokens_per_layer; - size_t per_layer_model_proj; - size_t per_layer_proj_norm; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - size_t per_layer_gate; - size_t per_layer_proj; - size_t post_per_layer_norm; - size_t layer_scalar; - - std::vector moe_w1_experts; - std::vector moe_w3_experts; - std::vector moe_w2_experts; - size_t moe_per_expert_scale = 0; - size_t router_proj = 0; - size_t router_scale = 0; - size_t post_ffn_norm_1 = 0; - size_t pre_ffn_norm_2 = 0; - size_t post_ffn_norm_2 = 0; - }; - - std::vector layers; - } weight_nodes_; - - uint32_t first_shared_layer_ = 0; - std::vector kv_share_map_; - std::vector shared_k_nodes_; - std::vector shared_v_nodes_; - - std::vector<__fp16> v_norm_ones_weight_; - size_t v_norm_ones_node_ = 0; - size_t v_norm_ones_global_node_ = 0; -}; - -class Gemma4VisionModel : public Model { - friend class Gemma4MmModel; -public: - struct PreprocessedImage { - std::vector pixel_values; - size_t height; - size_t width; - size_t patch_height; - size_t patch_width; - size_t num_patches; - }; - - Gemma4VisionModel(); - explicit Gemma4VisionModel(const Config& config); - ~Gemma4VisionModel() override = default; - - PreprocessedImage preprocess_image(const std::string& image_path); - size_t forward_vision(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend); - size_t build_vision_projector(CactusGraph* gb, size_t vision_features, ComputeBackend backend); - -protected: - size_t forward(const std::vector&, bool) override { - throw std::runtime_error("Gemma4VisionModel: use forward_vision() instead"); - } - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4VisionModel: build_attention unused"); - } - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Gemma4VisionModel: build_mlp unused"); - } - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4VisionModel: build_transformer_block unused"); - } - void load_weights_to_graph(CactusGraph* gb) override; - -private: - size_t build_vision_patch_embedding(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend); - size_t build_vision_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend); - size_t build_vision_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend); - size_t build_vision_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend); - size_t build_vision_pooler(CactusGraph* gb, size_t hidden, const PreprocessedImage& img, ComputeBackend backend); - std::pair build_2d_rope_nodes(CactusGraph* gb, const PreprocessedImage& img, size_t max_patches); - size_t build_padding_mask(CactusGraph* gb, size_t num_real, size_t max_patches); - - struct VisionWeightNodes { - size_t patch_input_proj = 0; - size_t position_table = 0; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t attn_q_norm = 0; - size_t attn_k_norm = 0; - size_t input_layernorm = 0; - size_t post_attention_layernorm = 0; - size_t pre_feedforward_layernorm = 0; - size_t post_feedforward_layernorm = 0; - size_t mlp_gate_proj = 0; - size_t mlp_up_proj = 0; - size_t mlp_down_proj = 0; - size_t layer_scalar = 0; - }; - - std::vector layers; - size_t embed_vision_proj = 0; - size_t post_proj_norm = 0; - } vision_weights_; - - std::vector<__fp16> vision_v_norm_ones_; - size_t vision_v_norm_ones_node_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool disable_npu_ = false; - - friend bool ::test_gemma4_vision(bool); -}; - -class Gemma4AudioModel : public Model { - friend class Gemma4MmModel; -public: - Gemma4AudioModel(); - explicit Gemma4AudioModel(const Config& config); - ~Gemma4AudioModel() override = default; - - struct ConformerContext { - size_t timing_fp16 = 0; - size_t front_pad = 0; - size_t back_pad = 0; - size_t seq_len = 0; - }; - - size_t forward_audio(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend); - - size_t build_audio_projector(CactusGraph* gb, size_t audio_features, ComputeBackend backend); - - size_t build_sscp(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend); - - ConformerContext build_conformer_context(CactusGraph* gb, size_t sscp_output); - - size_t build_conformer_ffw(CactusGraph* gb, size_t input, uint32_t layer_idx, - bool is_end, ComputeBackend backend); - size_t build_conformer_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend); - size_t build_conformer_lconv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend); - size_t build_conformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend); - - struct AudioWeightNodes { - size_t sscp_conv0_weight = 0; - size_t sscp_conv0_norm = 0; - size_t sscp_conv1_weight = 0; - size_t sscp_conv1_norm = 0; - size_t sscp_input_proj = 0; - - struct ClipBounds { - float in_min = -1e10f, in_max = 1e10f; - float out_min = -1e10f, out_max = 1e10f; - }; - - struct ConformerLayerWeights { - size_t ffw_start_1 = 0, ffw_start_2 = 0; - ClipBounds ffw_start_1_clip, ffw_start_2_clip; - size_t ffw_start_pre_norm = 0, ffw_start_post_norm = 0; - size_t attn_q = 0, attn_k = 0, attn_v = 0; - ClipBounds attn_q_clip, attn_k_clip, attn_v_clip; - size_t attn_per_dim_scale = 0; - size_t attn_rel_pos_proj = 0; - size_t attn_post = 0; - ClipBounds attn_post_clip; - size_t attn_pre_norm = 0, attn_post_norm = 0; - size_t lconv_start = 0, lconv_depthwise = 0, lconv_end = 0; - ClipBounds lconv_start_clip, lconv_end_clip; - size_t lconv_pre_norm = 0, lconv_conv_norm = 0; - size_t ffw_end_1 = 0, ffw_end_2 = 0; - ClipBounds ffw_end_1_clip, ffw_end_2_clip; - size_t ffw_end_pre_norm = 0, ffw_end_post_norm = 0; - size_t block_norm = 0; - }; - std::vector layers; - - size_t output_proj = 0; - size_t output_proj_bias = 0; - - size_t embed_audio_proj = 0; - } audio_weights_; - - std::vector<__fp16> audio_proj_norm_ones_; - size_t audio_proj_norm_ones_node_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool disable_npu_ = false; - std::vector<__fp16> npu_audio_input_scratch_; - std::vector<__fp16> npu_audio_output_scratch_; - std::vector<__fp16> npu_audio_reorder_scratch_; - - friend bool ::test_gemma4_audio(bool); - -protected: - size_t forward(const std::vector&, bool) override { - throw std::runtime_error("Gemma4AudioModel: use forward_audio() instead"); - } - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4AudioModel: build_attention unused"); - } - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Gemma4AudioModel: build_mlp unused"); - } - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4AudioModel: build_transformer_block unused"); - } - void load_weights_to_graph(CactusGraph* gb) override; -}; - -class Gemma4MmModel : public Model { -public: - Gemma4MmModel(); - explicit Gemma4MmModel(const Config& config); - ~Gemma4MmModel() override = default; - - bool init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - - uint32_t decode(const std::vector& tokens, - float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f) override; - - void prefill(const std::vector& tokens, size_t chunk_size = 256, - const std::string& profile_file = "") override; - - void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file = "") override; - - uint32_t decode_with_images( - const std::vector& tokens, - const std::vector& image_paths, - float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f) override; - - uint32_t decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - - void reset_cache() override; - std::vector get_image_embeddings(const std::string& image_path) override; - std::vector get_audio_embeddings(const std::vector& audio_features) override; - void compact_kv_cache() override; - void remove_thinking_tokens(const std::vector>& ranges) override; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override; - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct ForwardResult { - size_t final_hidden_node; - size_t seq_len; - }; - - ForwardResult forward_multimodal(CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend, bool use_cache); - - uint32_t decode_multimodal(const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty); - -public: - struct MultimodalInputs { - size_t hidden_node = 0; - size_t pli_hidden_source_node = 0; - std::vector pli_tokens; - size_t seq_len = 0; - }; - - const Gemma4VisionModel& vision_encoder() const { return vision_encoder_; } - Gemma4VisionModel& vision_encoder() { return vision_encoder_; } - const Gemma4AudioModel& audio_encoder() const { return audio_encoder_; } - Gemma4AudioModel& audio_encoder() { return audio_encoder_; } - const Gemma4Model& language_model() const { return language_model_; } - Gemma4Model& language_model() { return language_model_; } - MultimodalInputs build_multimodal_inputs( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend); - -private: - Gemma4VisionModel vision_encoder_; - Gemma4AudioModel audio_encoder_; - Gemma4Model language_model_; - - bool prefill_completed_ = false; - size_t last_token_count_ = 0; -}; - -} -} diff --git a/cactus-transpile/models/gemma4/model_gemma4_audio.cpp b/cactus-transpile/models/gemma4/model_gemma4_audio.cpp deleted file mode 100644 index c1829558c..000000000 --- a/cactus-transpile/models/gemma4/model_gemma4_audio.cpp +++ /dev/null @@ -1,777 +0,0 @@ -#include "model_gemma4.h" -#include "../../graph/graph.h" -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -static const float INV_LN2 = 1.0f / std::log(2.0f); -static const float GEMMA4_AUDIO_K_SCALE = std::log(1.0f + std::exp(1.0f)) / std::log(2.0f); - -static size_t shape_elements(const std::vector& shape) { - if (shape.empty()) return 0; - size_t total = 1; - for (int d : shape) { - if (d <= 0) return 0; - total *= static_cast(d); - } - return total; -} - -static size_t infer_npu_audio_max_frames(const std::vector& shape, size_t mel_bins) { - if (shape.size() == 2) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - if (s1 == mel_bins) return s0; - if (s0 == mel_bins) return s1; - } else if (shape.size() == 3) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - size_t s2 = static_cast(shape[2]); - if (s0 == 1 && s2 == mel_bins) return s1; // [1, T, F] - if (s0 == 1 && s1 == mel_bins) return s2; // [1, F, T] - if (s1 == mel_bins && s2 == 1) return s0; // [T, F, 1] - if (s0 == mel_bins && s2 == 1) return s1; // [F, T, 1] - } else if (shape.size() == 4) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - size_t s2 = static_cast(shape[2]); - size_t s3 = static_cast(shape[3]); - if (s0 == 1 && s1 == 1 && s3 == mel_bins) return s2; // [1, 1, T, F] - if (s0 == 1 && s1 == 1 && s2 == mel_bins) return s3; // [1, 1, F, T] - if (s0 == 1 && s1 == mel_bins && s3 == 1) return s2; // [1, F, T, 1] - if (s0 == 1 && s2 == mel_bins && s3 == 1) return s1; // [1, T, F, 1] - } - return 0; -} - -static bool pack_gemma4_audio_for_npu(const std::vector& mel_features, - size_t frames, - size_t mel_bins, - const std::vector& input_shape, - std::vector<__fp16>& packed) { - if (input_shape.empty()) return false; - const size_t total = shape_elements(input_shape); - if (total == 0) return false; - packed.resize(total); - std::fill(packed.begin(), packed.end(), static_cast<__fp16>(0.0f)); - - auto tm = [&](size_t t, size_t m) -> __fp16 { - return static_cast<__fp16>(mel_features[t * mel_bins + m]); - }; - - if (input_shape.size() == 4) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - const size_t s3 = static_cast(input_shape[3]); - if (s0 != 1) return false; - - if (s1 == 1 && s2 >= frames && s3 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(t * s3) + m] = tm(t, m); - return true; - } - if (s1 == 1 && s2 == mel_bins && s3 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(m * s3) + t] = tm(t, m); - return true; - } - if (s1 >= frames && s2 == mel_bins && s3 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[((t * s2 + m) * s3)] = tm(t, m); - return true; - } - if (s1 == mel_bins && s2 >= frames && s3 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[((m * s2 + t) * s3)] = tm(t, m); - return true; - } - return false; - } - - if (input_shape.size() == 3) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - - if (s0 == 1 && s1 >= frames && s2 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[t * s2 + m] = tm(t, m); - return true; - } - if (s0 == 1 && s1 == mel_bins && s2 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[m * s2 + t] = tm(t, m); - return true; - } - if (s0 >= frames && s1 == mel_bins && s2 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(t * s1 + m) * s2] = tm(t, m); - return true; - } - if (s0 == mel_bins && s1 >= frames && s2 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(m * s1 + t) * s2] = tm(t, m); - return true; - } - return false; - } - - if (input_shape.size() == 2) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - - if (s0 >= frames && s1 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[t * s1 + m] = tm(t, m); - return true; - } - if (s0 == mel_bins && s1 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[m * s1 + t] = tm(t, m); - return true; - } - return false; - } - - return false; -} - -struct NPUAudioOutputLayout { - std::vector dims; - size_t hidden_axis = SIZE_MAX; - size_t hidden_dim = 0; - size_t time_steps = 0; -}; - -static bool infer_npu_audio_output_layout(const std::vector& output_shape, - size_t elements_written, - size_t expected_hidden_dim, - NPUAudioOutputLayout& layout) { - if (elements_written == 0) return false; - - layout = NPUAudioOutputLayout{}; - for (int d : output_shape) { - if (d > 0) layout.dims.push_back(static_cast(d)); - } - - if (layout.dims.empty()) { - if (expected_hidden_dim == 0 || (elements_written % expected_hidden_dim) != 0) { - return false; - } - layout.dims = {elements_written / expected_hidden_dim, expected_hidden_dim}; - } - - const size_t rank = layout.dims.size(); - size_t hidden_axis = SIZE_MAX; - if (expected_hidden_dim > 0) { - if (layout.dims[rank - 1] == expected_hidden_dim) { - hidden_axis = rank - 1; - } else if (rank >= 2 && layout.dims[rank - 2] == expected_hidden_dim) { - hidden_axis = rank - 2; - } else { - for (size_t i = 0; i < rank; ++i) { - if (layout.dims[i] == expected_hidden_dim) { - hidden_axis = i; - break; - } - } - } - } - - if (hidden_axis == SIZE_MAX) { - hidden_axis = rank - 1; - } - - const size_t hidden_dim = layout.dims[hidden_axis]; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) { - return false; - } - - layout.hidden_axis = hidden_axis; - layout.hidden_dim = hidden_dim; - layout.time_steps = elements_written / hidden_dim; - return layout.time_steps > 0; -} - -static bool materialize_npu_audio_time_major(const __fp16* src, - const NPUAudioOutputLayout& layout, - size_t time_steps, - std::vector<__fp16>& dst) { - if (!src || layout.dims.empty() || layout.hidden_dim == 0 || layout.time_steps == 0) { - return false; - } - if (layout.hidden_axis >= layout.dims.size() || time_steps == 0 || time_steps > layout.time_steps) { - return false; - } - - const size_t rank = layout.dims.size(); - if (layout.hidden_axis == rank - 1) { - // Already [time, hidden] in row-major flattening. - return false; - } - - std::vector strides(rank, 1); - for (size_t i = rank; i-- > 1;) { - strides[i - 1] = strides[i] * layout.dims[i]; - } - - std::vector non_hidden_axes; - non_hidden_axes.reserve(rank - 1); - for (size_t i = 0; i < rank; ++i) { - if (i != layout.hidden_axis) non_hidden_axes.push_back(i); - } - - dst.assign(time_steps * layout.hidden_dim, static_cast<__fp16>(0.0f)); - - for (size_t t = 0; t < time_steps; ++t) { - size_t rem = t; - size_t base_offset = 0; - for (size_t i = non_hidden_axes.size(); i-- > 0;) { - size_t axis = non_hidden_axes[i]; - size_t dim = layout.dims[axis]; - size_t coord = rem % dim; - rem /= dim; - base_offset += coord * strides[axis]; - } - - for (size_t h = 0; h < layout.hidden_dim; ++h) { - size_t src_index = base_offset + h * strides[layout.hidden_axis]; - dst[t * layout.hidden_dim + h] = src[src_index]; - } - } - - return true; -} - -static size_t graph_clamp(CactusGraph* gb, size_t x, float lo, float hi) { - if (lo >= hi || std::isinf(lo) || std::isinf(hi)) return x; - x = gb->scalar_add(x, -lo); - x = gb->relu(x); - x = gb->scalar_add(x, lo); - x = gb->scalar_multiply(gb->scalar_add(x, -hi), -1.0f); - x = gb->relu(x); - x = gb->scalar_add(gb->scalar_multiply(x, -1.0f), hi); - return x; -} - -static size_t clipped_matmul(CactusGraph* gb, size_t input, size_t weight, - float in_min, float in_max, float out_min, float out_max, - ComputeBackend backend) { - size_t clamped_in = graph_clamp(gb, input, in_min, in_max); - size_t result = gb->matmul(clamped_in, weight, true, backend); - return graph_clamp(gb, result, out_min, out_max); -} - -static float read_scalar_weight(CactusGraph* gb, const std::string& path) { - size_t node = gb->mmap_weights(path); - const auto& buf = gb->get_output_buffer(node); - if (buf.precision == Precision::FP16) - return static_cast(*buf.data_as<__fp16>()); - if (buf.precision == Precision::FP32) - return *buf.data_as(); - return 0.0f; -} - -static Gemma4AudioModel::AudioWeightNodes::ClipBounds load_clip_bounds(CactusGraph* gb, const std::string& prefix) { - Gemma4AudioModel::AudioWeightNodes::ClipBounds cb{0, 0, 0, 0}; - std::string in_min_path = prefix + "_input_min.weights"; - if (!std::filesystem::exists(in_min_path)) - return cb; - cb.in_min = read_scalar_weight(gb, in_min_path); - cb.in_max = read_scalar_weight(gb, prefix + "_input_max.weights"); - cb.out_min = read_scalar_weight(gb, prefix + "_output_min.weights"); - cb.out_max = read_scalar_weight(gb, prefix + "_output_max.weights"); - return cb; -} - -static void get_conv2d_output_hw(const CactusGraph* gb, size_t node, size_t& h, size_t& w) { - const auto& buf = gb->get_output_buffer(node); - h = buf.shape[2]; - w = buf.shape[3]; -} - -Gemma4AudioModel::Gemma4AudioModel() : Model() {} -Gemma4AudioModel::Gemma4AudioModel(const Config& config) : Model(config) {} - -void Gemma4AudioModel::load_weights_to_graph(CactusGraph* gb) { - auto resolve = [&](const std::string& name) -> std::string { - return model_folder_path_ + "/" + name; - }; - auto resolve_existing = [&](const std::string& preferred, - const std::string& legacy = std::string()) -> std::string { - std::string preferred_path = resolve(preferred); - if (std::filesystem::exists(preferred_path)) { - return preferred_path; - } - if (!legacy.empty()) { - std::string legacy_path = resolve(legacy); - if (std::filesystem::exists(legacy_path)) { - return legacy_path; - } - } - return preferred_path; - }; - auto load_weight = [&](const std::string& preferred, - const std::string& legacy = std::string()) -> size_t { - return gb->mmap_weights(resolve_existing(preferred, legacy)); - }; - auto load_clip_from_weight = [&](const std::string& preferred, - const std::string& legacy = std::string()) - -> Gemma4AudioModel::AudioWeightNodes::ClipBounds { - std::string weight_path = resolve_existing(preferred, legacy); - static const std::string suffix = ".weights"; - if (weight_path.size() > suffix.size() && - weight_path.compare(weight_path.size() - suffix.size(), suffix.size(), suffix) == 0) { - return load_clip_bounds(gb, weight_path.substr(0, weight_path.size() - suffix.size())); - } - return Gemma4AudioModel::AudioWeightNodes::ClipBounds{0, 0, 0, 0}; - }; - - if (!disable_npu_ && npu::is_npu_available()) { - std::string npu_path = model_folder_path_ + "/audio_encoder.mlpackage"; - if (std::filesystem::exists(npu_path)) { - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - npu_encoder_->preallocate(input_shape, "mel", ""); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - CACTUS_LOG_WARN("npu", "[gemma4-audio] found audio_encoder.mlpackage but failed to enable NPU audio encoder; using CPU"); - } - } else { - CACTUS_LOG_WARN("npu", "[gemma4-audio] audio_encoder.mlpackage not found; using CPU audio encoder"); - } - } else if (!disable_npu_) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] NPU backend unavailable on this device; using CPU audio encoder"); - } - - if (!use_npu_encoder_) { - audio_weights_.sscp_conv0_weight = load_weight( - "audio_subsample_conv_projection_conv_0_conv.weights", - "audio_subsample_conv_projection_layer0_conv.weights"); - audio_weights_.sscp_conv0_norm = load_weight( - "audio_subsample_conv_projection_conv_0_norm.weights", - "audio_subsample_conv_projection_layer0_norm.weights"); - audio_weights_.sscp_conv1_weight = load_weight( - "audio_subsample_conv_projection_conv_1_conv.weights", - "audio_subsample_conv_projection_layer1_conv.weights"); - audio_weights_.sscp_conv1_norm = load_weight( - "audio_subsample_conv_projection_conv_1_norm.weights", - "audio_subsample_conv_projection_layer1_norm.weights"); - audio_weights_.sscp_input_proj = load_weight( - "audio_subsample_conv_projection_input_proj.weights", - "audio_subsample_conv_projection_input_proj_linear.weights"); - - audio_weights_.layers.resize(config_.audio_num_layers); - for (uint32_t i = 0; i < config_.audio_num_layers; i++) { - auto& layer = audio_weights_.layers[i]; - - auto layer_weight = [&](const std::string& preferred_suffix, - const std::string& legacy_suffix = std::string()) -> size_t { - std::string legacy = legacy_suffix.empty() ? preferred_suffix : legacy_suffix; - return load_weight( - "audio_conformer_" + std::to_string(i) + "_" + preferred_suffix, - "audio_layers_" + std::to_string(i) + "_" + legacy); - }; - auto layer_clip = [&](const std::string& preferred_suffix, - const std::string& legacy_suffix = std::string()) - -> Gemma4AudioModel::AudioWeightNodes::ClipBounds { - std::string legacy = legacy_suffix.empty() ? preferred_suffix : legacy_suffix; - return load_clip_from_weight( - "audio_conformer_" + std::to_string(i) + "_" + preferred_suffix, - "audio_layers_" + std::to_string(i) + "_" + legacy); - }; - - layer.ffw_start_1 = layer_weight("ffw_layer_start_ffw_layer_1.weights", "feed_forward1_ffw_layer_1.weights"); - layer.ffw_start_2 = layer_weight("ffw_layer_start_ffw_layer_2.weights", "feed_forward1_ffw_layer_2.weights"); - layer.ffw_start_1_clip = layer_clip("ffw_layer_start_ffw_layer_1.weights", "feed_forward1_ffw_layer_1.weights"); - layer.ffw_start_2_clip = layer_clip("ffw_layer_start_ffw_layer_2.weights", "feed_forward1_ffw_layer_2.weights"); - layer.ffw_start_pre_norm = layer_weight("ffw_layer_start_pre_layer_norm.weights", "feed_forward1_pre_layer_norm.weights"); - layer.ffw_start_post_norm = layer_weight("ffw_layer_start_post_layer_norm.weights", "feed_forward1_post_layer_norm.weights"); - - layer.attn_q = layer_weight("attention_attn_q_proj.weights", "self_attn_q_proj.weights"); - layer.attn_k = layer_weight("attention_attn_k_proj.weights", "self_attn_k_proj.weights"); - layer.attn_v = layer_weight("attention_attn_v_proj.weights", "self_attn_v_proj.weights"); - layer.attn_q_clip = layer_clip("attention_attn_q_proj.weights", "self_attn_q_proj.weights"); - layer.attn_k_clip = layer_clip("attention_attn_k_proj.weights", "self_attn_k_proj.weights"); - layer.attn_v_clip = layer_clip("attention_attn_v_proj.weights", "self_attn_v_proj.weights"); - layer.attn_per_dim_scale = layer_weight("attention_attn_per_dim_scale.weights", "self_attn_per_dim_scale.weights"); - layer.attn_rel_pos_proj = layer_weight( - "attention_attn_relative_position_embedding_pos_proj.weights", - "self_attn_relative_k_proj.weights"); - layer.attn_post = layer_weight("attention_post.weights", "self_attn_post.weights"); - layer.attn_post_clip = layer_clip("attention_post.weights", "self_attn_post.weights"); - layer.attn_pre_norm = layer_weight("attention_pre_attn_norm.weights", "norm_pre_attn.weights"); - layer.attn_post_norm = layer_weight("attention_post_norm.weights", "norm_post_attn.weights"); - - layer.lconv_start = layer_weight("lconv1d_linear_start.weights"); - layer.lconv_depthwise = layer_weight("lconv1d_depthwise_conv1d.weights"); - layer.lconv_end = layer_weight("lconv1d_linear_end.weights"); - layer.lconv_start_clip = layer_clip("lconv1d_linear_start.weights"); - layer.lconv_end_clip = layer_clip("lconv1d_linear_end.weights"); - layer.lconv_pre_norm = layer_weight("lconv1d_pre_layer_norm.weights"); - layer.lconv_conv_norm = layer_weight("lconv1d_conv_norm.weights"); - - layer.ffw_end_1 = layer_weight("ffw_layer_end_ffw_layer_1.weights", "feed_forward2_ffw_layer_1.weights"); - layer.ffw_end_2 = layer_weight("ffw_layer_end_ffw_layer_2.weights", "feed_forward2_ffw_layer_2.weights"); - layer.ffw_end_1_clip = layer_clip("ffw_layer_end_ffw_layer_1.weights", "feed_forward2_ffw_layer_1.weights"); - layer.ffw_end_2_clip = layer_clip("ffw_layer_end_ffw_layer_2.weights", "feed_forward2_ffw_layer_2.weights"); - layer.ffw_end_pre_norm = layer_weight("ffw_layer_end_pre_layer_norm.weights", "feed_forward2_pre_layer_norm.weights"); - layer.ffw_end_post_norm = layer_weight("ffw_layer_end_post_layer_norm.weights", "feed_forward2_post_layer_norm.weights"); - - layer.block_norm = layer_weight("norm.weights", "norm_out.weights"); - } - - audio_weights_.output_proj = load_weight("audio_output_proj.weights"); - audio_weights_.output_proj_bias = load_weight("audio_output_proj.bias"); - } - audio_weights_.embed_audio_proj = load_weight("embed_audio_proj.weights"); - - size_t proj_out_dim = config_.audio_output_proj_dims > 0 - ? static_cast(config_.audio_output_proj_dims) - : static_cast(config_.audio_hidden_dim); - audio_proj_norm_ones_.assign(proj_out_dim, static_cast<__fp16>(1.0f)); - audio_proj_norm_ones_node_ = gb->input({proj_out_dim}, Precision::FP16); - gb->set_external_input(audio_proj_norm_ones_node_, audio_proj_norm_ones_.data(), Precision::FP16); - - output_weight_node_id_ = 0; -} - -size_t Gemma4AudioModel::build_sscp(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend) { - size_t mel_bins = config_.audio_input_feat_size; - size_t conv0_ch = config_.audio_sscp_conv0_channels; - size_t conv1_ch = config_.audio_sscp_conv1_channels; - - size_t mel_input = gb->input({num_frames, mel_bins}, Precision::FP32); - gb->set_input(mel_input, mel_features.data(), Precision::FP32); - size_t mel_fp16 = gb->precision_cast(mel_input, Precision::FP16); - - size_t x = gb->reshape(mel_fp16, {1, 1, num_frames, mel_bins}); - - x = gb->conv2d_k3s2p1(x, audio_weights_.sscp_conv0_weight); - size_t h1, w1; - get_conv2d_output_hw(gb, x, h1, w1); - - x = gb->reshape(x, {conv0_ch, h1, w1}); - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h1 * w1, conv0_ch}); - x = gb->layernorm(x, audio_weights_.sscp_conv0_norm, config_.audio_sscp_conv_eps); - x = gb->reshape(x, {h1, w1, conv0_ch}); - x = gb->transposeN(x, {2, 0, 1}); - x = gb->relu(x); - x = gb->reshape(x, {1, conv0_ch, h1, w1}); - - x = gb->conv2d_k3s2p1(x, audio_weights_.sscp_conv1_weight); - size_t h2, w2; - get_conv2d_output_hw(gb, x, h2, w2); - - x = gb->reshape(x, {conv1_ch, h2, w2}); - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h2 * w2, conv1_ch}); - x = gb->layernorm(x, audio_weights_.sscp_conv1_norm, config_.audio_sscp_conv_eps); - x = gb->reshape(x, {h2, w2, conv1_ch}); - x = gb->transposeN(x, {2, 0, 1}); - x = gb->relu(x); - - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h2, w2 * conv1_ch}); - - return gb->matmul(x, audio_weights_.sscp_input_proj, true, backend); -} - -static std::vector compute_timing_signal(size_t num_positions, size_t hidden_dim, size_t max_past) { - size_t num_timescales = hidden_dim / 2; - float log_ts_inc = std::log(1.0e4f) / std::max(static_cast(num_timescales) - 1, 1); - - std::vector signal(num_positions * hidden_dim); - for (size_t p = 0; p < num_positions; p++) { - float pos = static_cast(max_past) - static_cast(p); - for (size_t i = 0; i < num_timescales; i++) { - float scaled = pos * std::exp(-static_cast(i) * log_ts_inc); - signal[p * hidden_dim + i] = std::sin(scaled); - signal[p * hidden_dim + num_timescales + i] = std::cos(scaled); - } - } - return signal; -} - -static size_t create_zero_pad_node(CactusGraph* gb, size_t time_len, size_t num_heads, size_t head_dim) { - if (time_len == 0) return 0; - std::vector<__fp16> zeros(time_len * num_heads * head_dim, static_cast<__fp16>(0.0f)); - size_t node = gb->input({time_len, num_heads, head_dim}, Precision::FP16); - gb->set_input(node, zeros.data(), Precision::FP16); - return node; -} - -Gemma4AudioModel::ConformerContext Gemma4AudioModel::build_conformer_context( - CactusGraph* gb, size_t sscp_output) { - - size_t num_heads = config_.audio_num_heads; - size_t head_dim = config_.audio_head_dim; - size_t hidden_dim = num_heads * head_dim; - size_t max_past = config_.audio_context_left > 0 ? config_.audio_context_left - 1 : 0; - size_t num_positions = max_past + config_.audio_context_right + 1; - size_t seq_len = gb->get_output_buffer(sscp_output).shape[0]; - - auto timing_signal = compute_timing_signal(num_positions, hidden_dim, max_past); - size_t timing_input = gb->input({num_positions, hidden_dim}, Precision::FP32); - gb->set_input(timing_input, timing_signal.data(), Precision::FP32); - size_t timing_fp16 = gb->precision_cast(timing_input, Precision::FP16); - - size_t front_pad_len = (seq_len >= num_positions) ? seq_len - num_positions : 0; - size_t back_pad_len = seq_len > 0 ? seq_len - 1 : 0; - - return { - timing_fp16, - create_zero_pad_node(gb, front_pad_len, num_heads, head_dim), - create_zero_pad_node(gb, back_pad_len, num_heads, head_dim), - seq_len - }; -} - -size_t Gemma4AudioModel::build_conformer_ffw(CactusGraph* gb, size_t input, uint32_t layer_idx, - bool is_end, ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float residual_weight = config_.audio_residual_weight; - float eps = config_.audio_rms_norm_eps; - - size_t residual = input; - - size_t pre_norm = is_end ? layer.ffw_end_pre_norm : layer.ffw_start_pre_norm; - size_t post_norm = is_end ? layer.ffw_end_post_norm : layer.ffw_start_post_norm; - size_t w1 = is_end ? layer.ffw_end_1 : layer.ffw_start_1; - size_t w2 = is_end ? layer.ffw_end_2 : layer.ffw_start_2; - - auto& c1 = is_end ? layer.ffw_end_1_clip : layer.ffw_start_1_clip; - auto& c2 = is_end ? layer.ffw_end_2_clip : layer.ffw_start_2_clip; - - size_t x = gb->rms_norm(input, pre_norm, eps); - - x = clipped_matmul(gb, x, w1, c1.in_min, c1.in_max, c1.out_min, c1.out_max, backend); - x = gb->silu(x); - x = clipped_matmul(gb, x, w2, c2.in_min, c2.in_max, c2.out_min, c2.out_max, backend); - - x = gb->rms_norm(x, post_norm, eps); - - x = gb->scalar_multiply(x, residual_weight); - return gb->add(residual, x); -} - -size_t Gemma4AudioModel::build_conformer_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float eps = config_.audio_rms_norm_eps; - size_t num_heads = config_.audio_num_heads; - size_t head_dim = config_.audio_head_dim; - size_t hidden_dim = num_heads * head_dim; - float logit_cap = config_.audio_logit_cap; - size_t max_past = config_.audio_context_left > 0 ? config_.audio_context_left - 1 : 0; - size_t num_positions = max_past + config_.audio_context_right + 1; - size_t seq_len = ctx.seq_len; - - size_t residual = input; - - size_t x = gb->rms_norm(input, layer.attn_pre_norm, eps); - - size_t q = clipped_matmul(gb, x, layer.attn_q, layer.attn_q_clip.in_min, layer.attn_q_clip.in_max, layer.attn_q_clip.out_min, layer.attn_q_clip.out_max, backend); - size_t k = clipped_matmul(gb, x, layer.attn_k, layer.attn_k_clip.in_min, layer.attn_k_clip.in_max, layer.attn_k_clip.out_min, layer.attn_k_clip.out_max, backend); - size_t v = clipped_matmul(gb, x, layer.attn_v, layer.attn_v_clip.in_min, layer.attn_v_clip.in_max, layer.attn_v_clip.out_min, layer.attn_v_clip.out_max, backend); - - float q_scale = (1.0f / std::sqrt(static_cast(head_dim))) * INV_LN2; - size_t q_dim_scale_fp16 = gb->precision_cast(layer.attn_per_dim_scale, Precision::FP16); - size_t q_dim_scale = gb->scalar_exp(q_dim_scale_fp16); - q_dim_scale = gb->scalar_add(q_dim_scale, 1.0f); - q_dim_scale = gb->scalar_log(q_dim_scale); - q_dim_scale = gb->scalar_multiply(q_dim_scale, q_scale); - - size_t q_flat = gb->reshape(q, {seq_len * num_heads, head_dim}); - q_flat = gb->multiply(q_flat, q_dim_scale); - q = gb->reshape(q_flat, {seq_len, hidden_dim}); - - k = gb->scalar_multiply(k, GEMMA4_AUDIO_K_SCALE); - - size_t sin_emb = gb->matmul(ctx.timing_fp16, layer.attn_rel_pos_proj, true, backend); - size_t sin_emb_3d = gb->reshape(sin_emb, {num_positions, num_heads, head_dim}); - size_t R = 2 * seq_len - 1; - - size_t rel_key; - if (seq_len >= num_positions) { - rel_key = (ctx.front_pad != 0) ? gb->concat(ctx.front_pad, sin_emb_3d, 0) : sin_emb_3d; - if (ctx.back_pad != 0) - rel_key = gb->concat(rel_key, ctx.back_pad, 0); - } else { - size_t start_idx = num_positions - seq_len; - size_t sin_emb_sliced = gb->slice(sin_emb_3d, 0, start_idx, seq_len); - rel_key = (ctx.back_pad != 0) ? gb->concat(sin_emb_sliced, ctx.back_pad, 0) : sin_emb_sliced; - } - - size_t rel_key_4d = gb->reshape(rel_key, {1, R, num_heads, head_dim}); - size_t q4 = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - size_t rel_bias = gb->rel_pos_bias(q4, rel_key_4d, 1.0f); - - size_t k4 = gb->reshape(k, {1, seq_len, num_heads, head_dim}); - size_t v4 = gb->reshape(v, {1, seq_len, num_heads, head_dim}); - - size_t attn = gb->attention_masked(q4, k4, v4, rel_bias, 1.0f, - true, backend, true, 0, max_past, logit_cap); - - size_t attn_out = gb->reshape(attn, {seq_len, hidden_dim}); - - size_t out = clipped_matmul(gb, attn_out, layer.attn_post, layer.attn_post_clip.in_min, layer.attn_post_clip.in_max, layer.attn_post_clip.out_min, layer.attn_post_clip.out_max, backend); - out = gb->rms_norm(out, layer.attn_post_norm, eps); - - return gb->add(residual, out); -} - -size_t Gemma4AudioModel::build_conformer_lconv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float eps = config_.audio_rms_norm_eps; - size_t kernel_size = config_.audio_conf_conv_kernel_size; - size_t hidden_dim = config_.audio_hidden_dim; - - size_t residual = input; - - size_t x = gb->rms_norm(input, layer.lconv_pre_norm, eps); - - x = clipped_matmul(gb, x, layer.lconv_start, layer.lconv_start_clip.in_min, layer.lconv_start_clip.in_max, layer.lconv_start_clip.out_min, layer.lconv_start_clip.out_max, backend); - - x = gb->glu(x, -1); - - const auto& x_buf = gb->get_output_buffer(x); - size_t seq_len = x_buf.shape[0]; - x = gb->reshape(x, {1, seq_len, hidden_dim}); - - x = gb->conv1d_causal(x, layer.lconv_depthwise, kernel_size); - - x = gb->reshape(x, {seq_len, hidden_dim}); - - x = gb->rms_norm(x, layer.lconv_conv_norm, eps); - x = gb->silu(x); - - x = clipped_matmul(gb, x, layer.lconv_end, layer.lconv_end_clip.in_min, layer.lconv_end_clip.in_max, layer.lconv_end_clip.out_min, layer.lconv_end_clip.out_max, backend); - - return gb->add(x, residual); -} - -size_t Gemma4AudioModel::build_conformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend) { - hidden = build_conformer_ffw(gb, hidden, layer_idx, false, backend); - hidden = build_conformer_attention(gb, hidden, layer_idx, ctx, backend); - hidden = build_conformer_lconv1d(gb, hidden, layer_idx, backend); - hidden = build_conformer_ffw(gb, hidden, layer_idx, true, backend); - return gb->rms_norm(hidden, audio_weights_.layers[layer_idx].block_norm, config_.audio_rms_norm_eps); -} - -size_t Gemma4AudioModel::forward_audio(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend) { - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector npu_input_shape = npu_encoder_->get_input_shape(); - size_t mel_bins = static_cast(config_.audio_input_feat_size); - size_t max_npu_frames = infer_npu_audio_max_frames(npu_input_shape, mel_bins); - size_t copy_frames = max_npu_frames > 0 ? std::min(num_frames, max_npu_frames) : num_frames; - size_t input_values = copy_frames * mel_bins; - - if (mel_features.size() < input_values) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] insufficient mel input values; falling back to CPU audio encoder"); - } else if (!pack_gemma4_audio_for_npu(mel_features, copy_frames, mel_bins, npu_input_shape, - npu_audio_input_scratch_)) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] unsupported NPU input shape; falling back to CPU audio encoder"); - } else { - size_t t1 = (copy_frames + 1) / 2; - size_t t2 = (t1 + 1) / 2; - size_t expected_out_dim = config_.audio_output_proj_dims > 0 - ? static_cast(config_.audio_output_proj_dims) - : static_cast(config_.audio_hidden_dim); - - std::vector out_shape = npu_encoder_->get_output_shape(); - size_t out_elements = npu_encoder_->get_output_buffer_size(); - if (out_elements == 0) out_elements = shape_elements(out_shape); - if (out_elements == 0) out_elements = std::max(1, t2 * expected_out_dim); - - if (npu_audio_output_scratch_.size() < out_elements) - npu_audio_output_scratch_.resize(out_elements); - - size_t written = npu_encoder_->encode( - npu_audio_input_scratch_.data(), npu_audio_output_scratch_.data(), npu_input_shape, "mel", ""); - - NPUAudioOutputLayout out_layout; - if (written > 0 && - infer_npu_audio_output_layout(out_shape, written, expected_out_dim, out_layout) && - out_layout.hidden_dim == expected_out_dim) { - const __fp16* src = npu_audio_output_scratch_.data(); - __fp16* cached_output = npu_encoder_->get_output_buffer(); - size_t cached_count = npu_encoder_->get_output_buffer_size(); - size_t required = out_layout.time_steps * out_layout.hidden_dim; - if (cached_output != nullptr && cached_count >= required) { - src = cached_output; - } - - size_t actual_time = std::min(t2, out_layout.time_steps); - if (actual_time > 0) { - const __fp16* final_src = src; - if (out_layout.hidden_axis != (out_layout.dims.size() - 1)) { - if (!materialize_npu_audio_time_major(src, out_layout, actual_time, - npu_audio_reorder_scratch_)) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] failed to reorder NPU output layout; falling back to CPU audio encoder"); - } else { - final_src = npu_audio_reorder_scratch_.data(); - } - } - - if (final_src != nullptr) { - size_t hidden = gb->input({actual_time, out_layout.hidden_dim}, Precision::FP16); - gb->set_input(hidden, final_src, Precision::FP16); - return hidden; - } - } - } - - if (written > 0 && out_layout.hidden_dim != expected_out_dim) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] NPU output dim mismatch; falling back to CPU audio encoder"); - } else { - CACTUS_LOG_WARN("npu", "audio encode failed, falling back to CPU"); - } - } - } - - size_t hidden = build_sscp(gb, mel_features, num_frames, backend); - ConformerContext ctx = build_conformer_context(gb, hidden); - for (uint32_t i = 0; i < config_.audio_num_layers; i++) - hidden = build_conformer_block(gb, hidden, i, ctx, backend); - - if (config_.audio_output_proj_dims > 0) { - hidden = gb->matmul(hidden, audio_weights_.output_proj, true, backend); - hidden = gb->add(hidden, audio_weights_.output_proj_bias); - } - - return hidden; -} - -size_t Gemma4AudioModel::build_audio_projector(CactusGraph* gb, size_t audio_features, ComputeBackend backend) { - size_t normed = gb->rms_norm(audio_features, audio_proj_norm_ones_node_, config_.audio_rms_norm_eps); - size_t projected = gb->matmul(normed, audio_weights_.embed_audio_proj, true, backend); - return gb->scalar_multiply(projected, 1.0f / 16.0f); -} - -} -} diff --git a/cactus-transpile/models/gemma4/model_gemma4_mm.cpp b/cactus-transpile/models/gemma4/model_gemma4_mm.cpp deleted file mode 100644 index 55be7c98d..000000000 --- a/cactus-transpile/models/gemma4/model_gemma4_mm.cpp +++ /dev/null @@ -1,459 +0,0 @@ -#include "model_gemma4.h" -#include "../../graph/graph.h" -#include -#include -#include - -namespace cactus { -namespace engine { - -Gemma4MmModel::Gemma4MmModel() : Model() { - config_.model_type = Config::ModelType::GEMMA4; -} - -Gemma4MmModel::Gemma4MmModel(const Config& config) - : Model(config), vision_encoder_(config), audio_encoder_(config), language_model_(config) {} - -bool Gemma4MmModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!Model::init(model_folder, context_size, system_prompt, false)) - return false; - - auto* shared_graph = static_cast(graph_handle_); - if (!shared_graph) - throw std::runtime_error("Shared graph was not initialized for Gemma4MmModel"); - - bool has_vision = config_.vision_num_layers > 0 || config_.vision_embed_dim > 0; - bool has_audio = config_.audio_num_layers > 0 || config_.audio_hidden_dim > 0; - - if (has_vision) { - if (!vision_encoder_.init(shared_graph, model_folder, context_size, "", false)) - throw std::runtime_error("Failed to initialize vision encoder"); - } - - if (has_audio) { - if (!audio_encoder_.init(shared_graph, model_folder, context_size, "", false)) - throw std::runtime_error("Failed to initialize audio encoder"); - } - - if (!language_model_.init(shared_graph, model_folder, context_size, system_prompt, false)) - throw std::runtime_error("Failed to initialize language model"); - - output_weight_node_id_ = language_model_.output_weight_node_id_; - - if (do_warmup) { - std::vector warmup_tokens = {2}; - language_model_.forward(warmup_tokens); - auto* gb2 = static_cast(language_model_.graph_handle_); - gb2->execute(); - language_model_.reset_cache(); - } - - return true; -} - -void Gemma4MmModel::reset_cache() { - Model::reset_cache(); - language_model_.reset_cache(); - prefill_completed_ = false; - last_token_count_ = 0; -} - -void Gemma4MmModel::compact_kv_cache() { - language_model_.compact_kv_cache(); -} - -void Gemma4MmModel::remove_thinking_tokens(const std::vector>& ranges) { - language_model_.remove_thinking_tokens(ranges); -} - -void Gemma4MmModel::load_weights_to_graph(CactusGraph*) { - output_weight_node_id_ = 0; -} - -Gemma4MmModel::ForwardResult Gemma4MmModel::forward_multimodal( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend, bool use_cache) { - - auto inputs = build_multimodal_inputs( - gb, tokens, image_paths, audio_features, audio_num_frames, backend); - - size_t final_hidden = language_model_.forward_from_embeddings( - gb, - inputs.hidden_node, - inputs.pli_hidden_source_node, - inputs.pli_tokens, - inputs.seq_len, - backend, - use_cache); - - return ForwardResult{final_hidden, inputs.seq_len}; -} - -Gemma4MmModel::MultimodalInputs Gemma4MmModel::build_multimodal_inputs( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend) { - - size_t vision_soft_node = 0; - size_t num_vision_soft_tokens = 0; - size_t audio_soft_node = 0; - size_t num_audio_soft_tokens = 0; - - if (!image_paths.empty()) { - auto preprocessed = vision_encoder_.preprocess_image(image_paths[0]); - size_t vision_output = vision_encoder_.forward_vision(gb, preprocessed, backend); - vision_soft_node = vision_encoder_.build_vision_projector(gb, vision_output, backend); - uint32_t k = config_.vision_pooling_kernel_size; - num_vision_soft_tokens = (preprocessed.patch_width / k) * (preprocessed.patch_height / k); - } - - if (audio_features && !audio_features->empty()) { - size_t audio_output = audio_encoder_.forward_audio(gb, *audio_features, audio_num_frames, backend); - audio_soft_node = audio_encoder_.build_audio_projector(gb, audio_output, backend); - const auto& audio_buf = gb->get_output_buffer(audio_soft_node); - num_audio_soft_tokens = audio_buf.shape[0]; - } - - uint32_t image_token_id = config_.image_token_id; - uint32_t audio_token_id = config_.audio_token_id; - uint32_t pad_token_id = config_.pad_token_id; - - std::vector sequence_nodes; - std::vector current_text; - std::vector pli_tokens; - size_t total_seq_len = 0; - size_t vision_offset = 0; - size_t audio_offset = 0; - - auto flush_text = [&]() { - if (current_text.empty()) return; - size_t seg_len = current_text.size(); - size_t input_node = gb->input({seg_len}, Precision::FP32); - - auto hidden = gb->scalar_multiply( - gb->embedding(language_model_.embedding_node_id_, input_node), - std::sqrt(static_cast(config_.hidden_dim))); - - std::vector input_data(seg_len); - for (size_t i = 0; i < seg_len; i++) - input_data[i] = static_cast(current_text[i]); - gb->set_input(input_node, input_data.data(), Precision::FP32); - - sequence_nodes.push_back(hidden); - for (auto t : current_text) - pli_tokens.push_back(t); - total_seq_len += seg_len; - current_text.clear(); - }; - - auto append_soft_region = [&](size_t soft_node, size_t& soft_offset, size_t total_soft_tokens, - size_t placeholder_count) { - size_t to_insert = std::min(placeholder_count, total_soft_tokens - soft_offset); - if (to_insert > 0) { - sequence_nodes.push_back(gb->slice(soft_node, 0, soft_offset, to_insert)); - for (size_t j = 0; j < to_insert; j++) - pli_tokens.push_back(pad_token_id); - total_seq_len += to_insert; - soft_offset += to_insert; - } - }; - - auto flush_vision_region = [&](size_t placeholder_count) { - append_soft_region(vision_soft_node, vision_offset, num_vision_soft_tokens, placeholder_count); - }; - - auto flush_audio_region = [&](size_t placeholder_count) { - append_soft_region(audio_soft_node, audio_offset, num_audio_soft_tokens, placeholder_count); - }; - - bool in_image_region = false; - bool in_audio_region = false; - size_t region_count = 0; - - for (size_t i = 0; i < tokens.size(); i++) { - uint32_t tok = tokens[i]; - bool is_vision_token = (tok == image_token_id && image_token_id != 0); - bool is_audio_token = (tok == audio_token_id && audio_token_id != 0); - - if (is_vision_token) { - if (in_audio_region) { - flush_audio_region(region_count); - in_audio_region = false; - } - if (!in_image_region) { - flush_text(); - in_image_region = true; - region_count = 0; - } - region_count++; - } else if (is_audio_token) { - if (in_image_region) { - flush_vision_region(region_count); - in_image_region = false; - } - if (!in_audio_region) { - flush_text(); - in_audio_region = true; - region_count = 0; - } - region_count++; - } else { - if (in_image_region) { - flush_vision_region(region_count); - in_image_region = false; - } - if (in_audio_region) { - flush_audio_region(region_count); - in_audio_region = false; - } - current_text.push_back(tok); - } - } - - if (in_image_region) - flush_vision_region(region_count); - if (in_audio_region) - flush_audio_region(region_count); - flush_text(); - - if (sequence_nodes.empty()) - throw std::runtime_error("No embedding nodes built"); - - size_t merged = sequence_nodes[0]; - for (size_t i = 1; i < sequence_nodes.size(); i++) - merged = gb->concat(merged, sequence_nodes[i], 0); - - return MultimodalInputs{ - .hidden_node = merged, - .pli_hidden_source_node = merged, - .pli_tokens = std::move(pli_tokens), - .seq_len = total_seq_len, - }; -} - -uint32_t Gemma4MmModel::decode_multimodal( - const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - - bool has_media = !image_paths.empty() || (audio_features && !audio_features->empty()); - - if (!has_media) { - prefill_completed_ = false; - last_token_count_ = tokens.size(); - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); - } - - if (temperature < 0) temperature = config_.default_temperature; - if (top_p < 0) top_p = config_.default_top_p; - if (top_k == 0) top_k = config_.default_top_k; - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - bool cache_empty = language_model_.kv_cache_.is_empty(); - bool need_prefill = cache_empty || !prefill_completed_; - - if (!need_prefill && tokens.size() <= last_token_count_) { - reset_cache(); - need_prefill = true; - } - - size_t seq_len_for_updates = 0; - size_t final_hidden_node = 0; - - if (need_prefill) { - auto result = forward_multimodal(gb, tokens, image_paths, audio_features, - audio_num_frames, backend, true); - final_hidden_node = result.final_hidden_node; - seq_len_for_updates = result.seq_len; - prefill_completed_ = true; - last_token_count_ = tokens.size(); - } else { - size_t delta = tokens.size() - last_token_count_; - if (delta == 0) delta = 1; - std::vector incremental_tokens(tokens.end() - delta, tokens.end()); - final_hidden_node = language_model_.forward(incremental_tokens, true); - seq_len_for_updates = incremental_tokens.size(); - last_token_count_ = tokens.size(); - } - - auto last_hidden = gb->index(final_hidden_node, seq_len_for_updates - 1, 0); - const auto& last_buf = gb->get_output_buffer(last_hidden); - last_hidden = gb->reshape(last_hidden, {1, last_buf.shape[0]}); - - auto logits_node = gb->matmul(last_hidden, language_model_.output_weight_node_id_, true, backend); - - if (config_.final_logit_softcapping > 0.0f) { - float inv_cap = 1.0f / config_.final_logit_softcapping; - logits_node = gb->scalar_multiply(logits_node, inv_cap); - logits_node = gb->tanh(logits_node); - logits_node = gb->scalar_multiply(logits_node, config_.final_logit_softcapping); - } - - size_t sampled_token = - language_model_.sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty, nullptr); - - if (!profile_file.empty()) - gb->execute(profile_file); - else - gb->execute(); - - compute_entropy(gb, logits_node, out_entropy); - - language_model_.post_execute_updates(gb, seq_len_for_updates); - language_model_.update_kv_cache(gb, seq_len_for_updates); - - auto* output_ptr = gb->get_output(sampled_token); - uint32_t result_token = *static_cast(output_ptr); - language_model_.record_sampled_token(result_token); - return result_token; -} - -size_t Gemma4MmModel::forward(const std::vector& tokens, bool use_cache) { - return language_model_.forward(tokens, use_cache); -} - -uint32_t Gemma4MmModel::decode(const std::vector& tokens, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - prefill_completed_ = false; - last_token_count_ = tokens.size(); - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -void Gemma4MmModel::prefill(const std::vector& tokens, size_t chunk_size, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - prefill_completed_ = false; - last_token_count_ = tokens.size(); - language_model_.prefill(tokens, chunk_size, profile_file); -} - -void Gemma4MmModel::prefill_with_images(const std::vector& tokens, - const std::vector& image_paths, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - - if (image_paths.empty()) { - prefill(tokens, get_prefill_chunk_size(), profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - auto result = forward_multimodal(gb, tokens, image_paths, nullptr, 0, backend, true); - - if (!profile_file.empty()) - gb->execute(profile_file); - else - gb->execute(); - - language_model_.post_execute_updates(gb, result.seq_len); - language_model_.update_kv_cache(gb, result.seq_len); - - prefill_completed_ = true; - last_token_count_ = tokens.size(); -} - -uint32_t Gemma4MmModel::decode_with_images( - const std::vector& tokens, const std::vector& image_paths, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - return decode_multimodal(tokens, image_paths, nullptr, 0, - temperature, top_p, top_k, profile_file, out_entropy, - min_p, repetition_penalty); -} - -uint32_t Gemma4MmModel::decode_with_audio( - const std::vector& tokens, const std::vector& audio_features, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty, - float* /*out_token_time_start*/, float* /*out_token_time_end*/) { - size_t num_frames = audio_features.size() / config_.audio_input_feat_size; - std::vector empty_images; - return decode_multimodal(tokens, empty_images, &audio_features, num_frames, - temperature, top_p, top_k, profile_file, out_entropy, - min_p, repetition_penalty); -} - -std::vector Gemma4MmModel::get_image_embeddings(const std::string& image_path) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - auto preprocessed = vision_encoder_.preprocess_image(image_path); - size_t vision_output = vision_encoder_.forward_vision(gb, preprocessed, backend); - size_t projected = vision_encoder_.build_vision_projector(gb, vision_output, backend); - - gb->execute(); - - const auto& buf = gb->get_output_buffer(projected); - size_t total = buf.total_size; - std::vector embedding(total); - const __fp16* fp16_data = buf.data_as<__fp16>(); - for (size_t i = 0; i < total; i++) - embedding[i] = static_cast(fp16_data[i]); - return embedding; -} - -std::vector Gemma4MmModel::get_audio_embeddings(const std::vector& audio_features) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - size_t num_frames = audio_features.size() / config_.audio_input_feat_size; - size_t audio_output = audio_encoder_.forward_audio(gb, audio_features, num_frames, backend); - size_t projected = audio_encoder_.build_audio_projector(gb, audio_output, backend); - - gb->execute(); - - const auto& buf = gb->get_output_buffer(projected); - size_t total = buf.total_size; - std::vector embedding(total); - const __fp16* fp16_data = buf.data_as<__fp16>(); - for (size_t i = 0; i < total; i++) - embedding[i] = static_cast(fp16_data[i]); - return embedding; -} - -size_t Gemma4MmModel::build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_attention should not be called directly on Gemma4MmModel"); -} - -size_t Gemma4MmModel::build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const { - throw std::runtime_error("build_mlp should not be called directly on Gemma4MmModel"); -} - -size_t Gemma4MmModel::build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_transformer_block should not be called directly on Gemma4MmModel"); -} - -} -} diff --git a/cactus-transpile/models/gemma4/model_gemma4_vision.cpp b/cactus-transpile/models/gemma4/model_gemma4_vision.cpp deleted file mode 100644 index 6d495bd4c..000000000 --- a/cactus-transpile/models/gemma4/model_gemma4_vision.cpp +++ /dev/null @@ -1,540 +0,0 @@ -#include "model_gemma4.h" -#include "../../graph/graph.h" -#include -#include -#include -#include - -extern "C" { - #include "../../../libs/stb/stb_image.h" - #include "../../../libs/stb/stb_image_resize2.h" -} - -namespace cactus { -namespace engine { - -static std::pair, std::vector> compute_2d_rope_tables( - const Gemma4VisionModel::PreprocessedImage& img, size_t max_patches, size_t head_dim, float theta) { - - size_t half_dim = head_dim / 2; - size_t freq_per_dim = half_dim / 2; - - std::vector cos_table(max_patches * head_dim, 1.0f); - std::vector sin_table(max_patches * head_dim, 0.0f); - - for (size_t py = 0; py < img.patch_height; py++) { - for (size_t px = 0; px < img.patch_width; px++) { - size_t patch_idx = py * img.patch_width + px; - - for (size_t i = 0; i < freq_per_dim; i++) { - float inv_freq = 1.0f / std::pow(theta, 2.0f * i / static_cast(half_dim)); - - float x_angle = static_cast(px) * inv_freq; - cos_table[patch_idx * head_dim + i] = std::cos(x_angle); - cos_table[patch_idx * head_dim + freq_per_dim + i] = std::cos(x_angle); - sin_table[patch_idx * head_dim + i] = std::sin(x_angle); - sin_table[patch_idx * head_dim + freq_per_dim + i] = std::sin(x_angle); - - float y_angle = static_cast(py) * inv_freq; - cos_table[patch_idx * head_dim + half_dim + i] = std::cos(y_angle); - cos_table[patch_idx * head_dim + half_dim + freq_per_dim + i] = std::cos(y_angle); - sin_table[patch_idx * head_dim + half_dim + i] = std::sin(y_angle); - sin_table[patch_idx * head_dim + half_dim + freq_per_dim + i] = std::sin(y_angle); - } - } - } - - return {cos_table, sin_table}; -} - -static std::pair, std::vector<__fp16>> compute_npu_rope_tables( - const Gemma4VisionModel::PreprocessedImage& img, size_t max_patches, size_t head_dim, float theta) { - - auto [cos_f32, sin_f32] = compute_2d_rope_tables(img, max_patches, head_dim, theta); - - std::vector<__fp16> cos_full(max_patches * head_dim); - std::vector<__fp16> sin_signed(max_patches * head_dim); - - for (size_t p = 0; p < max_patches; p++) { - size_t base = p * head_dim; - for (size_t d = 0; d < head_dim; d++) { - cos_full[base + d] = static_cast<__fp16>(cos_f32[base + d]); - sin_signed[base + d] = static_cast<__fp16>(sin_f32[base + d]); - } - } - - return {cos_full, sin_signed}; -} - -Gemma4VisionModel::Gemma4VisionModel() : Model() {} -Gemma4VisionModel::Gemma4VisionModel(const Config& config) : Model(config) {} - -void Gemma4VisionModel::load_weights_to_graph(CactusGraph* gb) { - auto resolve = [&](const std::string& name) -> std::string { - return model_folder_path_ + "/" + name; - }; - - if (!disable_npu_ && npu::is_npu_available()) { - std::string npu_path = model_folder_path_ + "/vision_encoder.mlpackage"; - if (std::filesystem::exists(npu_path)) { - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (input_shape.empty()) { - input_shape = {static_cast(config_.vision_default_output_length * - config_.vision_pooling_kernel_size * config_.vision_pooling_kernel_size), - static_cast(config_.vision_embed_dim)}; - } - npu_encoder_->preallocate(input_shape, "hidden_states", "output"); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - CACTUS_LOG_WARN("npu", "[gemma4-vision] found vision_encoder.mlpackage but failed to enable NPU vision encoder; using CPU"); - } - } else { - CACTUS_LOG_WARN("npu", "[gemma4-vision] vision_encoder.mlpackage not found; using CPU vision encoder"); - } - } else if (!disable_npu_) { - CACTUS_LOG_WARN("npu", "[gemma4-vision] NPU backend unavailable on this device; using CPU vision encoder"); - } - - vision_weights_.patch_input_proj = gb->mmap_weights(resolve("vision_patch_embedder_input_proj.weights")); - vision_weights_.position_table = gb->mmap_weights(resolve("vision_patch_embedder_position_embedding_table.weights")); - - if (!use_npu_encoder_) { - vision_weights_.layers.resize(config_.vision_num_layers); - for (uint32_t i = 0; i < config_.vision_num_layers; i++) { - auto& layer = vision_weights_.layers[i]; - std::string prefix = resolve("vision_encoder_layers_" + std::to_string(i) + "_"); - - layer.attn_q_weight = gb->mmap_weights(prefix + "self_attn_q_proj.weights"); - layer.attn_k_weight = gb->mmap_weights(prefix + "self_attn_k_proj.weights"); - layer.attn_v_weight = gb->mmap_weights(prefix + "self_attn_v_proj.weights"); - layer.attn_output_weight = gb->mmap_weights(prefix + "self_attn_o_proj.weights"); - layer.attn_q_norm = gb->mmap_weights(prefix + "self_attn_q_norm.weights"); - layer.attn_k_norm = gb->mmap_weights(prefix + "self_attn_k_norm.weights"); - layer.input_layernorm = gb->mmap_weights(prefix + "input_layernorm.weights"); - layer.post_attention_layernorm = gb->mmap_weights(prefix + "post_attention_layernorm.weights"); - layer.pre_feedforward_layernorm = gb->mmap_weights(prefix + "pre_feedforward_layernorm.weights"); - layer.post_feedforward_layernorm = gb->mmap_weights(prefix + "post_feedforward_layernorm.weights"); - layer.mlp_gate_proj = gb->mmap_weights(prefix + "mlp_gate_proj.weights"); - layer.mlp_up_proj = gb->mmap_weights(prefix + "mlp_up_proj.weights"); - layer.mlp_down_proj = gb->mmap_weights(prefix + "mlp_down_proj.weights"); - layer.layer_scalar = std::filesystem::exists(prefix + "layer_scalar.weights") ? gb->mmap_weights(prefix + "layer_scalar.weights") : 0; - } - } - - vision_weights_.embed_vision_proj = gb->mmap_weights(resolve("embed_vision_proj.weights")); - - size_t vision_head_dim = config_.vision_head_dim; - vision_v_norm_ones_.assign(vision_head_dim, static_cast<__fp16>(1.0f)); - vision_v_norm_ones_node_ = gb->input({vision_head_dim}, Precision::FP16); - gb->set_external_input(vision_v_norm_ones_node_, vision_v_norm_ones_.data(), Precision::FP16); - - vision_weights_.post_proj_norm = gb->mmap_weights(resolve("embed_vision_post_proj_norm.weights")); - - output_weight_node_id_ = 0; -} - -Gemma4VisionModel::PreprocessedImage Gemma4VisionModel::preprocess_image(const std::string& image_path) { - int w, h, channels; - unsigned char* data = stbi_load(image_path.c_str(), &w, &h, &channels, 3); - if (!data) - throw std::runtime_error("Failed to load image: " + image_path); - - uint32_t patch_size = config_.vision_patch_size; - uint32_t pooling_k = config_.vision_pooling_kernel_size; - uint32_t max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - uint32_t side_mult = pooling_k * patch_size; - - float total_px = static_cast(h) * static_cast(w); - float target_px = static_cast(max_patches) * static_cast(patch_size * patch_size); - float factor = std::sqrt(target_px / total_px); - - int target_h = static_cast(std::floor(factor * h / side_mult)) * side_mult; - int target_w = static_cast(std::floor(factor * w / side_mult)) * side_mult; - - if (target_h == 0) target_h = side_mult; - if (target_w == 0) target_w = side_mult; - - std::vector resized_data; - unsigned char* src = data; - if (target_h != h || target_w != w) { - resized_data.resize(target_h * target_w * 3); - stbir_resize_uint8_linear(data, w, h, 0, resized_data.data(), target_w, target_h, 0, - static_cast(3)); - src = resized_data.data(); - } - - size_t ph = target_h / patch_size; - size_t pw = target_w / patch_size; - size_t num_patches = ph * pw; - - std::vector pixels(3 * target_h * target_w); - for (int y = 0; y < target_h; y++) { - for (int x = 0; x < target_w; x++) { - size_t src_idx = (y * target_w + x) * 3; - for (int c = 0; c < 3; c++) { - pixels[c * target_h * target_w + y * target_w + x] = - static_cast(src[src_idx + c]) * config_.rescale_factor; - } - } - } - - stbi_image_free(data); - - return PreprocessedImage{ - std::move(pixels), - static_cast(target_h), - static_cast(target_w), - ph, pw, num_patches - }; -} - -size_t Gemma4VisionModel::build_vision_patch_embedding(CactusGraph* gb, const PreprocessedImage& img, - ComputeBackend backend) { - uint32_t patch_size = config_.vision_patch_size; - size_t patch_dim = 3 * patch_size * patch_size; - size_t num_patches = img.num_patches; - size_t hidden_size = config_.vision_embed_dim; - size_t pos_embed_size = config_.vision_position_embedding_size; - - size_t pixel_input = gb->input({3, img.height, img.width}, Precision::FP32); - gb->set_input(pixel_input, img.pixel_values.data(), Precision::FP32); - - size_t pixel_fp16 = gb->precision_cast(pixel_input, Precision::FP16); - size_t reshaped = gb->reshape(pixel_fp16, {3, img.patch_height, patch_size, img.patch_width, patch_size}); - size_t permuted = gb->transposeN(reshaped, {1, 3, 2, 4, 0}); - size_t patches = gb->reshape(permuted, {num_patches, patch_dim}); - - float mean = config_.image_mean; - float norm_std = config_.image_std; - patches = gb->scalar_multiply(gb->scalar_add(patches, -mean), 1.0f / norm_std); - - size_t projected = gb->matmul(patches, vision_weights_.patch_input_proj, true, backend); - - size_t pos_table = gb->reshape(vision_weights_.position_table, {2, pos_embed_size, hidden_size}); - - std::vector x_positions(num_patches); - std::vector y_positions(num_patches); - for (size_t py = 0; py < img.patch_height; py++) { - for (size_t px = 0; px < img.patch_width; px++) { - size_t idx = py * img.patch_width + px; - x_positions[idx] = static_cast(px); - y_positions[idx] = static_cast(py); - } - } - - size_t x_pos_input = gb->input({num_patches}, Precision::FP32); - gb->set_input(x_pos_input, x_positions.data(), Precision::FP32); - size_t y_pos_input = gb->input({num_patches}, Precision::FP32); - gb->set_input(y_pos_input, y_positions.data(), Precision::FP32); - - size_t x_table = gb->slice(pos_table, 0, 0, 1); - x_table = gb->reshape(x_table, {pos_embed_size, hidden_size}); - size_t y_table = gb->slice(pos_table, 0, 1, 1); - y_table = gb->reshape(y_table, {pos_embed_size, hidden_size}); - - size_t x_embed = gb->embedding(x_table, x_pos_input); - size_t y_embed = gb->embedding(y_table, y_pos_input); - size_t pos_embed = gb->add(x_embed, y_embed); - - size_t pos_embed_fp16 = gb->precision_cast(pos_embed, Precision::FP16); - - return gb->add(projected, pos_embed_fp16); -} - -std::pair Gemma4VisionModel::build_2d_rope_nodes( - CactusGraph* gb, const PreprocessedImage& img, size_t max_patches) { - - size_t head_dim = config_.vision_head_dim; - auto rope_tables = compute_2d_rope_tables(img, max_patches, head_dim, config_.vision_rope_theta); - auto& cos_table = rope_tables.first, sin_table = rope_tables.second; - - size_t cos_input = gb->input({max_patches, head_dim}, Precision::FP32); - gb->set_input(cos_input, cos_table.data(), Precision::FP32); - size_t cos_node = gb->precision_cast(cos_input, Precision::FP16); - cos_node = gb->reshape(cos_node, {1, max_patches, 1, head_dim}); - - size_t sin_input = gb->input({max_patches, head_dim}, Precision::FP32); - gb->set_input(sin_input, sin_table.data(), Precision::FP32); - size_t sin_node = gb->precision_cast(sin_input, Precision::FP16); - sin_node = gb->reshape(sin_node, {1, max_patches, 1, head_dim}); - - return {cos_node, sin_node}; -} - -size_t Gemma4VisionModel::build_padding_mask(CactusGraph* gb, size_t num_real, size_t max_patches) { - if (num_real >= max_patches) - return 0; - - std::vector mask(max_patches * max_patches, 0.0f); - for (size_t q = 0; q < num_real; q++) - for (size_t k = 0; k < num_real; k++) - mask[q * max_patches + k] = 1.0f; - size_t mask_input = gb->input({max_patches, max_patches}, Precision::FP32); - gb->set_input(mask_input, mask.data(), Precision::FP32); - size_t mask_node = gb->precision_cast(mask_input, Precision::FP16); - return gb->reshape(mask_node, {1, max_patches, max_patches}); -} - -size_t Gemma4VisionModel::build_vision_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - size_t head_dim = config_.vision_head_dim; - size_t num_heads = config_.vision_attention_heads; - size_t kv_heads = config_.vision_kv_heads; - - const auto& input_buf = gb->get_output_buffer(input); - size_t seq_len = input_buf.shape[0]; - - auto q = gb->matmul(input, layer.attn_q_weight, true, backend); - q = gb->reshape(q, {seq_len * num_heads, head_dim}); - q = gb->rms_norm(q, layer.attn_q_norm, config_.layer_norm_eps); - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - - auto k = gb->matmul(input, layer.attn_k_weight, true, backend); - k = gb->reshape(k, {seq_len * kv_heads, head_dim}); - k = gb->rms_norm(k, layer.attn_k_norm, config_.layer_norm_eps); - k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); - - auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); - auto v = gb->rms_norm(gb->reshape(v_proj, {seq_len * kv_heads, head_dim}), - vision_v_norm_ones_node_, config_.layer_norm_eps); - v = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); - - size_t half_dim = head_dim / 2; - - auto apply_2d_rope = [&](size_t tensor) -> size_t { - auto first_half = gb->slice(tensor, 3, 0, half_dim); - auto second_half = gb->slice(tensor, 3, half_dim, half_dim); - - auto rot_first_a = gb->slice(first_half, 3, 0, half_dim / 2); - auto rot_first_b = gb->slice(first_half, 3, half_dim / 2, half_dim / 2); - auto neg_b = gb->scalar_multiply(rot_first_b, -1.0f); - - size_t cos_x = gb->slice(cos_node, 3, 0, half_dim); - size_t sin_x = gb->slice(sin_node, 3, 0, half_dim); - - auto first_rotated = gb->add( - gb->multiply(first_half, cos_x), - gb->multiply(gb->concat(neg_b, rot_first_a, 3), sin_x)); - - auto rot_second_a = gb->slice(second_half, 3, 0, half_dim / 2); - auto rot_second_b = gb->slice(second_half, 3, half_dim / 2, half_dim / 2); - auto neg_sb = gb->scalar_multiply(rot_second_b, -1.0f); - - size_t cos_y = gb->slice(cos_node, 3, half_dim, half_dim); - size_t sin_y = gb->slice(sin_node, 3, half_dim, half_dim); - - auto second_rotated = gb->add( - gb->multiply(second_half, cos_y), - gb->multiply(gb->concat(neg_sb, rot_second_a, 3), sin_y)); - - return gb->concat(first_rotated, second_rotated, 3); - }; - - auto q_rot = apply_2d_rope(q); - auto k_rot = apply_2d_rope(k); - - size_t attn; - if (attn_mask_node != 0) - attn = gb->attention_masked(q_rot, k_rot, v, attn_mask_node, 1.f, false, backend); - else - attn = gb->attention(q_rot, k_rot, v, 1.f, false, backend); - - return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), - layer.attn_output_weight, true, backend); -} - -size_t Gemma4VisionModel::build_vision_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - auto gate = gb->gelu(gb->matmul(input, layer.mlp_gate_proj, true, backend)); - auto up = gb->matmul(input, layer.mlp_up_proj, true, backend); - return gb->matmul(gb->multiply(gate, up), layer.mlp_down_proj, true, backend); -} - -size_t Gemma4VisionModel::build_vision_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - - auto normed = gb->rms_norm(hidden, layer.input_layernorm, config_.layer_norm_eps); - auto attn_raw = build_vision_attention(gb, normed, layer_idx, cos_node, sin_node, attn_mask_node, backend); - auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm, config_.layer_norm_eps); - auto residual = gb->add(hidden, attn); - - auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm, config_.layer_norm_eps); - auto mlp_raw = build_vision_mlp(gb, pre_mlp, layer_idx, backend); - auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm, config_.layer_norm_eps); - - auto out = gb->add(residual, mlp); - - if (layer.layer_scalar != 0) - out = gb->multiply(out, layer.layer_scalar); - - return out; -} - -size_t Gemma4VisionModel::build_vision_pooler(CactusGraph* gb, size_t hidden, const PreprocessedImage& img, - ComputeBackend backend) { - size_t k = config_.vision_pooling_kernel_size; - size_t output_length = config_.vision_default_output_length; - size_t num_patches = img.num_patches; - size_t k_squared = k * k; - - if (num_patches == output_length) - return hidden; - - size_t max_x = img.patch_width; - size_t max_y = img.patch_height; - - std::vector pool_weights(num_patches * output_length, 0.0f); - std::vector valid_bins(output_length, false); - for (size_t py = 0; py < max_y; py++) { - for (size_t px = 0; px < max_x; px++) { - size_t patch_idx = py * max_x + px; - size_t kx = px / k; - size_t ky = py / k; - size_t kernel_idx = kx + (max_x / k) * ky; - if (kernel_idx < output_length) { - pool_weights[kernel_idx * num_patches + patch_idx] = 1.0f / static_cast(k_squared); - valid_bins[kernel_idx] = true; - } - } - } - - size_t weights_node = gb->input({output_length, num_patches}, Precision::FP32); - gb->set_input(weights_node, pool_weights.data(), Precision::FP32); - size_t weights_fp16 = gb->precision_cast(weights_node, Precision::FP16); - - size_t pooled = gb->matmul(weights_fp16, hidden, false, backend); - - size_t valid_count = 0; - for (bool v : valid_bins) if (v) valid_count++; - - if (valid_count == output_length) - return pooled; - - std::vector valid_slices; - for (size_t i = 0; i < output_length; i++) { - if (valid_bins[i]) { - auto slice = gb->slice(pooled, 0, i, 1); - valid_slices.push_back(slice); - } - } - - size_t stripped = valid_slices[0]; - for (size_t i = 1; i < valid_slices.size(); i++) - stripped = gb->concat(stripped, valid_slices[i], 0); - - return stripped; -} - -size_t Gemma4VisionModel::forward_vision(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend) { - size_t hidden = build_vision_patch_embedding(gb, img, backend); - - size_t num_real = img.num_patches; - size_t hidden_size = config_.vision_embed_dim; - uint32_t pooling_k = config_.vision_pooling_kernel_size; - size_t max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - bool can_use_npu_path = use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available(); - if (can_use_npu_path) { - std::vector npu_input_shape = npu_encoder_->get_input_shape(); - if (npu_input_shape.size() >= 2) { - size_t a = static_cast(npu_input_shape[0]); - size_t b = static_cast(npu_input_shape[1]); - if (b == hidden_size) { - max_patches = a; - } else if (a == hidden_size) { - max_patches = b; - } else if (a > 0) { - max_patches = a; - } - } - } - if (num_real > max_patches) { - if (can_use_npu_path) { - CACTUS_LOG_WARN("npu", "[gemma4-vision] image has more patches than NPU encoder supports; falling back to CPU vision encoder"); - can_use_npu_path = false; - max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - } - } - size_t num_padding = max_patches - num_real; - - if (num_padding > 0) { - std::vector<__fp16> zeros(num_padding * hidden_size, static_cast<__fp16>(0.0f)); - size_t pad_node = gb->input({num_padding, hidden_size}, Precision::FP16); - gb->set_input(pad_node, zeros.data(), Precision::FP16); - hidden = gb->concat(hidden, pad_node, 0); - } - - auto [cos_node, sin_node] = build_2d_rope_nodes(gb, img, max_patches); - size_t attn_mask_node = build_padding_mask(gb, num_real, max_patches); - - if (can_use_npu_path) { - gb->execute(); - - const auto& h_buf = gb->get_output_buffer(hidden); - if (h_buf.precision != Precision::FP16) - throw std::runtime_error("[gemma4-vision] expected FP16 hidden output for NPU path"); - std::vector<__fp16> hidden_fp16(max_patches * hidden_size); - memcpy(hidden_fp16.data(), h_buf.data_as<__fp16>(), max_patches * hidden_size * sizeof(__fp16)); - - size_t head_dim = config_.vision_head_dim; - auto [cos_fp16, sin_fp16] = compute_npu_rope_tables(img, max_patches, head_dim, config_.vision_rope_theta); - - std::vector<__fp16> additive_mask(max_patches * max_patches, static_cast<__fp16>(0.0f)); - if (num_padding > 0) { - for (size_t q = 0; q < max_patches; q++) - for (size_t k = 0; k < max_patches; k++) - if (q >= num_real || k >= num_real) - additive_mask[q * max_patches + k] = static_cast<__fp16>(-65504.0f); - } - - std::vector inputs = { - {"hidden_states", hidden_fp16.data(), {static_cast(max_patches), static_cast(hidden_size)}}, - {"cos_full", cos_fp16.data(), {static_cast(max_patches), 1, static_cast(head_dim)}}, - {"sin_signed", sin_fp16.data(), {static_cast(max_patches), 1, static_cast(head_dim)}}, - {"attention_mask", additive_mask.data(), {1, static_cast(max_patches), static_cast(max_patches)}}, - }; - - std::vector out_shape = npu_encoder_->get_output_shape(); - size_t out_elements = 1; - for (int d : out_shape) out_elements *= d; - size_t required = max_patches * hidden_size; - if (out_elements < required) - out_elements = required; - - std::vector<__fp16> npu_output(out_elements); - size_t written = npu_encoder_->encode_multimodal_input(inputs, npu_output.data(), "output"); - - if (written >= required) { - hidden = gb->input({max_patches, hidden_size}, Precision::FP16); - gb->set_input(hidden, npu_output.data(), Precision::FP16); - } else { - CACTUS_LOG_WARN("npu", "[gemma4-vision] encode_multimodal_input returned insufficient output, falling back to CPU"); - for (uint32_t i = 0; i < config_.vision_num_layers; i++) - hidden = build_vision_transformer_block(gb, hidden, i, cos_node, sin_node, attn_mask_node, backend); - } - } else { - for (uint32_t i = 0; i < config_.vision_num_layers; i++) - hidden = build_vision_transformer_block(gb, hidden, i, cos_node, sin_node, attn_mask_node, backend); - } - - if (num_padding > 0) - hidden = gb->slice(hidden, 0, 0, num_real); - - return build_vision_pooler(gb, hidden, img, backend); -} - -size_t Gemma4VisionModel::build_vision_projector(CactusGraph* gb, size_t vision_features, ComputeBackend backend) { - size_t projected = gb->matmul(vision_features, vision_weights_.embed_vision_proj, true, backend); - return gb->rms_norm(projected, vision_weights_.post_proj_norm, config_.layer_norm_eps); -} - -} -} diff --git a/cactus-transpile/models/model.h b/cactus-transpile/models/model.h deleted file mode 100644 index 13382cd96..000000000 --- a/cactus-transpile/models/model.h +++ /dev/null @@ -1,1773 +0,0 @@ -#pragma once - -#include "../engine/engine.h" -#include "../npu/npu.h" - -namespace cactus { -namespace engine { - - - -class QwenModel : public Model { -public: - QwenModel(); - explicit QwenModel(const Config& config); - ~QwenModel() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - }; - - std::vector layers; - } weight_nodes_; -}; - -class Qwen3p5Model : public Model { -public: - Qwen3p5Model(); - explicit Qwen3p5Model(const Config& config); - ~Qwen3p5Model() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - void reset_cache() override; - -private: - size_t build_gated_deltanet(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset); - - struct WeightNodeIDs { - size_t output_weight = 0; - size_t output_norm_weight = 0; - - enum class LayerType : uint8_t { ATTENTION, DELTANET }; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t input_layernorm_weight = 0; - size_t attn_q_norm_weight = 0; - size_t attn_k_norm_weight = 0; - size_t deltanet_qkv_weight = 0; - size_t deltanet_gate_weight = 0; - size_t deltanet_beta_weight = 0; - size_t deltanet_gate_bias = 0; - size_t deltanet_beta_bias = 0; - size_t deltanet_z_weight = 0; - size_t deltanet_conv_weight = 0; - size_t ffn_gate_weight = 0; - size_t ffn_up_weight = 0; - size_t ffn_down_weight = 0; - size_t post_attention_layernorm_weight = 0; - }; - - struct LayerEntry { - LayerType type = LayerType::ATTENTION; - LayerWeights weights; - }; - - std::vector layers; - } weight_nodes_; - - ConvCache conv_cache_; - std::vector conv_cache_state_nodes_; - bool last_forward_used_cache_ = false; - size_t deltanet_total_seq_len_ = 0; - size_t deltanet_heads_ = 0; - size_t deltanet_key_dim_ = 0; - size_t deltanet_value_dim_ = 0; - size_t deltanet_state_flat_dim_ = 0; - size_t deltanet_mixed_dim_ = 0; - size_t deltanet_conv_history_len_ = 0; - size_t deltanet_conv_flat_dim_ = 0; - size_t deltanet_cache_row_dim_ = 0; -}; - - - -class GemmaModel : public Model { -public: - GemmaModel(); - explicit GemmaModel(const Config& config); - ~GemmaModel() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - -private: - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - }; - - std::vector layers; - } weight_nodes_; -}; - -class GemmaModel3n : public Model { -public: - GemmaModel3n(); - explicit GemmaModel3n(const Config& config); - ~GemmaModel3n() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - std::vector get_kv_layer_dims() const override; - -private: - size_t forward_split(const std::vector& tokens, bool use_cache); - - size_t build_preamble(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input, size_t* streams); - void build_layer(CactusGraph* gb, uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t pos_offset, size_t pli, size_t* streams); - size_t build_output_head(CactusGraph* gb, size_t* streams, ComputeBackend backend); - void set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens); - - size_t build_laurel(CactusGraph* gb, size_t normed_input, uint32_t layer_idx, ComputeBackend backend) const; - size_t build_gaussian_topk(CactusGraph* gb, size_t input, float ppf) const; - size_t build_rms_norm_no_weight(CactusGraph* gb, size_t input, size_t num_rows, size_t row_dim) const; - size_t build_magnitude_normalize(CactusGraph* gb, size_t reference, size_t target) const; - - size_t build_altup_router_modalities(CactusGraph* gb, size_t stream0, uint32_t layer_idx, ComputeBackend backend) const; - void build_altup_predict(CactusGraph* gb, size_t modalities, uint32_t layer_idx, - const size_t* streams, size_t* predictions) const; - void build_altup_correct(CactusGraph* gb, size_t activated, size_t modalities, uint32_t layer_idx, - ComputeBackend backend, const size_t* predictions, size_t* corrected) const; - void build_per_layer_input(CactusGraph* gb, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend, size_t* streams) const; - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - size_t altup_proj_weights[3]; - size_t altup_unembed_proj_weights[3]; - size_t embed_tokens_per_layer; - size_t per_layer_model_proj; - size_t per_layer_proj_norm; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - - size_t altup_router_norm; - size_t altup_prediction_coefs; - size_t altup_correction_coefs; - size_t altup_correct_output_scale; - size_t altup_modality_router; - size_t laurel_left; - size_t laurel_right; - size_t laurel_norm; - size_t per_layer_gate; - size_t per_layer_proj; - size_t post_per_layer_norm; - }; - - std::vector layers; - } weight_nodes_; - - std::vector kv_share_map_; - std::vector shared_k_nodes_; - std::vector shared_v_nodes_; - - std::vector<__fp16> v_norm_ones_weight_; - size_t v_norm_ones_node_ = 0; -}; - -class Siglip2VisionModel : public Model { - friend class Lfm2VlModel; - -public: - struct VisionEmbeddingResult { - size_t combined_embeddings; - std::vector tile_embeddings; - }; - - Siglip2VisionModel(); - explicit Siglip2VisionModel(const Config& cfg); - ~Siglip2VisionModel() override = default; - virtual size_t forward_vision(const Siglip2Preprocessor::PreprocessedImage& preprocessed_image); - virtual size_t forward_vision(CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - std::vector get_image_embedding(const std::string& image_path); - Siglip2Preprocessor& get_preprocessor() { return preprocessor_; } - const Siglip2Preprocessor& get_preprocessor() const { return preprocessor_; } - -protected: - VisionEmbeddingResult build_vision_embeddings(CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - - size_t build_vision_transformer_layer(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - - size_t build_vision_attention(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - - size_t build_vision_mlp(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - - void load_weights_to_graph(CactusGraph* gb) override; - size_t forward(const std::vector& tokens, bool use_cache = false) override; - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - -protected: - struct VisionWeightNodeIDs { - size_t patch_embedding_weight; - size_t patch_embedding_bias; - size_t position_embedding; - size_t post_layernorm_weight; - size_t post_layernorm_bias; - - struct VisionLayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t attn_q_bias; - size_t attn_k_bias; - size_t attn_v_bias; - size_t attn_output_bias; - size_t layer_norm1_weight; - size_t layer_norm1_bias; - size_t layer_norm2_weight; - size_t layer_norm2_bias; - size_t mlp_fc1_weight; - size_t mlp_fc1_bias; - size_t mlp_fc2_weight; - size_t mlp_fc2_bias; - }; - - std::vector vision_layers; - } vision_weight_nodes_; - - Siglip2Preprocessor preprocessor_; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; -}; - - -class LFM2Model : public Model { - friend class Lfm2VlModel; - -public: - LFM2Model(); - explicit LFM2Model(const Config& config); - ~LFM2Model() override = default; - - bool is_cache_empty() const; - - bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; - bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - -protected: - using Model::forward; - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache); - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - size_t forward(CactusGraph* gb, const std::vector& tokens, ComputeBackend backend, bool use_cache = false); - size_t forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, ComputeBackend backend, bool use_cache = false); - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - void reset_cache() override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - - size_t conv_depthwise_weight; - size_t conv_in_proj_weight; - size_t conv_out_proj_weight; - - size_t input_layernorm_weight; - size_t post_attention_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - }; - - enum class LayerType : uint8_t { ATTENTION, CONV }; - - struct LayerEntry { - LayerType type; - LayerWeights weights; - }; - - std::vector layers; - } weight_nodes_; - - ConvCache conv_cache_; - std::vector conv_cache_bx_nodes_; - bool last_forward_used_cache_ = false; -}; - - -class LFM2MoEModel : public Model { -public: - LFM2MoEModel(); - explicit LFM2MoEModel(const Config& config); - ~LFM2MoEModel() override = default; - - bool is_cache_empty() const; - - bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; - bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - -protected: - using Model::forward; - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache); - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - size_t forward(CactusGraph* gb, const std::vector& tokens, ComputeBackend backend, bool use_cache = false); - size_t forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, ComputeBackend backend, bool use_cache = false); - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - void reset_cache() override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - bool is_dense_layer(uint32_t layer_idx) const; - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct ExpertWeights { - size_t w1_weight = 0; - size_t w3_weight = 0; - size_t w2_weight = 0; - }; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t attn_q_norm_weight = 0; - size_t attn_k_norm_weight = 0; - - size_t conv_depthwise_weight = 0; - size_t conv_in_proj_weight = 0; - size_t conv_out_proj_weight = 0; - - size_t input_layernorm_weight = 0; - size_t post_attention_layernorm_weight = 0; - size_t ffn_gate_weight = 0; - size_t ffn_up_weight = 0; - size_t ffn_down_weight = 0; - - size_t moe_router_weight = 0; - size_t moe_expert_bias = 0; - std::vector moe_experts; - }; - - enum class LayerType : uint8_t { ATTENTION, CONV }; - - struct LayerEntry { - LayerType type; - LayerWeights weights; - }; - - std::vector layers; - } weight_nodes_; - - ConvCache conv_cache_; - std::vector conv_cache_bx_nodes_; - bool last_forward_used_cache_ = false; -}; - - -class NomicModel : public Model { -public: - NomicModel(); - explicit NomicModel(const Config& config); - ~NomicModel() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - size_t build_standard_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_moe_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const; - - struct WeightNodeIDs { - size_t embedding_layernorm_weight; - size_t embedding_layernorm_bias; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_q_bias; - size_t attn_k_bias; - size_t attn_v_bias; - size_t attn_output_weight; - size_t attn_output_bias; - size_t ffn_up_weight; - size_t ffn_up_bias; - size_t ffn_norm_1_weight; - size_t ffn_norm_1_bias; - size_t ffn_down_weight; - size_t ffn_down_bias; - size_t ffn_norm_2_weight; - size_t ffn_norm_2_bias; - size_t mlp_router_layer_weight; - size_t mlp_experts_bias; - std::vector mlp_experts_mlp1_weight; - std::vector mlp_experts_mlp2_weight; - }; - - std::vector layers; - } weight_nodes_; -}; - -class WhisperModel : public Model { -public: - WhisperModel(); - explicit WhisperModel(const Config& config); - ~WhisperModel() override = default; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t,ComputeBackend, bool, size_t) override { - throw std::runtime_error("Whisper: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Whisper: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Whisper: build_transformer_block unused"); - } - - size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { - throw std::runtime_error("Whisper requires mel+token forward()."); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - - void run_encoder(const std::vector& audio_features); - void reset_graph_side_cache_nodes(); - - size_t run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only); - - void load_weights_to_graph(CactusGraph* gb) override; - - size_t build_encoder_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_encoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend); - - size_t build_decoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_conv1d(CactusGraph* gb, size_t input); - - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - - std::vector get_audio_embeddings(const std::vector& audio_features) override; - - void reset_cache() override; - -private: - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - size_t decoder_norm_weight; - size_t decoder_norm_bias; - size_t decoder_position_embeddings_weight; - - size_t encoder_position_embeddings; - size_t encoder_conv1_weight; - size_t encoder_conv1_bias; - size_t encoder_conv2_weight; - size_t encoder_conv2_bias; - size_t encoder_conv3_weight; - size_t encoder_conv3_bias; - size_t encoder_norm_weight; - size_t encoder_norm_bias; - - size_t encoder_output; - - struct LayerWeights { - size_t decoder_output_norm_bias; - size_t decoder_output_norm_weight; - size_t decoder_position_embeddings_weight; - size_t decoder_token_embeddings_weight; - - size_t decoder_encoder_attn_q_weight; - size_t decoder_encoder_attn_k_weight; - size_t decoder_encoder_attn_v_weight; - size_t decoder_encoder_attn_q_bias; - size_t decoder_encoder_attn_v_bias; - size_t decoder_encoder_attn_output_weight; - size_t decoder_encoder_attn_output_bias; - - size_t decoder_post_encoder_layernorm_weight; - size_t decoder_post_encoder_layernorm_bias; - - size_t decoder_ffn1_weight; - size_t decoder_ffn1_bias; - size_t decoder_ffn2_weight; - size_t decoder_ffn2_bias; - - size_t decoder_post_ffn_layernorm_weight; - size_t decoder_post_ffn_layernorm_bias; - - size_t decoder_self_attn_q_weight; - size_t decoder_self_attn_k_weight; - size_t decoder_self_attn_v_weight; - size_t decoder_self_attn_q_bias; - size_t decoder_self_attn_v_bias; - size_t decoder_self_attn_output_weight; - size_t decoder_self_attn_output_bias; - - size_t decoder_post_attn_layernorm_weight; - size_t decoder_post_attn_layernorm_bias; - - //Encoder layers - size_t encoder_ffn1_weight; - size_t encoder_ffn1_bias; - size_t encoder_ffn2_weight; - size_t encoder_ffn2_bias; - - size_t encoder_post_ffn_layernorm_weight; - size_t encoder_post_ffn_layernorm_bias; - - size_t encoder_self_attn_q_weight; - size_t encoder_self_attn_k_weight; - size_t encoder_self_attn_v_weight; - size_t encoder_self_attn_q_bias; - size_t encoder_self_attn_v_bias; - size_t encoder_self_attn_output_weight; - size_t encoder_self_attn_output_bias; - - size_t encoder_post_attn_layernorm_weight; - size_t encoder_post_attn_layernorm_bias; - }; - - std::vector layers; - } weight_nodes_; - - bool encoder_ready_ = false; - size_t last_new_tokens_; - size_t last_conv1_node_ = 0; - size_t last_conv2_node_ = 0; - size_t last_encoder_post_norm_node_ = 0; - size_t last_enc_plus_pos_node_ = 0; - size_t encoder_transformer_block_0 = 0; - size_t encoder_pre_gelu = 0; - size_t encoder_post_gelu = 0; - - std::vector encoder_block_out_nodes_; - - size_t encoder_output_persistent_ = 0; - - std::vector suppress_tokens_ = { - 1, - 2, - 7, - 8, - 9, - 10, - 14, - 25, - 26, - 27, - 28, - 29, - 31, - 58, - 59, - 60, - 61, - 62, - 63, - 90, - 91, - 92, - 93, - 359, - 503, - 522, - 542, - 873, - 893, - 902, - 918, - 922, - 931, - 1350, - 1853, - 1982, - 2460, - 2627, - 3246, - 3253, - 3268, - 3536, - 3846, - 3961, - 4183, - 4667, - 6585, - 6647, - 7273, - 9061, - 9383, - 10428, - 10929, - 11938, - 12033, - 12331, - 12562, - 13793, - 14157, - 14635, - 15265, - 15618, - 16553, - 16604, - 18362, - 18956, - 20075, - 21675, - 22520, - 26130, - 26161, - 26435, - 28279, - 29464, - 31650, - 32302, - 32470, - 36865, - 42863, - 47425, - 49870, - 50254, - 50258, - 50358, - 50359, - 50360, - 50361, - 50362 - }; - - std::vector begin_suppress_tokens_ = { - 220, - 50257 - }; - - bool first_decode_step_ = true; - - std::unordered_map suppress_bias_; - std::unordered_map suppress_bias_first_step_; - - std::vector encoder_k_persistent_; - std::vector encoder_v_persistent_; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - - uint32_t enc_layers_ = 0; - uint32_t dec_layers_ = 0; -}; - - -class MoonshineModel : public Model { -public: - MoonshineModel(); - explicit MoonshineModel(const Config& config); - ~MoonshineModel() override = default; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t,ComputeBackend, bool, size_t) override { - throw std::runtime_error("Moonshine: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Moonshine: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Moonshine: build_transformer_block unused"); - } - - size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { - throw std::runtime_error("Moonshine requires mel+token forward()."); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - - size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); - - void reset_graph_side_cache_nodes(); - - size_t build_decoder(const std::vector& tokens, bool use_cache, bool last_token_only); - - void load_weights_to_graph(CactusGraph* gb) override; - - size_t build_decoder_cross_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_encoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend); - - size_t build_decoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_conv1d(CactusGraph* gb, size_t input); - size_t build_audio_preprocessor(CactusGraph* gb, size_t input); - - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - - std::vector get_audio_embeddings(const std::vector& audio_features) override; - - void reset_cache() override; - -private: - struct WeightNodeIDs { - size_t output_weight; - - size_t decoder_norm_weight; - - size_t encoder_conv1_weight; - size_t encoder_conv2_weight; - size_t encoder_conv2_bias; - size_t encoder_conv3_weight; - size_t encoder_conv3_bias; - size_t encoder_norm_weight; - size_t encoder_norm_bias; - size_t encoder_layer_norm_weight; - - - struct LayerWeights { - //Decoder layers - - size_t decoder_encoder_attn_q_weight; - size_t decoder_encoder_attn_k_weight; - size_t decoder_encoder_attn_v_weight; - size_t decoder_encoder_attn_output_weight; - - size_t decoder_post_encoder_layernorm_weight; - - size_t decoder_ffn1_weight; - size_t decoder_ffn1_bias; - size_t decoder_ffn_gate_weight; - size_t decoder_ffn_gate_bias; - size_t decoder_ffn_up_weight; - size_t decoder_ffn_up_bias; - size_t decoder_ffn2_weight; - size_t decoder_ffn2_bias; - - size_t decoder_post_ffn_layernorm_weight; - - size_t decoder_self_attn_q_weight; - size_t decoder_self_attn_k_weight; - size_t decoder_self_attn_v_weight; - size_t decoder_self_attn_output_weight; - - size_t decoder_post_attn_layernorm_weight; - - //Encoder layers - size_t encoder_ffn1_weight; - size_t encoder_ffn1_bias; - size_t encoder_ffn_gate_weight; - size_t encoder_ffn_gate_bias; - size_t encoder_ffn_up_weight; - size_t encoder_ffn_up_bias; - size_t encoder_ffn2_weight; - size_t encoder_ffn2_bias; - - size_t encoder_post_ffn_layernorm_weight; - - size_t encoder_self_attn_q_weight; - size_t encoder_self_attn_k_weight; - size_t encoder_self_attn_v_weight; - size_t encoder_self_attn_output_weight; - - size_t encoder_post_attn_layernorm_weight; - }; - - std::vector encoder_layers; - std::vector decoder_layers; - } weight_nodes_; - - bool encoder_ready_ = false; - size_t last_new_tokens_; - std::vector encoder_output_host_; - std::vector encoder_output_shape_; - size_t last_conv1_node_ = 0; - size_t last_conv2_node_ = 0; - size_t last_encoder_post_norm_node_ = 0; - size_t last_enc_plus_pos_node_ = 0; - size_t encoder_transformer_block_0 = 0; - size_t encoder_pre_gelu = 0; - size_t encoder_post_gelu = 0; - - std::vector encoder_block_out_nodes_; - std::vector encoder_output_bytes_; - Precision encoder_output_precision_ = Precision::FP32; - - bool first_decode_step_ = true; - - std::vector encoder_k_persistent_; - std::vector encoder_v_persistent_; - - Precision encoder_kv_precision_ = Precision::FP32; - bool encoder_kv_ready_ = false; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - -}; - -class NeedleModel : public Model { -public: - NeedleModel(); - explicit NeedleModel(const Config& config); - ~NeedleModel() override = default; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Needle: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Needle: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Needle: build_transformer_block unused"); - } - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, - size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f) override; - void load_weights_to_graph(CactusGraph* gb) override; - void reset_cache() override; - - void run_encoder(const std::vector& encoder_tokens); - void reset_graph_side_cache_nodes(); - size_t run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only); - - size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - size_t build_decoder_cross_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - -private: - struct WeightNodeIDs { - size_t output_weight = 0; - size_t encoder_norm_weight = 0; - size_t decoder_norm_weight = 0; - - struct EncoderLayerWeights { - size_t attn_gate_weight = 0; - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t input_norm_weight = 0; - size_t attn_q_norm_weight = 0; - size_t attn_k_norm_weight = 0; - }; - - struct DecoderLayerWeights { - size_t self_attn_gate_weight = 0; - size_t cross_attn_gate_weight = 0; - size_t self_attn_q_weight = 0; - size_t self_attn_k_weight = 0; - size_t self_attn_v_weight = 0; - size_t self_attn_output_weight = 0; - size_t self_attn_q_norm_weight = 0; - size_t self_attn_k_norm_weight = 0; - - size_t encoder_attn_q_weight = 0; - size_t encoder_attn_k_weight = 0; - size_t encoder_attn_v_weight = 0; - size_t encoder_attn_output_weight = 0; - size_t encoder_attn_q_norm_weight = 0; - size_t encoder_attn_k_norm_weight = 0; - - size_t input_norm_weight = 0; - size_t post_attn_norm_weight = 0; - }; - - std::vector encoder_layers; - std::vector decoder_layers; - } weight_nodes_; - - bool encoder_ready_ = false; - size_t last_encoder_post_norm_node_ = 0; - std::vector encoder_k_persistent_; - std::vector encoder_v_persistent_; -}; - -class ParakeetModel : public Model { -public: - ParakeetModel(); - explicit ParakeetModel(const Config& config); - ~ParakeetModel() override = default; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Parakeet: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Parakeet: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Parakeet: build_transformer_block unused"); - } - - size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { - throw std::runtime_error("Parakeet requires audio feature forward()."); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - std::vector get_audio_embeddings(const std::vector& audio_features) override; - void reset_cache() override; - -private: - size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); - size_t build_ctc_logits(CactusGraph* gb, size_t hidden_states); - size_t build_subsampling(CactusGraph* gb, const std::vector& audio_features); - size_t build_relative_position_embeddings(CactusGraph* gb, size_t seq_len); - size_t build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); - size_t build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend); - size_t build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend); - size_t build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); - std::vector greedy_decode_tokens(CactusGraph* gb, size_t logits_node) const; - - struct WeightNodeIDs { - size_t ctc_head_weight = 0; - size_t ctc_head_bias = 0; - - size_t subsampling_conv0_weight = 0; - size_t subsampling_conv0_bias = 0; - size_t subsampling_depthwise1_weight = 0; - size_t subsampling_depthwise1_bias = 0; - size_t subsampling_pointwise1_weight = 0; - size_t subsampling_pointwise1_bias = 0; - size_t subsampling_depthwise2_weight = 0; - size_t subsampling_depthwise2_bias = 0; - size_t subsampling_pointwise2_weight = 0; - size_t subsampling_pointwise2_bias = 0; - size_t subsampling_linear_weight = 0; - size_t subsampling_linear_bias = 0; - - struct LayerWeights { - size_t ff1_linear1_weight = 0; - size_t ff1_linear1_bias = 0; - size_t ff1_linear2_weight = 0; - size_t ff1_linear2_bias = 0; - - size_t ff2_linear1_weight = 0; - size_t ff2_linear1_bias = 0; - size_t ff2_linear2_weight = 0; - size_t ff2_linear2_bias = 0; - - size_t self_attn_q_weight = 0; - size_t self_attn_q_bias = 0; - size_t self_attn_k_weight = 0; - size_t self_attn_k_bias = 0; - size_t self_attn_v_weight = 0; - size_t self_attn_v_bias = 0; - size_t self_attn_output_weight = 0; - size_t self_attn_output_bias = 0; - size_t self_attn_relative_k_weight = 0; - size_t self_attn_bias_u = 0; - size_t self_attn_bias_v = 0; - - size_t norm_ff1_weight = 0; - size_t norm_ff1_bias = 0; - size_t norm_self_attn_weight = 0; - size_t norm_self_attn_bias = 0; - size_t norm_conv_weight = 0; - size_t norm_conv_bias = 0; - size_t norm_ff2_weight = 0; - size_t norm_ff2_bias = 0; - size_t norm_out_weight = 0; - size_t norm_out_bias = 0; - - size_t conv_pointwise1_weight = 0; - size_t conv_pointwise1_bias = 0; - size_t conv_depthwise_weight = 0; - size_t conv_depthwise_bias = 0; - size_t conv_pointwise2_weight = 0; - size_t conv_pointwise2_bias = 0; - size_t conv_batchnorm_weight = 0; - size_t conv_batchnorm_bias = 0; - size_t conv_batchnorm_running_mean = 0; - size_t conv_batchnorm_running_var = 0; - }; - - std::vector layers; - } weight_nodes_; - - bool ctc_tokens_ready_ = false; - size_t ctc_emit_index_ = 0; - std::vector ctc_tokens_; - size_t last_input_token_count_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool has_cpu_encoder_weights_ = false; -}; - -class ParakeetTDTModel : public Model { -public: - ParakeetTDTModel(); - explicit ParakeetTDTModel(const Config& config); - ~ParakeetTDTModel() override = default; - - struct TDTToken { uint32_t id; float time_start; float time_end; }; - - struct ChunkStreamState { - bool initialized = false; - uint32_t last_token = 0; - std::vector> h; - std::vector> c; - }; - - struct ChunkStreamResult { - std::string text; - std::string confirmed_text; - std::string pending_text; - size_t token_count = 0; - size_t confirmed_token_count = 0; - double raw_decoder_tps = 0.0; - double raw_decoder_time_ms = 0.0; - float start_sec = 0.0f; - float confirmed_end_sec = 0.0f; - float resume_end_sec = 0.0f; - float end_sec = 0.0f; - }; - - ChunkStreamResult decode_chunk_stream( - const std::vector& audio_features, - size_t replay_start_frame, - size_t start_frame, - size_t end_frame, - ChunkStreamState& state); - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("ParakeetTDT: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("ParakeetTDT: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("ParakeetTDT: build_transformer_block unused"); - } - - size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { - throw std::runtime_error("ParakeetTDT requires audio feature forward()."); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - std::vector get_audio_embeddings(const std::vector& audio_features) override; - void reset_cache() override; - -private: - size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); - size_t build_subsampling(CactusGraph* gb, const std::vector& audio_features); - size_t build_relative_position_embeddings(CactusGraph* gb, size_t seq_len); - size_t build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); - size_t build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend); - size_t build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend); - size_t build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); - std::vector decode_tdt_tokens_with_state( - CactusGraph* gb, - size_t encoder_hidden_node, - size_t replay_start_frame, - size_t start_frame, - size_t end_frame, - ChunkStreamState* stream_state, - size_t* out_confirmed_count = nullptr, - double* out_raw_decoder_time_ms = nullptr) const; - std::vector greedy_decode_tdt_tokens(CactusGraph* gb, size_t encoder_hidden_node) const; - - struct WeightNodeIDs { - size_t subsampling_conv0_weight = 0; - size_t subsampling_conv0_bias = 0; - size_t subsampling_depthwise1_weight = 0; - size_t subsampling_depthwise1_bias = 0; - size_t subsampling_pointwise1_weight = 0; - size_t subsampling_pointwise1_bias = 0; - size_t subsampling_depthwise2_weight = 0; - size_t subsampling_depthwise2_bias = 0; - size_t subsampling_pointwise2_weight = 0; - size_t subsampling_pointwise2_bias = 0; - size_t subsampling_linear_weight = 0; - size_t subsampling_linear_bias = 0; - - struct LayerWeights { - size_t ff1_linear1_weight = 0; - size_t ff1_linear1_bias = 0; - size_t ff1_linear2_weight = 0; - size_t ff1_linear2_bias = 0; - - size_t ff2_linear1_weight = 0; - size_t ff2_linear1_bias = 0; - size_t ff2_linear2_weight = 0; - size_t ff2_linear2_bias = 0; - - size_t self_attn_q_weight = 0; - size_t self_attn_q_bias = 0; - size_t self_attn_k_weight = 0; - size_t self_attn_k_bias = 0; - size_t self_attn_v_weight = 0; - size_t self_attn_v_bias = 0; - size_t self_attn_output_weight = 0; - size_t self_attn_output_bias = 0; - size_t self_attn_relative_k_weight = 0; - size_t self_attn_bias_u = 0; - size_t self_attn_bias_v = 0; - - size_t norm_ff1_weight = 0; - size_t norm_ff1_bias = 0; - size_t norm_self_attn_weight = 0; - size_t norm_self_attn_bias = 0; - size_t norm_conv_weight = 0; - size_t norm_conv_bias = 0; - size_t norm_ff2_weight = 0; - size_t norm_ff2_bias = 0; - size_t norm_out_weight = 0; - size_t norm_out_bias = 0; - - size_t conv_pointwise1_weight = 0; - size_t conv_pointwise1_bias = 0; - size_t conv_depthwise_weight = 0; - size_t conv_depthwise_bias = 0; - size_t conv_pointwise2_weight = 0; - size_t conv_pointwise2_bias = 0; - size_t conv_batchnorm_weight = 0; - size_t conv_batchnorm_bias = 0; - size_t conv_batchnorm_running_mean = 0; - size_t conv_batchnorm_running_var = 0; - }; - - struct PredictorLayerWeights { - size_t weight_ih = 0; - size_t weight_hh = 0; - size_t bias = 0; - }; - - size_t predictor_embed = 0; - std::vector predictor_layers; - size_t joint_enc_weight = 0; - size_t joint_enc_bias = 0; - size_t joint_pred_weight = 0; - size_t joint_pred_bias = 0; - size_t joint_out_weight = 0; - size_t joint_out_bias = 0; - - std::vector layers; - } weight_nodes_; - - bool tdt_tokens_ready_ = false; - size_t tdt_emit_index_ = 0; - std::vector tdt_tokens_; - size_t last_input_token_count_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool has_cpu_encoder_weights_ = false; -}; - - -class Lfm2VlModel : public Model { -public: - Lfm2VlModel(); - explicit Lfm2VlModel(const Config& config); - ~Lfm2VlModel() override = default; - - bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; - size_t forward(const std::vector& tokens, bool use_cache = false) override; - - uint32_t decode(const std::vector& tokens, - float temperature = -1.0f, - float top_p = -1.0f, - size_t top_k = 0, - const std::string& profile_file = "", - float* out_entropy = nullptr, - float min_p = 0.15f, - float repetition_penalty = 1.1f) override; - - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - - void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file = "") override; - - uint32_t decode_with_images( - const std::vector& tokens, - const std::vector& image_paths, - float temperature = -1.0f, - float top_p = -1.0f, - size_t top_k = 0, - const std::string& profile_file = "", - float* out_entropy = nullptr, - float min_p = 0.15f, - float repetition_penalty = 1.1f) override; - - void reset_cache() override; - std::vector get_image_embeddings(const std::string& image_path) override; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override; - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct ProjectedTileFeature { - size_t node_id; - size_t token_count; - }; - - struct TextEmbeddingInput { - size_t input_node; - std::vector tokens; - }; - - struct MergedEmbeddingResult { - size_t node_id; - size_t seq_len; - }; - - struct ForwardImageResult { - size_t final_hidden_node; - size_t seq_len; - }; - std::vector get_image_features( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - - ForwardImageResult forward_images( - CactusGraph* gb, - const std::vector& tokens, - const std::vector& image_paths, - ComputeBackend backend, - bool use_cache); - size_t build_multimodal_projector( - CactusGraph* gb, - size_t image_features, - size_t tile_h, - size_t tile_w, - ComputeBackend backend); - size_t pixel_unshuffle(CactusGraph* gb, size_t hidden_states, size_t height, size_t width, size_t channels); - MergedEmbeddingResult merge_image_text_embeddings( - CactusGraph* gb, - const std::vector& tokens, - const std::vector>& image_embedding_nodes, - std::vector& text_embedding_inputs); - Siglip2VisionModel vision_tower_; - LFM2Model language_model_; - Siglip2Preprocessor preprocessor_; - struct ProjectorWeights { - size_t layer_norm_weight; - size_t layer_norm_bias; - size_t linear_1_weight; - size_t linear_1_bias; - size_t linear_2_weight; - size_t linear_2_bias; - } projector_weights_; - - bool vision_weights_loaded_ = false; - bool language_weights_loaded_ = false; - - bool image_prefill_completed_ = false; - size_t last_token_count_ = 0; -}; - -class SileroVADModel : public Model { -public: - static constexpr size_t CONTEXT_SIZE = 64; - static constexpr size_t CHUNK_SIZE = 512; - static constexpr size_t REFLECT_PAD_SIZE = 64; - static constexpr size_t HIDDEN_SIZE = 128; - static constexpr size_t GATE_SIZE = 512; - - SileroVADModel(); - explicit SileroVADModel(const Config& config); - ~SileroVADModel() override; - - bool init(const std::string& model_folder, size_t context_size = 0, - const std::string& system_prompt = "", bool do_warmup = false) override; - - struct SpeechTimestamp { - size_t start; - size_t end; - }; - - struct SpeechTimestampsOptions { - float threshold = 0.5f; - float neg_threshold = 0.0f; - int min_speech_duration_ms = 250; - float max_speech_duration_s = std::numeric_limits::infinity(); - int min_silence_duration_ms = 100; - int speech_pad_ms = 30; - int window_size_samples = 512; - int min_silence_at_max_speech = 98; - bool use_max_poss_sil_at_max_speech = true; - int sampling_rate = 16000; - }; - - float process_chunk(const std::vector& audio_chunk); - void reset_states(); - std::vector get_speech_timestamps(const std::vector& audio, const SpeechTimestampsOptions& options); - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("SileroVAD: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("SileroVAD: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("SileroVAD: build_transformer_block unused"); - } - - size_t forward(const std::vector&, bool = false) override { - throw std::runtime_error("SileroVAD: use process_chunk() instead"); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - void build_graph(); - - struct VADGraphNodes { - size_t input; - size_t h_prev; - size_t c_prev; - size_t h_new; - size_t c_new; - size_t output; - size_t lstm_output; - size_t encoder_output; - } graph_nodes_; - - struct VADWeightNodes { - size_t stft_basis; - struct EncoderBlock { - size_t conv_weight; - size_t conv_bias; - }; - std::vector encoder_blocks; - size_t lstm_weight_ih; - size_t lstm_weight_hh; - size_t lstm_bias_ih; - size_t lstm_bias_hh; - size_t output_conv_weight; - size_t output_conv_bias; - } weight_nodes_; - - struct VADState { - std::vector<__fp16> h; - std::vector<__fp16> c; - std::vector context; - std::vector input_buf; - std::vector<__fp16> input_fp16; - } state_; - - CactusGraph graph_; - std::string weights_path_; -}; - -class YoutuModel : public Model { -public: - YoutuModel(); - explicit YoutuModel(const Config& config); - ~YoutuModel() override = default; - - void reset_cache() override; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - std::vector get_kv_layer_dims() const override; - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - -private: - KVCache v_cache_; - std::vector cache_v_nodes_; - - struct WeightNodeIDs { - size_t output_weight = 0; - size_t output_norm_weight = 0; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_q_a_weight = 0; - size_t attn_q_a_norm_weight = 0; - size_t attn_q_b_weight = 0; - size_t attn_kv_a_weight = 0; - size_t attn_kv_a_norm_weight = 0; - size_t attn_kv_b_weight = 0; - size_t attn_output_weight = 0; - size_t input_layernorm_weight = 0; - size_t post_attention_layernorm_weight = 0; - size_t ffn_gate_weight = 0; - size_t ffn_up_weight = 0; - size_t ffn_down_weight = 0; - size_t attn_q_a_bias = 0; - size_t attn_kv_a_bias = 0; - size_t attn_output_bias = 0; - }; - - std::vector layers; - } weight_nodes_; -}; - -class PyAnnoteModel : public Model { -public: - PyAnnoteModel(); - explicit PyAnnoteModel(const Config& config); - ~PyAnnoteModel() override = default; - - bool init(const std::string& model_folder, size_t context_size = 0, - const std::string& system_prompt = "", bool do_warmup = false) override; - - std::vector diarize(const float* pcm_f32, size_t num_samples, size_t step_samples = 16000, bool raw_powerset = false); - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("PyAnnote: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("PyAnnote: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("PyAnnote: build_transformer_block unused"); - } - - size_t forward(const std::vector&, bool = false) override { - throw std::runtime_error("PyAnnote: use diarize() instead"); - } - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - void build_graph(); - - struct LSTMLayerWeights { - size_t w_ih_fwd, w_hh_fwd, b_ih_fwd, b_hh_fwd; - size_t w_ih_bwd, w_hh_bwd, b_ih_bwd, b_hh_bwd; - }; - - struct WeightNodeIDs { - size_t sinc_filters; - size_t wav_norm_weight, wav_norm_bias; - size_t norm0_weight, norm0_bias; - size_t conv1_weight, conv1_bias; - size_t norm1_weight, norm1_bias; - size_t conv2_weight, conv2_bias; - size_t norm2_weight, norm2_bias; - LSTMLayerWeights lstm_layers[4]; - size_t linear0_weight, linear0_bias; - size_t linear1_weight, linear1_bias; - size_t classifier_weight, classifier_bias; - } weight_nodes_; - - CactusGraph graph_; - size_t audio_input_ = 0; - size_t output_node_ = 0; - std::vector hamming_; - std::vector<__fp16> chunk_buf_; -}; - -class WeSpeakerModel : public Model { -public: - WeSpeakerModel(); - explicit WeSpeakerModel(const Config& config); - ~WeSpeakerModel() override = default; - - bool init(const std::string& model_folder, size_t context_size = 0, - const std::string& system_prompt = "", bool do_warmup = false) override; - - std::vector embed(const float* fbank_features, size_t num_features); - std::vector embed(const float* fbank_features, size_t num_features, - const float* mask_weights, size_t mask_num_frames); - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("WeSpeaker: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("WeSpeaker: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("WeSpeaker: build_transformer_block unused"); - } - - size_t forward(const std::vector&, bool = false) override { - throw std::runtime_error("WeSpeaker: use embed() instead"); - } - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - void build_graph(size_t num_frames); - void build_graph_masked(size_t num_frames); - - struct ResBlockWeights { - size_t conv1_w, conv1_b; - size_t conv2_w, conv2_b; - size_t shortcut_conv_w, shortcut_conv_b; - bool has_shortcut = false; - }; - - ResBlockWeights load_resblock(CactusGraph* gb, const std::string& prefix, bool has_shortcut); - size_t build_resblock(CactusGraph* gb, size_t x, const ResBlockWeights& rb, bool stride2); - size_t build_resnet_body(CactusGraph* gb, size_t x); - - struct WeightNodeIDs { - size_t conv1_w, conv1_b; - std::vector layer1, layer2, layer3, layer4; - size_t seg1_w, seg1_b; - } weight_nodes_; - - CactusGraph graph_; - size_t audio_input_ = 0; - size_t mask_input_ = 0; - size_t output_node_ = 0; - size_t current_num_frames_ = 0; - bool current_masked_ = false; - std::vector<__fp16> input_buf_; - std::vector mask_buf_; -}; - -} -} - -#include "gemma4/model_gemma4.h" diff --git a/cactus-transpile/models/model_gemma.cpp b/cactus-transpile/models/model_gemma.cpp deleted file mode 100644 index a128c8905..000000000 --- a/cactus-transpile/models/model_gemma.cpp +++ /dev/null @@ -1,192 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include -#include - - -namespace cactus { -namespace engine { - -GemmaModel::GemmaModel() : Model() {} - -GemmaModel::GemmaModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -void GemmaModel::post_init() { - kv_cache_.set_window_size(0, 0); -} - -void GemmaModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.pre_feedforward_layernorm_weight = gb->mmap_weights(layer_prefix + "pre_ffn_norm.weights"); - layer.post_feedforward_layernorm_weight = gb->mmap_weights(layer_prefix + "post_ffn_norm.weights"); - } - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - load_npu_prefill(npu_prefill_path); - } -} - -size_t GemmaModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); - q_proj = gb->rms_norm(q_proj, layer.attn_q_norm_weight, config_.layer_norm_eps); - q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); - - size_t num_kv_heads = config_.attention_kv_heads; - k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); - k_proj = gb->rms_norm(k_proj, layer.attn_k_norm_weight, config_.layer_norm_eps); - k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); - - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - bool is_global_attention = ((layer_idx + 1) % 6) == 0; - - if (config_.rope_theta > 0) { - float rope_freq = is_global_attention ? 1000000.0f : 10000.0f; - - q_proj_4d = gb->rope(q_proj_4d, rope_freq, position_offset); - k_proj_4d = gb->rope(k_proj_4d, rope_freq, position_offset); - } - - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - } - - size_t attn_output_4d; - size_t window_size = is_global_attention ? 0 : 512; - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim, window_size - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset, window_size); - } - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - return gb->matmul(attn_output, layer.attn_output_weight, true, backend); -} - - -size_t GemmaModel::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t gate_output = gb->matmul(input, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(input, layer.ffn_up_weight, true, backend); - size_t gate_gelu = gb->gelu(gate_output); - size_t gated = gb->multiply(gate_gelu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - -size_t GemmaModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - - auto normalized_attn = gb->rms_norm(attn_output, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - - auto after_attention = gb->add_clipped(hidden, normalized_attn); - - auto pre_mlp_norm = gb->rms_norm(after_attention, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, pre_mlp_norm, layer_idx, backend); - - auto normalized_mlp = gb->rms_norm(mlp_output, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - - return gb->add_clipped(after_attention, normalized_mlp); -} - - -size_t GemmaModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto seq_len = static_cast(tokens.size()); - - size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - float embed_scale = std::sqrt(static_cast(config_.hidden_dim)); - hidden = gb->scalar_multiply(hidden, embed_scale); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - return final_hidden; -} - -} -} \ No newline at end of file diff --git a/cactus-transpile/models/model_gemma3n.cpp b/cactus-transpile/models/model_gemma3n.cpp deleted file mode 100644 index 65a3b8173..000000000 --- a/cactus-transpile/models/model_gemma3n.cpp +++ /dev/null @@ -1,475 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -static const float RSQRT2 = 1.0f / std::sqrt(2.0f); - -GemmaModel3n::GemmaModel3n() : Model() {} - -GemmaModel3n::GemmaModel3n(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -std::vector GemmaModel3n::get_kv_layer_dims() const { - uint32_t n = config_.num_layers; - uint32_t num_shared = config_.num_kv_shared_layers; - uint32_t first_shared = (n > num_shared) ? n - num_shared : n; - - std::vector dims(n); - for (uint32_t i = 0; i < n; i++) { - dims[i] = (i >= first_shared) ? 0 : config_.attention_head_dim; - } - return dims; -} - -void GemmaModel3n::post_init() { - kv_cache_.set_window_size(0, 0); - - uint32_t n = config_.num_layers; - uint32_t num_shared = config_.num_kv_shared_layers; - uint32_t first_shared = (n > num_shared) ? n - num_shared : n; - - kv_share_map_.resize(n, -1); - shared_k_nodes_.resize(n, 0); - shared_v_nodes_.resize(n, 0); - - auto is_global_layer = [&](uint32_t idx) -> bool { - if (!config_.layer_types.empty() && idx < config_.layer_types.size()) { - const auto& lt = config_.layer_types[idx]; - return (lt == "global" || lt == "full" || lt == "full_attention"); - } - return (idx % 5) == 4; - }; - - for (uint32_t i = first_shared; i < n; i++) { - bool is_global = is_global_layer(i); - for (int j = static_cast(first_shared) - 1; j >= 0; j--) { - if (is_global_layer(j) == is_global) { - kv_share_map_[i] = j; - break; - } - } - } -} - -void GemmaModel3n::load_weights_to_graph(CactusGraph* gb) { - assert(config_.altup_num_inputs == 4 && "WeightNodeIDs altup arrays assume exactly 4 AltUp streams"); - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (int i = 0; i < 3; i++) { - auto idx = std::to_string(i); - weight_nodes_.altup_proj_weights[i] = gb->mmap_weights(model_folder_path_ + "/altup_proj_" + idx + ".weights"); - weight_nodes_.altup_unembed_proj_weights[i] = gb->mmap_weights(model_folder_path_ + "/altup_unembed_proj_" + idx + ".weights"); - } - - weight_nodes_.embed_tokens_per_layer = gb->mmap_embeddings(model_folder_path_ + "/embed_tokens_per_layer.weights"); - weight_nodes_.per_layer_model_proj = gb->mmap_weights(model_folder_path_ + "/per_layer_model_proj.weights"); - weight_nodes_.per_layer_proj_norm = gb->mmap_weights(model_folder_path_ + "/per_layer_proj_norm.weights"); - - uint32_t num_shared = config_.num_kv_shared_layers; - uint32_t first_shared = (config_.num_layers > num_shared) ? config_.num_layers - num_shared : config_.num_layers; - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - bool is_shared = (i >= first_shared); - - layer.attn_q_weight = gb->mmap_weights(prefix + "attn_q.weights"); - layer.attn_k_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k.weights"); - layer.attn_v_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(prefix + "post_attn_norm.weights"); - layer.pre_feedforward_layernorm_weight = gb->mmap_weights(prefix + "pre_ffn_norm.weights"); - layer.post_feedforward_layernorm_weight = gb->mmap_weights(prefix + "post_ffn_norm.weights"); - - layer.altup_router_norm = gb->mmap_weights(prefix + "altup_router_norm.weights"); - layer.altup_prediction_coefs = gb->mmap_weights(prefix + "altup_prediction_coefs.weights"); - layer.altup_correction_coefs = gb->mmap_weights(prefix + "altup_correction_coefs.weights"); - layer.altup_correct_output_scale = gb->mmap_weights(prefix + "altup_correct_output_scale.weights"); - layer.altup_modality_router = gb->mmap_weights(prefix + "altup_modality_router.weights"); - layer.laurel_left = gb->mmap_weights(prefix + "laurel_left.weights"); - layer.laurel_right = gb->mmap_weights(prefix + "laurel_right.weights"); - layer.laurel_norm = gb->mmap_weights(prefix + "laurel_norm.weights"); - layer.per_layer_gate = gb->mmap_weights(prefix + "per_layer_gate.weights"); - layer.per_layer_proj = gb->mmap_weights(prefix + "per_layer_proj.weights"); - layer.post_per_layer_norm = gb->mmap_weights(prefix + "post_per_layer_norm.weights"); - } - - size_t head_dim = config_.attention_head_dim; - v_norm_ones_weight_.assign(head_dim, static_cast<__fp16>(1.0f)); - v_norm_ones_node_ = gb->input({head_dim}, Precision::FP16); - gb->set_external_input(v_norm_ones_node_, v_norm_ones_weight_.data(), Precision::FP16); -} - - -size_t GemmaModel3n::build_rms_norm_no_weight(CactusGraph* gb, size_t input, size_t num_rows, size_t row_dim) const { - auto flat = gb->reshape(input, {num_rows, row_dim}); - return gb->rms_norm(flat, v_norm_ones_node_, config_.layer_norm_eps); -} - -size_t GemmaModel3n::build_magnitude_normalize(CactusGraph* gb, size_t reference, size_t target) const { - size_t rows = gb->get_output_buffer(reference).shape[0]; - auto ref_sq = gb->reshape(gb->mean(gb->multiply(reference, reference), 1), {rows, 1}); - auto tgt_sq = gb->reshape(gb->mean(gb->multiply(target, target), 1), {rows, 1}); - auto ref_mag = gb->scalar_sqrt(ref_sq); - auto tgt_mag = gb->scalar_sqrt(gb->scalar_add(tgt_sq, 1e-5f)); - auto ratio = gb->divide(ref_mag, tgt_mag); - return gb->multiply(target, ratio); -} - -size_t GemmaModel3n::build_gaussian_topk(CactusGraph* gb, size_t input, float ppf) const { - return gb->gaussian_topk(input, ppf); -} - -size_t GemmaModel3n::build_laurel(CactusGraph* gb, size_t normed_input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto x = gb->matmul(normed_input, layer.laurel_left, true, backend); - x = gb->matmul(x, layer.laurel_right, true, backend); - return gb->add(normed_input, gb->rms_norm(x, layer.laurel_norm, config_.layer_norm_eps)); -} - -size_t GemmaModel3n::build_altup_router_modalities(CactusGraph* gb, size_t stream0, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto x = gb->rms_norm(stream0, layer.altup_router_norm, config_.layer_norm_eps); - x = gb->scalar_multiply(x, 1.0f / static_cast(config_.hidden_dim)); - return gb->tanh(gb->matmul(x, layer.altup_modality_router, true, backend)); -} - -void GemmaModel3n::build_altup_predict(CactusGraph* gb, size_t modalities, uint32_t layer_idx, - const size_t* streams, size_t* predictions) const { - uint32_t n = config_.altup_num_inputs; - auto coefs = gb->matmul(modalities, weight_nodes_.layers[layer_idx].altup_prediction_coefs, true, ComputeBackend::CPU); - - size_t seq_len = gb->get_output_buffer(streams[0]).shape[0]; - auto fused = gb->altup_predict(coefs, streams, n); - for (uint32_t i = 0; i < n; i++) { - predictions[i] = gb->slice(fused, 0, i * seq_len, seq_len); - } -} - -void GemmaModel3n::build_altup_correct(CactusGraph* gb, size_t activated, size_t modalities, uint32_t layer_idx, - ComputeBackend backend, const size_t* predictions, size_t* corrected) const { - uint32_t n = config_.altup_num_inputs; - auto coefs = gb->scalar_add(gb->matmul(modalities, weight_nodes_.layers[layer_idx].altup_correction_coefs, true, backend), 1.0f); - auto innovation = gb->subtract(activated, predictions[0]); - - size_t seq_len = gb->get_output_buffer(predictions[0]).shape[0]; - auto fused = gb->altup_correct(coefs, innovation, predictions, n); - for (uint32_t i = 0; i < n; i++) { - corrected[i] = gb->slice(fused, 0, i * seq_len, seq_len); - } -} - -void GemmaModel3n::build_per_layer_input(CactusGraph* gb, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend, size_t* streams) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto gated = gb->multiply(streams[0], layer.altup_correct_output_scale); - gated = gb->gelu(gb->matmul(gated, layer.per_layer_gate, true, backend)); - - auto pli = gb->multiply(gated, gb->slice(pli_combined, 1, layer_idx * pli_dim, pli_dim)); - pli = gb->rms_norm(gb->matmul(pli, layer.per_layer_proj, true, backend), layer.post_per_layer_norm, config_.layer_norm_eps); - - for (uint32_t i = 1; i < config_.altup_num_inputs; i++) - streams[i] = gb->add(streams[i], pli); -} - - -size_t GemmaModel3n::build_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t seq_len = gb->get_output_buffer(input).shape[0]; - size_t num_heads = config_.attention_heads; - size_t kv_heads = config_.attention_kv_heads; - size_t head_dim = config_.attention_head_dim; - int share_src = (layer_idx < kv_share_map_.size()) ? kv_share_map_[layer_idx] : -1; - - bool is_global = false; - if (!config_.layer_types.empty() && layer_idx < config_.layer_types.size()) { - const auto& lt = config_.layer_types[layer_idx]; - is_global = (lt == "global" || lt == "full" || lt == "full_attention"); - } else { - is_global = (layer_idx % 5) == 4; - } - float rope_freq = is_global ? config_.rope_theta : config_.rope_local_base_freq; - size_t window = is_global ? 0 : config_.sliding_window; - - auto q = gb->matmul(input, layer.attn_q_weight, true, backend); - q = gb->reshape(q, {seq_len * num_heads, head_dim}); - q = gb->rms_norm(q, layer.attn_q_norm_weight, config_.layer_norm_eps); - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - auto q4 = gb->rope(q, rope_freq, position_offset); - - size_t k4, v4; - if (share_src >= 0 && shared_k_nodes_[share_src] != 0) { - k4 = shared_k_nodes_[share_src]; - v4 = shared_v_nodes_[share_src]; - } else { - auto k = gb->matmul(input, layer.attn_k_weight, true, backend); - k = gb->reshape(k, {seq_len * kv_heads, head_dim}); - k = gb->rms_norm(k, layer.attn_k_norm_weight, config_.layer_norm_eps); - k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); - k4 = gb->rope(k, rope_freq, position_offset); - - auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); - auto v = build_rms_norm_no_weight(gb, v_proj, seq_len * kv_heads, head_dim); - v4 = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); - - shared_k_nodes_[layer_idx] = k4; - shared_v_nodes_[layer_idx] = v4; - } - - if (use_cache && share_src < 0) { - cache_k_output_nodes_[layer_idx] = k4; - cache_v_output_nodes_[layer_idx] = v4; - } - - size_t cache_src = (share_src >= 0) ? static_cast(share_src) : layer_idx; - size_t attn; - if (use_cache && !kv_cache_.is_empty()) { - attn = gb->attention_int8_hybrid( - q4, k4, v4, attention_scale_, position_offset, - kv_cache_.get_keys_int8(cache_src), kv_cache_.get_values_int8(cache_src), - kv_cache_.get_key_scales(cache_src), kv_cache_.get_value_scales(cache_src), - kv_cache_.current_seq_len, kv_heads, head_dim, window); - } else { - attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); - } - - auto o_proj = gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), layer.attn_output_weight, true, backend); - return o_proj; -} - - -size_t GemmaModel3n::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto gate = gb->matmul(input, layer.ffn_gate_weight, true, backend); - auto up = gb->matmul(input, layer.ffn_up_weight, true, backend); - - if (layer_idx < config_.activation_sparsity_ppf.size() && config_.activation_sparsity_ppf[layer_idx] > 0.0f) - gate = build_gaussian_topk(gb, gate, config_.activation_sparsity_ppf[layer_idx]); - - auto gate_activated = gb->gelu(gate); - auto activated = gb->multiply(gate_activated, up); - return gb->matmul(activated, layer.ffn_down_weight, true, backend); -} - - -size_t GemmaModel3n::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto normed = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - - auto laurel = build_laurel(gb, normed, layer_idx, backend); - auto attn_raw = build_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto combined = gb->add(gb->add(hidden, attn), laurel); - auto residual = gb->scalar_multiply(combined, RSQRT2); - - auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - auto mlp_raw = build_mlp(gb, pre_mlp, layer_idx, backend); - auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - - auto block_out = gb->add(residual, mlp); - return block_out; -} - - -size_t GemmaModel3n::build_preamble(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input, size_t* streams) { - uint32_t num_layers = config_.num_layers; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - uint32_t num_altup = config_.altup_num_inputs; - - token_input = gb->input({seq_len}, Precision::FP32); - auto x = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - pli_input = gb->input({seq_len}, Precision::FP32); - auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), - std::sqrt(static_cast(pli_dim))); - auto pli_proj = gb->scalar_multiply(gb->matmul(x, weight_nodes_.per_layer_model_proj, true, backend), - 1.0f / std::sqrt(static_cast(config_.hidden_dim))); - pli_proj = gb->reshape(pli_proj, {seq_len * num_layers, pli_dim}); - pli_proj = gb->rms_norm(pli_proj, weight_nodes_.per_layer_proj_norm, config_.layer_norm_eps); - pli_proj = gb->reshape(pli_proj, {seq_len, num_layers * pli_dim}); - auto pli_combined = gb->scalar_multiply(gb->add(pli_proj, pli_embed), RSQRT2); - - streams[0] = x; - for (uint32_t i = 1; i < num_altup; i++) { - streams[i] = build_magnitude_normalize(gb, x, gb->matmul(x, weight_nodes_.altup_proj_weights[i - 1], true, backend)); - } - - return pli_combined; -} - -void GemmaModel3n::build_layer(CactusGraph* gb, uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t pos_offset, size_t pli, size_t* streams) { - auto modalities = build_altup_router_modalities(gb, streams[0], layer_idx, backend); - - size_t predictions[4]; - build_altup_predict(gb, modalities, layer_idx, streams, predictions); - - auto activated = build_transformer_block(gb, predictions[0], layer_idx, backend, use_cache, pos_offset); - - auto modalities_post = build_altup_router_modalities(gb, activated, layer_idx, backend); - - build_altup_correct(gb, activated, modalities_post, layer_idx, backend, predictions, streams); - - if (config_.hidden_size_per_layer_input > 0) { - build_per_layer_input(gb, pli, layer_idx, backend, streams); - } -} - -size_t GemmaModel3n::build_output_head(CactusGraph* gb, size_t* streams, ComputeBackend backend) { - uint32_t num_altup = config_.altup_num_inputs; - - for (uint32_t i = 1; i < num_altup; i++) { - streams[i] = gb->matmul(streams[i], weight_nodes_.altup_unembed_proj_weights[i - 1], true, backend); - streams[i] = build_magnitude_normalize(gb, streams[0], streams[i]); - } - - auto hidden = streams[0]; - for (uint32_t i = 1; i < num_altup; i++) - hidden = gb->add(hidden, streams[i]); - hidden = gb->scalar_multiply(hidden, 1.0f / static_cast(num_altup)); - - auto final_normed = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - return final_normed; -} - -void GemmaModel3n::set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens) { - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - gb->set_input(pli_input, input_data.data(), Precision::FP32); -} - - -size_t GemmaModel3n::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (tokens.empty()) - throw std::runtime_error("Token sequence cannot be empty"); - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - size_t token_input, pli_input, streams[4]; - auto pli = build_preamble(gb, tokens.size(), backend, token_input, pli_input, streams); - - for (uint32_t i = 0; i < config_.num_layers; i++) - build_layer(gb, i, backend, use_cache, pos_offset, pli, streams); - - set_token_inputs(gb, token_input, pli_input, tokens); - return build_output_head(gb, streams, backend); -} - - -size_t GemmaModel3n::forward_split(const std::vector& tokens, bool use_cache) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - size_t seq_len = tokens.size(); - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - uint32_t num_layers = config_.num_layers; - uint32_t first_shared = num_layers - config_.num_kv_shared_layers; - - size_t token_input, pli_input, streams[4]; - auto pli = build_preamble(gb, seq_len, backend, token_input, pli_input, streams); - - for (uint32_t i = 0; i < first_shared; i++) - build_layer(gb, i, backend, use_cache, pos_offset, pli, streams); - - size_t hidden_dim = config_.hidden_dim; - for (uint32_t i = 0; i < config_.altup_num_inputs; i++) { - streams[i] = gb->index(streams[i], seq_len - 1, 0); - streams[i] = gb->reshape(streams[i], {1, hidden_dim}); - } - - auto pli_last = gb->index(pli, seq_len - 1, 0); - pli_last = gb->reshape(pli_last, {1, num_layers * config_.hidden_size_per_layer_input}); - - for (uint32_t i = first_shared; i < num_layers; i++) - build_layer(gb, i, backend, use_cache, pos_offset, pli_last, streams); - - set_token_inputs(gb, token_input, pli_input, tokens); - return build_output_head(gb, streams, backend); -} - - -void GemmaModel3n::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (tokens.empty()) - return; - - static constexpr size_t SPLIT_PREFILL_MIN_TOKENS = 32; - bool use_split = config_.num_kv_shared_layers > 0 - && tokens.size() >= SPLIT_PREFILL_MIN_TOKENS - && !std::getenv("CACTUS_DISABLE_SPLIT_PREFILL"); - - if (!use_split) { - Model::prefill(tokens, chunk_size, profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - - auto process_chunk = [&](const std::vector& chunk) { - forward_split(chunk, true); - gb->execute(profile_file); - update_kv_cache(gb, chunk.size()); - }; - - if (tokens.size() <= chunk_size) { - process_chunk(tokens); - return; - } - - size_t num_full_chunks = (tokens.size() - 1) / chunk_size; - for (size_t i = 0; i < num_full_chunks; ++i) { - size_t start = i * chunk_size; - std::vector chunk(tokens.begin() + start, tokens.begin() + start + chunk_size); - if (i == 1) - gb->set_prefill_mode(true); - process_chunk(chunk); - } - - gb->set_prefill_mode(false); - size_t final_start = num_full_chunks * chunk_size; - std::vector final_chunk(tokens.begin() + final_start, tokens.end()); - process_chunk(final_chunk); -} - -} -} diff --git a/cactus-transpile/models/model_lfm2.cpp b/cactus-transpile/models/model_lfm2.cpp deleted file mode 100644 index 12d10cad9..000000000 --- a/cactus-transpile/models/model_lfm2.cpp +++ /dev/null @@ -1,389 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include -#include -#include -#include -#include -#include -namespace cactus { -namespace engine { -LFM2Model::LFM2Model() : Model() { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -LFM2Model::LFM2Model(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -void LFM2Model::post_init() { - if (config_.conv_L_cache > 0) { - Precision cache_precision = Precision::FP16; - size_t conv_window = config_.conv_L_cache > 0 ? config_.conv_L_cache - 1 : 0; - conv_cache_.init(config_.num_layers, config_.hidden_dim, conv_window, cache_precision); - - } - last_forward_used_cache_ = false; - -} -void LFM2Model::reset_cache() { - Model::reset_cache(); - - if (conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } -} -bool LFM2Model::is_cache_empty() const { - return kv_cache_.is_empty(); -} -bool LFM2Model::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - if (!Model::init(model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -bool LFM2Model::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!Model::init(external_graph, model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -void LFM2Model::load_weights_to_graph(CactusGraph* gb) { - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - } - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - } - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - - } - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer_entry = weight_nodes_.layers[i]; - auto& layer = layer_entry.weights; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - bool is_conv_layer = false; - if (i < config_.layer_types.size()) { - std::string layer_type = config_.layer_types[i]; - is_conv_layer = (layer_type == "conv" || layer_type == "CONV"); - - } - if (is_conv_layer) { - layer_entry.type = WeightNodeIDs::LayerType::CONV; - layer.conv_in_proj_weight = gb->mmap_weights(layer_prefix + "conv_in_proj.weights"); - layer.conv_out_proj_weight = gb->mmap_weights(layer_prefix + "conv_out_proj.weights"); - layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); - - } else { - layer_entry.type = WeightNodeIDs::LayerType::ATTENTION; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); - - } - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - - } - -} -size_t LFM2Model::build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - size_t in_proj = gb->matmul(input, layer.conv_in_proj_weight, true, backend); - const auto& in_proj_buf = gb->get_output_buffer(in_proj); - if (in_proj_buf.shape.size() != 2 || (in_proj_buf.shape[1] % 3) != 0) { - throw std::runtime_error("Conv in_proj output must be [L, 3*C]"); - } - - const size_t L = in_proj_buf.shape[0]; - const size_t C = in_proj_buf.shape[1] / 3; - - size_t triplet = gb->reshape(in_proj, {L, static_cast(3), C}); - size_t B = gb->slice(triplet, 1, 0, 1); - size_t Cg = gb->slice(triplet, 1, 1, 1); - size_t X = gb->slice(triplet, 1, 2, 1); - - B = gb->reshape(B, {L, C}); - - Cg = gb->reshape(Cg, {L, C}); - - X = gb->reshape(X, {L, C}); - size_t Bx = gb->multiply(B, X); - if (use_cache) { - conv_cache_bx_nodes_[layer_idx] = Bx; - - } else { - conv_cache_bx_nodes_[layer_idx] = 0; - - } - const auto& wbuf = gb->get_output_buffer(layer.conv_depthwise_weight); - size_t K = wbuf.shape.back(); - - size_t conv_w = layer.conv_depthwise_weight; - if (wbuf.shape.size() == 2) { - K = wbuf.shape[1]; - conv_w = gb->reshape(conv_w, {wbuf.shape[0], static_cast(1), K}); - } else if (wbuf.shape.size() != 3){ - throw std::runtime_error("Unexpected depthwise weight rank"); - } - K = wbuf.shape.back(); - - size_t conv_input_lc = Bx; - if (use_cache && conv_cache_.window_size > 0) { - auto view = conv_cache_.get_window(layer_idx); - std::vector segments; - if (view.len2 > 0) { - size_t left_node = gb->input({view.len2, C}, conv_cache_.precision); - gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); - segments.push_back(left_node); - - } - if (view.len1 > 0) { - size_t right_node = gb->input({view.len1, C}, conv_cache_.precision); - gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); - segments.push_back(right_node); - - } - if (!segments.empty()) { - conv_input_lc = segments[0]; - for (size_t idx = 1; idx < segments.size(); ++idx) { - conv_input_lc = gb->concat(conv_input_lc, segments[idx], 0); - - } - conv_input_lc = gb->concat(conv_input_lc, Bx, 0); - - } - } - - const auto& conv_input_buf = gb->get_output_buffer(conv_input_lc); - size_t total_len = conv_input_buf.shape[0]; - - size_t x_nlc = gb->reshape(conv_input_lc, {static_cast(1), total_len, C}); - const size_t dilation = 1; - - size_t y_nlc = gb->conv1d_causal(x_nlc, conv_w, K, dilation); - size_t start = total_len > L ? total_len - L : 0; - size_t y_slice = gb->slice(y_nlc, 1, start, L); - - size_t y_lc = gb->reshape(y_slice, {L, C}); - size_t gated = gb->multiply(Cg, y_lc); - size_t projected = gb->matmul(gated, layer.conv_out_proj_weight, true, backend); - return projected; -} - -size_t LFM2Model::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto q_proj_linear = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj_linear = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj_linear = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - const auto& q_shape = gb->get_output_buffer(q_proj_linear).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - - auto q_proj_reshaped = gb->reshape(q_proj_linear, {batch_seq * num_heads, head_dim}); - - auto q_proj_norm = gb->rms_norm(q_proj_reshaped, layer.attn_q_norm_weight, config_.layer_norm_eps); - - auto q_proj = gb->reshape(q_proj_norm, {batch_seq, num_heads * head_dim}); - size_t num_kv_heads = config_.attention_kv_heads; - auto k_proj_reshaped = gb->reshape(k_proj_linear, {batch_seq * num_kv_heads, head_dim}); - - auto k_proj_norm = gb->rms_norm(k_proj_reshaped, layer.attn_k_norm_weight, config_.layer_norm_eps); - - auto k_proj = gb->reshape(k_proj_norm, {batch_seq, num_kv_heads * head_dim}); - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj_linear, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); - } - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - } - - size_t attn_output_4d; - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - auto projected = gb->matmul(attn_output, layer.attn_output_weight, true, backend); - return projected; -} - -size_t LFM2Model::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, ComputeBackend backend) const { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto gate = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - auto up = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - auto activated = gb->multiply(gb->silu(gate), up); - - auto down = gb->matmul(activated, layer.ffn_down_weight, true, backend); - - return down; -} - -size_t LFM2Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - size_t block_output; - if (layer_entry.type == WeightNodeIDs::LayerType::CONV) { - block_output = build_conv1d(gb, normalized_input, layer_idx, backend, use_cache); - } else { - if (layer_idx < conv_cache_bx_nodes_.size()) { - conv_cache_bx_nodes_[layer_idx] = 0; - } - block_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - } - - auto after_block = gb->add(hidden, block_output); - auto normalized_after_block = gb->rms_norm(after_block, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_block, layer_idx, backend); - auto block_result = gb->add(after_block, mlp_output); - return block_result; -} - -size_t LFM2Model::forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, - ComputeBackend backend, bool use_cache) { - if (seq_len == 0) { - throw std::runtime_error("Sequence length must be greater than zero"); - } - - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - } - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - last_forward_used_cache_ = use_cache; - - if (!use_cache && conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } - size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - size_t hidden = input_embeddings; - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - return final_hidden; -} -size_t LFM2Model::forward(CactusGraph* gb, const std::vector& tokens, - ComputeBackend backend, bool use_cache) { - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - auto seq_len = static_cast(tokens.size()); - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - auto final_hidden = forward(gb, hidden, seq_len, backend, use_cache); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - return final_hidden; -} -size_t LFM2Model::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - return forward(gb, tokens, backend, use_cache); -} -void LFM2Model::post_execute_updates(CactusGraph* gb, size_t /*seq_len*/) { - if (conv_cache_bx_nodes_.empty()) { - return; - } - - if (!last_forward_used_cache_ || conv_cache_.window_size == 0) { - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - return; - } - size_t layer_count = std::min(conv_cache_bx_nodes_.size(), weight_nodes_.layers.size()); - for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { - if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::CONV) { - conv_cache_bx_nodes_[layer_idx] = 0; - continue; - } - size_t bx_node = conv_cache_bx_nodes_[layer_idx]; - if (bx_node != 0) { - conv_cache_.update(gb, layer_idx, bx_node); - } - conv_cache_bx_nodes_[layer_idx] = 0; - } - last_forward_used_cache_ = false; - -} -} -} \ No newline at end of file diff --git a/cactus-transpile/models/model_lfm2moe.cpp b/cactus-transpile/models/model_lfm2moe.cpp deleted file mode 100644 index e867531bb..000000000 --- a/cactus-transpile/models/model_lfm2moe.cpp +++ /dev/null @@ -1,456 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -LFM2MoEModel::LFM2MoEModel() : Model() { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -LFM2MoEModel::LFM2MoEModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -void LFM2MoEModel::post_init() { - if (config_.num_dense_layers > config_.num_layers) { - throw std::runtime_error("num_dense_layers cannot be greater than num_layers for LFM2MoEModel"); - } - if (config_.num_experts == 0 || config_.num_experts_per_tok == 0 || config_.moe_intermediate_dim == 0) { - throw std::runtime_error("LFM2MoEModel requires num_experts, num_experts_per_tok, and moe_intermediate_dim to be non-zero"); - } - if (!config_.use_expert_bias || !config_.norm_topk_prob) { - throw std::runtime_error("LFM2MoEModel currently requires use_expert_bias=true and norm_topk_prob=true"); - } - if (config_.num_experts_per_tok > config_.num_experts) { - throw std::runtime_error("num_experts_per_tok cannot be greater than num_experts"); - } - - if (config_.conv_L_cache > 0) { - Precision cache_precision = Precision::FP16; - size_t conv_window = config_.conv_L_cache > 0 ? config_.conv_L_cache - 1 : 0; - conv_cache_.init(config_.num_layers, config_.hidden_dim, conv_window, cache_precision); - - } - last_forward_used_cache_ = false; - -} - -bool LFM2MoEModel::is_dense_layer(uint32_t layer_idx) const { - return layer_idx < config_.num_dense_layers; -} -void LFM2MoEModel::reset_cache() { - Model::reset_cache(); - - if (conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } -} -bool LFM2MoEModel::is_cache_empty() const { - return kv_cache_.is_empty(); -} -bool LFM2MoEModel::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - if (!Model::init(model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -bool LFM2MoEModel::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!Model::init(external_graph, model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -void LFM2MoEModel::load_weights_to_graph(CactusGraph* gb) { - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - } - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - } - auto mmap_or_throw = [&](const std::string& path, bool embedding) -> size_t { - size_t node_id = embedding ? gb->mmap_embeddings(path) : gb->mmap_weights(path); - return node_id; - }; - - embedding_node_id_ = mmap_or_throw(embedding_file_path_, true); - weight_nodes_.output_norm_weight = mmap_or_throw(model_folder_path_ + "/output_norm.weights", false); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - - } else { - weight_nodes_.output_weight = mmap_or_throw(model_folder_path_ + "/output_weight.weights", false); - output_weight_node_id_ = weight_nodes_.output_weight; - - } - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer_entry = weight_nodes_.layers[i]; - auto& layer = layer_entry.weights; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - bool is_conv_layer = false; - if (i < config_.layer_types.size()) { - std::string layer_type = config_.layer_types[i]; - is_conv_layer = (layer_type == "conv" || layer_type == "CONV"); - - } - if (is_conv_layer) { - layer_entry.type = WeightNodeIDs::LayerType::CONV; - layer.conv_in_proj_weight = mmap_or_throw(layer_prefix + "conv_in_proj.weights", false); - layer.conv_out_proj_weight = mmap_or_throw(layer_prefix + "conv_out_proj.weights", false); - layer.conv_depthwise_weight = mmap_or_throw(layer_prefix + "conv_depthwise.weights", false); - - } else { - layer_entry.type = WeightNodeIDs::LayerType::ATTENTION; - layer.attn_q_weight = mmap_or_throw(layer_prefix + "attn_q.weights", false); - layer.attn_k_weight = mmap_or_throw(layer_prefix + "attn_k.weights", false); - layer.attn_v_weight = mmap_or_throw(layer_prefix + "attn_v.weights", false); - layer.attn_output_weight = mmap_or_throw(layer_prefix + "attn_output.weights", false); - layer.attn_q_norm_weight = mmap_or_throw(layer_prefix + "attn_q_norm.weights", false); - layer.attn_k_norm_weight = mmap_or_throw(layer_prefix + "attn_k_norm.weights", false); - - } - layer.input_layernorm_weight = mmap_or_throw(layer_prefix + "input_norm.weights", false); - layer.post_attention_layernorm_weight = mmap_or_throw(layer_prefix + "post_attn_norm.weights", false); - if (is_dense_layer(i)) { - layer.ffn_gate_weight = mmap_or_throw(layer_prefix + "ffn_gate.weights", false); - layer.ffn_up_weight = mmap_or_throw(layer_prefix + "ffn_up.weights", false); - layer.ffn_down_weight = mmap_or_throw(layer_prefix + "ffn_down.weights", false); - } else { - layer.moe_router_weight = mmap_or_throw(layer_prefix + "moe_router.weights", false); - layer.moe_expert_bias = mmap_or_throw(layer_prefix + "moe_expert_bias.weights", false); - layer.moe_experts.resize(config_.num_experts); - for (uint32_t expert_idx = 0; expert_idx < config_.num_experts; ++expert_idx) { - auto& expert = layer.moe_experts[expert_idx]; - std::string expert_prefix = layer_prefix + "moe_expert_" + std::to_string(expert_idx) + "_"; - expert.w1_weight = mmap_or_throw(expert_prefix + "w1.weights", false); - expert.w3_weight = mmap_or_throw(expert_prefix + "w3.weights", false); - expert.w2_weight = mmap_or_throw(expert_prefix + "w2.weights", false); - } - } - - } - -} -size_t LFM2MoEModel::build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - size_t in_proj = gb->matmul(input, layer.conv_in_proj_weight, true, backend); - const auto& in_proj_buf = gb->get_output_buffer(in_proj); - if (in_proj_buf.shape.size() != 2 || (in_proj_buf.shape[1] % 3) != 0) { - throw std::runtime_error("Conv in_proj output must be [L, 3*C]"); - } - - const size_t L = in_proj_buf.shape[0]; - const size_t C = in_proj_buf.shape[1] / 3; - - size_t triplet = gb->reshape(in_proj, {L, static_cast(3), C}); - size_t B = gb->slice(triplet, 1, 0, 1); - size_t Cg = gb->slice(triplet, 1, 1, 1); - size_t X = gb->slice(triplet, 1, 2, 1); - - B = gb->reshape(B, {L, C}); - - Cg = gb->reshape(Cg, {L, C}); - - X = gb->reshape(X, {L, C}); - size_t Bx = gb->multiply(B, X); - if (use_cache) { - conv_cache_bx_nodes_[layer_idx] = Bx; - - } else { - conv_cache_bx_nodes_[layer_idx] = 0; - - } - const auto& wbuf = gb->get_output_buffer(layer.conv_depthwise_weight); - size_t K = wbuf.shape.back(); - - size_t conv_w = layer.conv_depthwise_weight; - if (wbuf.shape.size() == 2) { - K = wbuf.shape[1]; - conv_w = gb->reshape(conv_w, {wbuf.shape[0], static_cast(1), K}); - } else if (wbuf.shape.size() != 3){ - throw std::runtime_error("Unexpected depthwise weight rank"); - } - K = wbuf.shape.back(); - - size_t conv_input_lc = Bx; - if (use_cache && conv_cache_.window_size > 0) { - auto view = conv_cache_.get_window(layer_idx); - std::vector segments; - if (view.len2 > 0) { - size_t left_node = gb->input({view.len2, C}, conv_cache_.precision); - gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); - segments.push_back(left_node); - - } - if (view.len1 > 0) { - size_t right_node = gb->input({view.len1, C}, conv_cache_.precision); - gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); - segments.push_back(right_node); - - } - if (!segments.empty()) { - conv_input_lc = segments[0]; - for (size_t idx = 1; idx < segments.size(); ++idx) { - conv_input_lc = gb->concat(conv_input_lc, segments[idx], 0); - - } - conv_input_lc = gb->concat(conv_input_lc, Bx, 0); - - } - } - - const auto& conv_input_buf = gb->get_output_buffer(conv_input_lc); - size_t total_len = conv_input_buf.shape[0]; - - size_t x_nlc = gb->reshape(conv_input_lc, {static_cast(1), total_len, C}); - const size_t dilation = 1; - - size_t y_nlc = gb->conv1d_causal(x_nlc, conv_w, K, dilation); - size_t start = total_len > L ? total_len - L : 0; - size_t y_slice = gb->slice(y_nlc, 1, start, L); - - size_t y_lc = gb->reshape(y_slice, {L, C}); - size_t gated = gb->multiply(Cg, y_lc); - size_t projected = gb->matmul(gated, layer.conv_out_proj_weight, true, backend); - return projected; -} - -size_t LFM2MoEModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto q_proj_linear = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj_linear = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj_linear = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - const auto& q_shape = gb->get_output_buffer(q_proj_linear).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - - auto q_proj_reshaped = gb->reshape(q_proj_linear, {batch_seq * num_heads, head_dim}); - - auto q_proj_norm = gb->rms_norm(q_proj_reshaped, layer.attn_q_norm_weight, config_.layer_norm_eps); - - auto q_proj = gb->reshape(q_proj_norm, {batch_seq, num_heads * head_dim}); - size_t num_kv_heads = config_.attention_kv_heads; - auto k_proj_reshaped = gb->reshape(k_proj_linear, {batch_seq * num_kv_heads, head_dim}); - - auto k_proj_norm = gb->rms_norm(k_proj_reshaped, layer.attn_k_norm_weight, config_.layer_norm_eps); - - auto k_proj = gb->reshape(k_proj_norm, {batch_seq, num_kv_heads * head_dim}); - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj_linear, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); - } - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - } - - size_t attn_output_4d; - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - auto projected = gb->matmul(attn_output, layer.attn_output_weight, true, backend); - return projected; -} - -size_t LFM2MoEModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, ComputeBackend backend) const { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - if (is_dense_layer(layer_idx)) { - auto gate = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - auto up = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - auto activated = gb->multiply(gb->silu(gate), up); - auto down = gb->matmul(activated, layer.ffn_down_weight, true, backend); - return down; - } - - auto router_logits = gb->matmul(normalized_h, layer.moe_router_weight, true, backend); - auto routing_probs = gb->sigmoid(router_logits); - auto scores_for_routing = gb->add(routing_probs, layer.moe_expert_bias); - auto topk_indices_and_values = gb->topk(scores_for_routing, config_.num_experts_per_tok); - auto topk_indices = gb->index(topk_indices_and_values, 0, 0); - - std::vector w1_weights, w3_weights, w2_weights; - w1_weights.reserve(config_.num_experts); - w3_weights.reserve(config_.num_experts); - w2_weights.reserve(config_.num_experts); - for (uint32_t expert_idx = 0; expert_idx < config_.num_experts; ++expert_idx) { - const auto& expert = layer.moe_experts[expert_idx]; - w1_weights.push_back(expert.w1_weight); - w3_weights.push_back(expert.w3_weight); - w2_weights.push_back(expert.w2_weight); - } - - return gb->moe_layer( - normalized_h, - routing_probs, - topk_indices, - w1_weights, - w3_weights, - w2_weights, - config_.num_experts, - config_.num_experts_per_tok, - config_.norm_topk_prob, - 1e-6f, - config_.routed_scaling_factor - ); -} - -size_t LFM2MoEModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - size_t block_output; - if (layer_entry.type == WeightNodeIDs::LayerType::CONV) { - block_output = build_conv1d(gb, normalized_input, layer_idx, backend, use_cache); - } else { - if (layer_idx < conv_cache_bx_nodes_.size()) { - conv_cache_bx_nodes_[layer_idx] = 0; - } - block_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - } - - auto after_block = gb->add(hidden, block_output); - auto normalized_after_block = gb->rms_norm(after_block, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_block, layer_idx, backend); - auto block_result = gb->add(after_block, mlp_output); - return block_result; -} - -size_t LFM2MoEModel::forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, - ComputeBackend backend, bool use_cache) { - if (seq_len == 0) { - throw std::runtime_error("Sequence length must be greater than zero"); - } - - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - } - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - last_forward_used_cache_ = use_cache; - - if (!use_cache && conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } - size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - size_t hidden = input_embeddings; - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - return final_hidden; -} -size_t LFM2MoEModel::forward(CactusGraph* gb, const std::vector& tokens, - ComputeBackend backend, bool use_cache) { - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - auto seq_len = static_cast(tokens.size()); - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - auto final_hidden = forward(gb, hidden, seq_len, backend, use_cache); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - return final_hidden; -} -size_t LFM2MoEModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - return forward(gb, tokens, backend, use_cache); -} -void LFM2MoEModel::post_execute_updates(CactusGraph* gb, size_t /*seq_len*/) { - if (conv_cache_bx_nodes_.empty()) { - return; - } - - if (!last_forward_used_cache_ || conv_cache_.window_size == 0) { - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - return; - } - size_t layer_count = std::min(conv_cache_bx_nodes_.size(), weight_nodes_.layers.size()); - for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { - if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::CONV) { - conv_cache_bx_nodes_[layer_idx] = 0; - continue; - } - size_t bx_node = conv_cache_bx_nodes_[layer_idx]; - if (bx_node != 0) { - conv_cache_.update(gb, layer_idx, bx_node); - } - conv_cache_bx_nodes_[layer_idx] = 0; - } - last_forward_used_cache_ = false; - -} -} -} diff --git a/cactus-transpile/models/model_lfm2vl.cpp b/cactus-transpile/models/model_lfm2vl.cpp deleted file mode 100644 index c4357ad0b..000000000 --- a/cactus-transpile/models/model_lfm2vl.cpp +++ /dev/null @@ -1,607 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Lfm2VlModel::Lfm2VlModel() : Model() { - config_.model_type = Config::ModelType::LFM2; -} - -Lfm2VlModel::Lfm2VlModel(const Config& config) - : Model(config), - vision_tower_(config), - language_model_(config) { - Siglip2Preprocessor::Config preprocessor_config; - preprocessor_config.patch_size = static_cast(config.vision_patch_size); - preprocessor_config.downsample_factor = static_cast(config.downsample_factor); - preprocessor_config.min_tiles = static_cast(config.min_tiles); - preprocessor_config.max_tiles = static_cast(config.max_tiles); - preprocessor_config.use_thumbnail = config.use_thumbnail; - preprocessor_config.min_image_tokens = static_cast(config.min_image_tokens); - preprocessor_config.max_image_tokens = static_cast(config.max_image_tokens); - preprocessor_config.max_num_patches = static_cast(config.max_num_patches); - preprocessor_config.tile_size = static_cast(config.tile_size); - preprocessor_config.max_pixels_tolerance = config.max_pixels_tolerance; - preprocessor_config.do_resize = true; - preprocessor_config.do_rescale = true; - preprocessor_config.do_normalize = true; - preprocessor_config.do_convert_rgb = true; - preprocessor_config.do_image_splitting = config.do_image_splitting; - preprocessor_config.rescale_factor = config.rescale_factor; - preprocessor_config.image_mean[0] = config.image_mean; - preprocessor_config.image_mean[1] = config.image_mean; - preprocessor_config.image_mean[2] = config.image_mean; - preprocessor_config.image_std[0] = config.image_std; - preprocessor_config.image_std[1] = config.image_std; - preprocessor_config.image_std[2] = config.image_std; - - preprocessor_ = Siglip2Preprocessor(preprocessor_config); -} - -bool Lfm2VlModel::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - - if (!Model::init(model_folder, context_size, system_prompt, false)) { - return false; - } - auto* shared_graph = static_cast(graph_handle_); - if (!shared_graph) { - throw std::runtime_error("Shared graph was not initialized for Lfm2VlModel"); - } - std::string vision_folder = model_folder; - if (!vision_tower_.init(shared_graph, vision_folder, context_size, "", false)) { - throw std::runtime_error("Failed to initialize vision tower"); - } - - vision_weights_loaded_ = true; - if (!language_model_.init(shared_graph, model_folder, context_size, system_prompt, false)) { - throw std::runtime_error("Failed to initialize language model"); - } - - language_model_.output_weight_node_id_ = output_weight_node_id_; - - language_weights_loaded_ = true; - - if (do_warmup) { - std::string warmup_text = system_prompt.empty() ? "Hello" : system_prompt; - auto warmup_tokens = tokenizer_->encode(warmup_text); - language_model_.decode(warmup_tokens, config_.default_temperature, config_.default_top_p, config_.default_top_k, ""); - language_model_.reset_cache(); - } - - return true; -} - -void Lfm2VlModel::reset_cache() { - Model::reset_cache(); - language_model_.reset_cache(); - image_prefill_completed_ = false; - last_token_count_ = 0; -} - -void Lfm2VlModel::load_weights_to_graph(CactusGraph* gb) { - namespace fs = std::filesystem; - fs::path base(model_folder_path_); - - auto resolve_weight = [&](const std::string& primary, const std::string& fallback = "") -> std::string { - fs::path primary_path = base / primary; - if (fs::exists(primary_path)) { - return primary_path.string(); - } - if (!fallback.empty()) { - fs::path fallback_path = base / fallback; - if (fs::exists(fallback_path)) { - return fallback_path.string(); - } - } - return primary_path.string(); - }; - - try { - projector_weights_.layer_norm_weight = gb->mmap_weights(resolve_weight("projector_layer_norm.weights")); - projector_weights_.layer_norm_bias = gb->mmap_weights(resolve_weight("projector_layer_norm.bias.weights")); - } catch (const std::exception&) { - projector_weights_.layer_norm_weight = 0; - projector_weights_.layer_norm_bias = 0; - } - - projector_weights_.linear_1_weight = gb->mmap_weights(resolve_weight("projector_linear_1.weights", "projector_linear1.weights")); - projector_weights_.linear_1_bias = gb->mmap_weights(resolve_weight("projector_linear_1.bias.weights", "projector_linear1.bias.weights")); - projector_weights_.linear_2_weight = gb->mmap_weights(resolve_weight("projector_linear_2.weights", "projector_linear2.weights")); - projector_weights_.linear_2_bias = gb->mmap_weights(resolve_weight("projector_linear_2.bias.weights", "projector_linear2.bias.weights")); - output_weight_node_id_ = gb->mmap_weights(resolve_weight("token_embeddings.weights")); -} - -size_t Lfm2VlModel::pixel_unshuffle(CactusGraph* gb, size_t hidden_states, - size_t height, size_t width, size_t channels) { - - const size_t factor = config_.downsample_factor; - const size_t new_height = height / factor; - const size_t new_width = width / factor; - size_t step1 = gb->reshape(hidden_states, {1, height, new_width, channels * factor}); - step1 = gb->transposeN(step1, {0, 2, 1, 3}); - size_t step2 = gb->reshape(step1, {1, new_width, new_height, channels * factor * factor}); - size_t result = gb->transposeN(step2, {0, 2, 1, 3}); - return result; -} - -size_t Lfm2VlModel::build_multimodal_projector(CactusGraph* gb, size_t image_features, - size_t tile_h, size_t tile_w, ComputeBackend backend) { - const size_t vision_hidden = config_.vision_embed_dim; - - const auto& input_buf = gb->get_output_buffer(image_features); - size_t image_features_fp16 = (input_buf.precision == Precision::FP16) - ? image_features - : gb->precision_cast(image_features, Precision::FP16); - - size_t unshuffled = pixel_unshuffle(gb, image_features_fp16, tile_h, tile_w, vision_hidden); - const size_t factor = config_.downsample_factor; - const size_t new_h = tile_h / factor; - const size_t new_w = tile_w / factor; - const size_t in_channels = vision_hidden * factor * factor; - const size_t seq_len = new_h * new_w; - size_t flattened = gb->reshape(unshuffled, {seq_len, in_channels}); - size_t normalized = flattened; - if (projector_weights_.layer_norm_weight != 0 && projector_weights_.layer_norm_bias != 0) { - normalized = gb->layernorm(flattened, projector_weights_.layer_norm_weight, - projector_weights_.layer_norm_bias, config_.layer_norm_eps); - } - size_t hidden = gb->matmul(normalized, projector_weights_.linear_1_weight, true, backend); - hidden = gb->add(hidden, projector_weights_.linear_1_bias); - hidden = gb->gelu(hidden); - size_t output = gb->matmul(hidden, projector_weights_.linear_2_weight, true, backend); - output = gb->add(output, projector_weights_.linear_2_bias); - return output; -} - -std::vector Lfm2VlModel::get_image_embeddings(const std::string& image_path) { - return vision_tower_.get_image_embedding(image_path); -} - -std::vector Lfm2VlModel::get_image_features( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - - size_t vision_output = vision_tower_.forward_vision(gb, preprocessed_image, backend); - std::vector projected_features; - projected_features.reserve(preprocessed_image.spatial_shapes.size()); - - size_t offset = 0; - for (size_t tile_idx = 0; tile_idx < preprocessed_image.spatial_shapes.size(); ++tile_idx) { - const auto& shape = preprocessed_image.spatial_shapes[tile_idx]; - const size_t tile_h = shape.first; - const size_t tile_w = shape.second; - const size_t tile_tokens = tile_h * tile_w; - const size_t factor = config_.downsample_factor; - if (factor == 0) { - throw std::runtime_error("Downsample factor must be greater than zero"); - } - if (tile_h % factor != 0 || tile_w % factor != 0) { - throw std::runtime_error("Tile dimensions must be divisible by downsample factor"); - } - const size_t new_h = tile_h / factor; - const size_t new_w = tile_w / factor; - const size_t projected_tokens = new_h * new_w; - - size_t tile_features = gb->slice(vision_output, 0, offset, tile_tokens); - offset += tile_tokens; - - size_t reshaped = gb->reshape(tile_features, {1, tile_h, tile_w, config_.vision_embed_dim}); - size_t projected = build_multimodal_projector(gb, reshaped, tile_h, tile_w, backend); - ProjectedTileFeature feature{}; - feature.node_id = projected; - feature.token_count = projected_tokens; - projected_features.push_back(feature); - - } - - return projected_features; -} - -Lfm2VlModel::MergedEmbeddingResult Lfm2VlModel::merge_image_text_embeddings( - CactusGraph* gb, - const std::vector& tokens, - const std::vector>& image_embedding_nodes, - std::vector& text_embedding_inputs) { - - text_embedding_inputs.clear(); - - Tokenizer* tokenizer = language_model_.get_tokenizer(); - if (!tokenizer) { - throw std::runtime_error("Tokenizer must be initialized before merging embeddings"); - } - - const uint32_t image_token_id = tokenizer->get_image_token_id(); - - auto get_token_id = [tokenizer](const std::string& token_text) -> uint32_t { - auto encoded = tokenizer->encode(token_text); - if (encoded.size() != 1) { - - throw std::runtime_error("Expected single token encoding for " + token_text); - } - return encoded[0]; - }; - - const uint32_t image_start_id = get_token_id("<|image_start|>"); - const uint32_t image_end_id = get_token_id("<|image_end|>"); - - std::vector sequence_nodes; - sequence_nodes.reserve(tokens.size() + image_embedding_nodes.size()); - - std::vector current_segment; - current_segment.reserve(tokens.size()); - - size_t total_seq_len = 0; - - auto flush_segment = [&](void) { - if (current_segment.empty()) { - - return; - } - - const size_t segment_len = current_segment.size(); - TextEmbeddingInput segment; - segment.tokens.swap(current_segment); - segment.input_node = gb->input({segment.tokens.size()}, Precision::FP32); - - const auto& embedding_buffer = gb->get_output_buffer(language_model_.embedding_node_id_); - - for (size_t i = 0; i < embedding_buffer.shape.size(); ++i) { - (void)i; - } - - size_t embedding_node = gb->embedding(language_model_.embedding_node_id_, segment.input_node); - - text_embedding_inputs.push_back(std::move(segment)); - - sequence_nodes.push_back(embedding_node); - - total_seq_len += segment_len; - - current_segment.clear(); - - }; - - size_t token_index = 0; - size_t image_index = 0; - - while (token_index < tokens.size()) { - uint32_t token_id = tokens[token_index]; - - if (token_id == image_start_id) { - flush_segment(); - - if (image_index >= image_embedding_nodes.size()) { - - throw std::runtime_error("Encountered <|image_start|> without corresponding image features"); - } - - current_segment.push_back(token_id); - ++token_index; - - const auto& tiles = image_embedding_nodes[image_index]; - size_t tile_index = 0; - while (token_index < tokens.size()) { - uint32_t inner_token = tokens[token_index]; - if (inner_token == image_token_id) { - flush_segment(); - - if (tile_index >= tiles.size()) { - - throw std::runtime_error("More placeholders than projected tile features"); - } - - const auto& tile = tiles[tile_index++]; - sequence_nodes.push_back(tile.node_id); - - total_seq_len += tile.token_count; - - for (size_t count = 0; count < tile.token_count; ++count) { - if (token_index >= tokens.size()) { - throw std::runtime_error("Insufficient tokens for projected features"); - } - if (tokens[token_index] != image_token_id) { - throw std::runtime_error("Unexpected token encountered within image feature span"); - } - ++token_index; - - } - continue; - } - - current_segment.push_back(inner_token); - - ++token_index; - - if (inner_token == image_end_id) { - flush_segment(); - break; - } - } - - if (tile_index != tiles.size()) { - if (tile_index < tiles.size()) { - for (size_t remaining = tile_index; remaining < tiles.size(); ++remaining) { - (void)tiles[remaining]; - } - } - throw std::runtime_error("Unused projected tile features remain after processing image block"); - } - - ++image_index; - - } else { - current_segment.push_back(token_id); - - ++token_index; - - } - } - - flush_segment(); - if (image_index != image_embedding_nodes.size()) { - throw std::runtime_error("Not all image features were consumed while merging embeddings"); - } - - if (sequence_nodes.empty()) { - throw std::runtime_error("Failed to build embedding sequence from provided tokens"); - } - - size_t merged = sequence_nodes[0]; - for (size_t idx = 1; idx < sequence_nodes.size(); ++idx) { - merged = gb->concat(merged, sequence_nodes[idx], 0); - - } - - return MergedEmbeddingResult{merged, total_seq_len}; -} - -size_t Lfm2VlModel::build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_attention should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const { - throw std::runtime_error("build_mlp should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_transformer_block should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t final_hidden = language_model_.forward(gb, tokens, backend, use_cache); - - return final_hidden; -} - -uint32_t Lfm2VlModel::decode(const std::vector& tokens, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (temperature < 0) { - temperature = config_.default_temperature; - } - if (top_p < 0) { - top_p = config_.default_top_p; - } - if (top_k == 0) { - top_k = config_.default_top_k; - } - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -void Lfm2VlModel::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - - language_model_.prefill(tokens, chunk_size, profile_file); -} - -void Lfm2VlModel::prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (tokens.empty()) { - return; - } - if (image_paths.empty()) { - prefill(tokens, get_prefill_chunk_size(), profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto forward_result = forward_images(gb, tokens, image_paths, backend, true); - gb->execute(profile_file); - - language_model_.post_execute_updates(gb, forward_result.seq_len); - language_model_.update_kv_cache(gb, forward_result.seq_len); - - image_prefill_completed_ = true; - last_token_count_ = tokens.size(); -} - -Lfm2VlModel::ForwardImageResult Lfm2VlModel::forward_images( - CactusGraph* gb, - const std::vector& tokens, - const std::vector& image_paths, - ComputeBackend backend, - bool use_cache) { - if (!gb) { - throw std::runtime_error("Graph must be initialized before forwarding"); - } - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - std::vector> all_image_embeddings; - all_image_embeddings.reserve(image_paths.size()); - for (const auto& image_path : image_paths) { - auto preprocessed = preprocessor_.preprocess_from_file(image_path); - - auto image_features = get_image_features(gb, preprocessed, backend); - - all_image_embeddings.push_back(std::move(image_features)); - } - - std::vector text_embedding_inputs; - text_embedding_inputs.reserve(tokens.size() / 4 + 1); - - auto merged_embeddings = merge_image_text_embeddings(gb, tokens, all_image_embeddings, text_embedding_inputs); - if (merged_embeddings.seq_len == 0) { - throw std::runtime_error("Merged embedding sequence length cannot be zero"); - } - - for (const auto& embedding_input : text_embedding_inputs) { - if (embedding_input.tokens.empty()) { - continue; - } - - std::vector segment_data(embedding_input.tokens.size()); - for (size_t i = 0; i < embedding_input.tokens.size(); ++i) { - segment_data[i] = static_cast(embedding_input.tokens[i]); - } - gb->set_input(embedding_input.input_node, segment_data.data(), Precision::FP32); - } - size_t final_hidden = language_model_.forward(gb, merged_embeddings.node_id, merged_embeddings.seq_len, backend, use_cache); - return ForwardImageResult{final_hidden, merged_embeddings.seq_len}; -} - -uint32_t Lfm2VlModel::decode_with_images( - const std::vector& tokens, - const std::vector& image_paths, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty) { - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (image_paths.empty()) { - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); - } - - if (temperature < 0) { - temperature = config_.default_temperature; - } - if (top_p < 0) { - top_p = config_.default_top_p; - } - if (top_k == 0) { - top_k = config_.default_top_k; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - bool cache_empty = language_model_.is_cache_empty(); - bool need_prefill = cache_empty || !image_prefill_completed_; - - if (!need_prefill && tokens.size() <= last_token_count_) { - - reset_cache(); - need_prefill = true; - } - - size_t seq_len_for_updates = 0; - size_t final_hidden_node = 0; - - if (need_prefill) { - auto forward_result = forward_images(gb, tokens, image_paths, backend, true); - - final_hidden_node = forward_result.final_hidden_node; - seq_len_for_updates = forward_result.seq_len; - image_prefill_completed_ = true; - last_token_count_ = tokens.size(); - } else { - size_t delta = tokens.size() - last_token_count_; - if (delta > tokens.size()) { - delta = tokens.size(); - } - if (delta == 0) { - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty for cached decode step"); - } - delta = 1; - - } - std::vector incremental_tokens(tokens.end() - delta, tokens.end()); - - final_hidden_node = language_model_.forward(gb, incremental_tokens, backend, true); - - seq_len_for_updates = incremental_tokens.size(); - last_token_count_ = tokens.size(); - } - - auto logits_node_id = gb->matmul(final_hidden_node, language_model_.output_weight_node_id_, true, backend); - size_t sampled_token_id = - language_model_.sample_token(gb, logits_node_id, temperature, top_p, top_k, min_p, repetition_penalty, nullptr); - if (!profile_file.empty()) { - gb->execute(profile_file); - - } else { - gb->execute(); - - } - - compute_entropy(gb, logits_node_id, out_entropy); - - language_model_.post_execute_updates(gb, seq_len_for_updates); - language_model_.update_kv_cache(gb, seq_len_for_updates); - - auto* output_ptr = gb->get_output(sampled_token_id); - uint32_t result = *static_cast(output_ptr); - language_model_.record_sampled_token(result); - return result; -} - -} -} diff --git a/cactus-transpile/models/model_moonshine.cpp b/cactus-transpile/models/model_moonshine.cpp deleted file mode 100644 index fe03f5983..000000000 --- a/cactus-transpile/models/model_moonshine.cpp +++ /dev/null @@ -1,596 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include -#include -#include -namespace cactus { -namespace engine { - -static size_t moonshine_downsampled_len(size_t audio_len) { - if (audio_len < 127) { - return 0; - } - size_t l1 = (audio_len - 127) / 64 + 1; - if (l1 < 7) { - return 0; - } - size_t l2 = (l1 - 7) / 3 + 1; - if (l2 < 3) { - return 0; - } - size_t l3 = (l2 - 1) / 2 + 1; - return l3; -} - -MoonshineModel::MoonshineModel() : Model() {} -MoonshineModel::MoonshineModel(const Config& config) : Model(config) { - weight_nodes_.encoder_layers.resize(config.num_encoder_layers); - weight_nodes_.decoder_layers.resize(config.num_decoder_layers); - float hd = static_cast(config.attention_head_dim); - if (hd <= 0.0f) { - hd = 64.0f; - } - attention_scale_ = 1.0f / std::sqrt(hd); - encoder_block_out_nodes_.resize(config.num_encoder_layers, 0); - encoder_k_persistent_.assign(config.num_decoder_layers, 0); - encoder_v_persistent_.assign(config.num_decoder_layers, 0); -} -void MoonshineModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - output_weight_node_id_ = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - if (output_weight_node_id_ == 0) { - output_weight_node_id_ = gb->mmap_weights(model_folder_path_ + "/output_layer.weights"); - } - weight_nodes_.output_weight = output_weight_node_id_; - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (!input_shape.empty()) { - npu_encoder_->preallocate(input_shape, "x", ""); - } - } - else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - if (!use_npu_encoder_) { - weight_nodes_.encoder_conv1_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_weight.weights"); - weight_nodes_.encoder_conv2_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_weight.weights"); - weight_nodes_.encoder_conv2_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_bias.bias"); - weight_nodes_.encoder_conv3_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv3_weight.weights"); - weight_nodes_.encoder_conv3_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv3_bias.bias"); - weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_norm_weight.weights"); - weight_nodes_.encoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/encoder_norm_bias.bias"); - weight_nodes_.encoder_layer_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_layer_norm_weight.weights"); - } - for (uint32_t i = 0; i < config_.num_decoder_layers; i++) { - auto& layer = weight_nodes_.decoder_layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.decoder_encoder_attn_k_weight = gb->mmap_weights(layer_prefix + "encoder_attn_k.weights"); - layer.decoder_encoder_attn_q_weight = gb->mmap_weights(layer_prefix + "encoder_attn_q.weights"); - layer.decoder_encoder_attn_v_weight = gb->mmap_weights(layer_prefix + "encoder_attn_v.weights"); - layer.decoder_encoder_attn_output_weight = gb->mmap_weights(layer_prefix + "encoder_attn_output.weights"); - layer.decoder_post_encoder_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - if (config_.decoder_act_gelu) { - layer.decoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.decoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - } - else { - layer.decoder_ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.decoder_ffn_gate_bias = gb->mmap_weights(layer_prefix + "ffn_gate.bias"); - layer.decoder_ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.decoder_ffn_up_bias = gb->mmap_weights(layer_prefix + "ffn_up.bias"); - } - layer.decoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.decoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - layer.decoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); - layer.decoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.decoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.decoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.decoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.decoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - } - if (!use_npu_encoder_) { - for (uint32_t i = 0; i < config_.num_encoder_layers; i++) { - auto& layer = weight_nodes_.encoder_layers[i]; - std::string layer_prefix = model_folder_path_ + "/encoder_layer_" + std::to_string(i) + "_"; - if (config_.encoder_act_gelu) { - layer.encoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.encoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - } - else { - layer.encoder_ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.encoder_ffn_gate_bias = gb->mmap_weights(layer_prefix + "ffn_gate.bias"); - layer.encoder_ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.encoder_ffn_up_bias = gb->mmap_weights(layer_prefix + "ffn_up.bias"); - } - layer.encoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.encoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - layer.encoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.encoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.encoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.encoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.encoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.encoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - } - } -} -static size_t build_encoder_mlp_gelu(CactusGraph* gb, size_t input, size_t w1, size_t b1, size_t w2, size_t b2, ComputeBackend backend, uint32_t /*layer_idx*/) { - auto ffn1_weight = gb->matmul(input, w1, true, backend); - auto ffn1_bias = gb->add(ffn1_weight, b1); - auto ffn1_act = gb->gelu(ffn1_bias); - auto ffn2_weight = gb->matmul(ffn1_act, w2, true, backend); - auto ffn2_bias = gb->add(ffn2_weight, b2); - return ffn2_bias; -} -static size_t build_decoder_mlp_silu(CactusGraph* gb, size_t input, size_t w_gate, size_t b_gate, size_t w_up, size_t b_up, size_t w2, size_t b2, ComputeBackend backend, uint32_t /*layer_idx*/) { - auto gate_weight = gb->matmul(input, w_gate, true, backend); - auto gate_bias = gb->add(gate_weight, b_gate); - auto gate_act = gb->silu(gate_bias); - auto up_weight = gb->matmul(input, w_up, true, backend); - auto up_bias = gb->add(up_weight, b_up); - auto intermediate = gb->multiply(gate_act, up_bias); - auto output_weight = gb->matmul(intermediate, w2, true, backend); - auto output_bias = gb->add(output_weight, b2); - return output_bias; -} -size_t MoonshineModel::build_decoder_cross_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool /*use_cache*/, size_t /*position_offset*/){ - const auto& layer = weight_nodes_.decoder_layers[layer_idx]; - size_t q = gb->matmul(input, layer.decoder_encoder_attn_q_weight, true, backend); - const auto& q_buf = gb->get_output_buffer(q); - if (q_buf.shape.size() != 2) { - throw std::runtime_error("encoder cross-attn: q must be [T_dec, D]"); - } - size_t T_dec = q_buf.shape[0]; - size_t q_heads = config_.attention_heads; - size_t kv_heads = config_.attention_kv_heads; - size_t head_dim = config_.attention_head_dim; - q = gb->reshape(q, {1, T_dec, q_heads, head_dim}); - size_t k_4d = 0; - size_t v_4d = 0; - bool is_pop = (encoder_k_persistent_[layer_idx] != 0 && gb->is_populated(encoder_k_persistent_[layer_idx])); - std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + ".encoder_attn."; - if (is_pop) { - k_4d = encoder_k_persistent_[layer_idx]; - v_4d = encoder_v_persistent_[layer_idx]; - } - else { - size_t enc_norm = last_encoder_post_norm_node_; - size_t k = gb->matmul(enc_norm, layer.decoder_encoder_attn_k_weight, true, backend); - size_t v = gb->matmul(enc_norm, layer.decoder_encoder_attn_v_weight, true, backend); - const auto& k_buf = gb->get_output_buffer(k); - if (k_buf.shape.size() != 2) { - throw std::runtime_error("encoder cross-attn: k must be [T_enc, D]"); - } - size_t T_enc = k_buf.shape[0]; - k_4d = gb->reshape(k, {1, T_enc, kv_heads, head_dim}); - v_4d = gb->reshape(v, {1, T_enc, kv_heads, head_dim}); - if (encoder_k_persistent_[layer_idx] == 0) { - encoder_k_persistent_[layer_idx] = gb->persistent(k_4d); - encoder_v_persistent_[layer_idx] = gb->persistent(v_4d); - } - k_4d = encoder_k_persistent_[layer_idx]; - v_4d = encoder_v_persistent_[layer_idx]; - } - size_t attn = gb->attention(q, k_4d, v_4d, attention_scale_, false); - attn = gb->reshape(attn, {T_dec, q_heads * head_dim}); - size_t output = gb->matmul(attn, layer.decoder_encoder_attn_output_weight, true, backend); - return output; -} -void MoonshineModel::reset_graph_side_cache_nodes() { - cache_k_output_nodes_.assign(config_.num_decoder_layers, 0); - cache_v_output_nodes_.assign(config_.num_decoder_layers, 0); -} -void MoonshineModel::reset_cache() { - Model::reset_cache(); - encoder_ready_ = false; - encoder_kv_ready_ = false; - first_decode_step_ = true; - encoder_output_host_.clear(); - auto* gb = static_cast(graph_handle_); - if (gb) { - if (last_encoder_post_norm_node_ != 0) { - gb->invalidate_persistent(last_encoder_post_norm_node_); - last_encoder_post_norm_node_ = 0; - } - for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { - if (encoder_k_persistent_[i] != 0) { - gb->invalidate_persistent(encoder_k_persistent_[i]); - encoder_k_persistent_[i] = 0; - } - if (encoder_v_persistent_[i] != 0) { - gb->invalidate_persistent(encoder_v_persistent_[i]); - encoder_v_persistent_[i] = 0; - } - } - } -} -size_t MoonshineModel::build_decoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ - const auto& layer = weight_nodes_.decoder_layers[layer_idx]; - size_t q_proj = gb->matmul(input, layer.decoder_self_attn_q_weight, true, backend); - size_t k_proj = gb->matmul(input, layer.decoder_self_attn_k_weight, true, backend); - size_t v_proj = gb->matmul(input, layer.decoder_self_attn_v_weight, true, backend); - std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + ".self_attn."; - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - if (q_shape.size() != 2) { - throw std::runtime_error("decoder self-attn: q must be [T_new, D]"); - } - size_t seq_new = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - size_t num_kv_heads = config_.attention_kv_heads; - size_t q_4d = gb->reshape(q_proj, {1, seq_new, num_heads, head_dim}); - size_t k_4d = gb->reshape(k_proj, {1, seq_new, num_kv_heads, head_dim}); - size_t v_4d = gb->reshape(v_proj, {1, seq_new, num_kv_heads, head_dim}); - size_t rot_dim = std::max(head_dim / 2, (size_t)32); - if (config_.rope_theta > 0) { - q_4d = gb->rope_gptj(q_4d, config_.rope_theta, position_offset, rot_dim); - k_4d = gb->rope_gptj(k_4d, config_.rope_theta, position_offset, rot_dim); - } - size_t final_k = k_4d; - size_t final_v = v_4d; - if (use_cache && !kv_cache_.is_empty()) { - auto k_view = kv_cache_.get_key_view(layer_idx); - auto v_view = kv_cache_.get_value_view(layer_idx); - if (!k_view.ptr1 || !v_view.ptr1) { - throw std::runtime_error("KV cache view is empty but kv_cache_.is_empty()==false"); - } - size_t cache_len = kv_cache_.current_seq_len; - size_t cache_k_node = gb->input( - {1, cache_len, num_kv_heads, head_dim} -, - kv_cache_.precision - ); - size_t cache_v_node = gb->input( - {1, cache_len, num_kv_heads, head_dim} -, - kv_cache_.precision - ); - if (k_view.ptr2 == nullptr && v_view.ptr2 == nullptr) { - gb->set_external_input(cache_k_node, const_cast(k_view.ptr1), kv_cache_.precision); - gb->set_external_input(cache_v_node, const_cast(v_view.ptr1), kv_cache_.precision); - } - else { - gb->set_external_input(cache_k_node, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); - gb->set_external_input(cache_v_node, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); - } - final_k = gb->concat(cache_k_node, k_4d, 1); - final_v = gb->concat(cache_v_node, v_4d, 1); - } - if (use_cache) { - cache_k_output_nodes_[layer_idx] = final_k; - cache_v_output_nodes_[layer_idx] = final_v; - } - else { - cache_k_output_nodes_[layer_idx] = k_4d; - cache_v_output_nodes_[layer_idx] = v_4d; - } - auto attn_out_4d = gb->attention(q_4d, final_k, final_v, attention_scale_, position_offset); - auto attn_out = gb->reshape(attn_out_4d, {seq_new, num_heads * head_dim}); - auto output = gb->matmul(attn_out, layer.decoder_self_attn_output_weight, true, backend); - return output; -} -size_t MoonshineModel::build_encoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t /*position_offset*/){ - const auto& layer = weight_nodes_.encoder_layers[layer_idx]; - if(use_cache) - throw std::runtime_error("The encoder attention layers are not auto-regressive, and thus don't use KV caching!"); - auto q_proj = gb->matmul(input, layer.encoder_self_attn_q_weight, true, backend); - auto v_proj = gb->matmul(input, layer.encoder_self_attn_v_weight, true, backend); - auto k_proj = gb->matmul(input, layer.encoder_self_attn_k_weight, true, backend); - std::string prefix = "model.encoder.layers." + std::to_string(layer_idx) + ".self_attn."; - size_t seq_len = gb->get_output_buffer(q_proj).shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - auto q = gb->reshape(q_proj, {1, seq_len, num_heads, head_dim}); - auto k = gb->reshape(k_proj, {1, seq_len, num_heads, head_dim}); - auto v = gb->reshape(v_proj, {1, seq_len, num_heads, head_dim}); - size_t rot_dim = static_cast(head_dim * config_.partial_rotary_factor); - if (rot_dim % 2 != 0) throw std::runtime_error("rot_dim must be even"); - if (config_.rope_theta > 0) { - q = gb->rope_gptj(q, config_.rope_theta, 0, rot_dim); - k = gb->rope_gptj(k, config_.rope_theta, 0, rot_dim); - } - auto attn = gb->attention(q, k, v, attention_scale_, false); - attn = gb->reshape(attn, {seq_len, num_heads * head_dim}); - auto output = gb->matmul(attn, layer.encoder_self_attn_output_weight, true, backend); - return output; -} -size_t MoonshineModel::build_audio_preprocessor(CactusGraph* gb, size_t input) -{ - size_t conv_input = input; - const auto& xbuf = gb->get_output_buffer(input); - if (xbuf.precision == Precision::INT8) { - conv_input = gb->precision_cast(input, Precision::FP16); - } - size_t conv1 = gb->conv1d(conv_input, weight_nodes_.encoder_conv1_weight, 64); - last_conv1_node_ = conv1; - size_t conv1_act = gb->tanh(conv1); - size_t group_norm = gb->groupnorm(conv1_act, weight_nodes_.encoder_norm_weight, weight_nodes_.encoder_norm_bias, 1); - - size_t conv2_with_bias = gb->conv1d_k7s3(group_norm, weight_nodes_.encoder_conv2_weight, weight_nodes_.encoder_conv2_bias); - size_t conv2_act = gb->gelu(conv2_with_bias); - size_t conv3 = gb->conv1d_k3(conv2_act, weight_nodes_.encoder_conv3_weight, 2); - auto bias3_shape = gb->get_output_buffer(weight_nodes_.encoder_conv3_bias).shape; - size_t C3 = bias3_shape[0]; - size_t bias3 = gb->reshape(weight_nodes_.encoder_conv3_bias, {1, C3, 1}); - size_t conv3_with_bias = gb->add(conv3, bias3); - size_t conv3_act = gb->gelu(conv3_with_bias); - const auto& buf = gb->get_output_buffer(conv3_act); - size_t conv_out_transposed; - if (buf.precision == Precision::FP16) { - conv_out_transposed = gb->transpose(conv3_act, ComputeBackend::CPU); - } - else { - size_t conv3_f16 = gb->precision_cast(conv3_act, Precision::FP16); - conv_out_transposed = gb->transpose(conv3_f16, ComputeBackend::CPU); - } - return conv_out_transposed; -} -size_t MoonshineModel::build_encoder_transformer_block( - CactusGraph* gb, - size_t hidden, - uint32_t layer_idx, - ComputeBackend backend, - bool use_cache, - size_t position_offset) -{ - const auto& layer = weight_nodes_.encoder_layers[layer_idx]; - size_t input_layernorm = gb->layernorm( - hidden, - layer.encoder_post_attn_layernorm_weight - ); - size_t self_attn_out = build_encoder_self_attention( - gb, input_layernorm, layer_idx, backend, use_cache, position_offset - ); - size_t post_attention_layernorm_input = gb->add(hidden, self_attn_out); - size_t post_attention_layernorm = gb->layernorm( - post_attention_layernorm_input, - layer.encoder_post_ffn_layernorm_weight - ); - std::string prefix = "model.encoder.layers." + std::to_string(layer_idx) + "."; - size_t mlp_out = build_encoder_mlp_gelu( - gb, post_attention_layernorm, layer.encoder_ffn1_weight, layer.encoder_ffn1_bias, - layer.encoder_ffn2_weight, layer.encoder_ffn2_bias, backend, layer_idx - ); - size_t out = gb->add(post_attention_layernorm_input, mlp_out); - if (layer_idx < encoder_block_out_nodes_.size()) { - encoder_block_out_nodes_[layer_idx] = out; - } - return out; -} -size_t MoonshineModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ - const auto& layer = weight_nodes_.decoder_layers[layer_idx]; - size_t input_layernorm = gb->layernorm(hidden, layer.decoder_post_attn_layernorm_weight); - size_t sa = build_decoder_self_attention(gb, input_layernorm, layer_idx, backend, use_cache, position_offset); - size_t x_post_sa = gb->add(hidden, sa); - size_t post_attention_layernorm = gb->layernorm(x_post_sa, layer.decoder_post_encoder_layernorm_weight); - size_t ca = build_decoder_cross_attention(gb, post_attention_layernorm, layer_idx, backend, use_cache, position_offset); - size_t x_post_ca = gb->add(x_post_sa, ca); - size_t final_layernorm = gb->layernorm(x_post_ca, layer.decoder_post_ffn_layernorm_weight); - size_t mlp_out = build_decoder_mlp_silu( - gb, final_layernorm, layer.decoder_ffn_gate_weight, layer.decoder_ffn_gate_bias, - layer.decoder_ffn_up_weight, layer.decoder_ffn_up_bias, - layer.decoder_ffn2_weight, layer.decoder_ffn2_bias, backend, layer_idx - ); - size_t x_post_ffn = gb->add(x_post_ca, mlp_out); - std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + "."; - return x_post_ffn; -} -size_t MoonshineModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) -{ - if (!audio_features.empty()) { - float min_val = audio_features[0]; - float max_val = audio_features[0]; - double sum_val = 0.0; - for (float val : audio_features) { - if (val < min_val) min_val = val; - if (val > max_val) max_val = val; - sum_val += val; - } - (void)sum_val; - } - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector input_shape = npu_encoder_->get_input_shape(); - if (input_shape.size() != 3) { - input_shape = {1, 1, static_cast(audio_features.size())}; - } - if (input_shape[1] != 1) { - throw std::runtime_error("Moonshine NPU encoder expects mono audio input"); - } - size_t expected_samples = static_cast(input_shape[2]); - std::vector<__fp16> audio_f16(expected_samples, 0); - size_t copy_samples = std::min(audio_features.size(), expected_samples); - if (copy_samples > 0) { - cactus_fp32_to_fp16(audio_features.data(), audio_f16.data(), copy_samples); - } - size_t T_enc = moonshine_downsampled_len(expected_samples); - size_t D_enc = static_cast(config_.hidden_dim); - if (T_enc == 0 || D_enc == 0) { - std::cout << "NPU encoder output has unexpected shape for input size " - << expected_samples << std::endl; - std::cout << "Falling back to CPU encoder path." << std::endl; - goto encoder_cpu_fallback; - } - - size_t total_elements = T_enc * D_enc; - std::vector<__fp16> npu_output(total_elements); - size_t elements_written = npu_encoder_->encode( - audio_f16.data(), - npu_output.data(), - input_shape, - "x", - "" - ); - if (elements_written > 0) { - size_t enc_output_node = gb->input({T_enc, D_enc}, Precision::FP16); - gb->set_input(enc_output_node, npu_output.data(), Precision::FP16); - size_t enc_output_persistent = gb->persistent(enc_output_node); - last_encoder_post_norm_node_ = enc_output_persistent; - return enc_output_persistent; - } - } -encoder_cpu_fallback: - auto backend = - (config_.default_backend == Config::Backend::CPU) - ? ComputeBackend::CPU - : ComputeBackend::NPU; - size_t audio_input = 0; - const std::vector* cpu_audio = &audio_features; - std::vector<__fp16> audio_f16(cpu_audio->size()); - if (!cpu_audio->empty()) { - cactus_fp32_to_fp16(cpu_audio->data(), audio_f16.data(), cpu_audio->size()); - } - size_t audio_length = cpu_audio->size(); - audio_input = gb->input({1, 1, audio_length}, Precision::FP16); - gb->set_input(audio_input, audio_f16.data(), Precision::FP16); - size_t conv2_transposed = build_audio_preprocessor(gb, audio_input); - const auto& conv_shape = gb->get_output_buffer(conv2_transposed).shape; - if (conv_shape.size() != 3 || conv_shape[0] != 1) - throw std::runtime_error("Conv2 transpose should be [1, T_enc, D]."); - size_t T_enc = conv_shape[1]; - size_t D_enc = conv_shape[2]; - size_t h = gb->reshape(conv2_transposed, {T_enc, D_enc}); - for (uint32_t i = 0; i < config_.num_encoder_layers; ++i){ - h = build_encoder_transformer_block(gb, h, i, backend, false, 0); - } - size_t h_norm = gb->layernorm( - h, - weight_nodes_.encoder_layer_norm_weight - ); - size_t h_norm_persistent = gb->persistent(h_norm); - last_encoder_post_norm_node_ = h_norm_persistent; - return h_norm_persistent; -} -size_t MoonshineModel::build_decoder(const std::vector& tokens, bool use_cache, bool last_token_only) { - auto* gb = static_cast(graph_handle_); - const size_t full_len = tokens.size(); - if (full_len == 0) { - throw std::runtime_error("Decoder token list cannot be empty."); - } - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - size_t start_idx = (use_cache && kv_cache_.current_seq_len > 0) ? full_len - 1 : 0; - size_t new_tokens = full_len - start_idx; - size_t position_offset = use_cache ? kv_cache_.current_seq_len : 0; - size_t tok_input = gb->input({new_tokens}, Precision::FP32); - std::vector tok_f(new_tokens); - for (size_t i = 0; i < new_tokens; i++) { - tok_f[i] = static_cast(tokens[start_idx + i]); - } - gb->set_input(tok_input, tok_f.data(), Precision::FP32); - size_t dec_hidden = gb->embedding(embedding_node_id_, tok_input); - for (uint32_t layer_idx = 0; layer_idx < config_.num_decoder_layers; ++layer_idx) { - dec_hidden = build_decoder_transformer_block( - gb, - dec_hidden, - layer_idx, - backend, - use_cache, - position_offset - ); - } - size_t dec_norm = gb->layernorm( - dec_hidden, - weight_nodes_.decoder_norm_weight - ); - size_t logits_input = dec_norm; - if (last_token_only) { - size_t row_index = new_tokens - 1; - logits_input = gb->slice(logits_input, 0, row_index, 1); - } - auto w_shape = gb->get_output_buffer(output_weight_node_id_).shape; - size_t logits = gb->matmul(logits_input, output_weight_node_id_, true, backend); - last_new_tokens_ = new_tokens; - return logits; -} -size_t MoonshineModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) -{ - auto* gb = static_cast(graph_handle_); - gb->clear_debug_nodes(); - build_encoder(gb, audio_features); - size_t logits = build_decoder(tokens, use_cache, true); - return logits; -} -std::vector MoonshineModel::get_audio_embeddings(const std::vector& audio_features) { - auto* gb = static_cast(graph_handle_); - build_encoder(gb, audio_features); - size_t pooled = gb->mean(last_encoder_post_norm_node_, 0); - gb->execute(); - const auto& output_buf = gb->get_output_buffer(pooled); - size_t hidden_dim = output_buf.total_size; - std::vector embedding(hidden_dim); - void* output_data = gb->get_output(pooled); - const float* output_ptr = static_cast(output_data); - std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); - reset_cache(); - return embedding; -} -uint32_t MoonshineModel::decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty, - float* /*out_token_time_start*/, - float* /*out_token_time_end*/) -{ - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (audio_features.empty()) - throw std::runtime_error("Audio features cannot be empty in Moonshine decode_with_audio"); - auto* gb = static_cast(graph_handle_); - gb->clear_debug_nodes(); - bool cold_start = !encoder_ready_; - size_t logits_node = 0; - uint32_t bos = static_cast(get_tokenizer()->get_bos_token()); - std::vector full_tokens; - full_tokens.reserve(tokens.size() + 1); - full_tokens.push_back(bos); - full_tokens.insert(full_tokens.end(), tokens.begin(), tokens.end()); - if (cold_start) - { - gb->soft_reset(); - kv_cache_.reset(); - kv_cache_.current_seq_len = 0; - reset_graph_side_cache_nodes(); - encoder_kv_ready_ = false; - first_decode_step_ = true; - build_encoder(gb, audio_features); - logits_node = build_decoder(full_tokens, false, false); - encoder_ready_ = true; - } - else - { - gb->soft_reset(); - reset_graph_side_cache_nodes(); - std::vector last_token_vec = { tokens.back() }; - logits_node = build_decoder(last_token_vec, true, true); - } - size_t sampled_token_id = sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty); - if (!profile_file.empty()) gb->execute(profile_file); - else gb->execute(); - compute_entropy(gb, logits_node, out_entropy); - post_execute_updates(gb, full_tokens.size()); - update_kv_cache(gb, last_new_tokens_); - auto* out_ptr = gb->get_output(sampled_token_id); - uint32_t sampled = *reinterpret_cast(out_ptr); - record_sampled_token(sampled); - return sampled; -} -} -} diff --git a/cactus-transpile/models/model_needle.cpp b/cactus-transpile/models/model_needle.cpp deleted file mode 100644 index e7220a97f..000000000 --- a/cactus-transpile/models/model_needle.cpp +++ /dev/null @@ -1,337 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include - -namespace cactus { -namespace engine { - -namespace { - -size_t delta_rms_norm(CactusGraph* gb, size_t input, size_t weight, float epsilon) { - return gb->rms_norm(input, gb->scalar_add(weight, 1.0f), epsilon); -} - -size_t normalize_qk_proj(CactusGraph* gb, size_t proj, size_t norm_weight, - size_t seq_len, size_t num_heads, size_t head_dim, float epsilon) { - proj = gb->reshape(proj, {seq_len * num_heads, head_dim}); - proj = delta_rms_norm(gb, proj, norm_weight, epsilon); - return gb->reshape(proj, {seq_len, num_heads * head_dim}); -} - -size_t apply_residual_gate(CactusGraph* gb, size_t residual, size_t update, size_t gate_weight) { - return gb->add_clipped(residual, gb->multiply(update, gb->sigmoid(gate_weight))); -} - -size_t embed_tokens(CactusGraph* gb, size_t embedding_id, const std::vector& tokens, size_t hidden_dim) { - size_t n = tokens.size(); - size_t input_node = gb->input({n}, Precision::FP32); - std::vector data(n); - for (size_t i = 0; i < n; ++i) data[i] = static_cast(tokens[i]); - gb->set_input(input_node, data.data(), Precision::FP32); - return gb->scalar_multiply(gb->embedding(embedding_id, input_node), std::sqrt(static_cast(hidden_dim))); -} - -} // namespace - -NeedleModel::NeedleModel() : Model() {} - -NeedleModel::NeedleModel(const Config& config) : Model(config) { - weight_nodes_.encoder_layers.resize(config.num_encoder_layers); - weight_nodes_.decoder_layers.resize(config.num_decoder_layers); - float hd = static_cast(config.attention_head_dim); - attention_scale_ = 1.0f / std::sqrt(hd > 0.0f ? hd : 64.0f); - encoder_k_persistent_.assign(config.num_decoder_layers, 0); - encoder_v_persistent_.assign(config.num_decoder_layers, 0); -} - -void NeedleModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_layer_norm_weight.weights"); - weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_encoder_layers; ++i) { - auto& l = weight_nodes_.encoder_layers[i]; - std::string p = model_folder_path_ + "/encoder_layer_" + std::to_string(i) + "_"; - l.attn_gate_weight = gb->mmap_weights(p + "attn_gate.weights"); - l.input_norm_weight = gb->mmap_weights(p + "input_norm.weights"); - l.attn_q_weight = gb->mmap_weights(p + "attn_q.weights"); - l.attn_k_weight = gb->mmap_weights(p + "attn_k.weights"); - l.attn_v_weight = gb->mmap_weights(p + "attn_v.weights"); - l.attn_output_weight = gb->mmap_weights(p + "attn_output.weights"); - l.attn_q_norm_weight = gb->mmap_weights(p + "attn_q_norm.weights"); - l.attn_k_norm_weight = gb->mmap_weights(p + "attn_k_norm.weights"); - } - - for (uint32_t i = 0; i < config_.num_decoder_layers; ++i) { - auto& l = weight_nodes_.decoder_layers[i]; - std::string p = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - l.self_attn_gate_weight = gb->mmap_weights(p + "self_attn_gate.weights"); - l.cross_attn_gate_weight = gb->mmap_weights(p + "cross_attn_gate.weights"); - l.input_norm_weight = gb->mmap_weights(p + "input_norm.weights"); - l.post_attn_norm_weight = gb->mmap_weights(p + "post_attn_norm.weights"); - l.self_attn_q_weight = gb->mmap_weights(p + "attn_q.weights"); - l.self_attn_k_weight = gb->mmap_weights(p + "attn_k.weights"); - l.self_attn_v_weight = gb->mmap_weights(p + "attn_v.weights"); - l.self_attn_output_weight = gb->mmap_weights(p + "attn_output.weights"); - l.self_attn_q_norm_weight = gb->mmap_weights(p + "attn_q_norm.weights"); - l.self_attn_k_norm_weight = gb->mmap_weights(p + "attn_k_norm.weights"); - l.encoder_attn_q_weight = gb->mmap_weights(p + "encoder_attn_q.weights"); - l.encoder_attn_k_weight = gb->mmap_weights(p + "encoder_attn_k.weights"); - l.encoder_attn_v_weight = gb->mmap_weights(p + "encoder_attn_v.weights"); - l.encoder_attn_output_weight = gb->mmap_weights(p + "encoder_attn_output.weights"); - l.encoder_attn_q_norm_weight = gb->mmap_weights(p + "encoder_attn_q_norm.weights"); - l.encoder_attn_k_norm_weight = gb->mmap_weights(p + "encoder_attn_k_norm.weights"); - } -} - -void NeedleModel::reset_graph_side_cache_nodes() { - cache_k_output_nodes_.assign(config_.num_decoder_layers, 0); - cache_v_output_nodes_.assign(config_.num_decoder_layers, 0); -} - -void NeedleModel::reset_cache() { - Model::reset_cache(); - encoder_ready_ = false; - auto* gb = static_cast(graph_handle_); - if (!gb) { reset_graph_side_cache_nodes(); return; } - - if (last_encoder_post_norm_node_ != 0) { - gb->invalidate_persistent(last_encoder_post_norm_node_); - last_encoder_post_norm_node_ = 0; - } - for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { - if (encoder_k_persistent_[i] != 0) { gb->invalidate_persistent(encoder_k_persistent_[i]); encoder_k_persistent_[i] = 0; } - if (encoder_v_persistent_[i] != 0) { gb->invalidate_persistent(encoder_v_persistent_[i]); encoder_v_persistent_[i] = 0; } - } - reset_graph_side_cache_nodes(); -} - -size_t NeedleModel::build_encoder_self_attention(CactusGraph* gb, size_t input, - uint32_t layer_idx, ComputeBackend backend, - bool /*use_cache*/, size_t /*position_offset*/) { - const auto& l = weight_nodes_.encoder_layers[layer_idx]; - auto q = gb->matmul(input, l.attn_q_weight, true, backend); - auto k = gb->matmul(input, l.attn_k_weight, true, backend); - auto v = gb->matmul(input, l.attn_v_weight, true, backend); - - size_t seq = gb->get_output_buffer(q).shape[0]; - size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; - - q = normalize_qk_proj(gb, q, l.attn_q_norm_weight, seq, nh, hd, config_.layer_norm_eps); - k = normalize_qk_proj(gb, k, l.attn_k_norm_weight, seq, nkv, hd, config_.layer_norm_eps); - - auto q4 = gb->reshape(q, {1, seq, nh, hd}); - auto k4 = gb->reshape(k, {1, seq, nkv, hd}); - auto v4 = gb->reshape(v, {1, seq, nkv, hd}); - if (config_.rope_theta > 0) { q4 = gb->rope(q4, config_.rope_theta, 0); k4 = gb->rope(k4, config_.rope_theta, 0); } - - auto attn = gb->attention(q4, k4, v4, attention_scale_, false); - return gb->matmul(gb->reshape(attn, {seq, nh * hd}), l.attn_output_weight, true, backend); -} - -size_t NeedleModel::build_decoder_self_attention(CactusGraph* gb, size_t input, - uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t position_offset) { - const auto& l = weight_nodes_.decoder_layers[layer_idx]; - auto q = gb->matmul(input, l.self_attn_q_weight, true, backend); - auto k = gb->matmul(input, l.self_attn_k_weight, true, backend); - auto v = gb->matmul(input, l.self_attn_v_weight, true, backend); - - size_t seq = gb->get_output_buffer(q).shape[0]; - size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; - - q = normalize_qk_proj(gb, q, l.self_attn_q_norm_weight, seq, nh, hd, config_.layer_norm_eps); - k = normalize_qk_proj(gb, k, l.self_attn_k_norm_weight, seq, nkv, hd, config_.layer_norm_eps); - - auto q4 = gb->reshape(q, {1, seq, nh, hd}); - auto k4 = gb->reshape(k, {1, seq, nkv, hd}); - auto v4 = gb->reshape(v, {1, seq, nkv, hd}); - if (config_.rope_theta > 0) { q4 = gb->rope(q4, config_.rope_theta, position_offset); k4 = gb->rope(k4, config_.rope_theta, position_offset); } - - size_t final_k = k4, final_v = v4; - if (use_cache && !kv_cache_.is_empty()) { - size_t cache_len = kv_cache_.current_seq_len; - size_t ck = gb->input({1, cache_len, nkv, hd}, kv_cache_.precision); - size_t cv = gb->input({1, cache_len, nkv, hd}, kv_cache_.precision); - auto kv = kv_cache_.get_key_view(layer_idx); - auto vv = kv_cache_.get_value_view(layer_idx); - if (!kv.ptr2 && !vv.ptr2) { - gb->set_external_input(ck, const_cast(kv.ptr1), kv_cache_.precision); - gb->set_external_input(cv, const_cast(vv.ptr1), kv_cache_.precision); - } else { - gb->set_external_input(ck, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); - gb->set_external_input(cv, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); - } - final_k = gb->concat(ck, k4, 1); - final_v = gb->concat(cv, v4, 1); - } - - cache_k_output_nodes_[layer_idx] = use_cache ? final_k : k4; - cache_v_output_nodes_[layer_idx] = use_cache ? final_v : v4; - - auto attn = gb->attention(q4, final_k, final_v, attention_scale_, position_offset); - return gb->matmul(gb->reshape(attn, {seq, nh * hd}), l.self_attn_output_weight, true, backend); -} - -size_t NeedleModel::build_decoder_cross_attention(CactusGraph* gb, size_t input, - uint32_t layer_idx, ComputeBackend backend, - bool /*use_cache*/, size_t /*position_offset*/) { - const auto& l = weight_nodes_.decoder_layers[layer_idx]; - size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; - - auto q = gb->matmul(input, l.encoder_attn_q_weight, true, backend); - size_t seq_dec = gb->get_output_buffer(q).shape[0]; - q = normalize_qk_proj(gb, q, l.encoder_attn_q_norm_weight, seq_dec, nh, hd, config_.layer_norm_eps); - auto q4 = gb->reshape(q, {1, seq_dec, nh, hd}); - - size_t k4 = 0, v4 = 0; - bool cached = encoder_k_persistent_[layer_idx] != 0 && gb->is_populated(encoder_k_persistent_[layer_idx]); - if (cached) { - k4 = encoder_k_persistent_[layer_idx]; - v4 = encoder_v_persistent_[layer_idx]; - } else { - auto kp = gb->matmul(last_encoder_post_norm_node_, l.encoder_attn_k_weight, true, backend); - auto vp = gb->matmul(last_encoder_post_norm_node_, l.encoder_attn_v_weight, true, backend); - size_t seq_enc = gb->get_output_buffer(kp).shape[0]; - kp = normalize_qk_proj(gb, kp, l.encoder_attn_k_norm_weight, seq_enc, nkv, hd, config_.layer_norm_eps); - k4 = gb->reshape(kp, {1, seq_enc, nkv, hd}); - v4 = gb->reshape(vp, {1, seq_enc, nkv, hd}); - if (encoder_k_persistent_[layer_idx] == 0) { - encoder_k_persistent_[layer_idx] = gb->persistent(k4); - encoder_v_persistent_[layer_idx] = gb->persistent(v4); - } - k4 = encoder_k_persistent_[layer_idx]; - v4 = encoder_v_persistent_[layer_idx]; - } - - auto attn = gb->attention(q4, k4, v4, attention_scale_, false); - return gb->matmul(gb->reshape(attn, {seq_dec, nh * hd}), l.encoder_attn_output_weight, true, backend); -} - -size_t NeedleModel::build_encoder_transformer_block(CactusGraph* gb, size_t hidden, - uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t position_offset) { - const auto& l = weight_nodes_.encoder_layers[layer_idx]; - auto normed = delta_rms_norm(gb, hidden, l.input_norm_weight, config_.layer_norm_eps); - auto attn = build_encoder_self_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - return apply_residual_gate(gb, hidden, attn, l.attn_gate_weight); -} - -size_t NeedleModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, - uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t position_offset) { - const auto& l = weight_nodes_.decoder_layers[layer_idx]; - auto normed = delta_rms_norm(gb, hidden, l.input_norm_weight, config_.layer_norm_eps); - auto self_out = build_decoder_self_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - auto after_self = apply_residual_gate(gb, hidden, self_out, l.self_attn_gate_weight); - auto cross_in = delta_rms_norm(gb, after_self, l.post_attn_norm_weight, config_.layer_norm_eps); - auto cross_out = build_decoder_cross_attention(gb, cross_in, layer_idx, backend, use_cache, position_offset); - return apply_residual_gate(gb, after_self, cross_out, l.cross_attn_gate_weight); -} - -void NeedleModel::run_encoder(const std::vector& encoder_tokens) { - auto* gb = static_cast(graph_handle_); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - auto hidden = embed_tokens(gb, embedding_node_id_, encoder_tokens, config_.hidden_dim); - - for (uint32_t i = 0; i < config_.num_encoder_layers; ++i) - hidden = build_encoder_transformer_block(gb, hidden, i, backend, false, 0); - - last_encoder_post_norm_node_ = gb->persistent( - delta_rms_norm(gb, hidden, weight_nodes_.encoder_norm_weight, config_.layer_norm_eps)); -} - -size_t NeedleModel::run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only) { - auto* gb = static_cast(graph_handle_); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - size_t pos_offset = use_cache ? kv_cache_.current_seq_len : 0; - - auto hidden = embed_tokens(gb, embedding_node_id_, tokens, config_.hidden_dim); - for (uint32_t i = 0; i < config_.num_decoder_layers; ++i) - hidden = build_decoder_transformer_block(gb, hidden, i, backend, use_cache, pos_offset); - - auto norm = delta_rms_norm(gb, hidden, weight_nodes_.decoder_norm_weight, config_.layer_norm_eps); - if (last_token_only) norm = gb->slice(norm, 0, tokens.size() - 1, 1); - return gb->matmul(norm, output_weight_node_id_, true, backend); -} - -size_t NeedleModel::forward(const std::vector& tokens, bool use_cache) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - reset_graph_side_cache_nodes(); - - if (!encoder_ready_ || !use_cache) { - std::vector enc(tokens.begin(), tokens.end() - 1); - std::vector seed(tokens.end() - 1, tokens.end()); - if (!use_cache) { reset_cache(); gb->soft_reset(); } - run_encoder(enc); - encoder_ready_ = true; - return run_decoder_step(seed, false, false); - } - return run_decoder_step(tokens, true, tokens.size() == 1); -} - -void NeedleModel::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& profile_file) { - if (tokens.empty()) return; - reset_cache(); - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - run_encoder(tokens); - if (!profile_file.empty()) gb->execute(profile_file); else gb->execute(); - encoder_ready_ = true; -} - -uint32_t NeedleModel::decode(const std::vector& tokens, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float /*min_p*/, float /*repetition_penalty*/) { - if (temperature < 0) temperature = config_.default_temperature; - if (top_p < 0) top_p = config_.default_top_p; - if (top_k == 0) top_k = config_.default_top_k; - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - reset_graph_side_cache_nodes(); - - bool cold_start = kv_cache_.is_empty(); - std::vector cold_tokens; - const std::vector* dec_tokens = &tokens; - if (!encoder_ready_) { - reset_cache(); gb->soft_reset(); - std::vector enc(tokens.begin(), tokens.end() - 1); - cold_tokens.assign(tokens.end() - 1, tokens.end()); - run_encoder(enc); - encoder_ready_ = true; - dec_tokens = &cold_tokens; - cold_start = true; - } - - size_t logits = cold_start - ? run_decoder_step(*dec_tokens, false, false) - : run_decoder_step(tokens, true, tokens.size() == 1); - - if (config_.final_logit_softcapping > 0.0f) { - float inv = 1.0f / config_.final_logit_softcapping; - logits = gb->scalar_multiply(gb->tanh(gb->scalar_multiply(logits, inv)), config_.final_logit_softcapping); - } - - auto sampled = sample_token(gb, logits, temperature, top_p, top_k, 0.0f, 1.0f); - if (!profile_file.empty()) gb->execute(profile_file); else gb->execute(); - - compute_entropy(gb, logits, out_entropy); - post_execute_updates(gb, tokens.size()); - update_kv_cache(gb, dec_tokens->size()); - return *static_cast(gb->get_output(sampled)); -} - -} // namespace engine -} // namespace cactus diff --git a/cactus-transpile/models/model_nomic.cpp b/cactus-transpile/models/model_nomic.cpp deleted file mode 100644 index a9477a865..000000000 --- a/cactus-transpile/models/model_nomic.cpp +++ /dev/null @@ -1,201 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include - - -namespace cactus { -namespace engine { - -NomicModel::NomicModel() : Model() {} - -NomicModel::NomicModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -void NomicModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.embedding_layernorm_weight = gb->mmap_weights(model_folder_path_ + "/embedding_layernorm.weight"); - weight_nodes_.embedding_layernorm_bias = gb->mmap_weights(model_folder_path_ + "/embedding_layernorm.bias"); - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_q_bias = gb->mmap_weights(layer_prefix + "attn_q.bias"); - layer.attn_k_bias = gb->mmap_weights(layer_prefix + "attn_k.bias"); - layer.attn_v_bias = gb->mmap_weights(layer_prefix + "attn_v.bias"); - - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.attn_output_bias = gb->mmap_weights(layer_prefix + "attn_output.bias"); - layer.ffn_norm_1_weight = gb->mmap_weights(layer_prefix + "norm1.weights"); - layer.ffn_norm_1_bias = gb->mmap_weights(layer_prefix + "norm1.bias"); - layer.ffn_norm_2_weight = gb->mmap_weights(layer_prefix + "norm2.weights"); - layer.ffn_norm_2_bias = gb->mmap_weights(layer_prefix + "norm2.bias"); - - if ((i + 1) % config_.moe_every_n_layers != 0) { - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.ffn_up_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.ffn_down_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - - } else { - layer.mlp_router_layer_weight = gb->mmap_weights(layer_prefix + "mlp_router.layer.weights"); - layer.mlp_experts_bias = gb->mmap_weights(layer_prefix + "mlp_experts.bias"); - - for (uint32_t j = 0; j < config_.num_experts; j++) { - layer.mlp_experts_mlp1_weight.push_back(gb->mmap_weights(layer_prefix + "mlp_expert_" + std::to_string(j) + ".mlp1.weights")); - layer.mlp_experts_mlp2_weight.push_back(gb->mmap_weights(layer_prefix + "mlp_expert_" + std::to_string(j) + ".mlp2.weights")); - } - } - } -} - -size_t NomicModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - if (use_cache) { - throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); - } - (void)position_offset; - - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - q_proj = gb->add(q_proj, layer.attn_q_bias); - - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - k_proj = gb->add(k_proj, layer.attn_k_bias); - - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - v_proj = gb->add(v_proj, layer.attn_v_bias); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - const size_t seq_len = q_shape[0]; - const size_t num_heads = config_.attention_heads; - const size_t head_dim = config_.attention_head_dim; - - if (num_heads == 0 || head_dim == 0) { - throw std::runtime_error("Invalid attention head configuration for Nomic model"); - } - - auto reshape_to_heads = [&](size_t tensor) { - return gb->reshape(tensor, {1, seq_len, num_heads, head_dim}); - }; - - auto q_proj_4d = reshape_to_heads(q_proj); - auto k_proj_4d = reshape_to_heads(k_proj); - auto v_proj_4d = reshape_to_heads(v_proj); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, 0); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, 0); - } - - auto attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, false); - auto attn_output = gb->reshape(attn_output_4d, {seq_len, num_heads * head_dim}); - - auto output = gb->matmul(attn_output, layer.attn_output_weight, true, backend); - output = gb->add(output, layer.attn_output_bias); - return output; -} - -size_t NomicModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - if ((layer_idx + 1) % config_.moe_every_n_layers != 0) { - return build_standard_mlp(gb, normalized_h, layer_idx, backend); - } else { - return build_moe_mlp(gb, normalized_h, layer_idx, backend); - } -} - -size_t NomicModel::build_standard_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto hidden = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - hidden = gb->add(hidden, layer.ffn_up_bias); - hidden = gb->gelu(hidden); - hidden = gb->matmul(hidden, layer.ffn_down_weight, true, backend); - hidden = gb->add(hidden, layer.ffn_down_bias); - return hidden; -} - -size_t NomicModel::build_moe_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - const size_t num_experts = config_.num_experts != 0 - ? config_.num_experts - : gb->get_output_buffer(layer.mlp_router_layer_weight).shape[0]; - - auto gate_weights = gb->matmul(normalized_h, layer.mlp_router_layer_weight, true, backend); - auto gate_probs = gb->softmax(gate_weights); - auto topk_result = gb->topk(gate_probs, config_.num_top_experts); - auto topk_idx = gb->index(topk_result, 0, 0); - - std::vector w1_weights, w2_weights; - for (size_t e = 0; e < num_experts; ++e) { - w1_weights.push_back(layer.mlp_experts_mlp1_weight[e]); - w2_weights.push_back(layer.mlp_experts_mlp2_weight[e]); - } - - auto moe_out = gb->moe_layer( - normalized_h, gate_probs, topk_idx, - w1_weights, w2_weights, - num_experts, config_.num_top_experts, - false, - 1e-6f, - 1.0f, - Activation::GELU - ); - - return gb->add(moe_out, layer.mlp_experts_bias); -} - -size_t NomicModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - if (use_cache) { - throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); - } - (void)position_offset; - const auto& layer = weight_nodes_.layers[layer_idx]; - auto attn_output = build_attention(gb, hidden, layer_idx, backend); - auto residual = gb->add(hidden, attn_output); - auto normalized_residual = gb->layernorm(residual, layer.ffn_norm_1_weight, layer.ffn_norm_1_bias, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_residual, layer_idx, backend); - auto final_residual = gb->add(normalized_residual, mlp_output); - auto normalized_final_residual = gb->layernorm(final_residual, layer.ffn_norm_2_weight, layer.ffn_norm_2_bias, config_.layer_norm_eps); - return normalized_final_residual; -} - -size_t NomicModel::forward(const std::vector& tokens, bool use_cache) { - if (use_cache) { - throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); - } - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - size_t seq_len = static_cast(tokens.size()); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t input_node_id = gb->input({seq_len}, Precision::FP32); - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - size_t hidden = gb->embedding(embedding_node_id_, input_node_id); - - hidden = gb->layernorm(hidden, weight_nodes_.embedding_layernorm_weight, weight_nodes_.embedding_layernorm_bias, config_.layer_norm_eps); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend); - } - - return hidden; -} - -} -} diff --git a/cactus-transpile/models/model_parakeet.cpp b/cactus-transpile/models/model_parakeet.cpp deleted file mode 100644 index 7aee878b3..000000000 --- a/cactus-transpile/models/model_parakeet.cpp +++ /dev/null @@ -1,860 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include -#include -#include -#include - -size_t shape_elements(const std::vector& shape) { - if (shape.empty()) return 0; - size_t total = 1; - for (int d : shape) { - if (d <= 0) return 0; - total *= static_cast(d); - } - return total; -} - -bool pack_parakeet_features_for_npu( - const std::vector<__fp16>& time_major_f16, - size_t frames, - size_t num_mels, - const std::vector& input_shape, - std::vector<__fp16>& packed) -{ - if (input_shape.empty()) return false; - const size_t total = shape_elements(input_shape); - if (total == 0) return false; - packed.assign(total, static_cast<__fp16>(0.0f)); - - auto tm = [&](size_t t, size_t m) -> __fp16 { - return time_major_f16[t * num_mels + m]; - }; - - if (input_shape.size() == 4) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - const size_t s3 = static_cast(input_shape[3]); - if (s0 != 1) return false; - - if (s1 == 1 && s2 >= frames && s3 == num_mels) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[(t * s3) + m] = tm(t, m); - } - } - return true; - } - if (s1 >= frames && s2 == num_mels && s3 == 1) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[((t * s2 + m) * s3)] = tm(t, m); - } - } - return true; - } - if (s1 == num_mels && s2 >= frames && s3 == 1) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[((m * s2 + t) * s3)] = tm(t, m); - } - } - return true; - } - if (s1 == 1 && s2 == num_mels && s3 >= frames) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[(m * s3) + t] = tm(t, m); - } - } - return true; - } - return false; - } - - if (input_shape.size() == 3) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - - if (s0 == 1 && s1 >= frames && s2 == num_mels) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[t * s2 + m] = tm(t, m); - } - } - return true; - } - if (s0 == 1 && s1 == num_mels && s2 >= frames) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[m * s2 + t] = tm(t, m); - } - } - return true; - } - if (s0 >= frames && s1 == num_mels && s2 == 1) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[(t * s1 + m) * s2] = tm(t, m); - } - } - return true; - } - if (s0 == num_mels && s1 >= frames && s2 == 1) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[(m * s1 + t) * s2] = tm(t, m); - } - } - return true; - } - return false; - } - - if (input_shape.size() == 2) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - - if (s0 >= frames && s1 == num_mels) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[t * s1 + m] = tm(t, m); - } - } - return true; - } - if (s0 == num_mels && s1 >= frames) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[m * s1 + t] = tm(t, m); - } - } - return true; - } - return false; - } - - return false; -} - -bool infer_npu_encoder_output_shape( - const std::vector& output_shape, - size_t elements_written, - size_t fallback_hidden_dim, - size_t& time_steps, - size_t& hidden_dim) -{ - if (elements_written == 0) return false; - - std::vector dims; - dims.reserve(output_shape.size()); - for (int d : output_shape) { - if (d > 0) dims.push_back(static_cast(d)); - } - - if (dims.size() >= 2) { - time_steps = dims[dims.size() - 2]; - hidden_dim = dims[dims.size() - 1]; - } else if (dims.size() == 1) { - hidden_dim = dims[0]; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; - time_steps = elements_written / hidden_dim; - } else { - hidden_dim = fallback_hidden_dim; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; - time_steps = elements_written / hidden_dim; - } - - if (time_steps == 0 || hidden_dim == 0) return false; - if (time_steps * hidden_dim > elements_written) return false; - return true; -} - -namespace cactus { -namespace engine { - -ParakeetModel::ParakeetModel() : Model() {} - -ParakeetModel::ParakeetModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); - - float hd = static_cast(config.attention_head_dim); - if (hd <= 0.0f) { - hd = 64.0f; - } - attention_scale_ = 1.0f / std::sqrt(hd); - - // Legacy converted Parakeet configs inherited generic rope_theta defaults. - // Keep RoPE disabled unless explicitly set to a non-default value. - if (std::fabs(config_.rope_theta - 10000.0f) < 1e-3f || - std::fabs(config_.rope_theta - 1000000.0f) < 1e-3f) { - config_.rope_theta = 0.0f; - } -} - -void ParakeetModel::load_weights_to_graph(CactusGraph* gb) { - weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); - weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); - weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); - weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); - weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); - weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); - weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); - weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); - weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); - weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); - weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); - weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); - - weight_nodes_.ctc_head_weight = gb->mmap_weights(model_folder_path_ + "/ctc_head_weight.weights"); - weight_nodes_.ctc_head_bias = gb->mmap_weights(model_folder_path_ + "/ctc_head_bias.bias"); - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (!input_shape.empty()) { - npu_encoder_->preallocate(input_shape, "x", ""); - } - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - const std::filesystem::path model_path(model_folder_path_); - has_cpu_encoder_weights_ = - std::filesystem::exists(model_path / "subsampling_conv0_weight.weights") && - std::filesystem::exists(model_path / "subsampling_linear_weight.weights"); - - if (has_cpu_encoder_weights_) { - weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); - weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); - weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); - weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); - weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); - weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); - weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); - weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); - weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); - weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); - weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); - weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); - } - - for (uint32_t i = 0; i < config_.num_layers; ++i) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - layer.ff1_linear1_weight = gb->mmap_weights(layer_prefix + "ff1_linear1.weights"); - layer.ff1_linear1_bias = gb->mmap_weights(layer_prefix + "ff1_linear1.bias"); - layer.ff1_linear2_weight = gb->mmap_weights(layer_prefix + "ff1_linear2.weights"); - layer.ff1_linear2_bias = gb->mmap_weights(layer_prefix + "ff1_linear2.bias"); - - layer.ff2_linear1_weight = gb->mmap_weights(layer_prefix + "ff2_linear1.weights"); - layer.ff2_linear1_bias = gb->mmap_weights(layer_prefix + "ff2_linear1.bias"); - layer.ff2_linear2_weight = gb->mmap_weights(layer_prefix + "ff2_linear2.weights"); - layer.ff2_linear2_bias = gb->mmap_weights(layer_prefix + "ff2_linear2.bias"); - - layer.self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); - layer.self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); - layer.self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); - layer.self_attn_k_bias = gb->mmap_weights(layer_prefix + "self_attn_k.bias"); - layer.self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); - layer.self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); - layer.self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); - layer.self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); - layer.self_attn_relative_k_weight = gb->mmap_weights(layer_prefix + "self_attn_relative_k.weights"); - layer.self_attn_bias_u = gb->mmap_weights(layer_prefix + "self_attn_bias_u.weights"); - layer.self_attn_bias_v = gb->mmap_weights(layer_prefix + "self_attn_bias_v.weights"); - - layer.norm_ff1_weight = gb->mmap_weights(layer_prefix + "norm_ff1.weights"); - layer.norm_ff1_bias = gb->mmap_weights(layer_prefix + "norm_ff1.bias"); - layer.norm_self_attn_weight = gb->mmap_weights(layer_prefix + "norm_self_attn.weights"); - layer.norm_self_attn_bias = gb->mmap_weights(layer_prefix + "norm_self_attn.bias"); - layer.norm_conv_weight = gb->mmap_weights(layer_prefix + "norm_conv.weights"); - layer.norm_conv_bias = gb->mmap_weights(layer_prefix + "norm_conv.bias"); - layer.norm_ff2_weight = gb->mmap_weights(layer_prefix + "norm_ff2.weights"); - layer.norm_ff2_bias = gb->mmap_weights(layer_prefix + "norm_ff2.bias"); - layer.norm_out_weight = gb->mmap_weights(layer_prefix + "norm_out.weights"); - layer.norm_out_bias = gb->mmap_weights(layer_prefix + "norm_out.bias"); - - layer.conv_pointwise1_weight = gb->mmap_weights(layer_prefix + "conv_pointwise1.weights"); - layer.conv_pointwise1_bias = gb->mmap_weights(layer_prefix + "conv_pointwise1.bias"); - layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); - layer.conv_depthwise_bias = gb->mmap_weights(layer_prefix + "conv_depthwise.bias"); - layer.conv_pointwise2_weight = gb->mmap_weights(layer_prefix + "conv_pointwise2.weights"); - layer.conv_pointwise2_bias = gb->mmap_weights(layer_prefix + "conv_pointwise2.bias"); - layer.conv_batchnorm_weight = gb->mmap_weights(layer_prefix + "conv_batchnorm_weight.weights"); - layer.conv_batchnorm_bias = gb->mmap_weights(layer_prefix + "conv_batchnorm_bias.bias"); - layer.conv_batchnorm_running_mean = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_mean.weights"); - layer.conv_batchnorm_running_var = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_var.weights"); - } -} - -size_t ParakeetModel::build_subsampling(CactusGraph* gb, const std::vector& audio_features) { - const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); - if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { - throw std::runtime_error("Parakeet expects audio_features with shape [num_mels, num_frames]"); - } - - const size_t frames = audio_features.size() / num_mels; - std::vector time_major(frames * num_mels); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &audio_features[m * frames]; - for (size_t t = 0; t < frames; ++t) { - time_major[t * num_mels + m] = src[t]; - } - } - - std::vector<__fp16> features_f16(time_major.size()); - cactus_fp32_to_fp16(time_major.data(), features_f16.data(), time_major.size()); - - size_t x = gb->input({1, 1, frames, num_mels}, Precision::FP16); - gb->set_input(x, features_f16.data(), Precision::FP16); - - x = gb->conv2d_k3s2p1(x, weight_nodes_.subsampling_conv0_weight, weight_nodes_.subsampling_conv0_bias); - x = gb->relu(x); - - x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise1_weight, weight_nodes_.subsampling_depthwise1_bias); - x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise1_weight, weight_nodes_.subsampling_pointwise1_bias); - x = gb->relu(x); - - x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise2_weight, weight_nodes_.subsampling_depthwise2_bias); - x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise2_weight, weight_nodes_.subsampling_pointwise2_bias); - x = gb->relu(x); - - const auto& conv_shape = gb->get_output_buffer(x).shape; - if (conv_shape.size() != 4 || conv_shape[0] != 1) { - throw std::runtime_error("Parakeet subsampling produced invalid shape"); - } - - const size_t C = conv_shape[1]; - const size_t T = conv_shape[2]; - const size_t W = conv_shape[3]; - - size_t t_major = gb->transposeN(x, {0, 2, 1, 3}, ComputeBackend::CPU); - size_t flattened = gb->reshape(t_major, {T, C * W}); - size_t projected = gb->matmul(flattened, weight_nodes_.subsampling_linear_weight, true, ComputeBackend::CPU); - projected = gb->add(projected, weight_nodes_.subsampling_linear_bias); - return projected; -} - -size_t ParakeetModel::build_relative_position_embeddings(CactusGraph* gb, size_t seq_len) { - const size_t hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - const size_t half_dim = hidden_dim / 2; - const size_t rel_len = 2 * seq_len - 1; - - std::vector pos_embed(rel_len * hidden_dim, 0.0f); - for (size_t p = 0; p < rel_len; ++p) { - const int rel_pos = static_cast(seq_len - 1) - static_cast(p); - for (size_t i = 0; i < half_dim; ++i) { - const float exponent = static_cast(2 * i) / static_cast(hidden_dim); - const float inv_freq = 1.0f / std::pow(10000.0f, exponent); - const float angle = static_cast(rel_pos) * inv_freq; - pos_embed[p * hidden_dim + 2 * i] = std::sin(angle); - if (2 * i + 1 < hidden_dim) { - pos_embed[p * hidden_dim + 2 * i + 1] = std::cos(angle); - } - } - } - - std::vector<__fp16> pos_embed_f16(pos_embed.size()); - cactus_fp32_to_fp16(pos_embed.data(), pos_embed_f16.data(), pos_embed.size()); - - size_t pos_node = gb->input({rel_len, hidden_dim}, Precision::FP16); - gb->set_input(pos_node, pos_embed_f16.data(), Precision::FP16); - return pos_node; -} - -size_t ParakeetModel::build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t q = gb->matmul(hidden, layer.self_attn_q_weight, true, backend); - q = gb->add(q, layer.self_attn_q_bias); - size_t k = gb->matmul(hidden, layer.self_attn_k_weight, true, backend); - k = gb->add(k, layer.self_attn_k_bias); - size_t v = gb->matmul(hidden, layer.self_attn_v_weight, true, backend); - v = gb->add(v, layer.self_attn_v_bias); - - const auto& q_shape = gb->get_output_buffer(q).shape; - if (q_shape.size() != 2) { - throw std::runtime_error("Parakeet self-attention expects [T, D]"); - } - - const size_t T = q_shape[0]; - const size_t q_heads = std::max(1, static_cast(config_.attention_heads)); - const size_t kv_heads = std::max(1, static_cast(config_.attention_kv_heads)); - const size_t head_dim = std::max(1, static_cast(config_.attention_head_dim)); - - size_t q4 = gb->reshape(q, {1, T, q_heads, head_dim}); - size_t k4 = gb->reshape(k, {1, T, kv_heads, head_dim}); - size_t v4 = gb->reshape(v, {1, T, kv_heads, head_dim}); - - size_t bias_u = gb->reshape(layer.self_attn_bias_u, {1, static_cast(1), q_heads, head_dim}); - size_t bias_v = gb->reshape(layer.self_attn_bias_v, {1, static_cast(1), q_heads, head_dim}); - size_t q_u4 = gb->add(q4, bias_u); - size_t q_v4 = gb->add(q4, bias_v); - - size_t rel_k_flat = gb->matmul(position_embeddings, layer.self_attn_relative_k_weight, true, backend); - size_t rel_k4 = gb->reshape(rel_k_flat, {1, 2 * T - 1, q_heads, head_dim}); - size_t rel_bias = gb->rel_pos_bias(q_v4, rel_k4, attention_scale_); - - size_t attn = gb->attention_masked(q_u4, k4, v4, rel_bias, attention_scale_, false, backend, true); - attn = gb->reshape(attn, {T, q_heads * head_dim}); - - size_t out = gb->matmul(attn, layer.self_attn_output_weight, true, backend); - out = gb->add(out, layer.self_attn_output_bias); - return out; -} - -size_t ParakeetModel::build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t w1 = second_ff ? layer.ff2_linear1_weight : layer.ff1_linear1_weight; - size_t b1 = second_ff ? layer.ff2_linear1_bias : layer.ff1_linear1_bias; - size_t w2 = second_ff ? layer.ff2_linear2_weight : layer.ff1_linear2_weight; - size_t b2 = second_ff ? layer.ff2_linear2_bias : layer.ff1_linear2_bias; - - size_t x = gb->matmul(hidden, w1, true, backend); - x = gb->add(x, b1); - - std::string act = config_.encoder_hidden_act; - std::transform(act.begin(), act.end(), act.begin(), ::tolower); - if (act.find("gelu") != std::string::npos) { - x = gb->gelu(x); - } else if (act == "relu") { - x = gb->relu(x); - } else { - x = gb->silu(x); - } - - x = gb->matmul(x, w2, true, backend); - x = gb->add(x, b2); - return x; -} - -size_t ParakeetModel::build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend) { - (void)backend; - const auto& layer = weight_nodes_.layers[layer_idx]; - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("Parakeet convolution module expects [T, D]"); - } - - const size_t T = hidden_shape[0]; - const size_t D = hidden_shape[1]; - - size_t x = gb->reshape(hidden, {1, T, D}); - x = gb->conv1d_pointwise(x, layer.conv_pointwise1_weight, layer.conv_pointwise1_bias); - x = gb->glu(x, -1); - - x = gb->conv1d_same_depthwise_k9(x, layer.conv_depthwise_weight, layer.conv_depthwise_bias); - - x = gb->batchnorm( - x, - layer.conv_batchnorm_weight, - layer.conv_batchnorm_bias, - layer.conv_batchnorm_running_mean, - layer.conv_batchnorm_running_var, - 2, - 1e-5f - ); - - std::string act = config_.encoder_hidden_act; - std::transform(act.begin(), act.end(), act.begin(), ::tolower); - if (act.find("gelu") != std::string::npos) { - x = gb->gelu(x); - } else if (act == "relu") { - x = gb->relu(x); - } else { - x = gb->silu(x); - } - - x = gb->conv1d_pointwise(x, layer.conv_pointwise2_weight, layer.conv_pointwise2_bias); - - x = gb->reshape(x, {T, D}); - return x; -} - -size_t ParakeetModel::build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t ff1_in = gb->layernorm(hidden, layer.norm_ff1_weight, layer.norm_ff1_bias); - size_t ff1 = build_feed_forward(gb, ff1_in, layer_idx, false, backend); - ff1 = gb->scalar_multiply(ff1, 0.5f); - size_t x = gb->add(hidden, ff1); - - size_t attn_in = gb->layernorm(x, layer.norm_self_attn_weight, layer.norm_self_attn_bias); - size_t attn = build_self_attention(gb, attn_in, position_embeddings, layer_idx, backend); - x = gb->add(x, attn); - - size_t conv_in = gb->layernorm(x, layer.norm_conv_weight, layer.norm_conv_bias); - size_t conv = build_convolution_module(gb, conv_in, layer_idx, backend); - x = gb->add(x, conv); - - size_t ff2_in = gb->layernorm(x, layer.norm_ff2_weight, layer.norm_ff2_bias); - size_t ff2 = build_feed_forward(gb, ff2_in, layer_idx, true, backend); - ff2 = gb->scalar_multiply(ff2, 0.5f); - x = gb->add(x, ff2); - - x = gb->layernorm(x, layer.norm_out_weight, layer.norm_out_bias); - return x; -} - -size_t ParakeetModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) { - const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); - if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { - throw std::runtime_error("Parakeet expects audio_features with shape [num_mels, num_frames]"); - } - const size_t frames = audio_features.size() / num_mels; - size_t expected_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - size_t expected_vocab_size = std::max(1, static_cast(config_.vocab_size)); - { - const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; - if (ctc_w_shape.size() == 2) { - expected_vocab_size = ctc_w_shape[0]; - expected_hidden_dim = ctc_w_shape[1]; - } else if (ctc_w_shape.size() == 3) { - expected_vocab_size = ctc_w_shape[0]; - expected_hidden_dim = ctc_w_shape[1]; - } - } - - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector time_major(frames * num_mels); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &audio_features[m * frames]; - for (size_t t = 0; t < frames; ++t) { - time_major[t * num_mels + m] = src[t]; - } - } - - std::vector<__fp16> time_major_f16(time_major.size()); - cactus_fp32_to_fp16(time_major.data(), time_major_f16.data(), time_major.size()); - - std::vector input_shape = npu_encoder_->get_input_shape(); - std::vector<__fp16> npu_input; - if (pack_parakeet_features_for_npu(time_major_f16, frames, num_mels, input_shape, npu_input)) { - npu_encoder_->preallocate(input_shape, "x", ""); - - std::vector output_shape = npu_encoder_->get_output_shape(); - size_t output_capacity = npu_encoder_->get_output_buffer_size(); - if (output_capacity == 0) { - output_capacity = shape_elements(output_shape); - } - if (output_capacity == 0) { - output_capacity = std::max( - 1, frames * std::max(1, static_cast(config_.hidden_dim))); - } - - std::vector<__fp16> npu_output(output_capacity); - const size_t elements_written = npu_encoder_->encode( - npu_input.data(), - npu_output.data(), - input_shape, - "x", - "" - ); - - size_t T_enc = 0; - size_t D_enc = 0; - if (elements_written > 0 && - infer_npu_encoder_output_shape(output_shape, elements_written, - static_cast(config_.hidden_dim), T_enc, D_enc)) { - const __fp16* src = npu_output.data(); - __fp16* cached_output = npu_encoder_->get_output_buffer(); - const size_t cached_count = npu_encoder_->get_output_buffer_size(); - const size_t required = T_enc * D_enc; - if (cached_output != nullptr && cached_count >= required) { - src = cached_output; - } - - // Accept encoder hidden output [T, hidden] or direct logits [T, vocab]. - if (D_enc == expected_hidden_dim || D_enc == expected_vocab_size) { - size_t enc_output = gb->input({T_enc, D_enc}, Precision::FP16); - gb->set_input(enc_output, src, Precision::FP16); - return enc_output; - } - } - } - } - - if (!has_cpu_encoder_weights_) { - throw std::runtime_error( - "Parakeet requires either CPU encoder weights or model.mlpackage encoder output."); - } - - ComputeBackend backend = ComputeBackend::CPU; - size_t hidden = build_subsampling(gb, audio_features); - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("Parakeet encoder expects subsampling output [T, D]"); - } - size_t position_embeddings = build_relative_position_embeddings(gb, hidden_shape[0]); - for (uint32_t i = 0; i < config_.num_layers; ++i) { - hidden = build_encoder_block(gb, hidden, position_embeddings, i, backend); - } - return hidden; -} - -size_t ParakeetModel::build_ctc_logits(CactusGraph* gb, size_t hidden_states) { - const auto& hidden_shape = gb->get_output_buffer(hidden_states).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("Parakeet CTC head expects hidden states [T, D]"); - } - const size_t T = hidden_shape[0]; - const size_t D = hidden_shape[1]; - - size_t hidden_nlc = gb->reshape(hidden_states, {1, T, D}); - const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; - size_t vocab_size = ctc_w_shape.empty() ? config_.vocab_size : ctc_w_shape[0]; - size_t logits_nlc = gb->conv1d_pointwise(hidden_nlc, weight_nodes_.ctc_head_weight, weight_nodes_.ctc_head_bias); - - return gb->reshape(logits_nlc, {T, vocab_size}); -} - -size_t ParakeetModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) { - (void)tokens; - (void)use_cache; - auto* gb = static_cast(graph_handle_); - gb->clear_debug_nodes(); - size_t encoder_out = build_encoder(gb, audio_features); - - const auto& enc_shape = gb->get_output_buffer(encoder_out).shape; - if (enc_shape.size() != 2) { - throw std::runtime_error("Parakeet encoder output must be rank-2 [T, D]"); - } - - size_t ctc_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - size_t ctc_vocab_size = std::max(1, static_cast(config_.vocab_size)); - const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; - if (ctc_w_shape.size() == 2) { - ctc_vocab_size = ctc_w_shape[0]; - ctc_hidden_dim = ctc_w_shape[1]; - } else if (ctc_w_shape.size() == 3) { - ctc_vocab_size = ctc_w_shape[0]; - ctc_hidden_dim = ctc_w_shape[1]; - } - - // If the NPU graph already emits [T, vocab] logits, skip CPU CTC head. - if (enc_shape[1] == ctc_vocab_size && ctc_vocab_size != ctc_hidden_dim) { - return encoder_out; - } - - if (enc_shape[1] != ctc_hidden_dim) { - throw std::runtime_error( - "Parakeet encoder output dim mismatch: expected hidden dim " + - std::to_string(ctc_hidden_dim) + ", got " + std::to_string(enc_shape[1])); - } - - return build_ctc_logits(gb, encoder_out); -} - -std::vector ParakeetModel::greedy_decode_tokens(CactusGraph* gb, size_t logits_node) const { - const auto& logits_buf = gb->get_output_buffer(logits_node); - if (logits_buf.shape.size() != 2) { - throw std::runtime_error("Parakeet logits must be rank-2 [T, vocab]"); - } - - const size_t T = logits_buf.shape[0]; - const size_t vocab_size = logits_buf.shape[1]; - std::vector frame_ids(T, 0); - - if (logits_buf.precision == Precision::FP32) { - const float* src = logits_buf.data_as(); - for (size_t t = 0; t < T; ++t) { - const float* row = src + t * vocab_size; - size_t best_idx = 0; - float best_val = row[0]; - for (size_t v = 1; v < vocab_size; ++v) { - if (row[v] > best_val) { - best_val = row[v]; - best_idx = v; - } - } - frame_ids[t] = static_cast(best_idx); - } - } else if (logits_buf.precision == Precision::FP16) { - const __fp16* src = logits_buf.data_as<__fp16>(); - for (size_t t = 0; t < T; ++t) { - const __fp16* row = src + t * vocab_size; - size_t best_idx = 0; - float best_val = static_cast(row[0]); - for (size_t v = 1; v < vocab_size; ++v) { - const float val = static_cast(row[v]); - if (val > best_val) { - best_val = val; - best_idx = v; - } - } - frame_ids[t] = static_cast(best_idx); - } - } else { - const int8_t* src = logits_buf.data_as(); - for (size_t t = 0; t < T; ++t) { - const int8_t* row = src + t * vocab_size; - size_t best_idx = 0; - int best_val = static_cast(row[0]); - for (size_t v = 1; v < vocab_size; ++v) { - const int val = static_cast(row[v]); - if (val > best_val) { - best_val = val; - best_idx = v; - } - } - frame_ids[t] = static_cast(best_idx); - } - } - - const uint32_t blank_id = config_.pad_token_id > 0 ? config_.pad_token_id : static_cast(vocab_size - 1); - std::vector decoded; - decoded.reserve(frame_ids.size()); - - uint32_t prev = blank_id; - for (uint32_t id : frame_ids) { - if (id == blank_id) { - prev = blank_id; - continue; - } - if (id == prev) { - continue; - } - decoded.push_back(id); - prev = id; - } - - return decoded; -} - -uint32_t ParakeetModel::decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty, - float* /*out_token_time_start*/, - float* /*out_token_time_end*/) -{ - (void)temperature; - (void)top_p; - (void)top_k; - (void)min_p; - (void)repetition_penalty; - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (audio_features.empty()) { - throw std::runtime_error("Audio features cannot be empty in Parakeet decode_with_audio"); - } - - auto* gb = static_cast(graph_handle_); - const bool new_request = !ctc_tokens_ready_ || tokens.empty() || tokens.size() < last_input_token_count_; - if (new_request) { - gb->soft_reset(); - size_t logits_node = forward(audio_features, tokens, false); - - if (!profile_file.empty()) { - gb->execute(profile_file); - } else { - gb->execute(); - } - - ctc_tokens_ = greedy_decode_tokens(gb, logits_node); - ctc_emit_index_ = 0; - ctc_tokens_ready_ = true; - } - - last_input_token_count_ = tokens.size(); - if (out_entropy) { - *out_entropy = 0.0f; - } - - if (ctc_emit_index_ < ctc_tokens_.size()) { - return ctc_tokens_[ctc_emit_index_++]; - } - return get_tokenizer()->get_eos_token(); -} - -std::vector ParakeetModel::get_audio_embeddings(const std::vector& audio_features) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - size_t hidden = build_encoder(gb, audio_features); - - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - size_t ctc_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - size_t ctc_vocab_size = std::max(1, static_cast(config_.vocab_size)); - const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; - if (ctc_w_shape.size() == 2) { - ctc_vocab_size = ctc_w_shape[0]; - ctc_hidden_dim = ctc_w_shape[1]; - } else if (ctc_w_shape.size() == 3) { - ctc_vocab_size = ctc_w_shape[0]; - ctc_hidden_dim = ctc_w_shape[1]; - } - - // Embeddings require hidden states; if NPU returned logits, rerun CPU encoder. - if (hidden_shape.size() == 2 && - hidden_shape[1] == ctc_vocab_size && - ctc_vocab_size != ctc_hidden_dim && - use_npu_encoder_) { - if (!has_cpu_encoder_weights_) { - throw std::runtime_error( - "Parakeet audio embeddings require hidden-state encoder output; " - "CPU encoder fallback weights are not available."); - } - const bool prev_use_npu = use_npu_encoder_; - use_npu_encoder_ = false; - gb->soft_reset(); - hidden = build_encoder(gb, audio_features); - use_npu_encoder_ = prev_use_npu; - } - - size_t pooled = gb->mean(hidden, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - const size_t hidden_dim = output_buf.total_size; - std::vector embedding(hidden_dim); - - if (output_buf.precision == Precision::FP32) { - const float* src = output_buf.data_as(); - std::copy(src, src + hidden_dim, embedding.begin()); - } else if (output_buf.precision == Precision::FP16) { - const __fp16* src = output_buf.data_as<__fp16>(); - Quantization::fp16_to_fp32(src, embedding.data(), hidden_dim); - } else { - const int8_t* src = output_buf.data_as(); - Quantization::int8_to_fp32(src, embedding.data(), hidden_dim, 1.0f); - } - - reset_cache(); - return embedding; -} - -void ParakeetModel::reset_cache() { - Model::reset_cache(); - ctc_tokens_ready_ = false; - ctc_emit_index_ = 0; - ctc_tokens_.clear(); - last_input_token_count_ = 0; -} - -} // namespace engine -} // namespace cactus \ No newline at end of file diff --git a/cactus-transpile/models/model_parakeet_tdt.cpp b/cactus-transpile/models/model_parakeet_tdt.cpp deleted file mode 100644 index da0e827d4..000000000 --- a/cactus-transpile/models/model_parakeet_tdt.cpp +++ /dev/null @@ -1,1107 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include "../kernel/kernel.h" -#include "../telemetry/telemetry.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -size_t shape_elements(const std::vector& shape); -bool pack_parakeet_features_for_npu( - const std::vector<__fp16>& time_major_f16, - size_t frames, - size_t num_mels, - const std::vector& input_shape, - std::vector<__fp16>& packed); - -namespace { - -constexpr uint32_t kMaxStreamDurationSkipFrames = 2; - -bool infer_npu_encoder_output_shape( - const std::vector& output_shape, - size_t elements_written, - size_t fallback_hidden_dim, - size_t& time_steps, - size_t& hidden_dim) -{ - if (elements_written == 0) return false; - - std::vector dims; - dims.reserve(output_shape.size()); - for (int d : output_shape) { - if (d > 0) dims.push_back(static_cast(d)); - } - - if (dims.size() >= 2) { - time_steps = dims[dims.size() - 2]; - hidden_dim = dims[dims.size() - 1]; - } else if (dims.size() == 1) { - hidden_dim = dims[0]; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; - time_steps = elements_written / hidden_dim; - } else { - hidden_dim = fallback_hidden_dim; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; - time_steps = elements_written / hidden_dim; - } - - if (time_steps == 0 || hidden_dim == 0) return false; - if (time_steps * hidden_dim > elements_written) return false; - return true; -} - -std::vector<__fp16> copy_buffer_to_fp16(const BufferDesc& buffer) { - std::vector<__fp16> out(buffer.total_size, static_cast<__fp16>(0.0f)); - if (buffer.total_size == 0) { - return out; - } - - if (buffer.precision == Precision::FP16) { - const __fp16* src = buffer.data_as<__fp16>(); - std::copy(src, src + buffer.total_size, out.begin()); - return out; - } - - if (buffer.precision == Precision::FP32) { - const float* src = buffer.data_as(); - cactus_fp32_to_fp16(src, out.data(), buffer.total_size); - return out; - } - - if (buffer.precision == Precision::INT8 && !buffer.is_interleaved) { - const int8_t* src = buffer.data_as(); - Quantization::int8_to_fp16(src, out.data(), buffer.total_size, 1.0f); - return out; - } - - throw std::runtime_error("Unsupported precision for FP16 conversion in ParakeetTDT"); -} - -size_t argmax_range(const BufferDesc& buffer, size_t offset, size_t length) { - if (length == 0 || (offset + length) > buffer.total_size) { - throw std::runtime_error("Invalid argmax range"); - } - - size_t best_idx = 0; - float best_val = -std::numeric_limits::infinity(); - - if (buffer.precision == Precision::FP16) { - const __fp16* src = buffer.data_as<__fp16>() + offset; - best_val = static_cast(src[0]); - for (size_t i = 1; i < length; ++i) { - float v = static_cast(src[i]); - if (v > best_val) { - best_val = v; - best_idx = i; - } - } - return best_idx; - } - - if (buffer.precision == Precision::FP32) { - const float* src = buffer.data_as() + offset; - best_val = src[0]; - for (size_t i = 1; i < length; ++i) { - float v = src[i]; - if (v > best_val) { - best_val = v; - best_idx = i; - } - } - return best_idx; - } - - if (buffer.precision == Precision::INT8) { - const int8_t* src = buffer.data_as() + offset; - best_val = static_cast(src[0]); - for (size_t i = 1; i < length; ++i) { - float v = static_cast(src[i]); - if (v > best_val) { - best_val = v; - best_idx = i; - } - } - return best_idx; - } - - throw std::runtime_error("Unsupported logits precision in argmax"); -} - -size_t argmax_range_with_bias(const BufferDesc& buffer, size_t offset, size_t length, - const std::unordered_map& bias) { - if (bias.empty()) { - return argmax_range(buffer, offset, length); - } - if (length == 0 || (offset + length) > buffer.total_size) { - throw std::runtime_error("Invalid argmax range"); - } - - size_t best_idx = 0; - float best_val = -std::numeric_limits::infinity(); - - if (buffer.precision == Precision::FP16) { - const __fp16* src = buffer.data_as<__fp16>() + offset; - for (size_t i = 0; i < length; ++i) { - float v = static_cast(src[i]); - auto it = bias.find(static_cast(i)); - if (it != bias.end()) v += it->second; - if (v > best_val) { best_val = v; best_idx = i; } - } - return best_idx; - } - - if (buffer.precision == Precision::FP32) { - const float* src = buffer.data_as() + offset; - for (size_t i = 0; i < length; ++i) { - float v = src[i]; - auto it = bias.find(static_cast(i)); - if (it != bias.end()) v += it->second; - if (v > best_val) { best_val = v; best_idx = i; } - } - return best_idx; - } - - if (buffer.precision == Precision::INT8) { - const int8_t* src = buffer.data_as() + offset; - for (size_t i = 0; i < length; ++i) { - float v = static_cast(src[i]); - auto it = bias.find(static_cast(i)); - if (it != bias.end()) v += it->second; - if (v > best_val) { best_val = v; best_idx = i; } - } - return best_idx; - } - - throw std::runtime_error("Unsupported logits precision in biased argmax"); -} - -} // namespace - -namespace cactus { -namespace engine { - -ParakeetTDTModel::ParakeetTDTModel() : Model() {} - -ParakeetTDTModel::ParakeetTDTModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); - - float hd = static_cast(config.attention_head_dim); - if (hd <= 0.0f) { - hd = 64.0f; - } - attention_scale_ = 1.0f / std::sqrt(hd); - - if (std::fabs(config_.rope_theta - 10000.0f) < 1e-3f || - std::fabs(config_.rope_theta - 1000000.0f) < 1e-3f) { - config_.rope_theta = 0.0f; - } -} - -void ParakeetTDTModel::load_weights_to_graph(CactusGraph* gb) { - weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); - weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); - weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); - weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); - weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); - weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); - weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); - weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); - weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); - weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); - weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); - weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); - - weight_nodes_.predictor_embed = gb->mmap_weights(model_folder_path_ + "/tdt_predictor_embed.weights"); - - size_t predictor_layers = static_cast(config_.predictor_num_layers); - if (predictor_layers == 0) { - while (true) { - const std::string prefix = model_folder_path_ + "/tdt_predictor_lstm_" + std::to_string(predictor_layers); - if (!std::filesystem::exists(prefix + "_weight_ih.weights") || - !std::filesystem::exists(prefix + "_weight_hh.weights") || - !std::filesystem::exists(prefix + "_bias.weights")) { - break; - } - ++predictor_layers; - } - } - if (predictor_layers == 0) { - predictor_layers = 1; - } - - weight_nodes_.predictor_layers.resize(predictor_layers); - for (size_t i = 0; i < predictor_layers; ++i) { - auto& layer = weight_nodes_.predictor_layers[i]; - const std::string prefix = model_folder_path_ + "/tdt_predictor_lstm_" + std::to_string(i); - layer.weight_ih = gb->mmap_weights(prefix + "_weight_ih.weights"); - layer.weight_hh = gb->mmap_weights(prefix + "_weight_hh.weights"); - layer.bias = gb->mmap_weights(prefix + "_bias.weights"); - } - - weight_nodes_.joint_enc_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_enc.weights"); - weight_nodes_.joint_enc_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_enc.bias"); - weight_nodes_.joint_pred_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_pred.weights"); - weight_nodes_.joint_pred_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_pred.bias"); - weight_nodes_.joint_out_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_out.weights"); - weight_nodes_.joint_out_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_out.bias"); - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (!input_shape.empty()) { - npu_encoder_->preallocate(input_shape, "x", ""); - } - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - const std::filesystem::path model_path(model_folder_path_); - has_cpu_encoder_weights_ = - std::filesystem::exists(model_path / "subsampling_conv0_weight.weights") && - std::filesystem::exists(model_path / "subsampling_linear_weight.weights"); - - if (has_cpu_encoder_weights_) { - weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); - weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); - weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); - weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); - weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); - weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); - weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); - weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); - weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); - weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); - weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); - weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); - } - - for (uint32_t i = 0; i < config_.num_layers; ++i) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - layer.ff1_linear1_weight = gb->mmap_weights(layer_prefix + "ff1_linear1.weights"); - layer.ff1_linear1_bias = gb->mmap_weights(layer_prefix + "ff1_linear1.bias"); - layer.ff1_linear2_weight = gb->mmap_weights(layer_prefix + "ff1_linear2.weights"); - layer.ff1_linear2_bias = gb->mmap_weights(layer_prefix + "ff1_linear2.bias"); - - layer.ff2_linear1_weight = gb->mmap_weights(layer_prefix + "ff2_linear1.weights"); - layer.ff2_linear1_bias = gb->mmap_weights(layer_prefix + "ff2_linear1.bias"); - layer.ff2_linear2_weight = gb->mmap_weights(layer_prefix + "ff2_linear2.weights"); - layer.ff2_linear2_bias = gb->mmap_weights(layer_prefix + "ff2_linear2.bias"); - - layer.self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); - layer.self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); - layer.self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); - layer.self_attn_k_bias = gb->mmap_weights(layer_prefix + "self_attn_k.bias"); - layer.self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); - layer.self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); - layer.self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); - layer.self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); - layer.self_attn_relative_k_weight = gb->mmap_weights(layer_prefix + "self_attn_relative_k.weights"); - layer.self_attn_bias_u = gb->mmap_weights(layer_prefix + "self_attn_bias_u.weights"); - layer.self_attn_bias_v = gb->mmap_weights(layer_prefix + "self_attn_bias_v.weights"); - - layer.norm_ff1_weight = gb->mmap_weights(layer_prefix + "norm_ff1.weights"); - layer.norm_ff1_bias = gb->mmap_weights(layer_prefix + "norm_ff1.bias"); - layer.norm_self_attn_weight = gb->mmap_weights(layer_prefix + "norm_self_attn.weights"); - layer.norm_self_attn_bias = gb->mmap_weights(layer_prefix + "norm_self_attn.bias"); - layer.norm_conv_weight = gb->mmap_weights(layer_prefix + "norm_conv.weights"); - layer.norm_conv_bias = gb->mmap_weights(layer_prefix + "norm_conv.bias"); - layer.norm_ff2_weight = gb->mmap_weights(layer_prefix + "norm_ff2.weights"); - layer.norm_ff2_bias = gb->mmap_weights(layer_prefix + "norm_ff2.bias"); - layer.norm_out_weight = gb->mmap_weights(layer_prefix + "norm_out.weights"); - layer.norm_out_bias = gb->mmap_weights(layer_prefix + "norm_out.bias"); - - layer.conv_pointwise1_weight = gb->mmap_weights(layer_prefix + "conv_pointwise1.weights"); - layer.conv_pointwise1_bias = gb->mmap_weights(layer_prefix + "conv_pointwise1.bias"); - layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); - layer.conv_depthwise_bias = gb->mmap_weights(layer_prefix + "conv_depthwise.bias"); - layer.conv_pointwise2_weight = gb->mmap_weights(layer_prefix + "conv_pointwise2.weights"); - layer.conv_pointwise2_bias = gb->mmap_weights(layer_prefix + "conv_pointwise2.bias"); - layer.conv_batchnorm_weight = gb->mmap_weights(layer_prefix + "conv_batchnorm_weight.weights"); - layer.conv_batchnorm_bias = gb->mmap_weights(layer_prefix + "conv_batchnorm_bias.bias"); - layer.conv_batchnorm_running_mean = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_mean.weights"); - layer.conv_batchnorm_running_var = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_var.weights"); - } -} - -size_t ParakeetTDTModel::build_subsampling(CactusGraph* gb, const std::vector& audio_features) { - const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); - if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { - throw std::runtime_error("ParakeetTDT expects audio_features with shape [num_mels, num_frames]"); - } - - const size_t frames = audio_features.size() / num_mels; - std::vector time_major(frames * num_mels); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &audio_features[m * frames]; - for (size_t t = 0; t < frames; ++t) { - time_major[t * num_mels + m] = src[t]; - } - } - - std::vector<__fp16> features_f16(time_major.size()); - cactus_fp32_to_fp16(time_major.data(), features_f16.data(), time_major.size()); - - size_t x = gb->input({1, 1, frames, num_mels}, Precision::FP16); - gb->set_input(x, features_f16.data(), Precision::FP16); - - x = gb->conv2d_k3s2p1(x, weight_nodes_.subsampling_conv0_weight, weight_nodes_.subsampling_conv0_bias); - x = gb->relu(x); - - x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise1_weight, weight_nodes_.subsampling_depthwise1_bias); - x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise1_weight, weight_nodes_.subsampling_pointwise1_bias); - x = gb->relu(x); - - x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise2_weight, weight_nodes_.subsampling_depthwise2_bias); - x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise2_weight, weight_nodes_.subsampling_pointwise2_bias); - x = gb->relu(x); - - const auto& conv_shape = gb->get_output_buffer(x).shape; - if (conv_shape.size() != 4 || conv_shape[0] != 1) { - throw std::runtime_error("ParakeetTDT subsampling produced invalid shape"); - } - - const size_t C = conv_shape[1]; - const size_t T = conv_shape[2]; - const size_t W = conv_shape[3]; - - size_t t_major = gb->transposeN(x, {0, 2, 1, 3}, ComputeBackend::CPU); - size_t flattened = gb->reshape(t_major, {T, C * W}); - size_t projected = gb->matmul(flattened, weight_nodes_.subsampling_linear_weight, true, ComputeBackend::CPU); - projected = gb->add(projected, weight_nodes_.subsampling_linear_bias); - return projected; -} - -size_t ParakeetTDTModel::build_relative_position_embeddings(CactusGraph* gb, size_t seq_len) { - const size_t hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - const size_t half_dim = hidden_dim / 2; - const size_t rel_len = 2 * seq_len - 1; - - std::vector pos_embed(rel_len * hidden_dim, 0.0f); - for (size_t p = 0; p < rel_len; ++p) { - const int rel_pos = static_cast(seq_len - 1) - static_cast(p); - for (size_t i = 0; i < half_dim; ++i) { - const float exponent = static_cast(2 * i) / static_cast(hidden_dim); - const float inv_freq = 1.0f / std::pow(10000.0f, exponent); - const float angle = static_cast(rel_pos) * inv_freq; - pos_embed[p * hidden_dim + 2 * i] = std::sin(angle); - if (2 * i + 1 < hidden_dim) { - pos_embed[p * hidden_dim + 2 * i + 1] = std::cos(angle); - } - } - } - - std::vector<__fp16> pos_embed_f16(pos_embed.size()); - cactus_fp32_to_fp16(pos_embed.data(), pos_embed_f16.data(), pos_embed.size()); - - size_t pos_node = gb->input({rel_len, hidden_dim}, Precision::FP16); - gb->set_input(pos_node, pos_embed_f16.data(), Precision::FP16); - return pos_node; -} - -size_t ParakeetTDTModel::build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t q = gb->matmul(hidden, layer.self_attn_q_weight, true, backend); - q = gb->add(q, layer.self_attn_q_bias); - size_t k = gb->matmul(hidden, layer.self_attn_k_weight, true, backend); - k = gb->add(k, layer.self_attn_k_bias); - size_t v = gb->matmul(hidden, layer.self_attn_v_weight, true, backend); - v = gb->add(v, layer.self_attn_v_bias); - - const auto& q_shape = gb->get_output_buffer(q).shape; - if (q_shape.size() != 2) { - throw std::runtime_error("ParakeetTDT self-attention expects [T, D]"); - } - - const size_t T = q_shape[0]; - const size_t q_heads = std::max(1, static_cast(config_.attention_heads)); - const size_t kv_heads = std::max(1, static_cast(config_.attention_kv_heads)); - const size_t head_dim = std::max(1, static_cast(config_.attention_head_dim)); - - size_t q4 = gb->reshape(q, {1, T, q_heads, head_dim}); - size_t k4 = gb->reshape(k, {1, T, kv_heads, head_dim}); - size_t v4 = gb->reshape(v, {1, T, kv_heads, head_dim}); - - size_t bias_u = gb->reshape(layer.self_attn_bias_u, {1, static_cast(1), q_heads, head_dim}); - size_t bias_v = gb->reshape(layer.self_attn_bias_v, {1, static_cast(1), q_heads, head_dim}); - size_t q_u4 = gb->add(q4, bias_u); - size_t q_v4 = gb->add(q4, bias_v); - - size_t rel_k_flat = gb->matmul(position_embeddings, layer.self_attn_relative_k_weight, true, backend); - size_t rel_k4 = gb->reshape(rel_k_flat, {1, 2 * T - 1, q_heads, head_dim}); - size_t rel_bias = gb->rel_pos_bias(q_v4, rel_k4, attention_scale_); - - size_t attn = gb->attention_masked(q_u4, k4, v4, rel_bias, attention_scale_, false, backend, true); - attn = gb->reshape(attn, {T, q_heads * head_dim}); - - size_t out = gb->matmul(attn, layer.self_attn_output_weight, true, backend); - out = gb->add(out, layer.self_attn_output_bias); - return out; -} - -size_t ParakeetTDTModel::build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - bool second_ff, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t w1 = second_ff ? layer.ff2_linear1_weight : layer.ff1_linear1_weight; - size_t b1 = second_ff ? layer.ff2_linear1_bias : layer.ff1_linear1_bias; - size_t w2 = second_ff ? layer.ff2_linear2_weight : layer.ff1_linear2_weight; - size_t b2 = second_ff ? layer.ff2_linear2_bias : layer.ff1_linear2_bias; - - size_t x = gb->matmul(hidden, w1, true, backend); - x = gb->add(x, b1); - - std::string act = config_.encoder_hidden_act; - std::transform(act.begin(), act.end(), act.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); - if (act.find("gelu") != std::string::npos) { - x = gb->gelu(x); - } else if (act == "relu") { - x = gb->relu(x); - } else { - x = gb->silu(x); - } - - x = gb->matmul(x, w2, true, backend); - x = gb->add(x, b2); - return x; -} - -size_t ParakeetTDTModel::build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend) { - (void)backend; - const auto& layer = weight_nodes_.layers[layer_idx]; - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("ParakeetTDT convolution module expects [T, D]"); - } - - const size_t T = hidden_shape[0]; - const size_t D = hidden_shape[1]; - - size_t x = gb->reshape(hidden, {1, T, D}); - x = gb->conv1d_pointwise(x, layer.conv_pointwise1_weight, layer.conv_pointwise1_bias); - x = gb->glu(x, -1); - - x = gb->conv1d_same_depthwise_k9(x, layer.conv_depthwise_weight, layer.conv_depthwise_bias); - - x = gb->batchnorm( - x, - layer.conv_batchnorm_weight, - layer.conv_batchnorm_bias, - layer.conv_batchnorm_running_mean, - layer.conv_batchnorm_running_var, - 2, - 1e-5f - ); - - std::string act = config_.encoder_hidden_act; - std::transform(act.begin(), act.end(), act.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); - if (act.find("gelu") != std::string::npos) { - x = gb->gelu(x); - } else if (act == "relu") { - x = gb->relu(x); - } else { - x = gb->silu(x); - } - - x = gb->conv1d_pointwise(x, layer.conv_pointwise2_weight, layer.conv_pointwise2_bias); - - x = gb->reshape(x, {T, D}); - return x; -} - -size_t ParakeetTDTModel::build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t ff1_in = gb->layernorm(hidden, layer.norm_ff1_weight, layer.norm_ff1_bias); - size_t ff1 = build_feed_forward(gb, ff1_in, layer_idx, false, backend); - ff1 = gb->scalar_multiply(ff1, 0.5f); - size_t x = gb->add(hidden, ff1); - - size_t attn_in = gb->layernorm(x, layer.norm_self_attn_weight, layer.norm_self_attn_bias); - size_t attn = build_self_attention(gb, attn_in, position_embeddings, layer_idx, backend); - x = gb->add(x, attn); - - size_t conv_in = gb->layernorm(x, layer.norm_conv_weight, layer.norm_conv_bias); - size_t conv = build_convolution_module(gb, conv_in, layer_idx, backend); - x = gb->add(x, conv); - - size_t ff2_in = gb->layernorm(x, layer.norm_ff2_weight, layer.norm_ff2_bias); - size_t ff2 = build_feed_forward(gb, ff2_in, layer_idx, true, backend); - ff2 = gb->scalar_multiply(ff2, 0.5f); - x = gb->add(x, ff2); - - x = gb->layernorm(x, layer.norm_out_weight, layer.norm_out_bias); - return x; -} - -size_t ParakeetTDTModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) { - const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); - if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { - throw std::runtime_error("ParakeetTDT expects audio_features with shape [num_mels, num_frames]"); - } - - const size_t frames = audio_features.size() / num_mels; - - size_t expected_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - { - const auto& joint_enc_shape = gb->get_output_buffer(weight_nodes_.joint_enc_weight).shape; - if (joint_enc_shape.size() == 2) { - expected_hidden_dim = joint_enc_shape[1]; - } - } - - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector time_major(frames * num_mels); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &audio_features[m * frames]; - for (size_t t = 0; t < frames; ++t) { - time_major[t * num_mels + m] = src[t]; - } - } - - std::vector<__fp16> time_major_f16(time_major.size()); - cactus_fp32_to_fp16(time_major.data(), time_major_f16.data(), time_major.size()); - - std::vector input_shape = npu_encoder_->get_input_shape(); - std::vector<__fp16> npu_input; - if (pack_parakeet_features_for_npu(time_major_f16, frames, num_mels, input_shape, npu_input)) { - npu_encoder_->preallocate(input_shape, "x", ""); - - std::vector output_shape = npu_encoder_->get_output_shape(); - size_t output_capacity = npu_encoder_->get_output_buffer_size(); - if (output_capacity == 0) { - output_capacity = shape_elements(output_shape); - } - if (output_capacity == 0) { - output_capacity = std::max(1, frames * expected_hidden_dim); - } - - std::vector<__fp16> npu_output(output_capacity); - const size_t elements_written = npu_encoder_->encode( - npu_input.data(), - npu_output.data(), - input_shape, - "x", - "" - ); - - size_t T_enc = 0; - size_t D_enc = 0; - if (elements_written > 0 && - infer_npu_encoder_output_shape(output_shape, elements_written, expected_hidden_dim, T_enc, D_enc)) { - const __fp16* src = npu_output.data(); - __fp16* cached_output = npu_encoder_->get_output_buffer(); - const size_t cached_count = npu_encoder_->get_output_buffer_size(); - const size_t required = T_enc * D_enc; - if (cached_output != nullptr && cached_count >= required) { - src = cached_output; - } - - if (D_enc == expected_hidden_dim) { - size_t enc_output = gb->input({T_enc, D_enc}, Precision::FP16); - gb->set_input(enc_output, src, Precision::FP16); - return enc_output; - } - } - } - } - - if (!has_cpu_encoder_weights_) { - throw std::runtime_error( - "Parakeet-TDT requires either CPU encoder weights or model.mlpackage encoder output."); - } - - ComputeBackend backend = ComputeBackend::CPU; - size_t hidden = build_subsampling(gb, audio_features); - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("ParakeetTDT encoder expects subsampling output [T, D]"); - } - - size_t position_embeddings = build_relative_position_embeddings(gb, hidden_shape[0]); - for (uint32_t i = 0; i < config_.num_layers; ++i) { - hidden = build_encoder_block(gb, hidden, position_embeddings, i, backend); - } - - return hidden; -} - -size_t ParakeetTDTModel::forward(const std::vector& audio_features, - const std::vector& tokens, - bool use_cache) { - (void)tokens; - (void)use_cache; - - auto* gb = static_cast(graph_handle_); - gb->clear_debug_nodes(); - return build_encoder(gb, audio_features); -} - -std::vector ParakeetTDTModel::decode_tdt_tokens_with_state( - CactusGraph* gb, - size_t encoder_hidden_node, - size_t replay_start_frame, - size_t start_frame, - size_t end_frame, - ChunkStreamState* stream_state, - size_t* out_confirmed_count, - double* out_raw_decoder_time_ms) const { - const auto& enc_buf = gb->get_output_buffer(encoder_hidden_node); - if (enc_buf.shape.size() != 2) { - throw std::runtime_error("ParakeetTDT encoder output must be rank-2 [T, D]"); - } - - const size_t T = enc_buf.shape[0]; - const size_t D = enc_buf.shape[1]; - if (T == 0 || D == 0) { - return {}; - } - - std::vector<__fp16> encoder_fp16 = copy_buffer_to_fp16(enc_buf); - - gb->soft_reset(); - - size_t frame_in = gb->input({1, D}, Precision::FP16); - size_t token_idx = gb->input({1}, Precision::FP32); - - size_t pred = gb->embedding(weight_nodes_.predictor_embed, token_idx); - - const size_t predictor_layers = weight_nodes_.predictor_layers.size(); - std::vector h_prev_nodes; - std::vector c_prev_nodes; - std::vector h_new_nodes; - std::vector c_new_nodes; - std::vector bias_hh_zero_nodes; - std::vector hidden_sizes; - - h_prev_nodes.reserve(predictor_layers); - c_prev_nodes.reserve(predictor_layers); - h_new_nodes.reserve(predictor_layers); - c_new_nodes.reserve(predictor_layers); - bias_hh_zero_nodes.reserve(predictor_layers); - hidden_sizes.reserve(predictor_layers); - - for (size_t i = 0; i < predictor_layers; ++i) { - const auto w_ih = weight_nodes_.predictor_layers[i].weight_ih; - const auto w_hh = weight_nodes_.predictor_layers[i].weight_hh; - const auto b_ih = weight_nodes_.predictor_layers[i].bias; - - if (gb->get_output_buffer(w_ih).precision != Precision::FP16 || - gb->get_output_buffer(w_hh).precision != Precision::FP16 || - gb->get_output_buffer(b_ih).precision != Precision::FP16) { - throw std::runtime_error( - "ParakeetTDT predictor LSTM weights must be FP16. " - "Re-convert the model with updated converter."); - } - - const auto& w_ih_shape = gb->get_output_buffer(w_ih).shape; - if (w_ih_shape.size() != 2 || (w_ih_shape[0] % 4) != 0) { - throw std::runtime_error("ParakeetTDT predictor LSTM weight_ih must be [4*hidden, input]"); - } - const size_t hidden_size = w_ih_shape[0] / 4; - hidden_sizes.push_back(hidden_size); - - size_t h_prev = gb->input({1, hidden_size}, Precision::FP16); - size_t c_prev = gb->input({1, hidden_size}, Precision::FP16); - size_t b_hh_zero = gb->input({4 * hidden_size}, Precision::FP16); - - size_t lstm_out = gb->lstm_cell(pred, h_prev, c_prev, w_ih, w_hh, b_ih, b_hh_zero); - size_t h_new = gb->slice(lstm_out, 2, 0, 1); - size_t c_new = gb->slice(lstm_out, 2, 1, 1); - - h_new = gb->reshape(h_new, {1, hidden_size}); - c_new = gb->reshape(c_new, {1, hidden_size}); - - pred = h_new; - - h_prev_nodes.push_back(h_prev); - c_prev_nodes.push_back(c_prev); - h_new_nodes.push_back(h_new); - c_new_nodes.push_back(c_new); - bias_hh_zero_nodes.push_back(b_hh_zero); - } - - size_t enc_proj = gb->matmul(frame_in, weight_nodes_.joint_enc_weight, true, ComputeBackend::CPU); - enc_proj = gb->add(enc_proj, weight_nodes_.joint_enc_bias); - - size_t pred_proj = gb->matmul(pred, weight_nodes_.joint_pred_weight, true, ComputeBackend::CPU); - pred_proj = gb->add(pred_proj, weight_nodes_.joint_pred_bias); - - size_t joint = gb->add(enc_proj, pred_proj); - joint = gb->relu(joint); - - size_t logits = gb->matmul(joint, weight_nodes_.joint_out_weight, true, ComputeBackend::CPU); - logits = gb->add(logits, weight_nodes_.joint_out_bias); - - const auto& logits_shape = gb->get_output_buffer(logits).shape; - if (logits_shape.size() != 2 || logits_shape[0] != 1) { - throw std::runtime_error("ParakeetTDT joint output must be [1, classes]"); - } - - const size_t total_classes = logits_shape[1]; - std::vector durations = config_.tdt_durations; - size_t duration_classes = durations.size(); - if (duration_classes == 0) { - const size_t fallback = static_cast(config_.tdt_num_durations); - if (fallback > 0) { - durations.resize(fallback); - for (size_t i = 0; i < fallback; ++i) { - durations[i] = static_cast(i); - } - duration_classes = fallback; - } - } - - if (duration_classes == 0 || duration_classes >= total_classes) { - throw std::runtime_error("ParakeetTDT duration classes are invalid"); - } - - const size_t token_classes = total_classes - duration_classes; - - uint32_t blank_id = config_.tdt_blank_id; - if (blank_id >= token_classes) { - const auto& emb_shape = gb->get_output_buffer(weight_nodes_.predictor_embed).shape; - if (emb_shape.size() == 2 && emb_shape[0] > 0) { - const size_t inferred_blank = emb_shape[0] - 1; - blank_id = static_cast(std::min(inferred_blank, token_classes - 1)); - } else { - blank_id = static_cast(token_classes - 1); - } - } - - const size_t time_limit = std::min(end_frame, T); - const size_t emit_begin = std::min(start_frame, time_limit); - const size_t time_begin = std::min(replay_start_frame, emit_begin); - - std::vector> h_state(predictor_layers); - std::vector> c_state(predictor_layers); - std::vector> bias_hh_zero_state(predictor_layers); - const bool can_resume_stream_state = - stream_state && - stream_state->initialized && - stream_state->h.size() == predictor_layers && - stream_state->c.size() == predictor_layers; - - for (size_t i = 0; i < predictor_layers; ++i) { - const bool can_resume_layer = - can_resume_stream_state && - stream_state->h[i].size() == hidden_sizes[i] && - stream_state->c[i].size() == hidden_sizes[i]; - if (can_resume_layer) { - h_state[i] = stream_state->h[i]; - c_state[i] = stream_state->c[i]; - } else { - h_state[i].assign(hidden_sizes[i], static_cast<__fp16>(0.0f)); - c_state[i].assign(hidden_sizes[i], static_cast<__fp16>(0.0f)); - } - bias_hh_zero_state[i].assign(4 * hidden_sizes[i], static_cast<__fp16>(0.0f)); - gb->set_input(bias_hh_zero_nodes[i], bias_hh_zero_state[i].data(), Precision::FP16); - } - - constexpr float kHopSec = 160.0f / 16000.0f; - const float frame_sec = kHopSec * static_cast(config_.subsampling_factor); - auto* tokenizer = get_tokenizer(); - - std::vector output_tokens; - output_tokens.reserve(T * 2); - - uint32_t last_token = blank_id; - if (can_resume_stream_state && - stream_state->last_token < token_classes) { - last_token = stream_state->last_token; - } - constexpr size_t kMaxSymbolsPerStep = 10; - size_t time_idx = time_begin; - const bool is_stream_mode = cactus::telemetry::isStreamMode(); - std::vector> snap_h = h_state; - std::vector> snap_c = c_state; - uint32_t snap_last_token = last_token; - size_t confirmed_count = 0; - double raw_decoder_time_ms = 0.0; - - while (time_idx < time_limit) { - bool advanced = false; - size_t symbols_added = 0; - - while (symbols_added < kMaxSymbolsPerStep) { - const __fp16* frame_ptr = encoder_fp16.data() + time_idx * D; - gb->set_input(frame_in, frame_ptr, Precision::FP16); - - float token_value = static_cast(last_token); - gb->set_input(token_idx, &token_value, Precision::FP32); - - for (size_t i = 0; i < predictor_layers; ++i) { - gb->set_input(h_prev_nodes[i], h_state[i].data(), Precision::FP16); - gb->set_input(c_prev_nodes[i], c_state[i].data(), Precision::FP16); - } - - const auto decoder_step_start = std::chrono::steady_clock::now(); - gb->execute(); - const auto decoder_step_end = std::chrono::steady_clock::now(); - raw_decoder_time_ms += - std::chrono::duration_cast( - decoder_step_end - decoder_step_start).count() / 1000.0; - - const auto& logits_buf = gb->get_output_buffer(logits); - const auto& bias = get_vocab_bias(); - const size_t best_token = argmax_range_with_bias(logits_buf, 0, token_classes, bias); - const size_t best_duration_idx = argmax_range(logits_buf, token_classes, duration_classes); - const uint32_t predicted_skip = durations[best_duration_idx]; - uint32_t skip = predicted_skip; - if (is_stream_mode && skip > kMaxStreamDurationSkipFrames) { - skip = kMaxStreamDurationSkipFrames; - } - - if (best_token != blank_id) { - if (stream_state && time_idx >= emit_begin && tokenizer) { - std::string piece = tokenizer->decode({static_cast(best_token)}); - if (!piece.empty() && piece[0] == ' ' && !output_tokens.empty()) { - snap_h = h_state; - snap_c = c_state; - snap_last_token = last_token; - confirmed_count = output_tokens.size(); - } - } - if (time_idx >= emit_begin) { - output_tokens.push_back( - {static_cast(best_token), time_idx * frame_sec, (time_idx + skip) * frame_sec}); - } - last_token = static_cast(best_token); - - for (size_t i = 0; i < predictor_layers; ++i) { - const auto& h_buf = gb->get_output_buffer(h_new_nodes[i]); - const auto& c_buf = gb->get_output_buffer(c_new_nodes[i]); - const __fp16* h_ptr = h_buf.data_as<__fp16>(); - const __fp16* c_ptr = c_buf.data_as<__fp16>(); - std::copy(h_ptr, h_ptr + hidden_sizes[i], h_state[i].begin()); - std::copy(c_ptr, c_ptr + hidden_sizes[i], c_state[i].begin()); - } - } - - ++symbols_added; - - if (skip > 0) { - time_idx += skip; - advanced = true; - break; - } - - if (best_token == blank_id) { - ++time_idx; - advanced = true; - break; - } - } - - if (!advanced) { - ++time_idx; - } - } - - if (out_confirmed_count) { - *out_confirmed_count = confirmed_count; - } - if (out_raw_decoder_time_ms) { - *out_raw_decoder_time_ms = raw_decoder_time_ms; - } - - if (stream_state) { - stream_state->initialized = true; - stream_state->last_token = snap_last_token; - stream_state->h = std::move(snap_h); - stream_state->c = std::move(snap_c); - } - - return output_tokens; -} - -std::vector ParakeetTDTModel::greedy_decode_tdt_tokens( - CactusGraph* gb, - size_t encoder_hidden_node) const { - return decode_tdt_tokens_with_state( - gb, - encoder_hidden_node, - 0, - 0, - std::numeric_limits::max(), - nullptr); -} - -uint32_t ParakeetTDTModel::decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty, - float* out_token_time_start, - float* out_token_time_end) -{ - (void)temperature; - (void)top_p; - (void)top_k; - (void)min_p; - (void)repetition_penalty; - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (audio_features.empty()) { - throw std::runtime_error("Audio features cannot be empty in ParakeetTDT decode_with_audio"); - } - - auto* gb = static_cast(graph_handle_); - - const bool new_request = !tdt_tokens_ready_ || tokens.empty() || tokens.size() < last_input_token_count_; - if (new_request) { - gb->soft_reset(); - size_t encoder_out = forward(audio_features, tokens, false); - - if (!profile_file.empty()) { - gb->execute(profile_file); - } else { - gb->execute(); - } - - tdt_tokens_ = greedy_decode_tdt_tokens(gb, encoder_out); - tdt_emit_index_ = 0; - tdt_tokens_ready_ = true; - } - - last_input_token_count_ = tokens.size(); - if (out_entropy) { - *out_entropy = 0.0f; - } - - if (tdt_emit_index_ < tdt_tokens_.size()) { - const auto& tok = tdt_tokens_[tdt_emit_index_++]; - if (out_token_time_start) *out_token_time_start = tok.time_start; - if (out_token_time_end) *out_token_time_end = tok.time_end; - return tok.id; - } - - return get_tokenizer()->get_eos_token(); -} - -ParakeetTDTModel::ChunkStreamResult ParakeetTDTModel::decode_chunk_stream( - const std::vector& audio_features, - size_t replay_start_frame, - size_t start_frame, - size_t end_frame, - ChunkStreamState& state) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (audio_features.empty()) { - throw std::runtime_error("Audio features cannot be empty in ParakeetTDT decode_chunk_stream"); - } - - auto* gb = static_cast(graph_handle_); - - gb->soft_reset(); - size_t encoder_out = forward(audio_features, {}, false); - gb->execute(); - - ChunkStreamResult result; - double raw_decoder_time_ms = 0.0; - std::vector tokens = decode_tdt_tokens_with_state( - gb, encoder_out, replay_start_frame, start_frame, end_frame, &state, - &result.confirmed_token_count, &raw_decoder_time_ms); - - result.token_count = tokens.size(); - result.raw_decoder_time_ms = raw_decoder_time_ms; - result.raw_decoder_tps = - (result.token_count > 0 && raw_decoder_time_ms > 0.0) - ? (static_cast(result.token_count) * 1000.0) / raw_decoder_time_ms - : 0.0; - constexpr float kHopSec = 160.0f / 16000.0f; - const float frame_sec = kHopSec * static_cast(config_.subsampling_factor); - result.start_sec = start_frame * frame_sec; - result.confirmed_end_sec = start_frame * frame_sec; - result.resume_end_sec = replay_start_frame * frame_sec; - result.end_sec = start_frame * frame_sec; - if (!tokens.empty()) { - result.start_sec = tokens.front().time_start; - result.end_sec = tokens.back().time_end; - if (result.confirmed_token_count > 0 && - result.confirmed_token_count <= tokens.size()) { - result.confirmed_end_sec = tokens[result.confirmed_token_count - 1].time_end; - result.resume_end_sec = result.confirmed_end_sec; - } - } - - auto* tokenizer = get_tokenizer(); - if (!tokenizer) { - throw std::runtime_error("Tokenizer unavailable in ParakeetTDT decode_chunk_stream"); - } - for (size_t i = 0; i < tokens.size(); ++i) { - const std::string piece = tokenizer->decode({tokens[i].id}); - result.text += piece; - if (i < result.confirmed_token_count) { - result.confirmed_text += piece; - } else { - result.pending_text += piece; - } - } - return result; -} - -std::vector ParakeetTDTModel::get_audio_embeddings(const std::vector& audio_features) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - size_t hidden = build_encoder(gb, audio_features); - size_t pooled = gb->mean(hidden, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - const size_t hidden_dim = output_buf.total_size; - std::vector embedding(hidden_dim); - - if (output_buf.precision == Precision::FP32) { - const float* src = output_buf.data_as(); - std::copy(src, src + hidden_dim, embedding.begin()); - } else if (output_buf.precision == Precision::FP16) { - const __fp16* src = output_buf.data_as<__fp16>(); - Quantization::fp16_to_fp32(src, embedding.data(), hidden_dim); - } else { - const int8_t* src = output_buf.data_as(); - Quantization::int8_to_fp32(src, embedding.data(), hidden_dim, 1.0f); - } - - reset_cache(); - return embedding; -} - -void ParakeetTDTModel::reset_cache() { - Model::reset_cache(); - tdt_tokens_ready_ = false; - tdt_emit_index_ = 0; - tdt_tokens_.clear(); - last_input_token_count_ = 0; -} - -} // namespace engine -} // namespace cactus diff --git a/cactus-transpile/models/model_pyannote.cpp b/cactus-transpile/models/model_pyannote.cpp deleted file mode 100644 index 44ac4c8c3..000000000 --- a/cactus-transpile/models/model_pyannote.cpp +++ /dev/null @@ -1,201 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include -#include -#include - -namespace cactus { -namespace engine { - -static constexpr size_t WINDOW_SAMPLES = 160000; -static constexpr size_t FRAMES_PER_WINDOW = 589; -static constexpr int NUM_CLASSES = 7; -static constexpr int NUM_SPEAKERS = 3; - -PyAnnoteModel::PyAnnoteModel() : Model() {} -PyAnnoteModel::PyAnnoteModel(const Config& config) : Model(config) {} - -void PyAnnoteModel::load_weights_to_graph(CactusGraph* gb) { - const std::string& p = model_folder_path_; - - weight_nodes_.sinc_filters = gb->mmap_weights(p + "/sincnet_sinc_filters.weights"); - weight_nodes_.wav_norm_weight = gb->mmap_weights(p + "/sincnet_wav_norm_weight.weights"); - weight_nodes_.wav_norm_bias = gb->mmap_weights(p + "/sincnet_wav_norm_bias.weights"); - weight_nodes_.norm0_weight = gb->mmap_weights(p + "/sincnet_norm0_weight.weights"); - weight_nodes_.norm0_bias = gb->mmap_weights(p + "/sincnet_norm0_bias.weights"); - weight_nodes_.conv1_weight = gb->mmap_weights(p + "/sincnet_conv1_weight.weights"); - weight_nodes_.conv1_bias = gb->mmap_weights(p + "/sincnet_conv1_bias.weights"); - weight_nodes_.norm1_weight = gb->mmap_weights(p + "/sincnet_norm1_weight.weights"); - weight_nodes_.norm1_bias = gb->mmap_weights(p + "/sincnet_norm1_bias.weights"); - weight_nodes_.conv2_weight = gb->mmap_weights(p + "/sincnet_conv2_weight.weights"); - weight_nodes_.conv2_bias = gb->mmap_weights(p + "/sincnet_conv2_bias.weights"); - weight_nodes_.norm2_weight = gb->mmap_weights(p + "/sincnet_norm2_weight.weights"); - weight_nodes_.norm2_bias = gb->mmap_weights(p + "/sincnet_norm2_bias.weights"); - - for (int layer = 0; layer < 4; ++layer) { - auto& lw = weight_nodes_.lstm_layers[layer]; - std::string fwd = p + "/lstm_fwd_" + std::to_string(layer); - std::string bwd = p + "/lstm_bwd_" + std::to_string(layer); - lw.w_ih_fwd = gb->mmap_weights(fwd + "_weight_ih.weights"); - lw.w_hh_fwd = gb->mmap_weights(fwd + "_weight_hh.weights"); - lw.b_ih_fwd = gb->mmap_weights(fwd + "_bias_ih.weights"); - lw.b_hh_fwd = gb->mmap_weights(fwd + "_bias_hh.weights"); - lw.w_ih_bwd = gb->mmap_weights(bwd + "_weight_ih.weights"); - lw.w_hh_bwd = gb->mmap_weights(bwd + "_weight_hh.weights"); - lw.b_ih_bwd = gb->mmap_weights(bwd + "_bias_ih.weights"); - lw.b_hh_bwd = gb->mmap_weights(bwd + "_bias_hh.weights"); - } - - weight_nodes_.linear0_weight = gb->mmap_weights(p + "/linear_0_weight.weights"); - weight_nodes_.linear0_bias = gb->mmap_weights(p + "/linear_0_bias.weights"); - weight_nodes_.linear1_weight = gb->mmap_weights(p + "/linear_1_weight.weights"); - weight_nodes_.linear1_bias = gb->mmap_weights(p + "/linear_1_bias.weights"); - weight_nodes_.classifier_weight = gb->mmap_weights(p + "/classifier_weight.weights"); - weight_nodes_.classifier_bias = gb->mmap_weights(p + "/classifier_bias.weights"); -} - -void PyAnnoteModel::build_graph() { - audio_input_ = graph_.input({1, 1, 160000}, Precision::FP16); - - size_t x = graph_.groupnorm(audio_input_, weight_nodes_.wav_norm_weight, weight_nodes_.wav_norm_bias, 1, 1e-5f); - x = graph_.conv1d(x, weight_nodes_.sinc_filters, 10); - x = graph_.abs(x); - x = graph_.maxpool1d(x, 3, 3); - x = graph_.groupnorm(x, weight_nodes_.norm0_weight, weight_nodes_.norm0_bias, 80, 1e-5f); - x = graph_.leaky_relu(x, 0.01f); - - x = graph_.conv1d(x, weight_nodes_.conv1_weight, weight_nodes_.conv1_bias, 1); - x = graph_.maxpool1d(x, 3, 3); - x = graph_.groupnorm(x, weight_nodes_.norm1_weight, weight_nodes_.norm1_bias, 60, 1e-5f); - x = graph_.leaky_relu(x, 0.01f); - - x = graph_.conv1d(x, weight_nodes_.conv2_weight, weight_nodes_.conv2_bias, 1); - x = graph_.maxpool1d(x, 3, 3); - x = graph_.groupnorm(x, weight_nodes_.norm2_weight, weight_nodes_.norm2_bias, 60, 1e-5f); - x = graph_.leaky_relu(x, 0.01f); - - x = graph_.transposeN(x, {0, 2, 1}); - - for (int layer = 0; layer < 4; ++layer) { - auto& lw = weight_nodes_.lstm_layers[layer]; - x = graph_.bilstm_sequence(x, - lw.w_ih_fwd, lw.w_hh_fwd, lw.b_ih_fwd, lw.b_hh_fwd, - lw.w_ih_bwd, lw.w_hh_bwd, lw.b_ih_bwd, lw.b_hh_bwd); - } - - const auto& bilstm_shape = graph_.get_output_buffer(x).shape; - size_t T = bilstm_shape[1]; - x = graph_.reshape(x, {T, bilstm_shape[2]}); - - x = graph_.add(graph_.matmul(x, weight_nodes_.linear0_weight, true), weight_nodes_.linear0_bias); - x = graph_.leaky_relu(x, 0.01f); - x = graph_.add(graph_.matmul(x, weight_nodes_.linear1_weight, true), weight_nodes_.linear1_bias); - x = graph_.leaky_relu(x, 0.01f); - - x = graph_.add(graph_.matmul(x, weight_nodes_.classifier_weight, true), weight_nodes_.classifier_bias); - x = graph_.softmax(x, -1); - x = graph_.reshape(x, {1, T, 7}); - - output_node_ = x; -} - -bool PyAnnoteModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - (void)context_size; - (void)system_prompt; - (void)do_warmup; - - if (initialized_) { - return true; - } - - model_folder_path_ = model_folder; - - try { - load_weights_to_graph(&graph_); - build_graph(); - - chunk_buf_.resize(WINDOW_SAMPLES); - - hamming_.resize(FRAMES_PER_WINDOW); - for (size_t i = 0; i < FRAMES_PER_WINDOW; ++i) - hamming_[i] = 0.54f - 0.46f * std::cos(2.0f * M_PI * i / (FRAMES_PER_WINDOW - 1)); - - initialized_ = true; - return true; - } catch (const std::exception& e) { - initialized_ = false; - return false; - } -} - -std::vector PyAnnoteModel::diarize(const float* pcm_f32, size_t num_samples, size_t step_samples, bool raw_powerset) { - if (!initialized_) throw std::runtime_error("PyAnnote model not initialized"); - - const size_t total_frames = std::max( - FRAMES_PER_WINDOW, - static_cast(std::round((double)num_samples * FRAMES_PER_WINDOW / WINDOW_SAMPLES)) - ); - - std::vector aggregated(total_frames * NUM_CLASSES, 0.0f); - std::vector weight_sum(total_frames, 0.0f); - - auto process_chunk = [&](size_t chunk_start) { - const size_t copy_len = std::min(WINDOW_SAMPLES, num_samples - chunk_start); - for (size_t i = 0; i < copy_len; ++i) - chunk_buf_[i] = static_cast<__fp16>(pcm_f32[chunk_start + i]); - std::fill(chunk_buf_.begin() + copy_len, chunk_buf_.end(), static_cast<__fp16>(0.0f)); - - graph_.set_input(audio_input_, chunk_buf_.data(), Precision::FP16); - graph_.execute(); - - const __fp16* chunk_scores = graph_.get_output_buffer(output_node_).data_as<__fp16>(); - const size_t frame_offset = static_cast(std::round((double)chunk_start * FRAMES_PER_WINDOW / WINDOW_SAMPLES)); - - for (size_t f = 0; f < FRAMES_PER_WINDOW; ++f) { - const size_t out_f = frame_offset + f; - if (out_f >= total_frames) break; - const float w = hamming_[f]; - weight_sum[out_f] += w; - for (int c = 0; c < NUM_CLASSES; ++c) - aggregated[out_f * NUM_CLASSES + c] += w * static_cast(chunk_scores[f * NUM_CLASSES + c]); - } - }; - - const size_t last_start = num_samples > WINDOW_SAMPLES ? num_samples - WINDOW_SAMPLES : 0; - bool last_processed = false; - for (size_t s = 0; s + WINDOW_SAMPLES <= num_samples; s += step_samples) { - if (s >= last_start) last_processed = true; - process_chunk(s); - } - if (!last_processed) - process_chunk(last_start); - - for (size_t f = 0; f < total_frames; ++f) { - if (weight_sum[f] > 0.0f) { - const float inv_w = 1.0f / weight_sum[f]; - float* agg = aggregated.data() + f * NUM_CLASSES; - for (int c = 0; c < NUM_CLASSES; ++c) - agg[c] *= inv_w; - } - } - - if (raw_powerset) { - return aggregated; - } - - std::vector speaker_probs(total_frames * NUM_SPEAKERS, 0.0f); - for (size_t f = 0; f < total_frames; ++f) { - const float* agg = aggregated.data() + f * NUM_CLASSES; - float* dst = speaker_probs.data() + f * NUM_SPEAKERS; - dst[0] = agg[1] + agg[4] + agg[5]; - dst[1] = agg[2] + agg[4] + agg[6]; - dst[2] = agg[3] + agg[5] + agg[6]; - } - - return speaker_probs; -} - -} -} diff --git a/cactus-transpile/models/model_qwen.cpp b/cactus-transpile/models/model_qwen.cpp deleted file mode 100644 index 601c05334..000000000 --- a/cactus-transpile/models/model_qwen.cpp +++ /dev/null @@ -1,171 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include -#include - - -namespace cactus { -namespace engine { - -QwenModel::QwenModel() : Model() {} - -QwenModel::QwenModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -void QwenModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - } - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - load_npu_prefill(npu_prefill_path); - } -} - -size_t QwenModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); - q_proj = gb->rms_norm(q_proj, layer.attn_q_norm_weight, config_.layer_norm_eps); - q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); - - size_t num_kv_heads = config_.attention_kv_heads; - k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); - k_proj = gb->rms_norm(k_proj, layer.attn_k_norm_weight, config_.layer_norm_eps); - k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); - - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); - } - - size_t attn_output_4d; - - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - } - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - return gb->matmul(attn_output, layer.attn_output_weight, true, backend); -} - - -size_t QwenModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - size_t gate_silu = gb->silu(gate_output); - size_t gated = gb->multiply(gate_silu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - - -size_t QwenModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - auto after_attention = gb->add(hidden, attn_output); - auto normalized_after_attention = gb->rms_norm(after_attention, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); - return gb->add(after_attention, mlp_output); -} - - -size_t QwenModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto seq_len = static_cast(tokens.size()); - - size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - return final_hidden; -} - -} -} \ No newline at end of file diff --git a/cactus-transpile/models/model_qwen3p5.cpp b/cactus-transpile/models/model_qwen3p5.cpp deleted file mode 100644 index 0f438e1c0..000000000 --- a/cactus-transpile/models/model_qwen3p5.cpp +++ /dev/null @@ -1,771 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Qwen3p5Model::Qwen3p5Model() : Model() { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_state_nodes_.assign(config_.num_layers, 0); -} - -Qwen3p5Model::Qwen3p5Model(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); - conv_cache_state_nodes_.assign(config.num_layers, 0); -} - -void Qwen3p5Model::post_init() { - bool has_deltanet_layer = false; - const WeightNodeIDs::LayerWeights* first_linear = nullptr; - for (const auto& layer : weight_nodes_.layers) { - if (layer.type == WeightNodeIDs::LayerType::DELTANET) { - has_deltanet_layer = true; - if (!first_linear) { - first_linear = &layer.weights; - } - break; - } - } - - deltanet_heads_ = static_cast(config_.attention_heads); - deltanet_key_dim_ = static_cast(config_.attention_head_dim); - deltanet_value_dim_ = static_cast(config_.attention_head_dim); - - if (has_deltanet_layer && - config_.linear_num_key_heads > 0 && - config_.linear_key_head_dim > 0) { - const size_t num_k_heads = static_cast(config_.linear_num_key_heads); - const size_t key_head_dim = static_cast(config_.linear_key_head_dim); - size_t num_v_heads = static_cast(config_.linear_num_value_heads > 0 - ? config_.linear_num_value_heads - : config_.linear_num_key_heads); - size_t value_head_dim = static_cast(config_.linear_value_head_dim); - - if (num_v_heads == 0) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention value head count in config"); - } - if (value_head_dim == 0) { - if (config_.linear_v_proj_dim > 0 && (config_.linear_v_proj_dim % num_v_heads == 0)) { - value_head_dim = static_cast(config_.linear_v_proj_dim / num_v_heads); - } else { - throw std::runtime_error("Qwen3p5Model cannot derive linear-attention value head dim from config"); - } - } - if (config_.linear_q_proj_dim > 0 && - static_cast(config_.linear_q_proj_dim) != num_k_heads * key_head_dim) { - throw std::runtime_error("Qwen3p5Model linear-attention q projection dim mismatch with key-head topology"); - } - if (config_.linear_v_proj_dim > 0 && - static_cast(config_.linear_v_proj_dim) != num_v_heads * value_head_dim) { - throw std::runtime_error("Qwen3p5Model linear-attention v projection dim mismatch with value-head topology"); - } - - deltanet_heads_ = num_v_heads; - deltanet_key_dim_ = key_head_dim; - deltanet_value_dim_ = value_head_dim; - } else if (has_deltanet_layer) { - auto* gb = static_cast(graph_handle_); - if (!gb) { - throw std::runtime_error("Qwen3p5Model cannot infer linear-attention dims: graph unavailable"); - } - if (!first_linear) { - throw std::runtime_error("Qwen3p5Model expected deltanet layer weights for dim inference"); - } - - const auto& q_buf = gb->get_output_buffer(first_linear->attn_q_weight); - const auto& v_buf = gb->get_output_buffer(first_linear->attn_v_weight); - const auto& n_buf = gb->get_output_buffer(first_linear->attn_q_norm_weight); - if (q_buf.shape.size() < 2 || v_buf.shape.size() < 2 || n_buf.shape.empty()) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention weight shapes"); - } - - const size_t q_proj_dim = q_buf.shape[0]; - const size_t v_proj_dim = v_buf.shape[0]; - const size_t inferred_key_dim = n_buf.shape[0]; - if (inferred_key_dim == 0 || q_proj_dim == 0 || v_proj_dim == 0) { - throw std::runtime_error("Qwen3p5Model failed to infer non-zero linear-attention dims"); - } - if (q_proj_dim % inferred_key_dim != 0) { - throw std::runtime_error("Qwen3p5Model q projection dim is not divisible by inferred key dim"); - } - - const size_t inferred_k_heads = q_proj_dim / inferred_key_dim; - const size_t inferred_v_heads = v_proj_dim / inferred_key_dim; - if (inferred_k_heads == 0 || inferred_v_heads == 0 || inferred_v_heads % inferred_k_heads != 0) { - throw std::runtime_error("Qwen3p5Model failed to infer compatible linear-attention head topology"); - } - - deltanet_heads_ = inferred_v_heads; - deltanet_key_dim_ = inferred_key_dim; - deltanet_value_dim_ = inferred_key_dim; - } - - if (deltanet_heads_ == 0 || deltanet_key_dim_ == 0 || deltanet_value_dim_ == 0) { - throw std::runtime_error("Qwen3p5Model requires non-zero deltanet heads/key/value dims"); - } - - deltanet_state_flat_dim_ = deltanet_heads_ * deltanet_key_dim_ * deltanet_value_dim_; - - deltanet_mixed_dim_ = 0; - if (config_.linear_q_proj_dim > 0 && config_.linear_v_proj_dim > 0) { - deltanet_mixed_dim_ = static_cast(2 * config_.linear_q_proj_dim + config_.linear_v_proj_dim); - } else if (has_deltanet_layer) { - auto* gb = static_cast(graph_handle_); - if (!gb || !first_linear) { - throw std::runtime_error("Qwen3p5Model cannot infer linear-attention mixed projection dim"); - } - const auto& q_buf = gb->get_output_buffer(first_linear->attn_q_weight); - const auto& v_buf = gb->get_output_buffer(first_linear->attn_v_weight); - if (q_buf.shape.size() < 2 || v_buf.shape.size() < 2) { - throw std::runtime_error("Qwen3p5Model invalid q/v projection shapes for mixed dim inference"); - } - deltanet_mixed_dim_ = (2 * q_buf.shape[0]) + v_buf.shape[0]; - } - - deltanet_conv_history_len_ = 0; - if (has_deltanet_layer) { - auto* gb = static_cast(graph_handle_); - if (!gb) { - throw std::runtime_error("Qwen3p5Model cannot infer linear-attention conv history: graph unavailable"); - } - for (const auto& layer : weight_nodes_.layers) { - if (layer.type != WeightNodeIDs::LayerType::DELTANET || layer.weights.deltanet_conv_weight == 0) { - continue; - } - const auto& conv_buf = gb->get_output_buffer(layer.weights.deltanet_conv_weight); - if (conv_buf.shape.empty()) { - continue; - } - const size_t kernel = conv_buf.shape.back(); - if (kernel > 1) { - deltanet_conv_history_len_ = std::max(deltanet_conv_history_len_, kernel - 1); - } - } - } - - if (deltanet_conv_history_len_ > 0 && deltanet_mixed_dim_ == 0) { - throw std::runtime_error("Qwen3p5Model linear-attention conv history requires non-zero mixed dim"); - } - deltanet_conv_flat_dim_ = deltanet_conv_history_len_ * deltanet_mixed_dim_; - deltanet_cache_row_dim_ = deltanet_state_flat_dim_ + deltanet_conv_flat_dim_; - if (deltanet_cache_row_dim_ == 0) { - throw std::runtime_error("Qwen3p5Model computed zero cache row dim"); - } - - conv_cache_.init(config_.num_layers, deltanet_cache_row_dim_, 1, Precision::FP32); - last_forward_used_cache_ = false; - deltanet_total_seq_len_ = 0; -} - -void Qwen3p5Model::reset_cache() { - Model::reset_cache(); - if (conv_cache_.window_size > 0) { - conv_cache_.reset(); - } - std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); - deltanet_total_seq_len_ = 0; - last_forward_used_cache_ = false; -} - -void Qwen3p5Model::post_execute_updates(CactusGraph* gb, size_t seq_len) { - if (!last_forward_used_cache_) { - std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); - return; - } - - const size_t layer_count = std::min(conv_cache_state_nodes_.size(), weight_nodes_.layers.size()); - for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { - if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::DELTANET) { - conv_cache_state_nodes_[layer_idx] = 0; - continue; - } - - const size_t state_node = conv_cache_state_nodes_[layer_idx]; - if (state_node != 0) { - conv_cache_.update(gb, layer_idx, state_node); - } - conv_cache_state_nodes_[layer_idx] = 0; - } - - deltanet_total_seq_len_ += seq_len; - last_forward_used_cache_ = false; -} - -void Qwen3p5Model::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - auto choose_existing_weight = [&](const std::vector& candidates) -> std::string { - for (const auto& p : candidates) { - if (std::filesystem::exists(p)) { - return p; - } - } - return std::string(); - }; - - auto mmap_required = [&](const std::vector& candidates, const std::string& weight_name) -> size_t { - const std::string path = choose_existing_weight(candidates); - if (path.empty()) { - throw std::runtime_error("Qwen3p5Model missing required weight: " + weight_name); - } - return gb->mmap_weights(path); - }; - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer_entry = weight_nodes_.layers[i]; - auto& layer = layer_entry.weights; - - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - bool is_deltanet = false; - if (i < config_.layer_types.size()) { - std::string lt = config_.layer_types[i]; - std::transform(lt.begin(), lt.end(), lt.begin(), ::tolower); - is_deltanet = (lt.find("deltanet") != std::string::npos || - lt.find("linear_attention") != std::string::npos || - lt.find("linear_attn") != std::string::npos); - } - layer_entry.type = is_deltanet ? WeightNodeIDs::LayerType::DELTANET : WeightNodeIDs::LayerType::ATTENTION; - - layer.attn_q_weight = mmap_required( - {layer_prefix + "attn_q.weights", layer_prefix + "linear_attn_q.weights"}, - "layer_" + std::to_string(i) + "_attn_q/linear_attn_q"); - layer.attn_k_weight = mmap_required( - {layer_prefix + "attn_k.weights", layer_prefix + "linear_attn_k.weights"}, - "layer_" + std::to_string(i) + "_attn_k/linear_attn_k"); - layer.attn_v_weight = mmap_required( - {layer_prefix + "attn_v.weights", layer_prefix + "linear_attn_v.weights"}, - "layer_" + std::to_string(i) + "_attn_v/linear_attn_v"); - layer.attn_output_weight = mmap_required( - {layer_prefix + "attn_output.weights", layer_prefix + "linear_attn_output.weights"}, - "layer_" + std::to_string(i) + "_attn_output/linear_attn_output"); - layer.input_layernorm_weight = mmap_required( - {layer_prefix + "input_norm.weights"}, - "layer_" + std::to_string(i) + "_input_norm"); - layer.attn_q_norm_weight = mmap_required( - {layer_prefix + "attn_q_norm.weights", layer_prefix + "linear_attn_norm.weights"}, - "layer_" + std::to_string(i) + "_attn_q_norm/linear_attn_norm"); - layer.attn_k_norm_weight = mmap_required( - {layer_prefix + "attn_k_norm.weights", layer_prefix + "linear_attn_norm.weights"}, - "layer_" + std::to_string(i) + "_attn_k_norm/linear_attn_norm"); - - if (is_deltanet) { - const std::string qkv_path = choose_existing_weight({ - layer_prefix + "linear_attn_qkv.weights" - }); - layer.deltanet_qkv_weight = qkv_path.empty() ? 0 : gb->mmap_weights(qkv_path); - - const std::string gate_path = choose_existing_weight({ - layer_prefix + "deltanet_gate.weights", - layer_prefix + "attn_gate.weights", - layer_prefix + "attn_f_gate.weights", - layer_prefix + "linear_attn_a.weights" - }); - const std::string beta_path = choose_existing_weight({ - layer_prefix + "deltanet_beta.weights", - layer_prefix + "attn_beta.weights", - layer_prefix + "attn_f_beta.weights", - layer_prefix + "linear_attn_b.weights" - }); - if (gate_path.empty() || beta_path.empty()) { - throw std::runtime_error( - "Qwen3p5Model deltanet layer missing gate/beta weights at layer " + std::to_string(i)); - } - layer.deltanet_gate_weight = gb->mmap_weights(gate_path); - layer.deltanet_beta_weight = gb->mmap_weights(beta_path); - - const std::string gate_bias_path = choose_existing_weight({ - layer_prefix + "linear_attn_A_log.weights" - }); - const std::string beta_bias_path = choose_existing_weight({ - layer_prefix + "linear_attn_dt_bias.weights" - }); - layer.deltanet_gate_bias = gate_bias_path.empty() ? 0 : gb->mmap_weights(gate_bias_path); - layer.deltanet_beta_bias = beta_bias_path.empty() ? 0 : gb->mmap_weights(beta_bias_path); - const std::string z_path = choose_existing_weight({ - layer_prefix + "linear_attn_z.weights" - }); - const std::string conv_path = choose_existing_weight({ - layer_prefix + "linear_attn_conv1d.weights" - }); - layer.deltanet_z_weight = z_path.empty() ? 0 : gb->mmap_weights(z_path); - layer.deltanet_conv_weight = conv_path.empty() ? 0 : gb->mmap_weights(conv_path); - } - - layer.ffn_gate_weight = mmap_required( - {layer_prefix + "ffn_gate.weights"}, - "layer_" + std::to_string(i) + "_ffn_gate"); - layer.ffn_up_weight = mmap_required( - {layer_prefix + "ffn_up.weights"}, - "layer_" + std::to_string(i) + "_ffn_up"); - layer.ffn_down_weight = mmap_required( - {layer_prefix + "ffn_down.weights"}, - "layer_" + std::to_string(i) + "_ffn_down"); - layer.post_attention_layernorm_weight = mmap_required( - {layer_prefix + "post_attn_norm.weights"}, - "layer_" + std::to_string(i) + "_post_attn_norm"); - } - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - load_npu_prefill(npu_prefill_path); - } -} - -size_t Qwen3p5Model::build_gated_deltanet(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - (void)position_offset; - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - - const auto& in_shape = gb->get_output_buffer(normalized_input).shape; - if (in_shape.size() < 2) { - throw std::runtime_error("Qwen3p5Model linear-attention expects rank-2 normalized input"); - } - const size_t seq_len = in_shape[0]; - - size_t q_proj = 0; - size_t k_proj = 0; - size_t v_proj = 0; - size_t mixed_qkv = 0; - size_t z_proj = layer.deltanet_z_weight ? gb->matmul(normalized_input, layer.deltanet_z_weight, true, backend) : 0; - - const size_t num_heads = deltanet_heads_; - const size_t key_dim = deltanet_key_dim_; - const size_t value_dim = deltanet_value_dim_; - const size_t v_proj_dim = num_heads * value_dim; - size_t qk_proj_dim = 0; - - if (layer.deltanet_qkv_weight != 0) { - mixed_qkv = gb->matmul(normalized_input, layer.deltanet_qkv_weight, true, backend); - const auto& mixed_shape = gb->get_output_buffer(mixed_qkv).shape; - if (mixed_shape.size() < 2 || mixed_shape[0] != seq_len) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention joint qkv projection shape"); - } - if (mixed_shape[1] <= v_proj_dim || ((mixed_shape[1] - v_proj_dim) % 2) != 0) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention joint qkv projection width"); - } - qk_proj_dim = (mixed_shape[1] - v_proj_dim) / 2; - } else { - q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - if (q_shape.size() < 2 || q_shape[0] != seq_len) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention q projection shape"); - } - qk_proj_dim = q_shape[1]; - size_t mixed_qk = gb->concat(q_proj, k_proj, 1); - mixed_qkv = gb->concat(mixed_qk, v_proj, 1); - } - - if (qk_proj_dim == 0 || qk_proj_dim % key_dim != 0 || v_proj_dim == 0) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention projection dimensions"); - } - const size_t num_k_heads = qk_proj_dim / key_dim; - if (num_heads % num_k_heads != 0) { - throw std::runtime_error("Qwen3p5Model requires num_v_heads divisible by num_k_heads"); - } - - const size_t mixed_proj_dim = qk_proj_dim + qk_proj_dim + v_proj_dim; - if (deltanet_mixed_dim_ != 0 && mixed_proj_dim != deltanet_mixed_dim_) { - throw std::runtime_error("Qwen3p5Model linear-attention mixed projection dim mismatch"); - } - const size_t state_flat_dim = deltanet_state_flat_dim_; - size_t prev_state_flat = 0; - size_t prev_conv_flat = 0; - if (use_cache && conv_cache_.window_size > 0) { - const auto view = conv_cache_.get_window(layer_idx); - if (conv_cache_.window_size == 1 && view.total_len == 1) { - const uint8_t* cache_row = nullptr; - if (view.len2 == 1 && view.ptr2 != nullptr) { - cache_row = static_cast(view.ptr2); - } else if (view.len1 == 1 && view.ptr1 != nullptr) { - cache_row = static_cast(view.ptr1); - } - if (cache_row != nullptr) { - prev_state_flat = gb->input({1, state_flat_dim}, conv_cache_.precision); - gb->set_external_input(prev_state_flat, const_cast(static_cast(cache_row)), - conv_cache_.precision); - if (deltanet_conv_flat_dim_ > 0) { - const size_t state_bytes = state_flat_dim * conv_cache_.element_size; - prev_conv_flat = gb->input({1, deltanet_conv_flat_dim_}, conv_cache_.precision); - gb->set_external_input( - prev_conv_flat, - const_cast(static_cast(cache_row + state_bytes)), - conv_cache_.precision); - } - } - } - - if (prev_state_flat == 0) { - std::vector segments; - if (view.len2 > 0) { - size_t left_node = gb->input({view.len2, deltanet_cache_row_dim_}, conv_cache_.precision); - gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); - segments.push_back(left_node); - } - if (view.len1 > 0) { - size_t right_node = gb->input({view.len1, deltanet_cache_row_dim_}, conv_cache_.precision); - gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); - segments.push_back(right_node); - } - - if (!segments.empty()) { - size_t stacked = segments[0]; - for (size_t i = 1; i < segments.size(); ++i) { - stacked = gb->concat(stacked, segments[i], 0); - } - if (view.total_len > 1) { - stacked = gb->slice(stacked, 0, view.total_len - 1, 1); - } - prev_state_flat = gb->slice(stacked, 1, 0, state_flat_dim); - if (deltanet_conv_flat_dim_ > 0) { - prev_conv_flat = gb->slice(stacked, 1, state_flat_dim, deltanet_conv_flat_dim_); - } - } - } - } - - size_t conv_source_with_history = mixed_qkv; - if (layer.deltanet_conv_weight) { - const auto& conv_wbuf = gb->get_output_buffer(layer.deltanet_conv_weight); - const size_t kernel = conv_wbuf.shape.back(); - if (kernel > 1 && use_cache && deltanet_conv_history_len_ > 0) { - size_t history_2d = 0; - if (prev_conv_flat != 0) { - history_2d = gb->reshape(prev_conv_flat, {deltanet_conv_history_len_, mixed_proj_dim}); - history_2d = gb->precision_cast(history_2d, Precision::FP16); - } else { - history_2d = gb->input({deltanet_conv_history_len_, mixed_proj_dim}, Precision::FP16); - std::vector<__fp16> zeros(deltanet_conv_flat_dim_, static_cast<__fp16>(0.0f)); - gb->set_input(history_2d, zeros.data(), Precision::FP16); - } - conv_source_with_history = gb->concat(history_2d, mixed_qkv, 0); - } - - const size_t total_conv_len = gb->get_output_buffer(conv_source_with_history).shape[0]; - size_t conv_input = gb->reshape(conv_source_with_history, {1, total_conv_len, mixed_proj_dim}); - size_t conv_out = gb->conv1d_causal(conv_input, layer.deltanet_conv_weight, kernel, 1); - const size_t conv_start = total_conv_len > seq_len ? total_conv_len - seq_len : 0; - conv_out = gb->slice(conv_out, 1, conv_start, seq_len); - mixed_qkv = gb->reshape(gb->silu(conv_out), {seq_len, mixed_proj_dim}); - } - - q_proj = gb->slice(mixed_qkv, 1, 0, qk_proj_dim); - k_proj = gb->slice(mixed_qkv, 1, qk_proj_dim, qk_proj_dim); - v_proj = gb->slice(mixed_qkv, 1, qk_proj_dim + qk_proj_dim, v_proj_dim); - - size_t q_4d = gb->reshape(q_proj, {1, seq_len, num_k_heads, key_dim}); - size_t k_4d = gb->reshape(k_proj, {1, seq_len, num_k_heads, key_dim}); - size_t v_4d = gb->reshape(v_proj, {1, seq_len, num_heads, value_dim}); - - size_t q_norm = gb->sum(gb->multiply(q_4d, q_4d), 3); - q_norm = gb->scalar_sqrt(gb->scalar_add(q_norm, 1e-6f)); - q_norm = gb->reshape(q_norm, {1, seq_len, num_k_heads, 1}); - q_4d = gb->divide(q_4d, q_norm); - - size_t k_norm = gb->sum(gb->multiply(k_4d, k_4d), 3); - k_norm = gb->scalar_sqrt(gb->scalar_add(k_norm, 1e-6f)); - k_norm = gb->reshape(k_norm, {1, seq_len, num_k_heads, 1}); - k_4d = gb->divide(k_4d, k_norm); - - size_t a_logits = gb->matmul(normalized_input, layer.deltanet_gate_weight, true, backend); - size_t b_logits = gb->matmul(normalized_input, layer.deltanet_beta_weight, true, backend); - a_logits = gb->precision_cast(a_logits, Precision::FP16); - b_logits = gb->precision_cast(b_logits, Precision::FP16); - if (layer.deltanet_beta_bias) { - size_t dt_bias_vec = layer.deltanet_beta_bias; - const auto& dt_buf = gb->get_output_buffer(dt_bias_vec); - if (!dt_buf.shape.empty() && dt_buf.shape[0] > num_heads) { - dt_bias_vec = gb->slice(dt_bias_vec, 0, 0, num_heads); - } - size_t dt_bias_2d = gb->reshape(dt_bias_vec, {1, num_heads}); - dt_bias_2d = gb->precision_cast(dt_bias_2d, Precision::FP16); - a_logits = gb->add(a_logits, dt_bias_2d); - } - size_t a_softplus = gb->scalar_log(gb->scalar_add(gb->scalar_exp(a_logits), 1.0f)); - - size_t gate_log; - if (layer.deltanet_gate_bias) { - size_t a_log_vec = layer.deltanet_gate_bias; - const auto& a_log_buf = gb->get_output_buffer(a_log_vec); - if (!a_log_buf.shape.empty() && a_log_buf.shape[0] > num_heads) { - a_log_vec = gb->slice(a_log_vec, 0, 0, num_heads); - } - size_t a_log_2d = gb->reshape(a_log_vec, {1, num_heads}); - a_log_2d = gb->precision_cast(a_log_2d, Precision::FP16); - size_t neg_exp_a = gb->scalar_multiply(gb->scalar_exp(a_log_2d), -1.0f); - gate_log = gb->multiply(neg_exp_a, a_softplus); - } else { - gate_log = gb->scalar_multiply(a_softplus, -1.0f); - } - size_t beta = gb->sigmoid(b_logits); - - size_t gate_3d = gb->reshape(gate_log, {1, seq_len, num_heads}); - size_t beta_3d = gb->reshape(beta, {1, seq_len, num_heads}); - - size_t initial_state; - - if (prev_state_flat != 0) { - initial_state = gb->reshape(prev_state_flat, {1, key_dim, num_heads, value_dim}); - } else { - initial_state = gb->input({1, key_dim, num_heads, value_dim}, Precision::FP16); - std::vector<__fp16> zeros(state_flat_dim, static_cast<__fp16>(0.0f)); - gb->set_input(initial_state, zeros.data(), Precision::FP16); - } - - size_t deltanet_out; - if (use_cache && seq_len == 1) { - deltanet_out = gb->gated_deltanet_decode(q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, 0.0f); - } else { - const size_t chunk_for_op = std::min(64, std::max(1, seq_len)); - deltanet_out = gb->gated_deltanet_prefill( - q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, chunk_for_op, 0.0f); - } - - size_t y_4d = gb->slice(deltanet_out, 1, 0, seq_len); - size_t state_tail = gb->slice(deltanet_out, 1, seq_len, key_dim); - - if (use_cache) { - size_t packed_cache = gb->reshape(state_tail, {1, state_flat_dim}); - if (deltanet_conv_flat_dim_ > 0) { - size_t history_2d = 0; - if (layer.deltanet_conv_weight) { - const size_t total_history_len = gb->get_output_buffer(conv_source_with_history).shape[0]; - if (total_history_len >= deltanet_conv_history_len_) { - const size_t hist_start = total_history_len - deltanet_conv_history_len_; - history_2d = gb->slice(conv_source_with_history, 0, hist_start, deltanet_conv_history_len_); - } else { - size_t pad_rows = deltanet_conv_history_len_ - total_history_len; - size_t pad = gb->input({pad_rows, mixed_proj_dim}, Precision::FP16); - std::vector<__fp16> zeros(pad_rows * mixed_proj_dim, static_cast<__fp16>(0.0f)); - gb->set_input(pad, zeros.data(), Precision::FP16); - history_2d = gb->concat(pad, conv_source_with_history, 0); - } - } else if (prev_conv_flat != 0) { - history_2d = gb->reshape(prev_conv_flat, {deltanet_conv_history_len_, mixed_proj_dim}); - history_2d = gb->precision_cast(history_2d, Precision::FP16); - } else { - history_2d = gb->input({deltanet_conv_history_len_, mixed_proj_dim}, Precision::FP16); - std::vector<__fp16> zeros(deltanet_conv_flat_dim_, static_cast<__fp16>(0.0f)); - gb->set_input(history_2d, zeros.data(), Precision::FP16); - } - - size_t history_flat = gb->reshape(history_2d, {1, deltanet_conv_flat_dim_}); - packed_cache = gb->concat(packed_cache, history_flat, 1); - } - packed_cache = gb->precision_cast(packed_cache, conv_cache_.precision); - conv_cache_state_nodes_[layer_idx] = packed_cache; - cache_k_output_nodes_[layer_idx] = 0; - cache_v_output_nodes_[layer_idx] = 0; - } else { - conv_cache_state_nodes_[layer_idx] = 0; - } - - size_t y_2d = gb->reshape(y_4d, {seq_len * num_heads, value_dim}); - if (z_proj != 0) { - size_t z_3d = gb->reshape(z_proj, {seq_len, num_heads, value_dim}); - size_t z_2d = gb->reshape(z_3d, {seq_len * num_heads, value_dim}); - y_2d = gb->multiply(gb->rms_norm(y_2d, layer.attn_q_norm_weight, config_.layer_norm_eps), gb->silu(z_2d)); - } - y_2d = gb->reshape(y_2d, {seq_len, num_heads * value_dim}); - return gb->matmul(y_2d, layer.attn_output_weight, true, backend); -} - -size_t Qwen3p5Model::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - - if (layer_entry.type == WeightNodeIDs::LayerType::DELTANET) { - return build_gated_deltanet(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - } - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - const size_t q_target_dim = num_heads * head_dim; - - size_t q_gate = 0; - const size_t q_proj_dim = q_shape.size() > 1 ? q_shape[1] : 0; - if (q_proj_dim == 2 * q_target_dim) { - size_t q_packed = gb->reshape(q_proj, {batch_seq, num_heads, 2 * head_dim}); - q_gate = gb->slice(q_packed, 2, head_dim, head_dim); - q_proj = gb->slice(q_packed, 2, 0, head_dim); - q_gate = gb->reshape(q_gate, {batch_seq, q_target_dim}); - q_proj = gb->reshape(q_proj, {batch_seq, q_target_dim}); - } - - const size_t q_norm_weight = gb->scalar_add(layer.attn_q_norm_weight, 1.0f); - q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); - q_proj = gb->rms_norm(q_proj, q_norm_weight, config_.layer_norm_eps); - q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); - - size_t num_kv_heads = config_.attention_kv_heads; - const size_t k_norm_weight = gb->scalar_add(layer.attn_k_norm_weight, 1.0f); - k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); - k_proj = gb->rms_norm(k_proj, k_norm_weight, config_.layer_norm_eps); - k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); - - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - float rotary_factor = config_.partial_rotary_factor; - if (rotary_factor <= 0.0f) { - rotary_factor = 0.25f; - } - size_t rot_dim = static_cast(static_cast(head_dim) * rotary_factor); - rot_dim = std::min(rot_dim, head_dim); - if ((rot_dim & 1u) != 0) { - rot_dim -= 1; - } - - auto apply_partial_rope = [&](size_t x_4d) -> size_t { - if (rot_dim == 0) { - return x_4d; - } - if (rot_dim >= head_dim) { - return gb->rope(x_4d, config_.rope_theta, position_offset); - } - size_t x_rot = gb->slice(x_4d, 3, 0, rot_dim); - size_t x_tail = gb->slice(x_4d, 3, rot_dim, head_dim - rot_dim); - x_rot = gb->rope(x_rot, config_.rope_theta, position_offset); - return gb->concat(x_rot, x_tail, 3); - }; - - q_proj_4d = apply_partial_rope(q_proj_4d); - k_proj_4d = apply_partial_rope(k_proj_4d); - } - - size_t attn_output_4d; - - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - conv_cache_state_nodes_[layer_idx] = 0; - } - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - if (q_gate != 0) { - attn_output = gb->multiply(attn_output, gb->sigmoid(q_gate)); - } - return gb->matmul(attn_output, layer.attn_output_weight, true, backend); -} - -size_t Qwen3p5Model::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx].weights; - size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - size_t gate_silu = gb->silu(gate_output); - size_t gated = gb->multiply(gate_silu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - -size_t Qwen3p5Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx].weights; - const size_t input_norm_weight = gb->scalar_add(layer.input_layernorm_weight, 1.0f); - auto normalized_input = gb->rms_norm(hidden, input_norm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - auto after_attention = gb->add(hidden, attn_output); - const size_t post_attn_norm_weight = gb->scalar_add(layer.post_attention_layernorm_weight, 1.0f); - auto normalized_after_attention = gb->rms_norm(after_attention, post_attn_norm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); - return gb->add(after_attention, mlp_output); -} - -size_t Qwen3p5Model::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - if (conv_cache_state_nodes_.size() != config_.num_layers) { - conv_cache_state_nodes_.assign(config_.num_layers, 0); - } - std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); - last_forward_used_cache_ = use_cache; - if (!use_cache && conv_cache_.window_size > 0) { - conv_cache_.reset(); - deltanet_total_seq_len_ = 0; - } - - auto seq_len = static_cast(tokens.size()); - const size_t kv_pos = use_cache ? kv_cache_.get_total_seq_len() : 0; - const size_t position_offset = use_cache ? std::max(kv_pos, deltanet_total_seq_len_) : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - static std::set skip_layers = {}; - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - if (skip_layers.count(layer_idx)) { - continue; - } - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - const size_t final_norm_weight = gb->scalar_add(weight_nodes_.output_norm_weight, 1.0f); - auto final_hidden = gb->rms_norm(hidden, final_norm_weight, config_.layer_norm_eps); - return final_hidden; -} - -} -} diff --git a/cactus-transpile/models/model_siglip2.cpp b/cactus-transpile/models/model_siglip2.cpp deleted file mode 100644 index 49c3c24b0..000000000 --- a/cactus-transpile/models/model_siglip2.cpp +++ /dev/null @@ -1,387 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Siglip2VisionModel::Siglip2VisionModel() : Model() { - config_.model_type = Config::ModelType::SIGLIP2; -} - -Siglip2VisionModel::Siglip2VisionModel(const Config& cfg) : Model(cfg) { - Siglip2Preprocessor::Config preprocessor_config; - preprocessor_config.patch_size = static_cast(config_.vision_patch_size); - preprocessor_config.downsample_factor = static_cast(config_.downsample_factor); - preprocessor_config.min_tiles = static_cast(config_.min_tiles); - preprocessor_config.max_tiles = static_cast(config_.max_tiles); - preprocessor_config.use_thumbnail = config_.use_thumbnail; - preprocessor_config.min_image_tokens = static_cast(config_.min_image_tokens); - preprocessor_config.max_image_tokens = static_cast(config_.max_image_tokens); - preprocessor_config.max_num_patches = static_cast(config_.max_num_patches); - preprocessor_config.tile_size = static_cast(config_.tile_size); - preprocessor_config.max_pixels_tolerance = config_.max_pixels_tolerance; - preprocessor_config.do_resize = true; - preprocessor_config.do_rescale = true; - preprocessor_config.do_normalize = true; - preprocessor_config.do_convert_rgb = true; - preprocessor_config.do_image_splitting = config_.do_image_splitting; - preprocessor_config.rescale_factor = config_.rescale_factor; - preprocessor_config.image_mean[0] = config_.image_mean; - preprocessor_config.image_mean[1] = config_.image_mean; - preprocessor_config.image_mean[2] = config_.image_mean; - preprocessor_config.image_std[0] = config_.image_std; - preprocessor_config.image_std[1] = config_.image_std; - preprocessor_config.image_std[2] = config_.image_std; - - preprocessor_ = Siglip2Preprocessor(preprocessor_config); -} - -void Siglip2VisionModel::load_weights_to_graph(CactusGraph* gb) { - std::string base = model_folder_path_ + "/"; - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - - std::vector typical_input_shape = { - static_cast(config_.max_num_patches), - static_cast(config_.vision_embed_dim) - }; - npu_encoder_->preallocate(typical_input_shape, "x", ""); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - // Always load patch embedding and position embedding weights (needed for both CPU and NPU paths) - vision_weight_nodes_.patch_embedding_weight = gb->mmap_weights(base + "vision_patch_embedding.weights"); - vision_weight_nodes_.patch_embedding_bias = gb->mmap_weights(base + "vision_patch_embedding.bias.weights"); - vision_weight_nodes_.position_embedding = gb->mmap_weights(base + "vision_position_embedding.weights"); - - if (!use_npu_encoder_) { - vision_weight_nodes_.vision_layers.resize(config_.vision_num_layers); - - vision_weight_nodes_.post_layernorm_weight = gb->mmap_weights(base + "vision_post_layernorm.weights"); - vision_weight_nodes_.post_layernorm_bias = gb->mmap_weights(base + "vision_post_layernorm.bias.weights"); - - for (uint32_t i = 0; i < vision_weight_nodes_.vision_layers.size(); ++i) { - auto& layer = vision_weight_nodes_.vision_layers[i]; - std::string prefix = base + "vision_layer_" + std::to_string(i) + "_"; - - layer.attn_q_weight = gb->mmap_weights(prefix + "self_attn_q.weights"); - layer.attn_q_bias = gb->mmap_weights(prefix + "self_attn_q.bias.weights"); - layer.attn_k_weight = gb->mmap_weights(prefix + "self_attn_k.weights"); - layer.attn_k_bias = gb->mmap_weights(prefix + "self_attn_k.bias.weights"); - layer.attn_v_weight = gb->mmap_weights(prefix + "self_attn_v.weights"); - layer.attn_v_bias = gb->mmap_weights(prefix + "self_attn_v.bias.weights"); - layer.attn_output_weight = gb->mmap_weights(prefix + "self_attn_out.weights"); - layer.attn_output_bias = gb->mmap_weights(prefix + "self_attn_out.bias.weights"); - - layer.layer_norm1_weight = gb->mmap_weights(prefix + "layer_norm1.weights"); - layer.layer_norm1_bias = gb->mmap_weights(prefix + "layer_norm1.bias.weights"); - layer.layer_norm2_weight = gb->mmap_weights(prefix + "layer_norm2.weights"); - layer.layer_norm2_bias = gb->mmap_weights(prefix + "layer_norm2.bias.weights"); - - layer.mlp_fc1_weight = gb->mmap_weights(prefix + "ffn_fc1.weights"); - layer.mlp_fc1_bias = gb->mmap_weights(prefix + "ffn_fc1.bias.weights"); - layer.mlp_fc2_weight = gb->mmap_weights(prefix + "ffn_fc2.weights"); - layer.mlp_fc2_bias = gb->mmap_weights(prefix + "ffn_fc2.bias.weights"); - } - } -} - -Siglip2VisionModel::VisionEmbeddingResult Siglip2VisionModel::build_vision_embeddings( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - const int num_tiles = preprocessed_image.num_tiles; - const int max_patches = preprocessed_image.max_patches_per_tile; - const int patch_dim = preprocessed_image.patch_dim; - - const size_t expected_size = static_cast(num_tiles) * static_cast(max_patches) * - static_cast(patch_dim); - if (preprocessed_image.pixel_values.size() != expected_size) { - throw std::runtime_error( - "Pixel values size mismatch: expected " + std::to_string(expected_size) + - " (tiles=" + std::to_string(num_tiles) + " * max_patches=" + std::to_string(max_patches) + - " * patch_dim=" + std::to_string(patch_dim) + ") but got " + - std::to_string(preprocessed_image.pixel_values.size())); - } - for (size_t i = 0; i < std::min(100, preprocessed_image.pixel_values.size()); ++i) { - float val = preprocessed_image.pixel_values[i]; - if (std::isnan(val) || std::isinf(val)) { - throw std::runtime_error( - "Invalid value in pixel_values at index " + std::to_string(i) + ": " + std::to_string(val)); - } - } - - size_t reshaped_weight = gb->reshape( - vision_weight_nodes_.patch_embedding_weight, - {static_cast(config_.vision_embed_dim), static_cast(patch_dim)}); - - size_t patch_bias = vision_weight_nodes_.patch_embedding_bias; - std::vector tile_embeddings; - tile_embeddings.reserve(static_cast(num_tiles)); - - for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { - const auto& shape = preprocessed_image.spatial_shapes[tile_idx]; - const int tile_h = shape.first; - const int tile_w = shape.second; - const int actual_patches = tile_h * tile_w; - if (actual_patches <= 0) { - continue; - } - - const float* tile_data = preprocessed_image.pixel_values.data() + - static_cast(tile_idx) * static_cast(max_patches) * - static_cast(patch_dim); - - size_t tile_input_fp32 = gb->input( - {static_cast(actual_patches), static_cast(patch_dim)}, Precision::FP32); - gb->set_input(tile_input_fp32, tile_data, Precision::FP32); - size_t tile_input = gb->precision_cast(tile_input_fp32, Precision::FP16); - size_t tile_patch = gb->matmul(tile_input, reshaped_weight, true, backend); - size_t tile_bias = gb->add(tile_patch, patch_bias); - size_t tile_pos = gb->bilinear_interpolation( - vision_weight_nodes_.position_embedding, - static_cast(tile_h), - static_cast(tile_w), - /*align_corners=*/false); - size_t tile_pos_cast = gb->precision_cast(tile_pos, Precision::FP16); - size_t tile_embed = gb->add(tile_bias, tile_pos_cast); - tile_embeddings.push_back(tile_embed); - } - - if (tile_embeddings.empty()) { - throw std::runtime_error("No valid tiles produced embeddings in build_vision_embeddings"); - } - auto concat_nodes = [&](const std::vector& nodes) { - if (nodes.empty()) { - throw std::runtime_error("Attempted to concatenate an empty node list"); - } - size_t combined = nodes.front(); - for (size_t i = 1; i < nodes.size(); ++i) { - combined = gb->concat(combined, nodes[i], /*axis=*/0); - } - return combined; - }; - - size_t embeddings = concat_nodes(tile_embeddings); - return VisionEmbeddingResult{embeddings, std::move(tile_embeddings)}; -} - -size_t Siglip2VisionModel::build_vision_attention(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t q = gb->matmul(hidden_states, layer.attn_q_weight, true, backend); - q = gb->add(q, layer.attn_q_bias); - size_t k = gb->matmul(hidden_states, layer.attn_k_weight, true, backend); - k = gb->add(k, layer.attn_k_bias); - size_t v = gb->matmul(hidden_states, layer.attn_v_weight, true, backend); - v = gb->add(v, layer.attn_v_bias); - const size_t num_heads = static_cast(config_.vision_attention_heads); - const size_t head_dim = static_cast(config_.vision_embed_dim / config_.vision_attention_heads); - const auto& q_buf = gb->get_output_buffer(q); - size_t seq_len = q_buf.shape[0]; - - size_t q_4d = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - size_t k_4d = gb->reshape(k, {1, seq_len, num_heads, head_dim}); - size_t v_4d = gb->reshape(v, {1, seq_len, num_heads, head_dim}); - - float scale = 1.0f / std::sqrt(static_cast(head_dim)); - size_t attn_output = gb->attention(q_4d, k_4d, v_4d, scale, false, backend); - size_t attn_2d = gb->reshape(attn_output, {seq_len, num_heads * head_dim}); - size_t output = gb->matmul(attn_2d, layer.attn_output_weight, true, backend); - output = gb->add(output, layer.attn_output_bias); - return output; -} - -size_t Siglip2VisionModel::build_vision_mlp(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t fc1_output = gb->matmul(hidden_states, layer.mlp_fc1_weight, true, backend); - fc1_output = gb->add(fc1_output, layer.mlp_fc1_bias); - size_t activated = gb->gelu(fc1_output); - size_t fc2_output = gb->matmul(activated, layer.mlp_fc2_weight, true, backend); - fc2_output = gb->add(fc2_output, layer.mlp_fc2_bias); - return fc2_output; -} - -size_t Siglip2VisionModel::build_vision_transformer_layer(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t residual = hidden_states; - size_t normalized = gb->layernorm(hidden_states, layer.layer_norm1_weight, - layer.layer_norm1_bias, config_.layer_norm_eps); - size_t attn_output = build_vision_attention(gb, normalized, layer_idx, backend); - hidden_states = gb->add(residual, attn_output); - residual = hidden_states; - normalized = gb->layernorm(hidden_states, layer.layer_norm2_weight, - layer.layer_norm2_bias, config_.layer_norm_eps); - size_t mlp_output = build_vision_mlp(gb, normalized, layer_idx, backend); - hidden_states = gb->add(residual, mlp_output); - return hidden_states; -} - -size_t Siglip2VisionModel::forward_vision( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - // NPU path: build patch embeddings + position embeddings on CPU, then run transformer on NPU - auto embedding_result = build_vision_embeddings(gb, preprocessed_image, backend); - - size_t total_patches = 0; - for (const auto& shape : preprocessed_image.spatial_shapes) { - total_patches += shape.first * shape.second; - } - - gb->execute(); - - const auto& embed_buffer = gb->get_output_buffer(embedding_result.combined_embeddings); - void* embed_ptr = gb->get_output(embedding_result.combined_embeddings); - - std::vector<__fp16> embed_f16; - const __fp16* input_ptr; - if (embed_buffer.precision == Precision::FP16) { - input_ptr = static_cast(embed_ptr); - } else { - embed_f16.resize(total_patches * config_.vision_embed_dim); - cactus_fp32_to_fp16(static_cast(embed_ptr), - embed_f16.data(), embed_f16.size()); - input_ptr = embed_f16.data(); - } - - std::vector input_shape = { - static_cast(total_patches), - static_cast(config_.vision_embed_dim) - }; - - __fp16* output_buffer = npu_encoder_->get_output_buffer(); - if (output_buffer) { - size_t elements = npu_encoder_->encode( - input_ptr, output_buffer, input_shape, "x", ""); - - if (elements > 0) { - gb->soft_reset(); - size_t vision_output = gb->input({total_patches, config_.vision_embed_dim}, - Precision::FP16); - gb->set_external_input(vision_output, output_buffer, Precision::FP16); - return vision_output; - } - } else { - std::vector<__fp16> npu_output(total_patches * config_.vision_embed_dim); - size_t elements = npu_encoder_->encode( - input_ptr, npu_output.data(), input_shape, "x", ""); - - if (elements > 0) { - gb->soft_reset(); - size_t vision_output = gb->input({total_patches, config_.vision_embed_dim}, - Precision::FP16); - gb->set_input(vision_output, npu_output.data(), Precision::FP16); - return vision_output; - } - } - - throw std::runtime_error("NPU encoder failed"); - } - - // CPU path: full forward pass through transformer layers - auto embedding_result = build_vision_embeddings(gb, preprocessed_image, backend); - - auto concat_nodes = [&](const std::vector& nodes) { - if (nodes.empty()) { - throw std::runtime_error("Attempted to concatenate an empty node list in forward_vision"); - } - size_t combined = nodes.front(); - for (size_t i = 1; i < nodes.size(); ++i) { - combined = gb->concat(combined, nodes[i], /*axis=*/0); - } - return combined; - }; - - std::vector tile_outputs; - tile_outputs.reserve(embedding_result.tile_embeddings.size()); - - for (size_t tile_idx = 0; tile_idx < embedding_result.tile_embeddings.size(); ++tile_idx) { - size_t hidden_states = embedding_result.tile_embeddings[tile_idx]; - - for (uint32_t layer_idx = 0; layer_idx < config_.vision_num_layers; ++layer_idx) { - hidden_states = build_vision_transformer_layer(gb, hidden_states, layer_idx, backend); - } - - hidden_states = gb->layernorm(hidden_states, - vision_weight_nodes_.post_layernorm_weight, - vision_weight_nodes_.post_layernorm_bias, - config_.layer_norm_eps); - tile_outputs.push_back(hidden_states); - } - - if (tile_outputs.empty()) { - throw std::runtime_error("No tile outputs generated in forward_vision"); - } - - size_t combined_output = concat_nodes(tile_outputs); - - return combined_output; -} - -size_t Siglip2VisionModel::forward_vision(const Siglip2Preprocessor::PreprocessedImage& preprocessed_image) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - return forward_vision(gb, preprocessed_image, backend); -} - -std::vector Siglip2VisionModel::get_image_embedding(const std::string& image_path) { - auto preprocessed = preprocessor_.preprocess_from_file(image_path); - size_t last_hidden_state = forward_vision(preprocessed); - - auto* gb = static_cast(graph_handle_); - size_t pooled = gb->mean(last_hidden_state, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - size_t hidden_dim = output_buf.total_size; - - std::vector embedding(hidden_dim); - void* output_data = gb->get_output(pooled); - const float* output_ptr = static_cast(output_data); - std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); - return embedding; -} - -size_t Siglip2VisionModel::forward(const std::vector&, bool) {return 0;} - -size_t Siglip2VisionModel::build_attention(CactusGraph*, size_t, uint32_t, - ComputeBackend, bool, size_t) { - return 0; -} - -size_t Siglip2VisionModel::build_mlp(CactusGraph*, size_t, uint32_t, - ComputeBackend) const { - return 0; -} - -size_t Siglip2VisionModel::build_transformer_block(CactusGraph*, size_t, uint32_t, - ComputeBackend, bool, size_t) { - return 0; -} - -} -} diff --git a/cactus-transpile/models/model_silero_vad.cpp b/cactus-transpile/models/model_silero_vad.cpp deleted file mode 100644 index 7573ac2de..000000000 --- a/cactus-transpile/models/model_silero_vad.cpp +++ /dev/null @@ -1,338 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include -#include - -namespace cactus { -namespace engine { - -SileroVADModel::SileroVADModel() : Model() { - weight_nodes_.encoder_blocks.resize(4); -} - -SileroVADModel::SileroVADModel(const Config& config) : Model(config) { - weight_nodes_.encoder_blocks.resize(4); -} - -SileroVADModel::~SileroVADModel() = default; - -void SileroVADModel::load_weights_to_graph(CactusGraph* gb) { - weight_nodes_.stft_basis = gb->mmap_weights(model_folder_path_ + "/stft_basis.weights"); - - for (uint32_t i = 0; i < 4; i++) { - std::string block_prefix = model_folder_path_ + "/encoder_block_" + std::to_string(i) + "_"; - weight_nodes_.encoder_blocks[i].conv_weight = gb->mmap_weights(block_prefix + "conv_weight.weights"); - weight_nodes_.encoder_blocks[i].conv_bias = gb->mmap_weights(block_prefix + "conv_bias.weights"); - } - - weight_nodes_.lstm_weight_ih = gb->mmap_weights(model_folder_path_ + "/lstm_weight_ih.weights"); - weight_nodes_.lstm_weight_hh = gb->mmap_weights(model_folder_path_ + "/lstm_weight_hh.weights"); - weight_nodes_.lstm_bias_ih = gb->mmap_weights(model_folder_path_ + "/lstm_bias_ih.weights"); - weight_nodes_.lstm_bias_hh = gb->mmap_weights(model_folder_path_ + "/lstm_bias_hh.weights"); - - weight_nodes_.output_conv_weight = gb->mmap_weights(model_folder_path_ + "/output_conv_weight.weights"); - weight_nodes_.output_conv_bias = gb->mmap_weights(model_folder_path_ + "/output_conv_bias.weights"); -} - -void SileroVADModel::build_graph() { - const size_t input_size = CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE; - - graph_nodes_.input = graph_.input({1, 1, input_size}, Precision::FP16); - graph_nodes_.h_prev = graph_.input({1, HIDDEN_SIZE}, Precision::FP16); - graph_nodes_.c_prev = graph_.input({1, HIDDEN_SIZE}, Precision::FP16); - - auto cplx = graph_.stft(graph_nodes_.input, weight_nodes_.stft_basis, 128, 129); - auto re = graph_.slice(cplx, 1, 0, 129); - auto im = graph_.slice(cplx, 1, 129, 129); - auto re_sq = graph_.multiply(re, re); - auto im_sq = graph_.multiply(im, im); - auto mag = graph_.scalar_sqrt(graph_.add(re_sq, im_sq)); - - const size_t strides[4] = {1, 2, 2, 1}; - auto x = mag; - for (uint32_t i = 0; i < 4; i++) { - auto conv = graph_.conv1d_k3(x, weight_nodes_.encoder_blocks[i].conv_weight, strides[i]); - auto bias_reshaped = graph_.reshape(weight_nodes_.encoder_blocks[i].conv_bias, - {1, graph_.get_output_buffer(weight_nodes_.encoder_blocks[i].conv_bias).shape[0], 1}); - auto with_bias = graph_.add(conv, bias_reshaped); - x = graph_.relu(with_bias); - } - - auto x_squeezed = graph_.reshape(x, {1, HIDDEN_SIZE}); - - graph_nodes_.lstm_output = graph_.lstm_cell( - x_squeezed, - graph_nodes_.h_prev, - graph_nodes_.c_prev, - weight_nodes_.lstm_weight_ih, - weight_nodes_.lstm_weight_hh, - weight_nodes_.lstm_bias_ih, - weight_nodes_.lstm_bias_hh - ); - - auto h_new = graph_.slice(graph_nodes_.lstm_output, 2, 0, 1); - auto c_new = graph_.slice(graph_nodes_.lstm_output, 2, 1, 1); - graph_nodes_.h_new = graph_.reshape(h_new, {1, HIDDEN_SIZE}); - graph_nodes_.c_new = graph_.reshape(c_new, {1, HIDDEN_SIZE}); - - auto h_relu = graph_.relu(graph_nodes_.h_new); - auto h_unsqueezed = graph_.reshape(h_relu, {1, HIDDEN_SIZE, 1}); - auto logits = graph_.conv1d(h_unsqueezed, weight_nodes_.output_conv_weight, weight_nodes_.output_conv_bias, 1); - graph_nodes_.output = graph_.sigmoid(logits); -} - -bool SileroVADModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - (void)context_size; - (void)system_prompt; - (void)do_warmup; - - if (initialized_) { - return true; - } - - state_.h.resize(HIDDEN_SIZE, (__fp16)0.0f); - state_.c.resize(HIDDEN_SIZE, (__fp16)0.0f); - state_.context.resize(CONTEXT_SIZE, 0.0f); - state_.input_buf.resize(CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE, 0.0f); - state_.input_fp16.resize(CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE); - - model_folder_path_ = model_folder; - - try { - load_weights_to_graph(&graph_); - build_graph(); - initialized_ = true; - return true; - } catch (const std::exception& e) { - initialized_ = false; - return false; - } -} - -float SileroVADModel::process_chunk(const std::vector& audio_chunk) { - const float* audio = audio_chunk.data(); - size_t samples = audio_chunk.size(); - - if (samples != CHUNK_SIZE) { - throw std::runtime_error("SileroVAD: Expected 512 samples"); - } - if (!initialized_) { - throw std::runtime_error("SileroVAD: Not initialized"); - } - - const size_t input_size = CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE; - - float* input_buf = state_.input_buf.data(); - __fp16* input_fp16 = state_.input_fp16.data(); - - std::memcpy(input_buf, state_.context.data(), CONTEXT_SIZE * sizeof(float)); - std::memcpy(input_buf + CONTEXT_SIZE, audio, CHUNK_SIZE * sizeof(float)); - - for (size_t i = 0; i < REFLECT_PAD_SIZE; i++) { - input_buf[CONTEXT_SIZE + CHUNK_SIZE + i] = - input_buf[CONTEXT_SIZE + CHUNK_SIZE - 2 - i]; - } - - cactus_fp32_to_fp16(input_buf, input_fp16, input_size); - - graph_.set_input(graph_nodes_.input, input_fp16, Precision::FP16); - graph_.set_input(graph_nodes_.h_prev, state_.h.data(), Precision::FP16); - graph_.set_input(graph_nodes_.c_prev, state_.c.data(), Precision::FP16); - graph_.execute(); - - void* h_out = graph_.get_output(graph_nodes_.h_new); - void* c_out = graph_.get_output(graph_nodes_.c_new); - void* vad_out = graph_.get_output(graph_nodes_.output); - - const __fp16* h_new = static_cast(h_out); - const __fp16* c_new = static_cast(c_out); - const __fp16* vad_score = static_cast(vad_out); - - std::memcpy(state_.h.data(), h_new, HIDDEN_SIZE * sizeof(__fp16)); - std::memcpy(state_.c.data(), c_new, HIDDEN_SIZE * sizeof(__fp16)); - - float vad_score_f32 = static_cast(vad_score[0]); - - std::memcpy(state_.context.data(), input_buf + CHUNK_SIZE, CONTEXT_SIZE * sizeof(float)); - - return vad_score_f32; -} - -void SileroVADModel::reset_states() { - state_.h.assign(HIDDEN_SIZE, (__fp16)0.0f); - state_.c.assign(HIDDEN_SIZE, (__fp16)0.0f); - state_.context.assign(CONTEXT_SIZE, 0.0f); -} - -std::vector SileroVADModel::get_speech_timestamps(const std::vector& audio, const SpeechTimestampsOptions& options) { - const size_t audio_length_samples = audio.size(); - const size_t window_size_samples = options.window_size_samples; - const float min_speech_samples = options.sampling_rate * options.min_speech_duration_ms / 1000.0f; - const float speech_pad_samples = options.sampling_rate * options.speech_pad_ms / 1000.0f; - const float max_speech_samples = options.sampling_rate * options.max_speech_duration_s - window_size_samples - 2 * speech_pad_samples; - const float min_silence_samples = options.sampling_rate * options.min_silence_duration_ms / 1000.0f; - const float neg_threshold = (options.neg_threshold == 0.0f) - ? std::max(options.threshold - 0.15f, 0.01f) - : options.neg_threshold; - reset_states(); - - std::vector speech_probs; - speech_probs.reserve(audio_length_samples / window_size_samples + 1); - - std::vector chunk(window_size_samples, 0.0f); - - for (size_t current_start = 0; current_start < audio_length_samples; current_start += window_size_samples) { - size_t remaining = audio_length_samples - current_start; - size_t copy_len = std::min(remaining, window_size_samples); - - std::memcpy(chunk.data(), audio.data() + current_start, copy_len * sizeof(float)); - - if (copy_len < window_size_samples) { - std::memset(chunk.data() + copy_len, 0, (window_size_samples - copy_len) * sizeof(float)); - } - - float speech_prob = process_chunk(chunk); - speech_probs.push_back(speech_prob); - } - - bool triggered = false; - std::vector speeches; - SpeechTimestamp current_speech = {0, 0}; - size_t temp_end = 0; - size_t prev_end = 0; - size_t next_start = 0; - std::vector> possible_ends; - - const float min_silence_samples_at_max_speech = options.sampling_rate * options.min_silence_at_max_speech / 1000.0f; - - for (size_t i = 0; i < speech_probs.size(); ++i) { - float speech_prob = speech_probs[i]; - size_t cur_sample = window_size_samples * i; - - if (speech_prob >= options.threshold && temp_end) { - size_t sil_dur = cur_sample - temp_end; - if (sil_dur > min_silence_samples_at_max_speech) { - possible_ends.push_back({temp_end, sil_dur}); - } - temp_end = 0; - if (next_start < prev_end) { - next_start = cur_sample; - } - } - - if (speech_prob >= options.threshold && !triggered) { - triggered = true; - current_speech.start = cur_sample; - continue; - } - - if (triggered && (cur_sample - current_speech.start > max_speech_samples)) { - if (options.use_max_poss_sil_at_max_speech && !possible_ends.empty()) { - auto max_silence = std::max_element(possible_ends.begin(), possible_ends.end(), - [](const std::pair& a, const std::pair& b) { - return a.second < b.second; - }); - prev_end = max_silence->first; - size_t dur = max_silence->second; - current_speech.end = prev_end; - speeches.push_back(current_speech); - current_speech = {0, 0}; - next_start = prev_end + dur; - - if (next_start < prev_end + cur_sample) { - current_speech.start = next_start; - } else { - triggered = false; - } - prev_end = next_start = temp_end = 0; - possible_ends.clear(); - } else { - if (prev_end) { - current_speech.end = prev_end; - speeches.push_back(current_speech); - current_speech = {0, 0}; - if (next_start < prev_end) { - triggered = false; - } else { - current_speech.start = next_start; - } - prev_end = next_start = temp_end = 0; - possible_ends.clear(); - } else { - current_speech.end = cur_sample; - speeches.push_back(current_speech); - current_speech = {0, 0}; - prev_end = next_start = temp_end = 0; - triggered = false; - possible_ends.clear(); - continue; - } - } - } - - if (speech_prob < neg_threshold && triggered) { - if (!temp_end) { - temp_end = cur_sample; - } - size_t sil_dur_now = cur_sample - temp_end; - - if (!options.use_max_poss_sil_at_max_speech && sil_dur_now > min_silence_samples_at_max_speech) { - prev_end = temp_end; - } - - if (sil_dur_now < min_silence_samples) { - continue; - } else { - current_speech.end = temp_end; - if ((current_speech.end - current_speech.start) > min_speech_samples) { - speeches.push_back(current_speech); - } - current_speech = {0, 0}; - prev_end = next_start = temp_end = 0; - triggered = false; - possible_ends.clear(); - continue; - } - } - } - - if (triggered && (audio_length_samples - current_speech.start) > min_speech_samples) { - current_speech.end = audio_length_samples; - speeches.push_back(current_speech); - } - - for (size_t i = 0; i < speeches.size(); ++i) { - if (i == 0) { - speeches[i].start = speeches[i].start > static_cast(speech_pad_samples) - ? speeches[i].start - static_cast(speech_pad_samples) - : 0; - } - if (i != speeches.size() - 1) { - size_t silence_duration = speeches[i + 1].start - speeches[i].end; - if (silence_duration < 2 * static_cast(speech_pad_samples)) { - speeches[i].end += silence_duration / 2; - speeches[i + 1].start = speeches[i + 1].start > silence_duration / 2 - ? speeches[i + 1].start - silence_duration / 2 - : 0; - } else { - speeches[i].end = std::min(audio_length_samples, speeches[i].end + static_cast(speech_pad_samples)); - speeches[i + 1].start = speeches[i + 1].start > static_cast(speech_pad_samples) - ? speeches[i + 1].start - static_cast(speech_pad_samples) - : 0; - } - } else { - speeches[i].end = std::min(audio_length_samples, speeches[i].end + static_cast(speech_pad_samples)); - } - } - - return speeches; -} - -size_t SileroVADModel::forward(const std::vector&, const std::vector&, bool) { - return 0; -} - -} -} diff --git a/cactus-transpile/models/model_wespeaker.cpp b/cactus-transpile/models/model_wespeaker.cpp deleted file mode 100644 index 62d4642c8..000000000 --- a/cactus-transpile/models/model_wespeaker.cpp +++ /dev/null @@ -1,204 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include - -namespace cactus { -namespace engine { - -WeSpeakerModel::WeSpeakerModel() : Model() {} -WeSpeakerModel::WeSpeakerModel(const Config& config) : Model(config) {} - -WeSpeakerModel::ResBlockWeights WeSpeakerModel::load_resblock(CactusGraph* gb, const std::string& prefix, bool has_shortcut) { - ResBlockWeights rb; - rb.has_shortcut = has_shortcut; - rb.conv1_w = gb->mmap_weights(prefix + "_conv1_weight.weights"); - rb.conv1_b = gb->mmap_weights(prefix + "_conv1_bias.weights"); - rb.conv2_w = gb->mmap_weights(prefix + "_conv2_weight.weights"); - rb.conv2_b = gb->mmap_weights(prefix + "_conv2_bias.weights"); - if (has_shortcut) { - rb.shortcut_conv_w = gb->mmap_weights(prefix + "_shortcut_0_weight.weights"); - rb.shortcut_conv_b = gb->mmap_weights(prefix + "_shortcut_0_bias.weights"); - } - return rb; -} - -size_t WeSpeakerModel::build_resblock(CactusGraph* gb, size_t x, const ResBlockWeights& rb, bool stride2) { - size_t identity = x; - - size_t out = stride2 ? gb->conv2d_k3s2p1(x, rb.conv1_w, rb.conv1_b) - : gb->conv2d_k3s1p1(x, rb.conv1_w, rb.conv1_b); - out = gb->relu(out); - out = gb->conv2d_k3s1p1(out, rb.conv2_w, rb.conv2_b); - - if (rb.has_shortcut) { - identity = gb->conv2d_k3s2p1(x, rb.shortcut_conv_w, rb.shortcut_conv_b); - } - - out = gb->add(out, identity); - out = gb->relu(out); - return out; -} - -void WeSpeakerModel::load_weights_to_graph(CactusGraph* gb) { - const std::string& p = model_folder_path_; - - weight_nodes_.conv1_w = gb->mmap_weights(p + "/resnet_conv1_weight.weights"); - weight_nodes_.conv1_b = gb->mmap_weights(p + "/resnet_conv1_bias.weights"); - - auto load_layer = [&](const std::string& layer_name, int num_blocks, bool first_has_shortcut) { - std::vector blocks; - for (int i = 0; i < num_blocks; ++i) { - std::string prefix = p + "/resnet_" + layer_name + "_" + std::to_string(i); - blocks.push_back(load_resblock(gb, prefix, i == 0 && first_has_shortcut)); - } - return blocks; - }; - - weight_nodes_.layer1 = load_layer("layer1", 3, false); - weight_nodes_.layer2 = load_layer("layer2", 4, true); - weight_nodes_.layer3 = load_layer("layer3", 6, true); - weight_nodes_.layer4 = load_layer("layer4", 3, true); - - weight_nodes_.seg1_w = gb->mmap_weights(p + "/resnet_seg_1_weight.weights"); - weight_nodes_.seg1_b = gb->mmap_weights(p + "/resnet_seg_1_bias.weights"); -} - -size_t WeSpeakerModel::build_resnet_body(CactusGraph* gb, size_t x) { - x = gb->conv2d_k3s1p1(x, weight_nodes_.conv1_w, weight_nodes_.conv1_b); - x = gb->relu(x); - - for (auto& rb : weight_nodes_.layer1) x = build_resblock(gb, x, rb, false); - for (size_t i = 0; i < weight_nodes_.layer2.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer2[i], i == 0); - for (size_t i = 0; i < weight_nodes_.layer3.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer3[i], i == 0); - for (size_t i = 0; i < weight_nodes_.layer4.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer4[i], i == 0); - return x; -} - -void WeSpeakerModel::build_graph(size_t num_frames) { - audio_input_ = graph_.input({1, 1, 80, num_frames}, Precision::FP16); - - size_t x = build_resnet_body(&graph_, audio_input_); - - x = graph_.stats_pool(x); - x = graph_.add(graph_.matmul(x, weight_nodes_.seg1_w, true), weight_nodes_.seg1_b); - - output_node_ = x; -} - -void WeSpeakerModel::build_graph_masked(size_t num_frames) { - audio_input_ = graph_.input({1, 1, 80, num_frames}, Precision::FP16); - - size_t x = build_resnet_body(&graph_, audio_input_); - - const auto& post_resnet = graph_.get_output_buffer(x); - size_t T_post = post_resnet.shape.back(); - - mask_input_ = graph_.input({1, T_post}, Precision::FP32); - - x = graph_.weighted_stats_pool(x, mask_input_); - x = graph_.add(graph_.matmul(x, weight_nodes_.seg1_w, true), weight_nodes_.seg1_b); - - output_node_ = x; -} - -bool WeSpeakerModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - (void)context_size; - (void)system_prompt; - (void)do_warmup; - - if (initialized_) { - return true; - } - - model_folder_path_ = model_folder; - - try { - load_weights_to_graph(&graph_); - initialized_ = true; - return true; - } catch (const std::exception& e) { - initialized_ = false; - return false; - } -} - -std::vector WeSpeakerModel::embed(const float* fbank_features, size_t num_features) { - if (!initialized_) throw std::runtime_error("WeSpeaker model not initialized"); - - static constexpr size_t NUM_MEL = 80; - const size_t num_frames = num_features / NUM_MEL; - if (num_frames == 0) throw std::runtime_error("WeSpeaker: empty fbank input"); - - if (num_frames != current_num_frames_ || current_masked_) { - graph_.hard_reset(); - load_weights_to_graph(&graph_); - build_graph(num_frames); - current_num_frames_ = num_frames; - current_masked_ = false; - } - - input_buf_.resize(num_features); - for (size_t i = 0; i < num_features; ++i) - input_buf_[i] = static_cast<__fp16>(fbank_features[i]); - - graph_.set_input(audio_input_, input_buf_.data(), Precision::FP16); - graph_.execute(); - - const auto& out_buf = graph_.get_output_buffer(output_node_); - const __fp16* out_data = out_buf.data_as<__fp16>(); - size_t total = out_buf.total_size; - - std::vector result(total); - for (size_t i = 0; i < total; ++i) - result[i] = static_cast(out_data[i]); - return result; -} - -std::vector WeSpeakerModel::embed(const float* fbank_features, size_t num_features, - const float* mask_weights, size_t mask_num_frames) { - if (!initialized_) throw std::runtime_error("WeSpeaker model not initialized"); - - static constexpr size_t NUM_MEL = 80; - const size_t num_frames = num_features / NUM_MEL; - if (num_frames == 0) throw std::runtime_error("WeSpeaker: empty fbank input"); - - if (num_frames != current_num_frames_ || !current_masked_) { - graph_.hard_reset(); - load_weights_to_graph(&graph_); - build_graph_masked(num_frames); - current_num_frames_ = num_frames; - current_masked_ = true; - } - - input_buf_.resize(num_features); - for (size_t i = 0; i < num_features; ++i) - input_buf_[i] = static_cast<__fp16>(fbank_features[i]); - graph_.set_input(audio_input_, input_buf_.data(), Precision::FP16); - - const auto& mask_shape = graph_.get_output_buffer(mask_input_); - size_t T_post = mask_shape.shape.back(); - - mask_buf_.resize(T_post); - for (size_t t = 0; t < T_post; ++t) { - size_t src_t = (t * mask_num_frames) / T_post; - if (src_t >= mask_num_frames) src_t = mask_num_frames - 1; - mask_buf_[t] = mask_weights[src_t]; - } - graph_.set_input(mask_input_, mask_buf_.data(), Precision::FP32); - - graph_.execute(); - - const auto& out_buf = graph_.get_output_buffer(output_node_); - const __fp16* out_data = out_buf.data_as<__fp16>(); - size_t total = out_buf.total_size; - - std::vector result(total); - for (size_t i = 0; i < total; ++i) - result[i] = static_cast(out_data[i]); - return result; -} - -} -} diff --git a/cactus-transpile/models/model_whisper.cpp b/cactus-transpile/models/model_whisper.cpp deleted file mode 100644 index 70ea31c92..000000000 --- a/cactus-transpile/models/model_whisper.cpp +++ /dev/null @@ -1,778 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include -#include -#include - -namespace { - -bool has_whisper_cpu_encoder_weights(const std::string& model_folder_path) { - const std::filesystem::path model_path(model_folder_path); - return std::filesystem::exists(model_path / "encoder_conv1_weight.weights") && - std::filesystem::exists(model_path / "encoder_position_embeddings.weights"); -} - -} // namespace - -namespace cactus { -namespace engine { - -WhisperModel::WhisperModel() : Model() {} - -WhisperModel::WhisperModel(const Config& config) : Model(config) { - enc_layers_ = config.num_encoder_layers > 0 ? config.num_encoder_layers : config.num_layers; - dec_layers_ = config.num_decoder_layers > 0 ? config.num_decoder_layers : config.num_layers; - - weight_nodes_.layers.resize(std::max(enc_layers_, dec_layers_)); - - float hd = static_cast(config.attention_head_dim); - if (hd <= 0.0f) { - hd = 64.0f; - } - - attention_scale_ = 1.0f / std::sqrt(hd); - - encoder_block_out_nodes_.resize(enc_layers_, 0); - encoder_k_persistent_.assign(enc_layers_, 0); - encoder_v_persistent_.assign(enc_layers_, 0); - - const float neg_inf = -std::numeric_limits::infinity(); - for (size_t tok : suppress_tokens_) { - suppress_bias_[static_cast(tok)] = neg_inf; - } - suppress_bias_first_step_ = suppress_bias_; - for (size_t tok : begin_suppress_tokens_) { - suppress_bias_first_step_[static_cast(tok)] = neg_inf; - } -} - -void WhisperModel::load_weights_to_graph(CactusGraph* gb) { - - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - - weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/decoder_norm.weights"); - weight_nodes_.decoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/decoder_norm.bias"); - weight_nodes_.decoder_position_embeddings_weight = gb->mmap_weights(model_folder_path_ + "/decoder_position_embeddings.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - - std::vector typical_input_shape = {1, static_cast(config_.num_mel_bins), 3000}; - npu_encoder_->preallocate(typical_input_shape, "x", ""); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - const bool has_cpu_encoder_weights = has_whisper_cpu_encoder_weights(model_folder_path_); - if (has_cpu_encoder_weights) { - weight_nodes_.encoder_position_embeddings = gb->mmap_weights(model_folder_path_ + "/encoder_position_embeddings.weights"); - weight_nodes_.encoder_conv1_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_weight.weights"); - weight_nodes_.encoder_conv1_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_bias.bias"); - weight_nodes_.encoder_conv2_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_weight.weights"); - weight_nodes_.encoder_conv2_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_bias.bias"); - weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_norm_weight.weights"); - weight_nodes_.encoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/encoder_norm_bias.bias"); - } - - const uint32_t max_layers = std::max(enc_layers_, dec_layers_); - for (uint32_t i = 0; i < max_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - - if (i < dec_layers_) { - // Decoder Layers - std::string layer_prefix = model_folder_path_ + "/decoder.layer_" + std::to_string(i) + "_"; - - layer.decoder_encoder_attn_k_weight = gb->mmap_weights(layer_prefix + "encoder_attn_k.weights"); - layer.decoder_encoder_attn_q_weight = gb->mmap_weights(layer_prefix + "encoder_attn_q.weights"); - layer.decoder_encoder_attn_v_weight = gb->mmap_weights(layer_prefix + "encoder_attn_v.weights"); - layer.decoder_encoder_attn_output_weight = gb->mmap_weights(layer_prefix + "encoder_attn_output.weights"); - layer.decoder_encoder_attn_q_bias = gb->mmap_weights(layer_prefix + "encoder_attn_q.bias"); - layer.decoder_encoder_attn_v_bias = gb->mmap_weights(layer_prefix + "encoder_attn_v.bias"); - layer.decoder_encoder_attn_output_bias = gb->mmap_weights(layer_prefix + "encoder_attn_output.bias"); - - layer.decoder_post_encoder_layernorm_weight = gb->mmap_weights(layer_prefix + "encoder_attn_norm.weights"); - layer.decoder_post_encoder_layernorm_bias = gb->mmap_weights(layer_prefix + "encoder_attn_norm.bias"); - - layer.decoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.decoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - layer.decoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.decoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - - layer.decoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); - layer.decoder_post_ffn_layernorm_bias = gb->mmap_weights(layer_prefix + "final_norm.bias"); - - layer.decoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); - layer.decoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); - layer.decoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); - layer.decoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); - layer.decoder_self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); - layer.decoder_self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); - layer.decoder_self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); - - layer.decoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "self_attn_norm.weights"); - layer.decoder_post_attn_layernorm_bias = gb->mmap_weights(layer_prefix + "self_attn_norm.bias"); - } - - if (has_cpu_encoder_weights && i < enc_layers_) { - std::string layer_prefix = model_folder_path_ + "/encoder.layer_" + std::to_string(i) + "_"; - - layer.encoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.encoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - layer.encoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.encoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - - layer.encoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); - layer.encoder_post_ffn_layernorm_bias = gb->mmap_weights(layer_prefix + "final_norm.bias"); - - layer.encoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); - layer.encoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); - layer.encoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); - layer.encoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); - layer.encoder_self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); - layer.encoder_self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); - layer.encoder_self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); - - layer.encoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "self_attn_norm.weights"); - layer.encoder_post_attn_layernorm_bias = gb->mmap_weights(layer_prefix + "self_attn_norm.bias"); - } - } -} - -size_t WhisperModel::build_encoder_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto ffn1_weight = gb->matmul(input, layer.encoder_ffn1_weight, true, backend); - auto ffn1_bias = gb->add(ffn1_weight, layer.encoder_ffn1_bias); - - encoder_pre_gelu = ffn1_bias; - - auto ffn1_act = gb->gelu_erf(ffn1_bias); - - encoder_post_gelu = ffn1_act; - - auto ffn2_weight = gb->matmul(ffn1_act, layer.encoder_ffn2_weight, true, backend); - auto ffn2_bias = gb->add(ffn2_weight, layer.encoder_ffn2_bias); - return ffn2_bias; -} - -size_t WhisperModel::build_decoder_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto ffn1_weight = gb->matmul(input, layer.decoder_ffn1_weight, true, backend); - auto ffn1_bias = gb->add(ffn1_weight, layer.decoder_ffn1_bias); - auto ffn1_act = gb->gelu_erf(ffn1_bias); - auto ffn2_weight = gb->matmul(ffn1_act, layer.decoder_ffn2_weight, true, backend); - auto ffn2_bias = gb->add(ffn2_weight, layer.decoder_ffn2_bias); - return ffn2_bias; -} - - - -size_t WhisperModel::build_encoder_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool /*use_cache*/, size_t /*position_offset*/){ - - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t q = gb->matmul(input, layer.decoder_encoder_attn_q_weight, true, backend); - q = gb->add(q, layer.decoder_encoder_attn_q_bias); - - const auto& q_buf = gb->get_output_buffer(q); - if (q_buf.shape.size() != 2) { - throw std::runtime_error("encoder cross-attn: q must be [T_dec, D]"); - } - - size_t T_dec = q_buf.shape[0]; - size_t q_heads = config_.attention_heads; - size_t kv_heads = config_.attention_kv_heads; - size_t head_dim = config_.attention_head_dim; - - q = gb->reshape(q, {1, T_dec, q_heads, head_dim}); - - size_t k_4d = 0; - size_t v_4d = 0; - - bool is_populated = (encoder_k_persistent_[layer_idx] != 0 && - gb->is_populated(encoder_k_persistent_[layer_idx])); - - if (is_populated) { - k_4d = encoder_k_persistent_[layer_idx]; - v_4d = encoder_v_persistent_[layer_idx]; - } else { - size_t enc_norm = last_encoder_post_norm_node_; - - size_t k = gb->matmul(enc_norm, layer.decoder_encoder_attn_k_weight, true, backend); - size_t v = gb->matmul(enc_norm, layer.decoder_encoder_attn_v_weight, true, backend); - v = gb->add(v, layer.decoder_encoder_attn_v_bias); - - const auto& k_buf = gb->get_output_buffer(k); - if (k_buf.shape.size() != 2) { - throw std::runtime_error("encoder cross-attn: k must be [T_enc, D]"); - } - size_t T_enc = k_buf.shape[0]; - - k_4d = gb->reshape(k, {1, T_enc, kv_heads, head_dim}); - v_4d = gb->reshape(v, {1, T_enc, kv_heads, head_dim}); - - if (encoder_k_persistent_[layer_idx] == 0) { - encoder_k_persistent_[layer_idx] = gb->persistent(k_4d); - encoder_v_persistent_[layer_idx] = gb->persistent(v_4d); - } - k_4d = encoder_k_persistent_[layer_idx]; - v_4d = encoder_v_persistent_[layer_idx]; - } - - size_t attn = gb->attention(q, k_4d, v_4d, attention_scale_, false); - - attn = gb->reshape(attn, {T_dec, q_heads * head_dim}); - size_t out = gb->matmul(attn, layer.decoder_encoder_attn_output_weight, true, backend); - out = gb->add(out, layer.decoder_encoder_attn_output_bias); - - return out; -} - -void WhisperModel::reset_graph_side_cache_nodes() { - cache_k_output_nodes_.assign(dec_layers_, 0); - cache_v_output_nodes_.assign(dec_layers_, 0); -} - -void WhisperModel::reset_cache() { - Model::reset_cache(); - encoder_ready_ = false; - first_decode_step_ = true; - - auto* gb = static_cast(graph_handle_); - if (gb) { - if (encoder_output_persistent_ != 0) { - gb->invalidate_persistent(encoder_output_persistent_); - encoder_output_persistent_ = 0; - } - for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { - if (encoder_k_persistent_[i] != 0) { - gb->invalidate_persistent(encoder_k_persistent_[i]); - encoder_k_persistent_[i] = 0; - } - if (encoder_v_persistent_[i] != 0) { - gb->invalidate_persistent(encoder_v_persistent_[i]); - encoder_v_persistent_[i] = 0; - } - } - } -} - -size_t WhisperModel::build_decoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q = gb->matmul(input, layer.decoder_self_attn_q_weight, true, backend); - q = gb->add(q, layer.decoder_self_attn_q_bias); - - auto k = gb->matmul(input, layer.decoder_self_attn_k_weight, true, backend); - auto v = gb->matmul(input, layer.decoder_self_attn_v_weight, true, backend); - v = gb->add(v, layer.decoder_self_attn_v_bias); - - const auto& q_shape = gb->get_output_buffer(q).shape; - if (q_shape.size() != 2) { - throw std::runtime_error("decoder self-attn: q must be [T_new, D]"); - } - - size_t seq_new = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - size_t num_kv_heads = config_.attention_kv_heads; - - auto q_4d = gb->reshape(q, {1, seq_new, num_heads, head_dim}); - auto k_4d = gb->reshape(k, {1, seq_new, num_kv_heads, head_dim}); - auto v_4d = gb->reshape(v, {1, seq_new, num_kv_heads, head_dim}); - - size_t final_k = k_4d; - size_t final_v = v_4d; - - if (use_cache && !kv_cache_.is_empty()) { - auto k_view = kv_cache_.get_key_view(layer_idx); - auto v_view = kv_cache_.get_value_view(layer_idx); - - if (!k_view.ptr1 || !v_view.ptr1) { - throw std::runtime_error("KV cache view is empty but kv_cache_.is_empty()==false"); - } - - size_t cache_len = kv_cache_.current_seq_len; - - size_t cache_k_node = gb->input( - {1, cache_len, num_kv_heads, head_dim}, - kv_cache_.precision - ); - size_t cache_v_node = gb->input( - {1, cache_len, num_kv_heads, head_dim}, - kv_cache_.precision - ); - - if (k_view.ptr2 == nullptr && v_view.ptr2 == nullptr) { - gb->set_external_input(cache_k_node, const_cast(k_view.ptr1), kv_cache_.precision); - gb->set_external_input(cache_v_node, const_cast(v_view.ptr1), kv_cache_.precision); - } else { - gb->set_external_input(cache_k_node, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); - gb->set_external_input(cache_v_node, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); - } - - final_k = gb->concat(cache_k_node, k_4d, 1); - final_v = gb->concat(cache_v_node, v_4d, 1); - } - - if (use_cache) { - cache_k_output_nodes_[layer_idx] = final_k; - cache_v_output_nodes_[layer_idx] = final_v; - } else { - cache_k_output_nodes_[layer_idx] = k_4d; - cache_v_output_nodes_[layer_idx] = v_4d; - } - - auto attn_out_4d = gb->attention(q_4d, final_k, final_v, attention_scale_, position_offset); - auto attn_out = gb->reshape(attn_out_4d, {seq_new, num_heads * head_dim}); - - auto output = gb->matmul(attn_out, layer.decoder_self_attn_output_weight, true, backend); - output = gb->add(output, layer.decoder_self_attn_output_bias); - return output; -} - -size_t WhisperModel::build_encoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t /*position_offset*/){ - const auto& layer = weight_nodes_.layers[layer_idx]; - - if(use_cache) - throw std::runtime_error("The encoder attention layers are not auto-regressive, and thus don't use KV caching!"); - - auto q = gb->matmul(input, layer.encoder_self_attn_q_weight, true, backend); - q = gb->add(q, layer.encoder_self_attn_q_bias); - auto v = gb->matmul(input, layer.encoder_self_attn_v_weight, true, backend); - v = gb->add(v, layer.encoder_self_attn_v_bias); - auto k = gb->matmul(input, layer.encoder_self_attn_k_weight, true, backend); - - size_t seq_len = gb->get_output_buffer(q).shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - k = gb->reshape(k, {1, seq_len, num_heads, head_dim}); - v = gb->reshape(v, {1, seq_len, num_heads, head_dim}); - - auto attn = gb->attention(q, k, v, attention_scale_, false); - - attn = gb->reshape(attn, {seq_len, num_heads * head_dim}); - - auto output = gb->matmul(attn, layer.encoder_self_attn_output_weight, true, backend); - output = gb->add(output, layer.encoder_self_attn_output_bias); - - return output; -} - -size_t WhisperModel::build_conv1d(CactusGraph* gb, size_t input) -{ - size_t conv_input = input; - const auto& xbuf = gb->get_output_buffer(input); - - if (xbuf.precision == Precision::INT8) { - conv_input = gb->precision_cast(input, Precision::FP16); - } - - size_t conv1 = gb->conv1d_k3(conv_input, weight_nodes_.encoder_conv1_weight, 1); - - auto bias1_shape = gb->get_output_buffer(weight_nodes_.encoder_conv1_bias).shape; - size_t C1 = bias1_shape[0]; - size_t bias1 = gb->reshape(weight_nodes_.encoder_conv1_bias, {1, C1, 1}); - conv1 = gb->add(conv1, bias1); - - last_conv1_node_ = conv1; - - conv1 = gb->gelu_erf(conv1); - - size_t conv2 = gb->conv1d_k3(conv1, weight_nodes_.encoder_conv2_weight, 2); - auto bias2_shape = gb->get_output_buffer(weight_nodes_.encoder_conv2_bias).shape; - size_t C2 = bias2_shape[0]; - size_t bias2 = gb->reshape(weight_nodes_.encoder_conv2_bias, {1, C2, 1}); - conv2 = gb->add(conv2, bias2); - - last_conv2_node_ = conv2; - - conv2 = gb->gelu_erf(conv2); - - const auto& buf = gb->get_output_buffer(conv2); - - size_t conv2_transposed; - if (buf.precision == Precision::FP16) { - conv2_transposed = gb->transpose(conv2, ComputeBackend::CPU); - } else { - size_t conv2_f16 = gb->precision_cast(conv2, Precision::FP16); - conv2_transposed = gb->transpose(conv2_f16, ComputeBackend::CPU); - } - - return conv2_transposed; -} - -size_t WhisperModel::build_encoder_transformer_block( - CactusGraph* gb, - size_t hidden, - uint32_t layer_idx, - ComputeBackend backend, - bool use_cache, - size_t position_offset) -{ - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t ln1 = gb->layernorm( - hidden, - layer.encoder_post_attn_layernorm_weight, - layer.encoder_post_attn_layernorm_bias - ); - - size_t sa = build_encoder_self_attention( - gb, ln1, layer_idx, backend, use_cache, position_offset - ); - - size_t x_post_sa = gb->add(hidden, sa); - - size_t ln2 = gb->layernorm( - x_post_sa, - layer.encoder_post_ffn_layernorm_weight, - layer.encoder_post_ffn_layernorm_bias - ); - - size_t ffn_out = build_encoder_mlp( - gb, ln2, layer_idx, backend - ); - - size_t out = gb->add(x_post_sa, ffn_out); - - if (layer_idx < encoder_block_out_nodes_.size()) { - encoder_block_out_nodes_[layer_idx] = out; - } - - return out; -} - -size_t WhisperModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t ln1 = gb->layernorm(hidden, layer.decoder_post_attn_layernorm_weight, layer.decoder_post_attn_layernorm_bias); - size_t sa = build_decoder_self_attention(gb, ln1, layer_idx, backend, use_cache, position_offset); - size_t x_post_sa = gb->add(hidden, sa); - - size_t ln2 = gb->layernorm(x_post_sa, layer.decoder_post_encoder_layernorm_weight, layer.decoder_post_encoder_layernorm_bias); - size_t ca = build_encoder_attention(gb, ln2, layer_idx, backend, use_cache, position_offset); - size_t x_post_ca = gb->add(x_post_sa, ca); - - size_t ln3 = gb->layernorm(x_post_ca,layer.decoder_post_ffn_layernorm_weight,layer.decoder_post_ffn_layernorm_bias); - size_t ffn_out = build_decoder_mlp(gb, ln3, layer_idx, backend); - size_t x_post_ffn = gb->add(x_post_ca, ffn_out); - - return x_post_ffn; - -} - -void WhisperModel::run_encoder(const std::vector& audio_features) -{ - const size_t mel_bins = config_.num_mel_bins; - if (audio_features.size() % mel_bins != 0) - throw std::runtime_error("Mel bins length must be divisible by " + std::to_string(mel_bins) + "."); - - size_t T_mel = audio_features.size() / mel_bins; - if (T_mel == 0) - throw std::runtime_error("Mel bins has zero frames."); - - auto* gb = static_cast(graph_handle_); - if (!gb) - throw std::runtime_error("Graph handle is null in run_encoder."); - - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector out_shape = npu_encoder_->get_output_shape(); - size_t T_enc, D_enc; - if (out_shape.size() == 3) { - T_enc = static_cast(out_shape[1]); - D_enc = static_cast(out_shape[2]); - } else if (out_shape.size() == 2) { - T_enc = static_cast(out_shape[0]); - D_enc = static_cast(out_shape[1]); - } else { - throw std::runtime_error("NPU encoder output has unexpected shape"); - } - - std::vector<__fp16> audio_features_f16(audio_features.size()); - cactus_fp32_to_fp16(audio_features.data(), audio_features_f16.data(), audio_features.size()); - - std::vector input_shape = {1, static_cast(mel_bins), static_cast(T_mel)}; - size_t total_elements = T_enc * D_enc; - std::vector<__fp16> npu_output(total_elements); - size_t elements_written = npu_encoder_->encode( - audio_features_f16.data(), - npu_output.data(), - input_shape, - "x", - "" - ); - - if (elements_written > 0) { - size_t enc_output_node = gb->input({T_enc, D_enc}, Precision::FP16); - gb->set_input(enc_output_node, npu_output.data(), Precision::FP16); - - size_t enc_output_persistent = gb->persistent(enc_output_node); - encoder_output_persistent_ = enc_output_persistent; - last_encoder_post_norm_node_ = enc_output_persistent; - return; - } - } - - if (!has_whisper_cpu_encoder_weights(model_folder_path_)) { - throw std::runtime_error( - "Whisper requires either CPU encoder weights or model.mlpackage encoder output."); - } - - auto backend = - (config_.default_backend == Config::Backend::CPU) - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t mel_input = 0; - std::vector<__fp16> audio_features_f16(audio_features.size()); - cactus_fp32_to_fp16(audio_features.data(), audio_features_f16.data(), audio_features.size()); - - mel_input = gb->input({1, mel_bins, T_mel}, Precision::FP16); - gb->set_input(mel_input, audio_features_f16.data(), Precision::FP16); - - size_t conv2_transposed = build_conv1d(gb, mel_input); - - const auto& conv_shape = gb->get_output_buffer(conv2_transposed).shape; - if (conv_shape.size() != 3 || conv_shape[0] != 1) - throw std::runtime_error("Conv2 transpose should be [1, T_enc, D]."); - - size_t T_enc = conv_shape[1]; - size_t D_enc = conv_shape[2]; - - size_t pos_slice = gb->slice(weight_nodes_.encoder_position_embeddings, 0, 0, T_enc); - - size_t h2d = gb->reshape(conv2_transposed, {T_enc, D_enc}); - - auto& h2d_buf = gb->get_output_buffer(h2d); - auto& pos_buf = gb->get_output_buffer(pos_slice); - - if (pos_buf.precision != h2d_buf.precision) { - pos_slice = gb->precision_cast(pos_slice, h2d_buf.precision); - } - - size_t h_pos = gb->add(h2d, pos_slice); - last_enc_plus_pos_node_ = h_pos; - - size_t h = h_pos; - for (uint32_t i = 0; i < enc_layers_; ++i){ - h = build_encoder_transformer_block(gb, h, i, backend, false, 0); - if (i == 0) { - encoder_transformer_block_0 = h; - } - } - - size_t h_norm = gb->layernorm( - h, - weight_nodes_.encoder_norm_weight, - weight_nodes_.encoder_norm_bias - ); - - size_t h_norm_persistent = gb->persistent(h_norm); - encoder_output_persistent_ = h_norm_persistent; - last_encoder_post_norm_node_ = h_norm_persistent; -} - - - -size_t WhisperModel::run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only) { - auto* gb = static_cast(graph_handle_); - - const size_t full_len = tokens.size(); - if (full_len == 0) { - throw std::runtime_error("Decoder token list cannot be empty."); - } - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t start_idx = (use_cache && kv_cache_.current_seq_len > 0) ? full_len - 1 : 0; - size_t new_tokens = full_len - start_idx; - - size_t tok_input = gb->input({new_tokens}, Precision::FP32); - std::vector tok_f(new_tokens); - for (size_t i = 0; i < new_tokens; i++) { - tok_f[i] = static_cast(tokens[start_idx + i]); - } - gb->set_input(tok_input, tok_f.data(), Precision::FP32); - - size_t dec_hidden = gb->embedding(embedding_node_id_, tok_input); - - size_t position_offset = kv_cache_.current_seq_len; - size_t dec_pos = gb->slice(weight_nodes_.decoder_position_embeddings_weight,0,position_offset,new_tokens); - - { - const auto& h_buf = gb->get_output_buffer(dec_hidden); - const auto& pos_buf = gb->get_output_buffer(dec_pos); - - size_t pos_node_for_add = dec_pos; - if (pos_buf.precision != h_buf.precision) { - pos_node_for_add = gb->precision_cast(dec_pos, h_buf.precision); - } - - dec_hidden = gb->add(dec_hidden, pos_node_for_add); - } - - for (uint32_t layer_idx = 0; layer_idx < dec_layers_; ++layer_idx) { - dec_hidden = build_decoder_transformer_block( - gb, - dec_hidden, - layer_idx, - backend, - use_cache, - position_offset - ); - } - - size_t dec_norm = gb->layernorm( - dec_hidden, - weight_nodes_.decoder_norm_weight, - weight_nodes_.decoder_norm_bias - ); - - size_t logits_input = dec_norm; - if (last_token_only) { - size_t row_index = new_tokens - 1; - logits_input = gb->slice(logits_input, 0, row_index, 1); - } - - auto w_shape = gb->get_output_buffer(output_weight_node_id_).shape; - - size_t logits = gb->matmul(logits_input, output_weight_node_id_, true, backend); - - last_new_tokens_ = new_tokens; - return logits; -} - - -size_t WhisperModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) -{ - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized"); - } - - if (!use_cache) { - kv_cache_.reset(); - kv_cache_.current_seq_len = 0; - reset_graph_side_cache_nodes(); - run_encoder(audio_features); - } - - return run_decoder_step(tokens, use_cache, false); -} - -std::vector WhisperModel::get_audio_embeddings(const std::vector& audio_features) { - run_encoder(audio_features); - - auto* gb = static_cast(graph_handle_); - - size_t pooled = gb->mean(encoder_output_persistent_, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - size_t hidden_dim = output_buf.total_size; - - std::vector embedding(hidden_dim); - void* output_data = gb->get_output(pooled); - const float* output_ptr = static_cast(output_data); - std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); - - reset_cache(); - return embedding; -} - -uint32_t WhisperModel::decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty, - float* /*out_token_time_start*/, - float* /*out_token_time_end*/) -{ - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (audio_features.empty()) - throw std::runtime_error("Mel bins cannot be empty in Whisper decode_with_audio"); - - auto* gb = static_cast(graph_handle_); - - bool cold_start = !encoder_ready_; - size_t logits_node = 0; - - std::vector full_tokens; - if (tokens.empty()) { - full_tokens.push_back(static_cast(get_tokenizer()->get_bos_token())); - } else { - full_tokens = tokens; - } - - if (cold_start) - { - gb->soft_reset(); - kv_cache_.reset(); - kv_cache_.current_seq_len = 0; - reset_graph_side_cache_nodes(); - - first_decode_step_ = true; - run_encoder(audio_features); - logits_node = run_decoder_step(full_tokens, false, false); - encoder_ready_ = true; - } - else - { - gb->soft_reset(); - reset_graph_side_cache_nodes(); - - std::vector last_token_vec = { tokens.back() }; - logits_node = run_decoder_step(last_token_vec, true, true); - } - - const auto& suppress_bias = first_decode_step_ ? suppress_bias_first_step_ : suppress_bias_; - if (first_decode_step_) first_decode_step_ = false; - size_t sampled_token_id = sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty, &suppress_bias); - if (!profile_file.empty()) gb->execute(profile_file); - else gb->execute(); - - - compute_entropy(gb, logits_node, out_entropy); - - post_execute_updates(gb, full_tokens.size()); - update_kv_cache(gb, last_new_tokens_); - - auto* out_ptr = gb->get_output(sampled_token_id); - uint32_t sampled = *reinterpret_cast(out_ptr); - record_sampled_token(sampled); - - return sampled; -} - - -} -} diff --git a/cactus-transpile/models/model_youtu.cpp b/cactus-transpile/models/model_youtu.cpp deleted file mode 100644 index cfd1028ab..000000000 --- a/cactus-transpile/models/model_youtu.cpp +++ /dev/null @@ -1,246 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include - -namespace cactus { -namespace engine { - -YoutuModel::YoutuModel() : Model() {} - -YoutuModel::YoutuModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -std::vector YoutuModel::get_kv_layer_dims() const { - return std::vector(config_.num_layers, static_cast(config_.qk_head_dim)); -} - -void YoutuModel::post_init() { - std::vector v_dims(config_.num_layers, static_cast(config_.v_head_dim)); - std::vector v_kv_heads(config_.num_layers, static_cast(config_.attention_kv_heads)); - v_cache_.init(config_.num_layers, kv_cache_.max_seq_len, - v_dims, v_kv_heads, Precision::INT8); - v_cache_.set_window_size(kv_cache_.window_size, kv_cache_.sink_size); - cache_v_nodes_.resize(config_.num_layers, 0); -} - -void YoutuModel::post_execute_updates(CactusGraph* gb, size_t seq_len) { - v_cache_.update_from_graph(gb, cache_v_nodes_, cache_v_nodes_, seq_len, - config_.num_layers); -} - -void YoutuModel::reset_cache() { - Model::reset_cache(); - v_cache_.reset(); -} - -void YoutuModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - if (config_.q_lora_rank == 0) { - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - } else { - layer.attn_q_a_weight = gb->mmap_weights(layer_prefix + "attn_q_a.weights"); - layer.attn_q_a_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_a_norm.weights"); - layer.attn_q_b_weight = gb->mmap_weights(layer_prefix + "attn_q_b.weights"); - } - layer.attn_kv_a_weight = gb->mmap_weights(layer_prefix + "attn_kv_a.weights"); - layer.attn_kv_a_norm_weight = gb->mmap_weights(layer_prefix + "attn_kv_a_norm.weights"); - layer.attn_kv_b_weight = gb->mmap_weights(layer_prefix + "attn_kv_b.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - if (config_.attention_bias) { - layer.attn_q_a_bias = gb->mmap_weights(layer_prefix + "attn_q_a_bias.weights"); - layer.attn_kv_a_bias = gb->mmap_weights(layer_prefix + "attn_kv_a_bias.weights"); - layer.attn_output_bias = gb->mmap_weights(layer_prefix + "attn_output_bias.weights"); - } - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - } -} - -static float yarn_get_mscale(float scale, float mscale) { - if (scale <= 1.0f) return 1.0f; - return 0.1f * mscale * logf(scale) + 1.0f; -} - -size_t YoutuModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - const size_t seq_len = gb->get_output_buffer(normalized_input).shape[0]; - const size_t num_heads = config_.attention_heads; - const size_t num_kv_heads = config_.attention_kv_heads; - const size_t kv_lora = config_.kv_lora_rank; - const size_t qk_head = config_.qk_head_dim; - const size_t qk_nope = config_.qk_nope_head_dim; - const size_t qk_rope = config_.qk_rope_head_dim; - const size_t v_dim = config_.v_head_dim; - const float eps = config_.layer_norm_eps; - - size_t q_full; - if (config_.q_lora_rank == 0) { - q_full = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - } else { - auto q_latent = gb->matmul(normalized_input, layer.attn_q_a_weight, true, backend); - if (config_.attention_bias) { - q_latent = gb->add(q_latent, layer.attn_q_a_bias); - } - q_latent = gb->rms_norm(q_latent, layer.attn_q_a_norm_weight, eps); - q_full = gb->matmul(q_latent, layer.attn_q_b_weight, true, backend); - } - - q_full = gb->reshape(q_full, {seq_len * num_heads, qk_head}); - auto q_nope = gb->slice(q_full, 1, 0, qk_nope); - auto q_rope_raw = gb->slice(q_full, 1, qk_nope, qk_rope); - - q_rope_raw = gb->reshape(q_rope_raw, {1, seq_len, num_heads, qk_rope}); - size_t q_rope_rotated; - if (config_.rope_interleave) { - q_rope_rotated = gb->rope_gptj(q_rope_raw, config_.rope_theta, position_offset, qk_rope); - } else { - q_rope_rotated = gb->rope(q_rope_raw, config_.rope_theta, position_offset); - } - q_rope_rotated = gb->reshape(q_rope_rotated, {seq_len * num_heads, qk_rope}); - - auto q_combined = gb->concat(q_nope, q_rope_rotated, -1); - auto q_4d = gb->reshape(q_combined, {1, seq_len, num_heads, qk_head}); - - auto kv_combined = gb->matmul(normalized_input, layer.attn_kv_a_weight, true, backend); - if (config_.attention_bias) { - kv_combined = gb->add(kv_combined, layer.attn_kv_a_bias); - } - auto kv_latent = gb->slice(kv_combined, 1, 0, kv_lora); - auto k_rope_raw = gb->slice(kv_combined, 1, kv_lora, qk_rope); - - kv_latent = gb->rms_norm(kv_latent, layer.attn_kv_a_norm_weight, eps); - - auto kv_decoded = gb->matmul(kv_latent, layer.attn_kv_b_weight, true, backend); - kv_decoded = gb->reshape(kv_decoded, {seq_len * num_kv_heads, qk_nope + v_dim}); - auto k_nope = gb->slice(kv_decoded, 1, 0, qk_nope); - auto v_flat = gb->slice(kv_decoded, 1, qk_nope, v_dim); - - k_rope_raw = gb->reshape(k_rope_raw, {1, seq_len, 1, qk_rope}); - size_t k_rope_rotated; - if (config_.rope_interleave) { - k_rope_rotated = gb->rope_gptj(k_rope_raw, config_.rope_theta, position_offset, qk_rope); - } else { - k_rope_rotated = gb->rope(k_rope_raw, config_.rope_theta, position_offset); - } - std::vector k_rope_copies(num_kv_heads, k_rope_rotated); - auto k_rope_4d = gb->cat(k_rope_copies, 2); - auto k_rope_flat = gb->reshape(k_rope_4d, {seq_len * num_kv_heads, qk_rope}); - - auto k_combined = gb->concat(k_nope, k_rope_flat, -1); - auto k_4d = gb->reshape(k_combined, {1, seq_len, num_kv_heads, qk_head}); - auto v_4d = gb->reshape(v_flat, {1, seq_len, num_kv_heads, v_dim}); - - if (use_cache && layer_idx < cache_k_output_nodes_.size()) { - cache_k_output_nodes_[layer_idx] = gb->reshape(k_4d, {seq_len * num_kv_heads, qk_head}); - cache_v_output_nodes_[layer_idx] = cache_k_output_nodes_[layer_idx]; - cache_v_nodes_[layer_idx] = gb->reshape(v_4d, {seq_len * num_kv_heads, v_dim}); - } - - float scale = 1.0f / sqrtf(static_cast(qk_head)); - if (config_.rope_mscale_all_dim != 0.0f && config_.rope_scaling_factor > 1.0f) { - float mscale = yarn_get_mscale(config_.rope_scaling_factor, config_.rope_mscale_all_dim); - scale *= mscale * mscale; - } - - size_t attn_4d; - if (use_cache && !kv_cache_.is_empty()) { - attn_4d = gb->attention_int8_hybrid( - q_4d, k_4d, v_4d, scale, position_offset, - kv_cache_.get_keys_int8(layer_idx), - v_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - v_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, qk_head, - kv_cache_.window_size, v_dim); - } else { - attn_4d = gb->attention(q_4d, k_4d, v_4d, scale, position_offset); - } - - auto attn_out = gb->reshape(attn_4d, {seq_len, num_heads * v_dim}); - size_t out = gb->matmul(attn_out, layer.attn_output_weight, true, backend); - if (config_.attention_bias) { - out = gb->add(out, layer.attn_output_bias); - } - return out; -} - - -size_t YoutuModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - size_t gate_silu = gb->silu(gate_output); - size_t gated = gb->multiply(gate_silu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - - -size_t YoutuModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - auto after_attention = gb->add_clipped(hidden, attn_output); - auto normalized_after_attention = gb->rms_norm(after_attention, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); - return gb->add_clipped(after_attention, mlp_output); -} - - -size_t YoutuModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - const size_t seq_len = tokens.size(); - const size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - -} -} diff --git a/cactus-transpile/npu/npu.cpp b/cactus-transpile/npu/npu.cpp deleted file mode 100644 index 69a9f80c7..000000000 --- a/cactus-transpile/npu/npu.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// NPU stub implementation for non-Apple platforms -// On Apple platforms, npu_ane.mm provides the real implementation - -#include "npu.h" - -namespace cactus { -namespace npu { - -std::unique_ptr create_encoder() { - return nullptr; -} - -std::unique_ptr create_prefill() { - return nullptr; -} - -bool is_npu_available() { - return false; -} - -} // namespace npu -} // namespace cactus diff --git a/cactus-transpile/npu/npu.h b/cactus-transpile/npu/npu.h deleted file mode 100644 index 5474c872f..000000000 --- a/cactus-transpile/npu/npu.h +++ /dev/null @@ -1,89 +0,0 @@ -#ifndef CACTUS_NPU_H -#define CACTUS_NPU_H - -#include -#include -#include -#include - -namespace cactus { -namespace npu { - - -struct NPUNamedInput { - std::string name; - const __fp16* data; - std::vector shape; -}; - -class NPUEncoder { -public: - virtual ~NPUEncoder() = default; - - virtual bool load(const std::string& model_path) = 0; - - virtual bool preallocate(const std::vector& input_shape, - const std::string& input_name = "x", - const std::string& output_name = "") = 0; - - virtual size_t encode(const __fp16* input, - __fp16* output, - const std::vector& shape, - const std::string& input_name = "x", - const std::string& output_name = "") = 0; - - virtual bool is_available() const = 0; - - virtual std::vector get_input_shape() const = 0; - - virtual std::vector get_output_shape() const = 0; - - virtual __fp16* get_output_buffer() = 0; - - virtual size_t get_output_buffer_size() const = 0; - - virtual size_t encode_multimodal_input( - const std::vector& inputs, - __fp16* output, - const std::string& output_name = "") = 0; -}; - -std::unique_ptr create_encoder(); - -bool is_npu_available(); - -struct NPUBufferRef { - const __fp16* data; - size_t count; -}; - -struct NPUPrefillDirectResult { - NPUBufferRef hidden; - std::vector k_caches; - std::vector v_caches; - bool valid; -}; - -class NPUPrefill { -public: - virtual ~NPUPrefill() = default; - virtual bool load(const std::string& model_path) = 0; - virtual bool is_available() const = 0; - virtual int get_chunk_size() const = 0; - virtual int get_hidden_dim() const = 0; - virtual int get_num_layers() const = 0; - virtual int get_num_kv_heads() const = 0; - virtual int get_head_dim() const = 0; - - virtual NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset = 0, - const std::string& input_name = "x") = 0; -}; - -std::unique_ptr create_prefill(); - -} // namespace npu -} // namespace cactus - -#endif // CACTUS_NPU_H \ No newline at end of file diff --git a/cactus-transpile/npu/npu_ane.h b/cactus-transpile/npu/npu_ane.h deleted file mode 100644 index c590fb0c4..000000000 --- a/cactus-transpile/npu/npu_ane.h +++ /dev/null @@ -1,154 +0,0 @@ -#ifndef CACTUS_NPU_ANE_H -#define CACTUS_NPU_ANE_H - -#include "npu.h" - -#if defined(__APPLE__) -#define CACTUS_HAS_ANE 1 -#else -#define CACTUS_HAS_ANE 0 -#endif - -namespace cactus { -namespace npu { - -#if CACTUS_HAS_ANE - - -class ANEEncoder : public NPUEncoder { -public: - ANEEncoder(); - ~ANEEncoder() override; - - ANEEncoder(const ANEEncoder&) = delete; - ANEEncoder& operator=(const ANEEncoder&) = delete; - - ANEEncoder(ANEEncoder&& other) noexcept; - ANEEncoder& operator=(ANEEncoder&& other) noexcept; - - bool load(const std::string& model_path) override; - - bool preallocate(const std::vector& input_shape, - const std::string& input_name = "x", - const std::string& output_name = "") override; - - size_t encode(const __fp16* input, - __fp16* output, - const std::vector& shape, - const std::string& input_name = "x", - const std::string& output_name = "") override; - - bool is_available() const override; - - std::vector get_input_shape() const override; - - std::vector get_output_shape() const override; - - __fp16* get_output_buffer() override; - - size_t get_output_buffer_size() const override; - - size_t encode_multimodal_input( - const std::vector& inputs, - __fp16* output, - const std::string& output_name = "") override; - -private: - void* impl_; -}; - -class ANEPrefill : public NPUPrefill { -public: - ANEPrefill(); - ~ANEPrefill() override; - - ANEPrefill(const ANEPrefill&) = delete; - ANEPrefill& operator=(const ANEPrefill&) = delete; - - ANEPrefill(ANEPrefill&& other) noexcept; - ANEPrefill& operator=(ANEPrefill&& other) noexcept; - - bool load(const std::string& model_path) override; - bool is_available() const override; - - int get_chunk_size() const override; - int get_hidden_dim() const override; - int get_num_layers() const override; - int get_num_kv_heads() const override; - int get_head_dim() const override; - - NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset = 0, - const std::string& input_name = "x") override; - -private: - void* impl_; - int chunk_size_ = 256; - int hidden_dim_ = 0; - int num_layers_ = 0; - int num_kv_heads_ = 0; - int head_dim_ = 0; -}; - -#else - -class ANEEncoder : public NPUEncoder { -public: - ANEEncoder() = default; - ~ANEEncoder() override = default; - - bool load(const std::string&) override { return false; } - - bool preallocate(const std::vector&, - const std::string& = "x", - const std::string& = "") override { return false; } - - size_t encode(const __fp16*, - __fp16*, - const std::vector&, - const std::string& = "x", - const std::string& = "") override { return 0; } - - bool is_available() const override { return false; } - - std::vector get_input_shape() const override { return {}; } - - std::vector get_output_shape() const override { return {}; } - - __fp16* get_output_buffer() override { return nullptr; } - - size_t get_output_buffer_size() const override { return 0; } - - size_t encode_multimodal_input( - const std::vector&, - __fp16*, - const std::string& = "") override { return 0; } -}; - -class ANEPrefill : public NPUPrefill { -public: - ANEPrefill() = default; - ~ANEPrefill() override = default; - - bool load(const std::string&) override { return false; } - bool is_available() const override { return false; } - - int get_chunk_size() const override { return 0; } - int get_hidden_dim() const override { return 0; } - int get_num_layers() const override { return 0; } - int get_num_kv_heads() const override { return 0; } - int get_head_dim() const override { return 0; } - - NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>&, - int = 0, - const std::string& = "x") override { return {}; } -}; - -#endif // CACTUS_HAS_ANE - -} // namespace npu -} // namespace cactus - -#endif // CACTUS_NPU_ANE_H \ No newline at end of file diff --git a/cactus-transpile/npu/npu_ane.mm b/cactus-transpile/npu/npu_ane.mm deleted file mode 100644 index e6c379486..000000000 --- a/cactus-transpile/npu/npu_ane.mm +++ /dev/null @@ -1,1305 +0,0 @@ -#include "npu_ane.h" - -#if CACTUS_HAS_ANE - -#import -#import -#include -#include -#include -#include -#include -#include -#include -#include "../graph/graph.h" - -namespace { - -static std::string to_lower_ascii(std::string value) { - std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { - return static_cast(std::tolower(c)); - }); - return value; -} - -static bool parse_compute_units(const char* raw_value, MLComputeUnits& out_units) { - if (!raw_value || raw_value[0] == '\0') { - return false; - } - - std::string value = to_lower_ascii(raw_value); - if (value == "all" || value == "default") { - out_units = MLComputeUnitsAll; - return true; - } - if (value == "cpu_and_ne" || value == "cpu-ne" || value == "cpu_ne" || - value == "ne" || value == "ane" || value == "cpuandneuralengine") { - out_units = MLComputeUnitsCPUAndNeuralEngine; - return true; - } - if (value == "cpu_and_gpu" || value == "cpu_gpu" || value == "cpu-gpu" || - value == "cpuandgpu") { - out_units = MLComputeUnitsCPUAndGPU; - return true; - } - if (value == "cpu_only" || value == "cpu-only" || value == "cpu") { - out_units = MLComputeUnitsCPUOnly; - return true; - } - - return false; -} - -static const char* compute_units_to_string(MLComputeUnits units) { - switch (units) { - case MLComputeUnitsAll: - return "ALL"; - case MLComputeUnitsCPUAndGPU: - return "CPU_AND_GPU"; - case MLComputeUnitsCPUOnly: - return "CPU_ONLY"; - case MLComputeUnitsCPUAndNeuralEngine: - return "CPU_AND_NE"; - } - return "UNKNOWN"; -} - -static bool model_path_looks_like_audio_encoder(NSString* model_path) { - if (!model_path) return false; - NSString* lower = [[model_path lastPathComponent] lowercaseString]; - return [lower containsString:@"audio_encoder"]; -} - -static bool model_path_looks_like_vision_encoder(NSString* model_path) { - if (!model_path) return false; - NSString* lower = [[model_path lastPathComponent] lowercaseString]; - return [lower containsString:@"vision_encoder"]; -} - -static void maybe_apply_compute_units_env(const char* env_name, MLComputeUnits& target) { - const char* raw = std::getenv(env_name); - if (!raw || raw[0] == '\0') return; - - MLComputeUnits parsed; - if (parse_compute_units(raw, parsed)) { - target = parsed; - return; - } - - CACTUS_LOG_WARN("npu", "Ignoring invalid " << env_name << "=" << raw); -} - -static MLComputeUnits resolve_compute_units_for_model(NSString* model_path, bool is_prefill) { - MLComputeUnits units = MLComputeUnitsCPUAndNeuralEngine; - - // Gemma4 multimodal encoders show better fidelity on CPU+GPU. - if (!is_prefill && model_path_looks_like_audio_encoder(model_path)) { - units = MLComputeUnitsCPUAndGPU; - } - if (!is_prefill && model_path_looks_like_vision_encoder(model_path)) { - units = MLComputeUnitsCPUAndGPU; - } - - maybe_apply_compute_units_env("CACTUS_ANE_COMPUTE_UNITS", units); - if (is_prefill) { - maybe_apply_compute_units_env("CACTUS_ANE_PREFILL_COMPUTE_UNITS", units); - } else { - maybe_apply_compute_units_env("CACTUS_ANE_ENCODER_COMPUTE_UNITS", units); - if (model_path_looks_like_audio_encoder(model_path)) { - maybe_apply_compute_units_env("CACTUS_ANE_AUDIO_COMPUTE_UNITS", units); - } - if (model_path_looks_like_vision_encoder(model_path)) { - maybe_apply_compute_units_env("CACTUS_ANE_VISION_COMPUTE_UNITS", units); - } - } - - return units; -} - -static bool should_recompile_mlpackage(NSString* mlpackage_path, NSString* mlmodelc_path) { - NSFileManager* fm = [NSFileManager defaultManager]; - if (![fm fileExistsAtPath:mlmodelc_path]) return true; - - const char* force = std::getenv("CACTUS_ANE_FORCE_RECOMPILE"); - if (force && force[0] != '\0' && std::strcmp(force, "0") != 0) { - return true; - } - - NSError* pkg_err = nil; - NSError* cache_err = nil; - NSDictionary* pkg_attr = [fm attributesOfItemAtPath:mlpackage_path error:&pkg_err]; - NSDictionary* cache_attr = [fm attributesOfItemAtPath:mlmodelc_path error:&cache_err]; - if (!pkg_attr || !cache_attr || pkg_err || cache_err) { - return false; - } - - NSDate* pkg_mtime = pkg_attr[NSFileModificationDate]; - NSDate* cache_mtime = cache_attr[NSFileModificationDate]; - if (!pkg_mtime || !cache_mtime) { - return false; - } - return ([pkg_mtime compare:cache_mtime] == NSOrderedDescending); -} - -static NSURL* resolve_or_compile_model_url(NSString* path, NSError** error) { - NSURL* modelURL = [NSURL fileURLWithPath:path]; - if (![path hasSuffix:@".mlpackage"]) { - return modelURL; - } - - BOOL isDir = NO; - if (![[NSFileManager defaultManager] fileExistsAtPath:path isDirectory:&isDir] || !isDir) { - CACTUS_LOG_ERROR("npu", "ANE mlpackage path is not a valid directory: " << [path UTF8String]); - return modelURL; - } - - NSString* cachedPath = [[path stringByDeletingPathExtension] stringByAppendingPathExtension:@"mlmodelc"]; - NSURL* cachedURL = [NSURL fileURLWithPath:cachedPath]; - NSFileManager* fm = [NSFileManager defaultManager]; - - if (!should_recompile_mlpackage(path, cachedPath)) { - return cachedURL; - } - - if ([fm fileExistsAtPath:cachedPath]) { - NSError* rmErr = nil; - [fm removeItemAtPath:cachedPath error:&rmErr]; - if (rmErr) { - CACTUS_LOG_WARN("npu", "Failed to remove stale mlmodelc: " << [[rmErr localizedDescription] UTF8String]); - } else { - CACTUS_LOG_INFO("npu", "Removed stale mlmodelc cache, recompiling: " << [cachedPath UTF8String]); - } - } - - NSURL* compiledURL = [MLModel compileModelAtURL:modelURL error:error]; - if (*error || !compiledURL) { - CACTUS_LOG_ERROR("npu", "ANE model compilation failed: " << [[*error localizedDescription] UTF8String]); - return modelURL; - } - - NSError* moveError = nil; - [fm moveItemAtURL:compiledURL toURL:cachedURL error:&moveError]; - if (moveError) { - CACTUS_LOG_WARN("npu", "Could not persist compiled mlmodelc cache: " << [[moveError localizedDescription] UTF8String]); - return compiledURL; - } - return cachedURL; -} - -} // namespace - -@interface CactusANEImpl : NSObject - -@property (nonatomic, strong) MLModel* model; -@property (nonatomic, strong) MLModelDescription* modelDescription; -@property (nonatomic, strong) MLMultiArray* cachedInputArray; -@property (nonatomic, strong) MLMultiArray* cachedOutputArray; -@property (nonatomic, strong) NSArray* cachedShape; -@property (nonatomic, strong) NSArray* cachedOutputShape; -@property (nonatomic, strong) NSString* cachedInputName; -@property (nonatomic, strong) NSString* cachedOutputName; -@property (nonatomic, assign) NSUInteger cachedOutputSize; -@property (nonatomic, strong) MLPredictionOptions* predictionOptions; -@property (nonatomic, strong) NSMutableDictionary* cachedMultiInputArrays; -@property (nonatomic, strong) MLMultiArray* cachedMultiOutputArray; -@property (nonatomic, strong) MLPredictionOptions* multiPredictionOptions; -@property (nonatomic, assign) BOOL multiInputPreallocated; -@property (nonatomic, assign) BOOL hasOutputBackings; -@property (nonatomic, assign) BOOL hasMultiOutputBackings; - -- (instancetype)initWithModelPath:(NSString*)path; -- (NSArray*)getInputShape; -- (NSArray*)getOutputShape; -- (BOOL)preallocateBuffersWithInput:(NSString*)inputName - shape:(NSArray*)shape - outputName:(NSString*)outputName; -- (BOOL)canUseCachedBufferWithInput:(NSString*)inputName - shape:(NSArray*)shape - outputName:(NSString*)outputName; -- (MLMultiArray*)predictWithInput:(NSString*)inputName - data:(const __fp16*)data - shape:(NSArray*)shape - outputName:(NSString*)outputName; -- (BOOL)preallocateMultiInputBuffersWithOutputName:(NSString*)outputName; -- (size_t)predictMultiInputWithDict:(NSMutableDictionary*)inputDict - outputData:(__fp16*)output - outputName:(NSString*)outputName; - -@end - -@implementation CactusANEImpl - -- (instancetype)initWithModelPath:(NSString*)path { - self = [super init]; - if (self) { - NSError* error = nil; - NSURL* modelURL = resolve_or_compile_model_url(path, &error); - - MLModelConfiguration* config = [[MLModelConfiguration alloc] init]; - config.computeUnits = resolve_compute_units_for_model(path, false); - CACTUS_LOG_INFO("npu", - "ANEEncoder compute units: " - << compute_units_to_string(config.computeUnits) - << " for " << [path UTF8String]); - - _model = [MLModel modelWithContentsOfURL:modelURL configuration:config error:&error]; - if (_model) { - _modelDescription = _model.modelDescription; - } - if (error) { - CACTUS_LOG_ERROR("npu", "ANE model load failed: " << [[error localizedDescription] UTF8String]); - } - } - return self; -} - -- (NSArray*)getInputShape { - if (!_modelDescription) return @[]; - - NSString* inputName = _cachedInputName; - if (!inputName) { - inputName = _modelDescription.inputDescriptionsByName.allKeys.firstObject; - } - - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.type == MLFeatureTypeMultiArray) { - return inputDesc.multiArrayConstraint.shape; - } - - return @[]; -} - -- (NSArray*)getOutputShape { - if (!_modelDescription) return @[]; - - NSString* outputName = _cachedOutputName; - if (!outputName) { - outputName = _modelDescription.outputDescriptionsByName.allKeys.firstObject; - } - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - return outputDesc.multiArrayConstraint.shape; - } - - return @[]; -} - -- (BOOL)preallocateBuffersWithInput:(NSString*)inputName - shape:(NSArray*)shape - outputName:(NSString*)outputName { - if (!_model) return NO; - - NSError* error = nil; - - MLMultiArrayDataType inputDataType = MLMultiArrayDataTypeFloat16; - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.multiArrayConstraint) { - inputDataType = inputDesc.multiArrayConstraint.dataType; - } - - _cachedInputArray = [[MLMultiArray alloc] - initWithShape:shape - dataType:inputDataType - error:&error]; - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE preallocate input array failed: " << [[error localizedDescription] UTF8String]); - return NO; - } - - _cachedShape = [shape copy]; - _cachedInputName = [inputName copy]; - _cachedOutputName = outputName ? [outputName copy] - : _modelDescription.outputDescriptionsByName.allKeys.firstObject; - - CACTUS_LOG_DEBUG("npu", - "[CactusANE] prealloc input name=" - << [_cachedInputName UTF8String] - << " shape=" << [[_cachedInputArray.shape description] UTF8String] - << " strides=" << [[_cachedInputArray.strides description] UTF8String] - << " dtype=" << (long)_cachedInputArray.dataType); - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[_cachedOutputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - NSArray* outputShape = outputDesc.multiArrayConstraint.shape; - MLMultiArrayDataType outputDataType = outputDesc.multiArrayConstraint.dataType; - - _cachedOutputArray = [[MLMultiArray alloc] - initWithShape:outputShape - dataType:outputDataType - error:&error]; - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE preallocate output array failed: " << [[error localizedDescription] UTF8String]); - return NO; - } - - _cachedOutputShape = [outputShape copy]; - _cachedOutputSize = 1; - for (NSNumber* dim in outputShape) { - _cachedOutputSize *= [dim unsignedIntegerValue]; - } - - CACTUS_LOG_DEBUG("npu", - "[CactusANE] prealloc output name=" - << [_cachedOutputName UTF8String] - << " shape=" << [[_cachedOutputArray.shape description] UTF8String] - << " strides=" << [[_cachedOutputArray.strides description] UTF8String] - << " dtype=" << (long)_cachedOutputArray.dataType); - - _predictionOptions = [[MLPredictionOptions alloc] init]; - _hasOutputBackings = NO; - if (_predictionOptions && _cachedOutputArray && _cachedOutputName) { - _predictionOptions.outputBackings = - @{ _cachedOutputName: _cachedOutputArray }; - _hasOutputBackings = YES; - } - } - - return YES; -} - -- (BOOL)canUseCachedBufferWithInput:(NSString*)inputName - shape:(NSArray*)shape - outputName:(NSString*)outputName { - if (!_cachedInputArray || !_cachedShape) return NO; - if (![_cachedInputName isEqualToString:inputName]) return NO; - if (_cachedShape.count != shape.count) return NO; - - for (NSUInteger i = 0; i < shape.count; i++) { - if (![_cachedShape[i] isEqualToNumber:shape[i]]) return NO; - } - - if (outputName && outputName.length > 0 && ![_cachedOutputName isEqualToString:outputName]) { - return NO; - } - - return YES; -} - -- (MLMultiArray*)predictWithInput:(NSString*)inputName - data:(const __fp16*)data - shape:(NSArray*)shape - outputName:(NSString*)outputName { - if (!_model) return nil; - - NSError* error = nil; - MLMultiArray* inputArray = nil; - - MLMultiArrayDataType expectedDataType = MLMultiArrayDataTypeFloat16; - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.multiArrayConstraint) { - expectedDataType = inputDesc.multiArrayConstraint.dataType; - } - - BOOL useCached = (expectedDataType == MLMultiArrayDataTypeFloat16) && - [self canUseCachedBufferWithInput:inputName shape:shape outputName:outputName]; - - if (useCached) { - inputArray = _cachedInputArray; - } else { - inputArray = [[MLMultiArray alloc] - initWithShape:shape - dataType:expectedDataType - error:&error]; - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE create input array failed: " << [[error localizedDescription] UTF8String]); - return nil; - } - } - - NSUInteger totalElements = 1; - for (NSNumber* dim in shape) { - totalElements *= [dim unsignedIntegerValue]; - } - - if (expectedDataType == MLMultiArrayDataTypeFloat16) { - __fp16* inputPtr = (__fp16*)inputArray.dataPointer; - memcpy(inputPtr, data, totalElements * sizeof(__fp16)); - } else { - float* inputPtr = (float*)inputArray.dataPointer; - for (NSUInteger i = 0; i < totalElements; i++) { - inputPtr[i] = (float)data[i]; - } - } - - MLFeatureValue* inputFeature = [MLFeatureValue featureValueWithMultiArray:inputArray]; - NSDictionary* inputDict = @{inputName: inputFeature}; - id inputProvider = [[MLDictionaryFeatureProvider alloc] - initWithDictionary:inputDict - error:&error]; - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE create feature provider failed: " << [[error localizedDescription] UTF8String]); - return nil; - } - - id outputProvider = nil; - - if (useCached && _hasOutputBackings && _predictionOptions) { - outputProvider = [_model predictionFromFeatures:inputProvider - options:_predictionOptions - error:&error]; - } else { - outputProvider = [_model predictionFromFeatures:inputProvider error:&error]; - } - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE prediction failed: " << [[error localizedDescription] UTF8String]); - return nil; - } - - NSString* outName = outputName; - if (!outName || outName.length == 0) { - outName = useCached ? _cachedOutputName - : _modelDescription.outputDescriptionsByName.allKeys.firstObject; - } - - MLFeatureValue* outputFeature = [outputProvider featureValueForName:outName]; - static bool logged_output_feature_layout = false; - if (!logged_output_feature_layout && outputFeature && outputFeature.multiArrayValue) { - MLMultiArray* outArr = outputFeature.multiArrayValue; - CACTUS_LOG_DEBUG("npu", - "[CactusANE] output feature name=" - << [outName UTF8String] - << " shape=" << [[outArr.shape description] UTF8String] - << " strides=" << [[outArr.strides description] UTF8String] - << " dtype=" << (long)outArr.dataType); - logged_output_feature_layout = true; - } - return outputFeature.multiArrayValue; -} - -static size_t copyMLArrayToFP16(MLMultiArray* array, __fp16* output) { - size_t count = array.count; - if (!array || !output || count == 0) return 0; - - std::vector dims; - std::vector strides; - const size_t rank = array.shape.count; - bool have_layout = rank == array.strides.count && rank > 0; - if (have_layout) { - dims.resize(rank); - strides.resize(rank); - for (size_t i = 0; i < rank; ++i) { - NSInteger d = [array.shape[i] integerValue]; - NSInteger s = [array.strides[i] integerValue]; - if (d <= 0 || s < 0) { - have_layout = false; - break; - } - dims[i] = static_cast(d); - strides[i] = static_cast(s); - } - } - - if (!have_layout) { - if (array.dataType == MLMultiArrayDataTypeFloat16) { - if (output != (__fp16*)array.dataPointer) { - memcpy(output, array.dataPointer, count * sizeof(__fp16)); - } - } else { - const float* src = (const float*)array.dataPointer; - for (size_t i = 0; i < count; i++) output[i] = (__fp16)src[i]; - } - return count; - } - - auto is_contiguous = [&]() -> bool { - if (!have_layout) return false; - size_t expected = 1; - for (size_t i = rank; i-- > 0;) { - if (dims[i] > 1 && strides[i] != expected) return false; - expected *= dims[i]; - } - return true; - }; - - if (false && is_contiguous()) { - if (array.dataType == MLMultiArrayDataTypeFloat16) { - if (output != (__fp16*)array.dataPointer) { - memcpy(output, array.dataPointer, count * sizeof(__fp16)); - } - } else { - const float* src = (const float*)array.dataPointer; - for (size_t i = 0; i < count; i++) output[i] = (__fp16)src[i]; - } - return count; - } - - std::vector idx(rank, 0); - if (array.dataType == MLMultiArrayDataTypeFloat16) { - const __fp16* src = (const __fp16*)array.dataPointer; - for (size_t i = 0; i < count; ++i) { - size_t offset = 0; - for (size_t d = 0; d < rank; ++d) { - offset += idx[d] * strides[d]; - } - output[i] = src[offset]; - - for (size_t d = rank; d-- > 0;) { - idx[d]++; - if (idx[d] < dims[d]) break; - idx[d] = 0; - } - } - } else { - const float* src = (const float*)array.dataPointer; - for (size_t i = 0; i < count; ++i) { - size_t offset = 0; - for (size_t d = 0; d < rank; ++d) { - offset += idx[d] * strides[d]; - } - output[i] = (__fp16)src[offset]; - - for (size_t d = rank; d-- > 0;) { - idx[d]++; - if (idx[d] < dims[d]) break; - idx[d] = 0; - } - } - } - return count; -} - -static void copyFP16ToMLArray(const __fp16* data, size_t count, MLMultiArray* array) { - if (!array || !data || count == 0) return; - - std::vector dims; - std::vector strides; - const size_t rank = array.shape.count; - bool have_layout = rank == array.strides.count && rank > 0; - if (have_layout) { - dims.resize(rank); - strides.resize(rank); - for (size_t i = 0; i < rank; ++i) { - NSInteger d = [array.shape[i] integerValue]; - NSInteger s = [array.strides[i] integerValue]; - if (d <= 0 || s < 0) { - have_layout = false; - break; - } - dims[i] = static_cast(d); - strides[i] = static_cast(s); - } - } - - if (!have_layout) { - if (array.dataType == MLMultiArrayDataTypeFloat16) { - memcpy(array.dataPointer, data, count * sizeof(__fp16)); - } else { - float* dst = (float*)array.dataPointer; - for (size_t i = 0; i < count; i++) dst[i] = (float)data[i]; - } - return; - } - - auto is_contiguous = [&]() -> bool { - if (!have_layout) return false; - size_t expected = 1; - for (size_t i = rank; i-- > 0;) { - if (dims[i] > 1 && strides[i] != expected) return false; - expected *= dims[i]; - } - return true; - }; - - if (false && is_contiguous()) { - if (array.dataType == MLMultiArrayDataTypeFloat16) { - memcpy(array.dataPointer, data, count * sizeof(__fp16)); - } else { - float* dst = (float*)array.dataPointer; - for (size_t i = 0; i < count; i++) dst[i] = (float)data[i]; - } - return; - } - - std::vector idx(rank, 0); - if (array.dataType == MLMultiArrayDataTypeFloat16) { - __fp16* dst = (__fp16*)array.dataPointer; - for (size_t i = 0; i < count; ++i) { - size_t offset = 0; - for (size_t d = 0; d < rank; ++d) { - offset += idx[d] * strides[d]; - } - dst[offset] = data[i]; - - for (size_t d = rank; d-- > 0;) { - idx[d]++; - if (idx[d] < dims[d]) break; - idx[d] = 0; - } - } - } else { - float* dst = (float*)array.dataPointer; - for (size_t i = 0; i < count; ++i) { - size_t offset = 0; - for (size_t d = 0; d < rank; ++d) { - offset += idx[d] * strides[d]; - } - dst[offset] = (float)data[i]; - - for (size_t d = rank; d-- > 0;) { - idx[d]++; - if (idx[d] < dims[d]) break; - idx[d] = 0; - } - } - } -} - -- (BOOL)preallocateMultiInputBuffersWithOutputName:(NSString*)outputName { - if (!_model || !_modelDescription) return NO; - - NSError* error = nil; - _cachedMultiInputArrays = [NSMutableDictionary new]; - - for (NSString* inputName in _modelDescription.inputDescriptionsByName) { - MLFeatureDescription* desc = _modelDescription.inputDescriptionsByName[inputName]; - if (desc.type != MLFeatureTypeMultiArray) continue; - - MLMultiArray* array = [[MLMultiArray alloc] - initWithShape:desc.multiArrayConstraint.shape - dataType:desc.multiArrayConstraint.dataType - error:&error]; - if (!array || error) return NO; - _cachedMultiInputArrays[inputName] = array; - } - - NSString* outName = (outputName && outputName.length > 0) - ? outputName : _modelDescription.outputDescriptionsByName.allKeys.firstObject; - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - _cachedMultiOutputArray = [[MLMultiArray alloc] - initWithShape:outputDesc.multiArrayConstraint.shape - dataType:outputDesc.multiArrayConstraint.dataType - error:&error]; - if (!_cachedMultiOutputArray || error) return NO; - - _multiPredictionOptions = [[MLPredictionOptions alloc] init]; - _hasMultiOutputBackings = NO; - } - - _multiInputPreallocated = YES; - return YES; -} - -- (size_t)predictMultiInputWithDict:(NSMutableDictionary*)inputDict - outputData:(__fp16*)output - outputName:(NSString*)outputName { - if (!_model) return 0; - - NSError* error = nil; - MLDictionaryFeatureProvider* provider = - [[MLDictionaryFeatureProvider alloc] initWithDictionary:inputDict error:&error]; - if (!provider || error) return 0; - - id result = nil; - if (_hasMultiOutputBackings && _multiPredictionOptions) { - result = [_model predictionFromFeatures:provider options:_multiPredictionOptions error:&error]; - } else { - result = [_model predictionFromFeatures:provider error:&error]; - } - if (!result || error) return 0; - - static bool logged_multi_layout = false; - if (!logged_multi_layout) { - for (NSString* key in inputDict) { - MLFeatureValue* inFeature = inputDict[key]; - if (inFeature && inFeature.multiArrayValue) { - MLMultiArray* arr = inFeature.multiArrayValue; - CACTUS_LOG_WARN("npu", - "[CactusANE] multi input name=" - << [key UTF8String] - << " shape=" << [[arr.shape description] UTF8String] - << " strides=" << [[arr.strides description] UTF8String] - << " dtype=" << (long)arr.dataType); - } - } - } - - MLFeatureValue* outFeature = [result featureValueForName:outputName]; - if (!outFeature || !outFeature.multiArrayValue) return 0; - if (!logged_multi_layout) { - MLMultiArray* outArr = outFeature.multiArrayValue; - CACTUS_LOG_WARN("npu", - "[CactusANE] multi output name=" - << [outputName UTF8String] - << " shape=" << [[outArr.shape description] UTF8String] - << " strides=" << [[outArr.strides description] UTF8String] - << " dtype=" << (long)outArr.dataType); - logged_multi_layout = true; - } - return copyMLArrayToFP16(outFeature.multiArrayValue, output); -} - -@end - -namespace cactus { -namespace npu { - -static bool g_npu_enabled = true; - -ANEEncoder::ANEEncoder() : impl_(nullptr) {} - -ANEEncoder::~ANEEncoder() { - if (impl_) { - (void)(__bridge_transfer CactusANEImpl*)impl_; - impl_ = nullptr; - } -} - -ANEEncoder::ANEEncoder(ANEEncoder&& other) noexcept : impl_(other.impl_) { - other.impl_ = nullptr; -} - -ANEEncoder& ANEEncoder::operator=(ANEEncoder&& other) noexcept { - if (this != &other) { - if (impl_) { - (void)(__bridge_transfer CactusANEImpl*)impl_; - } - impl_ = other.impl_; - other.impl_ = nullptr; - } - return *this; -} - -bool ANEEncoder::load(const std::string& model_path) { - @autoreleasepool { - CACTUS_LOG_INFO("npu", "ANEEncoder loading model: " << model_path); - NSString* path = [NSString stringWithUTF8String:model_path.c_str()]; - - if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { - CACTUS_LOG_WARN("npu", "not using NPU"); - return false; - } - - CactusANEImpl* impl = [[CactusANEImpl alloc] initWithModelPath:path]; - - if (impl && impl.model) { - impl_ = (__bridge_retained void*)impl; - CACTUS_LOG_INFO("npu", "ANEEncoder model loaded successfully: " << model_path); - return true; - } - CACTUS_LOG_ERROR("npu", "ANEEncoder model load failed: " << model_path); - return false; - } -} - -bool ANEEncoder::preallocate(const std::vector& input_shape, - const std::string& input_name, - const std::string& output_name) { - if (!impl_) return false; - - @autoreleasepool { - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - - NSMutableArray* shapeArray = [NSMutableArray array]; - for (int dim : input_shape) { - [shapeArray addObject:@(dim)]; - } - - NSString* inName = [NSString stringWithUTF8String:input_name.c_str()]; - NSString* outName = output_name.empty() - ? nil - : [NSString stringWithUTF8String:output_name.c_str()]; - - return [impl preallocateBuffersWithInput:inName shape:shapeArray outputName:outName]; - } -} - -size_t ANEEncoder::encode(const __fp16* input, - __fp16* output, - const std::vector& shape, - const std::string& input_name, - const std::string& output_name) { - if (!impl_ || !input || !output) return 0; - - @autoreleasepool { - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - - NSArray* shapeArray = impl.cachedShape; - bool shapeMatches = (shapeArray && shapeArray.count == shape.size()); - if (shapeMatches) { - for (size_t i = 0; i < shape.size(); ++i) { - if ([shapeArray[i] intValue] != shape[i]) { - shapeMatches = false; - break; - } - } - } - - if (!shapeMatches) { - NSMutableArray* newShapeArray = [NSMutableArray arrayWithCapacity:shape.size()]; - for (int dim : shape) { - [newShapeArray addObject:@(dim)]; - } - shapeArray = newShapeArray; - } - - NSString* inName = impl.cachedInputName; - if (!inName) { - inName = [NSString stringWithUTF8String:input_name.c_str()]; - } - NSString* outName = impl.cachedOutputName; - if (!outName && !output_name.empty()) { - outName = [NSString stringWithUTF8String:output_name.c_str()]; - } - - MLMultiArray* mlOutput = [impl predictWithInput:inName - data:input - shape:shapeArray - outputName:outName]; - - if (mlOutput) { - return copyMLArrayToFP16(mlOutput, output); - } - } - - return 0; -} - -bool ANEEncoder::is_available() const { - if (!impl_) return false; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - return impl.model != nil; -} - -std::vector ANEEncoder::get_input_shape() const { - std::vector result; - if (!impl_) return result; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - NSArray* shape = [impl getInputShape]; - for (NSNumber* dim in shape) { - result.push_back([dim intValue]); - } - return result; -} - -std::vector ANEEncoder::get_output_shape() const { - std::vector result; - if (!impl_) return result; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - NSArray* shape = [impl getOutputShape]; - for (NSNumber* dim in shape) { - result.push_back([dim intValue]); - } - return result; -} - -__fp16* ANEEncoder::get_output_buffer() { - if (!impl_) return nullptr; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - if (!impl.hasOutputBackings || !impl.cachedOutputArray) return nullptr; - return (__fp16*)impl.cachedOutputArray.dataPointer; -} - -size_t ANEEncoder::get_output_buffer_size() const { - if (!impl_) return 0; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - if (!impl.hasOutputBackings || !impl.cachedOutputArray) return 0; - return impl.cachedOutputSize; -} - -size_t ANEEncoder::encode_multimodal_input( - const std::vector& inputs, - __fp16* output, - const std::string& output_name) { - if (!impl_ || !output || inputs.empty()) return 0; - - @autoreleasepool { - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - - NSString* outName = output_name.empty() - ? impl.modelDescription.outputDescriptionsByName.allKeys.firstObject - : [NSString stringWithUTF8String:output_name.c_str()]; - - if (!impl.multiInputPreallocated) { - [impl preallocateMultiInputBuffersWithOutputName:outName]; - } - - NSMutableDictionary* inputDict = [NSMutableDictionary new]; - - for (const auto& input : inputs) { - NSString* nsName = [NSString stringWithUTF8String:input.name.c_str()]; - - size_t total = 1; - for (int dim : input.shape) total *= dim; - - MLMultiArray* array = impl.cachedMultiInputArrays[nsName]; - if (!array) { - MLMultiArrayDataType dataType = MLMultiArrayDataTypeFloat16; - MLFeatureDescription* desc = impl.modelDescription.inputDescriptionsByName[nsName]; - if (desc && desc.multiArrayConstraint) dataType = desc.multiArrayConstraint.dataType; - - NSMutableArray* shapeArray = [NSMutableArray arrayWithCapacity:input.shape.size()]; - for (int dim : input.shape) [shapeArray addObject:@(dim)]; - - NSError* arrayError = nil; - array = [[MLMultiArray alloc] initWithShape:shapeArray dataType:dataType error:&arrayError]; - if (!array || arrayError) return 0; - } - - copyFP16ToMLArray(input.data, total, array); - inputDict[nsName] = [MLFeatureValue featureValueWithMultiArray:array]; - } - - return [impl predictMultiInputWithDict:inputDict outputData:output outputName:outName]; - } -} - -std::unique_ptr create_encoder() { - return std::make_unique(); -} - -bool is_npu_available() { - CACTUS_LOG_INFO("npu", "is_npu_available: " << (g_npu_enabled ? "true" : "false")); - return g_npu_enabled; -} - -} // namespace npu -} // namespace cactus - -@interface CactusANEPrefillImpl : NSObject - -@property (nonatomic, strong) MLModel* model; -@property (nonatomic, strong) MLModelDescription* modelDescription; -@property (nonatomic, assign) int chunkSize; -@property (nonatomic, assign) int hiddenDim; -@property (nonatomic, assign) int numLayers; -@property (nonatomic, assign) int numKvHeads; -@property (nonatomic, assign) int headDim; - -@property (nonatomic, strong) MLMultiArray* cachedInputArray; -@property (nonatomic, strong) MLMultiArray* cachedOffsetArray; -@property (nonatomic, strong) NSMutableDictionary* cachedOutputArrays; -@property (nonatomic, strong) MLPredictionOptions* predictionOptions; -@property (nonatomic, assign) BOOL hasOutputBackings; - -- (instancetype)initWithModelPath:(NSString*)path; -- (void)preallocateBuffers; -- (BOOL)predictDirectWithInput:(NSString*)inputName - data:(const __fp16*)data - offset:(int)offset; -- (MLMultiArray*)getOutputArray:(NSString*)name; -@end - -@implementation CactusANEPrefillImpl - -- (instancetype)initWithModelPath:(NSString*)path { - self = [super init]; - if (self) { - NSError* error = nil; - NSURL* modelURL = resolve_or_compile_model_url(path, &error); - - MLModelConfiguration* config = [[MLModelConfiguration alloc] init]; - config.computeUnits = resolve_compute_units_for_model(path, true); - CACTUS_LOG_INFO("npu", - "ANEPrefill compute units: " - << compute_units_to_string(config.computeUnits) - << " for " << [path UTF8String]); - - _model = [MLModel modelWithContentsOfURL:modelURL configuration:config error:&error]; - if (_model) { - _modelDescription = _model.modelDescription; - [self inferModelDimensions]; - [self preallocateBuffers]; - } - if (error) { - CACTUS_LOG_ERROR("npu", "ANE prefill model load failed: " << [[error localizedDescription] UTF8String]); - } - } - return self; -} - -- (void)inferModelDimensions { - if (!_modelDescription) return; - - NSString* inputName = _modelDescription.inputDescriptionsByName.allKeys.firstObject; - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.type == MLFeatureTypeMultiArray) { - NSArray* shape = inputDesc.multiArrayConstraint.shape; - if (shape.count >= 2) { - _chunkSize = [shape[0] intValue]; - _hiddenDim = [shape[1] intValue]; - } - } - - int maxLayerIdx = -1; - for (NSString* outputName in _modelDescription.outputDescriptionsByName.allKeys) { - if ([outputName hasPrefix:@"k_"]) { - int layerIdx = [[outputName substringFromIndex:2] intValue]; - maxLayerIdx = MAX(maxLayerIdx, layerIdx); - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - NSArray* shape = outputDesc.multiArrayConstraint.shape; - if (shape.count >= 3) { - _numKvHeads = [shape[1] intValue]; - _headDim = [shape[2] intValue]; - } - } - } - } - _numLayers = maxLayerIdx + 1; -} - -- (void)preallocateBuffers { - if (!_model || !_modelDescription || _chunkSize == 0 || _hiddenDim == 0) return; - - NSError* error = nil; - - NSArray* inputShape = @[@(_chunkSize), @(_hiddenDim)]; - _cachedInputArray = [[MLMultiArray alloc] initWithShape:inputShape - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; - if (error) { - _cachedInputArray = nil; - return; - } - - if (_modelDescription.inputDescriptionsByName[@"offset"] != nil) { - _cachedOffsetArray = [[MLMultiArray alloc] initWithShape:@[@1] - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; - if (error) { - _cachedOffsetArray = nil; - } - } - - _cachedOutputArrays = [NSMutableDictionary dictionary]; - NSMutableDictionary* outputBackings = [NSMutableDictionary dictionary]; - - for (NSString* outputName in _modelDescription.outputDescriptionsByName.allKeys) { - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - NSArray* outputShape = outputDesc.multiArrayConstraint.shape; - - MLMultiArray* outputArray = [[MLMultiArray alloc] initWithShape:outputShape - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; - if (!error && outputArray) { - _cachedOutputArrays[outputName] = outputArray; - outputBackings[outputName] = outputArray; - } - } - } - - _predictionOptions = [[MLPredictionOptions alloc] init]; - _hasOutputBackings = NO; - if (@available(macOS 14.0, iOS 17.0, *)) { - _predictionOptions.outputBackings = outputBackings; - _hasOutputBackings = YES; - } -} - -- (BOOL)predictDirectWithInput:(NSString*)inputName - data:(const __fp16*)data - offset:(int)offset { - if (!_model || !_cachedInputArray) return NO; - - NSError* error = nil; - - NSUInteger totalElements = (NSUInteger)(_chunkSize * _hiddenDim); - __fp16* inputPtr = (__fp16*)_cachedInputArray.dataPointer; - memcpy(inputPtr, data, totalElements * sizeof(__fp16)); - - MLFeatureValue* inputFeature = [MLFeatureValue featureValueWithMultiArray:_cachedInputArray]; - NSMutableDictionary* inputDict = [NSMutableDictionary dictionaryWithObject:inputFeature forKey:inputName]; - - if (_cachedOffsetArray) { - ((__fp16*)_cachedOffsetArray.dataPointer)[0] = (__fp16)offset; - MLFeatureValue* offsetFeature = [MLFeatureValue featureValueWithMultiArray:_cachedOffsetArray]; - inputDict[@"offset"] = offsetFeature; - } - - id inputProvider = [[MLDictionaryFeatureProvider alloc] - initWithDictionary:inputDict - error:&error]; - if (error) return NO; - - id outputProvider = [_model predictionFromFeatures:inputProvider - options:_predictionOptions - error:&error]; - if (error || !outputProvider) return NO; - - if (!_hasOutputBackings) { - for (NSString* outputName in _cachedOutputArrays) { - MLFeatureValue* outFeature = [outputProvider featureValueForName:outputName]; - if (!outFeature || !outFeature.multiArrayValue) return NO; - - MLMultiArray* dst = _cachedOutputArrays[outputName]; - if (!dst) return NO; - - size_t copied = copyMLArrayToFP16(outFeature.multiArrayValue, (__fp16*)dst.dataPointer); - if (copied == 0) return NO; - } - } - - return YES; -} - -- (MLMultiArray*)getOutputArray:(NSString*)name { - return _cachedOutputArrays[name]; -} - -@end - -namespace cactus { -namespace npu { - -ANEPrefill::ANEPrefill() : impl_(nullptr) {} - -ANEPrefill::~ANEPrefill() { - if (impl_) { - (void)(__bridge_transfer CactusANEPrefillImpl*)impl_; - impl_ = nullptr; - } -} - -ANEPrefill::ANEPrefill(ANEPrefill&& other) noexcept : impl_(other.impl_), - chunk_size_(other.chunk_size_), hidden_dim_(other.hidden_dim_), - num_layers_(other.num_layers_), num_kv_heads_(other.num_kv_heads_), - head_dim_(other.head_dim_) { - other.impl_ = nullptr; -} - -ANEPrefill& ANEPrefill::operator=(ANEPrefill&& other) noexcept { - if (this != &other) { - if (impl_) { - (void)(__bridge_transfer CactusANEPrefillImpl*)impl_; - } - impl_ = other.impl_; - chunk_size_ = other.chunk_size_; - hidden_dim_ = other.hidden_dim_; - num_layers_ = other.num_layers_; - num_kv_heads_ = other.num_kv_heads_; - head_dim_ = other.head_dim_; - other.impl_ = nullptr; - } - return *this; -} - -bool ANEPrefill::load(const std::string& model_path) { - @autoreleasepool { - CACTUS_LOG_INFO("npu", "ANEPrefill loading model: " << model_path); - NSString* path = [NSString stringWithUTF8String:model_path.c_str()]; - - if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { - CACTUS_LOG_DEBUG("npu", "ANEPrefill model file not found: " << model_path); - return false; - } - - CactusANEPrefillImpl* impl = [[CactusANEPrefillImpl alloc] initWithModelPath:path]; - - if (impl && impl.model) { - impl_ = (__bridge_retained void*)impl; - chunk_size_ = impl.chunkSize; - hidden_dim_ = impl.hiddenDim; - num_layers_ = impl.numLayers; - num_kv_heads_ = impl.numKvHeads; - head_dim_ = impl.headDim; - CACTUS_LOG_INFO("npu", "ANEPrefill model loaded successfully: " << model_path); - return true; - } - CACTUS_LOG_ERROR("npu", "ANEPrefill model load failed: " << model_path); - return false; - } -} - -bool ANEPrefill::is_available() const { - if (!impl_) return false; - CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; - return impl.model != nil; -} - -int ANEPrefill::get_chunk_size() const { return chunk_size_; } -int ANEPrefill::get_hidden_dim() const { return hidden_dim_; } -int ANEPrefill::get_num_layers() const { return num_layers_; } -int ANEPrefill::get_num_kv_heads() const { return num_kv_heads_; } -int ANEPrefill::get_head_dim() const { return head_dim_; } - -NPUPrefillDirectResult ANEPrefill::prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset, - const std::string& input_name) { - - NPUPrefillDirectResult result; - result.valid = false; - result.hidden = {nullptr, 0}; - result.k_caches.resize(num_layers_, {nullptr, 0}); - result.v_caches.resize(num_layers_, {nullptr, 0}); - - if (!impl_) return result; - - @autoreleasepool { - CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; - - NSString* inName = [NSString stringWithUTF8String:input_name.c_str()]; - - BOOL success = [impl predictDirectWithInput:inName - data:embeddings.data() - offset:position_offset]; - - if (!success) return result; - - MLMultiArray* hiddenArray = [impl getOutputArray:@"hidden"]; - if (hiddenArray) { - result.hidden.data = (__fp16*)hiddenArray.dataPointer; - result.hidden.count = hiddenArray.count; - } - - for (int layer = 0; layer < num_layers_; layer++) { - NSString* kName = [NSString stringWithFormat:@"k_%d", layer]; - NSString* vName = [NSString stringWithFormat:@"v_%d", layer]; - - MLMultiArray* kArray = [impl getOutputArray:kName]; - MLMultiArray* vArray = [impl getOutputArray:vName]; - - if (kArray) { - result.k_caches[layer].data = (__fp16*)kArray.dataPointer; - result.k_caches[layer].count = kArray.count; - } - if (vArray) { - result.v_caches[layer].data = (__fp16*)vArray.dataPointer; - result.v_caches[layer].count = vArray.count; - } - } - - result.valid = true; - } - - return result; -} - -std::unique_ptr create_prefill() { - return std::make_unique(); -} - -} // namespace npu -} // namespace cactus - -#else // !CACTUS_HAS_ANE - -namespace cactus { -namespace npu { - -std::unique_ptr create_encoder() { - return nullptr; -} - -bool is_npu_available() { - return false; -} - -std::unique_ptr create_prefill() { - return nullptr; -} - -} // namespace npu -} // namespace cactus - -#endif // CACTUS_HAS_ANE diff --git a/cactus-transpile/telemetry/telemetry.cpp b/cactus-transpile/telemetry/telemetry.cpp deleted file mode 100644 index f45a7431b..000000000 --- a/cactus-transpile/telemetry/telemetry.cpp +++ /dev/null @@ -1,1546 +0,0 @@ -#include "telemetry/telemetry.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if defined(__APPLE__) -#include -#include -#endif -#include -#include -#include -#include -#include - -namespace cactus { -namespace telemetry { - -enum EventType { INIT = 0, COMPLETION = 1, EMBEDDING = 2, TRANSCRIPTION = 3, STREAM_TRANSCRIBE = 4 }; - -struct Event { - EventType type; - char model[128]; - bool success; - bool cloud_handoff; - double ttft_ms; - double prefill_tps; - double decode_tps; - double tps; - double response_time_ms; - double confidence; - double ram_usage_mb; - int tokens; - int prefill_tokens; - int decode_tokens; - double session_ttft_ms; - double session_tps; - double session_time_ms; - int session_tokens; - char message[256]; - char error[256]; - char function_calls[1024]; - std::chrono::system_clock::time_point timestamp; -}; -static bool enabled = false; -static int inference_active = 0; -static bool shutdown_called = false; -static bool atexit_registered = false; -static bool curl_initialized = false; -static bool cloud_disabled = false; -static std::string supabase_url = "https://vlqqczxwyaodtcdmdmlw.supabase.co"; -static std::string supabase_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZscXFjenh3eWFvZHRjZG1kbWx3Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTE1MTg2MzIsImV4cCI6MjA2NzA5NDYzMn0.nBzqGuK9j6RZ6mOPWU2boAC_5H9XDs-fPpo5P3WZYbI"; -static std::string device_id; -static std::string project_id; -static std::string cloud_key; -static std::string project_scope; -static std::string device_model; -static std::string device_os; -static std::string device_os_version; -static std::string device_brand; -static std::string cactus_version; -#ifndef CACTUS_DEFAULT_FRAMEWORK -#define CACTUS_DEFAULT_FRAMEWORK "cpp" -#endif -static std::string framework = CACTUS_DEFAULT_FRAMEWORK; -static std::string custom_cache_location; -static std::string app_id; -static std::string device_registered_file; -static std::string project_registered_file; -static bool device_registered = false; -static bool project_registered = false; -static bool ids_ready = false; -static bool in_stream_mode = false; - -static std::mutex telemetry_mutex; -static std::condition_variable telemetry_lifecycle_cv; - -enum class TelemetryLifecycleState { - Stopped, - Running, - ShuttingDown, -}; - -static TelemetryLifecycleState lifecycle_state = TelemetryLifecycleState::Stopped; - -static bool can_record_event_locked() { - return enabled && ids_ready && lifecycle_state == TelemetryLifecycleState::Running; -} - -struct CloudSendResult { - bool payload_ok = false; - bool project_registered_ok = false; - bool device_registered_ok = false; -}; - -struct CloudConfigurationStateSnapshot { - std::string supabase_url; - std::string supabase_key; - std::string device_id; - std::string project_id; - std::string cloud_key; - std::string project_scope; - std::string framework; - std::string cactus_version; - std::string device_model; - std::string device_os; - std::string device_os_version; - std::string device_brand; - std::string app_id; - std::string device_registered_file; - std::string project_registered_file; - bool enabled = false; - bool cloud_disabled = false; - bool device_registered = false; - bool project_registered = false; -}; - -static std::string new_uuid(); -static std::string format_timestamp(const std::chrono::system_clock::time_point& tp); -static std::string model_basename(const char* model_path) { - if (!model_path) return {}; - std::string m(model_path); - size_t pos = m.find_last_of("/\\"); - if (pos != std::string::npos && pos + 1 < m.size()) { - m = m.substr(pos + 1); - } - return m; -} - -// Forward declarations for helpers used before definition -static std::string event_type_to_string(EventType t); -static bool event_type_from_string(const std::string& s, EventType& t); -static bool extract_string_field(const std::string& line, const std::string& key, std::string& out); -static bool extract_json_field(const std::string& line, const std::string& key, std::string& out); -static bool extract_bool_field(const std::string& line, const std::string& key, bool& out); -static bool extract_double_field(const std::string& line, const std::string& key, double& out); -static bool extract_int_field(const std::string& line, const std::string& key, int& out); -static bool extract_double_field_raw(const std::string& line, const std::string& key, double& out); -static void process_events(const std::vector& fresh_events); -static std::string get_telemetry_dir_locked(); -static CloudConfigurationStateSnapshot capture_cloud_configuration_state_snapshot_locked(); - -class TelemetryDispatcher { -public: - TelemetryDispatcher() = default; - - static TelemetryDispatcher& instance() { - static std::once_flag once; - static TelemetryDispatcher* singleton = nullptr; - std::call_once(once, [] { - singleton = new TelemetryDispatcher(); - }); - return *singleton; - } - - void start() { - std::lock_guard lock(queue_mutex_); - if (running_) return; - stop_ = false; - processing_ = false; - running_ = true; - worker_thread_ = std::thread([this] { worker_loop(); }); - } - - bool enqueue(const Event& event) { - std::lock_guard lock(queue_mutex_); - if (!running_) return false; - pending_events_.push_back(event); - enqueue_seq_ += 1; - queue_cv_.notify_one(); - return true; - } - - void flush() { - uint64_t target = 0; - { - std::unique_lock lock(queue_mutex_); - if (running_) { - target = enqueue_seq_; - queue_cv_.notify_one(); - flush_cv_.wait(lock, [this, target] { - return processed_seq_ >= target && !processing_; - }); - } - } - process_with_io_lock({}); - } - - void stop() { - { - std::lock_guard lock(queue_mutex_); - stop_ = true; - queue_cv_.notify_all(); - } - - if (worker_thread_.joinable()) { - worker_thread_.join(); - } - - std::lock_guard lock(queue_mutex_); - running_ = false; - stop_ = false; - processing_ = false; - pending_events_.clear(); - enqueue_seq_ = 0; - processed_seq_ = 0; - } - -private: - void process_with_io_lock(const std::vector& batch) { - std::lock_guard io_guard(io_mutex_); - process_events(batch); - } - - void worker_loop() { - while (true) { - std::vector batch; - uint64_t processed_count = 0; - { - std::unique_lock lock(queue_mutex_); - queue_cv_.wait(lock, [this] { - return stop_ || !pending_events_.empty(); - }); - if (stop_ && pending_events_.empty()) { - break; - } - batch.assign(pending_events_.begin(), pending_events_.end()); - processed_count = static_cast(batch.size()); - pending_events_.clear(); - processing_ = true; - } - - process_with_io_lock(batch); - - { - std::lock_guard lock(queue_mutex_); - processed_seq_ += processed_count; - processing_ = false; - flush_cv_.notify_all(); - } - } - - std::lock_guard lock(queue_mutex_); - running_ = false; - processing_ = false; - flush_cv_.notify_all(); - } - std::mutex queue_mutex_; - std::mutex io_mutex_; - std::condition_variable queue_cv_; - std::condition_variable flush_cv_; - std::deque pending_events_; - std::thread worker_thread_; - bool stop_ = false; - bool running_ = false; - bool processing_ = false; - uint64_t enqueue_seq_ = 0; - uint64_t processed_seq_ = 0; -}; - -static void mkdir_p(const std::string& path) { - if (path.empty()) return; - - std::string current; - for (size_t i = 0; i < path.size(); ++i) { - current += path[i]; - if (path[i] == '/' && i > 0) { - mkdir(current.c_str(), 0755); - } - } - mkdir(path.c_str(), 0755); -} - -static std::string get_telemetry_dir_locked() { - const std::string& cache_location = custom_cache_location; - if (!cache_location.empty()) { - mkdir_p(cache_location); - return cache_location; - } - -#if defined(__ANDROID__) - { - std::ifstream f("/proc/self/cmdline"); - if (f.is_open()) { - std::string pkg; - std::getline(f, pkg, '\0'); - size_t colon = pkg.find(':'); - if (colon != std::string::npos) pkg = pkg.substr(0, colon); - if (!pkg.empty()) { - uid_t uid = getuid(); - int user_id = static_cast((uid - 10000) / 100000); - std::string dir = "/data/user/" + std::to_string(user_id) + "/" + pkg + "/cache/cactus/telemetry"; - mkdir_p(dir); - struct stat st; - if (stat(dir.c_str(), &st) == 0) return dir; - dir = "/data/data/" + pkg + "/cache/cactus/telemetry"; - mkdir_p(dir); - if (stat(dir.c_str(), &st) == 0) return dir; - } - } - } -#endif - - const char* home = getenv("HOME"); - if (!home) home = "/tmp"; - std::string dir = std::string(home) + "/Library/Caches/cactus/telemetry"; - mkdir((std::string(home) + "/Library").c_str(), 0755); - mkdir((std::string(home) + "/Library/Caches").c_str(), 0755); - mkdir((std::string(home) + "/Library/Caches/cactus").c_str(), 0755); - mkdir(dir.c_str(), 0755); - return dir; -} - -static std::string cloud_api_key_cache_file_locked() { - return get_telemetry_dir_locked() + "/cloud_api_key"; -} - -static std::string load_or_create_id(const std::string& file) { - std::ifstream in(file); - if (in.is_open()) { - std::string line; - if (std::getline(in, line) && !line.empty()) { - return line; - } - } - std::string id = new_uuid(); - std::ofstream out(file, std::ios::trunc); - if (out.is_open()) { - out << id; - } - return id; -} - -static bool load_registered_flag(const std::string& file) { - std::ifstream in(file); - if (!in.is_open()) return false; - std::string line; - if (std::getline(in, line) && !line.empty()) { - return line[0] == '1'; - } - return false; -} - -static void persist_registered_flag(const std::string& file) { - std::ofstream out(file, std::ios::trunc); - if (out.is_open()) { - out << "1"; - } -} - -static std::string sysctl_string(const char* key) { -#if defined(__APPLE__) - size_t size = 0; - if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) return {}; - std::string out(size, '\0'); - if (sysctlbyname(key, out.data(), &size, nullptr, 0) != 0) return {}; - if (!out.empty() && out.back() == '\0') out.pop_back(); - return out; -#else - (void)key; - return {}; -#endif -} - -static Event make_event(EventType type, const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message) { - Event e; - e.type = type; - e.success = success; - e.cloud_handoff = false; - e.ttft_ms = ttft_ms; - e.prefill_tps = 0.0; - e.decode_tps = tps; - e.tps = tps; - e.response_time_ms = response_time_ms; - e.confidence = 0.0; - e.ram_usage_mb = 0.0; - e.tokens = tokens; - e.prefill_tokens = 0; - e.decode_tokens = tokens; - e.session_ttft_ms = 0.0; - e.session_tps = 0.0; - e.session_time_ms = 0.0; - e.session_tokens = 0; - e.timestamp = std::chrono::system_clock::now(); - std::memset(e.model, 0, sizeof(e.model)); - std::memset(e.message, 0, sizeof(e.message)); - std::memset(e.error, 0, sizeof(e.error)); - std::memset(e.function_calls, 0, sizeof(e.function_calls)); - std::string safe_model = model_basename(model); - if (!safe_model.empty()) std::strncpy(e.model, safe_model.c_str(), sizeof(e.model)-1); - if (message) std::strncpy(e.message, message, sizeof(e.message)-1); - return e; -} - -static Event make_event_extended(EventType type, const char* model, const CompletionMetrics& metrics) { - Event e; - e.type = type; - e.success = metrics.success; - e.cloud_handoff = metrics.cloud_handoff; - e.ttft_ms = metrics.ttft_ms; - e.prefill_tps = metrics.prefill_tps; - e.decode_tps = metrics.decode_tps; - e.tps = metrics.decode_tps; - e.response_time_ms = metrics.response_time_ms; - e.confidence = metrics.confidence; - e.ram_usage_mb = metrics.ram_usage_mb; - e.tokens = static_cast(metrics.prefill_tokens + metrics.decode_tokens); - e.prefill_tokens = static_cast(metrics.prefill_tokens); - e.decode_tokens = static_cast(metrics.decode_tokens); - e.session_ttft_ms = 0.0; - e.session_tps = 0.0; - e.session_time_ms = 0.0; - e.session_tokens = 0; - e.timestamp = std::chrono::system_clock::now(); - std::memset(e.model, 0, sizeof(e.model)); - std::memset(e.message, 0, sizeof(e.message)); - std::memset(e.error, 0, sizeof(e.error)); - std::memset(e.function_calls, 0, sizeof(e.function_calls)); - std::string safe_model = model_basename(model); - if (!safe_model.empty()) std::strncpy(e.model, safe_model.c_str(), sizeof(e.model)-1); - if (!metrics.success && metrics.error_message) std::strncpy(e.error, metrics.error_message, sizeof(e.error)-1); - if (metrics.function_calls_json) std::strncpy(e.function_calls, metrics.function_calls_json, sizeof(e.function_calls)-1); - return e; -} - -static bool parse_event_line(const std::string& line, Event& out) { - std::string type_str; - if (!extract_string_field(line, "event_type", type_str)) return false; - EventType et; - if (!event_type_from_string(type_str, et)) return false; - - std::string model; - extract_string_field(line, "model", model); - bool success = false; - extract_bool_field(line, "success", success); - bool cloud_handoff = false; - extract_bool_field(line, "cloud_handoff", cloud_handoff); - double ttft = 0.0; - extract_double_field(line, "ttft", ttft); - double prefill_tps = 0.0; - extract_double_field(line, "prefill_tps", prefill_tps); - double decode_tps = 0.0; - extract_double_field(line, "decode_tps", decode_tps); - double tps = 0.0; - extract_double_field(line, "tps", tps); - if (decode_tps == 0.0 && tps != 0.0) decode_tps = tps; - double response_time = 0.0; - extract_double_field(line, "response_time", response_time); - double confidence = 0.0; - extract_double_field(line, "confidence", confidence); - double ram_usage_mb = 0.0; - extract_double_field(line, "ram_usage_mb", ram_usage_mb); - int tokens = 0; - extract_int_field(line, "tokens", tokens); - int prefill_tokens = 0; - extract_int_field(line, "prefill_tokens", prefill_tokens); - int decode_tokens = 0; - extract_int_field(line, "decode_tokens", decode_tokens); - double session_ttft_ms = 0.0; - extract_double_field(line, "session_ttft", session_ttft_ms); - double session_tps = 0.0; - extract_double_field(line, "session_tps", session_tps); - double session_time_ms = 0.0; - extract_double_field(line, "session_time_ms", session_time_ms); - int session_tokens = 0; - extract_int_field(line, "session_tokens", session_tokens); - std::string message; - extract_string_field(line, "message", message); - std::string error; - extract_string_field(line, "error", error); - std::string function_calls; - extract_json_field(line, "function_calls", function_calls); - double ts_ms = 0.0; - if (!extract_double_field_raw(line, "ts_ms", ts_ms)) { - extract_double_field(line, "ts_ms", ts_ms); - } - auto ts_point = std::chrono::system_clock::now(); - if (ts_ms > 0.0) { - ts_point = std::chrono::system_clock::time_point(std::chrono::milliseconds(static_cast(ts_ms))); - } - - out = make_event(et, - model.empty() ? nullptr : model.c_str(), - success, - ttft, - tps, - response_time, - tokens, - message.empty() ? nullptr : message.c_str()); - out.timestamp = ts_point; - out.cloud_handoff = cloud_handoff; - out.prefill_tps = prefill_tps; - out.decode_tps = decode_tps; - out.confidence = confidence; - out.ram_usage_mb = ram_usage_mb; - out.prefill_tokens = prefill_tokens; - out.decode_tokens = decode_tokens; - out.session_ttft_ms = session_ttft_ms; - out.session_tps = session_tps; - out.session_time_ms = session_time_ms; - out.session_tokens = session_tokens; - if (!error.empty()) std::strncpy(out.error, error.c_str(), sizeof(out.error)-1); - if (!function_calls.empty()) std::strncpy(out.function_calls, function_calls.c_str(), sizeof(out.function_calls)-1); - return true; -} - -static std::string event_type_to_string(EventType t) { - switch (t) { - case INIT: return "init"; - case COMPLETION: return "completion"; - case EMBEDDING: return "embedding"; - case TRANSCRIPTION: return "transcription"; - case STREAM_TRANSCRIBE: return "stream_transcribe"; - default: return "unknown"; - } -} - -static bool event_type_from_string(const std::string& s, EventType& t) { - if (s == "init") { t = INIT; return true; } - if (s == "completion") { t = COMPLETION; return true; } - if (s == "embedding") { t = EMBEDDING; return true; } - if (s == "transcription") { t = TRANSCRIPTION; return true; } - if (s == "stream_transcribe") { t = STREAM_TRANSCRIBE; return true; } - return false; -} - -static bool extract_string_field(const std::string& line, const std::string& key, std::string& out) { - std::string needle = "\"" + key + "\":"; - size_t pos = line.find(needle); - if (pos == std::string::npos) return false; - pos += needle.size(); - while (pos < line.size() && line[pos] == ' ') pos++; - if (pos >= line.size()) return false; - if (line[pos] == '"') { - pos++; - size_t end = line.find('"', pos); - if (end == std::string::npos) return false; - out = line.substr(pos, end - pos); - return true; - } - size_t end = line.find_first_of(",}", pos); - if (end == std::string::npos) end = line.size(); - out = line.substr(pos, end - pos); - if (out == "null") out.clear(); - return true; -} - -static bool extract_json_field(const std::string& line, const std::string& key, std::string& out) { - std::string needle = "\"" + key + "\":"; - size_t pos = line.find(needle); - if (pos == std::string::npos) return false; - pos += needle.size(); - while (pos < line.size() && line[pos] == ' ') pos++; - if (pos >= line.size()) return false; - - char start_char = line[pos]; - if (start_char == '[' || start_char == '{') { - char end_char = (start_char == '[') ? ']' : '}'; - int depth = 0; - size_t end = pos; - bool in_string = false; - bool escape_next = false; - - for (; end < line.size(); end++) { - if (escape_next) { - escape_next = false; - continue; - } - if (line[end] == '\\') { - escape_next = true; - continue; - } - if (line[end] == '"') { - in_string = !in_string; - continue; - } - if (in_string) continue; - - if (line[end] == start_char) { - depth++; - } else if (line[end] == end_char) { - depth--; - if (depth == 0) { - out = line.substr(pos, end - pos + 1); - return true; - } - } - } - return false; - } - - size_t end = line.find_first_of(",}", pos); - if (end == std::string::npos) end = line.size(); - out = line.substr(pos, end - pos); - if (out == "null") out.clear(); - return true; -} - -static bool extract_bool_field(const std::string& line, const std::string& key, bool& out) { - std::string raw; - if (!extract_string_field(line, key, raw)) return false; - if (raw == "true") { out = true; return true; } - if (raw == "false") { out = false; return true; } - return false; -} - -static bool extract_double_field(const std::string& line, const std::string& key, double& out) { - std::string raw; - if (!extract_string_field(line, key, raw)) return false; - try { - out = std::stod(raw); - return true; - } catch (...) { - return false; - } -} - -// Helper used when the field is already numeric (no quotes) in the log cache. -static bool extract_double_field_raw(const std::string& line, const std::string& key, double& out) { - std::string needle = "\"" + key + "\":"; - size_t pos = line.find(needle); - if (pos == std::string::npos) return false; - pos += needle.size(); - while (pos < line.size() && line[pos] == ' ') pos++; - if (pos >= line.size()) return false; - size_t end = line.find_first_of(",}", pos); - if (end == std::string::npos) end = line.size(); - std::string raw = line.substr(pos, end - pos); - try { - out = std::stod(raw); - return true; - } catch (...) { - return false; - } -} - -static bool extract_int_field(const std::string& line, const std::string& key, int& out) { - std::string raw; - if (!extract_string_field(line, key, raw)) return false; - try { - out = std::stoi(raw); - return true; - } catch (...) { - return false; - } -} - -static std::string new_uuid() { - static thread_local std::mt19937_64 rng(std::random_device{}()); - uint64_t a = rng(); - uint64_t b = rng(); - a = (a & 0xffffffffffff0fffULL) | 0x0000000000004000ULL; - b = (b & 0x3fffffffffffffffULL) | 0x8000000000000000ULL; - std::ostringstream oss; - oss << std::hex << std::setfill('0'); - oss << std::setw(8) << ((a >> 32) & 0xffffffffULL); - oss << "-" << std::setw(4) << ((a >> 16) & 0xffffULL); - oss << "-" << std::setw(4) << (a & 0xffffULL); - oss << "-" << std::setw(4) << ((b >> 48) & 0xffffULL); - oss << "-" << std::setw(12) << (b & 0xffffffffffffULL); - return oss.str(); -} - -static uint64_t fnv1a_64(const std::string& s) { - uint64_t h = 14695981039346656037ULL; - for (unsigned char c : s) { h ^= c; h *= 1099511628211ULL; } - return h; -} - -static std::string uuid_from_string(const std::string& input) { - uint64_t a = fnv1a_64(input); - uint64_t b = fnv1a_64(input + "\xff"); - a = (a & 0xffffffffffff0fffULL) | 0x0000000000004000ULL; - b = (b & 0x3fffffffffffffffULL) | 0x8000000000000000ULL; - std::ostringstream oss; - oss << std::hex << std::setfill('0'); - oss << std::setw(8) << ((a >> 32) & 0xffffffffULL); - oss << "-" << std::setw(4) << ((a >> 16) & 0xffffULL); - oss << "-" << std::setw(4) << (a & 0xffffULL); - oss << "-" << std::setw(4) << ((b >> 48) & 0xffffULL); - oss << "-" << std::setw(12) << (b & 0xffffffffffffULL); - return oss.str(); -} - -static std::string read_git_remote_url() { - const char* cwd_env = std::getenv("PWD"); - if (!cwd_env) return ""; - std::string dir = cwd_env; - while (!dir.empty()) { - std::ifstream f(dir + "/.git/config"); - if (f.is_open()) { - std::string line; - bool in_origin = false; - while (std::getline(f, line)) { - if (line.find("[remote \"origin\"]") != std::string::npos) { - in_origin = true; - } else if (in_origin) { - size_t pos = line.find("url = "); - if (pos != std::string::npos) return line.substr(pos + 6); - if (!line.empty() && line[0] == '[') break; - } - } - return ""; - } - size_t pos = dir.rfind('/'); - if (pos == 0 || pos == std::string::npos) break; - dir = dir.substr(0, pos); - } - return ""; -} - -#if defined(__APPLE__) -static std::string detect_app_id() { - CFBundleRef bundle = CFBundleGetMainBundle(); - if (!bundle) return ""; - CFStringRef id = (CFStringRef)CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("CFBundleIdentifier")); - if (!id) return ""; - char buf[256]; - return CFStringGetCString(id, buf, sizeof(buf), kCFStringEncodingUTF8) ? buf : ""; -} -#elif defined(__linux__) || defined(__ANDROID__) -static std::string detect_app_id() { - std::ifstream f("/proc/self/cmdline"); - if (!f.is_open()) return ""; - std::string s; - std::getline(f, s, '\0'); - size_t colon = s.find(':'); - if (colon != std::string::npos) s = s.substr(0, colon); - return s; -} -#else -static std::string detect_app_id() { return ""; } -#endif - -static std::string extract_android_package_from_path(const std::string& path) { - for (const char* prefix : {"/data/user/", "/data/user_de/", "/data/data/"}) { - size_t pos = path.find(prefix); - if (pos == std::string::npos) continue; - size_t start = pos + strlen(prefix); - if (strcmp(prefix, "/data/data/") != 0) { - start = path.find('/', start); - if (start == std::string::npos) continue; - start++; - } - size_t end = path.find('/', start); - if (end == std::string::npos) end = path.size(); - std::string pkg = path.substr(start, end - start); - if (!pkg.empty()) return pkg; - } - return ""; -} - -static std::string format_timestamp(const std::chrono::system_clock::time_point& tp) { - using namespace std::chrono; - auto secs = time_point_cast(tp); - auto ms = duration_cast(tp - secs).count(); - std::time_t t = system_clock::to_time_t(secs); - std::tm tm{}; -#if defined(_WIN32) - gmtime_s(&tm, &t); -#else - gmtime_r(&t, &tm); -#endif - char buf[32]; - std::snprintf(buf, sizeof(buf), "%04d-%02d-%02dT%02d:%02d:%02d.%03lldZ", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec, - static_cast(ms)); - return std::string(buf); -} - -[[maybe_unused]] static bool is_valid_uuid(const std::string& s) { - // Simple check used only for validation when accepting user-provided IDs. - if (s.size() != 36) return false; - for (size_t i = 0; i < s.size(); ++i) { - char c = s[i]; - if (i == 8 || i == 13 || i == 18 || i == 23) { - if (c != '-') return false; - } else { - bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - if (!hex) return false; - } - } - return true; -} - -static std::string escape_json_string(const char* str) { - if (!str) return ""; - std::string result; - result.reserve(std::strlen(str)); - for (const char* p = str; *p; ++p) { - switch (*p) { - case '"': result += "\\\""; break; - case '\\': result += "\\\\"; break; - case '\b': result += "\\b"; break; - case '\f': result += "\\f"; break; - case '\n': result += "\\n"; break; - case '\r': result += "\\r"; break; - case '\t': result += "\\t"; break; - default: - if (static_cast(*p) < 0x20) { - char buf[7]; - snprintf(buf, sizeof(buf), "\\u%04x", static_cast(*p)); - result += buf; - } else { - result += *p; - } - break; - } - } - return result; -} - -static void collect_device_info() { - struct utsname u; - if (uname(&u) == 0) { - const std::string sysname = u.sysname; - device_os = (sysname == "Darwin") ? "macos" : sysname; - device_model = u.machine; - device_os_version = u.release; - device_brand = "unknown"; -#if defined(__APPLE__) - // Prefer user-facing macOS info over Darwin kernel version. - std::string hw_model = sysctl_string("hw.model"); - std::string os_product = sysctl_string("kern.osproductversion"); - if (!hw_model.empty()) device_model = hw_model; - if (!os_product.empty()) device_os_version = os_product; - device_brand = "apple"; -#elif defined(__ANDROID__) - device_brand = "android"; -#endif - } -} - -static bool parse_version_line(const std::string& line) { - size_t start = line.find_first_not_of(" \t\r\n"); - size_t end = line.find_last_not_of(" \t\r\n"); - if (start != std::string::npos && end != std::string::npos) { - cactus_version = line.substr(start, end - start + 1); - return true; - } - return false; -} - -static void read_cactus_version() { - const char* version_paths[] = { - "CACTUS_VERSION", - "../CACTUS_VERSION", - "../../CACTUS_VERSION", - "../../../CACTUS_VERSION" - }; - - for (const char* path : version_paths) { - std::ifstream file(path); - if (file.is_open()) { - std::string line; - if (std::getline(file, line) && parse_version_line(line)) return; - } - } - -#if defined(__APPLE__) - CFBundleRef bundle = CFBundleGetMainBundle(); - if (bundle) { - CFURLRef url = CFBundleCopyResourceURL(bundle, CFSTR("CACTUS_VERSION"), NULL, NULL); - if (url) { - char fspath[4096]; - if (CFURLGetFileSystemRepresentation(url, true, (UInt8*)fspath, sizeof(fspath))) { - std::ifstream file(fspath); - if (file.is_open()) { - std::string line; - if (std::getline(file, line) && parse_version_line(line)) { - CFRelease(url); - return; - } - } - } - CFRelease(url); - } - } -#endif - -#ifdef CACTUS_COMPILE_TIME_VERSION - cactus_version = CACTUS_COMPILE_TIME_VERSION; - return; -#endif - - cactus_version = ""; -} - -static void apply_curl_tls_trust(CURL* curl) { - if (!curl) return; - const char* ca_bundle = std::getenv("CACTUS_CA_BUNDLE"); - if (ca_bundle && ca_bundle[0] != '\0') { - curl_easy_setopt(curl, CURLOPT_CAINFO, ca_bundle); - } -#if defined(__ANDROID__) - const char* ca_path = std::getenv("CACTUS_CA_PATH"); - if (ca_path && ca_path[0] != '\0') { - curl_easy_setopt(curl, CURLOPT_CAPATH, ca_path); - } else { - curl_easy_setopt(curl, CURLOPT_CAPATH, "/system/etc/security/cacerts"); - } -#endif -} - -static bool ensure_project_row_remote(CURL* curl, const CloudConfigurationStateSnapshot& snapshot) { - if (snapshot.project_id.empty() || snapshot.project_registered) return true; - std::string url = snapshot.supabase_url + "/rest/v1/projects"; - std::ostringstream payload; - payload << "[{"; - payload << "\"project_key\":\"" << snapshot.project_id << "\""; - if (!snapshot.project_scope.empty()) { - payload << ",\"name\":\"" << snapshot.project_scope << "\""; - } - payload << "}]"; - struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, "Content-Type: application/json"); - std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; - std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; - headers = curl_slist_append(headers, apikey_hdr.c_str()); - headers = curl_slist_append(headers, auth_hdr.c_str()); - headers = curl_slist_append(headers, "Prefer: resolution=ignore-duplicates"); - headers = curl_slist_append(headers, "Content-Profile: cactus"); - headers = curl_slist_append(headers, "Accept-Profile: cactus"); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - std::string body = payload.str(); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); - apply_curl_tls_trust(curl); - CURLcode res = curl_easy_perform(curl); - long code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (headers) curl_slist_free_all(headers); - bool ok = (res == CURLE_OK) && ((code >= 200 && code < 300) || code == 409); - return ok; -} - -static bool ensure_device_row_remote(CURL* curl, const CloudConfigurationStateSnapshot& snapshot) { - if (snapshot.device_registered) return true; - if (snapshot.device_id.empty()) return false; - std::string url = snapshot.supabase_url + "/rest/v1/devices"; - std::ostringstream payload; - payload << "[{"; - payload << "\"id\":\"" << snapshot.device_id << "\""; - payload << ",\"device_id\":\"" << snapshot.device_id << "\""; - if (!snapshot.device_model.empty()) payload << ",\"model\":\"" << snapshot.device_model << "\""; - if (!snapshot.device_os.empty()) payload << ",\"os\":\"" << snapshot.device_os << "\""; - if (!snapshot.device_os_version.empty()) payload << ",\"os_version\":\"" << snapshot.device_os_version << "\""; - if (!snapshot.device_brand.empty()) payload << ",\"brand\":\"" << snapshot.device_brand << "\""; - payload << "}]"; - struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, "Content-Type: application/json"); - std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; - std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; - headers = curl_slist_append(headers, apikey_hdr.c_str()); - headers = curl_slist_append(headers, auth_hdr.c_str()); - headers = curl_slist_append(headers, "Prefer: resolution=ignore-duplicates"); - headers = curl_slist_append(headers, "Content-Profile: cactus"); - headers = curl_slist_append(headers, "Accept-Profile: cactus"); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - std::string body = payload.str(); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); - apply_curl_tls_trust(curl); - CURLcode res = curl_easy_perform(curl); - long code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (headers) curl_slist_free_all(headers); - bool ok = (res == CURLE_OK) && ((code >= 200 && code < 300) || code == 409); - return ok; -} - -static bool send_payload(CURL* curl, const std::string& url, const std::string& body, const CloudConfigurationStateSnapshot& snapshot) { - struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, "Content-Type: application/json"); - std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; - std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; - headers = curl_slist_append(headers, apikey_hdr.c_str()); - headers = curl_slist_append(headers, auth_hdr.c_str()); - headers = curl_slist_append(headers, "Prefer: return=minimal"); - headers = curl_slist_append(headers, "Content-Profile: cactus"); - headers = curl_slist_append(headers, "Accept-Profile: cactus"); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); - apply_curl_tls_trust(curl); - CURLcode res = curl_easy_perform(curl); - long code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (headers) curl_slist_free_all(headers); - return (res == CURLE_OK) && (code >= 200 && code < 300); -} - -static CloudSendResult send_batch_to_cloud(const std::vector& local, const CloudConfigurationStateSnapshot& snapshot) { - CloudSendResult result; - if (!snapshot.enabled) return result; - if (local.empty()) { - result.payload_ok = true; - return result; - } - if (snapshot.cloud_disabled) return result; - if (snapshot.device_id.empty()) return result; - CURL* curl = curl_easy_init(); - if (!curl) return result; - result.project_registered_ok = ensure_project_row_remote(curl, snapshot); - if (!snapshot.device_registered) { - result.device_registered_ok = ensure_device_row_remote(curl, snapshot); - } else { - result.device_registered_ok = true; - } - std::string url = snapshot.supabase_url + "/rest/v1/logs"; - std::ostringstream payload; - payload << "["; - for (size_t i = 0; i < local.size(); ++i) { - const Event& e = local[i]; - payload << "{"; - payload << "\"event_type\":\"" << event_type_to_string(e.type) << "\","; - payload << "\"model\":\"" << escape_json_string(e.model) << "\","; - payload << "\"success\":" << (e.success ? "true" : "false") << ","; - payload << "\"cloud_handoff\":" << (e.cloud_handoff ? "true" : "false") << ","; - if (e.type == INIT || !e.success) { - payload << "\"ttft\":null,"; - payload << "\"prefill_tps\":null,"; - payload << "\"decode_tps\":null,"; - payload << "\"tps\":null,"; - } else { - payload << "\"ttft\":" << e.ttft_ms << ","; - payload << "\"prefill_tps\":" << e.prefill_tps << ","; - payload << "\"decode_tps\":" << e.decode_tps << ","; - payload << "\"tps\":" << e.tps << ","; - } - if (!e.success) { - payload << "\"response_time\":null,"; - payload << "\"ram_usage_mb\":null,"; - } else { - payload << "\"response_time\":" << e.response_time_ms << ","; - payload << "\"ram_usage_mb\":" << e.ram_usage_mb << ","; - } - if (e.type == INIT || !e.success) { - payload << "\"confidence\":null,"; - payload << "\"tokens\":null,"; - payload << "\"prefill_tokens\":null,"; - payload << "\"decode_tokens\":null,"; - } else { - payload << "\"confidence\":" << e.confidence << ","; - payload << "\"tokens\":" << e.tokens << ","; - payload << "\"prefill_tokens\":" << e.prefill_tokens << ","; - payload << "\"decode_tokens\":" << e.decode_tokens << ","; - } - if (e.type == STREAM_TRANSCRIBE) { - payload << "\"session_ttft\":" << e.session_ttft_ms << ","; - payload << "\"session_tps\":" << e.session_tps << ","; - payload << "\"session_time_ms\":" << e.session_time_ms << ","; - payload << "\"session_tokens\":" << e.session_tokens << ","; - } else { - payload << "\"session_ttft\":null,"; - payload << "\"session_tps\":null,"; - payload << "\"session_time_ms\":null,"; - payload << "\"session_tokens\":null,"; - } - payload << "\"created_at\":\"" << format_timestamp(e.timestamp) << "\","; - if (!snapshot.project_id.empty()) { - payload << "\"project_id\":\"" << snapshot.project_id << "\","; - } - if (!snapshot.cloud_key.empty()) { - payload << "\"key_hash\":\"" << snapshot.cloud_key << "\","; - } - payload << "\"framework\":\"" << snapshot.framework << "\","; - if (!snapshot.cactus_version.empty()) { - payload << "\"framework_version\":\"" << snapshot.cactus_version << "\","; - } - payload << "\"device_id\":\"" << snapshot.device_id << "\""; - if (!snapshot.app_id.empty()) { - payload << ",\"app_id\":\"" << escape_json_string(snapshot.app_id.c_str()) << "\""; - } else { - payload << ",\"app_id\":null"; - } - if (e.message[0] != '\0') { - payload << ",\"message\":\"" << escape_json_string(e.message) << "\""; - } else { - payload << ",\"message\":null"; - } - if (e.error[0] != '\0') { - payload << ",\"error\":\"" << escape_json_string(e.error) << "\""; - } else { - payload << ",\"error\":null"; - } - payload << ",\"function_calls\":null"; - payload << "}"; - if (i + 1 < local.size()) payload << ","; - } - payload << "]"; - result.payload_ok = send_payload(curl, url, payload.str(), snapshot); - curl_easy_cleanup(curl); - return result; -} - -static void write_events_to_cache_in_dir(const std::vector& local, const std::string& dir) { - for (const auto &e : local) { - std::ostringstream oss; - oss << "{\"event_type\":\"" << event_type_to_string(e.type) << "\","; - oss << "\"model\":\"" << escape_json_string(e.model) << "\","; - oss << "\"success\":" << (e.success ? "true" : "false") << ","; - oss << "\"cloud_handoff\":" << (e.cloud_handoff ? "true" : "false") << ","; - if (e.type == INIT || !e.success) { - oss << "\"ttft\":null,"; - oss << "\"prefill_tps\":null,"; - oss << "\"decode_tps\":null,"; - oss << "\"tps\":null,"; - } else { - oss << "\"ttft\":" << e.ttft_ms << ","; - oss << "\"prefill_tps\":" << e.prefill_tps << ","; - oss << "\"decode_tps\":" << e.decode_tps << ","; - oss << "\"tps\":" << e.tps << ","; - } - if (!e.success) { - oss << "\"response_time\":null,"; - oss << "\"ram_usage_mb\":null,"; - } else { - oss << "\"response_time\":" << e.response_time_ms << ","; - oss << "\"ram_usage_mb\":" << e.ram_usage_mb << ","; - } - if (e.type == INIT || !e.success) { - oss << "\"confidence\":null,"; - oss << "\"tokens\":null,"; - oss << "\"prefill_tokens\":null,"; - oss << "\"decode_tokens\":null,"; - } else { - oss << "\"confidence\":" << e.confidence << ","; - oss << "\"tokens\":" << e.tokens << ","; - oss << "\"prefill_tokens\":" << e.prefill_tokens << ","; - oss << "\"decode_tokens\":" << e.decode_tokens << ","; - } - if (e.type == STREAM_TRANSCRIBE) { - oss << "\"session_ttft\":" << e.session_ttft_ms << ","; - oss << "\"session_tps\":" << e.session_tps << ","; - oss << "\"session_time_ms\":" << e.session_time_ms << ","; - oss << "\"session_tokens\":" << e.session_tokens; - } else { - oss << "\"session_ttft\":null,"; - oss << "\"session_tps\":null,"; - oss << "\"session_time_ms\":null,"; - oss << "\"session_tokens\":null"; - } - oss << ",\"ts_ms\":" << std::chrono::duration_cast(e.timestamp.time_since_epoch()).count(); - if (e.message[0] != '\0') { - oss << ",\"message\":\"" << escape_json_string(e.message) << "\""; - } else { - oss << ",\"message\":null"; - } - if (e.error[0] != '\0') { - oss << ",\"error\":\"" << escape_json_string(e.error) << "\""; - } else { - oss << ",\"error\":null"; - } - oss << ",\"function_calls\":null"; - if (!app_id.empty()) { - oss << ",\"app_id\":\"" << escape_json_string(app_id.c_str()) << "\""; - } else { - oss << ",\"app_id\":null"; - } - oss << "}"; - std::string file = dir + "/" + event_type_to_string(e.type) + ".log"; - std::ofstream out(file, std::ios::app); - if (out.is_open()) { - out << oss.str() << "\n"; - out.close(); - } - } -} - -static std::vector load_cached_events_in_dir(const std::string& dir) { - std::vector events; - DIR* d = opendir(dir.c_str()); - if (!d) return events; - struct dirent* ent; - while ((ent = readdir(d)) != nullptr) { - std::string name = ent->d_name; - if (name.size() < 5 || name.substr(name.size() - 4) != ".log") continue; - std::ifstream in(dir + "/" + name); - std::string line; - while (std::getline(in, line)) { - Event e; - if (parse_event_line(line, e)) { - events.push_back(e); - } - } - } - closedir(d); - return events; -} - -static void clear_cache_files_in_dir(const std::string& dir) { - DIR* d = opendir(dir.c_str()); - if (!d) return; - struct dirent* ent; - while ((ent = readdir(d)) != nullptr) { - std::string name = ent->d_name; - if (name.size() < 5 || name.substr(name.size() - 4) != ".log") continue; - std::remove((dir + "/" + name).c_str()); - } - closedir(d); -} - -static CloudConfigurationStateSnapshot capture_cloud_configuration_state_snapshot_locked() { - CloudConfigurationStateSnapshot snapshot; - snapshot.supabase_url = supabase_url; - snapshot.supabase_key = supabase_key; - snapshot.device_id = device_id; - snapshot.project_id = project_id; - snapshot.cloud_key = cloud_key; - snapshot.project_scope = project_scope; - snapshot.framework = framework; - snapshot.cactus_version = cactus_version; - snapshot.device_model = device_model; - snapshot.device_os = device_os; - snapshot.device_os_version = device_os_version; - snapshot.device_brand = device_brand; - snapshot.app_id = app_id; - snapshot.device_registered_file = device_registered_file; - snapshot.project_registered_file = project_registered_file; - snapshot.enabled = enabled; - snapshot.cloud_disabled = cloud_disabled; - snapshot.device_registered = device_registered; - snapshot.project_registered = project_registered; - return snapshot; -} - -static void process_events(const std::vector& fresh_events) { - std::vector events; - CloudConfigurationStateSnapshot snapshot; - std::string telemetry_dir; - - { - std::lock_guard guard(telemetry_mutex); - telemetry_dir = get_telemetry_dir_locked(); - events = load_cached_events_in_dir(telemetry_dir); - if (!fresh_events.empty()) { - events.insert(events.end(), fresh_events.begin(), fresh_events.end()); - } - if (events.empty()) return; - snapshot = capture_cloud_configuration_state_snapshot_locked(); - } - - CloudSendResult send_result = send_batch_to_cloud(events, snapshot); - - { - std::lock_guard guard(telemetry_mutex); - if (send_result.project_registered_ok && !project_registered) { - if (!project_registered_file.empty()) { - persist_registered_flag(project_registered_file); - } - project_registered = true; - } - if (send_result.device_registered_ok && !device_registered) { - if (!device_registered_file.empty()) { - persist_registered_flag(device_registered_file); - } - device_registered = true; - } - - if (send_result.payload_ok) { - clear_cache_files_in_dir(telemetry_dir); - } else if (!fresh_events.empty()) { - write_events_to_cache_in_dir(fresh_events, telemetry_dir); - } - } -} - -void init(const char* project_id_param, const char* project_scope_param, const char* cloud_key_param) { - std::unique_lock lifecycle_guard(telemetry_mutex); - telemetry_lifecycle_cv.wait(lifecycle_guard, [] { - return lifecycle_state != TelemetryLifecycleState::ShuttingDown; - }); - - std::string scope = project_scope_param ? project_scope_param : "default"; - - bool cloud_disabled_from_env = false; - if (const char* env = std::getenv("CACTUS_NO_CLOUD_TELE")) { - if (env[0] != '\0' && !(env[0] == '0' && env[1] == '\0')) { - cloud_disabled_from_env = true; - } - } - const char* env_url = std::getenv("CACTUS_SUPABASE_URL"); - const char* env_key = std::getenv("CACTUS_SUPABASE_KEY"); - const char* env_project = std::getenv("CACTUS_PROJECT_ID"); - const char* env_cloud = std::getenv("CACTUS_CLOUD_KEY"); - - std::string dir = get_telemetry_dir_locked(); - - std::string device_file = dir + "/device_id"; - std::string device_flag_file = dir + "/device_registered"; - bool device_was_registered = load_registered_flag(device_flag_file); - std::string resolved_device_id = load_or_create_id(device_file); - - std::string resolved_project_id; - if (project_id_param && *project_id_param) { - resolved_project_id = project_id_param; - } else if (env_project && *env_project) { - resolved_project_id = env_project; - } else { - std::string remote_url = read_git_remote_url(); - resolved_project_id = remote_url.empty() ? resolved_device_id : uuid_from_string(remote_url); - } - - std::string project_flag_file = dir + "/project_reg_" + resolved_project_id.substr(0, 8); - bool project_was_registered = load_registered_flag(project_flag_file); - - std::string resolved_cloud_key; - if (cloud_key_param && *cloud_key_param) { - resolved_cloud_key = cloud_key_param; - } else if (env_cloud && *env_cloud) { - resolved_cloud_key = env_cloud; - } - - collect_device_info(); - if (cactus_version.empty()) read_cactus_version(); - - if (env_url && *env_url) supabase_url = env_url; - if (env_key && *env_key) supabase_key = env_key; - project_id = resolved_project_id; - project_scope = scope; - project_registered_file = project_flag_file; - cloud_key = resolved_cloud_key; - device_registered_file = device_flag_file; - device_id = resolved_device_id; - if (app_id.empty()) app_id = detect_app_id(); - - project_registered = project_was_registered; - device_registered = device_was_registered; - if (cloud_disabled_from_env) { - cloud_disabled = true; - } - - if (!curl_initialized) { - if (curl_global_init(CURL_GLOBAL_DEFAULT) == CURLE_OK) { - curl_initialized = true; - } - } - - if (!atexit_registered) { - atexit_registered = true; - std::atexit([](){ shutdown(); }); - } - - shutdown_called = false; - ids_ready = true; - enabled = true; - lifecycle_state = TelemetryLifecycleState::Running; - TelemetryDispatcher::instance().start(); -} - -void setEnabled(bool en) { - std::lock_guard guard(telemetry_mutex); - enabled = en; -} - -void setCloudDisabled(bool disabled) { - std::lock_guard guard(telemetry_mutex); - cloud_disabled = disabled; -} - -void setAppId(const char* id) { - std::lock_guard guard(telemetry_mutex); - if (id && id[0] != '\0') app_id = id; -} - -void setTelemetryEnvironment(const char* framework_str, const char* cache_location_str, const char* version_str) { - std::lock_guard guard(telemetry_mutex); - if (framework_str && framework_str[0] != '\0') { - framework = framework_str; - } - if (cache_location_str && cache_location_str[0] != '\0') { - custom_cache_location = cache_location_str; - if (app_id.empty()) { - std::string pkg = extract_android_package_from_path(custom_cache_location); - if (!pkg.empty()) app_id = pkg; - } - } - if (version_str && version_str[0] != '\0') { - cactus_version = version_str; - } -} - -void setCloudKey(const char* key) { - if (!key || key[0] == '\0') return; - std::lock_guard guard(telemetry_mutex); - cloud_key = key; -} - -void cacheCloudApiKey(const char* key) { - if (!key || key[0] == '\0') return; - std::lock_guard guard(telemetry_mutex); - const std::string file = cloud_api_key_cache_file_locked(); - std::ofstream out(file, std::ios::trunc); - if (out.is_open()) { - out << key; - } -} - -std::string loadCachedCloudApiKey() { - std::lock_guard guard(telemetry_mutex); - const std::string file = cloud_api_key_cache_file_locked(); - std::ifstream in(file); - if (!in.is_open()) return {}; - std::string line; - if (std::getline(in, line) && !line.empty()) { - return line; - } - return {}; -} - -void recordInit(const char* model, bool success, double response_time_ms, const char* message) { - double nan = std::numeric_limits::quiet_NaN(); - Event e = make_event(INIT, model, success, nan, nan, response_time_ms, 0, message); - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordCompletion(const char* model, const CompletionMetrics& metrics) { - Event e = make_event_extended(COMPLETION, model, metrics); - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordCompletion(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message) { - Event e = make_event(COMPLETION, model, success, ttft_ms, tps, response_time_ms, tokens, message); - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordEmbedding(const char* model, bool success, const char* message) { - Event e = make_event(EMBEDDING, model, success, 0.0, 0.0, 0.0, 0, message); - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double ram_usage_mb, const char* message) { - std::lock_guard guard(telemetry_mutex); - if (in_stream_mode) return; - Event e = make_event(TRANSCRIPTION, model, success, ttft_ms, tps, response_time_ms, tokens, message); - e.ram_usage_mb = ram_usage_mb; - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordStreamTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double session_ttft_ms, double session_tps, double session_time_ms, int session_tokens, const char* message) { - Event e = make_event(STREAM_TRANSCRIBE, model, success, ttft_ms, tps, response_time_ms, tokens, message); - e.session_ttft_ms = session_ttft_ms; - e.session_tps = session_tps; - e.session_time_ms = session_time_ms; - e.session_tokens = session_tokens; - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void setStreamMode(bool in_stream) { - std::lock_guard guard(telemetry_mutex); - in_stream_mode = in_stream; -} - -bool isStreamMode() { - std::lock_guard guard(telemetry_mutex); - return in_stream_mode; -} - -void markInference(bool active) { - std::lock_guard guard(telemetry_mutex); - if (active) { - inference_active += 1; - } else if (inference_active > 0) { - inference_active -= 1; - } -} - -void flush() { - TelemetryDispatcher::instance().flush(); -} - -void shutdown() { - { - std::lock_guard lifecycle_guard(telemetry_mutex); - if (shutdown_called) { - return; - } - - shutdown_called = true; - lifecycle_state = TelemetryLifecycleState::ShuttingDown; - } - - flush(); - - { - std::lock_guard lifecycle_guard(telemetry_mutex); - enabled = false; - ids_ready = false; - } - - TelemetryDispatcher::instance().stop(); - - { - std::lock_guard lifecycle_guard(telemetry_mutex); - if (curl_initialized) { - curl_initialized = false; - curl_global_cleanup(); - } - lifecycle_state = TelemetryLifecycleState::Stopped; - telemetry_lifecycle_cv.notify_all(); - } -} - -} // namespace telemetry -} // namespace cactus diff --git a/cactus-transpile/telemetry/telemetry.h b/cactus-transpile/telemetry/telemetry.h deleted file mode 100644 index cd7b49f45..000000000 --- a/cactus-transpile/telemetry/telemetry.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once -#include -#include - -namespace cactus { -namespace telemetry { - -struct CompletionMetrics { - bool success; - bool cloud_handoff; - double ttft_ms; - double prefill_tps; - double decode_tps; - double response_time_ms; - double confidence; - double ram_usage_mb; - size_t prefill_tokens; - size_t decode_tokens; - const char* error_message; - const char* function_calls_json; -}; - -void init(const char* project_id = nullptr, const char* project_scope = nullptr, const char* cloud_key = nullptr); -void setEnabled(bool enabled); -void setCloudDisabled(bool disabled); -void setAppId(const char* app_id); -void setTelemetryEnvironment(const char* framework, const char* cache_location, const char* version = nullptr); -void setCloudKey(const char* key); -void cacheCloudApiKey(const char* key); -std::string loadCachedCloudApiKey(); -void recordInit(const char* model, bool success, double response_time_ms, const char* message); -void recordCompletion(const char* model, const CompletionMetrics& metrics); -void recordCompletion(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message); -void recordEmbedding(const char* model, bool success, const char* message); -void recordTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double ram_usage_mb, const char* message); -void recordStreamTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double session_ttft_ms, double session_tps, double session_time_ms, int session_tokens, const char* message); -void setStreamMode(bool in_stream); -bool isStreamMode(); -void markInference(bool active); -void flush(); -void shutdown(); - -} // namespace telemetry -} // namespace cactus From 82486f657686d7b07f860cb855ca35497efbd6d7 Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Tue, 12 May 2026 09:27:26 -0700 Subject: [PATCH 07/11] Python changes --- python/src/cli.py | 55 ++++++++++++----------------------------------- 1 file changed, 14 insertions(+), 41 deletions(-) diff --git a/python/src/cli.py b/python/src/cli.py index fbb10f2f8..52ac64b1c 100644 --- a/python/src/cli.py +++ b/python/src/cli.py @@ -48,57 +48,30 @@ def _resolve_project_root() -> Path: WEIGHTS_VARIANT_CHOICES = ["auto", "apple", "standard"] -def _transpile_runtime_root() -> Path: - return PROJECT_ROOT / "cactus-transpile" - - -def _transpile_runtime_library_path() -> Path: +def _python_runtime_library_path() -> Path: suffix = ".dylib" if platform.system() == "Darwin" else ".so" - return _transpile_runtime_root() / "build" / f"libcactus{suffix}" + return PROJECT_ROOT / "cactus" / "build" / f"libcactus{suffix}" -def _ensure_transpile_runtime_library() -> Path: - runtime_root = _transpile_runtime_root() - cmake_lists = runtime_root / "CMakeLists.txt" - if not cmake_lists.exists(): +def _ensure_python_runtime_library() -> Path: + build_script = PROJECT_ROOT / "cactus" / "build.sh" + if not build_script.exists(): raise RuntimeError( - "The vendored transpiler runtime is missing.\n" - f"Expected: {cmake_lists}" + "The Cactus build script is missing.\n" + f"Expected: {build_script}" ) - library_path = _transpile_runtime_library_path() + library_path = _python_runtime_library_path() if library_path.exists(): return library_path - build_dir = runtime_root / "build" - build_dir.mkdir(parents=True, exist_ok=True) - jobs = str(os.cpu_count() or 4) - - print_color(YELLOW, "Building vendored transpiler runtime...") - configure = subprocess.run( - [ - "cmake", - "-S", - str(runtime_root), - "-B", - str(build_dir), - "-DCMAKE_RULE_MESSAGES=OFF", - "-DCMAKE_VERBOSE_MAKEFILE=OFF", - ], - cwd=PROJECT_ROOT, - ) - if configure.returncode != 0: - raise RuntimeError("Failed to configure the vendored transpiler runtime") - - build = subprocess.run( - ["cmake", "--build", str(build_dir), "-j", jobs], - cwd=PROJECT_ROOT, - ) + print_color(YELLOW, "Building Cactus shared runtime for transpiler...") + build = subprocess.run([str(build_script)], cwd=PROJECT_ROOT / "cactus") if build.returncode != 0: - raise RuntimeError("Failed to build the vendored transpiler runtime") + raise RuntimeError("Failed to build the Cactus shared runtime") if not library_path.exists(): raise RuntimeError( - "The vendored transpiler runtime build completed, but the library was not produced.\n" + "The Cactus build completed, but the shared library was not produced.\n" f"Expected: {library_path}" ) return library_path @@ -1095,7 +1068,7 @@ def _resolve_transpiled_manifest(path_value): def cmd_run_transpiled(args): """Run a saved transpiled component bundle.""" - transpile_lib = _ensure_transpile_runtime_library() + transpile_lib = _ensure_python_runtime_library() os.environ["CACTUS_LIB_PATH"] = str(transpile_lib) from .transpile.component_bundle_runtime import run_transpiled_bundle @@ -1123,7 +1096,7 @@ def cmd_transpile(args): print_color(RED, f"Error: transpiler script not found at {script_path}") return 1 - transpile_lib = _ensure_transpile_runtime_library() + transpile_lib = _ensure_python_runtime_library() command = [sys.executable, str(script_path), "--model-id", args.model_id] command.extend(getattr(args, "extra_args", []) or []) env = os.environ.copy() From cc5c318273f26e68d2ca0e6ce306d743b29d4851 Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Tue, 12 May 2026 16:29:21 -0700 Subject: [PATCH 08/11] CQ weights adoption --- python/examples/transpile_hf_model.py | 109 +++- python/src/cli.py | 52 +- .../src/transpile/component_bundle_runtime.py | 357 +++++++++++- python/src/transpile/lower.py | 158 +++++- python/src/transpile/model_adapters.py | 70 +-- python/src/transpile/optimize_graph.py | 119 +++- python/src/transpile/runtime_compat.py | 340 ++++++++++++ python/src/transpile/weight_binding.py | 58 +- python/src/transpile/weight_compat.py | 520 ++++++++++++++++++ .../tests/test_optimize_gemma4_attention.py | 104 ++++ python/tests/test_transpile_weight_compat.py | 80 +++ 11 files changed, 1834 insertions(+), 133 deletions(-) create mode 100644 python/src/transpile/runtime_compat.py create mode 100644 python/src/transpile/weight_compat.py create mode 100644 python/tests/test_optimize_gemma4_attention.py create mode 100644 python/tests/test_transpile_weight_compat.py diff --git a/python/examples/transpile_hf_model.py b/python/examples/transpile_hf_model.py index fc3664671..9ae49f42c 100644 --- a/python/examples/transpile_hf_model.py +++ b/python/examples/transpile_hf_model.py @@ -21,7 +21,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from src.graph import Graph +from src.transpile.runtime_compat import Graph from src.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features from src.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform from src.transpile.audio_preprocess import prepare_cactus_audio_features @@ -153,11 +153,17 @@ def _validate_weights_dir(weights_dir: str | None, *, model_id: str) -> Path | N manifest_path = root / "weights_manifest.json" if not manifest_path.exists(): - raise RuntimeError( - f"weights_dir is missing weights_manifest.json: {manifest_path}\n" - "\n" - f"Re-convert with the current converter:\n" - f" cactus convert {model_id} {root}\n" + has_weight_files = any(root.glob("*.weights")) + if not has_weight_files: + raise RuntimeError( + f"weights_dir is missing weights_manifest.json: {manifest_path}\n" + "\n" + f"Re-convert with the current converter:\n" + f" cactus convert {model_id} {root}\n" + ) + print( + "note=weights_manifest_missing using filename-based weight binding fallback " + f"for {root}" ) return root @@ -253,6 +259,17 @@ def _binding_entries_by_node_id( return result +def _binding_entries_by_value_id( + bindings: list[dict[str, object]], +) -> dict[str, dict[str, object]]: + result: dict[str, dict[str, object]] = {} + for binding in bindings: + value_id = binding.get("value_id") + if isinstance(value_id, str) and value_id: + result[value_id] = binding + return result + + def _write_graph_binding_manifest( *, artifact_dir: Path, @@ -360,27 +377,65 @@ def _write_component_bundle( if transpiled_graph is not None: existing_bindings = list(_serialize_json_compatible(transpiled_graph.bound_constant_bindings)) existing_by_node_id = _binding_entries_by_node_id(existing_bindings) + existing_by_value_id = _binding_entries_by_value_id(existing_bindings) + constant_value_ids = dict(getattr(transpiled_graph, "bound_constant_value_ids", {})) + value_to_node_id = {value_id: node_id for node_id, value_id in constant_value_ids.items()} constant_dir = component_dir / "bound_constants" for constant_tensor in transpiled_graph.bound_constants: - if int(constant_tensor.id) in existing_by_node_id: + node_id = int(constant_tensor.id) + if node_id in existing_by_node_id: continue constant_dir.mkdir(parents=True, exist_ok=True) - constant_filename = f"node_{int(constant_tensor.id)}.npy" + constant_filename = f"node_{node_id}.npy" constant_relpath = Path(component) / "bound_constants" / constant_filename constant_array = constant_tensor.numpy().copy() np.save(bundle_dir / constant_relpath, constant_array, allow_pickle=False) materialized_constant_bindings.append( { - "node_id": int(constant_tensor.id), - "value_id": f"materialized_constant_{int(constant_tensor.id)}", + "node_id": node_id, + "value_id": str(constant_value_ids.get(node_id, f"materialized_constant_{node_id}")), "path": str((bundle_dir / constant_relpath).relative_to(artifact_dir)), "kind": "saved_constant", - "source_name": f"materialized_constant_{int(constant_tensor.id)}", + "source_name": str(constant_value_ids.get(node_id, f"materialized_constant_{node_id}")), "format": "npy", "precision": int(constant_tensor.dtype), } ) + graph_constants = (optimized_graph or raw_graph).constants if (optimized_graph or raw_graph) is not None else {} + for value_id, constant_value in graph_constants.items(): + if value_id in existing_by_value_id: + continue + if any(binding.get("value_id") == value_id for binding in materialized_constant_bindings): + continue + if isinstance(constant_value, torch.Tensor): + constant_array = constant_value.detach().cpu().numpy().copy() + elif isinstance(constant_value, np.ndarray): + constant_array = np.ascontiguousarray(constant_value) + else: + continue + constant_dir.mkdir(parents=True, exist_ok=True) + safe_value_id = re.sub(r"[^A-Za-z0-9._-]+", "_", value_id).strip("._-") or "constant" + constant_filename = f"{safe_value_id}.npy" + constant_relpath = Path(component) / "bound_constants" / constant_filename + np.save(bundle_dir / constant_relpath, constant_array, allow_pickle=False) + materialized_constant_bindings.append( + { + "node_id": int(value_to_node_id.get(value_id, -1)), + "value_id": str(value_id), + "path": str((bundle_dir / constant_relpath).relative_to(artifact_dir)), + "kind": "saved_constant", + "source_name": str(value_id), + "format": "npy", + "precision": int( + Graph.FP16 if constant_array.dtype == np.float16 + else Graph.FP32 if constant_array.dtype == np.float32 + else Graph.INT8 if constant_array.dtype == np.int8 + else Graph.INT8 + ), + } + ) + graph_for_signature = optimized_graph or raw_graph if graph_for_signature is None: continue @@ -1882,14 +1937,30 @@ def _prepare_gemma4_multimodal_inputs( if audio_file and isinstance(batch.get("input_features"), torch.Tensor): feature_tensor = batch["input_features"] expected_mels = int(feature_tensor.shape[-1]) - native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( - audio_file, - expected_mels=expected_mels, - torch_dtype=torch_dtype, - ) - batch["input_features"] = native_audio - batch["input_features_mask"] = native_audio_mask - batch["native_audio_frames"] = native_audio_frames + try: + native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( + audio_file, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + except Exception as exc: + print(f"note=falling back to processor gemma4 audio features: {exc}") + batch["input_features"] = feature_tensor.to(dtype=torch_dtype) + fallback_mask = batch.get("input_features_mask") + if isinstance(fallback_mask, torch.Tensor): + batch["input_features_mask"] = fallback_mask.to(dtype=torch.bool) + native_audio_frames = int(fallback_mask.to(dtype=torch.int32).sum().item()) + else: + native_audio_frames = int(feature_tensor.shape[1]) + batch["input_features_mask"] = torch.ones( + (int(feature_tensor.shape[0]), native_audio_frames), + dtype=torch.bool, + ) + batch["native_audio_frames"] = native_audio_frames + else: + batch["input_features"] = native_audio + batch["input_features_mask"] = native_audio_mask + batch["native_audio_frames"] = native_audio_frames ordered_keys = [ key diff --git a/python/src/cli.py b/python/src/cli.py index 201c6ef4a..1682e82c8 100644 --- a/python/src/cli.py +++ b/python/src/cli.py @@ -46,6 +46,15 @@ def _resolve_project_root() -> Path: DEFAULT_MODEL_ID = "google/gemma-4-E2B-it" DEFAULT_TEST_MODEL_ID = "google/gemma-4-E2B-it" WEIGHTS_VARIANT_CHOICES = ["auto", "apple", "standard"] +MODEL_ID_ALIASES = { + "gemma4": DEFAULT_MODEL_ID, + "gemma4-e2b": DEFAULT_MODEL_ID, + "parakeet": "nvidia/parakeet-tdt-0.6b-v3", + "parakeet-tdt": "nvidia/parakeet-tdt-0.6b-v3", + "whisper": "openai/whisper-small", + "qwen": "Qwen/Qwen3-1.7B", + "lfm": "LiquidAI/LFM2-350M", +} def _python_runtime_library_path() -> Path: @@ -95,6 +104,11 @@ def print_color(color, message): print(f"{color}{message}{NC}") +def resolve_model_id_alias(model_id: str) -> str: + normalized = (model_id or "").strip() + return MODEL_ID_ALIASES.get(normalized.lower(), normalized) + + from .downloads import get_model_dir_name, get_weights_dir, download_from_hf as _download_from_hf_impl from .cq_download import ( combo_label, @@ -1128,12 +1142,25 @@ def cmd_run_transpiled(args): from .transpile.component_bundle_runtime import run_transpiled_bundle bundle_dir = getattr(args, "bundle_dir", None) or getattr(args, "model_id", None) + image_values: list[str] = [] + for attr_name in ("image_file", "image_files"): + value = getattr(args, attr_name, None) + if isinstance(value, str) and value.strip(): + image_values.append(value.strip()) + elif isinstance(value, (list, tuple)): + image_values.extend(str(item).strip() for item in value if str(item).strip()) + image_arg = getattr(args, "image", None) + if isinstance(image_arg, str) and image_arg.strip(): + image_values.append(image_arg.strip()) result = run_transpiled_bundle( bundle_dir, audio_file=getattr(args, "audio_file", None) or getattr(args, "audio", None), + image_files=tuple(image_values), prompt=getattr(args, "prompt", None), input_ids=getattr(args, "input_ids", None), weights_dir=getattr(args, "weights_dir", None), + system_prompt=getattr(args, "system", None), + enable_thinking=bool(getattr(args, "thinking", False)), ) print(json.dumps(result, indent=2, sort_keys=True)) if getattr(args, "result_json", None): @@ -1152,8 +1179,14 @@ def cmd_transpile(args): return 1 transpile_lib = _ensure_python_runtime_library() - command = [sys.executable, str(script_path), "--model-id", args.model_id] - command.extend(getattr(args, "extra_args", []) or []) + model_id = resolve_model_id_alias(args.model_id) + extra_args = list(getattr(args, "extra_args", []) or []) + command = [sys.executable, str(script_path), "--model-id", model_id] + if "--weights-dir" not in extra_args: + default_weights_dir = get_weights_dir(model_id) + if default_weights_dir.exists(): + command.extend(["--weights-dir", str(default_weights_dir)]) + command.extend(extra_args) env = os.environ.copy() env["CACTUS_LIB_PATH"] = str(transpile_lib) result = subprocess.run(command, cwd=PROJECT_ROOT, env=env) @@ -1162,7 +1195,8 @@ def cmd_transpile(args): def cmd_run(args): """Download model if needed and start interactive chat.""" - manifest_path = _resolve_transpiled_manifest(args.model_id) + model_id_or_path = getattr(args, "model_id", DEFAULT_MODEL_ID) + manifest_path = _resolve_transpiled_manifest(model_id_or_path) if manifest_path is not None: args.bundle_dir = str(manifest_path.parent.parent if manifest_path.parent.name == "components" else manifest_path.parent) return cmd_run_transpiled(args) @@ -1175,7 +1209,7 @@ def cmd_run(args): if api_key: os.environ["CACTUS_CLOUD_KEY"] = api_key - model_id = args.model_id + model_id = resolve_model_id_alias(args.model_id) if getattr(args, 'no_cloud_tele', False): os.environ["CACTUS_NO_CLOUD_TELE"] = "1" @@ -2257,13 +2291,17 @@ def create_parser(): help='Enable thinking/reasoning for models that support it') transpile_parser = subparsers.add_parser('transpile', help='Transpile a HuggingFace model into Cactus component graphs') - transpile_parser.add_argument('model_id', help='HuggingFace model ID to transpile') + transpile_parser.add_argument('model_id', help='HuggingFace model ID to transpile (aliases: gemma4, parakeet, whisper, qwen, lfm)') run_transpiled_parser = subparsers.add_parser('run-transpiled', help='Run a saved transpiled component bundle') run_transpiled_parser.add_argument('bundle_dir', help='Bundle root directory or manifest.json path from transpile_hf_model.py') run_transpiled_parser.add_argument('--file', dest='audio_file', default=None, help='Audio file for transcription bundles such as parakeet-tdt') + run_transpiled_parser.add_argument('--image', default=None, + help='Image file for multimodal transpiled bundles') + run_transpiled_parser.add_argument('--image-file', action='append', default=[], + help='Repeatable image input for multimodal transpiled bundles') run_transpiled_parser.add_argument('--weights-dir', default=None, help='Optional converted weights directory for bound-weight bundles') run_transpiled_parser.add_argument('--prompt', default=None, @@ -2272,6 +2310,10 @@ def create_parser(): help='Comma-separated token ids for causal-LM bundles') run_transpiled_parser.add_argument('--result-json', default=None, help='Optional path to save the result payload as JSON') + run_transpiled_parser.add_argument('--system', default=None, + help='Optional system prompt for multimodal transpiled bundles') + run_transpiled_parser.add_argument('--thinking', action='store_true', + help='Enable thinking/reasoning markers when supported by the transpiled bundle') transcribe_parser = subparsers.add_parser('transcribe', help='Download ASR model and run transcription') transcribe_parser.add_argument('model_id', nargs='?', default=DEFAULT_ASR_MODEL_ID, diff --git a/python/src/transpile/component_bundle_runtime.py b/python/src/transpile/component_bundle_runtime.py index 901b9eeb9..54a58e7bd 100644 --- a/python/src/transpile/component_bundle_runtime.py +++ b/python/src/transpile/component_bundle_runtime.py @@ -13,19 +13,25 @@ import numpy as np import torch -from src.cactus import _lib -from src.cactus import cactus_node_t -from src.graph import Graph -from src.graph import Tensor from src.tensor_io import CACTUS_MAGIC from src.tensor_io import FLAG_INTERLEAVED from src.tensor_io import align_offset from src.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features from src.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform from src.transpile.audio_preprocess import prepare_cactus_audio_features +from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.lower import transpile_preoptimized_ir +from src.transpile.optimize_graph import optimize_graph from src.transpile.parakeet_tdt_local import greedy_decode_parakeet_tdt_token_ids from src.transpile.parakeet_tdt_local import load_parakeet_tdt_config from src.transpile.parakeet_tdt_local import prepare_parakeet_tdt_audio_features +from src.transpile.runtime_compat import _lib +from src.transpile.runtime_compat import cactus_node_t +from src.transpile.runtime_compat import Graph +from src.transpile.runtime_compat import Tensor _HEADER_SIZE = 84 @@ -100,6 +106,17 @@ def load_saved_component_graph( weights_dir: str | Path | None = None, ) -> LoadedComponentGraph: root = Path(bundle_root).expanduser().resolve() + raw_ir_relpath = component_entry.get("raw_ir") + optimized_ir_relpath = component_entry.get("optimized_ir") + has_raw_ir = isinstance(raw_ir_relpath, str) and bool(raw_ir_relpath) + has_optimized_ir = isinstance(optimized_ir_relpath, str) and bool(optimized_ir_relpath) + if has_raw_ir or has_optimized_ir: + return _load_component_graph_from_ir( + bundle_root=root, + component_entry=component_entry, + weights_dir=weights_dir, + ) + graph_relpath = component_entry.get("graph") if not isinstance(graph_relpath, str) or not graph_relpath: raise ValueError(f"component entry is missing graph path: {component_entry}") @@ -132,6 +149,44 @@ def load_saved_component_graph( ) +def _load_component_graph_from_ir( + *, + bundle_root: Path, + component_entry: dict[str, object], + weights_dir: str | Path | None, +) -> LoadedComponentGraph: + raw_ir_relpath = component_entry.get("raw_ir") + optimized_ir_relpath = component_entry.get("optimized_ir") + has_raw_ir = isinstance(raw_ir_relpath, str) and bool(raw_ir_relpath) + has_optimized_ir = isinstance(optimized_ir_relpath, str) and bool(optimized_ir_relpath) + use_raw_ir = has_raw_ir and not has_optimized_ir + ir_relpath = str(raw_ir_relpath if use_raw_ir else optimized_ir_relpath) + ir_path = (bundle_root / ir_relpath).resolve() + payload = json.loads(ir_path.read_text()) + graph_payload = payload.get("graph") + if not isinstance(graph_payload, dict): + raise ValueError(f"saved IR payload is missing graph data: {ir_path}") + + ir_graph = _deserialize_saved_ir_graph( + graph_payload=graph_payload, + component_entry=component_entry, + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + if use_raw_ir: + canonicalize_exported_graph(ir_graph) + optimize_graph(ir_graph) + transpiled = transpile_preoptimized_ir(ir_graph) + return LoadedComponentGraph( + component=str(component_entry.get("component", "unknown")), + graph=transpiled.graph, + runtime_inputs=list(transpiled.runtime_inputs), + outputs=list(transpiled.outputs), + bound_constant_bindings=list(component_entry.get("bound_constant_bindings") or []), + bound_tensor_files=[], + ) + + def load_saved_component_graphs( bundle_dir_or_manifest: str | Path, *, @@ -157,14 +212,19 @@ def run_transpiled_bundle( bundle_dir_or_manifest: str | Path, *, audio_file: str | Path | None = None, + image_files: tuple[str, ...] = (), prompt: str | None = None, input_ids: str | list[int] | tuple[int, ...] | None = None, weights_dir: str | Path | None = None, torch_dtype: torch.dtype = torch.float16, + system_prompt: str | None = None, + enable_thinking: bool = False, ) -> dict[str, object]: + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + resolved_weights_dir = _default_weights_dir_for_manifest(manifest, explicit=weights_dir) component_graphs, manifest = load_saved_component_graphs( bundle_dir_or_manifest, - weights_dir=weights_dir, + weights_dir=resolved_weights_dir, ) family = str(manifest.get("family", "") or "") task = str(manifest.get("task", "") or "") @@ -177,7 +237,18 @@ def run_transpiled_bundle( audio_file=audio_file, torch_dtype=torch_dtype, ) - if task in {"causal_lm_logits", "multimodal_causal_lm_logits"}: + if task == "multimodal_causal_lm_logits": + return _run_multimodal_causal_lm_bundle( + component_graphs=component_graphs, + manifest=manifest, + prompt=prompt, + image_files=image_files, + audio_file=audio_file, + torch_dtype=torch_dtype, + system_prompt=system_prompt, + enable_thinking=enable_thinking, + ) + if task == "causal_lm_logits": return _run_causal_lm_logits_bundle( component_graphs=component_graphs, manifest=manifest, @@ -204,6 +275,166 @@ def run_transpiled_bundle( ) +def _default_weights_dir_for_manifest( + manifest: Mapping[str, object], + *, + explicit: str | Path | None, +) -> str | Path | None: + if explicit is not None: + return explicit + model_id = str(manifest.get("model_id", "") or "").strip() + if not model_id: + return None + try: + from src.downloads import get_weights_dir + + candidate = get_weights_dir(model_id) + except Exception: + return None + return candidate if candidate.exists() else None + + +def _deserialize_saved_ir_graph( + *, + graph_payload: Mapping[str, object], + component_entry: Mapping[str, object], + bundle_root: Path, + weights_dir: str | Path | None, +) -> IRGraph: + values_payload = graph_payload.get("values") + nodes_payload = graph_payload.get("nodes") + if not isinstance(values_payload, dict) or not isinstance(nodes_payload, list): + raise ValueError("saved optimized IR graph is missing values or nodes payload") + + values: dict[str, IRValue] = {} + for value_id, raw_value in values_payload.items(): + if not isinstance(value_id, str) or not isinstance(raw_value, dict): + continue + shape = raw_value.get("shape") + values[value_id] = IRValue( + id=str(raw_value.get("id", value_id)), + shape=None if shape is None else tuple(int(dim) for dim in shape), + dtype=None if raw_value.get("dtype") is None else str(raw_value.get("dtype")), + producer=None if raw_value.get("producer") is None else str(raw_value.get("producer")), + users=[str(item) for item in raw_value.get("users", [])], + meta=dict(raw_value.get("meta") or {}), + ) + + nodes: dict[str, IRNode] = {} + order: list[str] = [] + for raw_node in nodes_payload: + if not isinstance(raw_node, dict): + continue + node_id = str(raw_node["id"]) + node = IRNode( + id=node_id, + op=str(raw_node["op"]), + inputs=[str(item) for item in raw_node.get("inputs", [])], + outputs=[str(item) for item in raw_node.get("outputs", [])], + attrs=dict(raw_node.get("attrs") or {}), + meta=dict(raw_node.get("meta") or {}), + kind=str(raw_node.get("kind", "generic")), + ) + nodes[node_id] = node + order.append(node_id) + + constants_payload = graph_payload.get("constants") + if not isinstance(constants_payload, dict): + raise ValueError("saved optimized IR graph is missing constants payload") + + constant_bindings_by_value_id = { + str(binding.get("value_id")): binding + for binding in component_entry.get("bound_constant_bindings", []) + if isinstance(binding, dict) and isinstance(binding.get("value_id"), str) + } + + constants: dict[str, object] = {} + for value_id, serialized in constants_payload.items(): + if not isinstance(value_id, str): + continue + value = values.get(value_id) + if value is None: + continue + constants[value_id] = _deserialize_saved_ir_constant( + value=value, + serialized=serialized, + binding=constant_bindings_by_value_id.get(value_id), + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + + return IRGraph( + values=values, + nodes=nodes, + order=order, + inputs=[str(item) for item in graph_payload.get("inputs", [])], + outputs=[str(item) for item in graph_payload.get("outputs", [])], + constants=constants, + meta=dict(graph_payload.get("meta") or {}), + ) + + +def _deserialize_saved_ir_constant( + *, + value: IRValue, + serialized: object, + binding: Mapping[str, object] | None, + bundle_root: Path, + weights_dir: str | Path | None, +) -> object: + def _torch_dtype_from_name(name: str) -> torch.dtype | None: + return { + "torch.bool": torch.bool, + "torch.uint8": torch.uint8, + "torch.int8": torch.int8, + "torch.int16": torch.int16, + "torch.int32": torch.int32, + "torch.int64": torch.int64, + "torch.float16": torch.float16, + "torch.float32": torch.float32, + "torch.float64": torch.float64, + }.get(name) + + meta = value.meta if isinstance(value.meta, dict) else {} + if isinstance(meta.get("path"), str) and isinstance(meta.get("kind"), str): + # The lowerer ignores the constant payload when a weight binding is present + # in IRValue metadata and re-attaches the mmap-backed tensor directly. + return 0 + + if isinstance(binding, Mapping): + binding_format = str(binding.get("format", "tensor_io") or "tensor_io") + if binding_format == "npy": + tensor_path = _resolve_bound_tensor_path( + str(binding["path"]), + bundle_root=bundle_root, + weights_dir=weights_dir, + ) + array = np.load(tensor_path, mmap_mode="r") + return torch.from_numpy(np.asarray(array)) + + if isinstance(serialized, (str, int, float, bool)) or serialized is None: + return serialized + + if isinstance(serialized, list): + return serialized + + if isinstance(serialized, dict): + value_type = str(serialized.get("type", "")) + if value_type in {"torch.Tensor", "numpy.ndarray"}: + shape = tuple(int(dim) for dim in serialized.get("shape", []) or []) + if not shape: + dtype = _torch_dtype_from_name(str(serialized.get("dtype", ""))) or torch.float32 + zero_scalar = False if dtype is torch.bool else 0 + return torch.tensor(zero_scalar, dtype=dtype) + raise ValueError( + "saved optimized IR is missing a materialized constant payload for " + f"{value.id} with shape={shape}; expected a bound_constants entry" + ) + return dict(serialized) + + return serialized + + def execute_loaded_component_pipeline( components: list[LoadedComponentGraph], *, @@ -347,6 +578,119 @@ def _step( } +def _run_multimodal_causal_lm_bundle( + *, + component_graphs: dict[str, LoadedComponentGraph], + manifest: dict[str, object], + prompt: str | None, + image_files: tuple[str, ...], + audio_file: str | Path | None, + torch_dtype: torch.dtype, + system_prompt: str | None, + enable_thinking: bool, +) -> dict[str, object]: + required_components = ("vision_encoder", "audio_encoder", "lm_encoder", "decoder") + missing = [name for name in required_components if name not in component_graphs] + if missing: + raise ValueError( + "multimodal causal LM bundle requires components " + f"{required_components!r}, missing {missing!r}" + ) + + inputs_meta = manifest.get("inputs") if isinstance(manifest.get("inputs"), dict) else {} + resolved_prompt = prompt if prompt is not None else str(inputs_meta.get("prompt", "") or "") + if not resolved_prompt: + raise ValueError("provide --prompt for multimodal causal-LM bundles") + + resolved_image_files: tuple[str, ...] + if image_files: + resolved_image_files = tuple(str(Path(path).expanduser().resolve()) for path in image_files) + else: + stored_images = inputs_meta.get("image_files") + if isinstance(stored_images, list): + resolved_image_files = tuple(str(Path(path).expanduser().resolve()) for path in stored_images if isinstance(path, str) and path) + else: + resolved_image_files = () + if not resolved_image_files: + raise ValueError("provide --image or --image-file for multimodal causal-LM bundles") + + resolved_audio: str | None = None + if audio_file is not None: + resolved_audio = str(Path(audio_file).expanduser().resolve()) + else: + stored_audio = inputs_meta.get("audio_file") + if isinstance(stored_audio, str) and stored_audio: + resolved_audio = str(Path(stored_audio).expanduser().resolve()) + + from examples.transpile_hf_model import _patch_torch_flex_attention_compat + from examples.transpile_hf_model import _patch_transformers_torchvision_probe + from examples.transpile_hf_model import _prepare_gemma4_multimodal_inputs + + _patch_transformers_torchvision_probe() + _patch_torch_flex_attention_compat() + + from transformers import AutoProcessor + + model_source = str(manifest.get("model_source", "") or manifest.get("model_id", "") or "") + if not model_source: + raise ValueError("bundle manifest is missing model_source/model_id for multimodal preprocessing") + + processor = AutoProcessor.from_pretrained( + model_source, + local_files_only=Path(model_source).exists(), + trust_remote_code=True, + ) + prepared = _prepare_gemma4_multimodal_inputs( + processor, + prompt=resolved_prompt, + image_files=resolved_image_files, + audio_file=resolved_audio, + torch_dtype=torch_dtype, + system_prompt=system_prompt or "", + enable_thinking_if_supported=enable_thinking, + use_gemma4_chat_template=True, + ) + + _attach_component_io_names(manifest, component_graphs) + prepared_store = { + name: tensor.detach().cpu().numpy() + for name, tensor in zip(prepared.names, prepared.tensors, strict=True) + } + start = time.perf_counter() + store, _ = execute_loaded_component_pipeline( + [component_graphs[name] for name in required_components], + initial_store=prepared_store, + ) + end = time.perf_counter() + logits = np.asarray(store["logits"], dtype=np.float32) + if logits.ndim != 3: + raise RuntimeError(f"expected logits with shape [batch, seq, vocab], got {list(logits.shape)}") + + next_token_id = int(np.argmax(logits[0, -1])) + tokenizer = getattr(processor, "tokenizer", processor) + next_token = None + if hasattr(tokenizer, "decode"): + next_token = tokenizer.decode([next_token_id]) + + return { + "bundle_model_id": str(manifest.get("model_id", "") or ""), + "family": str(manifest.get("family", "") or ""), + "task": str(manifest.get("task", "") or ""), + "component_order": list(manifest.get("component_order", [])), + "prompt": resolved_prompt, + "image_files": list(resolved_image_files), + "audio_file": resolved_audio, + "input_shapes": { + name: list(np.asarray(value).shape) + for name, value in prepared_store.items() + }, + "output_shape": list(logits.shape), + "total_ms": (end - start) * 1000.0, + "next_token_id": next_token_id, + "next_token": next_token, + } + + def _run_causal_lm_logits_bundle( *, component_graphs: dict[str, LoadedComponentGraph], @@ -832,4 +1176,3 @@ def _to_numpy(value: Any) -> np.ndarray: if isinstance(value, torch.Tensor): return np.ascontiguousarray(value.detach().cpu().numpy()) raise TypeError(f"unsupported runtime value type: {type(value).__name__}") - diff --git a/python/src/transpile/lower.py b/python/src/transpile/lower.py index 8f20818b5..885b93ff0 100644 --- a/python/src/transpile/lower.py +++ b/python/src/transpile/lower.py @@ -7,8 +7,8 @@ import torch -from src.graph import Graph -from src.graph import Tensor +from src.transpile.runtime_compat import Graph +from src.transpile.runtime_compat import Tensor from src.transpile.capture_pytorch import CapturedModel from src.transpile.canonicalize.cleanup import canonicalize_exported_graph from src.transpile.graph_ir import IRGraph @@ -16,6 +16,7 @@ from src.transpile.graph_ir import IRValue from src.transpile.graph_ir import verify_ir from src.transpile.optimize_graph import optimize_graph +from src.transpile.weight_compat import ensure_binding_compatible from src.transpile.weight_binding import WeightBinding @@ -25,6 +26,7 @@ class TranspiledGraph: runtime_inputs: list[Tensor] bound_constants: list[Tensor] bound_constant_bindings: list[dict[str, object]] + bound_constant_value_ids: dict[int, str] outputs: list[Tensor] def set_input(self, index: int, data: Any, *, dtype: int | None = None) -> None: @@ -73,6 +75,7 @@ def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: runtime_inputs: list[Tensor] = [] bound_constants: list[Tensor] = [] bound_constant_bindings: list[dict[str, object]] = [] + bound_constant_value_ids: dict[int, str] = {} for value_id in ir.inputs: value = ir.values[value_id] @@ -83,10 +86,13 @@ def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: for value_id, const in ir.constants.items(): value = ir.values[value_id] binding = _lookup_weight_binding(value) - lowered_const = _lower_constant_value(g, value, const) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(g, value, const, binding=binding) env[value_id] = lowered_const if isinstance(lowered_const, Tensor): bound_constants.append(lowered_const) + bound_constant_value_ids[int(lowered_const.id)] = str(value_id) if binding is not None: bound_constant_bindings.append( { @@ -121,6 +127,7 @@ def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: runtime_inputs=runtime_inputs, bound_constants=bound_constants, bound_constant_bindings=bound_constant_bindings, + bound_constant_value_ids=bound_constant_value_ids, outputs=outputs, ) @@ -130,12 +137,18 @@ def _lower_input_value(g: Graph, value: IRValue) -> Tensor: raise ValueError(f"IR input missing shape or dtype: {value.id}") return g.input(shape=value.shape, dtype=_map_ir_dtype(value.dtype)) -def _lower_constant_value(g: Graph, value: IRValue, const: Any) -> Any: - binding = _lookup_weight_binding(value) +def _lower_constant_value( + g: Graph, + value: IRValue, + const: Any, + *, + binding: WeightBinding | None = None, +) -> Any: + if binding is None: + binding = _lookup_weight_binding(value) if binding is not None: _debug_mmap_binding(value.id, binding) if binding.kind == "embedding": - print("generating embedding tensor") # Observe which shit actually has embeddings generated return g.mmap_embeddings(binding.path) return g.mmap_weights(binding.path) @@ -244,10 +257,44 @@ def _matmul_output_dtype( return output_dtype +def _trim_padded_last_dim(g: Graph, tensor: Tensor, expected_shape: tuple[int, ...] | None) -> Tensor: + if expected_shape is None: + return tensor + actual_shape = tuple(int(dim) for dim in tensor.shape) + target_shape = tuple(int(dim) for dim in expected_shape) + if actual_shape == target_shape: + return tensor + if ( + len(actual_shape) == len(target_shape) + and len(actual_shape) >= 1 + and actual_shape[:-1] == target_shape[:-1] + and actual_shape[-1] > target_shape[-1] + ): + return g.slice(tensor, len(actual_shape) - 1, 0, target_shape[-1]) + return tensor + + def _legalize_for_transpose(g: Graph, x: Tensor) -> Tensor: return x +def _reshape_attention_output_for_linear( + g: Graph, + attn_out: Tensor, + target_shape: tuple[int, ...], +) -> Tensor: + actual_shape = tuple(int(dim) for dim in attn_out.shape) + target_shape = tuple(int(dim) for dim in target_shape) + if len(actual_shape) == 4 and len(target_shape) == 3 and actual_shape[0] == target_shape[0]: + batch, dim1, dim2, dim3 = actual_shape + _, target_seq, target_hidden = target_shape + if dim1 == target_seq and dim2 * dim3 == target_hidden: + return g.reshape(attn_out, target_shape) + if dim2 == target_seq and dim1 * dim3 == target_hidden: + return g.reshape(g.permute(attn_out, (0, 2, 1, 3)), target_shape) + return g.reshape(attn_out, target_shape) + + def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> list[Any]: op = node.op @@ -546,6 +593,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> pretransposed_rhs=True, output_dtype=matmul_output_dtype, ) + out = _trim_padded_last_dim(g, out, output_value.shape if output_value is not None else None) if node.attrs.get("has_bias"): bias = _ensure_tensor_dtype(g, bias, matmul_output_dtype) out = g.add(out, bias) @@ -567,6 +615,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> elif bias.dtype == Graph.FP32 and rhs.dtype not in (Graph.INT8, Graph.INT4): matmul_output_dtype = Graph.FP32 out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) + out = _trim_padded_last_dim(g, out, output_value.shape if output_value is not None else None) bias = _ensure_tensor_dtype(g, bias, matmul_output_dtype) out = g.add(bias, out) if out.dtype != output_dtype: @@ -650,6 +699,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> value = g.permute(_attention_tensor(env, node.inputs[2]), (0, 2, 1, 3)) if mask_tensor is not None: mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) out = g.attention( query, key, @@ -678,6 +728,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> mask_tensor = _tensor(env, node.inputs[input_index]) input_index += 1 mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) attn_out = g.attention( query, @@ -692,7 +743,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () if flat_shape: - attn_out = g.reshape(attn_out, flat_shape) + attn_out = _reshape_attention_output_for_linear(g, attn_out, flat_shape) if has_gate: gate = _tensor(env, node.inputs[input_index]) @@ -833,6 +884,8 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> mask_tensor = _ensure_fp16_tensor(g, mask_tensor) elif mask_tensor is not None: mask_tensor = _ensure_fp16_tensor(g, mask_tensor) + if mask_tensor is not None: + mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) attn_out = g.attention( query, @@ -847,7 +900,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> flat_shape_attr = tuple(int(v) for v in node.attrs.get("attention_output_shape", ())) flat_shape = _resolve_reshape_shape(tuple(attn_out.shape), flat_shape_attr) if flat_shape_attr else () if flat_shape: - attn_out = g.reshape(attn_out, flat_shape) + attn_out = _reshape_attention_output_for_linear(g, attn_out, flat_shape) if gate is not None: attn_out, gate = _legalize_elementwise_binary_inputs(g, attn_out, gate) @@ -1310,6 +1363,34 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> if dilation != 1: raise NotImplementedError(f"conv2d with dilation != 1 is unsupported: {dilation}") + if stride == 2 and padding == 0 and groups == 1 and weight.shape[2:] == (3, 3): + batch_size, channels, height, width = (int(dim) for dim in x.shape) + pad_dtype = _torch_dtype_for_graph_dtype(x.dtype) + top = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, 1, width), dtype=pad_dtype), + ) + bottom = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, 1, width), dtype=pad_dtype), + ) + x = g.cat([top, x, bottom], axis=2) + left = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, height + 2, 1), dtype=pad_dtype), + ) + right = _materialize_constant_tensor( + g, + torch.zeros((batch_size, channels, height + 2, 1), dtype=pad_dtype), + ) + x = g.cat([left, x, right], axis=3) + y = g.conv2d_k3s2p1(x, weight, bias=bias) + output_height = ((height - 3) // 2) + 1 + output_width = ((width - 3) // 2) + 1 + y = g.slice(y, axis=2, start=1, length=output_height) + y = g.slice(y, axis=3, start=1, length=output_width) + return [y] + if stride == 2 and padding == 1 and groups == 1 and weight.shape[2:] == (3, 3): return [g.conv2d_k3s2p1(x, weight, bias=bias)] @@ -1616,6 +1697,35 @@ def _resolve_attention_scale(node: IRNode, query: Tensor) -> float: return 1.0 +def _normalize_attention_mask_for_cactus(g: Graph, mask: Tensor, query: Tensor) -> Tensor: + """Convert broadcastable HF masks into the native Cactus attention contract.""" + + mask_shape = tuple(int(dim) for dim in mask.shape) + query_shape = tuple(int(dim) for dim in query.shape) + if len(query_shape) != 4: + return mask + + batch_size, seq_len, num_heads, _ = query_shape + if len(mask_shape) == 2 and batch_size == 1 and mask_shape[0] == seq_len: + return g.reshape(mask, (1, mask_shape[0], mask_shape[1])) + + if len(mask_shape) == 4: + if ( + mask_shape[0] == batch_size + and mask_shape[1] == 1 + and mask_shape[2] == seq_len + ): + return g.reshape(mask, (mask_shape[0], mask_shape[2], mask_shape[3])) + if ( + mask_shape[0] == batch_size + and mask_shape[1] == num_heads + and mask_shape[2] == seq_len + ): + return mask + + return mask + + def _lower_projected_attention_tensor( g: Graph, hidden: Tensor, @@ -1812,9 +1922,23 @@ def _try_lower_prefix_mask_advanced_index( return None if source_shape[: len(mask_shape)] != mask_shape: return None - selected = g.masked_select_prefix(source, mask) output_shape = _static_shape(ir.values[node.outputs[0]].shape if node.outputs and node.outputs[0] in ir.values else None) + if output_shape: + prefix_elems = 1 + for dim in mask_shape: + prefix_elems *= int(dim) + trailing_shape = source_shape[len(mask_shape) :] + flattened_shape = (prefix_elems, *trailing_shape) + flattened = source if source_shape == flattened_shape else g.reshape(source, flattened_shape) + expected_prefix = int(output_shape[0]) + if 0 <= expected_prefix <= prefix_elems: + selected = g.slice(flattened, axis=0, start=0, length=expected_prefix) + if tuple(int(dim) for dim in selected.shape) != tuple(int(dim) for dim in output_shape): + selected = g.reshape(selected, output_shape) + return selected + + selected = g.masked_select_prefix(source, mask) if output_shape is None or not output_shape: return selected if len(output_shape) != len(selected.shape): @@ -1882,11 +2006,7 @@ def _lower_binary_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tenso raise NotImplementedError(f"unsupported binary op: {op}") if isinstance(lhs_value, Tensor) and _is_scalar_like(rhs_value): - lhs_value = _ensure_tensor_dtype( - g, - lhs_value, - Graph.FP16 if lhs_value.dtype == Graph.FP16 else Graph.FP32, - ) + lhs_value = _ensure_scalar_math_tensor(g, lhs_value) scalar = float(rhs_value) if op == "add": return g.scalar_add(lhs_value, scalar) @@ -1899,11 +2019,7 @@ def _lower_binary_op(g: Graph, lhs_value: Any, rhs_value: Any, op: str) -> Tenso raise NotImplementedError(f"unsupported binary op: {op}") if _is_scalar_like(lhs_value) and isinstance(rhs_value, Tensor): - rhs_value = _ensure_tensor_dtype( - g, - rhs_value, - Graph.FP16 if rhs_value.dtype == Graph.FP16 else Graph.FP32, - ) + rhs_value = _ensure_scalar_math_tensor(g, rhs_value) scalar = float(lhs_value) if op == "add": return g.scalar_add(rhs_value, scalar) @@ -1996,7 +2112,7 @@ def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: def _ensure_scalar_math_tensor(g: Graph, tensor: Tensor) -> Tensor: - if tensor.dtype in (Graph.FP16, Graph.FP32): + if tensor.dtype == Graph.FP16: return tensor return g.precision_cast(tensor, Graph.FP16) @@ -2100,7 +2216,7 @@ def _flatten_to_2d_for_linear(g: Graph, tensor: Tensor) -> Tensor: leading = 1 for dim in shape[:-1]: leading *= int(dim) - return g.reshape(tensor, (leading, int(shape[-1]))) + return g.view(tensor, (leading, int(shape[-1]))) def _broadcast_batch_shape(lhs_batch: tuple[int, ...], rhs_batch: tuple[int, ...]) -> tuple[int, ...] | None: diff --git a/python/src/transpile/model_adapters.py b/python/src/transpile/model_adapters.py index 38476d8ec..c6aec13dd 100644 --- a/python/src/transpile/model_adapters.py +++ b/python/src/transpile/model_adapters.py @@ -1739,11 +1739,9 @@ def get_transpile_metadata(self): } -_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS = ( +_GEMMA4_DECODER_PIPELINE_IO_KEYS = ( "inputs_embeds", "per_layer_inputs", - "full_attention_mask", - "sliding_attention_mask", "position_ids", ) @@ -1918,7 +1916,7 @@ def _prepare_decoder_inputs( token_type_ids: torch.Tensor, image_features: torch.Tensor, audio_features: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.LongTensor]: + ) -> tuple[torch.Tensor, torch.Tensor, torch.LongTensor]: if self._native_merge_plan is not None: inputs_embeds, per_layer_inputs_tokens = _gemma4_apply_native_merge_plan( self.model, @@ -1968,45 +1966,9 @@ def _prepare_decoder_inputs( per_layer_inputs = inputs_embeds.new_empty((inputs_embeds.shape[0], inputs_embeds.shape[1], 0, 0)) position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) - if self._native_merge_plan is not None: - causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( - create_causal_mask=self._create_causal_mask, - create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, - config=self.backbone.config, - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - position_ids=position_ids, - ) - elif getattr(text_config, "use_bidirectional_attention", None) == "vision": - if not callable(self._create_causal_mask_mapping): - raise RuntimeError("Gemma4 component LM encoder requires create_causal_mask_mapping") - causal_mask_mapping = self._create_causal_mask_mapping( - self.multimodal_backbone.config, - inputs_embeds, - attention_mask, - None, - position_ids, - token_type_ids, - None, - is_training=self.training, - ) - else: - if not callable(self._create_masks_for_generate): - raise RuntimeError("Gemma4 component LM encoder requires create_masks_for_generate") - causal_mask_mapping = self._create_masks_for_generate( - self.multimodal_backbone.config, - inputs_embeds, - attention_mask, - None, - position_ids, - ) - full_attention_mask = causal_mask_mapping["full_attention"] - sliding_attention_mask = causal_mask_mapping.get("sliding_attention", full_attention_mask) return ( inputs_embeds, per_layer_inputs, - full_attention_mask, - sliding_attention_mask, position_ids, ) @@ -2091,27 +2053,35 @@ def get_transpile_metadata(self): class Gemma4DecoderAdapter(_Gemma4MultimodalComponentBase): def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): - super().__init__(model, input_names=_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS, weights_dir=weights_dir) + super().__init__(model, input_names=_GEMMA4_DECODER_PIPELINE_IO_KEYS, weights_dir=weights_dir) def forward( self, inputs_embeds: torch.Tensor, per_layer_inputs: torch.Tensor, - full_attention_mask: torch.Tensor, - sliding_attention_mask: torch.Tensor, position_ids: torch.LongTensor, ) -> torch.Tensor: normalized_per_layer_inputs = per_layer_inputs if normalized_per_layer_inputs.numel() == 0: normalized_per_layer_inputs = None + attention_mask = torch.ones( + position_ids.shape, + dtype=torch.long, + device=position_ids.device, + ) + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) hidden_states = _gemma4_text_backbone_forward( self.backbone, inputs_embeds=inputs_embeds, per_layer_inputs=normalized_per_layer_inputs, - causal_mask_mapping={ - "full_attention": full_attention_mask, - "sliding_attention": sliding_attention_mask, - }, + causal_mask_mapping=causal_mask_mapping, position_ids=position_ids, ) logits = self.model.lm_head(hidden_states[:, -1:, :]) @@ -2121,7 +2091,7 @@ def get_transpile_metadata(self): return { "graph": self._base_graph_meta( adapter_type=type(self).__name__, - input_names=_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS, + input_names=_GEMMA4_DECODER_PIPELINE_IO_KEYS, ), "import_hints": _gemma4_import_hints( self.backbone, @@ -2246,7 +2216,7 @@ def _require(component: str) -> None: module=lm_encoder, example_inputs=(input_ids, attention_mask, token_type_ids, image_features, audio_features), input_keys=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"), - output_keys=_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS, + output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, graph_meta={**common_graph_meta, "component": "lm_encoder"}, metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) @@ -2257,7 +2227,7 @@ def _require(component: str) -> None: component="decoder", module=decoder, example_inputs=tuple(decoder_inputs), - input_keys=_GEMMA4_DECODER_PIPELINE_OUTPUT_KEYS, + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, output_keys=("logits",), graph_meta={**common_graph_meta, "component": "decoder"}, metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, diff --git a/python/src/transpile/optimize_graph.py b/python/src/transpile/optimize_graph.py index 7c622dbf8..aab276617 100644 --- a/python/src/transpile/optimize_graph.py +++ b/python/src/transpile/optimize_graph.py @@ -62,6 +62,7 @@ def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig config = config or FusionConfig() verify_ir(graph) canonicalize_exported_graph(graph) + enable_gemma4_attention_block_fusions = not _is_gemma4_graph(graph) for _ in range(max_passes): changed = False @@ -80,9 +81,17 @@ def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig changed = True if config.enable_attention and fuse_attention(graph): changed = True - if config.enable_self_attention_block and fuse_self_attention_blocks(graph): + if ( + enable_gemma4_attention_block_fusions + and config.enable_self_attention_block + and fuse_self_attention_blocks(graph) + ): changed = True - if config.enable_attention_block and fuse_attention_blocks(graph): + if ( + enable_gemma4_attention_block_fusions + and config.enable_attention_block + and fuse_attention_blocks(graph) + ): changed = True if normalize_gemma4_decoder_attention_semantics(graph): changed = True @@ -487,7 +496,53 @@ def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: node = graph.nodes.get(node_id) if node is None or node.op not in {"attention", "scaled_dot_product_attention", "attention_block"}: continue - if str(node.meta.get("attention_layer_type") or "").strip().lower() != "sliding_attention": + layer_type = str(node.meta.get("attention_layer_type") or "").strip().lower() + if layer_type == "full_attention": + mask_input_index: int | None = None + if node.op == "attention_block": + if bool(node.attrs.get("has_mask", False)) and len(node.inputs) > 3: + mask_input_index = 3 + elif len(node.inputs) > 3: + mask_input_index = 3 + + if mask_input_index is not None: + mask_value_id = node.inputs[mask_input_index] + mask_value = graph.values.get(mask_value_id) + if mask_value is not None and mask_value.dtype in (None, "bool"): + del node.inputs[mask_input_index] + node.meta["gemma4_full_mask_elided"] = True + changed = True + + if node.op == "attention_block" and bool(node.attrs.get("has_mask", False)): + node.attrs["has_mask"] = False + changed = True + if not bool(node.attrs.get("is_causal", False)): + node.attrs["is_causal"] = True + changed = True + if "additive_mask" in node.attrs and not bool(node.attrs.get("additive_mask", False)): + node.attrs.pop("additive_mask", None) + changed = True + if int(node.attrs.get("window_size", 0)) == 0: + seq_len = 0 + attention_output_shape = node.attrs.get("attention_output_shape") + if isinstance(attention_output_shape, (list, tuple)) and len(attention_output_shape) >= 2: + try: + seq_len = int(attention_output_shape[1]) + except (TypeError, ValueError): + seq_len = 0 + if seq_len <= 0 and node.inputs: + query_value = graph.values.get(node.inputs[0]) + if query_value is not None and query_value.shape is not None and len(query_value.shape) >= 3: + seq_len = int(query_value.shape[2]) + if seq_len > 0: + # Gemma4 full-attention layers are semantically unwindowed, but using a + # window_size >= seq_len avoids the Apple Accelerate full-window fast path + # that has been unstable for transpiled grouped-query Gemma4 graphs. + node.attrs["window_size"] = seq_len + node.meta["gemma4_full_attention_window_compat"] = True + changed = True + continue + if layer_type != "sliding_attention": continue if bool(node.attrs.get("additive_mask", False)): continue @@ -502,22 +557,19 @@ def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: if mask_input_index is not None: mask_value_id = node.inputs[mask_input_index] mask_value = graph.values.get(mask_value_id) - if mask_value is None or mask_value.dtype not in (None, "bool"): - continue - mask_info = _extract_sliding_window_mask(graph, mask_value_id) - mask_input_name = _graph_input_name(graph, mask_value_id) - if mask_info is None and mask_input_name != "sliding_attention_mask": - continue - del node.inputs[mask_input_index] - node.meta["gemma4_sliding_mask_elided"] = True - if mask_info is not None: - node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) - if int(node.attrs.get("window_size", 0)) == 0 and int(mask_info["window_size"]) > 0: - node.attrs["window_size"] = int(mask_info["window_size"]) - node.meta.setdefault("window_size_source", "mask_pattern") - elif mask_input_name is not None: - node.meta["gemma4_elided_mask_input_name"] = mask_input_name - changed = True + if mask_value is not None and mask_value.dtype in (None, "bool"): + mask_info = _extract_sliding_window_mask(graph, mask_value_id) + mask_input_name = _graph_input_name(graph, mask_value_id) + del node.inputs[mask_input_index] + node.meta["gemma4_sliding_mask_elided"] = True + if mask_info is not None: + node.meta["mask_window_size_hint"] = int(mask_info["window_size"]) + if int(node.attrs.get("window_size", 0)) == 0 and int(mask_info["window_size"]) > 0: + node.attrs["window_size"] = int(mask_info["window_size"]) + node.meta.setdefault("window_size_source", "mask_pattern") + elif mask_input_name is not None: + node.meta["gemma4_elided_mask_input_name"] = mask_input_name + changed = True if node.op == "attention_block" and bool(node.attrs.get("has_mask", False)): node.attrs["has_mask"] = False @@ -723,14 +775,35 @@ def _materialize_shifted_constant(graph: IRGraph, value_id: str, offset: float, def _materialize_zero_like_constant(graph: IRGraph, value_id: str, *, suffix: str) -> str: base = graph.constants[value_id] - if not isinstance(base, torch.Tensor): - raise NotImplementedError(f"expected tensor constant for {value_id}, got {type(base).__name__}") - new_value_id = f"{value_id}_{suffix}" if new_value_id in graph.constants: return new_value_id - zero = torch.zeros_like(base.detach().cpu()) + if isinstance(base, torch.Tensor): + zero = torch.zeros_like(base.detach().cpu()) + else: + value = graph.values.get(value_id) + if value is None or value.shape is None or value.dtype is None: + raise NotImplementedError( + f"expected tensor constant metadata for {value_id}, got {type(base).__name__}" + ) + dtype_map = { + "bool": torch.bool, + "u8": torch.uint8, + "i8": torch.int8, + "i16": torch.int16, + "i32": torch.int32, + "i64": torch.int64, + "fp16": torch.float16, + "bf16": torch.bfloat16, + "fp32": torch.float32, + "fp64": torch.float64, + } + torch_dtype = dtype_map.get(str(value.dtype)) + if torch_dtype is None: + raise NotImplementedError(f"unsupported IR dtype for zero-like constant: {value.dtype}") + zero = torch.zeros(tuple(int(dim) for dim in value.shape), dtype=torch_dtype) + graph.constants[new_value_id] = zero graph.values[new_value_id] = IRValue( id=new_value_id, diff --git a/python/src/transpile/runtime_compat.py b/python/src/transpile/runtime_compat.py new file mode 100644 index 000000000..f0baa4b19 --- /dev/null +++ b/python/src/transpile/runtime_compat.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +import ctypes +import importlib +import sys +from typing import Any + + +class _MissingFFIFunction: + _cactus_missing_symbol = True + + def __init__(self, name: str): + self.__name__ = name + self.argtypes = None + self.restype = ctypes.c_int + + def __call__(self, *args: Any, **kwargs: Any) -> int: + raise RuntimeError(f"Cactus runtime is missing required symbol: {self.__name__}") + + +_ORIG_CDLL_GETATTR = ctypes.CDLL.__getattr__ + + +def _patched_cdll_getattr(self: ctypes.CDLL, name: str): + try: + return _ORIG_CDLL_GETATTR(self, name) + except AttributeError: + if not name.startswith("cactus_"): + raise + missing = _MissingFFIFunction(name) + setattr(self, name, missing) + return missing + + +def _load_runtime_modules(): + if "src.cactus" in sys.modules and "src.graph" in sys.modules: + return sys.modules["src.cactus"], sys.modules["src.graph"] + + ctypes.CDLL.__getattr__ = _patched_cdll_getattr + try: + cactus_module = importlib.import_module("src.cactus") + graph_module = importlib.import_module("src.graph") + finally: + ctypes.CDLL.__getattr__ = _ORIG_CDLL_GETATTR + return cactus_module, graph_module + + +def _patch_graph_runtime(graph_module, cactus_module) -> None: + Graph = graph_module.Graph + if getattr(Graph, "_transpile_runtime_compat_patched", False): + return + + _lib = cactus_module._lib + _err = cactus_module._err + cactus_node_t = cactus_module.cactus_node_t + + def _has_symbol(name: str) -> bool: + symbol = getattr(_lib, name, None) + return symbol is not None and not getattr(symbol, "_cactus_missing_symbol", False) + + def _ensure_compare_tensor(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP16): + return tensor + return self.precision_cast(tensor, Graph.FP16) + + def _ensure_scalar_tensor(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP16): + return tensor + return self.precision_cast(tensor, Graph.FP16) + + def _ensure_fp16_activation(self, tensor): + tensor = self._ensure_tensor(tensor) + if int(tensor.dtype) == int(Graph.FP32): + return self.precision_cast(tensor, Graph.FP16) + return tensor + + def _approx_nonzero_mask(self, tensor): + tensor = _ensure_scalar_tensor(self, tensor) + # Gemma4 compare paths in v2 only need stable 0/1-style masks for + # discrete values such as token-type ids and prebuilt boolean masks. + magnitude = self.abs(tensor) + shifted = self.scalar_add(magnitude, -0.5) + sharpened = self.scalar_multiply(shifted, 16.0) + return self.sigmoid(sharpened) + + orig_conv2d = Graph.conv2d + orig_not_equal = Graph.not_equal + orig_scalar_not_equal = Graph.scalar_not_equal + orig_transpose = Graph.transpose + orig_permute = Graph.permute + orig_scalar_add = Graph.scalar_add + orig_scalar_subtract = Graph.scalar_subtract + orig_scalar_multiply = Graph.scalar_multiply + orig_scalar_divide = Graph.scalar_divide + orig_scalar_exp = Graph.scalar_exp + orig_scalar_sqrt = Graph.scalar_sqrt + orig_scalar_cos = Graph.scalar_cos + orig_scalar_sin = Graph.scalar_sin + orig_scalar_log = Graph.scalar_log + orig_add = Graph.add + orig_subtract = Graph.subtract + orig_multiply = Graph.multiply + orig_divide = Graph.divide + orig_concat = Graph.concat + orig_cat = Graph.cat + orig_abs = Graph.abs + orig_pow = Graph.pow + + def matmul(self, a, b, pretransposed_rhs=False, backend=Graph.CPU, output_dtype=None): + a = _ensure_fp16_activation(self, a) + b = _ensure_fp16_activation(self, b) + out = cactus_node_t() + rc = _lib.cactus_graph_matmul( + self.h, + cactus_node_t(a.id), + cactus_node_t(b.id), + ctypes.c_bool(bool(pretransposed_rhs)), + ctypes.c_int32(int(backend)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError(_err("graph_matmul failed")) + result = self._tensor_from_node(out.value) + if output_dtype is not None and int(output_dtype) != int(result.dtype): + result = self.precision_cast(result, int(output_dtype)) + return result + + def gather(self, tensor, indices, axis=0): + tensor = self._ensure_tensor(tensor) + indices = self._ensure_tensor(indices) + if int(axis) != 0: + raise NotImplementedError( + f"transpiler runtime compatibility only supports gather(axis=0), got axis={axis}" + ) + out = cactus_node_t() + rc = _lib.cactus_graph_gather( + self.h, + cactus_node_t(tensor.id), + cactus_node_t(indices.id), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_gather failed") + return self._tensor_from_node(out.value) + + def conv2d(self, x, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + def _pair(value: Any) -> tuple[int, int]: + if isinstance(value, (tuple, list)): + if len(value) != 2: + raise ValueError(f"expected pair for conv2d parameter, got {value!r}") + return int(value[0]), int(value[1]) + return int(value), int(value) + + stride_hw = _pair(stride) + padding_hw = _pair(padding) + dilation_hw = _pair(dilation) + groups_int = int(groups) + kernel_hw = tuple(int(dim) for dim in self._ensure_tensor(weight).shape[-2:]) + + if _has_symbol("cactus_graph_conv2d"): + return orig_conv2d( + self, + x, + weight, + bias=bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + ) + + if ( + kernel_hw == (3, 3) + and stride_hw == (2, 2) + and padding_hw == (1, 1) + and dilation_hw == (1, 1) + ): + if groups_int == 1: + return self.conv2d_k3s2p1(x, weight, bias=bias) + if len(weight.shape) >= 1 and groups_int == int(weight.shape[0]): + return self.conv2d_depthwise_k3s2p1(x, weight, bias=bias) + + if ( + kernel_hw == (3, 3) + and stride_hw == (1, 1) + and padding_hw == (1, 1) + and dilation_hw == (1, 1) + and groups_int == 1 + and _has_symbol("cactus_graph_conv2d_k3s1p1") + ): + return self.conv2d_k3s1p1(x, weight, bias=bias) + + if ( + kernel_hw == (1, 1) + and stride_hw == (1, 1) + and padding_hw == (0, 0) + and dilation_hw == (1, 1) + and groups_int == 1 + ): + return self.conv2d_pointwise_1x1(x, weight, bias=bias) + + raise NotImplementedError( + "v2 runtime does not expose generic conv2d for this configuration: " + f"kernel={kernel_hw} stride={stride_hw} padding={padding_hw} " + f"dilation={dilation_hw} groups={groups_int}" + ) + + def scalar_not_equal(self, x, value): + if _has_symbol("cactus_graph_scalar_not_equal"): + return orig_scalar_not_equal(self, x, value) + x = _ensure_compare_tensor(self, x) + delta = self.scalar_add(x, -float(value)) + return _approx_nonzero_mask(self, delta) + + def not_equal(self, a, b): + if _has_symbol("cactus_graph_not_equal"): + return orig_not_equal(self, a, b) + a = _ensure_compare_tensor(self, a) + b = _ensure_compare_tensor(self, b) + delta = self.subtract(a, b) + return _approx_nonzero_mask(self, delta) + + def transpose(self, x, backend=Graph.CPU): + return orig_transpose(self, _ensure_scalar_tensor(self, x), backend=backend) + + def permute(self, x, permutation, backend=Graph.CPU): + return orig_permute(self, _ensure_scalar_tensor(self, x), permutation, backend=backend) + + def scalar_add(self, x, value): + return orig_scalar_add(self, _ensure_scalar_tensor(self, x), value) + + def scalar_subtract(self, x, value): + return orig_scalar_subtract(self, _ensure_scalar_tensor(self, x), value) + + def scalar_multiply(self, x, value): + return orig_scalar_multiply(self, _ensure_scalar_tensor(self, x), value) + + def scalar_divide(self, x, value): + return orig_scalar_divide(self, _ensure_scalar_tensor(self, x), value) + + def scalar_exp(self, x): + return orig_scalar_exp(self, _ensure_scalar_tensor(self, x)) + + def scalar_sqrt(self, x): + return orig_scalar_sqrt(self, _ensure_scalar_tensor(self, x)) + + def scalar_cos(self, x): + return orig_scalar_cos(self, _ensure_scalar_tensor(self, x)) + + def scalar_sin(self, x): + return orig_scalar_sin(self, _ensure_scalar_tensor(self, x)) + + def scalar_log(self, x): + return orig_scalar_log(self, _ensure_scalar_tensor(self, x)) + + def add(self, a, b): + return orig_add(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def subtract(self, a, b): + return orig_subtract(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def multiply(self, a, b): + return orig_multiply(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def divide(self, a, b): + return orig_divide(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b)) + + def concat(self, a, b, axis=0): + return orig_concat(self, _ensure_scalar_tensor(self, a), _ensure_scalar_tensor(self, b), axis=axis) + + def cat(self, tensors, axis=0): + legalized = [_ensure_scalar_tensor(self, tensor) for tensor in tensors] + return orig_cat(self, legalized, axis=axis) + + def abs(self, x): + return orig_abs(self, _ensure_scalar_tensor(self, x)) + + def pow(self, x, exponent): + return orig_pow(self, _ensure_scalar_tensor(self, x), exponent) + + Graph.matmul = matmul + Graph.gather = gather + Graph.conv2d = conv2d + Graph.scalar_not_equal = scalar_not_equal + Graph.not_equal = not_equal + Graph.transpose = transpose + Graph.permute = permute + Graph.scalar_add = scalar_add + Graph.scalar_subtract = scalar_subtract + Graph.scalar_multiply = scalar_multiply + Graph.scalar_divide = scalar_divide + Graph.scalar_exp = scalar_exp + Graph.scalar_sqrt = scalar_sqrt + Graph.scalar_cos = scalar_cos + Graph.scalar_sin = scalar_sin + Graph.scalar_log = scalar_log + Graph.add = add + Graph.subtract = subtract + Graph.multiply = multiply + Graph.divide = divide + Graph.concat = concat + Graph.cat = cat + Graph.abs = abs + Graph.pow = pow + Graph._transpile_runtime_compat_patched = True + + +_cactus_module, _graph_module = _load_runtime_modules() + +if hasattr(_cactus_module, "_lib"): + _lib_obj = _cactus_module._lib + if hasattr(_lib_obj, "cactus_graph_matmul"): + _lib_obj.cactus_graph_matmul.argtypes = [ + ctypes.c_void_p, + _cactus_module.cactus_node_t, + _cactus_module.cactus_node_t, + ctypes.c_bool, + ctypes.c_int32, + ctypes.POINTER(_cactus_module.cactus_node_t), + ] + _lib_obj.cactus_graph_matmul.restype = ctypes.c_int + if hasattr(_lib_obj, "cactus_graph_gather"): + _lib_obj.cactus_graph_gather.argtypes = [ + ctypes.c_void_p, + _cactus_module.cactus_node_t, + _cactus_module.cactus_node_t, + ctypes.POINTER(_cactus_module.cactus_node_t), + ] + _lib_obj.cactus_graph_gather.restype = ctypes.c_int + +_patch_graph_runtime(_graph_module, _cactus_module) + +_lib = _cactus_module._lib +_err = _cactus_module._err +cactus_node_t = _cactus_module.cactus_node_t +cactus_tensor_info_t = _cactus_module.cactus_tensor_info_t +Graph = _graph_module.Graph +Tensor = _graph_module.Tensor diff --git a/python/src/transpile/weight_binding.py b/python/src/transpile/weight_binding.py index da05532c0..c9d829d4b 100644 --- a/python/src/transpile/weight_binding.py +++ b/python/src/transpile/weight_binding.py @@ -115,6 +115,8 @@ class WeightBinding: _GEMMA4_AUDIO_TOWER_PREFIX = "model.audio_tower." _GEMMA_GLOBAL_FILENAMES: dict[str, tuple[str, str]] = { + "model.embed_tokens.weight": ("token_embeddings.weights", "embedding"), + "model.language_model.embed_tokens.weight": ("token_embeddings.weights", "embedding"), "model.language_model.embed_tokens_per_layer.weight": ("embed_tokens_per_layer.weights", "embedding"), "model.language_model.per_layer_model_projection.weight": ("per_layer_model_proj.weights", "weight"), "model.language_model.per_layer_projection_norm.weight": ("per_layer_proj_norm.weights", "weight"), @@ -128,6 +130,26 @@ class WeightBinding: "model.embed_audio.hard_embedding_norm.weight": ("embed_audio_hard_norm.weights", "weight"), } +_GEMMA_DECODER_LAYER_FILENAMES: dict[str, tuple[str, str]] = { + "layer_scalar": ("layer_scalar.weights", "weight"), + "self_attn.q_proj.weight": ("attn_q.weights", "weight"), + "self_attn.k_proj.weight": ("attn_k.weights", "weight"), + "self_attn.v_proj.weight": ("attn_v.weights", "weight"), + "self_attn.o_proj.weight": ("attn_output.weights", "weight"), + "self_attn.q_norm.weight": ("attn_q_norm.weights", "weight"), + "self_attn.k_norm.weight": ("attn_k_norm.weights", "weight"), + "mlp.gate_proj.weight": ("ffn_gate.weights", "weight"), + "mlp.up_proj.weight": ("ffn_up.weights", "weight"), + "mlp.down_proj.weight": ("ffn_down.weights", "weight"), + "input_layernorm.weight": ("input_norm.weights", "weight"), + "post_attention_layernorm.weight": ("post_attn_norm.weights", "weight"), + "pre_feedforward_layernorm.weight": ("pre_ffn_norm.weights", "weight"), + "post_feedforward_layernorm.weight": ("post_ffn_norm.weights", "weight"), + "post_per_layer_projection_layernorm.weight": ("post_per_layer_norm.weights", "weight"), + "per_layer_input_gate.weight": ("per_layer_gate.weights", "weight"), + "per_layer_projection.weight": ("per_layer_proj.weights", "weight"), +} + def _candidate_model_dir_names(model_name_or_path: str) -> list[str]: @@ -222,6 +244,15 @@ def _add_backbone_aliases(name: str) -> None: break for candidate in tuple(candidates): + for repeated_prefix, collapsed_prefix in ( + ("model.model.", "model."), + ("language_model.language_model.", "language_model."), + ("backbone.backbone.", "backbone."), + ): + if candidate.startswith(repeated_prefix): + tail = candidate[len(repeated_prefix) :] + _add(f"{collapsed_prefix}{tail}") + for prefix in ( "module.backbone.", "adapter.backbone.", @@ -401,6 +432,17 @@ def _add(filename: str | None, *, kind: str = "weight") -> None: if mapped is not None: _add(f"layer_{layer_index}_{mapped}") + gemma_layer_match = re.match( + r"^(?:model(?:\.language_model)?\.)?layers\.(\d+)\.(.+)$", + candidate, + ) + if gemma_layer_match: + layer_index = int(gemma_layer_match.group(1)) + suffix = gemma_layer_match.group(2) + mapped = _GEMMA_DECODER_LAYER_FILENAMES.get(suffix) + if mapped is not None: + _add(f"layer_{layer_index}_{mapped[0]}", kind=mapped[1]) + return filenames @@ -432,15 +474,15 @@ def resolve_weight_binding(*, weights_dir: str | None, source_name: str) -> Weig root = Path(weights_dir) if not root.exists(): return None + manifest: dict[str, object] = {} manifest_path = root / "weights_manifest.json" - if not manifest_path.exists(): - return None - try: - manifest = json.loads(manifest_path.read_text()) - except Exception: - return None - if not isinstance(manifest, dict): - return None + if manifest_path.exists(): + try: + loaded_manifest = json.loads(manifest_path.read_text()) + except Exception: + loaded_manifest = None + if isinstance(loaded_manifest, dict): + manifest = loaded_manifest for candidate_name in _normalized_source_candidates(source_name): entry = manifest.get(candidate_name) if not isinstance(entry, dict): diff --git a/python/src/transpile/weight_compat.py b/python/src/transpile/weight_compat.py new file mode 100644 index 000000000..cd72f71df --- /dev/null +++ b/python/src/transpile/weight_compat.py @@ -0,0 +1,520 @@ +from __future__ import annotations + +from dataclasses import replace +import shutil +import struct +from pathlib import Path + +import numpy as np + +from src.tensor_io import CACTUS_ALIGNMENT +from src.tensor_io import CACTUS_MAGIC +from src.tensor_io import FLAG_INTERLEAVED +from src.tensor_io import align_offset +from src.cq_convert.model_adapters.naming import gemma4_scale_factor +from src.transpile.runtime_compat import Graph +from src.transpile.weight_binding import WeightBinding +from src.cq_convert.quantization.cq import FLAG_ORTHOGONAL_ROTATION +from src.cq_convert.quantization.cq import GROUP_SIZE as CQ_GROUP_SIZE +from src.cq_convert.quantization.cq import PRECISION_CQ +from src.cq_convert.quantization.cq import make_codebook +from src.cq_convert.quantization.cq import make_hadamard_components +from src.cq_convert.quantization.cq import make_hadamard_matrix +from src.cq_convert.quantization.cq import make_orthogonal_rotation +from src.cq_convert.quantization.cq import pack_indices_lsb +from src.cq_convert.quantization.cq import quantize_hadamard +from src.cq_convert.quantization.cq import quantize_orthogonal +from src.cq_convert.quantization.cq import write_cq_tensor + + +_HEADER_SIZE = 84 +_INT8 = int(Graph.INT8) +_INT4 = int(Graph.INT4) + +_TOKEN_EMBEDDING_FILENAMES = { + "token_embeddings.weights", + "decoder_token_embeddings.weights", +} +_PER_LAYER_EMBEDDING_FILENAMES = { + "embed_tokens_per_layer.weights", +} + + +class _OpenedTensor: + def __init__( + self, + *, + path: Path, + precision: int, + shape: tuple[int, ...], + data: np.memmap, + scales: np.memmap | None, + group_size: int, + num_groups: int, + is_interleaved: bool, + original_n: int, + alignment: int, + ) -> None: + self.path = path + self.precision = precision + self.shape = shape + self.data = data + self.scales = scales + self.group_size = group_size + self.num_groups = num_groups + self.is_interleaved = is_interleaved + self.original_n = original_n + self.alignment = alignment + + +def ensure_binding_compatible(binding: WeightBinding, source_tensor: object | None = None) -> WeightBinding: + binding = ensure_embedding_binding_compatible(binding) + return _ensure_gemma4_weight_binding_compatible(binding, source_tensor=source_tensor) + + +def ensure_embedding_binding_compatible(binding: WeightBinding) -> WeightBinding: + if binding.kind != "embedding": + return binding + + source_path = Path(binding.path).expanduser().resolve() + opened = _open_cactus_tensor_file(source_path) + if opened.precision != _INT8: + return binding + if len(opened.shape) != 2 or opened.scales is None or opened.group_size <= 0: + return binding + + config = _embedding_cache_config(source_path.name, source_name=binding.source_name) + if config is None: + return binding + + compat_path = source_path.with_name(source_path.stem + f".cq{config['bits']}.weights") + _materialize_cq_embedding_cache( + opened, + compat_path, + bits=int(config["bits"]), + rotation=str(config["rotation"]), + ) + _cleanup_legacy_fp16_cache(source_path) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + +def _embedding_cache_config(filename: str, *, source_name: str) -> dict[str, object] | None: + normalized_source = str(source_name or "") + if filename in _PER_LAYER_EMBEDDING_FILENAMES or normalized_source.endswith("embed_tokens_per_layer.weight"): + return {"bits": 2, "rotation": "hadamard"} + if filename in _TOKEN_EMBEDDING_FILENAMES: + return {"bits": 4, "rotation": "orthogonal"} + return {"bits": 4, "rotation": "orthogonal"} + + +def _ensure_gemma4_weight_binding_compatible( + binding: WeightBinding, + *, + source_tensor: object | None, +) -> WeightBinding: + source_name = str(binding.source_name or "") + source_path = Path(binding.path).expanduser().resolve() + if not _is_gemma4_legacy_int4_binding(binding, source_path=source_path): + return binding + if source_tensor is None: + return binding + + opened = _open_cactus_tensor_file(source_path) + if opened.precision != _INT4 or len(opened.shape) != 2: + return binding + + compat_path = source_path.with_name(source_path.stem + ".cq4.weights") + _materialize_source_cq_weight( + source_tensor, + compat_path, + bits=4, + rotation="hadamard", + scale_factor=gemma4_scale_factor(source_path.name), + source_mtime_ns=source_path.stat().st_mtime_ns, + ) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + +def _is_gemma4_legacy_int4_binding(binding: WeightBinding, *, source_path: Path) -> bool: + if binding.kind != "weight": + return False + source_name = _normalize_gemma4_source_name(str(binding.source_name or "")) + gemma4_like_source = ( + source_name.startswith("model.language_model.") + or source_name.startswith("model.embed_vision.") + or source_name.startswith("model.embed_audio.") + or source_name.startswith("model.vision_tower.") + or source_name.startswith("model.audio_tower.") + ) + if not gemma4_like_source: + return False + parent_name = source_path.parent.name.lower() + if "gemma-4" not in parent_name and "gemma4" not in parent_name: + return False + return source_path.suffix == ".weights" and not source_path.name.endswith(".cq4.weights") + + +def _normalize_gemma4_source_name(source_name: str) -> str: + normalized = source_name.strip() + while True: + next_value = normalized + for prefix in ("module.", "adapter.", "model.model."): + if next_value.startswith(prefix): + next_value = next_value[len(prefix) :] + if next_value == normalized: + break + normalized = next_value + if normalized.startswith("language_model."): + return f"model.{normalized}" + if normalized.startswith("vision_tower.") or normalized.startswith("audio_tower."): + return f"model.{normalized}" + if normalized.startswith("embed_vision.") or normalized.startswith("embed_audio."): + return f"model.{normalized}" + return normalized + + +def _cleanup_legacy_fp16_cache(source_path: Path) -> None: + legacy = source_path.with_name(source_path.stem + ".fp16.weights") + if not legacy.exists(): + return + try: + legacy.unlink() + except OSError: + pass + + +def _materialize_source_cq_weight( + tensor: object, + out_path: Path, + *, + bits: int, + rotation: str, + scale_factor: float = 1.0, + source_mtime_ns: int | None = None, +) -> None: + if source_mtime_ns is not None and out_path.exists() and out_path.stat().st_mtime_ns >= source_mtime_ns: + return + + if rotation == "orthogonal": + cq = quantize_orthogonal(tensor, bits=bits) + else: + cq = quantize_hadamard(tensor, bits=bits, use_gptq=False) + if float(scale_factor) != 1.0: + cq = replace(cq, norms=(cq.norms.astype(np.float32) * float(scale_factor)).astype(np.float16)) + write_cq_tensor(out_path, cq) + + +def _open_cactus_tensor_file(path: str | Path) -> _OpenedTensor: + tensor_path = Path(path).expanduser().resolve() + with tensor_path.open("rb") as handle: + header = handle.read(_HEADER_SIZE) + if len(header) < _HEADER_SIZE: + raise RuntimeError(f"tensor file is too small for a Cactus header: {tensor_path}") + if header[:4] != CACTUS_MAGIC: + raise RuntimeError(f"tensor file is missing the CACT header: {tensor_path}") + + flags = struct.unpack_from(" 0) + precision = int(struct.unpack_from(" 0 else 0 + data_offset = align_offset(scales_offset + scales_bytes, alignment) if scales_bytes > 0 else aligned_header + + data_element_count = byte_size // np.dtype(dtype).itemsize + data = np.memmap(tensor_path, mode="r", dtype=dtype, offset=data_offset, shape=(data_element_count,)) + scales = None + if scales_bytes > 0: + scales = np.memmap( + tensor_path, + mode="r", + dtype=np.float16, + offset=scales_offset, + shape=(scales_bytes // np.dtype(np.float16).itemsize,), + ) + return _OpenedTensor( + path=tensor_path, + precision=precision, + shape=shape, + data=data, + scales=scales, + group_size=group_size, + num_groups=num_groups, + is_interleaved=bool(flags & FLAG_INTERLEAVED), + original_n=original_n, + alignment=alignment, + ) + + +def _materialize_cq_embedding_cache( + opened: _OpenedTensor, + out_path: Path, + *, + bits: int, + rotation: str, +) -> None: + src_mtime_ns = opened.path.stat().st_mtime_ns + if out_path.exists() and out_path.stat().st_mtime_ns >= src_mtime_ns: + return + + if len(opened.shape) != 2: + raise RuntimeError(f"expected rank-2 embedding tensor, got shape={opened.shape}") + + rows = int(opened.original_n or opened.shape[0]) + cols = int(opened.shape[1]) + if rotation == "hadamard" and cols % CQ_GROUP_SIZE != 0: + raise RuntimeError( + f"Hadamard CQ embedding conversion requires hidden dim divisible by {CQ_GROUP_SIZE}, " + f"got {cols} for {opened.path.name}" + ) + + input_scale = _estimate_embedding_input_scale(opened, rows=rows, cols=cols) + precision = int(PRECISION_CQ[int(bits)]) + group_size = cols if rotation == "orthogonal" else CQ_GROUP_SIZE + num_groups = cols // group_size + codebook = make_codebook(group_size, bits).astype(np.float16) + input_scale = np.asarray(input_scale, dtype=np.float16).reshape(cols) + recip = np.minimum(1.0 / np.maximum(input_scale.astype(np.float32), 1e-8), 65504.0).astype(np.float16) + + chunk_rows = _recommended_chunk_rows(cols=cols, rotation=rotation) + norms_tmp = out_path.with_suffix(out_path.suffix + ".norms.tmp") + data_tmp = out_path.with_suffix(out_path.suffix + ".data.tmp") + final_tmp = out_path.with_suffix(out_path.suffix + ".tmp") + + flags = 0 + trailer_parts: list[bytes] = [] + if rotation == "orthogonal": + flags |= FLAG_ORTHOGONAL_ROTATION + trailer_parts.append(make_orthogonal_rotation(group_size, seed=1234).astype(np.float16).tobytes()) + else: + left, right, perm = make_hadamard_components(group_size, seed=1234) + trailer_parts.extend([ + left.tobytes(), + right.tobytes(), + perm.astype(" current: + handle.write(b"\x00" * (data_offset - current)) + with data_tmp.open("rb") as packed_handle: + shutil.copyfileobj(packed_handle, handle, length=8 * 1024 * 1024) + final_tmp.replace(out_path) + finally: + for temp_path in (norms_tmp, data_tmp): + try: + temp_path.unlink() + except FileNotFoundError: + pass + + +def _recommended_chunk_rows(*, cols: int, rotation: str) -> int: + if rotation == "orthogonal": + if cols >= 8192: + return 8 + if cols >= 4096: + return 16 + return 64 + if cols >= 8192: + return 32 + if cols >= 4096: + return 48 + return 96 + + +def _estimate_embedding_input_scale( + opened: _OpenedTensor, + *, + rows: int, + cols: int, + max_sample_rows: int = 2048, + sample_chunk_rows: int = 64, +) -> np.ndarray: + if rows <= 0 or cols <= 0: + return np.ones((cols,), dtype=np.float16) + + stride = max(1, rows // max(1, max_sample_rows)) + sampled = 0 + accum = np.zeros((cols,), dtype=np.float64) + for start in range(0, rows, stride * sample_chunk_rows): + stop = min(start + sample_chunk_rows, rows) + chunk = _dequantize_int8_rows(opened, start, stop).astype(np.float32, copy=False) + accum += np.abs(chunk).sum(axis=0, dtype=np.float64) + sampled += int(stop - start) + if sampled >= max_sample_rows: + break + + if sampled <= 0: + return np.ones((cols,), dtype=np.float16) + + mean_abs = np.maximum(accum / float(sampled), 1e-6) + raw = np.power(mean_abs, -0.5, dtype=np.float64) + raw /= np.exp(np.mean(np.log(np.clip(raw, 1e-6, None)))) + return np.clip(raw, 1.0 / 8.0, 8.0).astype(np.float16) + + +def _quantize_hadamard_chunk( + chunk: np.ndarray, + *, + bits: int, + input_scale: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + work = chunk.astype(np.float32, copy=False) * input_scale.astype(np.float32, copy=False)[None, :] + rows, cols = work.shape + group_size = CQ_GROUP_SIZE + groups = cols // group_size + rotation = make_hadamard_matrix(group_size, seed=1234).astype(np.float32) + codebook = make_codebook(group_size, bits).astype(np.float32) + + indices = np.empty((rows, cols), dtype=np.uint8) + norms = np.empty((rows, groups), dtype=np.float16) + for group_index in range(groups): + lo = group_index * group_size + hi = lo + group_size + group = work[:, lo:hi] + row_norms = np.linalg.norm(group, axis=1).clip(min=1e-8).astype(np.float32, copy=False) + rotated = (group / row_norms[:, None]) @ rotation + idx = np.abs(rotated[..., None] - codebook[None, None, :]).argmin(axis=-1).astype(np.uint8) + indices[:, lo:hi] = idx + norms[:, group_index] = row_norms.astype(np.float16) + return pack_indices_lsb(indices, group_size, bits), norms + + +def _quantize_orthogonal_chunk( + chunk: np.ndarray, + *, + bits: int, + input_scale: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + work = chunk.astype(np.float32, copy=False) * input_scale.astype(np.float32, copy=False)[None, :] + rows, cols = work.shape + rotation = make_orthogonal_rotation(cols, seed=1234).astype(np.float32) + codebook = make_codebook(cols, bits).astype(np.float32) + norms = np.linalg.norm(work, axis=1).clip(min=1e-8).astype(np.float32, copy=False) + rotated = (work / norms[:, None]) @ rotation + indices = np.abs(rotated[..., None] - codebook[None, None, :]).argmin(axis=-1).astype(np.uint8) + return pack_indices_lsb(indices, cols, bits), norms.astype(np.float16).reshape(rows, 1) + + +def _write_cq_header( + handle, + *, + rows: int, + cols: int, + precision: int, + packed_bytes: int, + scales_bytes: int, + group_size: int, + num_groups: int, + flags: int, +) -> None: + handle.write(CACTUS_MAGIC) + handle.write(struct.pack(" np.ndarray: + if opened.scales is None or opened.group_size <= 0: + raise RuntimeError(f"INT8 embedding file is missing grouped scales: {opened.path}") + + rows = stop - start + cols = int(opened.shape[1]) + num_groups = int(opened.num_groups) + row_group = int(opened.group_size) + padded_rows = int(opened.shape[0]) + + if opened.is_interleaved: + block = 4 + padded_cols = ((cols + block - 1) // block) * block + q_blocks = opened.data.reshape(padded_rows // block, padded_cols // block, block, block) + s_blocks = opened.scales.reshape(padded_rows // block, num_groups, block) + block_start = start // block + block_end = (stop + block - 1) // block + q = np.asarray(q_blocks[block_start:block_end], dtype=np.int8).transpose(0, 2, 1, 3).reshape(-1, padded_cols) + s = np.asarray(s_blocks[block_start:block_end], dtype=np.float16).astype(np.float32).transpose(0, 2, 1).reshape(-1, num_groups) + row_offset = start - block_start * block + q = q[row_offset : row_offset + rows, :cols] + s = s[row_offset : row_offset + rows] + else: + q = np.asarray(opened.data, dtype=np.int8).reshape(padded_rows, cols)[start:stop] + s = np.asarray(opened.scales, dtype=np.float16).astype(np.float32).reshape(padded_rows, num_groups)[start:stop] + + out = np.empty((rows, cols), dtype=np.float16) + for group_index in range(num_groups): + lo = group_index * row_group + hi = min(lo + row_group, cols) + if lo >= hi: + break + scales = s[:, group_index : group_index + 1] + out[:, lo:hi] = (q[:, lo:hi].astype(np.float32) * scales).astype(np.float16) + return out diff --git a/python/tests/test_optimize_gemma4_attention.py b/python/tests/test_optimize_gemma4_attention.py new file mode 100644 index 000000000..12de67e9c --- /dev/null +++ b/python/tests/test_optimize_gemma4_attention.py @@ -0,0 +1,104 @@ +from src.transpile.graph_ir import IRGraph +from src.transpile.graph_ir import IRNode +from src.transpile.graph_ir import IRValue +from src.transpile.optimize_graph import normalize_gemma4_decoder_attention_semantics + + +def test_normalize_gemma4_full_attention_uses_sequence_window_compat() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 512), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 1, 800, 512), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 1, 800, 512), dtype="fp16"), + "full_attention_mask": IRValue( + id="full_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "proj": IRValue(id="proj", shape=(1536, 4096), dtype="fp16"), + "out": IRValue(id="out", shape=(1, 800, 1536), dtype="fp16", producer="attn"), + }, + nodes={ + "attn": IRNode( + id="attn", + op="attention_block", + inputs=["query", "key", "value", "full_attention_mask", "proj"], + outputs=["out"], + attrs={ + "has_mask": True, + "is_causal": False, + "window_size": 0, + "attention_output_shape": (1, 800, 4096), + }, + meta={"attention_layer_type": "full_attention"}, + ) + }, + order=["attn"], + inputs=["query", "key", "value", "full_attention_mask"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "input_names": ("query", "key", "value", "full_attention_mask"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value", "proj"] + assert graph.nodes["attn"].attrs["has_mask"] is False + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 800 + assert graph.nodes["attn"].meta["gemma4_full_attention_window_compat"] is True + + +def test_normalize_gemma4_sliding_attention_elides_runtime_mask() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 256), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 8, 800, 256), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 8, 800, 256), dtype="fp16"), + "sliding_attention_mask": IRValue( + id="sliding_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "proj": IRValue(id="proj", shape=(1536, 2048), dtype="fp16"), + "out": IRValue(id="out", shape=(1, 800, 1536), dtype="fp16", producer="attn"), + }, + nodes={ + "attn": IRNode( + id="attn", + op="attention_block", + inputs=["query", "key", "value", "sliding_attention_mask", "proj"], + outputs=["out"], + attrs={ + "has_mask": True, + "is_causal": False, + "window_size": 0, + "attention_output_shape": (1, 800, 2048), + }, + meta={"attention_layer_type": "sliding_attention"}, + ) + }, + order=["attn"], + inputs=["query", "key", "value", "sliding_attention_mask"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + "input_names": ("query", "key", "value", "sliding_attention_mask"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value", "proj"] + assert graph.nodes["attn"].attrs["has_mask"] is False + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 512 diff --git a/python/tests/test_transpile_weight_compat.py b/python/tests/test_transpile_weight_compat.py new file mode 100644 index 000000000..5ca6204c8 --- /dev/null +++ b/python/tests/test_transpile_weight_compat.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +from src.tensor_io import save_tensor_with_header +from src.transpile.weight_binding import WeightBinding +from src.transpile.weight_compat import _open_cactus_tensor_file +from src.transpile.weight_compat import ensure_binding_compatible +from src.transpile.weight_compat import ensure_embedding_binding_compatible + + +def _write_grouped_int8_embedding(path: Path, rows: int, cols: int) -> None: + rng = np.random.default_rng(1234) + tensor = rng.standard_normal((rows, cols), dtype=np.float32) + save_tensor_with_header(tensor, path, precision="INT8", model_type="generic") + + +def test_token_embedding_binding_upgrades_to_cq4(tmp_path: Path) -> None: + source = tmp_path / "token_embeddings.weights" + _write_grouped_int8_embedding(source, rows=8, cols=128) + + legacy_fp16 = tmp_path / "token_embeddings.fp16.weights" + legacy_fp16.write_bytes(b"legacy") + + binding = WeightBinding( + path=str(source), + kind="embedding", + source_name="model.embed_tokens.weight", + ) + compat = ensure_embedding_binding_compatible(binding) + + assert compat.path.endswith(".cq4.weights") + assert compat.kind == "embedding" + assert not legacy_fp16.exists() + + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 6 + assert opened.shape == (8, 128) + assert opened.scales is not None + + +def test_per_layer_embedding_binding_upgrades_to_cq2(tmp_path: Path) -> None: + source = tmp_path / "embed_tokens_per_layer.weights" + _write_grouped_int8_embedding(source, rows=8, cols=256) + + binding = WeightBinding( + path=str(source), + kind="embedding", + source_name="model.language_model.embed_tokens_per_layer.weight", + ) + compat = ensure_embedding_binding_compatible(binding) + + assert compat.path.endswith(".cq2.weights") + + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 4 + assert opened.shape == (8, 256) + assert opened.scales is not None + + +def test_gemma4_per_layer_projection_binding_upgrades_legacy_int4(tmp_path: Path) -> None: + rng = np.random.default_rng(1234) + source = tmp_path / "per_layer_model_proj.weights" + tensor = rng.standard_normal((8, 128), dtype=np.float32) + save_tensor_with_header(tensor, source, precision="INT4", model_type="generic") + + binding = WeightBinding( + path=str(source), + kind="weight", + source_name="module.model.model.language_model.per_layer_model_projection.weight", + ) + compat = ensure_binding_compatible(binding, source_tensor=tensor) + + assert compat.path.endswith(".cq4.weights") + opened = _open_cactus_tensor_file(compat.path) + assert opened.precision == 6 + assert opened.shape == (8, 128) + assert opened.scales is not None From 06c67a4da6a80ecccd9bc6c0f4962d4e71c4eb3b Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Tue, 12 May 2026 18:08:23 -0700 Subject: [PATCH 09/11] Added cli support to run transpiled models --- cactus-graph/src/execute.cpp | 44 +- python/src/cli.py | 146 ++++- python/src/graph.py | 10 +- .../src/transpile/component_bundle_runtime.py | 30 +- python/src/transpile/gemma4_runtime.py | 526 ++++++++++++++++++ python/src/transpile/lower.py | 115 +++- python/src/transpile/optimize_graph.py | 80 ++- python/src/transpile/weight_compat.py | 84 +-- python/tests/test_cli_transpile_defaults.py | 72 +++ .../tests/test_optimize_gemma4_attention.py | 120 ++++ python/tests/test_transpile_weight_compat.py | 19 + 11 files changed, 1155 insertions(+), 91 deletions(-) create mode 100644 python/src/transpile/gemma4_runtime.py create mode 100644 python/tests/test_cli_transpile_defaults.py diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index f2f7bc471..2a37ee681 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -297,11 +297,20 @@ void CactusGraph::execute(const std::string& profile_file) { BufferPool& pool = buffer_pool_; const size_t n = nodes_.size(); + auto get_env_int = [](const char* name, int fallback) -> int { + const char* val = std::getenv(name); + return val ? std::atoi(val) : fallback; + }; + + bool trace_execution = get_env_int("CACTUS_TRACE_EXECUTE", 0) != 0; bool need_debug = !profile_file.empty(); if (!need_debug) { static const bool env_debug = check_debug_env(); need_debug = env_debug; } + if (trace_execution) { + need_debug = true; + } if (!need_debug) { for (size_t i = 0; i < n; ++i) { @@ -331,11 +340,6 @@ void CactusGraph::execute(const std::string& profile_file) { } } - auto get_env_int = [](const char* name, int fallback) -> int { - const char* val = std::getenv(name); - return val ? std::atoi(val) : fallback; - }; - auto get_env_str = [](const char* name) -> std::string { const char* val = std::getenv(name); return val ? std::string(val) : std::string(); @@ -390,11 +394,28 @@ void CactusGraph::execute(const std::string& profile_file) { for (size_t node_idx = 0; node_idx < n; ++node_idx) { auto& node = nodes_[node_idx]; - if (node->op_type != OpType::INPUT) { - node->output_buffer.allocate_from_pool(pool); + if (node->op_type == OpType::INPUT) { + continue; } - if (enable_profiling && node->op_type != OpType::INPUT) { + node->output_buffer.allocate_from_pool(pool); + + if (trace_execution) { + std::cerr << "[cactus:execute] begin idx=" << node_idx + << " id=" << node->id + << " op=" << get_op_name(node->op_type) + << " shape=["; + for (size_t dim_idx = 0; dim_idx < node->output_buffer.shape.size(); ++dim_idx) { + if (dim_idx > 0) std::cerr << ","; + std::cerr << node->output_buffer.shape[dim_idx]; + } + std::cerr << "]" << std::endl; + } + + if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE) { + dispatch_node(*node, nodes_, node_index_map_); + populated_node_ids_.insert(node->id); + } else if (enable_profiling) { auto start = std::chrono::high_resolution_clock::now(); dispatch_node(*node, nodes_, node_index_map_); if (node->op_type == OpType::PERSISTENT) { @@ -419,6 +440,13 @@ void CactusGraph::execute(const std::string& profile_file) { populated_node_ids_.insert(node->id); } } + + if (trace_execution) { + std::cerr << "[cactus:execute] done idx=" << node_idx + << " id=" << node->id + << " op=" << get_op_name(node->op_type) + << std::endl; + } } std::unique_ptr capture_file_stream; diff --git a/python/src/cli.py b/python/src/cli.py index 1682e82c8..b8e5572e1 100644 --- a/python/src/cli.py +++ b/python/src/cli.py @@ -62,7 +62,11 @@ def _python_runtime_library_path() -> Path: return PROJECT_ROOT / "cactus" / "build" / f"libcactus{suffix}" -def _ensure_python_runtime_library() -> Path: +def _static_cactus_library_path() -> Path: + return PROJECT_ROOT / "cactus" / "build" / "libcactus.a" + + +def _build_static_cactus_library() -> Path: build_script = PROJECT_ROOT / "cactus" / "build.sh" if not build_script.exists(): raise RuntimeError( @@ -70,14 +74,118 @@ def _ensure_python_runtime_library() -> Path: f"Expected: {build_script}" ) + build = subprocess.run([str(build_script)], cwd=PROJECT_ROOT / "cactus") + if build.returncode != 0: + raise RuntimeError("Failed to build the Cactus static runtime") + + static_library_path = _static_cactus_library_path() + if not static_library_path.exists(): + raise RuntimeError( + "The Cactus build completed, but the static library was not produced.\n" + f"Expected: {static_library_path}" + ) + return static_library_path + + +def _public_cactus_api_symbols(static_library_path: Path) -> list[str]: + if platform.system() == "Darwin": + command = ["nm", "-gU", str(static_library_path)] + else: + command = ["nm", "-g", "--defined-only", str(static_library_path)] + + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError( + "Failed to inspect the Cactus static runtime symbols.\n" + f"Command: {' '.join(command)}\n" + f"{result.stderr.strip()}" + ) + + symbols: list[str] = [] + for line in result.stdout.splitlines(): + parts = line.split() + if not parts: + continue + symbol = parts[-1].strip() + normalized = symbol[1:] if symbol.startswith("_") else symbol + if not normalized.startswith("cactus_"): + continue + if symbol not in symbols: + symbols.append(symbol) + if not symbols: + raise RuntimeError(f"Could not find any public cactus_* symbols in {static_library_path}") + return symbols + + +def _link_python_runtime_library(*, static_library_path: Path, library_path: Path) -> None: + build_dir = library_path.parent + build_dir.mkdir(parents=True, exist_ok=True) + if library_path.exists(): + library_path.unlink() + + exported_symbols = _public_cactus_api_symbols(static_library_path) + system = platform.system() + if system == "Darwin": + compiler = shutil.which("clang++") or shutil.which("c++") + if not compiler: + raise RuntimeError("Failed to find a C++ compiler for linking libcactus.dylib") + + command = [ + compiler, + "-dynamiclib", + "-o", + str(library_path), + *[f"-Wl,-u,{symbol}" for symbol in exported_symbols], + str(static_library_path), + "-Wl,-install_name,@rpath/libcactus.dylib", + "-lcurl", + "-framework", + "Accelerate", + "-framework", + "CoreML", + "-framework", + "Foundation", + "-framework", + "Security", + "-framework", + "SystemConfiguration", + "-framework", + "CFNetwork", + ] + else: + compiler = shutil.which("g++") or shutil.which("c++") + if not compiler: + raise RuntimeError("Failed to find a C++ compiler for linking libcactus.so") + + command = [ + compiler, + "-shared", + "-o", + str(library_path), + *[f"-Wl,--undefined={symbol}" for symbol in exported_symbols], + str(static_library_path), + "-lcurl", + "-pthread", + "-ldl", + "-lm", + ] + + result = subprocess.run(command, cwd=build_dir) + if result.returncode != 0 or not library_path.exists(): + raise RuntimeError( + "Failed to link the Cactus shared runtime.\n" + f"Expected: {library_path}" + ) + + +def _ensure_python_runtime_library() -> Path: library_path = _python_runtime_library_path() if library_path.exists(): return library_path print_color(YELLOW, "Building Cactus shared runtime for transpiler...") - build = subprocess.run([str(build_script)], cwd=PROJECT_ROOT / "cactus") - if build.returncode != 0: - raise RuntimeError("Failed to build the Cactus shared runtime") + static_library_path = _build_static_cactus_library() + _link_python_runtime_library(static_library_path=static_library_path, library_path=library_path) if not library_path.exists(): raise RuntimeError( "The Cactus build completed, but the shared library was not produced.\n" @@ -1073,25 +1181,10 @@ def cmd_build_python(args): print(" Ubuntu: sudo apt-get install cmake") return 1 - cactus_dir = PROJECT_ROOT / "cactus" - build_script = cactus_dir / "build.sh" - if not build_script.exists(): - print_color(RED, f"Error: build.sh not found at {build_script}") - return 1 - - result = run_command(str(build_script), cwd=cactus_dir, check=False) - if result.returncode != 0: - print_color(RED, "Build failed") - return 1 - - if platform.system() == "Darwin": - lib_name = "libcactus.dylib" - else: - lib_name = "libcactus.so" - - lib_path = cactus_dir / "build" / lib_name - if not lib_path.exists(): - print_color(RED, f"Shared library not found at {lib_path}") + try: + lib_path = _ensure_python_runtime_library() + except Exception as exc: + print_color(RED, f"Error: {exc}") return 1 print_color(GREEN, "Python build complete!") @@ -1186,6 +1279,8 @@ def cmd_transpile(args): default_weights_dir = get_weights_dir(model_id) if default_weights_dir.exists(): command.extend(["--weights-dir", str(default_weights_dir)]) + if not getattr(args, "execute_after_transpile", False) and "--skip-execute" not in extra_args: + command.append("--skip-execute") command.extend(extra_args) env = os.environ.copy() env["CACTUS_LIB_PATH"] = str(transpile_lib) @@ -2292,6 +2387,11 @@ def create_parser(): transpile_parser = subparsers.add_parser('transpile', help='Transpile a HuggingFace model into Cactus component graphs') transpile_parser.add_argument('model_id', help='HuggingFace model ID to transpile (aliases: gemma4, parakeet, whisper, qwen, lfm)') + transpile_parser.add_argument( + '--execute-after-transpile', + action='store_true', + help='Also run the transpiled graph immediately. By default `cactus transpile` saves the bundle and skips execution.', + ) run_transpiled_parser = subparsers.add_parser('run-transpiled', help='Run a saved transpiled component bundle') run_transpiled_parser.add_argument('bundle_dir', diff --git a/python/src/graph.py b/python/src/graph.py index 541c74608..b6a257963 100644 --- a/python/src/graph.py +++ b/python/src/graph.py @@ -9,7 +9,11 @@ class Graph: INT8 = 0 FP16 = 1 FP32 = 2 - INT4 = 3 + CQ1 = 3 + CQ2 = 4 + CQ3 = 5 + CQ4 = 6 + INT4 = CQ1 CPU = 0 NPU = 1 ACT_SILU = 0 @@ -1038,7 +1042,7 @@ def _coerce_input_array(self, data, precision): arr = np.ascontiguousarray(arr, dtype=np.float16) elif precision == self.FP32: arr = np.ascontiguousarray(arr, dtype=np.float32) - elif precision == self.INT4: + elif precision in (self.INT4, self.CQ2, self.CQ3, self.CQ4): arr = np.ascontiguousarray(arr, dtype=np.uint8) else: raise RuntimeError("unsupported precision") @@ -1238,7 +1242,7 @@ def numpy(self): arr = np.ctypeslib.as_array((ctypes.c_float * num_elements).from_address(out_ptr.value)) elif precision == Graph.INT8: arr = np.ctypeslib.as_array((ctypes.c_int8 * num_elements).from_address(out_ptr.value)) - elif precision == Graph.INT4: + elif precision in (Graph.INT4, Graph.CQ2, Graph.CQ3, Graph.CQ4): arr = np.ctypeslib.as_array((ctypes.c_uint8 * int(info.byte_size)).from_address(out_ptr.value)) return arr.copy() else: diff --git a/python/src/transpile/component_bundle_runtime.py b/python/src/transpile/component_bundle_runtime.py index 54a58e7bd..b7a9b2062 100644 --- a/python/src/transpile/component_bundle_runtime.py +++ b/python/src/transpile/component_bundle_runtime.py @@ -20,6 +20,10 @@ from src.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform from src.transpile.audio_preprocess import prepare_cactus_audio_features from src.transpile.canonicalize.cleanup import canonicalize_exported_graph +from src.transpile.gemma4_runtime import ensure_transformers_supports_gemma4 +from src.transpile.gemma4_runtime import patch_torch_flex_attention_compat +from src.transpile.gemma4_runtime import patch_transformers_torchvision_probe +from src.transpile.gemma4_runtime import prepare_gemma4_multimodal_inputs from src.transpile.graph_ir import IRGraph from src.transpile.graph_ir import IRNode from src.transpile.graph_ir import IRValue @@ -40,6 +44,9 @@ Graph.FP16: np.float16, Graph.FP32: np.float32, Graph.INT4: np.uint8, + getattr(Graph, "CQ2", 4): np.uint8, + getattr(Graph, "CQ3", 5): np.uint8, + getattr(Graph, "CQ4", 6): np.uint8, } @@ -176,6 +183,11 @@ def _load_component_graph_from_ir( if use_raw_ir: canonicalize_exported_graph(ir_graph) optimize_graph(ir_graph) + else: + family = str(ir_graph.meta.get("adapter_family") or ir_graph.meta.get("family") or "").strip().lower() + component = str(ir_graph.meta.get("component", "") or component_entry.get("component", "") or "").strip().lower() + if family == "gemma4" and component == "decoder": + optimize_graph(ir_graph) transpiled = transpile_preoptimized_ir(ir_graph) return LoadedComponentGraph( component=str(component_entry.get("component", "unknown")), @@ -410,7 +422,7 @@ def _torch_dtype_from_name(name: str) -> torch.dtype | None: weights_dir=weights_dir, ) array = np.load(tensor_path, mmap_mode="r") - return torch.from_numpy(np.asarray(array)) + return torch.from_numpy(np.array(array, copy=True)) if isinstance(serialized, (str, int, float, bool)) or serialized is None: return serialized @@ -622,12 +634,14 @@ def _run_multimodal_causal_lm_bundle( if isinstance(stored_audio, str) and stored_audio: resolved_audio = str(Path(stored_audio).expanduser().resolve()) - from examples.transpile_hf_model import _patch_torch_flex_attention_compat - from examples.transpile_hf_model import _patch_transformers_torchvision_probe - from examples.transpile_hf_model import _prepare_gemma4_multimodal_inputs - - _patch_transformers_torchvision_probe() - _patch_torch_flex_attention_compat() + external_transformers_site_packages = ensure_transformers_supports_gemma4() + if external_transformers_site_packages: + print( + "note=using external transformers install for gemma4 runtime: " + f"{external_transformers_site_packages}" + ) + patch_transformers_torchvision_probe() + patch_torch_flex_attention_compat() from transformers import AutoProcessor @@ -640,7 +654,7 @@ def _run_multimodal_causal_lm_bundle( local_files_only=Path(model_source).exists(), trust_remote_code=True, ) - prepared = _prepare_gemma4_multimodal_inputs( + prepared = prepare_gemma4_multimodal_inputs( processor, prompt=resolved_prompt, image_files=resolved_image_files, diff --git a/python/src/transpile/gemma4_runtime.py b/python/src/transpile/gemma4_runtime.py new file mode 100644 index 000000000..e848f1e9d --- /dev/null +++ b/python/src/transpile/gemma4_runtime.py @@ -0,0 +1,526 @@ +from __future__ import annotations + +import builtins +import importlib.util +import sys +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +import numpy as np +import torch + +from src.transpile.audio_preprocess import prepare_native_gemma4_audio_features + + +_TORCHVISION_COMPAT_LIBRARIES: list[object] = [] + + +@dataclass +class PreparedInputs: + names: tuple[str, ...] + tensors: tuple[torch.Tensor, ...] + metadata: dict[str, object] + + +def _transformers_supports_model_module(module_name: str) -> bool: + try: + return importlib.util.find_spec(module_name) is not None + except Exception: + return False + + +def _candidate_external_site_packages() -> list[Path]: + candidates: list[Path] = [] + major_minor = f"python{sys.version_info.major}.{sys.version_info.minor}" + pyenv_versions = Path.home() / ".pyenv" / "versions" + if not pyenv_versions.exists(): + return candidates + for version_dir in sorted(pyenv_versions.iterdir(), reverse=True): + site_packages = version_dir / "lib" / major_minor / "site-packages" + if site_packages.exists(): + candidates.append(site_packages) + return candidates + + +def ensure_transformers_supports_gemma4() -> str | None: + target_module = "transformers.models.gemma4.modeling_gemma4" + if _transformers_supports_model_module(target_module): + return None + + for site_packages in _candidate_external_site_packages(): + candidate = site_packages / Path(*target_module.split(".")) + if not candidate.with_suffix(".py").exists() and not candidate.is_dir(): + continue + sys.path.insert(0, str(site_packages)) + for module_name in list(sys.modules): + root_name = module_name.split(".", 1)[0] + if root_name in {"transformers", "huggingface_hub", "tokenizers"}: + del sys.modules[module_name] + if _transformers_supports_model_module(target_module): + return str(site_packages) + try: + sys.path.remove(str(site_packages)) + except ValueError: + pass + return None + + +def _patch_torchvision_missing_nms_op() -> str | None: + """Keep mismatched torch/torchvision installs importable for processors.""" + + try: + import torchvision # type: ignore # noqa: F401 + return None + except RuntimeError as exc: + if "operator torchvision::nms does not exist" not in str(exc): + return None + except Exception: + return None + + try: + import torchvision.extension as tv_extension # type: ignore + if bool(getattr(tv_extension, "_HAS_OPS", False)): + return None + except Exception: + pass + + try: + library = torch.library.Library("torchvision", "DEF") + library.define("nms(Tensor dets, Tensor scores, float iou_threshold) -> Tensor") + _TORCHVISION_COMPAT_LIBRARIES.append(library) + return "defined missing torchvision::nms operator for torchvision import compatibility" + except Exception: + return None + + +def patch_transformers_torchvision_probe() -> str | None: + has_torchvision = importlib.util.find_spec("torchvision") is not None + has_lzma = importlib.util.find_spec("_lzma") is not None + + if not has_torchvision: + return None + + if has_lzma: + nms_patch_note = _patch_torchvision_missing_nms_op() + return nms_patch_note + + base_note: str | None = None + try: + import backports.lzma as backports_lzma # type: ignore + + sys.modules.setdefault("lzma", backports_lzma) + base_note = "using backports.lzma because this Python build is missing _lzma" + nms_patch_note = _patch_torchvision_missing_nms_op() + if nms_patch_note: + return f"{base_note}; {nms_patch_note}" + return base_note + except Exception: + pass + + class _InterpolationModeStub: + NEAREST = "nearest" + BILINEAR = "bilinear" + BICUBIC = "bicubic" + LANCZOS = "lanczos" + + class _TorchvisionFunctionalStub: + InterpolationMode = _InterpolationModeStub + + def __getattr__(self, name: str): + raise RuntimeError( + "torchvision functionality is unavailable because this Python build is missing _lzma; " + f"attempted to access torchvision.transforms.functional.{name}" + ) + + builtins.F = _TorchvisionFunctionalStub() + builtins.tvF = builtins.F + + import transformers.utils as tf_utils # type: ignore + import transformers.utils.import_utils as tf_import_utils # type: ignore + + @lru_cache + def _disabled() -> bool: + return False + + tf_import_utils.is_torchvision_available = _disabled + tf_import_utils.is_torchvision_v2_available = _disabled + tf_utils.is_torchvision_available = _disabled + tf_utils.is_torchvision_v2_available = _disabled + return "disabled torchvision import checks because this Python build is missing _lzma" + + +def patch_torch_flex_attention_compat() -> str | None: + try: + import torch.nn.attention.flex_attention as flex_attention # type: ignore + except Exception: + return None + + if hasattr(flex_attention, "AuxRequest"): + return None + + class _AuxRequest: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + flex_attention.AuxRequest = _AuxRequest # type: ignore[attr-defined] + return "installed torch flex_attention AuxRequest compatibility stub" + + +def _resolve_audio_sample_rate(processor: object) -> int: + for attr_name in ("feature_extractor", "tokenizer"): + child = getattr(processor, attr_name, None) + sample_rate = getattr(child, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + sample_rate = getattr(processor, "sampling_rate", None) + if isinstance(sample_rate, int) and sample_rate > 0: + return sample_rate + return 16000 + + +_GEMMA4_MULTIMODAL_INPUT_ORDER = ( + "input_ids", + "attention_mask", + "token_type_ids", + "pixel_values", + "pixel_position_ids", + "input_features", + "input_features_mask", +) + + +def _normalize_multimodal_prompt( + prompt: str, + *, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, +) -> str: + normalized = prompt.strip() + prefixes: list[str] = [] + + if image_token and num_images > 0: + image_count = normalized.count(image_token) + if image_count < num_images: + prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) + if audio_token and num_audio_segments > 0: + audio_count = normalized.count(audio_token) + if audio_count < num_audio_segments: + prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) + + if prefixes: + prefix = "\n".join(part for part in prefixes if part) + if normalized: + return f"{prefix}\n{normalized}" + return prefix + return normalized + + +def _build_gemma4_chat_prompt( + *, + prompt: str, + image_token: str | None, + num_images: int, + audio_token: str | None, + num_audio_segments: int, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, +) -> str: + result = "" + normalized_system = system_prompt.strip() + if enable_thinking_if_supported or normalized_system: + result += "<|turn>system\n" + if enable_thinking_if_supported: + result += "<|think|>" + result += normalized_system + result += "\n" + + result += "<|turn>user\n" + if image_token and num_images > 0: + for _ in range(num_images): + result += f"\n\n{image_token}\n\n" + result += prompt + if audio_token and num_audio_segments > 0: + result += "".join(audio_token for _ in range(num_audio_segments)) + result += "\n" + result += "<|turn>model\n" + return result + + +def _gemma4_split_cactus_newline_token_merges(batch: object) -> None: + input_ids = batch.get("input_ids") if hasattr(batch, "get") else None + if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: + return + + expansions = { + 108: (107, 107), + 109: (107, 107, 107), + } + if not any(int(token) in expansions for token in input_ids.reshape(-1).tolist()): + return + + lengths: list[list[int]] = [] + max_len = 0 + for row in input_ids.detach().cpu().tolist(): + row_lengths = [len(expansions.get(int(token), (int(token),))) for token in row] + lengths.append(row_lengths) + max_len = max(max_len, sum(row_lengths)) + + for key in ("input_ids", "attention_mask", "token_type_ids"): + value = batch.get(key) if hasattr(batch, "get") else None + if not isinstance(value, torch.Tensor) or value.ndim != 2: + continue + expanded = torch.full( + (value.shape[0], max_len), + 0, + dtype=value.dtype, + device=value.device, + ) + for row_idx, row in enumerate(value.detach().cpu().tolist()): + out: list[int] = [] + for token_idx, item in enumerate(row): + if key == "input_ids": + out.extend(expansions.get(int(item), (int(item),))) + else: + out.extend([int(item)] * lengths[row_idx][token_idx]) + expanded[row_idx, : len(out)] = torch.tensor(out, dtype=value.dtype, device=value.device) + batch[key] = expanded + + +def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: + if not image_files: + return [] + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for --image-file: {exc}") from exc + + images: list[object] = [] + for image_file in image_files: + path = Path(image_file).resolve() + if not path.exists(): + raise RuntimeError(f"image_file does not exist: {path}") + with Image.open(path) as image: + images.append(image.convert("RGB").copy()) + return images + + +def _get_processor_image_attr(processor: object, name: str, default: object) -> object: + image_processor = getattr(processor, "image_processor", None) + if image_processor is not None and hasattr(image_processor, name): + return getattr(image_processor, name) + if isinstance(image_processor, dict) and name in image_processor: + return image_processor[name] + return default + + +def _prepare_gemma4_native_image_tensors( + processor: object, + image_files: tuple[str, ...], +) -> tuple[torch.Tensor, torch.Tensor] | None: + if not image_files: + return None + try: + from PIL import Image # type: ignore + except Exception as exc: # pragma: no cover + raise RuntimeError(f"Pillow is required for Gemma4 native image preprocessing: {exc}") from exc + + patch_size = int(_get_processor_image_attr(processor, "patch_size", 16)) + pooling_kernel_size = int(_get_processor_image_attr(processor, "pooling_kernel_size", 3)) + max_soft_tokens = int(_get_processor_image_attr(processor, "max_soft_tokens", 280)) + rescale_factor = float(_get_processor_image_attr(processor, "rescale_factor", 1.0 / 255.0)) + max_patches = max_soft_tokens * pooling_kernel_size * pooling_kernel_size + side_multiple = pooling_kernel_size * patch_size + patch_dim = 3 * patch_size * patch_size + if patch_size <= 0 or pooling_kernel_size <= 0 or max_patches <= 0: + return None + + try: + resample_bilinear = Image.Resampling.BILINEAR + except AttributeError: # pragma: no cover + resample_bilinear = Image.BILINEAR + + pixel_batches: list[np.ndarray] = [] + position_batches: list[np.ndarray] = [] + for image_file in image_files: + path = Path(image_file).resolve() + with Image.open(path) as image: + rgb = image.convert("RGB") + width, height = rgb.size + target_pixels = float(max_patches * patch_size * patch_size) + factor = float(np.sqrt(target_pixels / max(1.0, float(width * height)))) + target_h = int(np.floor(factor * height / side_multiple)) * side_multiple + target_w = int(np.floor(factor * width / side_multiple)) * side_multiple + if target_h == 0: + target_h = side_multiple + if target_w == 0: + target_w = side_multiple + if (target_w, target_h) != rgb.size: + rgb = rgb.resize((target_w, target_h), resample=resample_bilinear) + array = np.asarray(rgb, dtype=np.float32) * rescale_factor + + patch_h = target_h // patch_size + patch_w = target_w // patch_size + num_patches = patch_h * patch_w + if num_patches > max_patches: + raise RuntimeError( + f"Gemma4 native image preprocessing produced {num_patches} patches, " + f"but max_patches={max_patches}" + ) + chw = np.transpose(array, (2, 0, 1)) + patches = ( + chw.reshape(3, patch_h, patch_size, patch_w, patch_size) + .transpose(1, 3, 2, 4, 0) + .reshape(num_patches, patch_dim) + ) + + padded_patches = np.zeros((max_patches, patch_dim), dtype=np.float32) + padded_patches[:num_patches] = patches + positions = np.full((max_patches, 2), -1, dtype=np.int64) + valid_positions = np.zeros((num_patches, 2), dtype=np.int64) + for patch_y in range(patch_h): + row_start = patch_y * patch_w + valid_positions[row_start : row_start + patch_w, 0] = np.arange(patch_w, dtype=np.int64) + valid_positions[row_start : row_start + patch_w, 1] = patch_y + positions[:num_patches] = valid_positions + + pixel_batches.append(padded_patches) + position_batches.append(positions) + + return ( + torch.from_numpy(np.stack(pixel_batches, axis=0)), + torch.from_numpy(np.stack(position_batches, axis=0)), + ) + + +def prepare_gemma4_multimodal_inputs( + processor: object | None, + *, + prompt: str, + image_files: tuple[str, ...], + audio_file: str | None, + torch_dtype: torch.dtype, + system_prompt: str = "", + enable_thinking_if_supported: bool = False, + use_gemma4_chat_template: bool = False, +) -> PreparedInputs: + if processor is None: + raise RuntimeError("multimodal Gemma4 transpile requires an AutoProcessor with image and audio support") + + images = _load_image_inputs(image_files) + audio_waveforms: list[np.ndarray] = [] + sample_rate: int | None = None + if audio_file: + sample_rate = _resolve_audio_sample_rate(processor) + from src.transpile.audio_preprocess import load_audio_waveform + + audio_waveforms.append(load_audio_waveform(audio_file, target_sample_rate=sample_rate)) + + image_token = getattr(processor, "image_token", None) + audio_token = getattr(processor, "audio_token", None) + processor_prompt = _normalize_multimodal_prompt( + prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + ) + normalized_prompt = processor_prompt + if use_gemma4_chat_template: + normalized_prompt = prompt.strip() + processor_prompt = _build_gemma4_chat_prompt( + prompt=normalized_prompt, + image_token=image_token if isinstance(image_token, str) else None, + num_images=len(images), + audio_token=audio_token if isinstance(audio_token, str) else None, + num_audio_segments=len(audio_waveforms), + system_prompt=system_prompt, + enable_thinking_if_supported=enable_thinking_if_supported, + ) + + batch = processor( + text=processor_prompt, + images=images or None, + audio=audio_waveforms or None, + return_tensors="pt", + ) + if use_gemma4_chat_template: + _gemma4_split_cactus_newline_token_merges(batch) + native_image_tensors = _prepare_gemma4_native_image_tensors(processor, image_files) + if native_image_tensors is not None: + batch["pixel_values"], batch["pixel_position_ids"] = native_image_tensors + if audio_file and isinstance(batch.get("input_features"), torch.Tensor): + feature_tensor = batch["input_features"] + expected_mels = int(feature_tensor.shape[-1]) + try: + native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( + audio_file, + expected_mels=expected_mels, + torch_dtype=torch_dtype, + ) + except Exception as exc: + print(f"note=falling back to processor gemma4 audio features: {exc}") + batch["input_features"] = feature_tensor.to(dtype=torch_dtype) + fallback_mask = batch.get("input_features_mask") + if isinstance(fallback_mask, torch.Tensor): + batch["input_features_mask"] = fallback_mask.to(dtype=torch.bool) + native_audio_frames = int(fallback_mask.to(dtype=torch.int32).sum().item()) + else: + native_audio_frames = int(feature_tensor.shape[1]) + batch["input_features_mask"] = torch.ones( + (int(feature_tensor.shape[0]), native_audio_frames), + dtype=torch.bool, + ) + batch["native_audio_frames"] = native_audio_frames + else: + batch["input_features"] = native_audio + batch["input_features_mask"] = native_audio_mask + batch["native_audio_frames"] = native_audio_frames + + ordered_keys = [ + key + for key in _GEMMA4_MULTIMODAL_INPUT_ORDER + if isinstance(batch.get(key), torch.Tensor) + ] + if not ordered_keys: + raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") + + tensors: list[torch.Tensor] = [] + for key in ordered_keys: + value = batch[key] + if not isinstance(value, torch.Tensor): + continue + if torch.is_floating_point(value): + value = value.to(dtype=torch_dtype) + tensors.append(value) + + metadata: dict[str, object] = { + "prompt": normalized_prompt, + "processor_prompt": processor_prompt, + "image_files": [str(Path(path).resolve()) for path in image_files], + "input_shapes": { + name: list(tensor.shape) + for name, tensor in zip(ordered_keys, tensors) + }, + } + if audio_file: + metadata["audio_file"] = str(Path(audio_file).resolve()) + if sample_rate is not None: + metadata["sample_rate"] = sample_rate + native_audio_frames = batch.get("native_audio_frames") + if isinstance(native_audio_frames, int): + metadata["native_audio_frames"] = native_audio_frames + + return PreparedInputs( + names=tuple(ordered_keys[: len(tensors)]), + tensors=tuple(tensors), + metadata=metadata, + ) + + +_patch_transformers_torchvision_probe = patch_transformers_torchvision_probe +_patch_torch_flex_attention_compat = patch_torch_flex_attention_compat +_prepare_gemma4_multimodal_inputs = prepare_gemma4_multimodal_inputs diff --git a/python/src/transpile/lower.py b/python/src/transpile/lower.py index 885b93ff0..e12d398cd 100644 --- a/python/src/transpile/lower.py +++ b/python/src/transpile/lower.py @@ -5,6 +5,7 @@ import os from typing import Any +import numpy as np import torch from src.transpile.runtime_compat import Graph @@ -56,6 +57,16 @@ class BroadcastAlias: kind: str +def _is_quantized_runtime_dtype(dtype: int) -> bool: + return int(dtype) in { + int(Graph.INT8), + int(getattr(Graph, "CQ1", 3)), + int(getattr(Graph, "CQ2", 4)), + int(getattr(Graph, "CQ3", 5)), + int(getattr(Graph, "CQ4", 6)), + } + + def transpile_captured(captured: CapturedModel) -> TranspiledGraph: return transpile_ir(captured.ir_graph) @@ -237,7 +248,7 @@ def _matmul_with_quantized_rhs_legalization( pretransposed_rhs: bool = False, output_dtype: int | None = None, ) -> Tensor: - if rhs.dtype in (Graph.INT8, Graph.INT4) and lhs.dtype == Graph.FP32: + if _is_quantized_runtime_dtype(rhs.dtype) and lhs.dtype == Graph.FP32: lhs = g.precision_cast(lhs, Graph.FP16) return g.matmul(lhs, rhs, pretransposed_rhs=pretransposed_rhs, output_dtype=output_dtype) @@ -250,7 +261,7 @@ def _matmul_output_dtype( ) -> int | None: if output_dtype != Graph.FP16: return output_dtype - if rhs.dtype in (Graph.INT8, Graph.INT4): + if _is_quantized_runtime_dtype(rhs.dtype): return output_dtype if lhs.dtype == Graph.FP32 or rhs.dtype == Graph.FP32: return Graph.FP32 @@ -584,7 +595,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> and output_dtype == Graph.FP16 ): matmul_output_dtype = Graph.FP32 - elif bias.dtype == Graph.FP32 and weight.dtype not in (Graph.INT8, Graph.INT4): + elif bias.dtype == Graph.FP32 and not _is_quantized_runtime_dtype(weight.dtype): matmul_output_dtype = Graph.FP32 out = _matmul_with_quantized_rhs_legalization( g, @@ -612,7 +623,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> matmul_output_dtype = _matmul_output_dtype(lhs, rhs, output_dtype=output_dtype) if lhs.dtype == Graph.FP16 and rhs.dtype == Graph.FP16 and bias.dtype == Graph.FP16 and output_dtype == Graph.FP16: matmul_output_dtype = Graph.FP32 - elif bias.dtype == Graph.FP32 and rhs.dtype not in (Graph.INT8, Graph.INT4): + elif bias.dtype == Graph.FP32 and not _is_quantized_runtime_dtype(rhs.dtype): matmul_output_dtype = Graph.FP32 out = _matmul_with_quantized_rhs_legalization(g, lhs, rhs, output_dtype=matmul_output_dtype) out = _trim_padded_last_dim(g, out, output_value.shape if output_value is not None else None) @@ -679,6 +690,8 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> return [h_new, c_new] if op in {"scaled_dot_product_attention", "attention"}: + if _should_lower_gemma4_decoder_attention_without_kernel(ir, node): + return [_lower_gemma4_decoder_attention_without_kernel(g, ir, env, node)] mask = node.attrs.get("mask") mask_tensor: Tensor | None = None additive_mask = bool(node.attrs.get("additive_mask", False)) @@ -2298,6 +2311,100 @@ def _build(dim: int, coord_prefix: tuple[int, ...]) -> Tensor: return _build(0, ()) +def _should_lower_gemma4_decoder_attention_without_kernel(ir: IRGraph, node: IRNode) -> bool: + family = str(ir.meta.get("adapter_family") or ir.meta.get("family") or "").strip().lower() + component = str(ir.meta.get("component", "") or "").strip().lower() + if family != "gemma4" or component != "decoder": + return False + return node.op in {"attention", "scaled_dot_product_attention"} + + +def _lower_gemma4_decoder_attention_without_kernel( + g: Graph, + ir: IRGraph, + env: dict[str, Any], + node: IRNode, +) -> Tensor: + query = _attention_tensor(env, node.inputs[0]) + key = _attention_tensor(env, node.inputs[1]) + value = _attention_tensor(env, node.inputs[2]) + + key_transposed = g.permute(key, (0, 1, 3, 2)) + scores = _lower_static_batched_matmul(g, query, key_transposed, output_dtype=Graph.FP16) + if scores is None: + raise NotImplementedError("Gemma4 decoder attention requires static batched matmul support") + + scale = float(node.attrs.get("scale", 1.0)) + if scale != 1.0: + scores = g.scalar_multiply(_ensure_scalar_math_tensor(g, scores), scale) + + additive_mask: Tensor | None = None + if len(node.inputs) > 3: + mask_tensor = _ensure_fp16_tensor(g, _tensor(env, node.inputs[3])) + if bool(node.attrs.get("additive_mask", False)): + additive_mask = mask_tensor + else: + additive_mask = _lower_binary_op(g, mask_tensor, 1.0, "subtract") + additive_mask = g.scalar_multiply(_ensure_scalar_math_tensor(g, additive_mask), 1.0e4) + else: + query_shape = tuple(int(dim) for dim in query.shape) + key_shape = tuple(int(dim) for dim in key.shape) + if len(query_shape) >= 4 and len(key_shape) >= 4: + query_seq = int(query_shape[-2]) + key_seq = int(key_shape[-2]) + additive_mask = _materialize_gemma4_attention_mask( + g, + query_seq=query_seq, + key_seq=key_seq, + is_causal=bool(node.attrs.get("is_causal", False)), + window_size=int(node.attrs.get("window_size", 0)), + dtype=torch.float16, + ) + + if additive_mask is not None: + scores = _lower_binary_op(g, scores, additive_mask, "add") + + probs = g.softmax(scores, axis=-1) + probs = _ensure_tensor_dtype(g, probs, value.dtype) + out = _lower_static_batched_matmul(g, probs, value, output_dtype=value.dtype) + if out is None: + raise NotImplementedError("Gemma4 decoder attention output requires static batched matmul support") + + output_value = ir.values.get(node.outputs[0]) + output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else out.dtype + if out.dtype != output_dtype: + out = g.precision_cast(out, output_dtype) + return out + + +def _materialize_gemma4_attention_mask( + g: Graph, + *, + query_seq: int, + key_seq: int, + is_causal: bool, + window_size: int, + dtype: torch.dtype, +) -> Tensor | None: + if query_seq <= 0 or key_seq <= 0: + return None + if not is_causal and window_size <= 0: + return None + + q_index = np.arange(query_seq, dtype=np.int32)[:, None] + k_index = np.arange(key_seq, dtype=np.int32)[None, :] + allowed = np.ones((query_seq, key_seq), dtype=np.bool_) + + if is_causal: + allowed &= k_index <= q_index + if window_size > 0 and window_size < key_seq: + allowed &= k_index >= (q_index - (window_size - 1)) + + mask = np.where(allowed, 0.0, -1.0e4).astype(np.float16) + tensor = torch.from_numpy(mask.reshape(1, 1, query_seq, key_seq)).to(dtype=dtype) + return _materialize_constant_tensor(g, tensor) + + def _legalize_matmul_inputs( g: Graph, lhs: Tensor, diff --git a/python/src/transpile/optimize_graph.py b/python/src/transpile/optimize_graph.py index aab276617..c8822bdac 100644 --- a/python/src/transpile/optimize_graph.py +++ b/python/src/transpile/optimize_graph.py @@ -491,6 +491,8 @@ def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: return False changed = False + if _assign_gemma4_decoder_attention_hints_from_graph_meta(graph): + changed = True sliding_window = _graph_sliding_window(graph) for node_id in list(graph.order): node = graph.nodes.get(node_id) @@ -507,8 +509,7 @@ def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: if mask_input_index is not None: mask_value_id = node.inputs[mask_input_index] - mask_value = graph.values.get(mask_value_id) - if mask_value is not None and mask_value.dtype in (None, "bool"): + if _gemma4_can_elide_attention_mask(graph, mask_value_id): del node.inputs[mask_input_index] node.meta["gemma4_full_mask_elided"] = True changed = True @@ -556,8 +557,7 @@ def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: if mask_input_index is not None: mask_value_id = node.inputs[mask_input_index] - mask_value = graph.values.get(mask_value_id) - if mask_value is not None and mask_value.dtype in (None, "bool"): + if _gemma4_can_elide_attention_mask(graph, mask_value_id): mask_info = _extract_sliding_window_mask(graph, mask_value_id) mask_input_name = _graph_input_name(graph, mask_value_id) del node.inputs[mask_input_index] @@ -594,6 +594,78 @@ def normalize_gemma4_decoder_attention_semantics(graph: IRGraph) -> bool: return changed +def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bool: + component = str(graph.meta.get("component", "") or "").strip().lower() + if component != "decoder": + return False + + layer_types = _graph_layer_types(graph) + if not layer_types: + return False + + attention_nodes: list[IRNode] = [] + for node_id in graph.order: + node = graph.nodes.get(node_id) + if node is None or node.op not in {"attention", "scaled_dot_product_attention", "attention_block"}: + continue + attention_nodes.append(node) + + if len(attention_nodes) != len(layer_types): + return False + + changed = False + for layer_index, (node, layer_type) in enumerate(zip(attention_nodes, layer_types, strict=True)): + if "attention_layer_type" not in node.meta: + node.meta["attention_layer_type"] = str(layer_type) + changed = True + if "attention_layer_index" not in node.meta: + node.meta["attention_layer_index"] = int(layer_index) + changed = True + return changed + + +def _gemma4_can_elide_attention_mask( + graph: IRGraph, + value_id: str, + *, + _visited: set[str] | None = None, +) -> bool: + mask_value = graph.values.get(value_id) + if mask_value is None: + return False + if mask_value.dtype in (None, "bool"): + return True + + input_name = _graph_input_name(graph, value_id) + if isinstance(input_name, str) and "attention_mask" in input_name: + return True + + if _visited is None: + _visited = set() + if value_id in _visited: + return False + _visited.add(value_id) + + node = producer(graph, strip_passthrough(graph, value_id)) + if node is None: + return False + if node.op in { + "logical_and", + "logical_or", + "scalar_equal", + "scalar_not_equal", + "greater", + "greater_equal", + "less", + "less_equal", + "aten.__and__.Tensor", + }: + return True + if node.op in {"precision_cast", "view", "expand", "permute", "slice", "index", "where"}: + return any(_gemma4_can_elide_attention_mask(graph, input_id, _visited=_visited) for input_id in node.inputs) + return False + + def fuse_self_attention_blocks(graph: IRGraph) -> bool: changed = False for node_id in list(graph.order): diff --git a/python/src/transpile/weight_compat.py b/python/src/transpile/weight_compat.py index cd72f71df..47e61ed7d 100644 --- a/python/src/transpile/weight_compat.py +++ b/python/src/transpile/weight_compat.py @@ -69,7 +69,7 @@ def __init__( def ensure_binding_compatible(binding: WeightBinding, source_tensor: object | None = None) -> WeightBinding: binding = ensure_embedding_binding_compatible(binding) - return _ensure_gemma4_weight_binding_compatible(binding, source_tensor=source_tensor) + return _ensure_legacy_int4_weight_binding_compatible(binding, source_tensor=source_tensor) def ensure_embedding_binding_compatible(binding: WeightBinding) -> WeightBinding: @@ -107,72 +107,74 @@ def _embedding_cache_config(filename: str, *, source_name: str) -> dict[str, obj return {"bits": 4, "rotation": "orthogonal"} -def _ensure_gemma4_weight_binding_compatible( +def _ensure_legacy_int4_weight_binding_compatible( binding: WeightBinding, *, source_tensor: object | None, ) -> WeightBinding: - source_name = str(binding.source_name or "") source_path = Path(binding.path).expanduser().resolve() - if not _is_gemma4_legacy_int4_binding(binding, source_path=source_path): - return binding - if source_tensor is None: - return binding - opened = _open_cactus_tensor_file(source_path) - if opened.precision != _INT4 or len(opened.shape) != 2: + if not _is_legacy_packed_int4_tensor(opened): return binding compat_path = source_path.with_name(source_path.stem + ".cq4.weights") + if compat_path.exists(): + if _can_materialize_compat_weight(source_tensor): + _materialize_source_cq_weight( + source_tensor, + compat_path, + bits=4, + rotation="hadamard", + scale_factor=_compat_weight_scale_factor(source_path), + source_mtime_ns=source_path.stat().st_mtime_ns, + ) + return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) + + if not _can_materialize_compat_weight(source_tensor): + raise RuntimeError( + "legacy packed INT4 weight is not directly executable in the v2 runtime and is " + f"missing a CQ companion file: {source_path.name}. " + "Rerun `cactus transpile ...` so the transpiler can materialize the CQ weights, " + "or generate them through `cq_convert` first." + ) + _materialize_source_cq_weight( source_tensor, compat_path, bits=4, rotation="hadamard", - scale_factor=gemma4_scale_factor(source_path.name), + scale_factor=_compat_weight_scale_factor(source_path), source_mtime_ns=source_path.stat().st_mtime_ns, ) return WeightBinding(path=str(compat_path), kind=binding.kind, source_name=binding.source_name) -def _is_gemma4_legacy_int4_binding(binding: WeightBinding, *, source_path: Path) -> bool: - if binding.kind != "weight": +def _is_legacy_packed_int4_tensor(opened: _OpenedTensor) -> bool: + if opened.precision != _INT4: return False - source_name = _normalize_gemma4_source_name(str(binding.source_name or "")) - gemma4_like_source = ( - source_name.startswith("model.language_model.") - or source_name.startswith("model.embed_vision.") - or source_name.startswith("model.embed_audio.") - or source_name.startswith("model.vision_tower.") - or source_name.startswith("model.audio_tower.") - ) - if not gemma4_like_source: + if len(opened.shape) != 2 or opened.scales is None or opened.group_size <= 0: return False - parent_name = source_path.parent.name.lower() - if "gemma-4" not in parent_name and "gemma4" not in parent_name: + if not opened.is_interleaved: return False - return source_path.suffix == ".weights" and not source_path.name.endswith(".cq4.weights") + return True -def _normalize_gemma4_source_name(source_name: str) -> str: - normalized = source_name.strip() - while True: - next_value = normalized - for prefix in ("module.", "adapter.", "model.model."): - if next_value.startswith(prefix): - next_value = next_value[len(prefix) :] - if next_value == normalized: - break - normalized = next_value - if normalized.startswith("language_model."): - return f"model.{normalized}" - if normalized.startswith("vision_tower.") or normalized.startswith("audio_tower."): - return f"model.{normalized}" - if normalized.startswith("embed_vision.") or normalized.startswith("embed_audio."): - return f"model.{normalized}" - return normalized +def _can_materialize_compat_weight(source_tensor: object | None) -> bool: + if source_tensor is None: + return False + if isinstance(source_tensor, (str, bytes, bytearray, Path)): + return False + if np.isscalar(source_tensor): + return False + return hasattr(source_tensor, "shape") +def _compat_weight_scale_factor(source_path: Path) -> float: + parent_name = source_path.parent.name.lower() + if "gemma-4" in parent_name or "gemma4" in parent_name: + return gemma4_scale_factor(source_path.name) + return 1.0 + def _cleanup_legacy_fp16_cache(source_path: Path) -> None: legacy = source_path.with_name(source_path.stem + ".fp16.weights") if not legacy.exists(): diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py new file mode 100644 index 000000000..4e0f3b3ce --- /dev/null +++ b/python/tests/test_cli_transpile_defaults.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from pathlib import Path + +from src import cli + + +def _fake_completed_process(returncode: int = 0): + class _Result: + def __init__(self, code: int): + self.returncode = code + + return _Result(returncode) + + +def test_cmd_transpile_defaults_to_skip_execute(monkeypatch) -> None: + parser = cli.create_parser() + args = cli.preprocess_eval_args(parser, ["transpile", "gemma4"]) + + captured: list[tuple[list[str], Path, dict[str, str]]] = [] + + monkeypatch.setattr(cli, "_ensure_python_runtime_library", lambda: Path("/tmp/libcactus.dylib")) + monkeypatch.setattr(cli, "get_weights_dir", lambda model_id: Path("/tmp/missing-weights")) + + def _fake_run(command, cwd=None, env=None): + captured.append((list(command), Path(cwd), dict(env or {}))) + return _fake_completed_process(0) + + monkeypatch.setattr(cli.subprocess, "run", _fake_run) + + rc = cli.cmd_transpile(args) + + assert rc == 0 + assert captured + command, cwd, env = captured[0] + assert command[:4] == [ + cli.sys.executable, + str(cli.PROJECT_ROOT / "python" / "examples" / "transpile_hf_model.py"), + "--model-id", + cli.DEFAULT_MODEL_ID, + ] + assert "--skip-execute" in command + assert cwd == cli.PROJECT_ROOT + assert env["CACTUS_LIB_PATH"] == "/tmp/libcactus.dylib" + + +def test_cmd_transpile_can_execute_immediately_when_requested(monkeypatch) -> None: + parser = cli.create_parser() + args = cli.preprocess_eval_args( + parser, + ["transpile", "gemma4", "--execute-after-transpile", "--artifact-dir", "/tmp/gemma4-bundle"], + ) + + captured: list[list[str]] = [] + + monkeypatch.setattr(cli, "_ensure_python_runtime_library", lambda: Path("/tmp/libcactus.dylib")) + monkeypatch.setattr(cli, "get_weights_dir", lambda model_id: Path("/tmp/missing-weights")) + + def _fake_run(command, cwd=None, env=None): + captured.append(list(command)) + return _fake_completed_process(0) + + monkeypatch.setattr(cli.subprocess, "run", _fake_run) + + rc = cli.cmd_transpile(args) + + assert rc == 0 + assert captured + command = captured[0] + assert "--skip-execute" not in command + assert "--artifact-dir" in command + assert "/tmp/gemma4-bundle" in command diff --git a/python/tests/test_optimize_gemma4_attention.py b/python/tests/test_optimize_gemma4_attention.py index 12de67e9c..65a7e745b 100644 --- a/python/tests/test_optimize_gemma4_attention.py +++ b/python/tests/test_optimize_gemma4_attention.py @@ -102,3 +102,123 @@ def test_normalize_gemma4_sliding_attention_elides_runtime_mask() -> None: assert graph.nodes["attn"].attrs["has_mask"] is False assert graph.nodes["attn"].attrs["is_causal"] is True assert graph.nodes["attn"].attrs["window_size"] == 512 + + +def test_normalize_gemma4_attention_recovers_layer_hints_from_graph_meta() -> None: + graph = IRGraph( + values={ + "query_0": IRValue(id="query_0", shape=(1, 8, 800, 256), dtype="fp16"), + "key_0": IRValue(id="key_0", shape=(1, 8, 800, 256), dtype="fp16"), + "value_0": IRValue(id="value_0", shape=(1, 8, 800, 256), dtype="fp16"), + "sliding_attention_mask": IRValue( + id="sliding_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "out_0": IRValue(id="out_0", shape=(1, 8, 800, 256), dtype="fp16", producer="attn_0"), + "query_1": IRValue(id="query_1", shape=(1, 8, 800, 256), dtype="fp16"), + "key_1": IRValue(id="key_1", shape=(1, 8, 800, 256), dtype="fp16"), + "value_1": IRValue(id="value_1", shape=(1, 8, 800, 256), dtype="fp16"), + "full_attention_mask": IRValue( + id="full_attention_mask", + shape=(1, 1, 800, 800), + dtype="bool", + ), + "out_1": IRValue(id="out_1", shape=(1, 8, 800, 256), dtype="fp16", producer="attn_1"), + }, + nodes={ + "attn_0": IRNode( + id="attn_0", + op="attention", + inputs=["query_0", "key_0", "value_0", "sliding_attention_mask"], + outputs=["out_0"], + attrs={"is_causal": False, "window_size": 0}, + ), + "attn_1": IRNode( + id="attn_1", + op="attention", + inputs=["query_1", "key_1", "value_1", "full_attention_mask"], + outputs=["out_1"], + attrs={"is_causal": False, "window_size": 0}, + ), + }, + order=["attn_0", "attn_1"], + inputs=[ + "query_0", + "key_0", + "value_0", + "sliding_attention_mask", + "query_1", + "key_1", + "value_1", + "full_attention_mask", + ], + outputs=["out_1"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + "layer_types": ("sliding_attention", "full_attention"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn_0"].meta["attention_layer_type"] == "sliding_attention" + assert graph.nodes["attn_0"].meta["attention_layer_index"] == 0 + assert graph.nodes["attn_0"].inputs == ["query_0", "key_0", "value_0"] + assert graph.nodes["attn_0"].attrs["is_causal"] is True + assert graph.nodes["attn_0"].attrs["window_size"] == 512 + assert graph.nodes["attn_1"].meta["attention_layer_type"] == "full_attention" + assert graph.nodes["attn_1"].meta["attention_layer_index"] == 1 + assert graph.nodes["attn_1"].inputs == ["query_1", "key_1", "value_1"] + assert graph.nodes["attn_1"].attrs["is_causal"] is True + assert graph.nodes["attn_1"].attrs["window_size"] == 800 + + +def test_normalize_gemma4_attention_elides_logical_fp16_mask() -> None: + graph = IRGraph( + values={ + "query": IRValue(id="query", shape=(1, 8, 800, 256), dtype="fp16"), + "key": IRValue(id="key", shape=(1, 8, 800, 256), dtype="fp16"), + "value": IRValue(id="value", shape=(1, 8, 800, 256), dtype="fp16"), + "mask_a": IRValue(id="mask_a", shape=(1, 1, 800, 800), dtype="bool"), + "mask_b": IRValue(id="mask_b", shape=(1, 1, 800, 800), dtype="bool"), + "mask_fp16": IRValue(id="mask_fp16", shape=(1, 1, 800, 800), dtype="fp16", producer="mask_and"), + "out": IRValue(id="out", shape=(1, 8, 800, 256), dtype="fp16", producer="attn"), + }, + nodes={ + "mask_and": IRNode( + id="mask_and", + op="logical_and", + inputs=["mask_a", "mask_b"], + outputs=["mask_fp16"], + ), + "attn": IRNode( + id="attn", + op="attention", + inputs=["query", "key", "value", "mask_fp16"], + outputs=["out"], + attrs={"is_causal": False, "window_size": 0}, + meta={"attention_layer_type": "sliding_attention"}, + ), + }, + order=["mask_and", "attn"], + inputs=["query", "key", "value", "mask_a", "mask_b"], + outputs=["out"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder", + "sliding_window": 512, + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn"].inputs == ["query", "key", "value"] + assert graph.nodes["attn"].attrs["is_causal"] is True + assert graph.nodes["attn"].attrs["window_size"] == 512 diff --git a/python/tests/test_transpile_weight_compat.py b/python/tests/test_transpile_weight_compat.py index 5ca6204c8..d5c09a970 100644 --- a/python/tests/test_transpile_weight_compat.py +++ b/python/tests/test_transpile_weight_compat.py @@ -78,3 +78,22 @@ def test_gemma4_per_layer_projection_binding_upgrades_legacy_int4(tmp_path: Path assert opened.precision == 6 assert opened.shape == (8, 128) assert opened.scales is not None + + +def test_decoder_binding_prefers_existing_cq4_companion(tmp_path: Path) -> None: + rng = np.random.default_rng(4321) + source = tmp_path / "layer_0_attn_q.weights" + tensor = rng.standard_normal((8, 128), dtype=np.float32) + save_tensor_with_header(tensor, source, precision="INT4", model_type="generic") + + companion = tmp_path / "layer_0_attn_q.cq4.weights" + save_tensor_with_header(tensor, companion, precision="FP16", model_type="generic") + + binding = WeightBinding( + path=str(source), + kind="weight", + source_name="module.backbone.layers.0.self_attn.q_proj.weight", + ) + compat = ensure_binding_compatible(binding, source_tensor=0) + + assert compat.path == str(companion) From 227273f004e6496e8a327d12f96dd9e321393a6f Mon Sep 17 00:00:00 2001 From: ParkiratS Date: Tue, 12 May 2026 18:57:14 -0700 Subject: [PATCH 10/11] Added missing graph ops --- cactus-engine/cactus_engine.h | 1 + cactus-engine/src/graph_ffi.cpp | 11 + cactus-graph/cactus_graph.h | 3 +- cactus-graph/src/builder.cpp | 17 + cactus-graph/src/execute.cpp | 3 +- cactus-graph/src/ops_math.cpp | 22 + cactus-graph/src/param_io.cpp | 1 + python/examples/transpile_hf_model.py | 33 +- python/src/cactus.py | 210 +++-- python/src/transpile/README.md | 1073 ++++++++++++++++++++++++ python/src/transpile/weight_binding.py | 58 ++ python/tests/test_graph.py | 40 + 12 files changed, 1382 insertions(+), 90 deletions(-) create mode 100644 python/src/transpile/README.md diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index b025cd0fe..8d6945109 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -292,6 +292,7 @@ CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, i CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); CACTUS_FFI_EXPORT int cactus_graph_concat( cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, diff --git a/cactus-engine/src/graph_ffi.cpp b/cactus-engine/src/graph_ffi.cpp index 8e61d5fe0..bcb3d469f 100644 --- a/cactus-engine/src/graph_ffi.cpp +++ b/cactus-engine/src/graph_ffi.cpp @@ -515,6 +515,17 @@ int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus } } +int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { + if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_cumsum"); + try { + *out = static_cast(as_graph(graph)->graph.cumsum(static_cast(x), axis)); + return 0; + } catch (const std::exception& e) { + last_error_message = e.what(); + return -1; + } +} + int cactus_graph_concat(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, cactus_node_t* out) { if (!graph || !out) { last_error_message = "Invalid args to cactus_graph_concat"; diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index f43b715f3..1b552fa64 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -91,7 +91,7 @@ enum class OpType { ABS, POW, FLATTEN, VIEW, MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, BILINEAR_INTERPOLATION, - SUM, MEAN, VARIANCE, MIN, MAX, + SUM, MEAN, VARIANCE, MIN, MAX, CUMSUM, RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, @@ -497,6 +497,7 @@ class CactusGraph { size_t variance(size_t input, int axis); size_t min(size_t input, int axis); size_t max(size_t input, int axis); + size_t cumsum(size_t input, int axis); size_t softmax(size_t input, int axis = -1); size_t topk(size_t input, size_t k); diff --git a/cactus-graph/src/builder.cpp b/cactus-graph/src/builder.cpp index c0e2f92b7..952f29bb3 100644 --- a/cactus-graph/src/builder.cpp +++ b/cactus-graph/src/builder.cpp @@ -213,6 +213,23 @@ size_t CactusGraph::variance(size_t input, int axis) { return reduction_op(OpTyp size_t CactusGraph::min(size_t input, int axis) { return reduction_op(OpType::MIN, input, axis); } size_t CactusGraph::max(size_t input, int axis) { return reduction_op(OpType::MAX, input, axis); } +size_t CactusGraph::cumsum(size_t input, int axis) { + const auto& input_buffer = get_output_buffer(input); + if (input_buffer.shape.empty()) { + throw std::runtime_error("Cumsum requires at least one dimension"); + } + + int actual_axis = axis; + if (actual_axis < 0) { + actual_axis += static_cast(input_buffer.shape.size()); + } + if (actual_axis < 0 || static_cast(actual_axis) >= input_buffer.shape.size()) { + throw std::runtime_error("Invalid axis for cumsum operation"); + } + + return add_node(OpType::CUMSUM, {input}, input_buffer.shape, {.axis = actual_axis, .output_precision = input_buffer.precision}); +} + size_t CactusGraph::rms_norm(size_t input, size_t weight, float epsilon) { OpParams params{.epsilon = epsilon}; return add_node(OpType::RMS_NORM, {input, weight}, {}, params); diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 2a37ee681..cc3e0c49c 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -118,6 +118,7 @@ static bool init_dispatch() { dispatch_flat[static_cast(OpType::VARIANCE)] = compute_reduce_node; dispatch_flat[static_cast(OpType::MIN)] = compute_reduce_node; dispatch_flat[static_cast(OpType::MAX)] = compute_reduce_node; + dispatch_flat[static_cast(OpType::CUMSUM)] = compute_reduce_node; dispatch_flat[static_cast(OpType::FLATTEN)] = compute_reshape_node; dispatch_flat[static_cast(OpType::VIEW)] = compute_reshape_node; dispatch_flat[static_cast(OpType::RESHAPE)] = compute_reshape_node; @@ -200,7 +201,7 @@ static const char* op_type_names[] = { "ABS", "POW", "FLATTEN", "VIEW", "MATMUL", "TRANSPOSE", "RESHAPE", "SLICE", "GATHER", "EMBEDDING", "BILINEAR_INTERPOLATION", - "SUM", "MEAN", "VARIANCE", "MIN", "MAX", + "SUM", "MEAN", "VARIANCE", "MIN", "MAX", "CUMSUM", "RMS_NORM", "ROPE", "ROPE_GPTJ", "SOFTMAX", "ATTENTION", "ATTENTION_INT8_HYBRID", "REL_POS_BIAS", "CONV1D_CAUSAL", "CONV1D_K3", "CONV1D_K7S3", "CONV1D", diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index bda707085..401603f52 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -220,6 +220,28 @@ void compute_reduce_node(GraphNode& node, const std::vector(axis) >= input_buffer.shape.size()) { + throw std::runtime_error("Cumsum axis out of range"); + } + + auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); + const __fp16* input = input_buffer.data_as<__fp16>(); + __fp16* output = node.output_buffer.data_as<__fp16>(); + + for (size_t outer = 0; outer < dims.outer; ++outer) { + for (size_t inner = 0; inner < dims.inner; ++inner) { + float running = 0.0f; + for (size_t axis_index = 0; axis_index < dims.axis_size; ++axis_index) { + size_t flat_index = ((outer * dims.axis_size) + axis_index) * dims.inner + inner; + running += static_cast(input[flat_index]); + output[flat_index] = static_cast<__fp16>(running); + } + } + } + return; + } + if (axis == -1) { switch (node.op_type) { case OpType::SUM: { diff --git a/cactus-graph/src/param_io.cpp b/cactus-graph/src/param_io.cpp index 599aa9203..91d8262a2 100644 --- a/cactus-graph/src/param_io.cpp +++ b/cactus-graph/src/param_io.cpp @@ -169,6 +169,7 @@ const Schema& op_schema(OpType op_type) { {OpType::VARIANCE, {{ParamField::Axis, FieldPersistence::Persistent}}}, {OpType::MIN, {{ParamField::Axis, FieldPersistence::Persistent}}}, {OpType::MAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::CUMSUM, {{ParamField::Axis, FieldPersistence::Persistent}}}, {OpType::INDEX, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::IndexValue, FieldPersistence::Persistent}}}, {OpType::CONCAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, {OpType::CAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, diff --git a/python/examples/transpile_hf_model.py b/python/examples/transpile_hf_model.py index 9ae49f42c..fda855056 100644 --- a/python/examples/transpile_hf_model.py +++ b/python/examples/transpile_hf_model.py @@ -9,6 +9,7 @@ import os import re import sys +from collections.abc import Mapping from collections import Counter from dataclasses import dataclass from dataclasses import fields @@ -46,6 +47,7 @@ from src.transpile.parakeet_tdt_local import greedy_decode_parakeet_tdt_token_ids from src.transpile.parakeet_tdt_local import load_parakeet_tdt_local_model from src.transpile.parakeet_tdt_local import prepare_parakeet_tdt_audio_features +from src.transpile.weight_compat import ensure_binding_compatible _TORCHVISION_COMPAT_LIBRARIES: list[object] = [] @@ -1601,6 +1603,28 @@ def _prepare_audio_inputs( ) +def _tokenize_text_prompt(tokenizer: object, prompt: str) -> torch.Tensor: + apply_chat_template = getattr(tokenizer, "apply_chat_template", None) + if callable(apply_chat_template): + try: + encoded = apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + ) + if isinstance(encoded, torch.Tensor) and encoded.ndim == 2: + return encoded.to(dtype=torch.long) + if isinstance(encoded, Mapping): + input_ids = encoded.get("input_ids") + if isinstance(input_ids, torch.Tensor) and input_ids.ndim == 2: + return input_ids.to(dtype=torch.long) + except Exception: + pass + encoded = tokenizer(prompt, return_tensors="pt") + return encoded["input_ids"].to(dtype=torch.long) + + def _prepare_text_inputs(tokenizer: object, *, prompt: str, input_ids_text: str | None) -> PreparedInputs: if input_ids_text: token_ids = [int(part.strip()) for part in input_ids_text.split(",") if part.strip()] @@ -1608,7 +1632,7 @@ def _prepare_text_inputs(tokenizer: object, *, prompt: str, input_ids_text: str raise ValueError("--input-ids was provided but no ids were parsed") input_ids = torch.tensor([token_ids], dtype=torch.long) else: - input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"] + input_ids = _tokenize_text_prompt(tokenizer, prompt) return PreparedInputs( names=("input_ids",), tensors=(input_ids,), @@ -2277,6 +2301,7 @@ def _lower_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: runtime_inputs = [] bound_constants = [] bound_constant_bindings = [] + bound_constant_value_ids: dict[int, str] = {} for value_id in ir.inputs: value = ir.values[value_id] @@ -2287,10 +2312,13 @@ def _lower_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: for value_id, const in ir.constants.items(): value = ir.values[value_id] binding = _lookup_weight_binding(value) - lowered_const = _lower_constant_value(graph, value, const) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(graph, value, const, binding=binding) env[value_id] = lowered_const if hasattr(lowered_const, "g") and hasattr(lowered_const, "id"): bound_constants.append(lowered_const) + bound_constant_value_ids[int(lowered_const.id)] = str(value_id) if binding is not None: bound_constant_bindings.append( { @@ -2325,6 +2353,7 @@ def _lower_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: runtime_inputs=runtime_inputs, bound_constants=bound_constants, bound_constant_bindings=bound_constant_bindings, + bound_constant_value_ids=bound_constant_value_ids, outputs=outputs, ) diff --git a/python/src/cactus.py b/python/src/cactus.py index 1485d9044..eac3f6a71 100644 --- a/python/src/cactus.py +++ b/python/src/cactus.py @@ -23,6 +23,16 @@ _lib = ctypes.CDLL(str(_LIB_PATH)) + +def _bind_optional(name, argtypes, restype): + try: + fn = getattr(_lib, name) + except AttributeError: + return None + fn.argtypes = argtypes + fn.restype = restype + return fn + cactus_graph_t = ctypes.c_void_p cactus_node_t = ctypes.c_uint64 @@ -92,19 +102,21 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_divide.restype = ctypes.c_int -_lib.cactus_graph_not_equal.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) -] -_lib.cactus_graph_not_equal.restype = ctypes.c_int +_bind_optional( + "cactus_graph_not_equal", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_precision_cast.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_precision_cast.restype = ctypes.c_int -_lib.cactus_graph_quantize_activations.argtypes = [ - cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) -] -_lib.cactus_graph_quantize_activations.restype = ctypes.c_int +_bind_optional( + "cactus_graph_quantize_activations", + [cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_scalar_add.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) @@ -122,14 +134,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_scalar_divide.restype = ctypes.c_int -_lib.cactus_graph_scalar_floor_divide.argtypes = [ - cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) -] -_lib.cactus_graph_scalar_floor_divide.restype = ctypes.c_int -_lib.cactus_graph_scalar_not_equal.argtypes = [ - cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) -] -_lib.cactus_graph_scalar_not_equal.restype = ctypes.c_int +_bind_optional( + "cactus_graph_scalar_floor_divide", + [cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_scalar_not_equal", + [cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_scalar_exp.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] @@ -150,14 +164,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_scalar_log.restype = ctypes.c_int -_lib.cactus_graph_masked_select_prefix.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) -] -_lib.cactus_graph_masked_select_prefix.restype = ctypes.c_int -_lib.cactus_graph_masked_scatter.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) -] -_lib.cactus_graph_masked_scatter.restype = ctypes.c_int +_bind_optional( + "cactus_graph_masked_select_prefix", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_masked_scatter", + [cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_abs.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) @@ -183,10 +199,11 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_reshape.restype = ctypes.c_int -_lib.cactus_graph_expand.argtypes = [ - cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t) -] -_lib.cactus_graph_expand.restype = ctypes.c_int +_bind_optional( + "cactus_graph_expand", + [cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_transpose.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] @@ -242,14 +259,16 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_bilinear_interpolation.restype = ctypes.c_int -_lib.cactus_graph_set_grouped_scales.argtypes = [ - cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_void_p -] -_lib.cactus_graph_set_grouped_scales.restype = ctypes.c_int -_lib.cactus_graph_set_interleaved.argtypes = [ - cactus_graph_t, cactus_node_t, ctypes.c_bool, ctypes.c_size_t -] -_lib.cactus_graph_set_interleaved.restype = ctypes.c_int +_bind_optional( + "cactus_graph_set_grouped_scales", + [cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_void_p], + ctypes.c_int, +) +_bind_optional( + "cactus_graph_set_interleaved", + [cactus_graph_t, cactus_node_t, ctypes.c_bool, ctypes.c_size_t], + ctypes.c_int, +) _lib.cactus_graph_release_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] _lib.cactus_graph_release_weight_pages.restype = ctypes.c_int _lib.cactus_graph_prefetch_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] @@ -396,10 +415,11 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_conv2d_pointwise_1x1.restype = ctypes.c_int -_lib.cactus_graph_conv2d_k3s1p1.argtypes = [ - cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) -] -_lib.cactus_graph_conv2d_k3s1p1.restype = ctypes.c_int +_bind_optional( + "cactus_graph_conv2d_k3s1p1", + [cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t)], + ctypes.c_int, +) _lib.cactus_graph_conv2d.argtypes = [ cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) @@ -497,11 +517,14 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_transcribe.restype = ctypes.c_int -_lib.cactus_detect_language.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_detect_language.restype = ctypes.c_int +_bind_optional( + "cactus_detect_language", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) _lib.cactus_embed.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_float), @@ -532,24 +555,33 @@ class cactus_tensor_info_t(ctypes.Structure): except AttributeError: pass -_lib.cactus_vad.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_vad.restype = ctypes.c_int - -_lib.cactus_diarize.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t -] -_lib.cactus_diarize.restype = ctypes.c_int - -_lib.cactus_embed_speaker.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, - ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, - ctypes.POINTER(ctypes.c_float), ctypes.c_size_t -] -_lib.cactus_embed_speaker.restype = ctypes.c_int +_bind_optional( + "cactus_vad", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) + +_bind_optional( + "cactus_diarize", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ], + ctypes.c_int, +) + +_bind_optional( + "cactus_embed_speaker", + [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ctypes.POINTER(ctypes.c_float), ctypes.c_size_t, + ], + ctypes.c_int, +) _lib.cactus_reset.argtypes = [ctypes.c_void_p] _lib.cactus_reset.restype = None @@ -572,16 +604,19 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_tokenize.restype = ctypes.c_int -_lib.cactus_decode_tokens.argtypes = [ - ctypes.c_void_p, - ctypes.POINTER(ctypes.c_uint32), - ctypes.c_size_t, - ctypes.c_float, - ctypes.c_float, - ctypes.c_size_t, - ctypes.POINTER(ctypes.c_uint32), -] -_lib.cactus_decode_tokens.restype = ctypes.c_int +_bind_optional( + "cactus_decode_tokens", + [ + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_uint32), + ctypes.c_size_t, + ctypes.c_float, + ctypes.c_float, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_uint32), + ], + ctypes.c_int, +) _lib.cactus_score_window.argtypes = [ ctypes.c_void_p, @@ -601,19 +636,22 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_rag_query.restype = ctypes.c_int -_lib.cactus_stream_transcribe_start.argtypes = [ctypes.c_void_p, ctypes.c_char_p] -_lib.cactus_stream_transcribe_start.restype = ctypes.c_void_p - -_lib.cactus_stream_transcribe_process.argtypes = [ - ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, - ctypes.c_char_p, ctypes.c_size_t -] -_lib.cactus_stream_transcribe_process.restype = ctypes.c_int - -_lib.cactus_stream_transcribe_stop.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t -] -_lib.cactus_stream_transcribe_stop.restype = ctypes.c_int +_bind_optional("cactus_stream_transcribe_start", [ctypes.c_void_p, ctypes.c_char_p], ctypes.c_void_p) + +_bind_optional( + "cactus_stream_transcribe_process", + [ + ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t, + ctypes.c_char_p, ctypes.c_size_t, + ], + ctypes.c_int, +) + +_bind_optional( + "cactus_stream_transcribe_stop", + [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t], + ctypes.c_int, +) _lib.cactus_index_init.argtypes = [ctypes.c_char_p, ctypes.c_size_t] _lib.cactus_index_init.restype = ctypes.c_void_p diff --git a/python/src/transpile/README.md b/python/src/transpile/README.md new file mode 100644 index 000000000..ce57bb68d --- /dev/null +++ b/python/src/transpile/README.md @@ -0,0 +1,1073 @@ +# Cactus Transpiler + +This folder contains the Python side of the Cactus transpiler: the code that takes +a Hugging Face or PyTorch model, captures it with `torch.export`, converts that +export into Cactus' intermediate representation (`IRGraph`), canonicalizes and +optimizes the IR, lowers it into a runtime `Graph`, and optionally saves or runs +the result. + +The transpiler has two main output shapes: + +- A single lowered graph for simple tasks such as text logits or encoder-only audio. +- A component bundle for models that are easier to split into staged graphs, most + notably Gemma4 multimodal and Parakeet TDT. + +This README is a code-first walkthrough of the actual flow in the repo today. + +## Related Entry Points + +These files are outside `python/src/transpile`, but they are the public top-level +entry points for the transpiler: + +- `python/src/cli.py` +- `python/examples/transpile_hf_model.py` + +`cactus transpile` is a thin CLI wrapper around `python/examples/transpile_hf_model.py`. +The example script is the real top-level transpile program. + +## CLI Entry Flow + +### `cactus transpile` + +`python/src/cli.py` resolves the transpiler script, ensures the Python runtime +library is available, and forwards most arguments to the example script: + +```python +def cmd_transpile(args): + script_path = PROJECT_ROOT / "python" / "examples" / "transpile_hf_model.py" + transpile_lib = _ensure_python_runtime_library() + model_id = resolve_model_id_alias(args.model_id) + extra_args = list(getattr(args, "extra_args", []) or []) + command = [sys.executable, str(script_path), "--model-id", model_id] + if "--weights-dir" not in extra_args: + default_weights_dir = get_weights_dir(model_id) + if default_weights_dir.exists(): + command.extend(["--weights-dir", str(default_weights_dir)]) + if not getattr(args, "execute_after_transpile", False) and "--skip-execute" not in extra_args: + command.append("--skip-execute") + command.extend(extra_args) +``` + +Two important details: + +- `cactus transpile` saves artifacts and skips execution by default. +- The CLI only has an explicit `--execute-after-transpile` flag; most other + transpile options are forwarded as unknown args to `transpile_hf_model.py`. + +### `cactus run-transpiled` + +Saved bundles are executed through `cmd_run_transpiled()` in `python/src/cli.py`: + +```python +def cmd_run_transpiled(args): + transpile_lib = _ensure_python_runtime_library() + os.environ["CACTUS_LIB_PATH"] = str(transpile_lib) + from .transpile.component_bundle_runtime import run_transpiled_bundle + + result = run_transpiled_bundle( + bundle_dir, + audio_file=getattr(args, "audio_file", None) or getattr(args, "audio", None), + image_files=tuple(image_values), + prompt=getattr(args, "prompt", None), + input_ids=getattr(args, "input_ids", None), + weights_dir=getattr(args, "weights_dir", None), + system_prompt=getattr(args, "system", None), + enable_thinking=bool(getattr(args, "thinking", False)), + ) +``` + +`cactus run ` also auto-detects transpiled bundles by looking for a +`manifest.json` and dispatches to the same runtime path. + +## End-To-End Timeline + +## 1. Parse Args, Infer Task, Load Model Bundle + +The real transpile program lives in `python/examples/transpile_hf_model.py`. +Its `main()` function decides what kind of model/task it is dealing with, validates +optional converted weights, and loads the HF model plus tokenizer/processor: + +```python +if args.task == "auto": + inferred_task = _infer_task_from_config(args.model_id) + ... + if image_files or (args.audio_file and has_multimodal_config and inferred_task == "causal_lm_logits"): + task = "multimodal_causal_lm_logits" + else: + task = inferred_task +else: + task = args.task + +validated_weights_dir = _validate_weights_dir(args.weights_dir.strip() or None, model_id=args.model_id) +weights_dir = str(validated_weights_dir) if validated_weights_dir is not None else None + +model_source, processor_or_tokenizer, model, model_config = _load_transformers_bundle( + model_id=args.model_id, + task=task, + torch_dtype=torch_dtype, + token=args.token, + trust_remote_code=args.trust_remote_code, + local_files_only=args.local_files_only, +) +``` + +### What decides the task? + +`_infer_task_from_config()` looks at: + +- `config.json` architectures +- model type +- known special cases like Parakeet TDT and Whisper +- model id substrings as a last fallback + +Today the main task shapes are: + +- `causal_lm_logits` +- `multimodal_causal_lm_logits` +- `ctc_logits` +- `encoder_hidden_states` +- `tdt_transcription` + +## 2. Prepare Example Inputs + +Before capture, the transpiler builds one concrete example input batch. That +example batch drives both `torch.export` capture and later optional execution. + +The high-level branches are: + +- `_prepare_text_inputs()` for text-only causal LM capture +- `_prepare_audio_inputs()` for speech/audio tasks +- `_prepare_gemma4_multimodal_inputs()` for Gemma4 multimodal + +Example: + +```python +if task == "causal_lm_logits": + prepared = _prepare_text_inputs(...) +elif task == "multimodal_causal_lm_logits": + prepared = _prepare_gemma4_multimodal_inputs(...) +elif task == "tdt_transcription": + prepared = _prepare_audio_inputs(...) +else: + prepared = _prepare_audio_inputs(...) +``` + +The `PreparedInputs` object carries: + +- `names`: logical input names like `input_ids` or `input_features` +- `tensors`: the actual example tensors +- `metadata`: prompt/audio/image/sample-rate info that later gets written into + manifests + +## 3. Canonicalize The Model Interface + +The transpiler does not try to capture every model's native HF forward signature +as-is. It first wraps the model in an adapter that exposes a smaller, more stable +task-specific interface. + +That happens in `model_adapters.py`: + +```python +def canonicalize_model_interface(model, task="causal_lm_logits", *, input_names=None, weights_dir=None): + family = _family_key(model) + ... + if task == "causal_lm_logits": + ... + elif task == "multimodal_causal_lm_logits": + adapter_factory = lambda inner_model: Gemma4MultimodalCausalLMLogitsAdapter( + inner_model, + input_names=resolved_input_names, + weights_dir=weights_dir, + ) + elif task == "ctc_logits": + adapter_factory = lambda inner_model: CTCLogitsAdapter(...) + elif task == "encoder_hidden_states": + adapter_factory = lambda inner_model: EncoderHiddenStatesAdapter(...) +``` + +This step is where family-specific knowledge lives: + +- Gemma/Gemma3/Gemma4/Qwen causal LM wrappers +- Gemma4 multimodal wrappers +- generic CTC and encoder wrappers + +The adapter is also where graph metadata and import hints get attached. + +## 4. Decide Between Monolithic Capture And Component Capture + +After input prep, the script asks `build_component_module_specs()` whether the +current model supports a split component pipeline: + +```python +component_specs = build_component_module_specs( + model, + task=task, + named_tensors=_named_tensor_store(prepared), + weights_dir=weights_dir, +) +``` + +Today that returns component specs for: + +- Gemma4 multimodal: `vision_encoder`, `audio_encoder`, `lm_encoder`, `decoder` +- Parakeet TDT: `audio_encoder`, `decoder` + +If component specs exist and `--component-pipeline` is `auto` or `on`, the script +takes the component path. Otherwise it captures a single wrapped model graph. + +## 5A. Component Pipeline Path + +The component path is driven by `_run_component_pipeline_transpile()` in +`python/examples/transpile_hf_model.py`: + +```python +for spec in component_specs: + captured_components[spec.component] = capture_component_spec(spec, fusion_config=fusion_config) +``` + +Each component spec is captured by `component_pipeline.capture_component_spec()`: + +```python +captured = capture_model(wrapped, spec.example_inputs) +raw_ir_graph = copy.deepcopy(captured.ir_graph) +optimized_ir_graph = copy.deepcopy(captured.ir_graph) +canonicalize_exported_graph(optimized_ir_graph) +optimize_graph(optimized_ir_graph, config=fusion_config) +transpiled_graph = transpile_preoptimized_ir(copy.deepcopy(optimized_ir_graph)) +``` + +So each component goes through the same capture/import/canonicalize/optimize/lower +pipeline independently. + +For Gemma4 multimodal, `model_adapters.py` builds the specs by first computing +example image/audio features, then generating example inputs for the downstream +`lm_encoder` and `decoder` stages. + +## 5B. Monolithic Capture Path + +The non-component path wraps the canonical adapter in `TranspileWrapper` and runs +the full graph through capture: + +```python +wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval() + +captured = capture_model(wrapper, prepared.tensors) +raw_ir_graph = copy.deepcopy(captured.ir_graph) + +canonicalize_exported_graph(captured.ir_graph) +optimize_graph(captured.ir_graph, config=fusion_config) +tg = _lower_preoptimized_ir(captured.ir_graph) +``` + +`TranspileWrapper` mainly exists to pass `weights_dir` into the graph metadata: + +```python +def get_transpile_metadata(self) -> dict[str, object]: + ... + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir +``` + +## 6. Capture With `torch.export` + +`capture_pytorch.py` owns the `torch.export` boundary: + +```python +normalized_kwargs = _inject_export_safe_kwargs(model, _normalize_kwargs(kwargs)) +... +transpile_metadata = _collect_transpile_metadata(model, example_args, example_kwargs) + +ep = export(model, args=example_args, kwargs=example_kwargs, strict=strict) +... +ir_graph = import_captured_to_ir(raw_captured, strict=strict) +``` + +Important behavior here: + +- HF-style `use_cache=False` and `return_dict=False` are injected automatically + when the model signature supports them. +- Metadata providers are collected from any module that defines + `get_transpile_metadata()` or `transpile_metadata()`. +- The captured export is immediately imported into the repo's own IR. + +The capture output is a `CapturedModel` dataclass holding: + +- the exported program +- the FX graph +- the lifted state dict +- the imported `IRGraph` +- the example inputs +- the collected metadata + +## 7. Import Exported FX Into `IRGraph` + +`import_ir.py` converts the exported FX/Export graph into the repo's intermediate +representation: + +```python +ir = IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[], constants={}, meta=dict(graph_meta)) +ctx = ImportContext(strict=strict, transpile_metadata=transpile_metadata) +weights_dir = resolve_transpile_weights_dir(ir.meta) + +for node in captured.graph.nodes: + if node.op == "placeholder": + ... + if node.op == "get_attr": + ... + if node.op == "call_function": + import_call_function(...) + if node.op == "output": + import_output(ir, node, ctx) + +apply_import_semantics(ir) +annotate_ir_components(ir) +verify_ir(ir) +``` + +### What gets imported? + +- Real graph inputs become `IRGraph.inputs` +- lifted parameters/buffers/constants become `IRGraph.constants` +- FX `call_function` nodes become `IRNode`s via `importers.py` + +### How weight binding metadata enters the IR + +When a captured constant corresponds to a known converted weight file, +`import_captured_to_ir()` resolves that mapping early: + +```python +binding = resolve_weight_binding(weights_dir=weights_dir, source_name=target) +... +ir.values[value_id_str].meta.update( + { + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } +) +``` + +That metadata is what later lets lowering use `g.mmap_weights()` or +`g.mmap_embeddings()` instead of embedding the constant payload into the graph. + +## 8. Importers: FX Target To Canonical IR Node + +`importers.py` is the large target-dispatch layer. It normalizes FX/PyTorch targets +to canonical op names and creates `IRNode`s with stable attributes. + +The dispatch point is: + +```python +def import_call_function(ir, node, ctx, *, shape, dtype, torch_op): + op = normalize_target(torch_op) + import_hint = ctx.lookup_import_hint(node, torch_op=torch_op, op_name=op) + importer = OP_IMPORTERS.get(op) + if importer is None: + import_opaque_call_function(...) + else: + importer(...) + _apply_import_hint(ir, node, import_hint) +``` + +Some notable importer behaviors: + +- `normalize.py` reduces many `aten.*` spellings into one canonical name. +- Unknown ops are imported as `kind="opaque"` IR nodes rather than being silently + dropped. +- Import hints can inject attrs/meta into matching nodes after import. +- Some nested exported subgraphs such as `wrap_with_set_grad_enabled` are inlined + explicitly in `import_ir.py`. + +## 9. Import-Time Semantic Rewrites + +Immediately after raw import, `import_semantics.py` performs a first semantic pass: + +```python +def apply_import_semantics(graph: IRGraph) -> IRGraph: + _rewrite_explicit_attention(graph) + _tag_explicit_linear(graph) + _rewrite_early_rope(graph) + rebuild_graph(graph) + verify_ir(graph) + return graph +``` + +This step is intentionally early. It rewrites export patterns into higher-level +meaning before the main optimizer sees them. + +Examples: + +- `scaled_dot_product_attention` becomes semantic `attention` +- explicit linears get tagged +- RoPE patterns can become semantic `rope` before later fusion passes + +## 10. Canonical Cleanup + +`canonicalize/cleanup.py` turns imported IR into a stricter canonical form: + +```python +def canonicalize_exported_graph(graph: IRGraph, *, max_passes: int = 8) -> IRGraph: + for _ in range(max_passes): + for node_id in list(graph.order): + _canonicalize_node(graph, node) + rebuild_graph(graph) + + for node_id in list(graph.order): + _simplify_node(graph, node) + rebuild_graph(graph) + + if _legalize_precisions(graph): + rebuild_graph(graph) + + dce(graph) +``` + +What this pass is responsible for: + +- renaming ops into canonical spellings +- turning shape-only ops into a stable `view` +- rewriting `transpose` and `movedim` into `permute` +- constant-folding trivial expressions +- materializing `ones`, `zeros`, and `arange` constants +- removing no-op casts/views/slices +- inserting FP16 legalizing casts where the runtime requires them +- dead-code elimination + +It also records unsupported canonical ops in `graph.meta["canonical_unsupported_op_counts"]`. + +## 11. Fusion And Graph Optimization + +`optimize_graph.py` runs the semantic fusion passes: + +```python +def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: + canonicalize_exported_graph(graph) + ... + if config.enable_rms_norm: + fuse_rms_norm(graph) + fuse_rms_norm_scale_multiply(graph) + if config.enable_rope: + fuse_rope(graph) + if config.enable_attention: + fuse_attention(graph) + ... + normalize_gemma4_decoder_attention_semantics(graph) + ... + annotate_gold_patterns(graph) + _prune_unused_inputs(graph) +``` + +This is where recognizable subgraphs collapse into semantic or fused nodes such as: + +- `rms_norm` +- `rope` +- `attention` +- `attention_block` +- `self_attention_block` +- `conv_module` +- `lstm_cell` +- `gated_deltanet_prefill` +- `gated_deltanet_decode` + +### Gemma4-specific optimization logic + +Gemma4 gets extra handling in this pass: + +- import hints and graph metadata carry layer types and sliding window info +- exported masks may be elided and rewritten into `window_size` +- some decoder attention behavior is normalized specifically for the saved runtime + +## 12. Lower IR Into A Cactus `Graph` + +`lower.py` is the last compiler stage. It maps `IRValue`s and `IRNode`s into the +runtime `Graph` object from `src.graph`. + +The high-level structure is: + +```python +def transpile_preoptimized_ir(ir: IRGraph) -> TranspiledGraph: + g = Graph() + env = {} + + for value_id in ir.inputs: + tensor = _lower_input_value(g, ir.values[value_id]) + env[value_id] = tensor + + for value_id, const in ir.constants.items(): + binding = _lookup_weight_binding(value) + if binding is not None: + binding = ensure_binding_compatible(binding, source_tensor=const) + lowered_const = _lower_constant_value(g, value, const, binding=binding) + env[value_id] = lowered_const + + for node_id in ir.order: + outputs = _lower_ir_node(g, ir.nodes[node_id], env, ir) +``` + +### Constant lowering + +This is the critical constant binding behavior: + +```python +def _lower_constant_value(g, value, const, *, binding=None): + if binding is not None: + if binding.kind == "embedding": + return g.mmap_embeddings(binding.path) + return g.mmap_weights(binding.path) + + if tensor_value.numel() == 1: + return tensor_value.item() + + return _materialize_constant_tensor(g, tensor_value) +``` + +So constants can become: + +- `g.mmap_weights(path)` +- `g.mmap_embeddings(path)` +- an inline scalar +- a materialized tensor node baked into the graph + +### Where weight compatibility is repaired + +`weight_compat.py` sits between IR metadata and actual lowering. It can rewrite a +binding to a more executable companion file, for example: + +- convert INT8 embeddings to a cached CQ embedding format +- materialize CQ4 companion files for legacy packed INT4 tensors + +### What lowering implements + +`_lower_ir_node()` is the actual op dispatcher. It covers: + +- elementwise math and comparisons +- views, reshape, expand, permute, transpose +- matmul and linear +- embedding and gather/index paths +- reductions and softmax +- conv1d/conv2d +- norms +- fused attention-family nodes +- fused LSTM and Conformer-style blocks + +If an IR op survives all prior passes and still has no lowering, `lower.py` raises: + +```python +raise NotImplementedError(f"unsupported IR op in lowering: {op}") +``` + +### Gemma4 decoder attention fallback + +One especially important special case is that Gemma4 decoder attention can bypass +the runtime attention kernel and be lowered manually as matmul + mask + softmax: + +```python +def _should_lower_gemma4_decoder_attention_without_kernel(ir, node): + family = str(ir.meta.get("adapter_family") or ir.meta.get("family") or "").strip().lower() + component = str(ir.meta.get("component", "") or "").strip().lower() + if family != "gemma4" or component != "decoder": + return False + return node.op in {"attention", "scaled_dot_product_attention"} +``` + +That path exists because the transpiled Gemma4 decoder has some runtime/kernel +compatibility constraints that are easier to control in lowered graph code. + +## 13. Save Artifacts + +After lowering, `transpile_hf_model.py` writes the IR and bundle artifacts. + +For the monolithic path it saves: + +- `raw_ir.json` +- `optimized_ir.json` +- `graph.cactus` +- `graph_bindings.json` +- component subgraphs under `components/` + +For the component path it saves: + +- `raw_ir.json` +- `optimized_ir.json` +- per-component IRs +- `components/manifest.json` +- one lowered graph per component + +The component bundle writer is `_write_component_bundle()`: + +```python +manifest_path = bundle_dir / "manifest.json" +_write_json( + manifest_path, + { + "model_id": model_id, + "model_source": model_source, + "task": task, + "family": family, + "component_order": component_order, + "inputs": _serialize_json_compatible(inputs_metadata), + "components": manifest_components, + }, +) +``` + +Why both a `.cactus` graph and a binding manifest? + +Because `Graph.save()` stores the graph structure, but the file-backed weight +bindings need to be reattached later. The manifests carry: + +- logical input/output names +- runtime node ids +- paths to bound weights or saved `.npy` constants + +## 14. Optional Execution And Reference Compare + +If execution is enabled, the example script will run the lowered graph immediately. + +By task: + +- text LM: run graph, report next token +- Gemma4 multimodal: run pipeline, report next token +- Parakeet TDT: run component encoder + custom decode loop, report transcript +- generic audio: run encoder graph and report output shapes + +Many paths also compare the transpiled output against a PyTorch reference forward +unless `--skip-reference-compare` is set. + +## 15. Run A Saved Bundle Later + +`component_bundle_runtime.py` is the runtime-side loader for saved bundles. + +The public entry is: + +```python +def run_transpiled_bundle(bundle_dir_or_manifest, *, audio_file=None, image_files=(), prompt=None, input_ids=None, weights_dir=None, ...): + bundle_root, manifest = load_component_bundle_manifest(bundle_dir_or_manifest) + component_graphs, manifest = load_saved_component_graphs( + bundle_dir_or_manifest, + weights_dir=resolved_weights_dir, + ) + ... +``` + +There are two ways a saved component can be reloaded: + +1. Load a serialized `.cactus` graph and rebind its constants. +2. Rebuild from saved raw/optimized IR JSON and re-run lowering. + +That second path is handled by `_load_component_graph_from_ir()`: + +```python +ir_graph = _deserialize_saved_ir_graph(...) +if use_raw_ir: + canonicalize_exported_graph(ir_graph) + optimize_graph(ir_graph) +else: + if family == "gemma4" and component == "decoder": + optimize_graph(ir_graph) +transpiled = transpile_preoptimized_ir(ir_graph) +``` + +So saved IR artifacts are not just for inspection; they are also a fallback loading +format that can be re-lowered on demand. + +### Bound constant rebinding at runtime + +The runtime loader reattaches `.weights` and `.npy` files with +`graph.set_external_input(...)`: + +```python +tensor = graph._tensor_from_node(node_id) +graph.set_external_input(tensor, int(tensor_file.data.ctypes.data), dtype=tensor_file.precision) +... +if tensor_file.scales is not None and tensor_file.group_size > 0: + _lib.cactus_graph_set_grouped_scales(...) +``` + +That is why the binding manifests are essential. + +## Major Situations And How They Are Handled + +### Hugging Face models that normally return caches or `ModelOutput` + +`capture_pytorch.py` forces export-safe kwargs: + +- `use_cache=False` +- `return_dict=False` + +This keeps the capture boundary tensor-only. + +### Weights directory exists but has no manifest + +`transpile_hf_model.py` allows a filename-based fallback: + +```python +if not manifest_path.exists(): + has_weight_files = any(root.glob("*.weights")) + if not has_weight_files: + raise RuntimeError(...) + print("note=weights_manifest_missing using filename-based weight binding fallback ...") +``` + +### Weights directory resolves, but no bindings match + +The transpile script fails hard and suggests re-running conversion: + +- component path: if component graphs resolve `0` bindings +- monolithic path: if full optimized IR resolves `0` bindings + +### Local snapshot exists but is incomplete + +`_load_model_source()` will reject incomplete local snapshots when +`--local-files-only` is set. + +### Gemma4 transformers support is missing locally + +`transpile_hf_model.py` and `gemma4_runtime.py` search alternate site-packages and +patch import compatibility for: + +- missing `transformers.models.gemma4` +- missing `_lzma` +- torchvision import probes +- missing `torchvision::nms` +- missing `flex_attention.AuxRequest` + +### Gemma4 local checkpoint key mismatches + +`_repair_gemma4_checkpoint_weights()` remaps local checkpoint keys before loading +the model. + +### Gemma4 multimodal prompt drift + +The transpiler normalizes prompt and tokenization details to match native Cactus: + +- inserts missing image/audio placeholder tokens +- builds a Cactus-style chat prompt when requested +- expands HF newline-merge tokens `108` and `109` back into repeated newline token + `107` +- can replace processor-generated image/audio tensors with native-like versions + +### Parakeet TDT does not run as a single generic forward + +`parakeet_tdt_local.py` provides: + +- a local PyTorch reconstruction of the model +- audio preprocessing +- a greedy TDT decode loop +- component specs for an encoder graph and a recurrent decoder-step graph + +### Some runtime symbols are not present in the current shared library + +`runtime_compat.py` monkey-patches the Python `Graph` wrapper so the transpiler can +still build graphs against the current runtime. It provides compatibility behavior +for items like: + +- generic `matmul` +- `gather(axis=0)` +- limited `conv2d` +- `not_equal` +- scalar math legalization + +### Opaque import is allowed, but execution may still fail later + +If `importers.py` does not know a target, it creates an opaque IR node instead of +dropping it. That preserves information, but lowering still needs a real lowering +case. Opaque nodes are therefore a sign that later lowering may stop with +`unsupported IR op`. + +## Artifact Layout + +A typical transpile artifact directory looks like this: + +```text +transpiled// + raw_ir.json + optimized_ir.json + graph.cactus + graph_bindings.json + result.json + components/ + manifest.json + vision_encoder/ + graph.cactus + bound_constants/ + ... + audio_encoder/ + graph.cactus + lm_encoder/ + graph.cactus + decoder/ + graph.cactus +``` + +Notes: + +- `graph.cactus` is the monolithic lowered graph, when one exists. +- `components/manifest.json` is the portable bundle manifest. +- `graph_bindings.json` and per-component `bound_constant_bindings` tell the loader + how to reattach mmap-backed weights or saved `.npy` constants. + +## File-By-File Map + +### Core transpiler files + +| File | Responsibility | +| --- | --- | +| `audio_preprocess.py` | WAV loading, resampling, generic log-mel extraction, Cactus audio frontend wrappers, Parakeet-native features, Gemma4-native audio features. | +| `capture_pytorch.py` | `torch.export` capture wrapper, export-safe kwargs injection, metadata collection, shape propagation, and `CapturedModel` creation. | +| `component_bundle_runtime.py` | Loader and executor for saved transpiled bundles; reloads `.cactus` graphs or re-lowers saved IR, then runs task-specific bundle executors. | +| `component_partition.py` | Heuristic component labeling of IR nodes/values and subgraph extraction into `audio_encoder` / `vision_encoder` / `lm_encoder` / `decoder`. | +| `component_pipeline.py` | Capture/optimize/lower loop for one component spec, plus runtime execution of a staged pipeline of components. | +| `gemma4_runtime.py` | Shared Gemma4 runtime helpers used by bundle execution: processor compatibility, prompt normalization, native-like multimodal preprocessing. | +| `graph_ir.py` | Definitions for `IRValue`, `IRNode`, `IRGraph`, deep-copy behavior, and IR consistency verification. | +| `import_ir.py` | Top-level importer from exported FX graph into `IRGraph`; resolves lifted constants, weight binding metadata, and inlined subgraphs. | +| `import_semantics.py` | Early semantic rewrites done immediately after import, especially attention and RoPE recognition. | +| `importers.py` | Large target-to-IR importer registry; converts normalized PyTorch ops into canonical `IRNode`s. | +| `lower.py` | Final lowering from optimized IR into runtime `Graph`, including op lowering, constant binding, fused op handling, and execution wrapper. | +| `model_adapters.py` | Family- and task-specific forward adapters, Gemma4 multimodal pipeline adapters, task canonicalization, and component spec builders. | +| `model_patterns.py` | Catalog of "gold patterns" used to annotate recognizable transformer structures after optimization. | +| `normalize.py` | Target-name normalization (`aten.*` to canonical names) and dtype-name normalization into IR spellings. | +| `ops.py` | Canonical op registry and alias table for IR ops, including backend op names and attr schemas. | +| `optimize_graph.py` | Main fusion/optimization pass driver, pattern fusion, Gemma4 attention normalization, and gold-pattern annotation. | +| `parakeet_tdt_local.py` | Local Parakeet TDT model reconstruction, config loader, decoder loop, and component-spec generation. | +| `runtime_compat.py` | Runtime wrapper patches that make the current Python `Graph` API usable for transpiled graphs even when some C symbols are absent. | +| `weight_binding.py` | Resolves IR constants to converted `.weights` files by manifest or filename heuristics; also resolves default weights dirs. | +| `weight_compat.py` | Makes bound weights executable with the current runtime by materializing compatibility companion files when needed. | + +### Canonicalization subdirectory + +| File | Responsibility | +| --- | --- | +| `canonicalize/cleanup.py` | Multi-pass canonical cleanup, constant folding, no-op removal, attr normalization, and precision legalization. | +| `canonicalize/utils.py` | Shared graph rebuild, DCE, node removal, dtype helpers, and constant materialization utilities. | + +### Fusion subdirectory + +| File | Responsibility | +| --- | --- | +| `fusion/__init__.py` | Re-exports the fusion matchers used by `optimize_graph.py`. | +| `fusion/common.py` | Shared fusion helper types and graph traversal helpers. | +| `fusion/linear.py` | Matcher for linear projection patterns. | +| `fusion/rms_norm.py` | Matcher for RMSNorm computational subgraphs. | +| `fusion/rope.py` | Matcher for rotary embedding patterns. | +| `fusion/attention.py` | Matchers for attention, attention blocks, self-attention blocks, and GQA-style projection patterns. | +| `fusion/conv.py` | Matcher for Conformer-like convolution modules. | +| `fusion/lstm.py` | Matcher for LSTM cell subgraphs. | +| `fusion/mlp.py` | Matcher for gated MLP branches such as GELU/SiLU gate-up-down patterns. | +| `fusion/rel_pos_bias.py` | Matcher for relative-position-bias subgraphs. | +| `fusion/deltanet.py` | Matcher for fused gated DeltaNet patterns. | + +## CLI Usage + +## 1. Basic Transpile + +The simplest public command is: + +```bash +cactus transpile +``` + +By default this: + +- resolves the runtime library +- launches `python/examples/transpile_hf_model.py` +- saves artifacts under `./transpiled//` +- adds `--skip-execute`, so it does not run the graph unless you ask + +To also execute right after transpiling: + +```bash +cactus transpile --execute-after-transpile +``` + +## 2. Forwarded Transpile Options + +Most useful transpile options are parsed by `python/examples/transpile_hf_model.py` +and simply forwarded through `cactus transpile`. + +Common ones: + +- `--task` +- `--prompt` +- `--input-ids` +- `--audio-file` +- `--image-file` +- `--weights-dir` +- `--artifact-dir` +- `--component-pipeline` +- `--skip-reference-compare` +- `--graph-filename` +- `--no-fuse-rms-norm` +- `--no-fuse-rope` +- `--no-fuse-attention` + +Examples: + +### Text causal LM + +```bash +cactus transpile \ + --prompt "The capital of France is" \ + --artifact-dir ./transpiled/text_demo +``` + +### Text causal LM with explicit converted weights + +```bash +cactus transpile \ + --weights-dir /path/to/converted_weights \ + --prompt "Write a haiku about compilers" +``` + +### Gemma4 multimodal + +```bash +cactus transpile \ + --task multimodal_causal_lm_logits \ + --image-file /path/to/image.jpg \ + --audio-file /path/to/audio.wav \ + --prompt "Describe what is happening in the image and audio." \ + --weights-dir /path/to/converted_weights +``` + +### Whisper-style encoder capture + +```bash +cactus transpile \ + --audio-file /path/to/sample.wav \ + --task encoder_hidden_states +``` + +### Parakeet TDT + +```bash +cactus transpile \ + --audio-file /path/to/sample.wav \ + --task tdt_transcription \ + --weights-dir /path/to/converted_weights +``` + +If you want the full direct script help: + +```bash +python python/examples/transpile_hf_model.py --help +``` + +## 3. Run A Saved Transpiled Bundle + +Use the bundle root directory or a `manifest.json` path: + +```bash +cactus run-transpiled /path/to/transpiled/model +``` + +For direct bundle execution, audio input is passed with `--file`. The `--audio` +flag belongs to `cactus run`, which may forward into the same runtime after +bundle auto-detection. + +### Text causal LM bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --prompt "The capital of France is" +``` + +Or pass token ids directly: + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --input-ids 2,13,42 +``` + +### Multimodal Gemma4 bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --image-file /path/to/image.jpg \ + --file /path/to/audio.wav \ + --prompt "What do you observe?" \ + --system "Answer concisely." +``` + +### Audio encoder / transcription bundle + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --file /path/to/sample.wav +``` + +### Bundles with bound weights + +If the bundle expects converted mmap weights, pass the weights directory when the +saved paths are not directly reachable: + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --weights-dir /path/to/converted_weights \ + --prompt "Hello" +``` + +### Save result payload + +```bash +cactus run-transpiled /path/to/transpiled/model \ + --prompt "Hello" \ + --result-json ./result.json +``` + +## 4. `cactus run` Also Works + +`cactus run` detects transpiled bundles automatically: + +```bash +cactus run /path/to/transpiled/model --prompt "Hello" +``` + +If you are passing media through `cactus run`, use `--image` and `--audio` +rather than `run-transpiled`'s direct `--file` audio flag. + +Internally, `cli.py` checks whether the path contains a transpiled manifest and +redirects to `cmd_run_transpiled()`. + +## Current Runtime Limitations + +As of the current code: + +- Saved bundle execution is implemented for `causal_lm_logits`, + `multimodal_causal_lm_logits`, `encoder_hidden_states`, and + `tdt_transcription`. +- `ctc_logits` can be transpiled, but `run_transpiled_bundle()` does not currently + implement a saved-bundle executor for that task. +- The component pipeline is model-family-specific; it is not a universal split-IR + path. + +## Useful Environment Variables + +- `CACTUS_TRANSPILER_WEIGHTS_DIR`: + fallback weights directory used by `weight_binding.py` +- `CACTUS_TRANSPILER_WEIGHTS_DIR_`: + family-specific weights directory override +- `CACTUS_TRANSPILER_DEBUG_MMAP=1`: + log constant binding and fallback behavior during lowering +- `CACTUS_GEMMA4_CAPTURE_FP32=1`: + temporarily promote some Gemma4 text modules to FP32 during capture on CPU + +## Practical Summary + +If you only want the high-level lifecycle, it is: + +1. `cactus transpile` launches `python/examples/transpile_hf_model.py`. +2. The script infers a task, loads the HF model, and creates one example input batch. +3. `model_adapters.py` wraps the model into a smaller canonical task interface. +4. `capture_pytorch.py` runs `torch.export`. +5. `import_ir.py` + `importers.py` convert the export to `IRGraph`. +6. `import_semantics.py` adds early semantic rewrites. +7. `canonicalize/cleanup.py` normalizes and simplifies the IR. +8. `optimize_graph.py` fuses patterns into higher-level semantic ops. +9. `lower.py` builds a runtime `Graph`, mmaps bound weights when possible, and + materializes everything else. +10. The script writes `raw_ir.json`, `optimized_ir.json`, `graph.cactus`, and + `components/manifest.json`. +11. `cactus run-transpiled` later reloads those artifacts with + `component_bundle_runtime.py`, rebinds external tensors, and executes the + task-specific bundle runner. diff --git a/python/src/transpile/weight_binding.py b/python/src/transpile/weight_binding.py index c9d829d4b..a04a0978d 100644 --- a/python/src/transpile/weight_binding.py +++ b/python/src/transpile/weight_binding.py @@ -109,6 +109,52 @@ class WeightBinding: "norm_out.bias": "norm_out.bias", } +_WHISPER_GLOBAL_FILENAMES: dict[str, tuple[str, str]] = { + "decoder.embed_tokens.weight": ("decoder_token_embeddings.weights", "embedding"), + "decoder.embed_positions.weight": ("decoder_position_embeddings.weights", "embedding"), + "decoder.layer_norm.weight": ("decoder_norm.weights", "weight"), + "decoder.layer_norm.bias": ("decoder_norm.bias", "weight"), + "proj_out.weight": ("output_weight.weights", "weight"), + "encoder.embed_positions.weight": ("encoder_position_embeddings.weights", "embedding"), + "encoder.conv1.bias": ("encoder_conv1_bias.bias", "weight"), + "encoder.conv1.weight": ("encoder_conv1_weight.weights", "weight"), + "encoder.conv2.bias": ("encoder_conv2_bias.bias", "weight"), + "encoder.conv2.weight": ("encoder_conv2_weight.weights", "weight"), + "encoder.layer_norm.bias": ("encoder_norm_bias.bias", "weight"), + "encoder.layer_norm.weight": ("encoder_norm_weight.weights", "weight"), +} + +_WHISPER_LAYER_FILENAMES: dict[str, str] = { + "self_attn.q_proj.weight": "self_attn_q.weights", + "self_attn.k_proj.weight": "self_attn_k.weights", + "self_attn.v_proj.weight": "self_attn_v.weights", + "self_attn.q_proj.bias": "self_attn_q.bias", + "self_attn.v_proj.bias": "self_attn_v.bias", + "self_attn.out_proj.weight": "self_attn_output.weights", + "self_attn.out_proj.bias": "self_attn_output.bias", + "self_attn_layer_norm.weight": "self_attn_norm.weights", + "self_attn_layer_norm.bias": "self_attn_norm.bias", + "encoder_attn.q_proj.weight": "encoder_attn_q.weights", + "encoder_attn.k_proj.weight": "encoder_attn_k.weights", + "encoder_attn.v_proj.weight": "encoder_attn_v.weights", + "encoder_attn.out_proj.weight": "encoder_attn_output.weights", + "encoder_attn.o_proj.weight": "encoder_attn_output.weights", + "encoder_attn.q_proj.bias": "encoder_attn_q.bias", + "encoder_attn.v_proj.bias": "encoder_attn_v.bias", + "encoder_attn.out_proj.bias": "encoder_attn_output.bias", + "encoder_attn.o_proj.bias": "encoder_attn_output.bias", + "encoder_attn_layer_norm.weight": "encoder_attn_norm.weights", + "encoder_attn_layer_norm.bias": "encoder_attn_norm.bias", + "fc1.weight": "mlp_fc1.weights", + "fc1.bias": "mlp_fc1.bias", + "fc2.weight": "mlp_fc2.weights", + "fc2.bias": "mlp_fc2.bias", + "final_layer_norm.weight": "final_norm.weights", + "final_layernorm.weight": "final_norm.weights", + "final_layer_norm.bias": "final_norm.bias", + "final_layernorm.bias": "final_norm.bias", +} + _GEMMA3N_VISION_TOWER_PREFIX = "model.vision_tower.timm_model." _GEMMA3N_AUDIO_TOWER_PREFIX = "model.audio_tower." _GEMMA4_VISION_TOWER_PREFIX = "model.vision_tower." @@ -387,6 +433,9 @@ def _add(filename: str | None, *, kind: str = "weight") -> None: for candidate in _normalized_source_candidates(source_name): _add(_PARAKEET_GLOBAL_FILENAMES.get(candidate)) + whisper_global = _WHISPER_GLOBAL_FILENAMES.get(candidate) + if whisper_global is not None: + _add(whisper_global[0], kind=whisper_global[1]) gemma_global = _GEMMA_GLOBAL_FILENAMES.get(candidate) if gemma_global is not None: _add(gemma_global[0], kind=gemma_global[1]) @@ -432,6 +481,15 @@ def _add(filename: str | None, *, kind: str = "weight") -> None: if mapped is not None: _add(f"layer_{layer_index}_{mapped}") + whisper_layer_match = re.match(r"^(encoder|decoder)\.layers\.(\d+)\.(.+)$", candidate) + if whisper_layer_match: + block = whisper_layer_match.group(1) + layer_index = int(whisper_layer_match.group(2)) + suffix = whisper_layer_match.group(3) + mapped = _WHISPER_LAYER_FILENAMES.get(suffix) + if mapped is not None: + _add(f"{block}.layer_{layer_index}_{mapped}") + gemma_layer_match = re.match( r"^(?:model(?:\.language_model)?\.)?layers\.(\d+)\.(.+)$", candidate, diff --git a/python/tests/test_graph.py b/python/tests/test_graph.py index e83b9e90d..e1f0c8615 100644 --- a/python/tests/test_graph.py +++ b/python/tests/test_graph.py @@ -453,6 +453,46 @@ def test_softmax_2d(self): np.testing.assert_allclose(out, expected, atol=5e-2) +class TestGraphCumsum(unittest.TestCase): + + def test_cumsum_last_axis(self): + g = Graph() + a = g.input((2, 4)) + y = a.cumsum(axis=-1) + + data = np.array([[1, 2, 3, 4], [0.5, 1.5, 2.5, 3.5]], dtype=np.float16) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = np.cumsum(data.astype(np.float32), axis=-1).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=5e-2) + + def test_save_load_roundtrip_cumsum(self): + g = Graph() + a = g.input((3, 2)) + y = a.cumsum(axis=0) + + data = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float16) + g.set_input(a, data) + g.execute() + expected = y.numpy() + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "cumsum_graph.cg" + g.save(path) + + loaded = Graph.load(path) + loaded_a = Tensor(loaded, a.id, a.shape, a.dtype) + loaded_y = Tensor(loaded, y.id, y.shape, y.dtype) + + loaded.set_input(loaded_a, data) + loaded.execute() + + out = loaded_y.numpy() + np.testing.assert_allclose(out, expected, atol=1e-2) + + class TestGraphSaveLoad(unittest.TestCase): def _rebind_tensor(self, graph, tensor): From ffe6971bac636077849af908e0da78a3a17d6180 Mon Sep 17 00:00:00 2001 From: Brian Ling Date: Wed, 13 May 2026 02:38:30 -0700 Subject: [PATCH 11/11] qwen layer debug and auto casting of rmsnorm, activation to fp16 (#639) --- python/src/transpile/lower.py | 57 +++- .../tests/test_transpile_qwen35_layer_diff.py | 276 ++++++++++++++++++ 2 files changed, 321 insertions(+), 12 deletions(-) create mode 100644 python/tests/test_transpile_qwen35_layer_diff.py diff --git a/python/src/transpile/lower.py b/python/src/transpile/lower.py index e12d398cd..eb46f27c8 100644 --- a/python/src/transpile/lower.py +++ b/python/src/transpile/lower.py @@ -634,19 +634,19 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> return [out] if op == "relu": - return [g.relu(_tensor(env, node.inputs[0]))] + return [g.relu(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] if op == "silu": - return [g.silu(_tensor(env, node.inputs[0]))] + return [g.silu(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] if op == "gelu": - return [g.gelu(_tensor(env, node.inputs[0]))] + return [g.gelu(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] if op == "gelu_erf": - return [g.gelu_erf(_tensor(env, node.inputs[0]))] + return [g.gelu_erf(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] if op == "sigmoid": - return [g.sigmoid(_tensor(env, node.inputs[0]))] + return [g.sigmoid(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] if op == "glu": return [g.glu(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] @@ -656,7 +656,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> return [_lower_softplus(g, x)] if op == "tanh": - return [g.tanh(_tensor(env, node.inputs[0]))] + return [g.tanh(_ensure_runtime_activation_tensor(g, _tensor(env, node.inputs[0])))] if op == "softmax": return [g.softmax(_tensor(env, node.inputs[0]), axis=int(node.attrs.get("axis", -1)))] @@ -1430,14 +1430,14 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> return [g.conv2d(x, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups)] if op == "layer_norm": - x = _tensor(env, node.inputs[0]) - weight = _tensor(env, node.inputs[1]) - bias = _tensor(env, node.inputs[2]) if len(node.inputs) > 2 else None + x = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[0])) + weight = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[1])) + bias = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[2])) if len(node.inputs) > 2 else None return [g.layer_norm(x, weight, bias=bias, eps=float(node.attrs["eps"]))] if op == "rms_norm": - x = _tensor(env, node.inputs[0]) - weight = _tensor(env, node.inputs[1]) + x = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[0])) + weight = _ensure_runtime_norm_tensor(g, _tensor(env, node.inputs[1])) reshape_back: tuple[int, ...] | None = None if len(x.shape) > 2: reshape_back = tuple(int(dim) for dim in x.shape) @@ -1606,7 +1606,14 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> pretransposed_rhs=True, ) z_proj = g.reshape(z_proj, (seq_len * num_v_heads, value_dim)) - y_2d = g.multiply(g.rms_norm(y_2d, norm_weight, eps=eps), g.silu(z_proj)) + y_2d = g.multiply( + g.rms_norm( + _ensure_runtime_norm_tensor(g, y_2d), + _ensure_runtime_norm_tensor(g, norm_weight), + eps=eps, + ), + g.silu(_ensure_runtime_activation_tensor(g, z_proj)), + ) return [g.reshape(y_2d, (batch_size, seq_len, num_v_heads * value_dim))] @@ -2124,6 +2131,32 @@ def _ensure_fp16_tensor(g: Graph, tensor: Tensor) -> Tensor: return g.precision_cast(tensor, Graph.FP16) +def _ensure_runtime_activation_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + if os.environ.get("CACTUS_TRANSPILER_DEBUG_ACTIVATION_CASTS") == "1": + print( + "[transpile:activation-cast] " + f"from_dtype={int(tensor.dtype)} " + f"to_dtype={int(Graph.FP16)} " + f"node_id={int(tensor.id)}" + ) + return g.precision_cast(tensor, Graph.FP16) + + +def _ensure_runtime_norm_tensor(g: Graph, tensor: Tensor) -> Tensor: + if tensor.dtype == Graph.FP16: + return tensor + if os.environ.get("CACTUS_TRANSPILER_DEBUG_NORM_CASTS") == "1": + print( + "[transpile:norm-cast] " + f"from_dtype={int(tensor.dtype)} " + f"to_dtype={int(Graph.FP16)} " + f"node_id={int(tensor.id)}" + ) + return g.precision_cast(tensor, Graph.FP16) + + def _ensure_scalar_math_tensor(g: Graph, tensor: Tensor) -> Tensor: if tensor.dtype == Graph.FP16: return tensor diff --git a/python/tests/test_transpile_qwen35_layer_diff.py b/python/tests/test_transpile_qwen35_layer_diff.py new file mode 100644 index 000000000..b5dc0f817 --- /dev/null +++ b/python/tests/test_transpile_qwen35_layer_diff.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import os +import sys +import unittest +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +try: + import torch +except Exception: # pragma: no cover + torch = None # type: ignore[assignment] + +try: + from transformers import AutoModelForCausalLM +except Exception: # pragma: no cover + AutoModelForCausalLM = None # type: ignore[assignment] + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +if torch is not None and AutoModelForCausalLM is not None: + from src.transpile.capture_pytorch import capture_model + from src.transpile.canonicalize.cleanup import canonicalize_exported_graph + from src.transpile.model_adapters import canonicalize_model_interface + from src.transpile.optimize_graph import optimize_graph + from src.transpile.lower import transpile_preoptimized_ir + + +_RUN_QWEN35_LAYER_DIFF = os.environ.get("CACTUS_RUN_LOCAL_QWEN35_LAYER_DIFF_TEST") == "1" +_DEFAULT_MODEL_ID = os.environ.get("CACTUS_TEST_QWEN35_MODEL_ID", "Qwen/Qwen3.5-2B") +_DEFAULT_INPUT_IDS = os.environ.get("CACTUS_TEST_QWEN35_INPUT_IDS", "151644,8948,198,9707") +_DEFAULT_WEIGHTS_DIR = os.environ.get( + "CACTUS_TEST_QWEN35_WEIGHTS_DIR", + str((Path(__file__).resolve().parents[2] / "weights" / "qwen3.5-2b").resolve()), +) + + +def _resolve_local_model_path(model_id: str) -> str: + candidate = Path(model_id).expanduser() + if candidate.exists(): + return str(candidate.resolve()) + + cache_root = ( + Path.home() + / ".cache" + / "huggingface" + / "hub" + / ("models--" + model_id.replace("/", "--")) + ) + snapshots_dir = cache_root / "snapshots" + if not snapshots_dir.exists(): + raise FileNotFoundError(f"could not find local snapshot for {model_id!r} under {snapshots_dir}") + snapshots = sorted(path for path in snapshots_dir.iterdir() if path.is_dir()) + if not snapshots: + raise FileNotFoundError(f"no snapshots found for {model_id!r} under {snapshots_dir}") + return str(snapshots[-1]) + + +def _parse_input_ids(raw: str) -> list[int]: + token_ids = [int(part.strip()) for part in raw.split(",") if part.strip()] + if not token_ids: + raise ValueError("CACTUS_TEST_QWEN35_INPUT_IDS produced no token ids") + return token_ids + + +def _summarize_ops(ir) -> str: + counts: dict[str, int] = {} + for node_id in ir.order: + op_name = str(ir.nodes[node_id].op) + counts[op_name] = counts.get(op_name, 0) + 1 + items = sorted(counts.items(), key=lambda item: (-item[1], item[0])) + preview = items[:8] + return ",".join(f"{name}:{count}" for name, count in preview) + + +@dataclass +class DiffStats: + label: str + shape: tuple[int, ...] + max_abs_diff: float + mean_abs_diff: float + argmax_ref: int | None + argmax_transpiled: int | None + raw_ir_nodes: int + optimized_ir_nodes: int + top_ops: str + + +if torch is not None and AutoModelForCausalLM is not None: + class _WeightsBoundTranspileWrapper(torch.nn.Module): + def __init__(self, module: torch.nn.Module, *, weights_dir: str | None): + super().__init__() + self.module = module + self.weights_dir = weights_dir + + def forward(self, *args: torch.Tensor) -> torch.Tensor: + return self.module(*args) + + def get_transpile_metadata(self) -> dict[str, object]: + metadata: dict[str, object] = {} + provider = getattr(self.module, "get_transpile_metadata", None) + if callable(provider): + provided = provider() + if isinstance(provided, dict): + metadata.update(provided) + graph_meta: dict[str, object] = {} + base_graph = metadata.get("graph", {}) + if isinstance(base_graph, dict): + graph_meta.update(base_graph) + if self.weights_dir: + graph_meta["weights_dir"] = self.weights_dir + metadata["graph"] = graph_meta + return metadata + + + class Qwen35CheckpointWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module, checkpoint_index: int, *, weights_dir: str | None = None): + super().__init__() + adapter = canonicalize_model_interface(model, task="causal_lm_logits", weights_dir=weights_dir) + if adapter.family != "qwen3_5": + raise ValueError(f"expected qwen3_5 adapter, got family={adapter.family}") + self.adapter = adapter.module + self.checkpoint_index = int(checkpoint_index) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + _, checkpoints = self.adapter.debug_forward(input_ids) + return checkpoints[self.checkpoint_index] + + + class Qwen35LogitsWrapper(torch.nn.Module): + def __init__(self, model: torch.nn.Module, *, weights_dir: str | None = None): + super().__init__() + adapter = canonicalize_model_interface(model, task="causal_lm_logits", weights_dir=weights_dir) + if adapter.family != "qwen3_5": + raise ValueError(f"expected qwen3_5 adapter, got family={adapter.family}") + self.adapter = adapter.module + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + logits, _ = self.adapter.debug_forward(input_ids) + return logits + + +class TestTranspileQwen35LayerDiff(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + if not _RUN_QWEN35_LAYER_DIFF: + raise unittest.SkipTest("set CACTUS_RUN_LOCAL_QWEN35_LAYER_DIFF_TEST=1 to run") + if torch is None: + raise unittest.SkipTest("torch is not available") + if AutoModelForCausalLM is None: + raise unittest.SkipTest("transformers is not available") + + local_model_path = _resolve_local_model_path(_DEFAULT_MODEL_ID) + cls.model_id = _DEFAULT_MODEL_ID + cls.model_path = local_model_path + cls.input_ids_list = _parse_input_ids(_DEFAULT_INPUT_IDS) + cls.input_ids = torch.tensor([cls.input_ids_list], dtype=torch.long) + cls.weights_dir = None + candidate_weights_dir = Path(_DEFAULT_WEIGHTS_DIR).expanduser().resolve() + if candidate_weights_dir.exists(): + cls.weights_dir = str(candidate_weights_dir) + + common_kwargs: dict[str, object] = { + "local_files_only": True, + "torch_dtype": torch.float16, + "device_map": None, + "low_cpu_mem_usage": True, + } + token = os.environ.get("HF_TOKEN") + if token: + common_kwargs["token"] = token + + cls.model = AutoModelForCausalLM.from_pretrained(local_model_path, **common_kwargs).eval() + + adapter = canonicalize_model_interface(cls.model, task="causal_lm_logits", weights_dir=cls.weights_dir) + if adapter.family != "qwen3_5": + raise unittest.SkipTest(f"expected qwen3_5 adapter, got {adapter.family}") + cls.num_layers = int(adapter.module.backbone.config.num_hidden_layers) + + print("") + print(f"[qwen35-layer-diff] model_id={cls.model_id}") + print(f"[qwen35-layer-diff] model_path={cls.model_path}") + print(f"[qwen35-layer-diff] input_ids={cls.input_ids_list}") + print(f"[qwen35-layer-diff] num_layers={cls.num_layers}") + print(f"[qwen35-layer-diff] weights_dir={cls.weights_dir}") + + @classmethod + def tearDownClass(cls) -> None: + model = getattr(cls, "model", None) + if model is not None: + del cls.model + + def _run_wrapper_diff(self, label: str, wrapper: torch.nn.Module) -> DiffStats: + wrapper = wrapper.eval() + with torch.no_grad(): + reference = wrapper(self.input_ids).detach().float().cpu().numpy() + + transpile_wrapper = _WeightsBoundTranspileWrapper(wrapper, weights_dir=self.weights_dir).eval() + captured = capture_model(transpile_wrapper, (self.input_ids,)) + raw_ir_nodes = len(captured.ir_graph.order) + canonicalize_exported_graph(captured.ir_graph) + optimize_graph(captured.ir_graph) + optimized_ir_nodes = len(captured.ir_graph.order) + top_ops = _summarize_ops(captured.ir_graph) + transpiled = transpile_preoptimized_ir(captured.ir_graph) + transpiled.set_inputs([self.input_ids.cpu().numpy()]) + actual = transpiled.execute()[0].numpy().astype(np.float32) + + diff = np.abs(reference.astype(np.float32) - actual) + max_abs_diff = float(np.max(diff)) + mean_abs_diff = float(np.mean(diff)) + + argmax_ref: int | None = None + argmax_transpiled: int | None = None + if reference.ndim == 3: + argmax_ref = int(np.argmax(reference[0, -1])) + argmax_transpiled = int(np.argmax(actual[0, -1])) + + print( + "[qwen35-layer-diff] " + f"{label} " + f"shape={tuple(reference.shape)} " + f"raw_ir_nodes={raw_ir_nodes} " + f"optimized_ir_nodes={optimized_ir_nodes} " + f"max_abs_diff={max_abs_diff:.6f} " + f"mean_abs_diff={mean_abs_diff:.6f} " + f"top_ops={top_ops}" + ) + if argmax_ref is not None and argmax_transpiled is not None: + print( + "[qwen35-layer-diff] " + f"{label} " + f"argmax_ref={argmax_ref} " + f"argmax_transpiled={argmax_transpiled}" + ) + + return DiffStats( + label=label, + shape=tuple(reference.shape), + max_abs_diff=max_abs_diff, + mean_abs_diff=mean_abs_diff, + argmax_ref=argmax_ref, + argmax_transpiled=argmax_transpiled, + raw_ir_nodes=raw_ir_nodes, + optimized_ir_nodes=optimized_ir_nodes, + top_ops=top_ops, + ) + + def test_qwen35_layer_by_layer_debug_diff(self) -> None: + layer_stats: list[DiffStats] = [] + for layer_index in range(self.num_layers): + wrapper = Qwen35CheckpointWrapper(self.model, checkpoint_index=layer_index, weights_dir=self.weights_dir) + layer_stats.append(self._run_wrapper_diff(f"layer_{layer_index}", wrapper)) + + norm_wrapper = Qwen35CheckpointWrapper(self.model, checkpoint_index=self.num_layers, weights_dir=self.weights_dir) + norm_stats = self._run_wrapper_diff("final_norm", norm_wrapper) + logits_stats = self._run_wrapper_diff("logits", Qwen35LogitsWrapper(self.model, weights_dir=self.weights_dir)) + + worst_layer = max(layer_stats, key=lambda item: item.max_abs_diff) + print( + "[qwen35-layer-diff] " + f"worst_layer={worst_layer.label} " + f"worst_layer_max_abs_diff={worst_layer.max_abs_diff:.6f} " + f"worst_layer_mean_abs_diff={worst_layer.mean_abs_diff:.6f}" + ) + print( + "[qwen35-layer-diff] " + f"final_norm_max_abs_diff={norm_stats.max_abs_diff:.6f} " + f"logits_max_abs_diff={logits_stats.max_abs_diff:.6f}" + ) + + self.assertEqual(len(layer_stats), self.num_layers) + self.assertTrue(np.isfinite(logits_stats.max_abs_diff)) + self.assertTrue(np.isfinite(logits_stats.mean_abs_diff))