diff --git a/docs/cactus_transpiler.md b/docs/cactus_transpiler.md index 9e840999e..433a44793 100644 --- a/docs/cactus_transpiler.md +++ b/docs/cactus_transpiler.md @@ -297,6 +297,85 @@ result = outputs[0].numpy() print("Output shape:", result.shape) # (1, 32) ``` +### Generic JAX User Graphs + +JAX and Flax models can be bundled by providing params and one or more graph +entrypoints. The caller owns tokenization, masks, sampling, and any prefill/decode +loop; Cactus captures the supplied functions, writes FP16 mmap weights, and saves +component graphs. + +```python +import jax.numpy as jnp +import numpy as np + +from cactus.transpile.capture_jax import JaxGraphSpec +from cactus.transpile.jax_user_graph_bundle import build_jax_user_graph_bundle +from cactus.transpile.jax_user_graph_bundle import load_jax_user_graph_bundle + + +def model(params, x): + return jnp.maximum(x @ params["w"] + params["b"], 0) + + +params = {"w": jnp.ones((4, 3), jnp.float16), "b": jnp.zeros((3,), jnp.float16)} +example_x = jnp.ones((2, 4), jnp.float16) + +result = build_jax_user_graph_bundle( + params=params, + output_dir="/tmp/my-jax-bundle", + model_id="my-jax-model", + specs=(JaxGraphSpec(name="forward", fn=model, example_args=(example_x,)),), +) + +loaded = load_jax_user_graph_bundle(result.output_dir) +y = loaded.execute("forward", np.ones((2, 4), np.float16))[0].numpy() +``` + +For encoder-decoder models, provide each graph boundary explicitly: + +```python +params, model, tokenizer, config = load_needle_model() +src = tokenizer.encode("What time is it in Tokyo?") +tgt = tokenizer.encode("") +src_mask = make_padding_mask(src, config.pad_token_id) +tgt_mask = make_causal_mask(tgt.shape[1]) +encoder_out, enc_mask = model.apply({"params": params}, src, src_mask, method=model.encode_text) + +result = build_jax_user_graph_bundle( + params=params, + output_dir="weights/needle", + model_id="needle", + task="encoder-decoder", + specs=( + JaxGraphSpec( + name="encoder", + fn=lambda params, src, src_mask: model.apply( + {"params": params}, src, src_mask, method=model.encode_text + ), + example_args=(src, src_mask), + output_names=("encoder_out", "encoder_mask"), + ), + JaxGraphSpec( + name="decoder_prefill", + fn=lambda params, tgt, encoder_out, tgt_mask, cross_mask: model.apply( + {"params": params}, tgt, encoder_out, tgt_mask, cross_mask, method=model.decode + ), + example_args=(tgt, encoder_out, tgt_mask, enc_mask), + output_names=("logits",), + ), + ), +) + +loaded = load_jax_user_graph_bundle(result.output_dir) +encoder_out, enc_mask = loaded.execute("encoder", src, src_mask) +logits = loaded.execute("decoder_prefill", tgt, encoder_out.numpy(), tgt_mask, enc_mask.numpy())[0].numpy() +next_token = int(np.argmax(logits[0, -1])) +``` + +Each component is saved under `components//` with `graph.cactus`, +`raw_ir.json`, and `optimized_ir.json`. For fast decode, expose a separate +`decoder_step` graph with cache tensors as explicit inputs and outputs. + --- ## Artifact Layout diff --git a/python/cactus/transpile/capture_jax.py b/python/cactus/transpile/capture_jax.py new file mode 100644 index 000000000..7033f74ca --- /dev/null +++ b/python/cactus/transpile/capture_jax.py @@ -0,0 +1,1661 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +import copy +from dataclasses import dataclass +import math +from typing import Any + +import numpy as np +import torch + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.graph_ir import verify_ir +from cactus.transpile.jax_semantic_rewrites import apply_jax_semantic_rewrites +from cactus.transpile.lower import TranspiledGraph +from cactus.transpile.lower import transpile_ir +from cactus.transpile.weight_binding import resolve_weight_binding + + +_BINARY_OPS = { + "add": "add", + "sub": "subtract", + "mul": "multiply", + "div": "divide", +} + +_BINARY_EVALUATORS = { + "add": np.add, + "sub": np.subtract, + "mul": np.multiply, + "div": np.divide, +} + +_UNARY_OPS = { + "logistic": "sigmoid", + "tanh": "tanh", + "sqrt": "scalar_sqrt", + "exp": "scalar_exp", + "log": "scalar_log", + "cos": "scalar_cos", + "sin": "scalar_sin", + "neg": "negate", + "abs": "abs", +} + +_UNARY_EVALUATORS = { + "sqrt": np.sqrt, + "exp": np.exp, + "log": np.log, + "cos": np.cos, + "sin": np.sin, + "neg": np.negative, + "logistic": lambda value: 1.0 / (1.0 + np.exp(-value)), + "tanh": np.tanh, +} + +_COMPARE_OPS = { + "eq": "equal", + "ne": "not_equal", + "lt": "less", + "le": "less_equal", + "gt": "greater", + "ge": "greater_equal", +} + +_RHS_SCALAR_COMPARE_OPS = { + "eq": "scalar_equal", + "ne": "scalar_not_equal", + "lt": "scalar_less", + "le": "scalar_less_equal", + "gt": "scalar_greater", + "ge": "scalar_greater_equal", +} + +_LHS_SCALAR_COMPARE_OPS = { + "eq": "scalar_equal", + "ne": "scalar_not_equal", + "lt": "scalar_greater", + "le": "scalar_greater_equal", + "gt": "scalar_less", + "ge": "scalar_less_equal", +} + + +def _dtype_to_ir(dtype: Any) -> str: + dtype = np.dtype(dtype) + if dtype.name == "bfloat16": + return "fp32" + if dtype == np.dtype(np.float16): + return "fp16" + if dtype == np.dtype(np.float32): + return "fp32" + if dtype == np.dtype(np.float64): + return "fp32" + if dtype == np.dtype(np.int8): + return "int8" + if dtype in { + np.dtype(np.int16), + np.dtype(np.int32), + np.dtype(np.int64), + np.dtype(np.uint8), + np.dtype(np.uint16), + np.dtype(np.uint32), + np.dtype(np.uint64), + np.dtype(np.bool_), + }: + return "fp32" + raise NotImplementedError(f"unsupported JAX dtype: {dtype}") + + +def _shape_dtype_from_aval(aval: Any) -> tuple[tuple[int, ...] | None, str | None]: + shape = getattr(aval, "shape", None) + dtype = getattr(aval, "dtype", None) + ir_shape = None if shape is None else tuple(int(dim) for dim in shape) + if ir_shape == (): + ir_shape = (1,) + ir_dtype = None if dtype is None else _dtype_to_ir(dtype) + return ir_shape, ir_dtype + + +@dataclass(frozen=True) +class _SyntheticAval: + shape: tuple[int, ...] + dtype: Any + + +def _constant_to_torch(value: Any) -> torch.Tensor: + array = np.asarray(value) + if array.dtype.name == "bfloat16": + array = array.astype(np.float32) + if array.dtype == np.dtype(np.float64): + array = array.astype(np.float32) + if np.issubdtype(array.dtype, np.floating): + compare_array = array.astype(np.float32, copy=False) + array = np.where(compare_array < -1.0e30, -65504.0, array) + array = np.where(compare_array > 1.0e30, 65504.0, array) + elif array.dtype in { + np.dtype(np.int16), + np.dtype(np.int32), + np.dtype(np.int64), + np.dtype(np.uint8), + np.dtype(np.uint16), + np.dtype(np.uint32), + np.dtype(np.uint64), + np.dtype(np.bool_), + }: + array = array.astype(np.float32) + return torch.from_numpy(np.array(array)) + + +def _tree_path_name(path: Sequence[Any]) -> str: + parts: list[str] = [] + for entry in path: + if hasattr(entry, "key"): + part = str(entry.key) + elif hasattr(entry, "name"): + part = str(entry.name) + elif hasattr(entry, "idx"): + part = str(entry.idx) + else: + part = str(entry) + parts.append(part) + return ".".join(parts) or "param" + + +def _flatten_named_leaves(tree_util: Any, params: Any) -> list[tuple[str, np.ndarray]]: + leaves: list[tuple[str, np.ndarray]] = [] + for path, value in tree_util.tree_flatten_with_path(params)[0]: + leaves.append((_tree_path_name(path), np.asarray(value))) + return leaves + + +def _weight_binding_fields(meta: dict[str, object]) -> dict[str, object] | None: + path = meta.get("path") + kind = meta.get("kind") + source_name = meta.get("source_name") + if isinstance(path, str) and isinstance(kind, str) and isinstance(source_name, str): + return {"path": path, "kind": kind, "source_name": source_name} + return None + + +def _propagate_weight_binding_meta(graph: IRGraph) -> None: + changed = True + while changed: + changed = False + for value_id in list(graph.constants): + value = graph.values[value_id] + if _weight_binding_fields(value.meta) is not None: + continue + if value.meta.get("derived_by_op") != "convert_element_type": + continue + derived_from = value.meta.get("derived_from_value_ids") + if not isinstance(derived_from, (tuple, list)): + continue + for source_id in derived_from: + source_value = graph.values.get(str(source_id)) + if source_value is None: + continue + binding_meta = _weight_binding_fields(source_value.meta) + if binding_meta is None: + continue + value.meta.update(binding_meta) + value.meta["materialized_from_value_id"] = str(source_id) + graph.meta.setdefault("weight_bindings", {})[value_id] = dict(value.meta) + changed = True + break + + +def _binding_meta( + *, + name: str, + weights_dir: str | None, + explicit: dict[str, dict[str, str]], +) -> dict[str, str]: + if name in explicit: + return dict(explicit[name]) + binding = resolve_weight_binding(weights_dir=weights_dir, source_name=name) + if binding is None: + return {} + return { + "path": binding.path, + "kind": binding.kind, + "source_name": binding.source_name, + } + + +def _freeze_leading_param_inputs( + graph: IRGraph, + named_leaves: Sequence[tuple[str, np.ndarray]], + *, + weights_dir: str | None, + explicit: dict[str, dict[str, str]], +) -> None: + if not named_leaves: + return + if len(graph.inputs) < len(named_leaves): + raise ValueError( + f"JAX graph has {len(graph.inputs)} inputs, cannot freeze {len(named_leaves)} parameter leaves" + ) + param_input_ids = list(graph.inputs[: len(named_leaves)]) + graph.inputs = graph.inputs[len(named_leaves) :] + graph.meta["weight_bindings"] = {} + for value_id, (name, leaf_array) in zip(param_input_ids, named_leaves, strict=True): + value = graph.values[value_id] + tensor = _constant_to_torch(leaf_array) + value.shape = tuple(int(dim) for dim in tensor.shape) + value.dtype = _dtype_to_ir(tensor.numpy().dtype) + value.meta.update( + { + "source_name": name, + "jax_param_input": True, + **_binding_meta(name=name, weights_dir=weights_dir, explicit=explicit), + } + ) + graph.constants[value_id] = tensor + if "path" in value.meta: + graph.meta["weight_bindings"][value_id] = dict(value.meta) + + +def _literal_value(literal: Any) -> Any: + value = getattr(literal, "val", literal) + if isinstance(value, np.ndarray): + return value.item() if value.ndim == 0 else value + if hasattr(value, "item"): + try: + return value.item() + except Exception: + return value + return value + + +class _JaxImportContext: + def __init__(self) -> None: + self.var_ids: dict[int, str] = {} + self.gather_index_aliases: dict[str, str] = {} + self.next_value_id = 0 + self.next_node_id = 0 + + def value_id(self, var: Any) -> str: + key = id(var) + existing = self.var_ids.get(key) + if existing is not None: + return existing + name = f"v{self.next_value_id}" + self.next_value_id += 1 + self.var_ids[key] = name + return name + + def bind_value(self, var: Any, value_id: str) -> None: + self.var_ids[id(var)] = value_id + + def node_id(self, op: str) -> str: + name = f"n{self.next_node_id}_{op}" + self.next_node_id += 1 + return name + + +@dataclass(frozen=True) +class JaxGraphSpec: + name: str + fn: Callable[..., Any] + example_args: Sequence[Any] + role: str = "generic" + input_names: Sequence[str] | None = None + output_names: Sequence[str] | None = None + graph_meta: dict[str, object] | None = None + + +@dataclass +class CapturedJaxGraph: + spec: JaxGraphSpec + raw_ir_graph: IRGraph + ir_graph: IRGraph + graph: TranspiledGraph + + def execute(self, *args: Any) -> list[Any]: + if len(args) != len(self.spec.example_args): + raise ValueError( + f"graph {self.spec.name!r} expected {len(self.spec.example_args)} inputs, got {len(args)}" + ) + self.graph.set_inputs( + [ + _coerce_runtime_input(arg, example) + for arg, example in zip(args, self.spec.example_args, strict=True) + ] + ) + return self.graph.execute() + + +@dataclass +class CapturedJaxGraphBundle: + graphs: dict[str, CapturedJaxGraph] + params: Any + weights_dir: str | None = None + + def execute(self, graph_name: str, *args: Any) -> list[Any]: + if graph_name not in self.graphs: + available = ", ".join(sorted(self.graphs)) or "" + raise ValueError(f"unknown JAX graph {graph_name!r}; available graphs: {available}") + return self.graphs[graph_name].execute(*args) + + +def _add_value(graph: IRGraph, value_id: str, aval: Any, *, producer: str | None = None) -> None: + shape, dtype = _shape_dtype_from_aval(aval) + graph.add_value(IRValue(id=value_id, shape=shape, dtype=dtype, producer=producer)) + + +def _coerce_runtime_input(value: Any, example: Any) -> np.ndarray: + array = np.asarray(value) + example_array = np.asarray(example) + target_dtype = example_array.dtype + if target_dtype.name == "bfloat16": + target_dtype = np.dtype(np.float32) + if array.dtype != target_dtype: + array = array.astype(target_dtype, copy=False) + if array.shape == (): + array = array.reshape((1,)) + return array + + +def _add_constant( + graph: IRGraph, + ctx: _JaxImportContext, + var: Any, + value: Any, + *, + source_name: str, + meta: dict[str, object] | None = None, +) -> str: + value_id = ctx.value_id(var) + if value_id in graph.values: + return value_id + tensor = _constant_to_torch(value) + graph.add_value( + IRValue( + id=value_id, + shape=tuple(int(dim) for dim in tensor.shape), + dtype=_dtype_to_ir(tensor.numpy().dtype), + producer=None, + meta={"source_name": source_name, **dict(meta or {})}, + ) + ) + graph.constants[value_id] = tensor + return value_id + + +def _register_node( + graph: IRGraph, + node: IRNode, + *, + out_avals: Sequence[Any], +) -> None: + graph.add_node(node) + graph.order.append(node.id) + for output_id, aval in zip(node.outputs, out_avals, strict=True): + shape, dtype = _shape_dtype_from_aval(aval) + graph.values[output_id].shape = shape + graph.values[output_id].dtype = dtype + + +def _primitive_name(eqn: Any) -> str: + primitive = getattr(eqn, "primitive", None) + return str(getattr(primitive, "name", primitive)) + + +def _is_literal(var: Any) -> bool: + return type(var).__name__ == "Literal" + + +def _ensure_literal_constant(graph: IRGraph, ctx: _JaxImportContext, literal: Any) -> str: + value_id = ctx.value_id(literal) + if value_id in graph.values: + return value_id + value = _literal_value(literal) + return _add_constant(graph, ctx, literal, value, source_name=f"literal:{value!r}") + + +def _derived_meta(input_ids: Sequence[str], *, op: str) -> dict[str, object]: + if not input_ids: + return {"derived_by_op": op} + return {"derived_from_value_ids": tuple(input_ids), "derived_by_op": op} + + +def _input_ids(graph: IRGraph, ctx: _JaxImportContext, invars: Sequence[Any]) -> list[str]: + result: list[str] = [] + for var in invars: + if _is_literal(var): + result.append(_ensure_literal_constant(graph, ctx, var)) + else: + result.append(ctx.value_id(var)) + return result + + +def _out_avals(eqn: Any) -> tuple[Any, ...]: + return tuple(getattr(var, "aval", None) for var in eqn.outvars) + + +def _literal_number(var: Any) -> float | None: + if not _is_literal(var): + return None + value = _literal_value(var) + if isinstance(value, (bool, np.bool_)): + return float(value) + if isinstance(value, (int, float, np.number)): + return float(value) + return None + + +def _constant_scalar(graph: IRGraph, value_id: str) -> float | None: + value = graph.constants.get(value_id) + if value is None: + return None + if graph.values[value_id].meta.get("jax_closed_constant"): + return None + array = np.asarray(value) + if array.shape != (): + return None + return float(array.item()) + + +def _constant_singleton_scalar(graph: IRGraph, value_id: str) -> float | None: + value = graph.constants.get(value_id) + if value is None: + return None + array = np.asarray(value) + if array.size != 1: + return None + return float(array.reshape(()).item()) + + +def _constant_scalar_or_singleton(graph: IRGraph, value_id: str) -> float | None: + scalar = _constant_scalar(graph, value_id) + if scalar is not None: + return scalar + singleton = _constant_singleton_scalar(graph, value_id) + if singleton is not None: + return singleton + value = graph.constants.get(value_id) + if value is None: + return None + array = np.asarray(value) + if array.size == 0: + return None + first = float(array.reshape(-1)[0].item()) + if np.allclose(array, first, rtol=0.0, atol=0.0): + return first + return None + + +def _replace_scalar_constant_with_broadcast( + graph: IRGraph, + value_id: str, + output_var: Any, + *, + source_op: str, +) -> str: + scalar = _constant_singleton_scalar(graph, value_id) + if scalar is None: + return value_id + shape = tuple(int(dim) for dim in getattr(output_var.aval, "shape", ())) + if not shape or _product(shape) <= 1: + return value_id + dtype = getattr(output_var.aval, "dtype", np.float32) + replacement_id = f"{value_id}__{source_op}_broadcast" + if replacement_id in graph.values: + return replacement_id + tensor = _constant_to_torch(np.full(shape, scalar, dtype=np.dtype(dtype))) + graph.add_value( + IRValue( + id=replacement_id, + shape=shape, + dtype=_dtype_to_ir(tensor.numpy().dtype), + producer=None, + meta={ + "source_name": f"broadcasted_scalar:{source_op}", + **_derived_meta((value_id,), op=f"{source_op}:scalar_broadcast"), + }, + ) + ) + graph.constants[replacement_id] = tensor + return replacement_id + + +def _where_scalar_value(value: float) -> float: + if value < -1.0e30: + return -65504.0 + if value > 1.0e30: + return 65504.0 + return value + + +def _constant_array(graph: IRGraph, value_id: str) -> np.ndarray | None: + value = graph.constants.get(value_id) + if value is None: + return None + if graph.values[value_id].meta.get("jax_closed_constant"): + return None + return np.asarray(value) + + +def _try_fold_constant_unary( + graph: IRGraph, + ctx: _JaxImportContext, + *, + prim: str, + input_id: str, + outvar: Any, +) -> bool: + evaluator = _UNARY_EVALUATORS.get(prim) + array = _constant_array(graph, input_id) + if evaluator is None or array is None: + return False + _add_constant( + graph, + ctx, + outvar, + evaluator(array), + source_name=f"folded:{prim}", + meta=_derived_meta((input_id,), op=prim), + ) + return True + + +def _try_fold_constant_binary( + graph: IRGraph, + ctx: _JaxImportContext, + *, + prim: str, + input_ids: Sequence[str], + outvar: Any, +) -> bool: + evaluator = _BINARY_EVALUATORS.get(prim) + if evaluator is None: + return False + lhs_array = _constant_array(graph, input_ids[0]) + rhs_array = _constant_array(graph, input_ids[1]) + if lhs_array is None or rhs_array is None: + return False + _add_constant( + graph, + ctx, + outvar, + evaluator(lhs_array, rhs_array), + source_name=f"folded:{prim}", + meta=_derived_meta(input_ids, op=prim), + ) + return True + + +def _alias_output(ctx: _JaxImportContext, outvar: Any, source_id: str) -> None: + ctx.bind_value(outvar, source_id) + + +def _product(values: Sequence[int]) -> int: + result = 1 + for value in values: + result *= int(value) + return result + + +def _conv_static_int(values: Sequence[int], *, name: str) -> int: + ints = tuple(int(value) for value in values) + if not ints: + return 1 + if any(value != ints[0] for value in ints): + raise NotImplementedError(f"JAX conv_general_dilated requires uniform {name}, got {ints}") + return ints[0] + + +def _conv_padding_int(padding: Sequence[Sequence[int]]) -> int: + pairs = tuple((int(left), int(right)) for left, right in padding) + if not pairs: + return 0 + if any(left != right for left, right in pairs): + raise NotImplementedError(f"JAX conv_general_dilated requires symmetric padding, got {pairs}") + return _conv_static_int(tuple(left for left, _ in pairs), name="padding") + + +def _conv_output_permutation(out_spec: Sequence[int]) -> tuple[int, ...]: + spec = tuple(int(axis) for axis in out_spec) + return tuple(spec.index(final_axis) for final_axis in range(len(spec))) + + +def _jaxpr_invars(jaxpr_like: Any) -> Sequence[Any]: + return tuple(getattr(getattr(jaxpr_like, "jaxpr", jaxpr_like), "invars", ())) + + +def _jaxpr_outvars(jaxpr_like: Any) -> Sequence[Any]: + return tuple(getattr(getattr(jaxpr_like, "jaxpr", jaxpr_like), "outvars", ())) + + +def _jaxpr_eqns(jaxpr_like: Any) -> Sequence[Any]: + return tuple(getattr(getattr(jaxpr_like, "jaxpr", jaxpr_like), "eqns", ())) + + +def _jaxpr_constvars(jaxpr_like: Any) -> Sequence[Any]: + return tuple(getattr(getattr(jaxpr_like, "jaxpr", jaxpr_like), "constvars", ())) + + +def _jaxpr_consts(jaxpr_like: Any) -> Sequence[Any]: + return tuple(getattr(jaxpr_like, "consts", ()) or ()) + + +def _inline_jaxpr( + graph: IRGraph, + ctx: _JaxImportContext, + jaxpr_like: Any, + input_ids: Sequence[str], + outvars: Sequence[Any], +) -> None: + inner_output_ids = _inline_jaxpr_outputs(graph, ctx, jaxpr_like, input_ids) + if len(outvars) != len(inner_output_ids): + raise NotImplementedError("nested JAXPR output arity mismatch") + for outer_var, inner_output_id in zip(outvars, inner_output_ids, strict=True): + _alias_output(ctx, outer_var, inner_output_id) + + +def _inline_jaxpr_outputs( + graph: IRGraph, + ctx: _JaxImportContext, + jaxpr_like: Any, + input_ids: Sequence[str], +) -> list[str]: + invars = _jaxpr_invars(jaxpr_like) + if len(input_ids) != len(invars): + raise NotImplementedError("nested JAXPR input arity mismatch") + invar_keys = {id(var) for var in invars} + for inner_eqn in _jaxpr_eqns(jaxpr_like): + for outvar in inner_eqn.outvars: + if id(outvar) not in invar_keys: + ctx.var_ids.pop(id(outvar), None) + for inner_var, input_id in zip(invars, input_ids, strict=True): + ctx.bind_value(inner_var, input_id) + for index, (constvar, const) in enumerate(zip(_jaxpr_constvars(jaxpr_like), _jaxpr_consts(jaxpr_like), strict=True)): + _add_constant(graph, ctx, constvar, const, source_name=f"nested_const_{index}") + for inner_eqn in _jaxpr_eqns(jaxpr_like): + _import_eqn(graph, ctx, inner_eqn) + return [ctx.value_id(inner_var) for inner_var in _jaxpr_outvars(jaxpr_like)] + + +def _generated_value( + graph: IRGraph, + ctx: _JaxImportContext, + *, + stem: str, + shape: tuple[int, ...] | None, + dtype: str | None, + producer: str, + meta: dict[str, object] | None = None, +) -> str: + value_id = f"v{ctx.next_value_id}_{stem}" + ctx.next_value_id += 1 + if value_id in graph.values: + raise ValueError(f"duplicate generated IR value id: {value_id}") + graph.values[value_id] = IRValue( + id=value_id, + shape=shape, + dtype=dtype, + producer=producer, + meta=dict(meta or {}), + ) + return value_id + + +def _register_generated_node( + graph: IRGraph, + node: IRNode, + *, + output_shapes: Sequence[tuple[int, ...] | None], + output_dtypes: Sequence[str | None], +) -> None: + if node.id in graph.nodes: + raise ValueError(f"duplicate IR node id: {node.id}") + graph.nodes[node.id] = node + graph.order.append(node.id) + for output_id, shape, dtype in zip(node.outputs, output_shapes, output_dtypes, strict=True): + graph.values[output_id].shape = shape + graph.values[output_id].dtype = dtype + + +def _scan_slice_input( + graph: IRGraph, + ctx: _JaxImportContext, + value_id: str, + *, + iteration: int, + input_index: int, +) -> str: + source_value = graph.values[value_id] + if source_value.shape is None or not source_value.shape: + return value_id + slice_node_id = ctx.node_id("scan_slice") + sliced_shape = (1, *tuple(int(dim) for dim in source_value.shape[1:])) + sliced_id = _generated_value( + graph, + ctx, + stem=f"scan{iteration}_{input_index}", + shape=sliced_shape, + dtype=source_value.dtype, + producer=slice_node_id, + meta={ + "derived_from_value_ids": (value_id,), + "derived_by_op": "scan_slice", + }, + ) + slice_node = IRNode( + slice_node_id, + "slice", + [value_id], + [sliced_id], + attrs={"axis": 0, "start": iteration, "end": iteration + 1, "step": 1}, + meta={"jax_generated": "scan_unroll"}, + ) + _register_generated_node(graph, slice_node, output_shapes=(sliced_shape,), output_dtypes=(source_value.dtype,)) + view_node_id = ctx.node_id("scan_slice_view") + output_shape = tuple(int(dim) for dim in source_value.shape[1:]) + if output_shape == () and sliced_shape == (1,): + return sliced_id + output_id = _generated_value( + graph, + ctx, + stem=f"scan{iteration}_{input_index}_view", + shape=output_shape, + dtype=source_value.dtype, + producer=view_node_id, + meta={ + "derived_from_value_ids": (sliced_id,), + "derived_by_op": "scan_slice_view", + }, + ) + view_node = IRNode( + view_node_id, + "view", + [sliced_id], + [output_id], + attrs={"shape": output_shape}, + meta={"jax_generated": "scan_unroll"}, + ) + _register_generated_node(graph, view_node, output_shapes=(output_shape,), output_dtypes=(source_value.dtype,)) + return output_id + + +def _add_iota_constant(graph: IRGraph, ctx: _JaxImportContext, eqn: Any) -> None: + params = dict(getattr(eqn, "params", {}) or {}) + shape = tuple(int(dim) for dim in params["shape"]) + dimension = int(params["dimension"]) + dtype = params.get("dtype", getattr(eqn.outvars[0].aval, "dtype", np.float32)) + values = np.arange(shape[dimension], dtype=np.dtype(dtype)) + reshape = [1] * len(shape) + reshape[dimension] = shape[dimension] + values = np.broadcast_to(values.reshape(reshape), shape) + _add_constant(graph, ctx, eqn.outvars[0], values, source_name=f"iota:{shape}:{dimension}") + + +def _import_eqn(graph: IRGraph, ctx: _JaxImportContext, eqn: Any) -> None: + prim = _primitive_name(eqn) + params = dict(getattr(eqn, "params", {}) or {}) + + if prim in {"jit", "remat2"}: + _inline_jaxpr(graph, ctx, params["jaxpr"], _input_ids(graph, ctx, eqn.invars), eqn.outvars) + return + + if prim in {"custom_jvp_call", "custom_jvp_call_jaxpr"}: + call_jaxpr = params.get("call_jaxpr") or params.get("fun_jaxpr") or params.get("jaxpr") + if call_jaxpr is None: + raise NotImplementedError("JAX custom_jvp_call import requires a primal call_jaxpr") + _inline_jaxpr(graph, ctx, call_jaxpr, _input_ids(graph, ctx, eqn.invars), eqn.outvars) + return + + if prim == "scan": + length = int(params.get("length", 1)) + num_consts = int(params.get("num_consts", 0)) + num_carry = int(params.get("num_carry", len(eqn.outvars))) + input_ids = _input_ids(graph, ctx, eqn.invars) + const_ids = input_ids[:num_consts] + carry_ids = input_ids[num_consts : num_consts + num_carry] + xs_ids = input_ids[num_consts + num_carry :] + if len(eqn.outvars) != num_carry: + raise NotImplementedError("JAX scan import currently supports carry-only scan outputs") + for iteration in range(length): + sliced_xs = [ + _scan_slice_input(graph, ctx, value_id, iteration=iteration, input_index=index) + for index, value_id in enumerate(xs_ids) + ] + body_outputs = _inline_jaxpr_outputs(graph, ctx, params["jaxpr"], [*const_ids, *carry_ids, *sliced_xs]) + if len(body_outputs) != num_carry: + raise NotImplementedError("JAX scan body output arity mismatch") + carry_ids = body_outputs + for outvar, carry_id in zip(eqn.outvars, carry_ids, strict=True): + _alias_output(ctx, outvar, carry_id) + return + + if prim == "stop_gradient": + _alias_output(ctx, eqn.outvars[0], _input_ids(graph, ctx, eqn.invars)[0]) + return + + if prim == "iota": + _add_iota_constant(graph, ctx, eqn) + return + + node = _node_for_eqn(graph, ctx, eqn) + if node is None: + return + _register_node(graph, node, out_avals=_out_avals(eqn)) + + +def _node_for_eqn(graph: IRGraph, ctx: _JaxImportContext, eqn: Any) -> IRNode | None: + prim = _primitive_name(eqn) + inputs = _input_ids(graph, ctx, eqn.invars) + outputs = [ctx.value_id(var) for var in eqn.outvars] + node_id = ctx.node_id(prim) + params = dict(getattr(eqn, "params", {}) or {}) + meta = {"jax_primitive": prim} + + if prim in _BINARY_OPS: + if _try_fold_constant_binary(graph, ctx, prim=prim, input_ids=inputs, outvar=eqn.outvars[0]): + return None + if prim == "div": + lhs_literal = _literal_number(eqn.invars[0]) + if lhs_literal is not None: + reciprocal_id = f"{outputs[0]}__reciprocal" + reciprocal_node = IRNode( + node_id, + "pow", + [inputs[1]], + [reciprocal_id], + attrs={"exponent": -1.0}, + meta=meta, + ) + _register_node(graph, reciprocal_node, out_avals=(eqn.outvars[0].aval,)) + if lhs_literal == 1.0: + _alias_output(ctx, eqn.outvars[0], reciprocal_id) + return None + return IRNode( + ctx.node_id("scalar_div_mul"), + "scalar_multiply", + [reciprocal_id], + outputs, + attrs={"value": lhs_literal}, + meta=meta, + ) + if prim in _BINARY_OPS: + if len(inputs) == 2 and len(outputs) == 1: + inputs = [ + _replace_scalar_constant_with_broadcast(graph, input_id, eqn.outvars[0], source_op=prim) + for input_id in inputs + ] + return IRNode(node_id, _BINARY_OPS[prim], inputs, outputs, meta=meta) + if prim in _UNARY_OPS: + if _try_fold_constant_unary(graph, ctx, prim=prim, input_id=inputs[0], outvar=eqn.outvars[0]): + return None + return IRNode(node_id, _UNARY_OPS[prim], inputs, outputs, meta=meta) + if prim == "log1p": + add_id = f"{outputs[0]}__log1p_add" + _register_node( + graph, + IRNode( + node_id, + "scalar_add", + inputs, + [add_id], + attrs={"value": 1.0}, + meta=meta, + ), + out_avals=(eqn.outvars[0].aval,), + ) + return IRNode(ctx.node_id("log1p_log"), "scalar_log", [add_id], outputs, meta=meta) + if prim == "expm1": + exp_id = f"{outputs[0]}__expm1_exp" + _register_node( + graph, + IRNode(node_id, "scalar_exp", inputs, [exp_id], meta=meta), + out_avals=(eqn.outvars[0].aval,), + ) + return IRNode(ctx.node_id("expm1_sub"), "scalar_add", [exp_id], outputs, attrs={"value": -1.0}, meta=meta) + if prim in _COMPARE_OPS: + lhs_scalar = _constant_scalar(graph, inputs[0]) + rhs_scalar = _constant_scalar(graph, inputs[1]) + if rhs_scalar is not None: + return IRNode(node_id, _RHS_SCALAR_COMPARE_OPS[prim], [inputs[0]], outputs, attrs={"value": rhs_scalar}, meta=meta) + if lhs_scalar is not None: + return IRNode(node_id, _LHS_SCALAR_COMPARE_OPS[prim], [inputs[1]], outputs, attrs={"value": lhs_scalar}, meta=meta) + return IRNode(node_id, _COMPARE_OPS[prim], inputs, outputs, meta=meta) + if prim == "and": + return IRNode(node_id, "logical_and", inputs, outputs, meta=meta) + if prim == "or": + return IRNode(node_id, "logical_or", inputs, outputs, meta=meta) + if prim == "square": + return IRNode(node_id, "multiply", [inputs[0], inputs[0]], outputs, meta=meta) + if prim == "rsqrt": + return IRNode(node_id, "pow", inputs, outputs, attrs={"exponent": -0.5}, meta=meta) + if prim == "integer_pow": + exponent = int(params["y"]) + if exponent == 2: + return IRNode(node_id, "multiply", [inputs[0], inputs[0]], outputs, meta=meta) + return IRNode(node_id, "pow", inputs, outputs, attrs={"exponent": float(exponent)}, meta=meta) + if prim == "pow": + rhs_literal = _literal_number(eqn.invars[1]) + if rhs_literal is not None: + return IRNode(node_id, "pow", [inputs[0]], outputs, attrs={"exponent": float(rhs_literal)}, meta=meta) + lhs_literal = _literal_number(eqn.invars[0]) + if lhs_literal is not None and lhs_literal > 0.0: + intermediate = f"{outputs[0]}__logmul" + scale_node = IRNode( + node_id, + "scalar_multiply", + [inputs[1]], + [intermediate], + attrs={"value": math.log(lhs_literal)}, + meta=meta, + ) + _register_node(graph, scale_node, out_avals=(eqn.outvars[0].aval,)) + return IRNode(ctx.node_id("pow_exp"), "scalar_exp", [intermediate], outputs, meta=meta) + raise NotImplementedError("JAX pow import only supports positive scalar base") + if prim == "dot_general": + dimension_numbers = params.get("dimension_numbers") + if dimension_numbers is not None: + ((lhs_contract, rhs_contract), (lhs_batch, rhs_batch)) = dimension_numbers + lhs_shape = tuple(getattr(eqn.invars[0].aval, "shape", ())) + rhs_shape = tuple(getattr(eqn.invars[1].aval, "shape", ())) + lhs_contract = tuple(int(dim) for dim in lhs_contract) + rhs_contract = tuple(int(dim) for dim in rhs_contract) + lhs_batch = tuple(int(dim) for dim in lhs_batch) + rhs_batch = tuple(int(dim) for dim in rhs_batch) + if lhs_batch or rhs_batch: + if tuple(lhs_shape[dim] for dim in lhs_batch) != tuple(rhs_shape[dim] for dim in rhs_batch): + raise NotImplementedError("JAX dot_general batch dimensions must have matching sizes") + expected_rhs_contract = (len(rhs_batch),) + if ( + tuple(lhs_contract) != (len(lhs_shape) - 1,) + or tuple(rhs_contract) != expected_rhs_contract + or len(rhs_shape) > 2 + ): + batch_shape = tuple(lhs_shape[dim] for dim in lhs_batch) + lhs_contract_shape = tuple(lhs_shape[dim] for dim in lhs_contract) + rhs_contract_shape = tuple(rhs_shape[dim] for dim in rhs_contract) + if lhs_contract_shape != rhs_contract_shape: + raise NotImplementedError("JAX dot_general contraction dimensions must have matching sizes") + lhs_non_contract = tuple( + dim for dim in range(len(lhs_shape)) if dim not in set(lhs_batch) | set(lhs_contract) + ) + rhs_non_contract = tuple( + dim for dim in range(len(rhs_shape)) if dim not in set(rhs_batch) | set(rhs_contract) + ) + lhs_order = lhs_batch + lhs_non_contract + lhs_contract + rhs_order = rhs_batch + rhs_contract + rhs_non_contract + lhs_non_shape = tuple(lhs_shape[dim] for dim in lhs_non_contract) + rhs_non_shape = tuple(rhs_shape[dim] for dim in rhs_non_contract) + lhs_matrix_shape = batch_shape + (_product(lhs_non_shape), _product(lhs_contract_shape)) + rhs_matrix_shape = batch_shape + (_product(rhs_contract_shape), _product(rhs_non_shape)) + output_shape = batch_shape + lhs_non_shape + rhs_non_shape + dtype = getattr(eqn.outvars[0].aval, "dtype", getattr(eqn.invars[0].aval, "dtype", np.float32)) + + lhs_id = inputs[0] + if lhs_order != tuple(range(len(lhs_shape))): + lhs_id = f"{outputs[0]}__lhs_transpose" + _register_node( + graph, + IRNode( + ctx.node_id("dot_general_lhs_transpose"), + "permute", + [inputs[0]], + [lhs_id], + attrs={"permutation": lhs_order}, + meta=meta, + ), + out_avals=(_SyntheticAval(tuple(lhs_shape[dim] for dim in lhs_order), dtype),), + ) + lhs_2d = f"{outputs[0]}__lhs_reshape" + _register_node( + graph, + IRNode( + ctx.node_id("dot_general_lhs_reshape"), + "reshape", + [lhs_id], + [lhs_2d], + attrs={"shape": lhs_matrix_shape}, + meta=meta, + ), + out_avals=(_SyntheticAval(lhs_matrix_shape, dtype),), + ) + + rhs_id = inputs[1] + if rhs_order != tuple(range(len(rhs_shape))): + rhs_id = f"{outputs[0]}__rhs_transpose" + _register_node( + graph, + IRNode( + ctx.node_id("dot_general_rhs_transpose"), + "permute", + [inputs[1]], + [rhs_id], + attrs={"permutation": rhs_order}, + meta=meta, + ), + out_avals=(_SyntheticAval(tuple(rhs_shape[dim] for dim in rhs_order), dtype),), + ) + rhs_2d = f"{outputs[0]}__rhs_reshape" + _register_node( + graph, + IRNode( + ctx.node_id("dot_general_rhs_reshape"), + "reshape", + [rhs_id], + [rhs_2d], + attrs={"shape": rhs_matrix_shape}, + meta=meta, + ), + out_avals=(_SyntheticAval(rhs_matrix_shape, dtype),), + ) + + matmul_id = f"{outputs[0]}__matmul" + _register_node( + graph, + IRNode(ctx.node_id("dot_general_matmul"), "matmul", [lhs_2d, rhs_2d], [matmul_id], meta=meta), + out_avals=(_SyntheticAval(batch_shape + (lhs_matrix_shape[-2], rhs_matrix_shape[-1]), dtype),), + ) + return IRNode( + ctx.node_id("dot_general_output_reshape"), + "reshape", + [matmul_id], + outputs, + attrs={"shape": output_shape}, + meta=meta, + ) + return IRNode(node_id, "matmul", inputs, outputs, meta=meta) + if prim == "conv_general_dilated": + if len(inputs) != 2 or len(outputs) != 1: + raise NotImplementedError("JAX conv_general_dilated import supports input and kernel only") + if int(params.get("batch_group_count", 1)) != 1: + raise NotImplementedError("JAX conv_general_dilated batch_group_count is unsupported") + lhs_dilation = tuple(int(value) for value in params.get("lhs_dilation", ())) + if any(value != 1 for value in lhs_dilation): + raise NotImplementedError(f"JAX conv_general_dilated lhs_dilation is unsupported: {lhs_dilation}") + dimension_numbers = params.get("dimension_numbers") + if dimension_numbers is None: + raise NotImplementedError("JAX conv_general_dilated requires static dimension_numbers") + + lhs_spec = tuple(int(axis) for axis in dimension_numbers.lhs_spec) + rhs_spec = tuple(int(axis) for axis in dimension_numbers.rhs_spec) + out_spec = tuple(int(axis) for axis in dimension_numbers.out_spec) + lhs_shape = tuple(int(dim) for dim in getattr(eqn.invars[0].aval, "shape", ())) + rhs_shape = tuple(int(dim) for dim in getattr(eqn.invars[1].aval, "shape", ())) + out_shape = tuple(int(dim) for dim in getattr(eqn.outvars[0].aval, "shape", ())) + rank = len(lhs_shape) + if rank not in {3, 4}: + raise NotImplementedError(f"JAX conv_general_dilated supports 1D/2D convs, got rank {rank}") + if tuple(sorted(lhs_spec)) != tuple(range(rank)) or tuple(sorted(rhs_spec)) != tuple(range(rank)): + raise NotImplementedError("JAX conv_general_dilated requires complete dimension specs") + if tuple(sorted(out_spec)) != tuple(range(rank)): + raise NotImplementedError("JAX conv_general_dilated requires complete output dimension spec") + + stride = _conv_static_int(params.get("window_strides", (1,) * (rank - 2)), name="stride") + dilation = _conv_static_int(params.get("rhs_dilation", (1,) * (rank - 2)), name="rhs_dilation") + padding = _conv_padding_int(params.get("padding", ((0, 0),) * (rank - 2))) + groups = int(params.get("feature_group_count", 1)) + dtype = getattr(eqn.outvars[0].aval, "dtype", getattr(eqn.invars[0].aval, "dtype", np.float32)) + + x_id = inputs[0] + x_shape = tuple(lhs_shape[axis] for axis in lhs_spec) + if lhs_spec != tuple(range(rank)): + x_id = f"{outputs[0]}__conv_lhs" + _register_node( + graph, + IRNode( + ctx.node_id("conv_lhs_transpose"), + "permute", + [inputs[0]], + [x_id], + attrs={"permutation": lhs_spec}, + meta=meta, + ), + out_avals=(_SyntheticAval(x_shape, dtype),), + ) + + weight_id = inputs[1] + weight_shape = tuple(rhs_shape[axis] for axis in rhs_spec) + if rhs_spec != tuple(range(rank)): + weight_id = f"{outputs[0]}__conv_rhs" + _register_node( + graph, + IRNode( + ctx.node_id("conv_rhs_transpose"), + "permute", + [inputs[1]], + [weight_id], + attrs={"permutation": rhs_spec}, + meta=meta, + ), + out_avals=(_SyntheticAval(weight_shape, dtype),), + ) + + conv_id = outputs[0] + output_permutation = _conv_output_permutation(out_spec) + conv_shape = tuple(out_shape[axis] for axis in out_spec) + if output_permutation != tuple(range(rank)): + conv_id = f"{outputs[0]}__conv" + _register_node( + graph, + IRNode( + node_id, + "conv1d" if rank == 3 else "conv2d", + [x_id, weight_id], + [conv_id], + attrs={"stride": stride, "padding": padding, "dilation": dilation, "groups": groups}, + meta=meta, + ), + out_avals=(_SyntheticAval(conv_shape, dtype),), + ) + if conv_id == outputs[0]: + return None + return IRNode( + ctx.node_id("conv_output_transpose"), + "permute", + [conv_id], + outputs, + attrs={"permutation": output_permutation}, + meta=meta, + ) + if prim == "reshape": + if inputs[0] in ctx.gather_index_aliases: + ctx.gather_index_aliases[outputs[0]] = ctx.gather_index_aliases[inputs[0]] + new_shape = tuple(int(dim) for dim in params["new_sizes"]) + input_shape = tuple(int(dim) for dim in getattr(eqn.invars[0].aval, "shape", ())) + source_value = graph.values.get(inputs[0]) + graph_input_shape = tuple(int(dim) for dim in (source_value.shape or ())) if source_value is not None else input_shape + if new_shape == () and graph_input_shape == (1,): + _alias_output(ctx, eqn.outvars[0], inputs[0]) + return None + return IRNode( + node_id, + "reshape", + inputs, + outputs, + attrs={"shape": new_shape}, + meta=meta, + ) + if prim == "squeeze": + shape = tuple(int(dim) for dim in getattr(eqn.outvars[0].aval, "shape", ())) + return IRNode(node_id, "reshape", inputs, outputs, attrs={"shape": shape}, meta=meta) + if prim == "transpose": + permutation = params.get("permutation") + if permutation is None: + permutation = params.get("permutation_or_none") + if permutation is None: + raise NotImplementedError("JAX transpose missing permutation") + return IRNode( + node_id, + "permute", + inputs, + outputs, + attrs={"permutation": tuple(int(dim) for dim in permutation)}, + meta=meta, + ) + if prim == "convert_element_type": + array = _constant_array(graph, inputs[0]) + if array is not None: + _add_constant( + graph, + ctx, + eqn.outvars[0], + array.astype(np.dtype(params["new_dtype"])), + source_name="folded:convert_element_type", + meta=_derived_meta(inputs, op=prim), + ) + return None + return IRNode( + node_id, + "precision_cast", + inputs, + outputs, + attrs={"dtype": _dtype_to_ir(params["new_dtype"])}, + meta=meta, + ) + if prim == "broadcast_in_dim": + shape = tuple(int(dim) for dim in params["shape"]) + scalar = _constant_singleton_scalar(graph, inputs[0]) + if scalar is not None: + _add_constant( + graph, + ctx, + eqn.outvars[0], + np.full(shape, scalar, dtype=np.float32), + source_name="folded:broadcast_in_dim", + meta=_derived_meta(inputs, op=prim), + ) + return None + if inputs[0] in ctx.gather_index_aliases: + ctx.gather_index_aliases[outputs[0]] = ctx.gather_index_aliases[inputs[0]] + broadcast_dimensions = tuple(int(dim) for dim in params.get("broadcast_dimensions", ())) + input_shape = tuple(graph.values[inputs[0]].shape or ()) + if broadcast_dimensions and broadcast_dimensions != tuple(range(len(input_shape))): + reshape_shape = [1] * len(shape) + for input_axis, output_axis in enumerate(broadcast_dimensions): + reshape_shape[output_axis] = input_shape[input_axis] + reshaped_shape = tuple(reshape_shape) + if reshaped_shape == shape: + return IRNode(node_id, "view", inputs, outputs, attrs={"shape": reshaped_shape}, meta=meta) + reshaped_id = _generated_value( + graph, + ctx, + stem="broadcast_base", + shape=reshaped_shape, + dtype=graph.values[inputs[0]].dtype, + producer=node_id, + meta=_derived_meta(inputs, op=f"{prim}:reshape"), + ) + reshape_node = IRNode(node_id, "view", inputs, [reshaped_id], attrs={"shape": reshaped_shape}, meta=meta) + _register_generated_node( + graph, + reshape_node, + output_shapes=(reshaped_shape,), + output_dtypes=(graph.values[inputs[0]].dtype,), + ) + node_id = ctx.node_id(prim) + return IRNode(node_id, "expand", [reshaped_id], outputs, attrs={"shape": shape}, meta=meta) + return IRNode(node_id, "expand", inputs, outputs, attrs={"shape": shape}, meta=meta) + if prim == "concatenate": + return IRNode(node_id, "cat", inputs, outputs, attrs={"axis": int(params["dimension"])}, meta=meta) + if prim == "stack": + axis = int(params.get("axis", 0)) + input_shape = tuple(getattr(eqn.invars[0].aval, "shape", ())) + rank = len(input_shape) + 1 + if axis < 0: + axis += rank + if axis < 0 or axis > len(input_shape): + raise NotImplementedError(f"JAX stack axis out of range: {axis}") + dtype = getattr(eqn.outvars[0].aval, "dtype", getattr(eqn.invars[0].aval, "dtype", np.float32)) + expanded_inputs: list[str] = [] + for input_index, input_id in enumerate(inputs): + expanded_id = f"{outputs[0]}__stack_{input_index}" + expanded_shape = input_shape[:axis] + (1,) + input_shape[axis:] + _register_node( + graph, + IRNode( + ctx.node_id("stack_view"), + "view", + [input_id], + [expanded_id], + attrs={"shape": expanded_shape}, + meta=meta, + ), + out_avals=(_SyntheticAval(expanded_shape, dtype),), + ) + expanded_inputs.append(expanded_id) + return IRNode(node_id, "cat", expanded_inputs, outputs, attrs={"axis": axis}, meta=meta) + if prim == "tile": + reps = tuple(int(value) for value in params.get("reps", ())) + if any(rep <= 0 for rep in reps): + raise NotImplementedError(f"JAX tile import requires positive reps, got {reps}") + input_shape = tuple(int(dim) for dim in getattr(eqn.invars[0].aval, "shape", ())) + if len(reps) < len(input_shape): + reps = (1,) * (len(input_shape) - len(reps)) + reps + current_id = inputs[0] + current_shape = input_shape + dtype = getattr(eqn.invars[0].aval, "dtype", getattr(eqn.outvars[0].aval, "dtype", np.float32)) + if len(reps) > len(current_shape): + current_shape = (1,) * (len(reps) - len(current_shape)) + current_shape + viewed_id = f"{outputs[0]}__tile_view" + _register_node( + graph, + IRNode( + ctx.node_id("tile_view"), + "view", + [current_id], + [viewed_id], + attrs={"shape": current_shape}, + meta=meta, + ), + out_avals=(_SyntheticAval(current_shape, dtype),), + ) + current_id = viewed_id + for axis, rep in enumerate(reps): + if rep == 1: + continue + next_id = outputs[0] if axis == len(reps) - 1 and all(factor == 1 for factor in reps[axis + 1 :]) else f"{outputs[0]}__tile_axis_{axis}" + next_shape = current_shape[:axis] + (current_shape[axis] * rep,) + current_shape[axis + 1 :] + _register_node( + graph, + IRNode( + ctx.node_id("tile_cat"), + "cat", + [current_id] * rep, + [next_id], + attrs={"axis": axis}, + meta=meta, + ), + out_avals=(_SyntheticAval(next_shape, dtype),), + ) + current_id = next_id + current_shape = next_shape + if current_id != outputs[0]: + _alias_output(ctx, eqn.outvars[0], current_id) + return None + if prim == "split": + axis = int(params.get("axis", 0)) + start = 0 + for output_id, output_var, size in zip(outputs, eqn.outvars, params.get("sizes", ()), strict=True): + end = start + int(size) + _register_node( + graph, + IRNode( + ctx.node_id("split_slice"), + "slice", + inputs, + [output_id], + attrs={"axis": axis, "start": start, "end": end, "step": 1}, + meta=meta, + ), + out_avals=(output_var.aval,), + ) + start = end + return None + if prim == "slice": + starts = tuple(int(value) for value in params["start_indices"]) + limits = tuple(int(value) for value in params["limit_indices"]) + raw_strides = params.get("strides") + strides = (1,) * len(starts) if raw_strides is None else tuple(int(value) for value in raw_strides) + changed_axes = [axis for axis, (start, limit) in enumerate(zip(starts, limits, strict=True)) if start != 0 or limit != getattr(eqn.invars[0].aval, "shape", ())[axis]] + if not changed_axes: + _alias_output(ctx, eqn.outvars[0], inputs[0]) + return None + if len(changed_axes) != 1: + raise NotImplementedError("JAX slice import supports one non-trivial axis") + axis = changed_axes[0] + return IRNode( + node_id, + "slice", + inputs, + outputs, + attrs={"axis": axis, "start": starts[axis], "end": limits[axis], "step": strides[axis]}, + meta=meta, + ) + if prim == "select_n": + if len(inputs) != 3: + raise NotImplementedError("JAX select_n import supports ternary select") + ctx.gather_index_aliases[outputs[0]] = inputs[1] + false_scalar = _constant_scalar(graph, inputs[1]) + true_scalar = _constant_scalar(graph, inputs[2]) + where_inputs = [inputs[0]] + attrs: dict[str, object] = {} + if true_scalar is None: + where_inputs.append(inputs[2]) + else: + attrs["true_is_scalar"] = True + attrs["true_value"] = _where_scalar_value(true_scalar) + if false_scalar is None: + where_inputs.append(inputs[1]) + else: + attrs["false_is_scalar"] = True + attrs["false_value"] = _where_scalar_value(false_scalar) + return IRNode(node_id, "where", where_inputs, outputs, attrs=attrs, meta=meta) + if prim == "gather": + if len(inputs) != 2: + raise NotImplementedError("JAX gather import supports embedding-style gather") + indices_id = ctx.gather_index_aliases.get(inputs[1], inputs[1]) + index_shape = tuple(getattr(eqn.invars[1].aval, "shape", ())) + if index_shape and index_shape[-1] == 1: + squeezed_id = f"{indices_id}__squeezed" + squeeze_shape = tuple(int(dim) for dim in index_shape[:-1]) + squeeze_node = IRNode( + node_id, + "reshape", + [indices_id], + [squeezed_id], + attrs={"shape": squeeze_shape}, + meta=meta, + ) + _register_node(graph, squeeze_node, out_avals=(eqn.invars[1].aval,)) + graph.values[squeezed_id].shape = squeeze_shape + if indices_id in ctx.gather_index_aliases: + ctx.gather_index_aliases[squeezed_id] = ctx.gather_index_aliases[indices_id] + indices_id = squeezed_id + node_id = ctx.node_id("gather_embedding") + indices_id = ctx.gather_index_aliases.get(indices_id, indices_id) + return IRNode(node_id, "embedding", [inputs[0], indices_id], outputs, meta=meta) + if prim in {"max", "min"}: + lhs_literal = _literal_number(eqn.invars[0]) + rhs_literal = _literal_number(eqn.invars[1]) + identity = -math.inf if prim == "max" else math.inf + if lhs_literal == identity: + _alias_output(ctx, eqn.outvars[0], inputs[1]) + return None + if rhs_literal == identity: + _alias_output(ctx, eqn.outvars[0], inputs[0]) + return None + + lhs_scalar = _constant_scalar(graph, inputs[0]) + rhs_scalar = _constant_scalar(graph, inputs[1]) + condition_id = f"{outputs[0]}__{prim}_condition" + condition_shape = tuple(getattr(eqn.outvars[0].aval, "shape", ())) + condition_dtype = np.bool_ + if rhs_scalar is not None: + compare_op = "scalar_greater" if prim == "max" else "scalar_less" + _register_node( + graph, + IRNode( + ctx.node_id(f"{prim}_condition"), + compare_op, + [inputs[0]], + [condition_id], + attrs={"value": rhs_scalar}, + meta=meta, + ), + out_avals=(_SyntheticAval(condition_shape, condition_dtype),), + ) + attrs = {"false_is_scalar": True, "false_value": _where_scalar_value(rhs_scalar)} + return IRNode(ctx.node_id(f"{prim}_where"), "where", [condition_id, inputs[0]], outputs, attrs=attrs, meta=meta) + if lhs_scalar is not None: + compare_op = "scalar_less" if prim == "max" else "scalar_greater" + _register_node( + graph, + IRNode( + ctx.node_id(f"{prim}_condition"), + compare_op, + [inputs[1]], + [condition_id], + attrs={"value": lhs_scalar}, + meta=meta, + ), + out_avals=(_SyntheticAval(condition_shape, condition_dtype),), + ) + attrs = {"true_is_scalar": True, "true_value": _where_scalar_value(lhs_scalar)} + return IRNode(ctx.node_id(f"{prim}_where"), "where", [condition_id, inputs[1]], outputs, attrs=attrs, meta=meta) + + compare_op = "greater" if prim == "max" else "less" + _register_node( + graph, + IRNode(ctx.node_id(f"{prim}_condition"), compare_op, inputs, [condition_id], meta=meta), + out_avals=(_SyntheticAval(condition_shape, condition_dtype),), + ) + return IRNode(ctx.node_id(f"{prim}_where"), "where", [condition_id, inputs[0], inputs[1]], outputs, meta=meta) + if prim in {"reduce_sum", "reduce_max", "reduce_min"}: + reduce_ops = { + "reduce_sum": "sum", + "reduce_max": "max", + "reduce_min": "min", + } + return IRNode( + node_id, + reduce_ops[prim], + inputs, + outputs, + attrs={"axis": tuple(int(axis) for axis in params.get("axes", ()))}, + meta=meta, + ) + + raise NotImplementedError(f"unsupported JAX primitive: {prim}") + + +def capture_jax_function( + fn: Callable[..., Any], + example_args: Sequence[Any], + *, + constant_names: Sequence[str] | None = None, + weight_bindings: dict[str, dict[str, str]] | None = None, + weights_dir: str | None = None, + graph_meta: dict[str, object] | None = None, +) -> IRGraph: + try: + import jax + except Exception as exc: # pragma: no cover - depends on optional dependency + raise RuntimeError("capture_jax_function requires the optional jax package") from exc + + closed = jax.make_jaxpr(fn)(*example_args) + jaxpr = closed.jaxpr + constants = tuple(getattr(closed, "consts", ()) or ()) + names = tuple(constant_names or ()) + bindings = weight_bindings or {} + ctx = _JaxImportContext() + graph = IRGraph( + values={}, + nodes={}, + order=[], + inputs=[], + outputs=[], + constants={}, + meta={ + "frontend": "jax", + "adapter_family": "generic", + **({"weights_dir": weights_dir} if weights_dir else {}), + **dict(graph_meta or {}), + }, + ) + + for var in jaxpr.invars: + value_id = ctx.value_id(var) + _add_value(graph, value_id, var.aval) + graph.inputs.append(value_id) + + for index, (var, value) in enumerate(zip(jaxpr.constvars, constants, strict=True)): + value_id = ctx.value_id(var) + name = names[index] if index < len(names) else f"const_{index}" + tensor = _constant_to_torch(value) + graph.add_value( + IRValue( + id=value_id, + shape=tuple(int(dim) for dim in tensor.shape), + dtype=_dtype_to_ir(tensor.numpy().dtype), + meta={ + "source_name": name, + "jax_closed_constant": True, + **_binding_meta(name=name, weights_dir=weights_dir, explicit=bindings), + }, + ) + ) + graph.constants[value_id] = tensor + if "path" in graph.values[value_id].meta: + graph.meta.setdefault("weight_bindings", {})[value_id] = dict(graph.values[value_id].meta) + + for eqn in jaxpr.eqns: + _import_eqn(graph, ctx, eqn) + + graph.outputs = [ctx.value_id(var) for var in jaxpr.outvars] + apply_jax_semantic_rewrites(graph) + verify_ir(graph) + return graph + + +def capture_jax_function_with_params( + fn: Callable[..., Any], + params: Any, + example_args: Sequence[Any], + *, + weights_dir: str | None = None, + weight_bindings: dict[str, dict[str, str]] | None = None, + graph_meta: dict[str, object] | None = None, +) -> IRGraph: + try: + import jax + except Exception as exc: # pragma: no cover - depends on optional dependency + raise RuntimeError("capture_jax_function_with_params requires the optional jax package") from exc + + named_leaves = _flatten_named_leaves(jax.tree_util, params) + flat_leaves, treedef = jax.tree_util.tree_flatten(params) + if len(flat_leaves) != len(named_leaves): + raise ValueError("JAX param flattening produced mismatched leaf counts") + leaf_count = len(flat_leaves) + + def bound_fn(*flat_params_and_args: Any) -> Any: + flat_param_values = flat_params_and_args[:leaf_count] + runtime_args = flat_params_and_args[leaf_count:] + rebuilt_params = jax.tree_util.tree_unflatten(treedef, flat_param_values) + return fn(rebuilt_params, *runtime_args) + + graph = capture_jax_function( + bound_fn, + (*flat_leaves, *example_args), + weight_bindings=weight_bindings, + weights_dir=weights_dir, + graph_meta=graph_meta, + ) + explicit = weight_bindings or {} + _freeze_leading_param_inputs( + graph, + named_leaves, + weights_dir=weights_dir, + explicit=explicit, + ) + _propagate_weight_binding_meta(graph) + verify_ir(graph) + return graph + + +def capture_jax_graphs( + params: Any, + specs: Sequence[JaxGraphSpec], + *, + weights_dir: str | None = None, + weight_bindings: dict[str, dict[str, str]] | None = None, + graph_meta: dict[str, object] | None = None, +) -> CapturedJaxGraphBundle: + seen_names: set[str] = set() + for spec in specs: + if spec.name in seen_names: + raise ValueError(f"duplicate JAX graph spec name: {spec.name!r}") + seen_names.add(spec.name) + if spec.input_names is not None and len(spec.input_names) != len(spec.example_args): + raise ValueError( + f"graph {spec.name!r} has {len(spec.input_names)} input names for " + f"{len(spec.example_args)} example inputs" + ) + + captured: dict[str, CapturedJaxGraph] = {} + for spec in specs: + ir = capture_jax_function_with_params( + spec.fn, + params, + tuple(spec.example_args), + weights_dir=weights_dir, + weight_bindings=weight_bindings, + graph_meta={ + **dict(graph_meta or {}), + **dict(spec.graph_meta or {}), + "jax_graph_name": spec.name, + "jax_graph_role": spec.role, + **({"input_names": tuple(spec.input_names)} if spec.input_names is not None else {}), + **({"output_names": tuple(spec.output_names)} if spec.output_names is not None else {}), + }, + ) + raw_ir = copy.deepcopy(ir) + transpiled_graph = transpile_ir(ir) + captured[spec.name] = CapturedJaxGraph( + spec=spec, + raw_ir_graph=raw_ir, + ir_graph=ir, + graph=transpiled_graph, + ) + return CapturedJaxGraphBundle(graphs=captured, params=params, weights_dir=weights_dir) + + +__all__ = [ + "CapturedJaxGraph", + "CapturedJaxGraphBundle", + "JaxGraphSpec", + "capture_jax_function", + "capture_jax_function_with_params", + "capture_jax_graphs", +] diff --git a/python/cactus/transpile/jax_semantic_rewrites.py b/python/cactus/transpile/jax_semantic_rewrites.py new file mode 100644 index 000000000..57e23637b --- /dev/null +++ b/python/cactus/transpile/jax_semantic_rewrites.py @@ -0,0 +1,571 @@ +from __future__ import annotations + +import numpy as np + +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.import_semantics import apply_import_semantics + + +def _constant_scalar(graph: IRGraph, value_id: str) -> float | None: + value = graph.constants.get(value_id) + if value is None: + return None + if graph.values[value_id].meta.get("jax_closed_constant"): + return None + array = np.asarray(value) + if array.shape != (): + return None + return float(array.item()) + + +def _constant_singleton_scalar(graph: IRGraph, value_id: str) -> float | None: + value = graph.constants.get(value_id) + if value is None: + return None + array = np.asarray(value) + if array.size != 1: + return None + return float(array.reshape(()).item()) + + +def _constant_scalar_or_singleton(graph: IRGraph, value_id: str) -> float | None: + scalar = _constant_scalar(graph, value_id) + if scalar is not None: + return scalar + singleton = _constant_singleton_scalar(graph, value_id) + if singleton is not None: + return singleton + value = graph.constants.get(value_id) + if value is None: + return None + array = np.asarray(value) + if array.size == 0: + return None + first = float(array.reshape(-1)[0].item()) + if np.allclose(array, first, rtol=0.0, atol=0.0): + return first + return None + + +def _constant_array(graph: IRGraph, value_id: str) -> np.ndarray | None: + value = graph.constants.get(value_id) + if value is None: + return None + if graph.values[value_id].meta.get("jax_closed_constant"): + return None + return np.asarray(value) + + +def _where_scalar_value(value: float) -> float: + if value < -1.0e30: + return -65504.0 + if value > 1.0e30: + return 65504.0 + return value + + +def _rebuild_users(graph: IRGraph) -> None: + for value in graph.values.values(): + value.users.clear() + for node_id in graph.order: + node = graph.nodes[node_id] + for input_id in node.inputs: + graph.values[input_id].users.append(node_id) + + +def _producer(graph: IRGraph, value_id: str) -> IRNode | None: + value = graph.values.get(value_id) + if value is None or value.producer is None: + return None + return graph.nodes.get(value.producer) + + +def _strip_simple_wrappers(graph: IRGraph, value_id: str) -> str: + current = value_id + for _ in range(8): + producer = _producer(graph, current) + if producer is None or producer.op not in {"precision_cast", "reshape", "view", "expand"} or not producer.inputs: + return current + current = producer.inputs[0] + return current + + +def _trace_rms_denominator(graph: IRGraph, value_id: str) -> str | None: + sqrt_node = _producer(graph, value_id) + if sqrt_node is None or sqrt_node.op != "scalar_sqrt" or not sqrt_node.inputs: + return None + add_node = _producer(graph, sqrt_node.inputs[0]) + if add_node is None or add_node.op != "add": + return None + for add_input in add_node.inputs: + mean_id = _strip_simple_wrappers(graph, add_input) + mean_node = _producer(graph, mean_id) + if mean_node is not None and mean_node.op == "divide" and len(mean_node.inputs) == 2: + divisor = _constant_scalar_or_singleton(graph, mean_node.inputs[1]) + if divisor is None: + continue + mean_id = _strip_simple_wrappers(graph, mean_node.inputs[0]) + mean_node = _producer(graph, mean_id) + if mean_node is None or mean_node.op not in {"mean", "sum"} or not mean_node.inputs: + continue + square_node = _producer(graph, _strip_simple_wrappers(graph, mean_node.inputs[0])) + if square_node is None or square_node.op != "multiply" or len(square_node.inputs) != 2: + continue + lhs = _strip_simple_wrappers(graph, square_node.inputs[0]) + rhs = _strip_simple_wrappers(graph, square_node.inputs[1]) + if lhs == rhs: + return lhs + return None + + +def _trace_rms_numerator(graph: IRGraph, value_id: str, source_id: str) -> tuple[str, str] | None: + current = value_id + producer = _producer(graph, current) + if producer is not None and producer.op == "precision_cast" and producer.inputs: + current = producer.inputs[0] + producer = _producer(graph, current) + if producer is None or producer.op != "multiply" or len(producer.inputs) != 2: + return None + lhs, rhs = producer.inputs + if _strip_simple_wrappers(graph, lhs) == source_id: + return lhs, rhs + if _strip_simple_wrappers(graph, rhs) == source_id: + return rhs, lhs + return None + + +def _base_weight_value_id(graph: IRGraph, value_id: str) -> str: + current = value_id + for _ in range(8): + producer = _producer(graph, current) + if producer is None or producer.op not in {"precision_cast", "reshape", "view", "expand"} or not producer.inputs: + return current + current = producer.inputs[0] + return current + + +def _rewrite_jax_rms_norms(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes[node_id] + if node.op == "multiply" and len(node.inputs) == 2: + rewritten = _try_rewrite_jax_multiply_rms_norm(graph, node) + if rewritten: + continue + if node.op != "divide" or len(node.inputs) != 2: + continue + source_id = _trace_rms_denominator(graph, node.inputs[1]) + if source_id is None: + continue + numerator = _trace_rms_numerator(graph, node.inputs[0], source_id) + if numerator is None: + continue + x_id, weight_id = numerator + weight_id = _base_weight_value_id(graph, weight_id) + weight_value = graph.values.get(weight_id) + x_value = graph.values.get(x_id) + if weight_value is None or x_value is None: + continue + if weight_value.shape is None or len(weight_value.shape) != 1: + continue + if x_value.shape is None or int(weight_value.shape[0]) != int(x_value.shape[-1]): + continue + node.op = "rms_norm" + node.inputs = [x_id, weight_id] + node.attrs = {"eps": 1.0e-6} + node.kind = "semantic" + node.meta = {**node.meta, "rewritten_from": "jax_rms_norm"} + + +def _trace_rms_inv_std(graph: IRGraph, value_id: str) -> tuple[str, float] | None: + current = _strip_simple_wrappers(graph, value_id) + producer = _producer(graph, current) + if producer is None: + return None + if producer.op == "pow" and float(producer.attrs.get("exponent", 0.0)) == -0.5 and producer.inputs: + add_id = producer.inputs[0] + elif producer.op == "divide" and len(producer.inputs) == 2: + add_id = producer.inputs[1] + else: + return None + + add_node = _producer(graph, _strip_simple_wrappers(graph, add_id)) + if add_node is None or add_node.op != "add" or len(add_node.inputs) != 2: + return None + + eps = 1.0e-6 + source_id: str | None = None + for add_input in add_node.inputs: + scalar = _constant_scalar(graph, add_input) + if scalar is not None: + eps = float(scalar) + continue + mean_node = _producer(graph, _strip_simple_wrappers(graph, add_input)) + if mean_node is None or mean_node.op not in {"mean", "sum"} or not mean_node.inputs: + continue + square_node = _producer(graph, _strip_simple_wrappers(graph, mean_node.inputs[0])) + if square_node is None or square_node.op != "multiply" or len(square_node.inputs) != 2: + continue + lhs = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, square_node.inputs[0])) + rhs = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, square_node.inputs[1])) + if _strip_simple_wrappers(graph, lhs) == _strip_simple_wrappers(graph, rhs): + source_id = _strip_simple_wrappers(graph, lhs) + if source_id is None: + return None + return source_id, eps + + +def _try_trace_rms_norm_multiply_branch(graph: IRGraph, value_id: str) -> tuple[str, float] | None: + node = _producer(graph, _strip_simple_wrappers(graph, value_id)) + if node is None or len(node.inputs) != 2: + return None + if node.op == "divide": + source_id = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, node.inputs[0])) + traced_source_id = _trace_rms_denominator(graph, node.inputs[1]) + if traced_source_id is None: + return None + if _strip_simple_wrappers(graph, source_id) == _strip_simple_wrappers(graph, traced_source_id): + eps = _trace_rms_denominator_eps(graph, node.inputs[1]) + return _strip_simple_wrappers(graph, traced_source_id), eps + return None + if node.op != "multiply": + return None + for source_candidate, inv_candidate in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + inv = _trace_rms_inv_std(graph, inv_candidate) + if inv is None: + continue + source_id, eps = inv + candidate_id = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, source_candidate)) + if _strip_simple_wrappers(graph, candidate_id) == _strip_simple_wrappers(graph, source_id): + return source_id, eps + return None + + +def _trace_rms_denominator_eps(graph: IRGraph, value_id: str) -> float: + sqrt_node = _producer(graph, value_id) + if sqrt_node is None or sqrt_node.op != "scalar_sqrt" or not sqrt_node.inputs: + return 1.0e-6 + add_node = _producer(graph, sqrt_node.inputs[0]) + if add_node is None or add_node.op != "add": + return 1.0e-6 + for add_input in add_node.inputs: + eps = _constant_scalar_or_singleton(graph, add_input) + if eps is not None: + return float(eps) + return 1.0e-6 + + +def _try_rewrite_jax_multiply_rms_norm(graph: IRGraph, node: IRNode) -> bool: + for norm_id, weight_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + weight_id = _base_weight_value_id(graph, weight_id) + weight_value = graph.values.get(weight_id) + if weight_value is None or weight_value.shape is None or len(weight_value.shape) != 1: + continue + traced = _try_trace_rms_norm_multiply_branch(graph, norm_id) + if traced is None: + continue + source_id, eps = traced + source_producer = _producer(graph, source_id) + if source_producer is not None and source_producer.op == "subtract": + continue + source_value = graph.values.get(source_id) + if source_value is None or source_value.shape is None: + continue + if int(weight_value.shape[0]) != int(source_value.shape[-1]): + continue + node.op = "rms_norm" + node.inputs = [source_id, weight_id] + node.attrs = {"eps": eps} + node.kind = "semantic" + node.meta = {**node.meta, "rewritten_from": "jax_rms_norm_multiply"} + return True + return False + + +def _rewrite_jax_silus(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes[node_id] + if node.op != "multiply" or len(node.inputs) != 2: + continue + for source_id, sigmoid_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + sigmoid_node = _producer(graph, _strip_simple_wrappers(graph, sigmoid_id)) + if sigmoid_node is None or sigmoid_node.op not in {"sigmoid", "logistic"} or len(sigmoid_node.inputs) != 1: + continue + if _strip_simple_wrappers(graph, source_id) != _strip_simple_wrappers(graph, sigmoid_node.inputs[0]): + continue + node.op = "silu" + node.inputs = [source_id] + node.attrs = {} + node.kind = "semantic" + node.meta = {**node.meta, "rewritten_from": "jax_silu"} + break + + +def _record_jax_semantic_pattern_counts(graph: IRGraph) -> None: + counts: dict[str, int] = {} + for node in graph.nodes.values(): + rewritten_from = node.meta.get("rewritten_from") + if isinstance(rewritten_from, str) and rewritten_from.startswith("jax_"): + name = rewritten_from.removeprefix("jax_") + counts[name] = counts.get(name, 0) + 1 + if counts: + graph.meta["jax_semantic_patterns"] = counts + + +def _trace_precision_cast_source(graph: IRGraph, value_id: str) -> str: + producer = _producer(graph, value_id) + if producer is not None and producer.op == "precision_cast" and producer.inputs: + return producer.inputs[0] + return value_id + + +def _trace_batch_norm_centered(graph: IRGraph, value_id: str) -> tuple[str, str] | None: + node = _producer(graph, _strip_simple_wrappers(graph, value_id)) + if node is None or node.op != "subtract" or len(node.inputs) != 2: + return None + lhs = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, node.inputs[0])) + mean_id = _base_weight_value_id(graph, node.inputs[1]) + source_value = graph.values.get(lhs) + mean_value = graph.values.get(mean_id) + if source_value is None or source_value.shape is None: + return None + if mean_value is None or mean_value.shape is None or len(mean_value.shape) != 1: + return None + if int(source_value.shape[-1]) != int(mean_value.shape[0]): + return None + return lhs, mean_id + + +def _trace_batch_norm_inv_scale(graph: IRGraph, value_id: str) -> tuple[str, str, float] | None: + node = _producer(graph, _strip_simple_wrappers(graph, value_id)) + if node is None or node.op != "multiply" or len(node.inputs) != 2: + return None + for inv_id, scale_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + scale_id = _base_weight_value_id(graph, scale_id) + scale_value = graph.values.get(scale_id) + if scale_value is None or scale_value.shape is None or len(scale_value.shape) != 1: + continue + inv_node = _producer(graph, _strip_simple_wrappers(graph, inv_id)) + if inv_node is None or inv_node.op not in {"pow", "divide"} or not inv_node.inputs: + continue + if inv_node.op == "pow" and float(inv_node.attrs.get("exponent", 0.0)) != -0.5: + continue + add_id = inv_node.inputs[0] if inv_node.op == "pow" else inv_node.inputs[1] + add_node = _producer(graph, _strip_simple_wrappers(graph, add_id)) + if add_node is None or add_node.op != "add" or len(add_node.inputs) != 2: + continue + eps = 1.0e-5 + var_id: str | None = None + for add_input in add_node.inputs: + scalar = _constant_scalar_or_singleton(graph, add_input) + if scalar is not None: + eps = float(scalar) + continue + candidate = _base_weight_value_id(graph, add_input) + candidate_value = graph.values.get(candidate) + if candidate_value is not None and candidate_value.shape is not None and len(candidate_value.shape) == 1: + var_id = candidate + if var_id is None: + continue + var_value = graph.values.get(var_id) + if var_value is None or var_value.shape is None or int(var_value.shape[0]) != int(scale_value.shape[0]): + continue + return var_id, scale_id, eps + return None + + +def _trace_batch_norm_affine(graph: IRGraph, value_id: str) -> tuple[str, str, str, str, float] | None: + node = _producer(graph, _strip_simple_wrappers(graph, value_id)) + if node is None or node.op != "multiply" or len(node.inputs) != 2: + return None + for centered_id, inv_scale_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + centered = _trace_batch_norm_centered(graph, centered_id) + inv_scale = _trace_batch_norm_inv_scale(graph, inv_scale_id) + if centered is None or inv_scale is None: + continue + source_id, mean_id = centered + var_id, scale_id, eps = inv_scale + channel_count = int(graph.values[scale_id].shape[0]) # type: ignore[index] + if int(graph.values[mean_id].shape[0]) != channel_count: # type: ignore[index] + continue + if int(graph.values[var_id].shape[0]) != channel_count: # type: ignore[index] + continue + return source_id, scale_id, mean_id, var_id, eps + return None + + +def _rewrite_jax_batch_norms(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes[node_id] + if node.op != "add" or len(node.inputs) != 2: + continue + for affine_id, bias_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + bias_id = _base_weight_value_id(graph, bias_id) + bias_value = graph.values.get(bias_id) + if bias_value is None or bias_value.shape is None or len(bias_value.shape) != 1: + continue + traced = _trace_batch_norm_affine(graph, affine_id) + if traced is None: + continue + source_id, scale_id, mean_id, var_id, eps = traced + source_value = graph.values.get(source_id) + if source_value is None or source_value.shape is None or not source_value.shape: + continue + channel_count = int(bias_value.shape[0]) + if int(source_value.shape[-1]) != channel_count: + continue + node.op = "batch_norm" + node.inputs = [source_id, scale_id, bias_id, mean_id, var_id] + node.attrs = {"axis": len(source_value.shape) - 1, "eps": eps} + node.kind = "semantic" + node.meta = {**node.meta, "rewritten_from": "jax_batch_norm"} + break + + +def _branch_uses_rms_norm_of(graph: IRGraph, branch_value_id: str, residual_value_id: str) -> bool: + residual_base = _strip_simple_wrappers(graph, residual_value_id) + stack = [branch_value_id] + visited: set[str] = set() + while stack and len(visited) < 512: + current = stack.pop() + if current in visited: + continue + visited.add(current) + node = _producer(graph, _strip_simple_wrappers(graph, current)) + if node is None: + continue + if node.op == "rms_norm" and node.inputs: + if _strip_simple_wrappers(graph, node.inputs[0]) == residual_base: + return True + stack.extend(node.inputs) + return False + + +def _rewrite_prenorm_residual_adds_to_clipped(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes[node_id] + if node.op != "add" or len(node.inputs) != 2: + continue + lhs, rhs = node.inputs + if _branch_uses_rms_norm_of(graph, rhs, lhs) or _branch_uses_rms_norm_of(graph, lhs, rhs): + node.op = "add_clipped" + node.kind = "semantic" + node.meta = {**node.meta, "rewritten_from": "prenorm_residual_add"} + + +def _prune_dead_nodes(graph: IRGraph) -> None: + live_values = set(graph.outputs) + live_nodes: set[str] = set() + stack = list(graph.outputs) + while stack: + value_id = stack.pop() + value = graph.values.get(value_id) + if value is None or value.producer is None: + continue + node = graph.nodes.get(value.producer) + if node is None or node.id in live_nodes: + continue + live_nodes.add(node.id) + for input_id in node.inputs: + if input_id not in live_values: + live_values.add(input_id) + stack.append(input_id) + + removed_nodes = set(graph.nodes) - live_nodes + if not removed_nodes: + return + for node_id in removed_nodes: + node = graph.nodes.pop(node_id) + for output_id in node.outputs: + graph.values.pop(output_id, None) + graph.constants.pop(output_id, None) + graph.order = [node_id for node_id in graph.order if node_id in live_nodes] + + +def _unique_graph_id(existing: set[str], base: str) -> str: + candidate = base + suffix = 0 + while candidate in existing: + suffix += 1 + candidate = f"{base}_{suffix}" + return candidate + + +def _legalize_remaining_jax_reductions(graph: IRGraph) -> None: + new_order: list[str] = [] + for node_id in list(graph.order): + node = graph.nodes[node_id] + if node.op not in {"sum", "mean", "min", "max"} or len(node.inputs) != 1 or len(node.outputs) != 1: + new_order.append(node_id) + continue + input_value = graph.values.get(node.inputs[0]) + output_value = graph.values.get(node.outputs[0]) + if input_value is None or output_value is None or input_value.dtype == "fp16": + new_order.append(node_id) + continue + + cast_input_id = _unique_graph_id(set(graph.values), f"{node.outputs[0]}__reduce_fp16_input") + cast_input_node_id = _unique_graph_id(set(graph.nodes), f"{node.id}_reduce_input_cast") + graph.add_node( + IRNode( + cast_input_node_id, + "precision_cast", + [node.inputs[0]], + [cast_input_id], + attrs={"dtype": "fp16"}, + meta={ + "jax_reduce_legalization": "input_fp16", + "source_dtype": input_value.dtype, + }, + ) + ) + graph.values[cast_input_id].shape = input_value.shape + graph.values[cast_input_id].dtype = "fp16" + node.inputs = [cast_input_id] + new_order.append(cast_input_node_id) + new_order.append(node_id) + + if output_value.dtype == "fp16": + continue + + original_output_id = node.outputs[0] + reduce_output_id = _unique_graph_id(set(graph.values), f"{original_output_id}__reduce_fp16") + node.outputs = [reduce_output_id] + graph.values[reduce_output_id] = IRValue( + id=reduce_output_id, + shape=output_value.shape, + dtype="fp16", + producer=node.id, + meta={ + "jax_reduce_legalization": "fp16_output", + "source_output": original_output_id, + }, + ) + cast_output_node_id = _unique_graph_id(set(graph.nodes), f"{node.id}_reduce_output_cast") + graph.nodes[cast_output_node_id] = IRNode( + cast_output_node_id, + "precision_cast", + [reduce_output_id], + [original_output_id], + attrs={"dtype": output_value.dtype}, + meta={ + "jax_reduce_legalization": "restore_output_dtype", + "source_dtype": "fp16", + }, + ) + output_value.producer = cast_output_node_id + new_order.append(cast_output_node_id) + graph.order = new_order + + +def apply_jax_semantic_rewrites(graph: IRGraph) -> None: + _rewrite_jax_batch_norms(graph) + _rewrite_jax_rms_norms(graph) + _rewrite_jax_silus(graph) + apply_import_semantics(graph) + _rewrite_prenorm_residual_adds_to_clipped(graph) + _record_jax_semantic_pattern_counts(graph) + _prune_dead_nodes(graph) + _legalize_remaining_jax_reductions(graph) + _rebuild_users(graph) diff --git a/python/cactus/transpile/jax_user_graph_bundle.py b/python/cactus/transpile/jax_user_graph_bundle.py new file mode 100644 index 000000000..ef196a298 --- /dev/null +++ b/python/cactus/transpile/jax_user_graph_bundle.py @@ -0,0 +1,531 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from dataclasses import fields +import json +from pathlib import Path +from typing import Any + +import numpy as np +import torch + +from cactus.convert.cactus_adapters.tensor_io import save_tensor_with_header +from cactus.transpile.capture_jax import capture_jax_graphs +from cactus.transpile.capture_jax import JaxGraphSpec +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.lower import TranspiledGraph +from cactus.transpile.runtime_compat import Graph +from cactus.transpile.runtime_compat import Tensor + + +@dataclass(frozen=True) +class JaxUserGraphBundleResult: + bundle: Any + output_dir: Path + components_manifest_path: Path + weights_dir: Path + + +@dataclass +class LoadedJaxUserGraph: + name: str + graph: Graph + runtime_inputs: list[Tensor] + outputs: list[Tensor] + logical_inputs: list[str] + logical_outputs: list[str] + + def set_inputs(self, inputs: Sequence[Any]) -> None: + if len(inputs) != len(self.runtime_inputs): + raise ValueError( + f"graph {self.name!r} expected {len(self.runtime_inputs)} inputs, got {len(inputs)}" + ) + for tensor, value in zip(self.runtime_inputs, inputs, strict=True): + self.graph.set_input(tensor, np.asarray(value)) + + def execute(self, *inputs: Any) -> list[Tensor]: + self.set_inputs(inputs) + self.graph.execute() + return self.outputs + + +@dataclass +class LoadedJaxUserGraphBundle: + root: Path + manifest_path: Path + manifest: dict[str, object] + graphs: dict[str, LoadedJaxUserGraph] + + def execute(self, graph_name: str, *inputs: Any) -> list[Tensor]: + if graph_name not in self.graphs: + available = ", ".join(sorted(self.graphs)) or "" + raise ValueError(f"unknown JAX user graph {graph_name!r}; available graphs: {available}") + return self.graphs[graph_name].execute(*inputs) + + +def flatten_jax_params(params: object) -> dict[str, np.ndarray]: + import jax + + return { + _tree_path_name(path): np.asarray(value) + for path, value in jax.tree_util.tree_flatten_with_path(params)[0] + } + + +def _tree_path_name(path: Sequence[Any]) -> str: + parts: list[str] = [] + for entry in path: + if hasattr(entry, "key"): + part = str(entry.key) + elif hasattr(entry, "name"): + part = str(entry.name) + elif hasattr(entry, "idx"): + part = str(entry.idx) + else: + part = str(entry) + parts.append(part) + return ".".join(parts) or "param" + + +def write_fp16_weights_manifest( + weights_dir: str | Path, + params: dict[str, object], + *, + exclude: set[str] | None = None, +) -> Path: + weights_root = Path(weights_dir) + weights_root.mkdir(parents=True, exist_ok=True) + excluded = exclude or set() + manifest: dict[str, dict[str, str]] = {} + for name, value in params.items(): + if name in excluded: + continue + filename = f"{name}.weights" + save_tensor_with_header(np.asarray(value), weights_root / filename, precision="FP16") + manifest[name] = {"filename": filename, "kind": "weight"} + manifest_path = weights_root / "weights_manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + return manifest_path + + +def load_jax_user_graph_bundle(bundle_dir_or_manifest: str | Path) -> LoadedJaxUserGraphBundle: + manifest_path = _resolve_components_manifest(bundle_dir_or_manifest) + root = manifest_path.parent.parent + manifest = json.loads(manifest_path.read_text()) + weights_dir = _resolve_manifest_path(root, manifest.get("weights_dir")) + graphs: dict[str, LoadedJaxUserGraph] = {} + for component in manifest.get("components", []): + if not isinstance(component, dict): + continue + name = str(component.get("component", "") or "") + graph_relpath = component.get("graph") + if not name or not isinstance(graph_relpath, str) or not graph_relpath: + continue + graph = Graph.load(_resolve_manifest_path(root, graph_relpath)) + for binding in component.get("bound_constant_bindings", []) or []: + if not isinstance(binding, dict): + continue + node_id = binding.get("node_id") + path = binding.get("path") + if not isinstance(node_id, int) or not isinstance(path, str): + continue + graph.bind_mmap_weights( + graph._tensor_from_node(int(node_id)), + _resolve_weight_path(root, weights_dir, path), + ) + graphs[name] = LoadedJaxUserGraph( + name=name, + graph=graph, + runtime_inputs=[ + graph._tensor_from_node(int(node_id)) + for node_id in component.get("runtime_input_node_ids", []) or [] + ], + outputs=[ + graph._tensor_from_node(int(node_id)) + for node_id in component.get("output_node_ids", []) or [] + ], + logical_inputs=[str(value) for value in component.get("logical_inputs", []) or []], + logical_outputs=[str(value) for value in component.get("logical_outputs", []) or []], + ) + return LoadedJaxUserGraphBundle( + root=root, + manifest_path=manifest_path, + manifest=manifest, + graphs=graphs, + ) + + +def _resolve_components_manifest(bundle_dir_or_manifest: str | Path) -> Path: + path = Path(bundle_dir_or_manifest).expanduser() + if path.is_dir(): + components_manifest = path / "components" / "manifest.json" + if components_manifest.exists(): + return components_manifest + if path.name == "components" and (path / "manifest.json").exists(): + return path / "manifest.json" + if path.exists(): + return path + raise FileNotFoundError(f"JAX user graph bundle manifest not found: {path}") + + +def _resolve_manifest_path(root: Path, value: object) -> str: + if not isinstance(value, str) or not value: + return str(root) + path = Path(value).expanduser() + return str(path if path.is_absolute() else root / path) + + +def _resolve_weight_path(root: Path, weights_dir: str, value: str) -> str: + path = Path(value).expanduser() + if path.is_absolute(): + return str(path) + candidate = Path(weights_dir) / path + if candidate.exists(): + return str(candidate) + return str(root / path) + + +def _embed_materialized_bound_constants(transpiled_graph: TranspiledGraph) -> None: + external_node_ids = { + int(binding["node_id"]) + for binding in transpiled_graph.bound_constant_bindings + } + for constant in transpiled_graph.bound_constants: + if int(constant.id) in external_node_ids: + continue + transpiled_graph.graph.mark_embedded_input(constant) + + +def _serialize_json_compatible(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _serialize_json_compatible(inner) for key, inner in value.items()} + if isinstance(value, (list, tuple)): + return [_serialize_json_compatible(inner) for inner in value] + if isinstance(value, Path): + return str(value) + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, torch.dtype): + return str(value) + if isinstance(value, torch.Tensor): + payload: dict[str, object] = { + "type": "torch.Tensor", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if value.numel() == 1: + payload["data"] = value.detach().cpu().reshape(-1)[0].item() + return payload + if isinstance(value, np.ndarray): + payload = { + "type": "numpy.ndarray", + "dtype": str(value.dtype), + "shape": list(value.shape), + } + if value.size == 1: + payload["data"] = np.asarray(value).reshape(-1)[0].item() + return payload + if hasattr(value, "__dataclass_fields__"): + return { + field.name: _serialize_json_compatible(getattr(value, field.name)) + for field in fields(value) + } + try: + return repr(value) + except Exception: + return f"<{type(value).__module__}.{type(value).__name__}>" + + +def _ir_graph_to_dict(graph: IRGraph) -> dict[str, object]: + return { + "meta": _serialize_json_compatible(graph.meta), + "inputs": list(graph.inputs), + "outputs": list(graph.outputs), + "constants": { + value_id: _serialize_json_compatible(constant) + for value_id, constant in graph.constants.items() + }, + "values": { + value_id: _serialize_json_compatible(value) + for value_id, value in graph.values.items() + }, + "nodes": [ + _serialize_json_compatible(graph.nodes[node_id]) + for node_id in graph.order + ], + } + + +def _write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def build_jax_user_graph_bundle( + *, + params: Any, + specs: Sequence[JaxGraphSpec], + output_dir: str | Path, + model_id: str, + task: str = "generic", + family: str = "jax_user_graph", + inputs_metadata: dict[str, object] | None = None, + graph_meta: dict[str, object] | None = None, + weight_arrays: dict[str, object] | None = None, + exclude_weights: set[str] | None = None, + weights_dir: str | Path | None = None, +) -> JaxUserGraphBundleResult: + arrays = weight_arrays if weight_arrays is not None else flatten_jax_params(params) + root = Path(output_dir) + weights_root = Path(weights_dir) if weights_dir is not None else root + write_fp16_weights_manifest(weights_root, arrays, exclude=exclude_weights) + + bundle = capture_jax_graphs( + params, + specs, + weights_dir=str(weights_root), + graph_meta={ + "frontend": "jax", + "adapter_family": "generic", + "graph_family": "jax_user_graph_bundle", + **dict(graph_meta or {}), + }, + ) + return _write_jax_user_graph_bundle( + bundle=bundle, + specs=specs, + output_dir=root, + weights_dir=weights_root, + model_id=model_id, + task=task, + family=family, + inputs_metadata=inputs_metadata, + ) + + +def build_jax_generation_graph_bundle( + *, + params: Any, + output_dir: str | Path, + model_id: str, + encoder: JaxGraphSpec | None = None, + decoder_prefill: JaxGraphSpec | None = None, + decoder_step: JaxGraphSpec | None = None, + task: str = "text-generation", + family: str = "jax_user_graph", + inputs_metadata: dict[str, object] | None = None, + graph_meta: dict[str, object] | None = None, + weight_arrays: dict[str, object] | None = None, + exclude_weights: set[str] | None = None, + weights_dir: str | Path | None = None, + max_cache_seq_len: int | None = None, + cache_sink_size: int = 4, +) -> JaxUserGraphBundleResult: + root = Path(output_dir) + weights_root = Path(weights_dir) if weights_dir is not None else root + arrays = weight_arrays if weight_arrays is not None else flatten_jax_params(params) + write_fp16_weights_manifest(weights_root, arrays, exclude=exclude_weights) + + input_specs = tuple(spec for spec in (encoder, decoder_prefill, decoder_step) if spec is not None) + bundle = capture_jax_generation_graphs( + params, + encoder=encoder, + decoder_prefill=decoder_prefill, + decoder_step=decoder_step, + weights_dir=str(weights_root), + graph_meta={ + "frontend": "jax", + "adapter_family": "generic", + "graph_family": "jax_user_graph_bundle", + **dict(graph_meta or {}), + }, + max_cache_seq_len=max_cache_seq_len, + cache_sink_size=cache_sink_size, + ) + return _write_jax_user_graph_bundle( + bundle=bundle, + specs=input_specs, + output_dir=root, + weights_dir=weights_root, + model_id=model_id, + task=task, + family=family, + inputs_metadata=inputs_metadata, + ) + + +def capture_jax_generation_graphs( + params: Any, + *, + encoder: JaxGraphSpec | None = None, + decoder_prefill: JaxGraphSpec | None = None, + decoder_step: JaxGraphSpec | None = None, + weights_dir: str | None = None, + weight_bindings: dict[str, dict[str, str]] | None = None, + graph_meta: dict[str, object] | None = None, + max_cache_seq_len: int | None = None, + cache_sink_size: int = 4, +) -> Any: + specs: list[JaxGraphSpec] = [] + + def _with_meta( + spec: JaxGraphSpec | None, + *, + name: str, + role: str, + component: str, + use_cache: bool, + ) -> None: + if spec is None: + return + cache_meta: dict[str, object] = {} + if use_cache: + cache_meta = { + "use_internal_kv_cache": True, + "cache_sink_size": int(cache_sink_size), + **({"max_cache_seq_len": int(max_cache_seq_len)} if max_cache_seq_len is not None else {}), + } + specs.append( + JaxGraphSpec( + name=spec.name or name, + role=spec.role if spec.role != "generic" else role, + fn=spec.fn, + example_args=spec.example_args, + input_names=spec.input_names, + output_names=spec.output_names, + graph_meta={ + "component": component, + **cache_meta, + **dict(spec.graph_meta or {}), + }, + ) + ) + + _with_meta(encoder, name="encoder", role="encoder", component="encoder", use_cache=False) + _with_meta( + decoder_prefill, + name="decoder_prefill", + role="decoder_prefill", + component="decoder_prefill_chunk", + use_cache=True, + ) + _with_meta( + decoder_step, + name="decoder_step", + role="decoder_step", + component="decoder_step", + use_cache=True, + ) + if not specs: + raise ValueError("capture_jax_generation_graphs requires at least one graph spec") + return capture_jax_graphs( + params, + specs, + weights_dir=weights_dir, + weight_bindings=weight_bindings, + graph_meta={ + "frontend": "jax", + "adapter_family": "generic", + "graph_family": "generation", + **dict(graph_meta or {}), + }, + ) + + +def _write_jax_user_graph_bundle( + *, + bundle: Any, + specs: Sequence[JaxGraphSpec], + output_dir: Path, + weights_dir: Path, + model_id: str, + task: str, + family: str, + inputs_metadata: dict[str, object] | None, +) -> JaxUserGraphBundleResult: + root = output_dir + weights_root = weights_dir + component_root = root / "components" + component_root.mkdir(parents=True, exist_ok=True) + + components = [] + component_order = [spec.name for spec in specs] + base_payload = { + "model_id": model_id, + "model_source": "jax_user_graph", + "task": task, + "family": family, + "inputs": _serialize_json_compatible(dict(inputs_metadata or {})), + } + for name, captured in bundle.graphs.items(): + component_dir = component_root / name + component_dir.mkdir(parents=True, exist_ok=True) + graph_path = component_dir / "graph.cactus" + raw_ir_path = component_dir / "raw_ir.json" + optimized_ir_path = component_dir / "optimized_ir.json" + _write_json( + raw_ir_path, + { + **base_payload, + "component": name, + "graph": _ir_graph_to_dict(captured.raw_ir_graph), + }, + ) + _write_json( + optimized_ir_path, + { + **base_payload, + "component": name, + "graph": _ir_graph_to_dict(captured.ir_graph), + }, + ) + _embed_materialized_bound_constants(captured.graph) + captured.graph.graph.save(str(graph_path)) + components.append( + { + "component": name, + "directory": str(component_dir.relative_to(root)), + "raw_ir": str(raw_ir_path.relative_to(root)), + "optimized_ir": str(optimized_ir_path.relative_to(root)), + "graph": str(graph_path.relative_to(root)), + "inputs": list(captured.ir_graph.inputs), + "outputs": list(captured.ir_graph.outputs), + "logical_inputs": list(captured.spec.input_names or ()), + "logical_outputs": list(captured.spec.output_names or ()), + "node_count": len(captured.ir_graph.order), + "weight_binding_count": len(captured.graph.bound_constant_bindings), + "runtime_input_node_ids": [int(tensor.id) for tensor in captured.graph.runtime_inputs], + "output_node_ids": [int(tensor.id) for tensor in captured.graph.outputs], + "cache_state_node_ids": [ + { + "layer_key": str(layer_key), + "key": int(key_tensor.id), + "value": int(value_tensor.id), + } + for layer_key, key_tensor, value_tensor in getattr(captured.graph, "cache_state_tensors", []) + ], + "bound_constant_bindings": captured.graph.bound_constant_bindings, + } + ) + + manifest = { + "model_id": model_id, + "model_source": "jax_user_graph", + "task": task, + "family": family, + "component_order": component_order, + "inputs": _serialize_json_compatible(dict(inputs_metadata or {})), + "components": components, + "weights_dir": str(weights_root), + "weights_manifest": str(weights_root / "weights_manifest.json"), + } + manifest_path = component_root / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + return JaxUserGraphBundleResult( + bundle=bundle, + output_dir=root, + components_manifest_path=manifest_path, + weights_dir=weights_root, + ) diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index b2f33db41..a06aa61f7 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -533,7 +533,7 @@ def _lower_constant_value( f"unsupported IR constant type for {value.id}: {type(const).__name__}" ) - if tensor_value.numel() == 1: + if tensor_value.numel() == 1 and value.shape in {None, ()}: return tensor_value.item() return _materialize_constant_tensor(g, tensor_value) @@ -865,7 +865,7 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> if len(base_shape) != 4: raise NotImplementedError(f"gqa_repeat_kv alias requires 4D base tensor, got {base_shape}") if len(target_shape) != 4: - raise NotImplementedError(f"gqa_repeat_kv reshape requires 4D target shape, got {target_shape}") + return [g.reshape(_materialize_broadcast_alias(g, source), target_shape)] batch, kv_heads, seq_len, head_dim = base_shape if target_shape[0] != batch or target_shape[2] != seq_len or target_shape[3] != head_dim: raise NotImplementedError( @@ -931,15 +931,20 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> return [g.permute(x, permutation)] if op == "permute": - x = _legalize_for_transpose(g, _tensor(env, node.inputs[0])) + source = env[node.inputs[0]] + if isinstance(source, BroadcastAlias): + x = _materialize_broadcast_alias(g, source) + else: + x = _tensor(env, node.inputs[0]) + x = _legalize_for_transpose(g, x) permutation = tuple(_normalize_dim(int(dim), len(x.shape)) for dim in node.attrs["permutation"]) if len(permutation) == 2 and permutation == (1, 0): return [g.transpose(x)] return [g.permute(x, permutation)] if op == "matmul": - lhs = _tensor(env, node.inputs[0]) - rhs = _tensor(env, node.inputs[1]) + lhs = _tensor_or_materialized_alias(g, env, node.inputs[0]) + rhs = _tensor_or_materialized_alias(g, env, node.inputs[1]) output_value = ir.values.get(node.outputs[0]) output_dtype = ( _map_ir_dtype(output_value.dtype) @@ -1487,7 +1492,11 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> return [g.cat(reshaped, axis=axis)] if op == "slice": - x = _tensor(env, node.inputs[0]) + source = env[node.inputs[0]] + if isinstance(source, BroadcastAlias): + x = _materialize_broadcast_alias(g, source) + else: + x = _tensor(env, node.inputs[0]) axis = _normalize_dim(int(node.attrs["axis"]), len(x.shape)) start = int(node.attrs["start"]) end = int(node.attrs["end"]) @@ -2239,6 +2248,36 @@ def _tensor(env: dict[str, Any], value_id: str) -> Tensor: return value +def _tensor_or_materialized_alias(g: Graph, env: dict[str, Any], value_id: str) -> Tensor: + value = env.get(value_id) + if isinstance(value, BroadcastAlias): + return _materialize_broadcast_alias(g, value) + return _tensor(env, value_id) + + +def _materialize_broadcast_alias(g: Graph, value: BroadcastAlias) -> Tensor: + if value.kind != "gqa_repeat_kv": + raise TypeError(f"unsupported broadcast alias: {value.kind}") + base_shape = tuple(int(dim) for dim in value.tensor.shape) + logical_shape = tuple(int(dim) for dim in value.logical_shape) + if len(base_shape) != 4 or len(logical_shape) not in {4, 5}: + raise TypeError(f"gqa_repeat_kv alias expects 4D->4D/5D shape, got {base_shape} -> {logical_shape}") + batch, kv_heads, seq_len, head_dim = base_shape + if len(logical_shape) == 5: + if logical_shape[0] != batch or logical_shape[1] != kv_heads or logical_shape[3] != seq_len or logical_shape[4] != head_dim: + raise TypeError(f"gqa_repeat_kv alias shape mismatch: {base_shape} -> {logical_shape}") + base = g.reshape(value.tensor, (batch, kv_heads, 1, seq_len, head_dim)) + return _lower_repeat(g, base, (1, 1, logical_shape[2], 1, 1)) + if logical_shape[0] != batch or logical_shape[2] != seq_len or logical_shape[3] != head_dim: + raise TypeError(f"gqa_repeat_kv alias shape mismatch: {base_shape} -> {logical_shape}") + if logical_shape[1] % max(kv_heads, 1) != 0: + raise TypeError(f"gqa_repeat_kv head count mismatch: {base_shape} -> {logical_shape}") + repeats = logical_shape[1] // max(kv_heads, 1) + expanded = g.reshape(value.tensor, (batch, kv_heads, 1, seq_len, head_dim)) + repeated = _lower_repeat(g, expanded, (1, 1, repeats, 1, 1)) + return g.reshape(repeated, logical_shape) + + def _attention_tensor(env: dict[str, Any], value_id: str) -> Tensor: try: value = env[value_id] diff --git a/python/tests/test_capture_jax.py b/python/tests/test_capture_jax.py new file mode 100644 index 000000000..a02f89e46 --- /dev/null +++ b/python/tests/test_capture_jax.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +jax = pytest.importorskip("jax") +import jax.numpy as jnp + +from cactus.convert.cactus_adapters.tensor_io import save_tensor_with_header +from cactus.transpile.capture_jax import capture_jax_function +from cactus.transpile.capture_jax import capture_jax_function_with_params +from cactus.transpile.lower import transpile_ir + + +def _execute_ir(ir, *inputs: object) -> list[np.ndarray]: + graph = transpile_ir(ir) + graph.set_inputs([np.asarray(value) for value in inputs]) + return [output.numpy() for output in graph.execute()] + + +def _assert_close(actual: object, expected: object, *, atol: float = 8e-2, rtol: float = 8e-2) -> None: + np.testing.assert_allclose(np.asarray(actual, dtype=np.float32), np.asarray(expected, dtype=np.float32), atol=atol, rtol=rtol) + + +def _write_fp16_manifest(weights_dir: Path, params: dict[str, object]) -> None: + weights_dir.mkdir(parents=True, exist_ok=True) + manifest = {} + for name, value in params.items(): + filename = f"{name}.weights" + save_tensor_with_header(np.asarray(value), weights_dir / filename, precision="FP16") + manifest[name] = {"filename": filename, "kind": "weight"} + (weights_dir / "weights_manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + +def test_capture_jax_function_imports_common_tensor_ops() -> None: + x = jnp.asarray([[0.2, -0.4, 0.7], [1.0, -1.2, 0.3]], dtype=jnp.float16) + w = jnp.asarray([[0.3, -0.5], [0.2, 0.4], [-0.7, 0.1]], dtype=jnp.float16) + b = jnp.asarray([0.05, -0.02], dtype=jnp.float16) + + def fn(values): + hidden = jnp.matmul(values, w) + b + gated = hidden * jax.nn.sigmoid(hidden) + return jnp.concatenate([gated, jnp.tanh(gated)], axis=-1) + + ir = capture_jax_function(fn, (x,), constant_names=("w", "b")) + got = _execute_ir(ir, x)[0] + + assert ir.meta["frontend"] == "jax" + assert ir.meta["adapter_family"] == "generic" + _assert_close(got, fn(x)) + + +def test_capture_jax_params_resolve_mmap_weights(tmp_path: Path) -> None: + params = { + "w1": jnp.asarray([[0.1, -0.2, 0.3], [0.4, 0.2, -0.1]], dtype=jnp.float16), + "b1": jnp.asarray([0.01, -0.02, 0.03], dtype=jnp.float16), + "w2": jnp.asarray([[0.2], [-0.3], [0.4]], dtype=jnp.float16), + } + x = jnp.asarray([[1.0, -2.0]], dtype=jnp.float16) + _write_fp16_manifest(tmp_path, params) + + def fn(model_params, values): + hidden = jax.nn.gelu(values @ model_params["w1"] + model_params["b1"]) + return hidden @ model_params["w2"] + + ir = capture_jax_function_with_params(fn, params, (x,), weights_dir=str(tmp_path)) + graph = transpile_ir(ir) + graph.set_inputs([np.asarray(x)]) + got = graph.execute()[0].numpy() + + bound_sources = {binding["source_name"] for binding in graph.bound_constant_bindings} + assert bound_sources == set(params) + _assert_close(got, fn(params, x)) + + +def test_capture_jax_handles_scalar_and_broadcast_boundaries() -> None: + scalar = jnp.asarray(2.0, dtype=jnp.float16) + x = jnp.asarray([[1.0, -2.0, 3.0]], dtype=jnp.float16) + + def fn(runtime_scalar, values): + constant = jnp.asarray(0.25, dtype=jnp.float16) + return values * runtime_scalar + jnp.broadcast_to(constant, values.shape) + + ir = capture_jax_function(fn, (scalar, x)) + got = _execute_ir(ir, scalar, x)[0] + + _assert_close(got, fn(scalar, x)) + + +def test_capture_jax_gqa_repeat_then_matmul_matches_jax() -> None: + q = jnp.ones((1, 4, 3, 2), dtype=jnp.float16) + k = jnp.asarray(np.arange(12, dtype=np.float16).reshape(1, 2, 3, 2) / 10.0) + + def fn(query, key): + repeated = jnp.repeat(key, 2, axis=1) + return jnp.matmul(query, jnp.swapaxes(repeated, -1, -2)) + + ir = capture_jax_function(fn, (q, k)) + got = _execute_ir(ir, q, k)[0] + + _assert_close(got, fn(q, k)) + + +def test_capture_jax_rms_norm_pattern_is_visible_to_optimizer() -> None: + x = jnp.asarray(np.linspace(-1.0, 1.0, 16, dtype=np.float16).reshape(1, 2, 8)) + weight = jnp.asarray(np.linspace(0.8, 1.2, 8, dtype=np.float16)) + + def fn(values): + rms = jnp.sqrt(jnp.mean(values.astype(jnp.float32) ** 2, axis=-1, keepdims=True) + 1.0e-6) + return ((values / rms) * weight).astype(jnp.float16) + + ir = capture_jax_function(fn, (x,), constant_names=("weight",)) + got = _execute_ir(ir, x)[0] + + assert any(ir.nodes[node_id].op == "rms_norm" for node_id in ir.order) + _assert_close(got, fn(x)) + + +def test_capture_flax_module_smoke_when_available() -> None: + pytest.importorskip("flax") + from flax import linen as nn + + class TinyFlaxModule(nn.Module): + @nn.compact + def __call__(self, token_ids, features): + embedded = nn.Embed(8, 4)(token_ids) + projected = nn.Dense(4)(features) + hidden = nn.LayerNorm()(embedded + projected) + return nn.Dense(3)(jax.nn.silu(hidden)) + + model = TinyFlaxModule() + token_ids = jnp.asarray([[1, 2]], dtype=jnp.int32) + features = jnp.asarray([[[0.2, -0.1], [0.4, 0.3]]], dtype=jnp.float16) + params = model.init(jax.random.PRNGKey(0), token_ids, features)["params"] + + def fn(model_params, ids, feats): + return model.apply({"params": model_params}, ids, feats) + + ir = capture_jax_function_with_params(fn, params, (token_ids, features)) + got = _execute_ir(ir, token_ids, features)[0] + + assert not any(ir.nodes[node_id].op == "layer_norm" for node_id in ir.order) + _assert_close(got, fn(params, token_ids, features), atol=1.2e-1, rtol=1.2e-1) diff --git a/python/tests/test_capture_jax_user_graph_bundle.py b/python/tests/test_capture_jax_user_graph_bundle.py new file mode 100644 index 000000000..6eee8d625 --- /dev/null +++ b/python/tests/test_capture_jax_user_graph_bundle.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +jax = pytest.importorskip("jax") +import jax.numpy as jnp + +from cactus.transpile.capture_jax import JaxGraphSpec +from cactus.transpile.jax_user_graph_bundle import build_jax_user_graph_bundle +from cactus.transpile.jax_user_graph_bundle import load_jax_user_graph_bundle + + +def _assert_close(actual: object, expected: object, *, atol: float = 8e-2, rtol: float = 8e-2) -> None: + np.testing.assert_allclose(np.asarray(actual, dtype=np.float32), np.asarray(expected, dtype=np.float32), atol=atol, rtol=rtol) + + +def _toy_export(): + params = { + "encoder_w": jnp.asarray([[0.2, -0.1, 0.4], [0.3, 0.5, -0.2]], dtype=jnp.float16), + "decoder_w": jnp.asarray([[0.1, -0.3], [0.4, 0.2], [-0.2, 0.6]], dtype=jnp.float16), + "out_b": jnp.asarray([0.01, -0.02], dtype=jnp.float16), + } + source = jnp.asarray([[[1.0, -0.5], [0.25, 0.75]]], dtype=jnp.float16) + target = jnp.asarray([[[0.4, -0.1, 0.2]]], dtype=jnp.float16) + + def encoder_fn(model_params, source_features): + encoded = source_features @ model_params["encoder_w"] + return encoded * jax.nn.sigmoid(encoded) + + def decoder_fn(model_params, target_features, encoder_out): + context = jnp.mean(encoder_out, axis=1, keepdims=True) + return (target_features + context) @ model_params["decoder_w"] + model_params["out_b"] + + encoder_out = encoder_fn(params, source) + specs = ( + JaxGraphSpec( + name="encoder", + role="encoder", + fn=encoder_fn, + example_args=(source,), + input_names=("source_features",), + output_names=("encoder_out",), + ), + JaxGraphSpec( + name="decoder", + role="generic", + fn=decoder_fn, + example_args=(target, encoder_out), + input_names=("target_features", "encoder_out"), + output_names=("logits",), + ), + ) + return params, specs, source, target, encoder_fn, decoder_fn + + +def test_jax_user_graph_bundle_writes_manifest_graphs_and_weights(tmp_path: Path) -> None: + params, specs, source, target, encoder_fn, decoder_fn = _toy_export() + result = build_jax_user_graph_bundle( + params=params, + specs=specs, + output_dir=tmp_path / "bundle", + model_id="toy-jax", + task="generic", + inputs_metadata={"owner": "client"}, + ) + + encoder_out = result.bundle.execute("encoder", source)[0].numpy() + logits = result.bundle.execute("decoder", target, encoder_out)[0].numpy() + manifest = json.loads(result.components_manifest_path.read_text()) + + assert manifest["model_source"] == "jax_user_graph" + assert manifest["component_order"] == ["encoder", "decoder"] + assert (tmp_path / "bundle/components/encoder/graph.cactus").exists() + assert (tmp_path / "bundle/components/decoder/graph.cactus").exists() + for component in manifest["components"]: + raw_ir_path = tmp_path / "bundle" / component["raw_ir"] + optimized_ir_path = tmp_path / "bundle" / component["optimized_ir"] + assert raw_ir_path == tmp_path / "bundle" / component["directory"] / "raw_ir.json" + assert optimized_ir_path == tmp_path / "bundle" / component["directory"] / "optimized_ir.json" + assert raw_ir_path.exists() + assert optimized_ir_path.exists() + assert json.loads(raw_ir_path.read_text())["graph"]["meta"]["frontend"] == "jax" + assert json.loads(optimized_ir_path.read_text())["graph"]["outputs"] == component["outputs"] + assert (tmp_path / "bundle/weights_manifest.json").exists() + _assert_close(encoder_out, encoder_fn(params, source)) + _assert_close(logits, decoder_fn(params, target, encoder_fn(params, source))) + + +def test_jax_user_graph_bundle_loads_saved_graphs_and_mmap_weights(tmp_path: Path) -> None: + params, specs, source, target, encoder_fn, decoder_fn = _toy_export() + result = build_jax_user_graph_bundle( + params=params, + specs=specs, + output_dir=tmp_path / "bundle", + model_id="toy-jax", + ) + loaded = load_jax_user_graph_bundle(result.output_dir) + + encoder_out = loaded.execute("encoder", source)[0].numpy() + logits = loaded.execute("decoder", target, encoder_out)[0].numpy() + + assert set(loaded.graphs) == {"encoder", "decoder"} + assert loaded.graphs["encoder"].logical_inputs == ["source_features"] + assert loaded.graphs["decoder"].logical_outputs == ["logits"] + _assert_close(encoder_out, encoder_fn(params, source)) + _assert_close(logits, decoder_fn(params, target, encoder_fn(params, source))) + + +def test_jax_user_graph_bundle_supports_external_weights_dir(tmp_path: Path) -> None: + params = { + "w": jnp.asarray([[0.5], [-0.25]], dtype=jnp.float16), + "b": jnp.asarray([0.1], dtype=jnp.float16), + } + x = jnp.asarray([[2.0, -1.0]], dtype=jnp.float16) + + def fn(model_params, values): + return values @ model_params["w"] + model_params["b"] + + result = build_jax_user_graph_bundle( + params=params, + specs=(JaxGraphSpec(name="project", fn=fn, example_args=(x,), input_names=("x",), output_names=("y",)),), + output_dir=tmp_path / "bundle", + weights_dir=tmp_path / "weights", + model_id="external-weights", + ) + loaded = load_jax_user_graph_bundle(result.components_manifest_path) + manifest = json.loads(result.components_manifest_path.read_text()) + + assert result.weights_dir == tmp_path / "weights" + assert (tmp_path / "weights/weights_manifest.json").exists() + assert manifest["weights_dir"] == str(tmp_path / "weights") + _assert_close(loaded.execute("project", x)[0].numpy(), fn(params, x)) + + +def test_jax_user_graph_bundle_binds_duplicate_equal_params_by_path(tmp_path: Path) -> None: + params = { + "a": jnp.zeros((2,), dtype=jnp.float16), + "b": jnp.zeros((2,), dtype=jnp.float16), + } + weight_arrays = { + "a": np.asarray([1.0, 1.0], dtype=np.float16), + "b": np.asarray([2.0, 2.0], dtype=np.float16), + } + x = jnp.asarray([10.0, 20.0], dtype=jnp.float16) + + def fn(model_params, values): + return values + model_params["b"] + + result = build_jax_user_graph_bundle( + params=params, + weight_arrays=weight_arrays, + specs=(JaxGraphSpec(name="forward", fn=fn, example_args=(x,)),), + output_dir=tmp_path / "bundle", + model_id="duplicate-param-bindings", + ) + loaded = load_jax_user_graph_bundle(result.output_dir) + bindings = loaded.manifest["components"][0]["bound_constant_bindings"] + + assert [binding["source_name"] for binding in bindings] == ["b"] + _assert_close(loaded.execute("forward", x)[0].numpy(), np.asarray(x) + weight_arrays["b"]) + + +def test_jax_user_graph_execute_coerces_inputs_to_example_dtype(tmp_path: Path) -> None: + params = {"w": jnp.asarray([[0.5], [-0.25]], dtype=jnp.float16)} + x = jnp.asarray([[2.0, -1.0]], dtype=jnp.float16) + + def fn(model_params, values): + return values @ model_params["w"] + + result = build_jax_user_graph_bundle( + params=params, + specs=(JaxGraphSpec(name="project", fn=fn, example_args=(x,)),), + output_dir=tmp_path / "bundle", + model_id="dtype-coercion", + ) + + got = result.bundle.execute("project", np.asarray(x, dtype=np.float32))[0].numpy() + _assert_close(got, fn(params, x)) + + +def test_jax_user_graph_rejects_wrong_input_count(tmp_path: Path) -> None: + params = {"w": jnp.asarray([[1.0]], dtype=jnp.float16)} + x = jnp.asarray([[2.0]], dtype=jnp.float16) + + def fn(model_params, values): + return values @ model_params["w"] + + result = build_jax_user_graph_bundle( + params=params, + specs=(JaxGraphSpec(name="project", fn=fn, example_args=(x,)),), + output_dir=tmp_path / "bundle", + model_id="errors", + ) + + with pytest.raises(ValueError, match="expected 1 inputs"): + result.bundle.execute("project")