diff --git a/docs/cactus_transpiler.md b/docs/cactus_transpiler.md index 433a44793..90d18f0a3 100644 --- a/docs/cactus_transpiler.md +++ b/docs/cactus_transpiler.md @@ -368,13 +368,15 @@ result = build_jax_user_graph_bundle( loaded = load_jax_user_graph_bundle(result.output_dir) encoder_out, enc_mask = loaded.execute("encoder", src, src_mask) -logits = loaded.execute("decoder_prefill", tgt, encoder_out.numpy(), tgt_mask, enc_mask.numpy())[0].numpy() +logits = loaded.execute("decoder_prefill", tgt, encoder_out, tgt_mask, enc_mask)[0].numpy() next_token = int(np.argmax(logits[0, -1])) ``` Each component is saved under `components//` with `graph.cactus`, -`raw_ir.json`, and `optimized_ir.json`. For fast decode, expose a separate -`decoder_step` graph with cache tensors as explicit inputs and outputs. +`raw_ir.json`, and `optimized_ir.json`. For decode, expose a separate +`decoder_step` graph. The generic JAX path does not create or manage internal +KV cache yet; component output `Tensor`s can be passed directly into another +component. Use `loaded.reset()` between independent generations. --- diff --git a/python/cactus/transpile/jax_semantic_rewrites.py b/python/cactus/transpile/jax_semantic_rewrites.py index 57e23637b..8ffe518f4 100644 --- a/python/cactus/transpile/jax_semantic_rewrites.py +++ b/python/cactus/transpile/jax_semantic_rewrites.py @@ -1,5 +1,8 @@ from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass + import numpy as np from cactus.transpile.graph_ir import IRGraph @@ -8,39 +11,19 @@ from cactus.transpile.import_semantics import apply_import_semantics -def _constant_scalar(graph: IRGraph, value_id: str) -> float | None: - value = graph.constants.get(value_id) - if value is None: - return None - if graph.values[value_id].meta.get("jax_closed_constant"): - return None - array = np.asarray(value) - if array.shape != (): - return None - return float(array.item()) - - -def _constant_singleton_scalar(graph: IRGraph, value_id: str) -> float | None: - value = graph.constants.get(value_id) - if value is None: - return None - array = np.asarray(value) - if array.size != 1: - return None - return float(array.reshape(()).item()) +@dataclass(frozen=True) +class JaxPattern: + name: str + apply: Callable[[IRGraph], None] def _constant_scalar_or_singleton(graph: IRGraph, value_id: str) -> float | None: - scalar = _constant_scalar(graph, value_id) - if scalar is not None: - return scalar - singleton = _constant_singleton_scalar(graph, value_id) - if singleton is not None: - return singleton value = graph.constants.get(value_id) if value is None: return None array = np.asarray(value) + if graph.values[value_id].meta.get("jax_closed_constant") and array.size != 1: + return None if array.size == 0: return None first = float(array.reshape(-1)[0].item()) @@ -49,23 +32,6 @@ def _constant_scalar_or_singleton(graph: IRGraph, value_id: str) -> float | None return None -def _constant_array(graph: IRGraph, value_id: str) -> np.ndarray | None: - value = graph.constants.get(value_id) - if value is None: - return None - if graph.values[value_id].meta.get("jax_closed_constant"): - return None - return np.asarray(value) - - -def _where_scalar_value(value: float) -> float: - if value < -1.0e30: - return -65504.0 - if value > 1.0e30: - return 65504.0 - return value - - def _rebuild_users(graph: IRGraph) -> None: for value in graph.values.values(): value.users.clear() @@ -92,195 +58,321 @@ def _strip_simple_wrappers(graph: IRGraph, value_id: str) -> str: return current -def _trace_rms_denominator(graph: IRGraph, value_id: str) -> str | None: - sqrt_node = _producer(graph, value_id) - if sqrt_node is None or sqrt_node.op != "scalar_sqrt" or not sqrt_node.inputs: - return None - add_node = _producer(graph, sqrt_node.inputs[0]) - if add_node is None or add_node.op != "add": +def _trace_reduction_source(graph: IRGraph, value_id: str) -> str | None: + current = _strip_simple_wrappers(graph, value_id) + node = _producer(graph, current) + if node is not None and node.op == "divide" and len(node.inputs) == 2: + divisor = _constant_scalar_or_singleton(graph, node.inputs[1]) + if divisor is None: + return None + current = _strip_simple_wrappers(graph, node.inputs[0]) + node = _producer(graph, current) + if node is None or node.op not in {"mean", "sum"} or not node.inputs: return None - for add_input in add_node.inputs: - mean_id = _strip_simple_wrappers(graph, add_input) - mean_node = _producer(graph, mean_id) - if mean_node is not None and mean_node.op == "divide" and len(mean_node.inputs) == 2: - divisor = _constant_scalar_or_singleton(graph, mean_node.inputs[1]) - if divisor is None: - continue - mean_id = _strip_simple_wrappers(graph, mean_node.inputs[0]) - mean_node = _producer(graph, mean_id) - if mean_node is None or mean_node.op not in {"mean", "sum"} or not mean_node.inputs: - continue - square_node = _producer(graph, _strip_simple_wrappers(graph, mean_node.inputs[0])) - if square_node is None or square_node.op != "multiply" or len(square_node.inputs) != 2: - continue - lhs = _strip_simple_wrappers(graph, square_node.inputs[0]) - rhs = _strip_simple_wrappers(graph, square_node.inputs[1]) - if lhs == rhs: - return lhs - return None + return _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, node.inputs[0])) -def _trace_rms_numerator(graph: IRGraph, value_id: str, source_id: str) -> tuple[str, str] | None: - current = value_id - producer = _producer(graph, current) - if producer is not None and producer.op == "precision_cast" and producer.inputs: - current = producer.inputs[0] - producer = _producer(graph, current) - if producer is None or producer.op != "multiply" or len(producer.inputs) != 2: +def _trace_square_reduction_source(graph: IRGraph, value_id: str, term_trace) -> str | None: + current = _strip_simple_wrappers(graph, value_id) + node = _producer(graph, current) + if node is not None and node.op == "divide" and len(node.inputs) == 2: + divisor = _constant_scalar_or_singleton(graph, node.inputs[1]) + if divisor is None: + return None + current = _strip_simple_wrappers(graph, node.inputs[0]) + node = _producer(graph, current) + if node is None or node.op not in {"mean", "sum"} or not node.inputs: return None - lhs, rhs = producer.inputs - if _strip_simple_wrappers(graph, lhs) == source_id: - return lhs, rhs - if _strip_simple_wrappers(graph, rhs) == source_id: - return rhs, lhs - return None + square_node = _producer(graph, _strip_simple_wrappers(graph, node.inputs[0])) + if square_node is None or square_node.op != "multiply" or len(square_node.inputs) != 2: + return None + lhs = term_trace(graph, square_node.inputs[0]) + rhs = term_trace(graph, square_node.inputs[1]) + if lhs is None or rhs is None: + return None + if _strip_simple_wrappers(graph, lhs) != _strip_simple_wrappers(graph, rhs): + return None + return _strip_simple_wrappers(graph, lhs) -def _base_weight_value_id(graph: IRGraph, value_id: str) -> str: - current = value_id - for _ in range(8): - producer = _producer(graph, current) - if producer is None or producer.op not in {"precision_cast", "reshape", "view", "expand"} or not producer.inputs: - return current - current = producer.inputs[0] - return current +def _trace_raw_square_source(graph: IRGraph, value_id: str) -> str | None: + return _strip_simple_wrappers(graph, _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, value_id))) -def _rewrite_jax_rms_norms(graph: IRGraph) -> None: - for node_id in list(graph.order): - node = graph.nodes[node_id] - if node.op == "multiply" and len(node.inputs) == 2: - rewritten = _try_rewrite_jax_multiply_rms_norm(graph, node) - if rewritten: - continue - if node.op != "divide" or len(node.inputs) != 2: - continue - source_id = _trace_rms_denominator(graph, node.inputs[1]) - if source_id is None: - continue - numerator = _trace_rms_numerator(graph, node.inputs[0], source_id) - if numerator is None: - continue - x_id, weight_id = numerator - weight_id = _base_weight_value_id(graph, weight_id) - weight_value = graph.values.get(weight_id) - x_value = graph.values.get(x_id) - if weight_value is None or x_value is None: - continue - if weight_value.shape is None or len(weight_value.shape) != 1: - continue - if x_value.shape is None or int(weight_value.shape[0]) != int(x_value.shape[-1]): - continue - node.op = "rms_norm" - node.inputs = [x_id, weight_id] - node.attrs = {"eps": 1.0e-6} - node.kind = "semantic" - node.meta = {**node.meta, "rewritten_from": "jax_rms_norm"} +def _trace_mean_square_source(graph: IRGraph, value_id: str) -> str | None: + return _trace_square_reduction_source(graph, value_id, _trace_raw_square_source) -def _trace_rms_inv_std(graph: IRGraph, value_id: str) -> tuple[str, float] | None: +def _trace_layer_norm_variance_source(graph: IRGraph, value_id: str) -> str | None: current = _strip_simple_wrappers(graph, value_id) - producer = _producer(graph, current) - if producer is None: + node = _producer(graph, current) + if node is not None and node.op == "where" and node.inputs: + current = _strip_simple_wrappers(graph, node.inputs[-1]) + node = _producer(graph, current) + if node is None or node.op != "subtract" or len(node.inputs) != 2: + return None + mean_square_source = _trace_mean_square_source(graph, node.inputs[0]) + square_mean_node = _producer(graph, _strip_simple_wrappers(graph, node.inputs[1])) + if mean_square_source is None or square_mean_node is None or square_mean_node.op != "multiply": + return None + if len(square_mean_node.inputs) != 2: + return None + lhs_mean_source = _trace_reduction_source(graph, square_mean_node.inputs[0]) + rhs_mean_source = _trace_reduction_source(graph, square_mean_node.inputs[1]) + if lhs_mean_source is None or rhs_mean_source is None: return None - if producer.op == "pow" and float(producer.attrs.get("exponent", 0.0)) == -0.5 and producer.inputs: - add_id = producer.inputs[0] - elif producer.op == "divide" and len(producer.inputs) == 2: - add_id = producer.inputs[1] - else: + if _strip_simple_wrappers(graph, lhs_mean_source) != _strip_simple_wrappers(graph, rhs_mean_source): return None + if _strip_simple_wrappers(graph, mean_square_source) != _strip_simple_wrappers(graph, lhs_mean_source): + return None + return mean_square_source + + +def _trace_layer_norm_centered_source(graph: IRGraph, value_id: str) -> str | None: + centered = _trace_layer_norm_centered(graph, value_id) + return None if centered is None else centered[0] + - add_node = _producer(graph, _strip_simple_wrappers(graph, add_id)) +def _trace_layer_norm_centered(graph: IRGraph, value_id: str) -> tuple[str, str] | None: + centered_id = _strip_simple_wrappers(graph, value_id) + node = _producer(graph, centered_id) + if node is None or node.op != "subtract" or len(node.inputs) != 2: + return None + source_id = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, node.inputs[0])) + mean_source_id = _trace_reduction_source(graph, node.inputs[1]) + if mean_source_id is None: + return None + if _strip_simple_wrappers(graph, source_id) != _strip_simple_wrappers(graph, mean_source_id): + return None + return _strip_simple_wrappers(graph, source_id), centered_id + + +def _trace_inv_std_addend(graph: IRGraph, value_id: str) -> str | None: + current = _strip_simple_wrappers(graph, value_id) + node = _producer(graph, current) + if node is None: + return None + if node.op == "pow" and node.inputs: + exponent = float(node.attrs.get("exponent", 0.0)) + if exponent == -0.5: + return node.inputs[0] + if exponent == -1.0: + return _trace_inv_std_addend(graph, node.inputs[0]) + if node.op == "scalar_sqrt" and node.inputs: + return node.inputs[0] + if node.op == "divide" and len(node.inputs) == 2: + numerator = _constant_scalar_or_singleton(graph, node.inputs[0]) + if numerator == 1.0: + return _trace_inv_std_addend(graph, node.inputs[1]) or node.inputs[1] + return _trace_inv_std_addend(graph, node.inputs[1]) + return None + + +def _trace_norm_inv_std( + graph: IRGraph, + value_id: str, + default_eps: float, + variance_trace, +) -> tuple[str, float] | None: + add_id = _trace_inv_std_addend(graph, value_id) + add_node = _producer(graph, _strip_simple_wrappers(graph, add_id)) if add_id is not None else None if add_node is None or add_node.op != "add" or len(add_node.inputs) != 2: return None - eps = 1.0e-6 + eps = default_eps source_id: str | None = None for add_input in add_node.inputs: - scalar = _constant_scalar(graph, add_input) + scalar = _constant_scalar_or_singleton(graph, add_input) if scalar is not None: eps = float(scalar) continue - mean_node = _producer(graph, _strip_simple_wrappers(graph, add_input)) - if mean_node is None or mean_node.op not in {"mean", "sum"} or not mean_node.inputs: - continue - square_node = _producer(graph, _strip_simple_wrappers(graph, mean_node.inputs[0])) - if square_node is None or square_node.op != "multiply" or len(square_node.inputs) != 2: - continue - lhs = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, square_node.inputs[0])) - rhs = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, square_node.inputs[1])) - if _strip_simple_wrappers(graph, lhs) == _strip_simple_wrappers(graph, rhs): - source_id = _strip_simple_wrappers(graph, lhs) + variance_source = variance_trace(graph, add_input) + if variance_source is not None: + source_id = variance_source if source_id is None: return None return source_id, eps -def _try_trace_rms_norm_multiply_branch(graph: IRGraph, value_id: str) -> tuple[str, float] | None: - node = _producer(graph, _strip_simple_wrappers(graph, value_id)) - if node is None or len(node.inputs) != 2: +def _trace_rms_inv_std(graph: IRGraph, value_id: str) -> tuple[str, float] | None: + return _trace_norm_inv_std(graph, value_id, 1.0e-6, _trace_mean_square_source) + + +def _trace_layer_norm_inv_std(graph: IRGraph, value_id: str) -> tuple[str, float] | None: + def _variance_source(g: IRGraph, candidate_id: str) -> str | None: + return _trace_layer_norm_variance_source(g, candidate_id) or _trace_square_reduction_source( + g, + candidate_id, + _trace_layer_norm_centered_source, + ) + + return _trace_norm_inv_std(graph, value_id, 1.0e-5, _variance_source) + + +def _valid_last_dim_weight(graph: IRGraph, weight_id: str, source_id: str) -> str | None: + weight_id = _strip_simple_wrappers(graph, weight_id) + weight_value = graph.values.get(weight_id) + source_value = graph.values.get(source_id) + if weight_value is None or source_value is None: return None - if node.op == "divide": - source_id = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, node.inputs[0])) - traced_source_id = _trace_rms_denominator(graph, node.inputs[1]) - if traced_source_id is None: - return None - if _strip_simple_wrappers(graph, source_id) == _strip_simple_wrappers(graph, traced_source_id): - eps = _trace_rms_denominator_eps(graph, node.inputs[1]) - return _strip_simple_wrappers(graph, traced_source_id), eps + if weight_value.shape is None or source_value.shape is None: return None - if node.op != "multiply": + if len(weight_value.shape) != 1 or int(weight_value.shape[0]) != int(source_value.shape[-1]): return None - for source_candidate, inv_candidate in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - inv = _trace_rms_inv_std(graph, inv_candidate) - if inv is None: - continue - source_id, eps = inv - candidate_id = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, source_candidate)) - if _strip_simple_wrappers(graph, candidate_id) == _strip_simple_wrappers(graph, source_id): - return source_id, eps - return None + return weight_id -def _trace_rms_denominator_eps(graph: IRGraph, value_id: str) -> float: - sqrt_node = _producer(graph, value_id) - if sqrt_node is None or sqrt_node.op != "scalar_sqrt" or not sqrt_node.inputs: - return 1.0e-6 - add_node = _producer(graph, sqrt_node.inputs[0]) - if add_node is None or add_node.op != "add": - return 1.0e-6 - for add_input in add_node.inputs: - eps = _constant_scalar_or_singleton(graph, add_input) - if eps is not None: - return float(eps) - return 1.0e-6 +def _rewrite_norm_node(node: IRNode, op: str, inputs: list[str], eps: float, rewritten_from: str) -> None: + node.op = op + node.inputs = inputs + node.attrs = {"eps": eps} + node.kind = "semantic" + node.meta = {**node.meta, "rewritten_from": rewritten_from} + + +def _trace_norm_branch(graph: IRGraph, value_id: str, source_trace, inv_trace) -> tuple[str, float] | None: + node = _producer(graph, _strip_simple_wrappers(graph, value_id)) + if node is None or len(node.inputs) != 2: + return None + pairs = ((node.inputs[0], node.inputs[1]),) if node.op == "divide" else () + if node.op == "multiply": + pairs = ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])) + for source_candidate, inv_candidate in pairs: + source_id = source_trace(graph, source_candidate) + inv = inv_trace(graph, inv_candidate) + if source_id is None or inv is None: + continue + inv_source_id, eps = inv + if _strip_simple_wrappers(graph, source_id) == _strip_simple_wrappers(graph, inv_source_id): + return source_id, eps + return None -def _try_rewrite_jax_multiply_rms_norm(graph: IRGraph, node: IRNode) -> bool: +def _trace_weighted_norm(graph: IRGraph, value_id: str, branch_trace) -> tuple[str, str, float] | None: + node = _producer(graph, _strip_simple_wrappers(graph, value_id)) + if node is None or node.op != "multiply" or len(node.inputs) != 2: + return None for norm_id, weight_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - weight_id = _base_weight_value_id(graph, weight_id) - weight_value = graph.values.get(weight_id) - if weight_value is None or weight_value.shape is None or len(weight_value.shape) != 1: - continue - traced = _try_trace_rms_norm_multiply_branch(graph, norm_id) + traced = branch_trace(graph, norm_id) if traced is None: continue source_id, eps = traced - source_producer = _producer(graph, source_id) - if source_producer is not None and source_producer.op == "subtract": - continue - source_value = graph.values.get(source_id) - if source_value is None or source_value.shape is None: + weight_id = _valid_last_dim_weight(graph, weight_id, source_id) + if weight_id is None: continue - if int(weight_value.shape[0]) != int(source_value.shape[-1]): + return source_id, weight_id, eps + return None + + +def _trace_factored_weighted_norm( + graph: IRGraph, + value_id: str, + source_trace, + inv_trace, +) -> tuple[str, str, float] | None: + node = _producer(graph, _strip_simple_wrappers(graph, value_id)) + if node is None or node.op != "multiply" or len(node.inputs) != 2: + return None + for source_candidate, scaled_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + source_id = source_trace(graph, source_candidate) + scaled_node = _producer(graph, _strip_simple_wrappers(graph, scaled_id)) + if source_id is None or scaled_node is None or scaled_node.op != "multiply" or len(scaled_node.inputs) != 2: + continue + for inv_id, weight_id in ((scaled_node.inputs[0], scaled_node.inputs[1]), (scaled_node.inputs[1], scaled_node.inputs[0])): + inv = inv_trace(graph, inv_id) + if inv is None: + continue + inv_source_id, eps = inv + if _strip_simple_wrappers(graph, source_id) != _strip_simple_wrappers(graph, inv_source_id): + continue + weight_id = _valid_last_dim_weight(graph, weight_id, source_id) + if weight_id is not None: + return source_id, weight_id, eps + return None + + +def _trace_weighted_rms_norm(graph: IRGraph, value_id: str) -> tuple[str, str, float] | None: + traced = _trace_weighted_norm( + graph, + value_id, + lambda g, candidate: _trace_norm_branch(g, candidate, _trace_raw_square_source, _trace_rms_inv_std), + ) + if traced is not None: + source_id, _, _ = traced + source_producer = _producer(graph, source_id) + if source_producer is None or source_producer.op != "subtract": + return traced + return _trace_factored_weighted_norm(graph, value_id, _trace_raw_square_source, _trace_rms_inv_std) + + +def _trace_weighted_layer_norm(graph: IRGraph, value_id: str) -> tuple[str, str, float] | None: + traced = _trace_weighted_norm( + graph, + value_id, + lambda g, candidate: _trace_norm_branch(g, candidate, _trace_layer_norm_centered_source, _trace_layer_norm_inv_std), + ) + if traced is not None: + return traced + return _trace_factored_weighted_norm( + graph, + value_id, + _trace_layer_norm_centered_source, + _trace_layer_norm_inv_std, + ) + + +def _rewrite_jax_rms_norms(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes[node_id] + if node.op == "multiply" and len(node.inputs) == 2: + traced = _trace_weighted_rms_norm(graph, node.outputs[0] if node.outputs else "") + if traced is not None: + source_id, weight_id, eps = traced + _rewrite_norm_node(node, "rms_norm", [source_id, weight_id], eps, "jax_rms_norm_multiply") + continue + if node.op == "divide" and len(node.inputs) == 2: + denominator = _trace_rms_inv_std(graph, node.inputs[1]) + if denominator is None: + continue + source_id, eps = denominator + numerator = _producer(graph, _strip_simple_wrappers(graph, node.inputs[0])) + if numerator is None or numerator.op != "multiply" or len(numerator.inputs) != 2: + continue + for x_id, weight_id in ((numerator.inputs[0], numerator.inputs[1]), (numerator.inputs[1], numerator.inputs[0])): + if _strip_simple_wrappers(graph, x_id) != _strip_simple_wrappers(graph, source_id): + continue + weight_id = _valid_last_dim_weight(graph, weight_id, x_id) + if weight_id is not None: + _rewrite_norm_node(node, "rms_norm", [x_id, weight_id], eps, "jax_rms_norm") + break + + +def _rewrite_jax_layer_norms(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes[node_id] + if node.op == "add" and len(node.inputs) == 2: + for affine_id, bias_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + bias_id = _strip_simple_wrappers(graph, bias_id) + bias_value = graph.values.get(bias_id) + if bias_value is None or bias_value.shape is None or len(bias_value.shape) != 1: + continue + traced_affine = _trace_weighted_layer_norm(graph, affine_id) + if traced_affine is None: + continue + source_id, weight_id, eps = traced_affine + source_value = graph.values.get(source_id) + if source_value is None or source_value.shape is None: + continue + if int(bias_value.shape[0]) != int(source_value.shape[-1]): + continue + _rewrite_norm_node(node, "layer_norm", [source_id, weight_id, bias_id], eps, "jax_layer_norm") + break continue - node.op = "rms_norm" - node.inputs = [source_id, weight_id] - node.attrs = {"eps": eps} - node.kind = "semantic" - node.meta = {**node.meta, "rewritten_from": "jax_rms_norm_multiply"} - return True - return False + if node.op == "multiply" and len(node.inputs) == 2: + traced = _trace_weighted_layer_norm(graph, node.outputs[0] if node.outputs else "") + if traced is not None: + source_id, weight_id, eps = traced + _rewrite_norm_node(node, "layer_norm", [source_id, weight_id], eps, "jax_layer_norm_multiply") def _rewrite_jax_silus(graph: IRGraph) -> None: @@ -302,31 +394,13 @@ def _rewrite_jax_silus(graph: IRGraph) -> None: break -def _record_jax_semantic_pattern_counts(graph: IRGraph) -> None: - counts: dict[str, int] = {} - for node in graph.nodes.values(): - rewritten_from = node.meta.get("rewritten_from") - if isinstance(rewritten_from, str) and rewritten_from.startswith("jax_"): - name = rewritten_from.removeprefix("jax_") - counts[name] = counts.get(name, 0) + 1 - if counts: - graph.meta["jax_semantic_patterns"] = counts - - -def _trace_precision_cast_source(graph: IRGraph, value_id: str) -> str: - producer = _producer(graph, value_id) - if producer is not None and producer.op == "precision_cast" and producer.inputs: - return producer.inputs[0] - return value_id - - def _trace_batch_norm_centered(graph: IRGraph, value_id: str) -> tuple[str, str] | None: node = _producer(graph, _strip_simple_wrappers(graph, value_id)) if node is None or node.op != "subtract" or len(node.inputs) != 2: return None - lhs = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, node.inputs[0])) - mean_id = _base_weight_value_id(graph, node.inputs[1]) - source_value = graph.values.get(lhs) + source_id = _trace_precision_cast_source(graph, _strip_simple_wrappers(graph, node.inputs[0])) + mean_id = _strip_simple_wrappers(graph, node.inputs[1]) + source_value = graph.values.get(source_id) mean_value = graph.values.get(mean_id) if source_value is None or source_value.shape is None: return None @@ -334,7 +408,7 @@ def _trace_batch_norm_centered(graph: IRGraph, value_id: str) -> tuple[str, str] return None if int(source_value.shape[-1]) != int(mean_value.shape[0]): return None - return lhs, mean_id + return source_id, mean_id def _trace_batch_norm_inv_scale(graph: IRGraph, value_id: str) -> tuple[str, str, float] | None: @@ -342,7 +416,7 @@ def _trace_batch_norm_inv_scale(graph: IRGraph, value_id: str) -> tuple[str, str if node is None or node.op != "multiply" or len(node.inputs) != 2: return None for inv_id, scale_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - scale_id = _base_weight_value_id(graph, scale_id) + scale_id = _strip_simple_wrappers(graph, scale_id) scale_value = graph.values.get(scale_id) if scale_value is None or scale_value.shape is None or len(scale_value.shape) != 1: continue @@ -362,7 +436,7 @@ def _trace_batch_norm_inv_scale(graph: IRGraph, value_id: str) -> tuple[str, str if scalar is not None: eps = float(scalar) continue - candidate = _base_weight_value_id(graph, add_input) + candidate = _strip_simple_wrappers(graph, add_input) candidate_value = graph.values.get(candidate) if candidate_value is not None and candidate_value.shape is not None and len(candidate_value.shape) == 1: var_id = candidate @@ -401,7 +475,7 @@ def _rewrite_jax_batch_norms(graph: IRGraph) -> None: if node.op != "add" or len(node.inputs) != 2: continue for affine_id, bias_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): - bias_id = _base_weight_value_id(graph, bias_id) + bias_id = _strip_simple_wrappers(graph, bias_id) bias_value = graph.values.get(bias_id) if bias_value is None or bias_value.shape is None or len(bias_value.shape) != 1: continue @@ -454,6 +528,223 @@ def _rewrite_prenorm_residual_adds_to_clipped(graph: IRGraph) -> None: node.meta = {**node.meta, "rewritten_from": "prenorm_residual_add"} +def _shape(graph: IRGraph, value_id: str) -> tuple[int, ...] | None: + value = graph.values.get(value_id) + if value is None or value.shape is None: + return None + return tuple(int(dim) for dim in value.shape) + + +def _trace_view_input(graph: IRGraph, value_id: str) -> str: + current = value_id + for _ in range(8): + node = _producer(graph, current) + if node is None or node.op not in {"view", "reshape", "precision_cast"} or not node.inputs: + return current + current = node.inputs[0] + return current + + +def _trace_gqa_repeat_base(graph: IRGraph, value_id: str) -> str: + current = _trace_view_input(graph, value_id) + shape = _shape(graph, current) + if shape is None or len(shape) != 5: + return current + expand = _producer(graph, current) + if expand is None or expand.op != "expand" or not expand.inputs: + return current + base = _trace_view_input(graph, expand.inputs[0]) + base_shape = _shape(graph, base) + if base_shape is None or len(base_shape) != 4: + return current + if base_shape[0] == shape[0] and base_shape[1] == shape[1] and base_shape[2] == shape[3] and base_shape[3] == shape[4]: + return base + return current + + +def _trace_jax_attention_logits(graph: IRGraph, value_id: str) -> tuple[str, str, float, str | None, bool] | None: + current = _trace_view_input(graph, value_id) + node = _producer(graph, current) + mask_id: str | None = None + additive_mask = False + if node is not None and node.op == "where" and len(node.inputs) >= 2: + mask_id = node.inputs[0] + current = _trace_view_input(graph, node.inputs[1]) + node = _producer(graph, current) + if node is not None and node.op == "add" and len(node.inputs) == 2: + for score_id, additive_mask_id in ((node.inputs[0], node.inputs[1]), (node.inputs[1], node.inputs[0])): + score_node = _producer(graph, _trace_view_input(graph, score_id)) + if score_node is not None and score_node.op == "divide": + mask_id = additive_mask_id + additive_mask = True + current = _trace_view_input(graph, score_id) + node = score_node + break + if node is None or node.op != "divide" or len(node.inputs) != 2: + return None + divisor = _constant_scalar_or_singleton(graph, node.inputs[1]) + if divisor is None or divisor == 0.0: + return None + matmul = _producer(graph, _trace_view_input(graph, node.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + query_id = _trace_view_input(graph, matmul.inputs[0]) + key_t_id = _trace_view_input(graph, matmul.inputs[1]) + key_t_node = _producer(graph, key_t_id) + if key_t_node is None or key_t_node.op != "permute" or len(key_t_node.inputs) != 1: + return None + if tuple(int(dim) for dim in key_t_node.attrs.get("permutation", ())) != (0, 1, 3, 2): + return None + key_id = _trace_gqa_repeat_base(graph, key_t_node.inputs[0]) + return query_id, key_id, 1.0 / float(divisor), mask_id, additive_mask + + +def _trace_jax_attention_probs(graph: IRGraph, value_id: str) -> tuple[str, str, float, str | None, bool] | None: + current = _trace_view_input(graph, value_id) + cast = _producer(graph, current) + if cast is not None and cast.op == "precision_cast" and cast.inputs: + current = _trace_view_input(graph, cast.inputs[0]) + divide = _producer(graph, current) + if divide is None or divide.op != "divide" or len(divide.inputs) != 2: + return None + exp_node = _producer(graph, _trace_view_input(graph, divide.inputs[0])) + if exp_node is None or exp_node.op != "scalar_exp" or not exp_node.inputs: + return None + subtract = _producer(graph, _trace_view_input(graph, exp_node.inputs[0])) + if subtract is None or subtract.op != "subtract" or not subtract.inputs: + return None + return _trace_jax_attention_logits(graph, subtract.inputs[0]) + + +def _rewrite_attention_node( + node: IRNode, + query_id: str, + key_id: str, + value_id: str, + mask_id: str | None, + *, + additive_mask: bool, + scale: float, + output_layout: str, + is_causal: bool, + rewritten_from: str, +) -> None: + node.op = "attention" + node.inputs = [query_id, key_id, value_id] + ([] if mask_id is None else [mask_id]) + node.attrs = { + "scale": float(scale), + "is_causal": bool(is_causal), + "window_size": 0, + "q_layout": "bhsd", + "k_layout": "bhsd", + "v_layout": "bhsd", + "output_layout": output_layout, + **({"additive_mask": True} if additive_mask else {}), + } + node.kind = "semantic" + node.meta = {**node.meta, "rewritten_from": rewritten_from} + + +def _trace_attention_output(graph: IRGraph, node: IRNode) -> tuple[str, str, str, float, str | None, bool] | None: + if len(node.inputs) != 1 or len(node.outputs) != 1: + return None + matmul = _producer(graph, _trace_view_input(graph, node.inputs[0])) + if matmul is None or matmul.op != "matmul" or len(matmul.inputs) != 2: + return None + traced = _trace_jax_attention_probs(graph, matmul.inputs[0]) + if traced is None: + return None + query_id, key_id, scale, mask_id, additive_mask = traced + return query_id, key_id, _trace_gqa_repeat_base(graph, matmul.inputs[1]), scale, mask_id, additive_mask + + +def _valid_attention_qkv_shapes( + query_shape: tuple[int, ...] | None, + key_shape: tuple[int, ...] | None, + value_shape: tuple[int, ...] | None, +) -> bool: + if query_shape is None or key_shape is None or value_shape is None: + return False + if len(query_shape) != 4 or len(key_shape) != 4 or len(value_shape) != 4: + return False + if query_shape[0] != key_shape[0] or key_shape[0] != value_shape[0]: + return False + return key_shape[1:] == value_shape[1:] + + +def _valid_attention_mask_shape( + mask_shape: tuple[int, ...] | None, + query_shape: tuple[int, ...], + key_shape: tuple[int, ...], +) -> bool: + return mask_shape == (query_shape[0], query_shape[1], query_shape[2], key_shape[2]) + + +def _rewrite_jax_prefill_attentions(graph: IRGraph) -> None: + for node_id in list(graph.order): + node = graph.nodes[node_id] + if node.op not in {"permute", "view", "reshape"}: + continue + is_decoder_step = str(graph.meta.get("component", "") or "").lower() == "decoder_step" + if node.op == "permute" and tuple(int(dim) for dim in node.attrs.get("permutation", ())) != (0, 2, 1, 3): + continue + if node.op in {"view", "reshape"} and not is_decoder_step: + continue + output_shape = _shape(graph, node.outputs[0]) + if output_shape is None or len(output_shape) != 4: + continue + traced = _trace_attention_output(graph, node) + if traced is None: + continue + query_id, key_id, value_id, scale, mask_id, additive_mask = traced + query_shape = _shape(graph, query_id) + key_shape = _shape(graph, key_id) + value_shape = _shape(graph, value_id) + if not _valid_attention_qkv_shapes(query_shape, key_shape, value_shape): + continue + assert query_shape is not None and key_shape is not None + if mask_id is not None and not _valid_attention_mask_shape(_shape(graph, mask_id), query_shape, key_shape): + continue + is_self_attention = query_shape[2] == key_shape[2] + if is_decoder_step: + if query_shape[2] != 1 or (not is_self_attention and mask_id is None): + continue + _rewrite_attention_node( + node, + query_id, + key_id, + value_id, + mask_id, + additive_mask=additive_mask, + scale=scale, + output_layout="bhsd", + is_causal=is_self_attention, + rewritten_from="jax_decoder_step_self_attention" if is_self_attention else "jax_decoder_step_cross_attention", + ) + continue + if output_shape != (query_shape[0], query_shape[2], query_shape[1], query_shape[3]): + continue + _rewrite_attention_node( + node, + query_id, + key_id, + value_id, + mask_id, + additive_mask=additive_mask, + scale=scale, + output_layout="bthd", + is_causal=mask_id is None and is_self_attention, + rewritten_from="jax_prefill_attention", + ) + + +def _trace_precision_cast_source(graph: IRGraph, value_id: str) -> str: + producer = _producer(graph, value_id) + if producer is not None and producer.op == "precision_cast" and producer.inputs: + return producer.inputs[0] + return value_id + + def _prune_dead_nodes(graph: IRGraph) -> None: live_values = set(graph.outputs) live_nodes: set[str] = set() @@ -559,13 +850,24 @@ def _legalize_remaining_jax_reductions(graph: IRGraph) -> None: graph.order = new_order +JAX_PATTERNS = ( + JaxPattern("batch_norm", _rewrite_jax_batch_norms), + JaxPattern("layer_norm", _rewrite_jax_layer_norms), + JaxPattern("rms_norm", _rewrite_jax_rms_norms), + JaxPattern("silu", _rewrite_jax_silus), + JaxPattern("attention", _rewrite_jax_prefill_attentions), + JaxPattern("prenorm_add_clipped", _rewrite_prenorm_residual_adds_to_clipped), +) + + +def _apply_pattern(graph: IRGraph, pattern: JaxPattern) -> None: + pattern.apply(graph) + + def apply_jax_semantic_rewrites(graph: IRGraph) -> None: - _rewrite_jax_batch_norms(graph) - _rewrite_jax_rms_norms(graph) - _rewrite_jax_silus(graph) + for pattern in JAX_PATTERNS: + _apply_pattern(graph, pattern) apply_import_semantics(graph) - _rewrite_prenorm_residual_adds_to_clipped(graph) - _record_jax_semantic_pattern_counts(graph) _prune_dead_nodes(graph) _legalize_remaining_jax_reductions(graph) _rebuild_users(graph) diff --git a/python/cactus/transpile/jax_user_graph_bundle.py b/python/cactus/transpile/jax_user_graph_bundle.py index ef196a298..60ab0d65d 100644 --- a/python/cactus/transpile/jax_user_graph_bundle.py +++ b/python/cactus/transpile/jax_user_graph_bundle.py @@ -42,13 +42,29 @@ def set_inputs(self, inputs: Sequence[Any]) -> None: f"graph {self.name!r} expected {len(self.runtime_inputs)} inputs, got {len(inputs)}" ) for tensor, value in zip(self.runtime_inputs, inputs, strict=True): - self.graph.set_input(tensor, np.asarray(value)) + if isinstance(value, Tensor): + if tuple(int(dim) for dim in value.shape) != tuple(int(dim) for dim in tensor.shape): + raise ValueError( + f"graph {self.name!r} input shape mismatch for node {tensor.id}: " + f"expected {tensor.shape}, got {value.shape}" + ) + if int(value.dtype) != int(tensor.dtype): + raise ValueError( + f"graph {self.name!r} input dtype mismatch for node {tensor.id}: " + f"expected {tensor.dtype}, got {value.dtype}" + ) + self.graph.set_input(tensor, value.numpy()) + else: + self.graph.set_input(tensor, np.asarray(value)) def execute(self, *inputs: Any) -> list[Tensor]: self.set_inputs(inputs) self.graph.execute() return self.outputs + def reset(self) -> None: + self.graph.hard_reset() + @dataclass class LoadedJaxUserGraphBundle: @@ -63,6 +79,16 @@ def execute(self, graph_name: str, *inputs: Any) -> list[Tensor]: raise ValueError(f"unknown JAX user graph {graph_name!r}; available graphs: {available}") return self.graphs[graph_name].execute(*inputs) + def reset(self, graph_name: str | None = None) -> None: + if graph_name is not None: + if graph_name not in self.graphs: + available = ", ".join(sorted(self.graphs)) or "" + raise ValueError(f"unknown JAX user graph {graph_name!r}; available graphs: {available}") + self.graphs[graph_name].reset() + return + for graph in self.graphs.values(): + graph.reset() + def flatten_jax_params(params: object) -> dict[str, np.ndarray]: import jax @@ -319,8 +345,6 @@ def build_jax_generation_graph_bundle( weight_arrays: dict[str, object] | None = None, exclude_weights: set[str] | None = None, weights_dir: str | Path | None = None, - max_cache_seq_len: int | None = None, - cache_sink_size: int = 4, ) -> JaxUserGraphBundleResult: root = Path(output_dir) weights_root = Path(weights_dir) if weights_dir is not None else root @@ -340,8 +364,6 @@ def build_jax_generation_graph_bundle( "graph_family": "jax_user_graph_bundle", **dict(graph_meta or {}), }, - max_cache_seq_len=max_cache_seq_len, - cache_sink_size=cache_sink_size, ) return _write_jax_user_graph_bundle( bundle=bundle, @@ -364,8 +386,6 @@ def capture_jax_generation_graphs( weights_dir: str | None = None, weight_bindings: dict[str, dict[str, str]] | None = None, graph_meta: dict[str, object] | None = None, - max_cache_seq_len: int | None = None, - cache_sink_size: int = 4, ) -> Any: specs: list[JaxGraphSpec] = [] @@ -375,17 +395,9 @@ def _with_meta( name: str, role: str, component: str, - use_cache: bool, ) -> None: if spec is None: return - cache_meta: dict[str, object] = {} - if use_cache: - cache_meta = { - "use_internal_kv_cache": True, - "cache_sink_size": int(cache_sink_size), - **({"max_cache_seq_len": int(max_cache_seq_len)} if max_cache_seq_len is not None else {}), - } specs.append( JaxGraphSpec( name=spec.name or name, @@ -396,26 +408,23 @@ def _with_meta( output_names=spec.output_names, graph_meta={ "component": component, - **cache_meta, **dict(spec.graph_meta or {}), }, ) ) - _with_meta(encoder, name="encoder", role="encoder", component="encoder", use_cache=False) + _with_meta(encoder, name="encoder", role="encoder", component="encoder") _with_meta( decoder_prefill, name="decoder_prefill", role="decoder_prefill", component="decoder_prefill_chunk", - use_cache=True, ) _with_meta( decoder_step, name="decoder_step", role="decoder_step", component="decoder_step", - use_cache=True, ) if not specs: raise ValueError("capture_jax_generation_graphs requires at least one graph spec") diff --git a/python/tests/test_capture_jax.py b/python/tests/test_capture_jax.py index a02f89e46..4267d8560 100644 --- a/python/tests/test_capture_jax.py +++ b/python/tests/test_capture_jax.py @@ -1,15 +1,11 @@ from __future__ import annotations -import json -from pathlib import Path - import numpy as np import pytest jax = pytest.importorskip("jax") import jax.numpy as jnp -from cactus.convert.cactus_adapters.tensor_io import save_tensor_with_header from cactus.transpile.capture_jax import capture_jax_function from cactus.transpile.capture_jax import capture_jax_function_with_params from cactus.transpile.lower import transpile_ir @@ -25,57 +21,6 @@ def _assert_close(actual: object, expected: object, *, atol: float = 8e-2, rtol: np.testing.assert_allclose(np.asarray(actual, dtype=np.float32), np.asarray(expected, dtype=np.float32), atol=atol, rtol=rtol) -def _write_fp16_manifest(weights_dir: Path, params: dict[str, object]) -> None: - weights_dir.mkdir(parents=True, exist_ok=True) - manifest = {} - for name, value in params.items(): - filename = f"{name}.weights" - save_tensor_with_header(np.asarray(value), weights_dir / filename, precision="FP16") - manifest[name] = {"filename": filename, "kind": "weight"} - (weights_dir / "weights_manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") - - -def test_capture_jax_function_imports_common_tensor_ops() -> None: - x = jnp.asarray([[0.2, -0.4, 0.7], [1.0, -1.2, 0.3]], dtype=jnp.float16) - w = jnp.asarray([[0.3, -0.5], [0.2, 0.4], [-0.7, 0.1]], dtype=jnp.float16) - b = jnp.asarray([0.05, -0.02], dtype=jnp.float16) - - def fn(values): - hidden = jnp.matmul(values, w) + b - gated = hidden * jax.nn.sigmoid(hidden) - return jnp.concatenate([gated, jnp.tanh(gated)], axis=-1) - - ir = capture_jax_function(fn, (x,), constant_names=("w", "b")) - got = _execute_ir(ir, x)[0] - - assert ir.meta["frontend"] == "jax" - assert ir.meta["adapter_family"] == "generic" - _assert_close(got, fn(x)) - - -def test_capture_jax_params_resolve_mmap_weights(tmp_path: Path) -> None: - params = { - "w1": jnp.asarray([[0.1, -0.2, 0.3], [0.4, 0.2, -0.1]], dtype=jnp.float16), - "b1": jnp.asarray([0.01, -0.02, 0.03], dtype=jnp.float16), - "w2": jnp.asarray([[0.2], [-0.3], [0.4]], dtype=jnp.float16), - } - x = jnp.asarray([[1.0, -2.0]], dtype=jnp.float16) - _write_fp16_manifest(tmp_path, params) - - def fn(model_params, values): - hidden = jax.nn.gelu(values @ model_params["w1"] + model_params["b1"]) - return hidden @ model_params["w2"] - - ir = capture_jax_function_with_params(fn, params, (x,), weights_dir=str(tmp_path)) - graph = transpile_ir(ir) - graph.set_inputs([np.asarray(x)]) - got = graph.execute()[0].numpy() - - bound_sources = {binding["source_name"] for binding in graph.bound_constant_bindings} - assert bound_sources == set(params) - _assert_close(got, fn(params, x)) - - def test_capture_jax_handles_scalar_and_broadcast_boundaries() -> None: scalar = jnp.asarray(2.0, dtype=jnp.float16) x = jnp.asarray([[1.0, -2.0, 3.0]], dtype=jnp.float16) @@ -104,13 +49,24 @@ def fn(query, key): _assert_close(got, fn(q, k)) -def test_capture_jax_rms_norm_pattern_is_visible_to_optimizer() -> None: - x = jnp.asarray(np.linspace(-1.0, 1.0, 16, dtype=np.float16).reshape(1, 2, 8)) +@pytest.mark.parametrize("variant", ["mean_rsqrt", "sum_div_rsqrt", "pow_neg_half", "inv_sqrt", "weight_first"]) +def test_capture_jax_rms_norm_common_spellings_lower_to_rms_norm(variant: str) -> None: + x = jnp.asarray(np.linspace(-650.0, 640.0, 16, dtype=np.float16).reshape(1, 2, 8)) weight = jnp.asarray(np.linspace(0.8, 1.2, 8, dtype=np.float16)) def fn(values): - rms = jnp.sqrt(jnp.mean(values.astype(jnp.float32) ** 2, axis=-1, keepdims=True) + 1.0e-6) - return ((values / rms) * weight).astype(jnp.float16) + squared = values.astype(jnp.float32) * values.astype(jnp.float32) + mean_square = jnp.mean(squared, axis=-1, keepdims=True) + if variant == "sum_div_rsqrt": + inv = jax.lax.rsqrt(jnp.sum(squared, axis=-1, keepdims=True) / values.shape[-1] + 1.0e-5) + return values * inv * weight + if variant == "pow_neg_half": + return values * ((mean_square + 1.0e-5) ** -0.5) * weight + if variant == "inv_sqrt": + return values * (1.0 / jnp.sqrt(mean_square + 1.0e-5)) * weight + if variant == "weight_first": + return weight * (values * jax.lax.rsqrt(mean_square + 1.0e-5)) + return values * jax.lax.rsqrt(mean_square + 1.0e-5) * weight ir = capture_jax_function(fn, (x,), constant_names=("weight",)) got = _execute_ir(ir, x)[0] @@ -119,6 +75,101 @@ def fn(values): _assert_close(got, fn(x)) +def test_capture_jax_zero_centered_rms_norm_lowers_to_rms_norm() -> None: + x = jnp.asarray(np.linspace(-73.0, 73.0, 512, dtype=np.float16).reshape(1, 1, 512)) + scale = jnp.zeros((512,), dtype=jnp.float16) + + def fn(values): + rms = jnp.sqrt(jnp.mean(values.astype(jnp.float32) ** 2, axis=-1, keepdims=True) + 1.0e-6) + return ((1.0 + scale) * values / rms).astype(jnp.float16) + + ir = capture_jax_function(fn, (x,), constant_names=("scale",)) + got = _execute_ir(ir, x)[0] + + assert any(ir.nodes[node_id].op == "rms_norm" for node_id in ir.order) + _assert_close(got, fn(x)) + + +@pytest.mark.parametrize("variant", ["mean_rsqrt", "sum_div_rsqrt", "sqrt_div", "no_bias"]) +def test_capture_jax_layer_norm_common_spellings_lower_to_layer_norm(variant: str) -> None: + x = jnp.asarray(np.linspace(-650.0, 640.0, 16, dtype=np.float16).reshape(1, 2, 8)) + weight = jnp.asarray(np.linspace(0.8, 1.2, 8, dtype=np.float16)) + bias = jnp.asarray(np.linspace(-0.2, 0.2, 8, dtype=np.float16)) + + def fn(values): + centered = values - jnp.mean(values.astype(jnp.float32), axis=-1, keepdims=True) + if variant == "sum_div_rsqrt": + var = jnp.sum(centered.astype(jnp.float32) * centered.astype(jnp.float32), axis=-1, keepdims=True) / values.shape[-1] + normed = centered * jax.lax.rsqrt(var + 1.0e-5) + elif variant == "sqrt_div": + var = jnp.mean(centered.astype(jnp.float32) * centered.astype(jnp.float32), axis=-1, keepdims=True) + normed = centered / jnp.sqrt(var + 1.0e-5) + else: + var = jnp.mean(centered.astype(jnp.float32) * centered.astype(jnp.float32), axis=-1, keepdims=True) + normed = centered * jax.lax.rsqrt(var + 1.0e-5) + if variant == "no_bias": + return normed * weight + return normed * weight + bias + + ir = capture_jax_function(fn, (x,), constant_names=("weight", "bias")) + got = _execute_ir(ir, x)[0] + + assert any(ir.nodes[node_id].op == "layer_norm" for node_id in ir.order) + _assert_close(got, fn(x)) + + +def test_capture_jax_prenorm_residual_add_lowers_to_add_clipped() -> None: + x = jnp.asarray(np.linspace(-0.5, 0.5, 16, dtype=np.float16).reshape(1, 2, 8)) + weight = jnp.asarray(np.linspace(0.8, 1.2, 8, dtype=np.float16)) + + def fn(values): + rms = jnp.sqrt(jnp.mean(values.astype(jnp.float32) ** 2, axis=-1, keepdims=True) + 1.0e-6) + branch = (values / rms) * weight + return values + branch + + ir = capture_jax_function(fn, (x,), constant_names=("weight",)) + got = _execute_ir(ir, x)[0] + + assert any(ir.nodes[node_id].op == "add_clipped" for node_id in ir.order) + _assert_close(got, fn(x)) + + +def test_capture_jax_prefill_attention_pattern_lowers_to_attention() -> None: + q = jnp.asarray(np.linspace(-0.5, 0.5, 1 * 2 * 3 * 4, dtype=np.float16).reshape(1, 2, 3, 4)) + k = jnp.asarray(np.linspace(0.25, -0.25, 1 * 2 * 3 * 4, dtype=np.float16).reshape(1, 2, 3, 4)) + v = jnp.asarray(np.linspace(-0.75, 0.75, 1 * 2 * 3 * 4, dtype=np.float16).reshape(1, 2, 3, 4)) + mask = jnp.asarray(np.where(np.tril(np.ones((1, 2, 3, 3), dtype=np.bool_)), 0.0, -1.0e4), dtype=jnp.float16) + + def fn(query, key, value, additive_mask): + scores = (query @ key.transpose(0, 1, 3, 2)) / jnp.sqrt(jnp.asarray(4.0, dtype=jnp.float32)) + probs = jax.nn.softmax(scores + additive_mask, axis=-1) + return (probs @ value).transpose(0, 2, 1, 3).reshape(1, 3, 8) + + ir = capture_jax_function(fn, (q, k, v, mask)) + got = _execute_ir(ir, q, k, v, mask)[0] + + assert any(ir.nodes[node_id].op == "attention" for node_id in ir.order) + _assert_close(got, fn(q, k, v, mask)) + + +def test_capture_jax_attention_keeps_broadcast_mask_decomposed() -> None: + q = jnp.asarray(np.linspace(-0.5, 0.5, 2 * 4 * 5 * 8, dtype=np.float16).reshape(2, 4, 5, 8)) + k = jnp.asarray(np.linspace(0.25, -0.25, 2 * 4 * 5 * 8, dtype=np.float16).reshape(2, 4, 5, 8)) + v = jnp.asarray(np.linspace(-0.75, 0.75, 2 * 4 * 5 * 8, dtype=np.float16).reshape(2, 4, 5, 8)) + mask = jnp.asarray(np.where(np.tril(np.ones((1, 1, 5, 5), dtype=np.bool_)), 0.0, -1.0e4), dtype=jnp.float16) + + def fn(query, key, value, additive_mask): + scores = (query @ key.transpose(0, 1, 3, 2)) / jnp.sqrt(jnp.asarray(8.0, dtype=jnp.float32)) + probs = jax.nn.softmax(scores + additive_mask, axis=-1) + return (probs @ value).transpose(0, 2, 1, 3).reshape(2, 5, 32) + + ir = capture_jax_function(fn, (q, k, v, mask)) + got = _execute_ir(ir, q, k, v, mask)[0] + + assert not any(ir.nodes[node_id].op == "attention" for node_id in ir.order) + _assert_close(got, fn(q, k, v, mask)) + + def test_capture_flax_module_smoke_when_available() -> None: pytest.importorskip("flax") from flax import linen as nn @@ -142,5 +193,5 @@ def fn(model_params, ids, feats): ir = capture_jax_function_with_params(fn, params, (token_ids, features)) got = _execute_ir(ir, token_ids, features)[0] - assert not any(ir.nodes[node_id].op == "layer_norm" for node_id in ir.order) + assert any(ir.nodes[node_id].op == "layer_norm" for node_id in ir.order) _assert_close(got, fn(params, token_ids, features), atol=1.2e-1, rtol=1.2e-1) diff --git a/python/tests/test_capture_jax_user_graph_bundle.py b/python/tests/test_capture_jax_user_graph_bundle.py index 6eee8d625..ac272cf74 100644 --- a/python/tests/test_capture_jax_user_graph_bundle.py +++ b/python/tests/test_capture_jax_user_graph_bundle.py @@ -10,6 +10,7 @@ import jax.numpy as jnp from cactus.transpile.capture_jax import JaxGraphSpec +from cactus.transpile.jax_user_graph_bundle import build_jax_generation_graph_bundle from cactus.transpile.jax_user_graph_bundle import build_jax_user_graph_bundle from cactus.transpile.jax_user_graph_bundle import load_jax_user_graph_bundle @@ -38,32 +39,32 @@ def decoder_fn(model_params, target_features, encoder_out): encoder_out = encoder_fn(params, source) specs = ( JaxGraphSpec( - name="encoder", - role="encoder", - fn=encoder_fn, - example_args=(source,), - input_names=("source_features",), - output_names=("encoder_out",), + name="encoder", role="encoder", fn=encoder_fn, example_args=(source,), + input_names=("source_features",), output_names=("encoder_out",), ), JaxGraphSpec( - name="decoder", - role="generic", - fn=decoder_fn, - example_args=(target, encoder_out), - input_names=("target_features", "encoder_out"), - output_names=("logits",), + name="decoder", role="generic", fn=decoder_fn, example_args=(target, encoder_out), + input_names=("target_features", "encoder_out"), output_names=("logits",), ), ) return params, specs, source, target, encoder_fn, decoder_fn -def test_jax_user_graph_bundle_writes_manifest_graphs_and_weights(tmp_path: Path) -> None: +def _build_toy_bundle(tmp_path: Path, **kwargs): params, specs, source, target, encoder_fn, decoder_fn = _toy_export() result = build_jax_user_graph_bundle( params=params, specs=specs, output_dir=tmp_path / "bundle", model_id="toy-jax", + **kwargs, + ) + return result, params, source, target, encoder_fn, decoder_fn + + +def test_jax_user_graph_bundle_writes_manifest_graphs_and_weights(tmp_path: Path) -> None: + result, params, source, target, encoder_fn, decoder_fn = _build_toy_bundle( + tmp_path, task="generic", inputs_metadata={"owner": "client"}, ) @@ -77,12 +78,8 @@ def test_jax_user_graph_bundle_writes_manifest_graphs_and_weights(tmp_path: Path assert (tmp_path / "bundle/components/encoder/graph.cactus").exists() assert (tmp_path / "bundle/components/decoder/graph.cactus").exists() for component in manifest["components"]: - raw_ir_path = tmp_path / "bundle" / component["raw_ir"] - optimized_ir_path = tmp_path / "bundle" / component["optimized_ir"] - assert raw_ir_path == tmp_path / "bundle" / component["directory"] / "raw_ir.json" - assert optimized_ir_path == tmp_path / "bundle" / component["directory"] / "optimized_ir.json" - assert raw_ir_path.exists() - assert optimized_ir_path.exists() + raw_ir_path, optimized_ir_path = (tmp_path / "bundle" / component[key] for key in ("raw_ir", "optimized_ir")) + assert raw_ir_path.exists() and optimized_ir_path.exists() assert json.loads(raw_ir_path.read_text())["graph"]["meta"]["frontend"] == "jax" assert json.loads(optimized_ir_path.read_text())["graph"]["outputs"] == component["outputs"] assert (tmp_path / "bundle/weights_manifest.json").exists() @@ -91,23 +88,18 @@ def test_jax_user_graph_bundle_writes_manifest_graphs_and_weights(tmp_path: Path def test_jax_user_graph_bundle_loads_saved_graphs_and_mmap_weights(tmp_path: Path) -> None: - params, specs, source, target, encoder_fn, decoder_fn = _toy_export() - result = build_jax_user_graph_bundle( - params=params, - specs=specs, - output_dir=tmp_path / "bundle", - model_id="toy-jax", - ) + result, params, source, target, encoder_fn, decoder_fn = _build_toy_bundle(tmp_path) loaded = load_jax_user_graph_bundle(result.output_dir) - encoder_out = loaded.execute("encoder", source)[0].numpy() + encoder_out = loaded.execute("encoder", source)[0] logits = loaded.execute("decoder", target, encoder_out)[0].numpy() assert set(loaded.graphs) == {"encoder", "decoder"} assert loaded.graphs["encoder"].logical_inputs == ["source_features"] assert loaded.graphs["decoder"].logical_outputs == ["logits"] - _assert_close(encoder_out, encoder_fn(params, source)) + _assert_close(encoder_out.numpy(), encoder_fn(params, source)) _assert_close(logits, decoder_fn(params, target, encoder_fn(params, source))) + loaded.reset() def test_jax_user_graph_bundle_supports_external_weights_dir(tmp_path: Path) -> None: @@ -133,68 +125,72 @@ def fn(model_params, values): assert result.weights_dir == tmp_path / "weights" assert (tmp_path / "weights/weights_manifest.json").exists() assert manifest["weights_dir"] == str(tmp_path / "weights") - _assert_close(loaded.execute("project", x)[0].numpy(), fn(params, x)) + _assert_close(loaded.execute("project", np.asarray(x, dtype=np.float32))[0].numpy(), fn(params, x)) -def test_jax_user_graph_bundle_binds_duplicate_equal_params_by_path(tmp_path: Path) -> None: +def test_jax_generation_decoder_step_fuses_attention_without_internal_cache(tmp_path: Path) -> None: params = { - "a": jnp.zeros((2,), dtype=jnp.float16), - "b": jnp.zeros((2,), dtype=jnp.float16), - } - weight_arrays = { - "a": np.asarray([1.0, 1.0], dtype=np.float16), - "b": np.asarray([2.0, 2.0], dtype=np.float16), + "wq": jnp.eye(16, dtype=jnp.float16), + "wk": jnp.eye(16, dtype=jnp.float16), + "wv": jnp.eye(16, dtype=jnp.float16), } - x = jnp.asarray([10.0, 20.0], dtype=jnp.float16) + x = jnp.asarray(np.linspace(-0.75, 0.75, 16, dtype=np.float16).reshape(1, 1, 16)) - def fn(model_params, values): - return values + model_params["b"] + def decoder_step(model_params, values): + q = (values @ model_params["wq"]).reshape(1, 1, 2, 8).transpose(0, 2, 1, 3) + k = (values @ model_params["wk"]).reshape(1, 1, 2, 8).transpose(0, 2, 1, 3) + v = (values @ model_params["wv"]).reshape(1, 1, 2, 8).transpose(0, 2, 1, 3) + scores = (q @ k.transpose(0, 1, 3, 2)) / jnp.sqrt(jnp.asarray(8.0, dtype=jnp.float32)) + probs = jax.nn.softmax(scores, axis=-1).astype(jnp.float16) + return (probs @ v).transpose(0, 2, 1, 3).reshape(1, 1, 16) - result = build_jax_user_graph_bundle( + result = build_jax_generation_graph_bundle( params=params, - weight_arrays=weight_arrays, - specs=(JaxGraphSpec(name="forward", fn=fn, example_args=(x,)),), + decoder_step=JaxGraphSpec(name="decoder_step", fn=decoder_step, example_args=(x,), output_names=("hidden",)), output_dir=tmp_path / "bundle", - model_id="duplicate-param-bindings", + model_id="decoder-step-attention", ) - loaded = load_jax_user_graph_bundle(result.output_dir) - bindings = loaded.manifest["components"][0]["bound_constant_bindings"] + graph = result.bundle.graphs["decoder_step"].ir_graph + attention_nodes = [node for node in graph.nodes.values() if node.op == "attention"] - assert [binding["source_name"] for binding in bindings] == ["b"] - _assert_close(loaded.execute("forward", x)[0].numpy(), np.asarray(x) + weight_arrays["b"]) + assert "use_internal_kv_cache" not in graph.meta + assert len(attention_nodes) == 1 + assert result.bundle.graphs["decoder_step"].graph.cache_state_tensors == [] + _assert_close(result.bundle.execute("decoder_step", x)[0].numpy(), decoder_step(params, x)) -def test_jax_user_graph_execute_coerces_inputs_to_example_dtype(tmp_path: Path) -> None: - params = {"w": jnp.asarray([[0.5], [-0.25]], dtype=jnp.float16)} - x = jnp.asarray([[2.0, -1.0]], dtype=jnp.float16) - - def fn(model_params, values): - return values @ model_params["w"] - - result = build_jax_user_graph_bundle( +def test_jax_generation_decoder_step_fuses_cross_attention(tmp_path: Path) -> None: + params = { + "wq": jnp.eye(16, dtype=jnp.float16), + } + x = jnp.asarray(np.linspace(-0.75, 0.75, 16, dtype=np.float16).reshape(1, 1, 16)) + key = jnp.asarray(np.linspace(-0.5, 0.5, num=1 * 2 * 4 * 8).reshape(1, 2, 4, 8), dtype=jnp.float16) + value = jnp.asarray(np.linspace(0.75, -0.25, num=1 * 2 * 4 * 8).reshape(1, 2, 4, 8), dtype=jnp.float16) + mask = jnp.asarray([[[[True, True, True, False]]]], dtype=jnp.bool_) + + def decoder_step(model_params, values, cross_k, cross_v, cross_mask): + q = (values @ model_params["wq"]).reshape(1, 1, 2, 8).transpose(0, 2, 1, 3) + scores = (q @ cross_k.transpose(0, 1, 3, 2)) / jnp.sqrt(jnp.asarray(8.0, dtype=jnp.float32)) + scores = jnp.where(cross_mask, scores, jnp.finfo(scores.dtype).min) + probs = jax.nn.softmax(scores, axis=-1).astype(jnp.float16) + return (probs @ cross_v).transpose(0, 2, 1, 3).reshape(1, 1, 16) + + result = build_jax_generation_graph_bundle( params=params, - specs=(JaxGraphSpec(name="project", fn=fn, example_args=(x,)),), + decoder_step=JaxGraphSpec( + name="decoder_step", fn=decoder_step, example_args=(x, key, value, mask), output_names=("hidden",), + ), output_dir=tmp_path / "bundle", - model_id="dtype-coercion", + model_id="cross-attention-step", ) - - got = result.bundle.execute("project", np.asarray(x, dtype=np.float32))[0].numpy() - _assert_close(got, fn(params, x)) - - -def test_jax_user_graph_rejects_wrong_input_count(tmp_path: Path) -> None: - params = {"w": jnp.asarray([[1.0]], dtype=jnp.float16)} - x = jnp.asarray([[2.0]], dtype=jnp.float16) - - def fn(model_params, values): - return values @ model_params["w"] - - result = build_jax_user_graph_bundle( - params=params, - specs=(JaxGraphSpec(name="project", fn=fn, example_args=(x,)),), - output_dir=tmp_path / "bundle", - model_id="errors", + graph = result.bundle.graphs["decoder_step"].ir_graph + attention_nodes = [node for node in graph.nodes.values() if node.op == "attention"] + + assert len(attention_nodes) == 1 + assert attention_nodes[0].meta["rewritten_from"] == "jax_decoder_step_cross_attention" + assert len(attention_nodes[0].inputs) == 4 + assert result.bundle.graphs["decoder_step"].graph.cache_state_tensors == [] + _assert_close( + result.bundle.execute("decoder_step", x, key, value, mask)[0].numpy(), + decoder_step(params, x, key, value, mask), ) - - with pytest.raises(ValueError, match="expected 1 inputs"): - result.bundle.execute("project")