Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 173 additions & 0 deletions python/cactus/transpile/model_adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -4177,6 +4177,92 @@ def get_transpile_metadata(self):
}


class Qwen3LMEncoderStepAdapter(torch.nn.Module):
def __init__(self, model: torch.nn.Module):
super().__init__()
self.backbone = model.model

def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
inputs_embeds = self.backbone.embed_tokens(input_ids)
return inputs_embeds, position_ids.to(dtype=torch.int64)

def get_transpile_metadata(self):
return {
"graph": {
**_transpile_graph_meta(
self.backbone,
adapter_family="qwen3",
adapter_type=type(self).__name__,
input_names=("input_ids", "position_ids"),
),
}
}


class Qwen3LMEncoderTextChunkAdapter(Qwen3LMEncoderStepAdapter):
pass


class Qwen3EmbedsCausalLMStepAdapter(torch.nn.Module):
def __init__(self, model: torch.nn.Module):
super().__init__()
self.model = model
self.backbone = model.model

def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
seq_len = int(inputs_embeds.shape[1])
allowed_positions = torch.tril(
torch.ones((seq_len, seq_len), dtype=torch.bool, device=inputs_embeds.device),
).view(1, 1, seq_len, seq_len)
allowed_values = torch.zeros(
(1, 1, seq_len, seq_len),
dtype=inputs_embeds.dtype,
device=inputs_embeds.device,
)
blocked_values = torch.ones(
(1, 1, seq_len, seq_len),
dtype=inputs_embeds.dtype,
device=inputs_embeds.device,
) * torch.finfo(inputs_embeds.dtype).min
causal_mask = torch.where(allowed_positions, allowed_values, blocked_values)

hidden_states = inputs_embeds
text_position_ids = position_ids.to(dtype=torch.int64)
position_embeddings = self.backbone.rotary_emb(hidden_states, text_position_ids)
for decoder_layer in self.backbone.layers[: self.backbone.config.num_hidden_layers]:
hidden_states = decoder_layer(
hidden_states,
position_embeddings=position_embeddings,
attention_mask=causal_mask,
position_ids=text_position_ids,
past_key_values=None,
use_cache=False,
)
hidden_states = self.backbone.norm(hidden_states)
return self.model.lm_head(hidden_states[:, -1:, :])

def get_transpile_metadata(self):
sliding_window = getattr(self.backbone.config, "sliding_window", None)
layer_types = list(getattr(self.backbone.config, "layer_types", []))
return {
"graph": {
**_transpile_graph_meta(
self.model,
adapter_family="qwen3",
adapter_type=type(self).__name__,
input_names=("inputs_embeds", "position_ids"),
),
"num_hidden_layers": int(self.backbone.config.num_hidden_layers),
"layer_types": tuple(layer_types),
"sliding_window": None if sliding_window is None else int(sliding_window),
}
}


class Qwen3EmbedsCausalLMPrefillChunkAdapter(Qwen3EmbedsCausalLMStepAdapter):
pass


def _build_qwen_causal_lm_component_specs(
model: torch.nn.Module,
*,
Expand Down Expand Up @@ -4205,6 +4291,14 @@ def _build_qwen_causal_lm_component_specs(
"task": "causal_lm_logits",
"adapter_family": family,
}

chunk_components = {"lm_encoder_step", "lm_encoder_text_chunk", "decoder_media_step", "decoder_prefill_chunk"}
wants_chunked = bool(chunk_components & requested_set)
if wants_chunked and family != "qwen3":
raise RuntimeError(
f"text chunked-prefill components are only supported for the qwen3 family, got {family}"
)

specs: list[ComponentModuleSpec] = []
if "decoder" in requested_set:
specs.append(ComponentModuleSpec(
Expand Down Expand Up @@ -4237,6 +4331,85 @@ def _build_qwen_causal_lm_component_specs(
},
metadata={"family": family, "task": "causal_lm_logits"},
))

if wants_chunked:
lm_encoder_step = Qwen3LMEncoderStepAdapter(model).eval()
lm_encoder_text_chunk = Qwen3LMEncoderTextChunkAdapter(model).eval()
decoder_media_step = Qwen3EmbedsCausalLMStepAdapter(model).eval()
decoder_prefill_chunk = Qwen3EmbedsCausalLMPrefillChunkAdapter(model).eval()
max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512)
prefill_chunk_size = max(1, int(os.environ.get("CACTUS_QWEN_PREFILL_CHUNK", "128") or "128"))
prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1]))
chunk_input_ids = input_ids[:, :prefill_chunk_size].contiguous()
chunk_position_ids = torch.arange(
prefill_chunk_size,
dtype=torch.long,
device=input_ids.device,
).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous()
with torch.no_grad():
chunk_embeds, chunk_pos_out = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids)
step_input_ids = input_ids[:, :1]
step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64)
with torch.no_grad():
step_embeds, step_pos_out = lm_encoder_step(step_input_ids, step_position_ids)

if "lm_encoder_step" in requested_set:
specs.append(ComponentModuleSpec(
component="lm_encoder_step",
module=lm_encoder_step,
example_inputs=(step_input_ids, step_position_ids),
input_keys=("input_ids", "position_ids"),
output_keys=("inputs_embeds", "position_ids"),
graph_meta={**common_graph_meta, "component": "lm_encoder_step"},
metadata={"family": family, "task": "causal_lm_logits"},
))
if "lm_encoder_text_chunk" in requested_set:
specs.append(ComponentModuleSpec(
component="lm_encoder_text_chunk",
module=lm_encoder_text_chunk,
example_inputs=(chunk_input_ids, chunk_position_ids),
input_keys=("input_ids", "position_ids"),
output_keys=("inputs_embeds", "position_ids"),
graph_meta={**common_graph_meta, "component": "lm_encoder_text_chunk"},
metadata={"family": family, "task": "causal_lm_logits"},
))
if "decoder_prefill_chunk" in requested_set:
specs.append(ComponentModuleSpec(
component="decoder_prefill_chunk",
module=decoder_prefill_chunk,
example_inputs=(chunk_embeds, chunk_pos_out),
input_keys=("inputs_embeds", "position_ids"),
output_keys=("logits",),
graph_meta={
**common_graph_meta,
"component": "decoder_prefill_chunk",
"use_internal_kv_cache": True,
"use_internal_conv_cache": True,
"use_internal_gated_deltanet_cache": True,
"max_cache_seq_len": max_cache_seq_len,
"cache_sink_size": 4,
"prefill_chunk_size": prefill_chunk_size,
},
metadata={"family": family, "task": "causal_lm_logits"},
))
if "decoder_media_step" in requested_set:
specs.append(ComponentModuleSpec(
component="decoder_media_step",
module=decoder_media_step,
example_inputs=(step_embeds, step_pos_out),
input_keys=("inputs_embeds", "position_ids"),
output_keys=("logits",),
graph_meta={
**common_graph_meta,
"component": "decoder_media_step",
"use_internal_kv_cache": True,
"use_internal_conv_cache": True,
"use_internal_gated_deltanet_cache": True,
"max_cache_seq_len": max_cache_seq_len,
"cache_sink_size": 4,
},
metadata={"family": family, "task": "causal_lm_logits"},
))
return specs


Expand Down
Loading