Skip to content

feat: add ViT encoder ACL Graph capture for Qwen3-VL. #1866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sunbaosong wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add ViT encoder ACL Graph capture for Qwen3-VL. #1866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions xllm/core/common/global_flags.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,9 @@ DECLARE_int32(max_tokens_for_graph_mode);

DECLARE_int32(acl_graph_decode_batch_size_limit);

DECLARE_bool(enable_encoder_graph);
DECLARE_string(encoder_graph_budgets);

DECLARE_bool(enable_chunked_prefill);

DECLARE_string(master_node_addr);
Expand Down
16 changes: 16 additions & 0 deletions xllm/core/framework/config/execution_config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,14 @@ DEFINE_int32(acl_graph_decode_batch_size_limit,
"When actual decode batch_size > this value, ACL graph decode "
"falls back to eager mode to avoid OOM.");

DEFINE_bool(enable_encoder_graph,
false,
"Whether to enable ACL graph for vision encoder");

DEFINE_string(encoder_graph_budgets,
"256,512",
"Comma-separated token budgets for encoder graph buckets");

DEFINE_bool(enable_shm,
false,
"Whether to enable shared memory for executing model.");
Expand Down Expand Up @@ -91,6 +99,8 @@ void ExecutionConfig::from_flags() {
XLLM_CONFIG_ASSIGN_FROM_FLAG(enable_graph_vmm_pool);
XLLM_CONFIG_ASSIGN_FROM_FLAG(max_tokens_for_graph_mode);
XLLM_CONFIG_ASSIGN_FROM_FLAG(acl_graph_decode_batch_size_limit);
XLLM_CONFIG_ASSIGN_FROM_FLAG(enable_encoder_graph);
XLLM_CONFIG_ASSIGN_FROM_FLAG(encoder_graph_budgets);
XLLM_CONFIG_ASSIGN_FROM_FLAG(enable_shm);
XLLM_CONFIG_ASSIGN_FROM_FLAG(use_contiguous_input_buffer);
XLLM_CONFIG_ASSIGN_FROM_FLAG(input_shm_size);
Expand All @@ -106,6 +116,8 @@ void ExecutionConfig::from_json(const JsonReader& json) {
XLLM_CONFIG_ASSIGN_FROM_JSON(enable_graph_vmm_pool);
XLLM_CONFIG_ASSIGN_FROM_JSON(max_tokens_for_graph_mode);
XLLM_CONFIG_ASSIGN_FROM_JSON(acl_graph_decode_batch_size_limit);
XLLM_CONFIG_ASSIGN_FROM_JSON(enable_encoder_graph);
XLLM_CONFIG_ASSIGN_FROM_JSON(encoder_graph_budgets);
XLLM_CONFIG_ASSIGN_FROM_JSON(enable_shm);
XLLM_CONFIG_ASSIGN_FROM_JSON(use_contiguous_input_buffer);
XLLM_CONFIG_ASSIGN_FROM_JSON(input_shm_size);
Expand All @@ -130,6 +142,10 @@ void ExecutionConfig::append_config_json(
config_json, default_config, max_tokens_for_graph_mode);
APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
config_json, default_config, acl_graph_decode_batch_size_limit);
APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
config_json, default_config, enable_encoder_graph);
APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
config_json, default_config, encoder_graph_budgets);
APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
config_json, default_config, enable_shm);
APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
Expand Down
6 changes: 6 additions & 0 deletions xllm/core/framework/config/execution_config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ class ExecutionConfig final {
"enable_graph_vmm_pool",
"max_tokens_for_graph_mode",
"acl_graph_decode_batch_size_limit",
"enable_encoder_graph",
"encoder_graph_budgets",
"enable_shm",
"use_contiguous_input_buffer",
"input_shm_size",
Expand All @@ -69,6 +71,10 @@ class ExecutionConfig final {

PROPERTY(int32_t, acl_graph_decode_batch_size_limit) = 16;

PROPERTY(bool, enable_encoder_graph) = false;

PROPERTY(std::string, encoder_graph_budgets) = "256,512";

PROPERTY(bool, enable_shm) = false;

PROPERTY(bool, use_contiguous_input_buffer) = true;
Expand Down
9 changes: 9 additions & 0 deletions xllm/core/framework/model/causal_vlm.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ class CausalVLM : public CausalLM {
virtual torch::Tensor get_input_embeddings(
const torch::Tensor& input_ids,
const ModelInputParams& input_params) = 0;
virtual void init_encoder_graph_manager(const ModelArgs& args,
const torch::Device& device) {}
};

template <typename Model>
Expand Down Expand Up @@ -161,6 +163,13 @@ class CausalVLMImpl : public CausalVLM {

torch::Device device() const override { return options_.device(); }

void init_encoder_graph_manager(const ModelArgs& args,
const torch::Device& device) override {
if constexpr (detail::has_init_encoder_graph_manager<Model>::value) {
model_->init_encoder_graph_manager(args, device);
}
}

const torch::TensorOptions& options() const override { return options_; }

private:
Expand Down
12 changes: 12 additions & 0 deletions xllm/core/framework/model/model_traits.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ namespace xllm {
struct ModelInputParams;
struct ModelGraphMetadataState;

class ModelLoader;

namespace layer {
class LmHead;
class WordEmbedding;
Expand Down Expand Up @@ -167,6 +169,16 @@ struct has_pooler<T,
std::declval<const torch::Tensor&>()))>>
: std::true_type {};

template <typename T, typename = void>
struct has_init_encoder_graph_manager : std::false_type {};

template <typename T>
struct has_init_encoder_graph_manager<
T,
std::void_t<decltype(std::declval<T>()->init_encoder_graph_manager(
std::declval<const ModelArgs&>(),
std::declval<const torch::Device&>()))>> : std::true_type {};

#if defined(USE_NPU)
template <typename T, typename = void>
struct has_get_npu_lm_head : std::false_type {};
Expand Down
3 changes: 2 additions & 1 deletion xllm/core/framework/model_context.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ bool should_enable_async_tiling_copy_stream() {
// ATB copy-stream teardown is not reversible for the same context on the
// current CANN/PTA stack, so contexts that may enter graph capture must not
// pre-create the helper stream.
if (::xllm::ExecutionConfig::get_instance().enable_graph()) {
if (::xllm::ExecutionConfig::get_instance().enable_graph() ||
::xllm::ExecutionConfig::get_instance().enable_encoder_graph()) {
return false;
}
return util::get_bool_env("ATB_USE_TILING_COPY_STREAM", false);
Expand Down
3 changes: 2 additions & 1 deletion xllm/core/layers/npu/npu_base_layer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ atb::Status BaseLayer::execute_node(atb_speed::Model::Node& node,
// However, libtorch_npu current stream is set to default stream after
// capture ends, causing inconsistency between ATB context and the actual
// execution stream
if (::xllm::ExecutionConfig::get_instance().enable_graph()) {
if (::xllm::ExecutionConfig::get_instance().enable_graph() ||
::xllm::ExecutionConfig::get_instance().enable_encoder_graph()) {
void* stream = c10_npu::getCurrentNPUStream(device_.index()).stream();
context_->SetExecuteStream(stream);
}
Expand Down
4 changes: 4 additions & 0 deletions xllm/core/runtime/options.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,10 @@ struct Options {
// maximum number of tokens for graph execution
PROPERTY(int32_t, max_tokens_for_graph_mode) = 2048;

PROPERTY(bool, enable_encoder_graph) = false;

PROPERTY(std::string, encoder_graph_budgets) = "256,512";

// beam width for beam search
PROPERTY(int32_t, beam_width) = 128;

Expand Down
Loading
Loading