Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions cactus-engine/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ set(ENGINE_SOURCES
src/sp.cpp
src/constraints.cpp
src/model.cpp
src/model_npu.cpp
src/engine_image.cpp
src/index.cpp
src/index_ffi.cpp
Expand All @@ -67,6 +68,16 @@ if(APPLE)
enable_language(OBJCXX)
list(APPEND ENGINE_SOURCES src/npu_ane.mm)
set_source_files_properties(src/npu_ane.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc")

# --- GPU backend (Metal text decoder, justin/v2-gpu) ---
# The GPU model lives in src/gpu/. Its Objective-C++ implementation
# depends on cactus-kernels-gpu (which we add as a subdirectory).
if(NOT TARGET cactus-kernels-gpu)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus-kernels-gpu
${CMAKE_CURRENT_BINARY_DIR}/cactus-kernels-gpu)
endif()
list(APPEND ENGINE_SOURCES src/gpu/gpu_model.mm)
set_source_files_properties(src/gpu/gpu_model.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc")
else()
list(APPEND ENGINE_SOURCES src/npu.cpp)
endif()
Expand Down Expand Up @@ -110,6 +121,8 @@ if(APPLE)
${COREML_FRAMEWORK} ${FOUNDATION_FRAMEWORK}
${SECURITY_FRAMEWORK} ${SYSTEMCONFIGURATION_FRAMEWORK} ${CFNETWORK_FRAMEWORK}
)
# GPU backend: link the kernel dispatch library + its Metal framework deps.
target_link_libraries(cactus_engine PUBLIC cactus-kernels-gpu)
endif()

# --- Version ---
Expand Down
46 changes: 14 additions & 32 deletions cactus-engine/src/engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,36 +50,6 @@ class NPUEncoder {
std::unique_ptr<NPUEncoder> create_encoder();
bool is_npu_available();

struct NPUBufferRef {
const __fp16* data;
size_t count;
};

struct NPUPrefillDirectResult {
NPUBufferRef hidden;
std::vector<NPUBufferRef> k_caches;
std::vector<NPUBufferRef> v_caches;
bool valid;
};

class NPUPrefill {
public:
virtual ~NPUPrefill() = default;
virtual bool load(const std::string& model_path) = 0;
virtual bool is_available() const = 0;
virtual int get_chunk_size() const = 0;
virtual int get_hidden_dim() const = 0;
virtual int get_num_layers() const = 0;
virtual int get_num_kv_heads() const = 0;
virtual int get_head_dim() const = 0;
virtual NPUPrefillDirectResult prefill_chunk_direct(
const std::vector<__fp16>& embeddings,
int position_offset = 0,
const std::string& input_name = "x") = 0;
};

std::unique_ptr<NPUPrefill> create_prefill();

} // namespace npu
namespace engine {

Expand Down Expand Up @@ -638,13 +608,17 @@ class Model {
void set_cache_window(size_t window_size, size_t sink_size = 4);
size_t get_cache_size() const { return cache_total_seq_len_; }

bool load_npu_prefill(const std::string& model_path);
bool has_npu_prefill() const { return false; }
size_t get_prefill_chunk_size() const { return 128; }
double last_prefill_cache_copy_ms() const { return last_prefill_cache_copy_ms_; }
size_t last_prefill_padding_tokens() const { return last_prefill_padding_tokens_; }
size_t last_prefill_scalar_tail_tokens() const { return last_prefill_scalar_tail_tokens_; }

bool load_npu_audio_encoder(const std::string& model_path);
bool has_npu_audio_encoder() const { return npu_audio_encoder_ != nullptr; }

bool load_npu_vision_encoder(const std::string& model_path);
bool has_npu_vision_encoder() const { return npu_vision_encoder_ != nullptr; }

void remove_thinking_tokens(const std::vector<std::pair<size_t, size_t>>& ranges);
void compact_kv_cache() {}

Expand Down Expand Up @@ -750,6 +724,14 @@ class Model {
Component* lm_encoder_media_chunk_ = nullptr;

std::string family_;
std::string npu_audio_encoder_mlpackage_;
std::string npu_vision_encoder_mlpackage_;

std::unique_ptr<npu::NPUEncoder> npu_audio_encoder_;
std::unique_ptr<npu::NPUEncoder> npu_vision_encoder_;

bool audio_encode_via_npu(const std::vector<float>& audio_features);
bool vision_encode_via_npu(const std::vector<float>& pixel_values);

std::map<std::string, std::vector<uint8_t>> media_features_;
std::map<std::string, std::vector<size_t>> media_feature_shapes_;
Expand Down
105 changes: 105 additions & 0 deletions cactus-engine/src/gpu/gpu_model.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/* GPUModel — the GPU-resident text decoder.
*
* Parallel to `cactus::engine::Model` (the CPU graph executor) and
* `cactus::npu::ANEEncoder` (the ANE encoder dispatcher). This class is
* the third backend: full GPU prefill + decode for the text decoder.
*
* Lifecycle:
* 1. Load: opens the bundle's `gpu_plan.json`, mmaps weights.bin /
* scales.bin / embedding.bin, wraps them as MTLBuffers (shared mode,
* zero copy). Allocates KV cache as PRIVATE MTLBuffers (one per
* layer × direction). Builds Metal pipelines per the plan.
* 2. Prefill: encode N tokens worth of work into a command buffer,
* commit, kick off the next chunk before waiting.
* 3. Decode: per-token MTLCommandBuffer, pipelined ~4 deep. Sampling
* happens on GPU; only the sampled token id crosses to CPU.
*
* All weights stay GPU-resident. KV cache never copies back. Sampling on GPU.
* That's the entire perf story; everything else is plumbing. */
#pragma once

#include <cstddef>
#include <cstdint>
#include <memory>
#include <string>
#include <vector>

#if defined(__APPLE__)
#define CACTUS_HAS_GPU 1
#else
#define CACTUS_HAS_GPU 0
#endif

namespace cactus {
namespace gpu {
struct Context;
struct Buffer;
struct Pipeline;
struct CommandBuffer;
} // namespace gpu

namespace engine {

#if CACTUS_HAS_GPU

class GPUModel {
public:
GPUModel();
~GPUModel();

GPUModel(const GPUModel&) = delete;
GPUModel& operator=(const GPUModel&) = delete;

// Load a GPU bundle from disk. `bundle_dir` is the bundle root (the
// dir containing components/manifest.json). Reads
// `components/gpu/gpu_plan.json` and the three `*.bin` files.
bool load(const std::string& bundle_dir, size_t max_context_size);
bool is_loaded() const;

// Run prefill on N input tokens, building up the KV cache. Returns
// the sampled first response token (top of distribution).
// Internally pipelines command buffers so the GPU stays busy.
uint32_t prefill_and_sample(const std::vector<uint32_t>& tokens,
float temperature,
float top_p,
size_t top_k);

// Generate one token. Reads/writes KV cache in place. The previously
// sampled token id (or initial position) is passed via input_token.
uint32_t decode_one(uint32_t input_token,
float temperature,
float top_p,
size_t top_k);

// Reset the KV cache to length zero. Cheap (no buffer deallocation).
void reset_cache();

// Plumbing for telemetry / health.
size_t kv_cache_length() const;
size_t kv_cache_capacity() const;

private:
struct Impl;
std::unique_ptr<Impl> impl_;
};

#else // CACTUS_HAS_GPU

// Non-Apple stub: every call fails / returns 0.
class GPUModel {
public:
GPUModel() = default;
~GPUModel() = default;
bool load(const std::string&, size_t) { return false; }
bool is_loaded() const { return false; }
uint32_t prefill_and_sample(const std::vector<uint32_t>&, float, float, size_t) { return 0; }
uint32_t decode_one(uint32_t, float, float, size_t) { return 0; }
void reset_cache() {}
size_t kv_cache_length() const { return 0; }
size_t kv_cache_capacity() const { return 0; }
};

#endif // CACTUS_HAS_GPU

} // namespace engine
} // namespace cactus
Loading
Loading