Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,19 @@ tests/assets/test_wav_info.txt
vision_activations.json
gemma-4-eap-extras/
.githooks/
experiments/matrix/.build/
experiments/matrix/.matrix_*.yaml
experiments/matrix/.tmp_matrix_*.yaml
experiments/matrix/logs/
experiments/matrix/results/
experiments/matrix/parakeet_litert_export/out*/
experiments/matrix/parakeet_litert_export/cpu_compare_*.json

tests/android/build*/
tests/android/__pycache__/
tests/android/onnxruntime_llm_bench
tests/build*/
tests/assets/

third_party/
ruy
20 changes: 20 additions & 0 deletions android/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,23 @@ add_custom_target(cactus_static ALL
)

set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

add_executable(cactus_llm_bench
${PROJECT_ROOT}/tests/android/cactus_llm_bench.cpp
)
target_include_directories(cactus_llm_bench PRIVATE ${PROJECT_ROOT}/cactus ${CACTUS_ENGINE_DIR} ${CACTUS_ENGINE_DIR}/src)
target_compile_definitions(cactus_llm_bench PRIVATE PLATFORM_CPU_ONLY=1)
target_compile_options(cactus_llm_bench PRIVATE -O3 -fvisibility=hidden -fvisibility-inlines-hidden)
target_link_libraries(cactus_llm_bench PRIVATE cactus_engine cactus_graph cactus_kernels ${LOG_LIB} android)
if(EXISTS "${CACTUS_MBEDTLS_LIBRARY}" AND EXISTS "${CACTUS_MBEDX509_LIBRARY}" AND EXISTS "${CACTUS_MBEDCRYPTO_LIBRARY}")
target_link_libraries(cactus_llm_bench PRIVATE
"${CACTUS_MBEDTLS_LIBRARY}"
"${CACTUS_MBEDX509_LIBRARY}"
"${CACTUS_MBEDCRYPTO_LIBRARY}"
)
endif()
target_link_options(cactus_llm_bench PRIVATE -Wl,-z,max-page-size=16384)
set_target_properties(cactus_llm_bench PROPERTIES
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
)
9 changes: 8 additions & 1 deletion android/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,16 @@ CACTUS_CURL_ROOT=${CACTUS_CURL_ROOT:-"$PROJECT_ROOT/cactus-engine/libs/curl"}
if [ -z "$ANDROID_NDK_HOME" ]; then
if [ -n "$ANDROID_HOME" ]; then
ANDROID_NDK_HOME=$(ls -d "$ANDROID_HOME/ndk/"* 2>/dev/null | sort -V | tail -1)
elif [ -d "$HOME/Library/Android/sdk" ]; then
fi
if [ -z "$ANDROID_NDK_HOME" ] && [ -d "$HOME/Library/Android/sdk" ]; then
ANDROID_NDK_HOME=$(ls -d "$HOME/Library/Android/sdk/ndk/"* 2>/dev/null | sort -V | tail -1)
fi
if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/Caskroom/android-ndk" ]; then
ANDROID_NDK_HOME=$(ls -d /opt/homebrew/Caskroom/android-ndk/*/AndroidNDK*.app/Contents/NDK 2>/dev/null | sort -V | tail -1)
fi
if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/share/android-ndk" ]; then
ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk"
fi
fi

if [ -z "$ANDROID_NDK_HOME" ] || [ ! -d "$ANDROID_NDK_HOME" ]; then
Expand Down
9 changes: 9 additions & 0 deletions cactus-engine/cactus_engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,15 @@ CACTUS_FFI_EXPORT int cactus_score_window(
size_t buffer_size
);

CACTUS_FFI_EXPORT int cactus_benchmark_tokens(
cactus_model_t model,
const uint32_t* prompt_tokens,
size_t prompt_token_len,
size_t decode_token_len,
char* response_buffer,
size_t buffer_size
);

CACTUS_FFI_EXPORT int cactus_transcribe(
cactus_model_t model,
const char* audio_file_path, // NULL if using pcm_buffer
Expand Down
80 changes: 80 additions & 0 deletions cactus-engine/src/complete.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "telemetry.h"
#include "cactus_kernels.h"
#include "wav.h"
#include <algorithm>
#include <chrono>
#include <cstdint>
#include <cstring>
Expand Down Expand Up @@ -1127,4 +1128,83 @@ int cactus_score_window(
}
}

int cactus_benchmark_tokens(
cactus_model_t model,
const uint32_t* prompt_tokens,
size_t prompt_token_len,
size_t decode_token_len,
char* response_buffer,
size_t buffer_size
) {
if (!model || !prompt_tokens || prompt_token_len == 0 || !response_buffer || buffer_size == 0) {
handle_error_response("Invalid parameters", response_buffer, buffer_size);
return -1;
}

try {
auto* handle = static_cast<CactusModelHandle*>(model);
std::vector<uint32_t> prompt(prompt_tokens, prompt_tokens + prompt_token_len);

auto start_time = std::chrono::high_resolution_clock::now();
double peak_ram_usage_mb = get_ram_usage_mb();
auto sample_peak_ram = [&]() {
peak_ram_usage_mb = std::max(peak_ram_usage_mb, get_ram_usage_mb());
};
handle->model->reset_cache();
if (decode_token_len == 0) {
handle->model->prefill(prompt, handle->model->get_prefill_chunk_size(), "", false);
} else if (prompt.size() > 1) {
handle->model->prefill(
std::vector<uint32_t>(prompt.begin(), prompt.end() - 1),
handle->model->get_prefill_chunk_size()
);
}
sample_peak_ram();
uint32_t next_token = decode_token_len > 0 ? handle->model->decode({prompt.back()}, 0.0f, 1.0f, 1) : 0;
sample_peak_ram();
auto first_token_time = std::chrono::high_resolution_clock::now();

size_t generated = decode_token_len > 0 ? 1 : 0;
while (generated < decode_token_len) {
next_token = handle->model->decode({next_token}, 0.0f, 1.0f, 1);
sample_peak_ram();
++generated;
}

auto end_time = std::chrono::high_resolution_clock::now();
double ttft_ms = std::chrono::duration_cast<std::chrono::microseconds>(first_token_time - start_time).count() / 1000.0;
double total_ms = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count() / 1000.0;
double decode_ms = std::max(0.0, total_ms - ttft_ms);
double prefill_tps = ttft_ms > 0.0 ? (static_cast<double>(prompt_token_len) * 1000.0) / ttft_ms : 0.0;
double decode_tps = (generated > 1 && decode_ms > 0.0) ? (static_cast<double>(generated - 1) * 1000.0) / decode_ms : 0.0;

std::ostringstream oss;
oss << "{"
<< "\"success\":true,"
<< "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << ttft_ms << ","
<< "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_ms << ","
<< "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","
<< "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","
<< "\"prompt_tokens\":" << prompt_token_len << ","
<< "\"completion_tokens\":" << generated << ","
<< "\"peak_ram_usage_mb\":" << std::fixed << std::setprecision(2) << peak_ram_usage_mb << ","
<< "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb()
<< "}";

std::string result = oss.str();
if (result.size() >= buffer_size) {
handle_error_response("Response buffer too small", response_buffer, buffer_size);
return -1;
}
std::strcpy(response_buffer, result.c_str());
return static_cast<int>(result.size());
} catch (const std::exception& e) {
handle_error_response(e.what(), response_buffer, buffer_size);
return -1;
} catch (...) {
handle_error_response("Unknown error during token benchmark", response_buffer, buffer_size);
return -1;
}
}

}
32 changes: 25 additions & 7 deletions cactus-engine/src/engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -580,7 +580,8 @@ class Model {
size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr,
float min_p = 0.15f, float repetition_penalty = 1.1f);

void prefill(const std::vector<uint32_t>& tokens, size_t chunk_size = 256, const std::string& profile_file = "");
void prefill(const std::vector<uint32_t>& tokens, size_t chunk_size = 128, const std::string& profile_file = "",
bool prepare_decode = true);

void prefill_with_images(const std::vector<uint32_t>& tokens, const std::vector<std::string>& image_paths,
const std::string& profile_file = "");
Expand Down Expand Up @@ -629,7 +630,7 @@ class Model {

bool load_npu_prefill(const std::string& model_path);
bool has_npu_prefill() const { return false; }
size_t get_prefill_chunk_size() const { return 256; }
size_t get_prefill_chunk_size() const { return 128; }

void remove_thinking_tokens(const std::vector<std::pair<size_t, size_t>>& ranges);
void compact_kv_cache() {}
Expand All @@ -649,7 +650,7 @@ class Model {
std::string path;
};

struct CacheStateEntry {
struct CacheStateBinding {
std::string layer_key;
int key_node_id = -1;
int value_node_id = -1;
Expand All @@ -663,22 +664,33 @@ class Model {
std::vector<int> output_node_ids;
std::vector<std::string> logical_outputs;
std::vector<Binding> bindings;
std::vector<CacheStateBinding> cache_states;
std::unique_ptr<CactusGraph> graph;
std::vector<std::vector<uint8_t>> input_buffers;
std::vector<CacheStateEntry> cache_states;
};

void copy_cache_state(const Component& src, Component& dst);

bool load_manifest();
bool setup_tokenizer();
bool load_components();
bool load_components(const std::unordered_set<std::string>& required_components);
bool bind_runtime_buffers(Component& comp);
void run_step(uint32_t token_id, size_t position, bool read_logits);
void run_encoder_step(uint32_t token_id, size_t position);
void run_media_step(size_t position, const uint8_t* feature_row, size_t feature_row_bytes,
Precision feature_precision);
void copy_component_outputs_to_inputs(const Component& source, Component& target);
void copy_encoder_outputs_to_decoder(const Component& enc);
void copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index);
void copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset);
bool cache_states_compatible(const Component& source, const Component& target) const;
void copy_cache_states(const Component& source, Component& target);
void reset_component_cache_states(Component& comp);
size_t component_chunk_tokens(const Component& comp, const std::string& input_name) const;
size_t component_output_tokens(const Component& comp, const std::string& output_name) const;
size_t run_chunked_prefill(const std::vector<uint32_t>& tokens, size_t start_position, size_t chunk_size,
bool prepare_decode);
void run_full_context_text();
void write_int_input(Component& comp, const std::string& name, int64_t value);
void write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value);
void write_bytes_input(Component& comp, const std::string& name, const void* data, size_t byte_size);
int input_index(const Component& comp, const std::string& name) const;
int output_index(const Component& comp, const std::string& name) const;
Expand All @@ -693,6 +705,10 @@ class Model {
std::map<std::string, Component> components_;
Component* encoder_ = nullptr;
Component* decoder_ = nullptr;
Component* decoder_prefill_ = nullptr;
Component* prefill_encoder_ = nullptr;
enum class DecodeRoute { CACHED_STEP, DIRECT_DECODER_STEP, FULL_CONTEXT_TEXT };
DecodeRoute decode_route_ = DecodeRoute::CACHED_STEP;
Component* vision_encoder_ = nullptr;
Component* audio_encoder_ = nullptr;
Component* lm_encoder_media_step_ = nullptr;
Expand All @@ -710,6 +726,8 @@ class Model {
bool initialized_ = false;
size_t cache_total_seq_len_ = 0;
size_t cache_max_seq_len_ = 4096;
size_t last_logit_position_ = 0;
std::vector<uint32_t> context_tokens_;

static constexpr size_t MAX_TOKEN_HISTORY = 128;
std::vector<uint32_t> token_history_;
Expand Down
Loading