From 7399b642fa332ee302491708e536735f3a05fac5 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Tue, 19 May 2026 11:31:07 -0700 Subject: [PATCH 01/12] Fix transpiled LLM runtime and optimize chunked prefill Signed-off-by: Noah Cylich --- .gitignore | 13 +- android/build.sh | 9 +- cactus-engine/cactus_engine.h | 9 + cactus-engine/src/complete.cpp | 80 ++++ cactus-engine/src/engine.h | 33 +- cactus-engine/src/model.cpp | 539 +++++++++++++++++++++- cactus-engine/tests/chat.cpp | 76 ++- cactus-graph/src/ops_cache.cpp | 32 +- cactus-kernels/src/threading.h | 9 +- cactus/build.sh | 11 +- python/cactus/cli/compile.py | 8 + python/cactus/cli/run.py | 161 +------ python/cactus/cli/transpile.py | 23 +- python/cactus/transpile/hf_model.py | 17 +- python/cactus/transpile/lower.py | 89 ++++ python/cactus/transpile/model_adapters.py | 344 +++++++++++--- python/cactus/transpile/model_profiles.py | 19 +- 17 files changed, 1216 insertions(+), 256 deletions(-) diff --git a/.gitignore b/.gitignore index 585b41e21..70c95bcef 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,15 @@ tests/assets/test_wav_info.txt vision_activations.json gemma-4-eap-extras/ .githooks/ -*.cg \ No newline at end of file +*.cg + +experiments/matrix/.build/ +experiments/matrix/.matrix_*.yaml +experiments/matrix/logs/ +experiments/matrix/results/ + +tests/android/build*/ +tests/android/__pycache__/ +tests/assets/ + +third_party/ diff --git a/android/build.sh b/android/build.sh index 9d6f2d1e2..370a3fd83 100755 --- a/android/build.sh +++ b/android/build.sh @@ -12,9 +12,16 @@ CACTUS_CURL_ROOT=${CACTUS_CURL_ROOT:-"$PROJECT_ROOT/cactus-engine/libs/curl"} if [ -z "$ANDROID_NDK_HOME" ]; then if [ -n "$ANDROID_HOME" ]; then ANDROID_NDK_HOME=$(ls -d "$ANDROID_HOME/ndk/"* 2>/dev/null | sort -V | tail -1) - elif [ -d "$HOME/Library/Android/sdk" ]; then + fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "$HOME/Library/Android/sdk" ]; then ANDROID_NDK_HOME=$(ls -d "$HOME/Library/Android/sdk/ndk/"* 2>/dev/null | sort -V | tail -1) fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/Caskroom/android-ndk" ]; then + ANDROID_NDK_HOME=$(ls -d /opt/homebrew/Caskroom/android-ndk/*/AndroidNDK*.app/Contents/NDK 2>/dev/null | sort -V | tail -1) + fi + if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/share/android-ndk" ]; then + ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" + fi fi if [ -z "$ANDROID_NDK_HOME" ] || [ ! -d "$ANDROID_NDK_HOME" ]; then diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index ce65fbf08..cb575ec21 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -74,6 +74,15 @@ CACTUS_FFI_EXPORT int cactus_score_window( size_t buffer_size ); +CACTUS_FFI_EXPORT int cactus_benchmark_tokens( + cactus_model_t model, + const uint32_t* prompt_tokens, + size_t prompt_token_len, + size_t decode_token_len, + char* response_buffer, + size_t buffer_size +); + CACTUS_FFI_EXPORT int cactus_transcribe( cactus_model_t model, const char* audio_file_path, // NULL if using pcm_buffer diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index a9ab196b5..f2c74c854 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -4,6 +4,7 @@ #include "telemetry.h" #include "cactus_kernels.h" #include "wav.h" +#include #include #include #include @@ -1116,4 +1117,83 @@ int cactus_score_window( } } +int cactus_benchmark_tokens( + cactus_model_t model, + const uint32_t* prompt_tokens, + size_t prompt_token_len, + size_t decode_token_len, + char* response_buffer, + size_t buffer_size +) { + if (!model || !prompt_tokens || prompt_token_len == 0 || !response_buffer || buffer_size == 0) { + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + + try { + auto* handle = static_cast(model); + std::vector prompt(prompt_tokens, prompt_tokens + prompt_token_len); + + auto start_time = std::chrono::high_resolution_clock::now(); + double peak_ram_usage_mb = get_ram_usage_mb(); + auto sample_peak_ram = [&]() { + peak_ram_usage_mb = std::max(peak_ram_usage_mb, get_ram_usage_mb()); + }; + handle->model->reset_cache(); + if (decode_token_len == 0) { + handle->model->prefill(prompt, handle->model->get_prefill_chunk_size(), "", false); + } else if (prompt.size() > 1) { + handle->model->prefill( + std::vector(prompt.begin(), prompt.end() - 1), + handle->model->get_prefill_chunk_size() + ); + } + sample_peak_ram(); + uint32_t next_token = decode_token_len > 0 ? handle->model->decode({prompt.back()}, 0.0f, 1.0f, 1) : 0; + sample_peak_ram(); + auto first_token_time = std::chrono::high_resolution_clock::now(); + + size_t generated = decode_token_len > 0 ? 1 : 0; + while (generated < decode_token_len) { + next_token = handle->model->decode({next_token}, 0.0f, 1.0f, 1); + sample_peak_ram(); + ++generated; + } + + auto end_time = std::chrono::high_resolution_clock::now(); + double ttft_ms = std::chrono::duration_cast(first_token_time - start_time).count() / 1000.0; + double total_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; + double decode_ms = std::max(0.0, total_ms - ttft_ms); + double prefill_tps = ttft_ms > 0.0 ? (static_cast(prompt_token_len) * 1000.0) / ttft_ms : 0.0; + double decode_tps = (generated > 1 && decode_ms > 0.0) ? (static_cast(generated - 1) * 1000.0) / decode_ms : 0.0; + + std::ostringstream oss; + oss << "{" + << "\"success\":true," + << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << ttft_ms << "," + << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_ms << "," + << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << "," + << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << "," + << "\"prompt_tokens\":" << prompt_token_len << "," + << "\"completion_tokens\":" << generated << "," + << "\"peak_ram_usage_mb\":" << std::fixed << std::setprecision(2) << peak_ram_usage_mb << "," + << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() + << "}"; + + std::string result = oss.str(); + if (result.size() >= buffer_size) { + handle_error_response("Response buffer too small", response_buffer, buffer_size); + return -1; + } + std::strcpy(response_buffer, result.c_str()); + return static_cast(result.size()); + } catch (const std::exception& e) { + handle_error_response(e.what(), response_buffer, buffer_size); + return -1; + } catch (...) { + handle_error_response("Unknown error during token benchmark", response_buffer, buffer_size); + return -1; + } +} + } diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 2c60b3dce..4d44308e6 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -576,7 +576,8 @@ class Model { size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + void prefill(const std::vector& tokens, size_t chunk_size = 128, const std::string& profile_file = "", + bool prepare_decode = true); void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, const std::string& profile_file = ""); @@ -615,7 +616,7 @@ class Model { bool load_npu_prefill(const std::string& model_path); bool has_npu_prefill() const { return false; } - size_t get_prefill_chunk_size() const { return 256; } + size_t get_prefill_chunk_size() const { return 128; } void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} @@ -635,6 +636,12 @@ class Model { std::string path; }; + struct CacheStateBinding { + std::string layer_key; + int key_node_id = -1; + int value_node_id = -1; + }; + struct Component { std::string name; std::string graph_path; @@ -643,16 +650,30 @@ class Model { std::vector output_node_ids; std::vector logical_outputs; std::vector bindings; + std::vector cache_states; std::unique_ptr graph; std::vector> input_buffers; }; bool load_manifest(); bool setup_tokenizer(); - bool load_components(); + bool load_components(const std::unordered_set& required_components); bool bind_runtime_buffers(Component& comp); void run_step(uint32_t token_id, size_t position, bool read_logits); + void run_encoder_step(uint32_t token_id, size_t position); + void copy_component_outputs_to_inputs(const Component& source, Component& target); + void copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index); + void copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset); + bool cache_states_compatible(const Component& source, const Component& target) const; + void copy_cache_states(const Component& source, Component& target); + void reset_component_cache_states(Component& comp); + size_t component_chunk_tokens(const Component& comp, const std::string& input_name) const; + size_t component_output_tokens(const Component& comp, const std::string& output_name) const; + size_t run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, + bool prepare_decode); + void run_full_context_text(); void write_int_input(Component& comp, const std::string& name, int64_t value); + void write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value); int input_index(const Component& comp, const std::string& name) const; uint32_t argmax_last_logits(); @@ -660,12 +681,18 @@ class Model { std::map components_; Component* encoder_ = nullptr; Component* decoder_ = nullptr; + Component* decoder_prefill_ = nullptr; + Component* prefill_encoder_ = nullptr; + enum class DecodeRoute { CACHED_STEP, DIRECT_DECODER_STEP, FULL_CONTEXT_TEXT }; + DecodeRoute decode_route_ = DecodeRoute::CACHED_STEP; Config config_; std::unique_ptr tokenizer_; bool initialized_ = false; size_t cache_total_seq_len_ = 0; size_t cache_max_seq_len_ = 4096; + size_t last_logit_position_ = 0; + std::vector context_tokens_; static constexpr size_t MAX_TOKEN_HISTORY = 128; std::vector token_history_; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 45b784ce7..6c85157b8 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -1,5 +1,6 @@ #include "engine.h" #include "cactus_graph.h" +#include "cactus_kernels.h" #define PICOJSON_USE_INT64 #include "picojson.h" @@ -17,10 +18,70 @@ #include #include #include +#include namespace cactus { namespace engine { +float read_scalar_value(Precision precision, const uint8_t* data, size_t index) { + const uint8_t* ptr = data + PrecisionTraits::byte_offset_of(precision, index); + switch (precision) { + case Precision::FP32: + return *reinterpret_cast(ptr); + case Precision::FP16: + return static_cast(*reinterpret_cast(ptr)); + case Precision::INT8: + return static_cast(*reinterpret_cast(ptr)); + default: + return 0.0f; + } +} + +void write_scalar_value(Precision precision, uint8_t* data, size_t index, float value) { + uint8_t* ptr = data + PrecisionTraits::byte_offset_of(precision, index); + switch (precision) { + case Precision::FP32: + *reinterpret_cast(ptr) = value; + break; + case Precision::FP16: + *reinterpret_cast<__fp16*>(ptr) = static_cast<__fp16>(value); + break; + case Precision::INT8: + *reinterpret_cast(ptr) = static_cast(value); + break; + default: + break; + } +} + +bool copy_component_tensor(CactusGraph& source_graph, + const BufferDesc& src_desc, + size_t src_node, + const BufferDesc& dst_desc, + std::vector& dst_buffer, + size_t dst_element_offset, + size_t element_count, + const std::string& name) { + const auto* src_ptr = static_cast(source_graph.get_output(src_node)); + if (src_desc.precision == dst_desc.precision) { + size_t dst_offset = PrecisionTraits::byte_offset_of(dst_desc.precision, dst_element_offset); + std::memcpy( + dst_buffer.data() + dst_offset, + src_ptr, + PrecisionTraits::packed_size_of(src_desc.precision, element_count)); + return true; + } + if (name != "position_ids" && name != "attention_mask") return false; + for (size_t i = 0; i < element_count; ++i) { + write_scalar_value( + dst_desc.precision, + dst_buffer.data(), + dst_element_offset + i, + read_scalar_value(src_desc.precision, src_ptr, i)); + } + return true; +} + void ConvCache::init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision) { num_layers = layers; hidden_size = hidden_dim; @@ -142,16 +203,62 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_ERROR("model", "Tokenizer init failed for bundle: " << bundle_dir); return false; } - if (!load_components()) return false; - - encoder_ = components_.count("lm_encoder_step") ? &components_.at("lm_encoder_step") : nullptr; - decoder_ = components_.count("decoder_step") ? &components_.at("decoder_step") : nullptr; - if (!encoder_ || !decoder_) { - CACTUS_LOG_ERROR("model", "Bundle missing lm_encoder_step or decoder_step components"); + std::string encoder_name; + std::string decoder_name; + std::unordered_set required_components; + bool has_chunked_prefill = components_.count("lm_encoder_step") + && components_.count("decoder_media_step") + && components_.count("lm_encoder_text_chunk") + && components_.count("decoder_prefill_chunk"); + if (has_chunked_prefill) { + encoder_name = "lm_encoder_step"; + decoder_name = "decoder_media_step"; + decode_route_ = DecodeRoute::CACHED_STEP; + required_components = { + encoder_name, + decoder_name, + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + }; + } else if (components_.count("decoder_step") + && input_index(components_.at("decoder_step"), "input_ids") >= 0 + && input_index(components_.at("decoder_step"), "position_ids") >= 0) { + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::DIRECT_DECODER_STEP; + required_components = {decoder_name}; + } else if (components_.count("lm_encoder_step") && components_.count("decoder_step")) { + encoder_name = "lm_encoder_step"; + decoder_name = "decoder_step"; + decode_route_ = DecodeRoute::CACHED_STEP; + required_components = {encoder_name, decoder_name}; + if (components_.count("decoder_prefill_chunk")) { + required_components.insert("decoder_prefill_chunk"); + } + if (components_.count("lm_encoder_text_chunk")) { + required_components.insert("lm_encoder_text_chunk"); + } + } else if (components_.count("text_lm_encoder") && components_.count("decoder")) { + encoder_name = "text_lm_encoder"; + decoder_name = "decoder"; + decode_route_ = DecodeRoute::FULL_CONTEXT_TEXT; + required_components = {encoder_name, decoder_name}; + } else { + CACTUS_LOG_ERROR("model", "Bundle missing lm_encoder_step+decoder_step or text_lm_encoder+decoder components"); return false; } - if (!bind_runtime_buffers(*encoder_)) return false; + if (!load_components(required_components)) return false; + if (!encoder_name.empty()) encoder_ = &components_.at(encoder_name); + decoder_ = &components_.at(decoder_name); + if (components_.count("decoder_prefill_chunk") && components_.at("decoder_prefill_chunk").graph) { + decoder_prefill_ = &components_.at("decoder_prefill_chunk"); + } + if (components_.count("lm_encoder_text_chunk") && components_.at("lm_encoder_text_chunk").graph) { + prefill_encoder_ = &components_.at("lm_encoder_text_chunk"); + } + if (encoder_ && !bind_runtime_buffers(*encoder_)) return false; + if (prefill_encoder_ && !bind_runtime_buffers(*prefill_encoder_)) return false; if (!bind_runtime_buffers(*decoder_)) return false; + if (decoder_prefill_ && !bind_runtime_buffers(*decoder_prefill_)) return false; cache_max_seq_len_ = context_size; initialized_ = true; @@ -199,6 +306,16 @@ bool Model::load_manifest() { comp.bindings.push_back(std::move(bd)); } } + if (c.count("cache_state_node_ids")) { + for (const auto& sv : c.at("cache_state_node_ids").get()) { + const auto& s = sv.get(); + CacheStateBinding cs; + if (s.count("layer_key")) cs.layer_key = s.at("layer_key").get(); + if (s.count("key")) cs.key_node_id = static_cast(s.at("key").get()); + if (s.count("value")) cs.value_node_id = static_cast(s.at("value").get()); + comp.cache_states.push_back(std::move(cs)); + } + } components_[comp.name] = std::move(comp); } return true; @@ -218,8 +335,9 @@ bool Model::setup_tokenizer() { return tokenizer_->load_vocabulary_with_config(vocab, merges, cfg); } -bool Model::load_components() { +bool Model::load_components(const std::unordered_set& required_components) { for (auto& [name, comp] : components_) { + if (!required_components.empty() && !required_components.count(name)) continue; if (comp.graph_path.empty()) continue; fs::path full = fs::path(bundle_dir_) / comp.graph_path; try { @@ -231,8 +349,14 @@ bool Model::load_components() { for (const auto& b : comp.bindings) { if (b.node_id < 0) continue; try { - comp.graph->bind_mmap_weights(static_cast(b.node_id), - (fs::path(bundle_dir_) / b.path).string()); + fs::path weight_path(b.path); + if (weight_path.is_absolute()) { + fs::path local = fs::path(bundle_dir_) / weight_path.filename(); + if (fs::exists(local)) weight_path = local; + } else { + weight_path = fs::path(bundle_dir_) / weight_path; + } + comp.graph->bind_mmap_weights(static_cast(b.node_id), weight_path.string()); } catch (const std::exception& e) { CACTUS_LOG_ERROR("model", "bind " << b.path << ": " << e.what()); return false; @@ -261,31 +385,385 @@ int Model::input_index(const Component& comp, const std::string& name) const { } void Model::write_int_input(Component& comp, const std::string& name, int64_t value) { + write_int_input_at(comp, name, 0, value); +} + +void Model::write_int_input_at(Component& comp, const std::string& name, size_t index, int64_t value) { int idx = input_index(comp, name); if (idx < 0) return; size_t node_id = static_cast(comp.runtime_input_node_ids[idx]); const auto& desc = comp.graph->get_output_buffer(node_id); auto& buf = comp.input_buffers[idx]; + if (index >= desc.total_size) return; + size_t offset = PrecisionTraits::byte_offset_of(desc.precision, index); + auto* dst = buf.data() + offset; switch (desc.precision) { case Precision::FP32: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; case Precision::FP16: - *reinterpret_cast<__fp16*>(buf.data()) = static_cast<__fp16>(value); + *reinterpret_cast<__fp16*>(dst) = static_cast<__fp16>(value); break; case Precision::INT8: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; default: - *reinterpret_cast(buf.data()) = static_cast(value); + *reinterpret_cast(dst) = static_cast(value); break; } } void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { + if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { + write_int_input(*decoder_, "input_ids", static_cast(token_id)); + write_int_input(*decoder_, "position_ids", static_cast(position)); + decoder_->graph->execute(); + return; + } + run_encoder_step(token_id, position); + copy_component_outputs_to_inputs(*encoder_, *decoder_); + decoder_->graph->execute(); +} + +void Model::run_encoder_step(uint32_t token_id, size_t position) { write_int_input(*encoder_, "input_ids", static_cast(token_id)); write_int_input(*encoder_, "position_ids", static_cast(position)); encoder_->graph->execute(); +} + +void Model::copy_component_outputs_to_inputs(const Component& source, Component& target) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + std::fill(target.input_buffers[dst_idx].begin(), target.input_buffers[dst_idx].end(), 0); + size_t elements = std::min(src_desc.total_size, dst_desc.total_size); + if (!copy_component_tensor(*source.graph, src_desc, src_node, dst_desc, target.input_buffers[dst_idx], 0, elements, out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +void Model::copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + size_t chunk_tokens = component_chunk_tokens(target, out_name); + if (chunk_tokens <= token_index || chunk_tokens == 0) { + throw std::runtime_error("chunk prefill token index exceeds input capacity for " + out_name); + } + if (dst_desc.total_size % chunk_tokens != 0) { + throw std::runtime_error("chunk prefill input shape is not token-aligned for " + out_name); + } + size_t elements_per_token = dst_desc.total_size / chunk_tokens; + if (src_desc.total_size != elements_per_token) { + throw std::runtime_error("component output/input token shape mismatch for " + out_name); + } + if (!copy_component_tensor( + *source.graph, + src_desc, + src_node, + dst_desc, + target.input_buffers[dst_idx], + token_index * elements_per_token, + src_desc.total_size, + out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +void Model::copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset) { + for (size_t i = 0; i < source.output_node_ids.size() && i < source.logical_outputs.size(); ++i) { + const std::string& out_name = source.logical_outputs[i]; + int dst_idx = input_index(target, out_name); + if (dst_idx < 0) continue; + size_t src_node = static_cast(source.output_node_ids[i]); + const auto& src_desc = source.graph->get_output_buffer(src_node); + size_t dst_node = static_cast(target.runtime_input_node_ids[dst_idx]); + const auto& dst_desc = target.graph->get_output_buffer(dst_node); + size_t src_tokens = component_output_tokens(source, out_name); + size_t dst_tokens = component_chunk_tokens(target, out_name); + if (src_tokens == 0 || dst_tokens == 0 || token_offset + src_tokens > dst_tokens) { + throw std::runtime_error("chunk prefill output range exceeds input capacity for " + out_name); + } + if (src_desc.total_size % src_tokens != 0 || dst_desc.total_size % dst_tokens != 0) { + throw std::runtime_error("chunk prefill output/input shape is not token-aligned for " + out_name); + } + size_t src_elements_per_token = src_desc.total_size / src_tokens; + size_t dst_elements_per_token = dst_desc.total_size / dst_tokens; + if (src_elements_per_token != dst_elements_per_token) { + throw std::runtime_error("component output/input token shape mismatch for " + out_name); + } + if (!copy_component_tensor( + *source.graph, + src_desc, + src_node, + dst_desc, + target.input_buffers[dst_idx], + token_offset * dst_elements_per_token, + src_desc.total_size, + out_name)) { + throw std::runtime_error("component output/input precision mismatch for " + out_name); + } + } +} + +bool Model::cache_states_compatible(const Component& source, const Component& target) const { + if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) return false; + for (size_t i = 0; i < source.cache_states.size(); ++i) { + const auto& src = source.cache_states[i]; + const auto& dst = target.cache_states[i]; + if (src.layer_key != dst.layer_key) return false; + if (src.key_node_id < 0 || src.value_node_id < 0 || dst.key_node_id < 0 || dst.value_node_id < 0) return false; + } + return true; +} + +void Model::copy_cache_states(const Component& source, Component& target) { + if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) { + throw std::runtime_error("prefill and step cache states are not compatible"); + } + for (size_t i = 0; i < source.cache_states.size(); ++i) { + const auto& src = source.cache_states[i]; + const auto& dst = target.cache_states[i]; + if (src.layer_key != dst.layer_key) { + throw std::runtime_error("prefill and step cache layer mismatch: " + src.layer_key + " != " + dst.layer_key); + } + for (auto [src_node, dst_node] : {std::pair{src.key_node_id, dst.key_node_id}, std::pair{src.value_node_id, dst.value_node_id}}) { + const auto& src_desc = source.graph->get_output_buffer(static_cast(src_node)); + const auto& dst_desc = target.graph->get_output_buffer(static_cast(dst_node)); + if (src_desc.precision != dst_desc.precision) { + std::ostringstream oss; + oss << "prefill and step cache precision mismatch at layer " << src.layer_key + << ": " << static_cast(src_desc.precision) + << " vs " << static_cast(dst_desc.precision); + throw std::runtime_error(oss.str()); + } + void* src_ptr = source.graph->get_output(static_cast(src_node)); + void* dst_ptr = target.graph->get_output(static_cast(dst_node)); + if (src_desc.byte_size == dst_desc.byte_size) { + std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); + continue; + } + if (src_desc.precision == Precision::INT8) { + auto* src_meta = static_cast(src_ptr); + auto* dst_meta = static_cast(dst_ptr); + const size_t src_current = static_cast(src_meta[0]); + const size_t src_max = static_cast(src_meta[1]); + const size_t kv_heads = static_cast(src_meta[2]); + const size_t head_dim = static_cast(src_meta[3]); + const size_t sink = static_cast(src_meta[4]); + if (src_max == 0 || kv_heads == 0 || head_dim == 0) { + throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); + } + const size_t groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + const size_t int8_stride = kv_heads * head_dim; + const size_t scale_stride = kv_heads * groups; + const size_t row_bytes = int8_stride + scale_stride * sizeof(float); + const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; + if (dst_max == 0) { + throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + } + const size_t dst_current = std::min(src_current, dst_max); + dst_meta[0] = dst_current; + dst_meta[1] = dst_max; + dst_meta[2] = kv_heads; + dst_meta[3] = head_dim; + dst_meta[4] = std::min(sink, dst_current); + std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); + + const auto* src_i8 = static_cast(src_ptr) + 64; + const auto* src_scales = reinterpret_cast( + static_cast(src_ptr) + 64 + src_max * int8_stride); + auto* dst_i8 = static_cast(dst_ptr) + 64; + auto* dst_scales = reinterpret_cast( + static_cast(dst_ptr) + 64 + dst_max * int8_stride); + auto copy_rows = [&](size_t dst_row, size_t src_row, size_t rows) { + if (rows == 0) return; + std::memcpy( + dst_i8 + dst_row * int8_stride, + src_i8 + src_row * int8_stride, + rows * int8_stride); + std::memcpy( + dst_scales + dst_row * scale_stride, + src_scales + src_row * scale_stride, + rows * scale_stride * sizeof(float)); + }; + if (src_current <= dst_max) { + copy_rows(0, 0, src_current); + } else { + const size_t copied_sink = std::min(sink, dst_max); + const size_t tail_rows = dst_max - copied_sink; + copy_rows(0, 0, copied_sink); + if (tail_rows > 0) copy_rows(copied_sink, src_current - tail_rows, tail_rows); + } + continue; + } + if (PrecisionTraits::is_cq(src_desc.precision) || src_desc.byte_size < 64 || dst_desc.byte_size < 64) { + std::ostringstream oss; + oss << "prefill and step cache buffer mismatch at layer " << src.layer_key + << ": " << src_desc.byte_size << " bytes vs " << dst_desc.byte_size << " bytes"; + throw std::runtime_error(oss.str()); + } + + auto* src_meta = static_cast(src_ptr); + auto* dst_meta = static_cast(dst_ptr); + const size_t src_current = static_cast(src_meta[0]); + const size_t src_max = static_cast(src_meta[1]); + const size_t kv_heads = static_cast(src_meta[2]); + const size_t head_dim = static_cast(src_meta[3]); + const size_t sink = static_cast(src_meta[4]); + if (src_max == 0 || kv_heads == 0 || head_dim == 0) { + throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); + } + const size_t row_bytes = kv_heads * head_dim * PrecisionTraits::size_of(src_desc.precision); + const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; + if (dst_max == 0) { + throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + } + const size_t dst_current = std::min(src_current, dst_max); + dst_meta[0] = dst_current; + dst_meta[1] = dst_max; + dst_meta[2] = kv_heads; + dst_meta[3] = head_dim; + dst_meta[4] = std::min(sink, dst_current); + std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); + const auto* src_rows = static_cast(src_ptr) + 64; + auto* dst_rows = static_cast(dst_ptr) + 64; + if (src_current <= dst_max) { + std::memcpy(dst_rows, src_rows, src_current * row_bytes); + } else { + const size_t copied_sink = std::min(sink, dst_max); + const size_t tail_rows = dst_max - copied_sink; + if (copied_sink > 0) { + std::memcpy(dst_rows, src_rows, copied_sink * row_bytes); + } + if (tail_rows > 0) { + std::memcpy( + dst_rows + copied_sink * row_bytes, + src_rows + (src_current - tail_rows) * row_bytes, + tail_rows * row_bytes); + } + } + } + } +} + +void Model::reset_component_cache_states(Component& comp) { + for (const auto& state : comp.cache_states) { + for (int node_id : {state.key_node_id, state.value_node_id}) { + if (node_id < 0) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); + if (!desc.get_data()) continue; + void* ptr = comp.graph->get_output(static_cast(node_id)); + if (!ptr) continue; + auto* metadata = static_cast(ptr); + metadata[0] = 0; + metadata[1] = 0; + } + } +} + +size_t Model::component_chunk_tokens(const Component& comp, const std::string& input_name) const { + int idx = input_index(comp, input_name); + if (idx < 0) return 0; + const auto& desc = comp.graph->get_output_buffer(static_cast(comp.runtime_input_node_ids[idx])); + if (desc.shape.size() >= 2 && desc.shape[0] == 1) return desc.shape[1]; + return desc.shape.empty() ? 0 : desc.shape[0]; +} + +size_t Model::component_output_tokens(const Component& comp, const std::string& output_name) const { + for (size_t i = 0; i < comp.logical_outputs.size() && i < comp.output_node_ids.size(); ++i) { + if (comp.logical_outputs[i] != output_name) continue; + const auto& desc = comp.graph->get_output_buffer(static_cast(comp.output_node_ids[i])); + if (desc.shape.size() >= 2 && desc.shape[0] == 1) return desc.shape[1]; + return desc.shape.empty() ? 0 : desc.shape[0]; + } + return 0; +} + +size_t Model::run_chunked_prefill(const std::vector& tokens, size_t start_position, size_t chunk_size, bool prepare_decode) { + if (decode_route_ != DecodeRoute::CACHED_STEP || !encoder_ || !decoder_ || !decoder_prefill_) return 0; + if (start_position != 0) return 0; + if (!cache_states_compatible(*decoder_prefill_, *decoder_)) return 0; + size_t component_tokens = component_chunk_tokens(*decoder_prefill_, "inputs_embeds"); + if (component_tokens <= 1) return 0; + size_t effective_chunk = chunk_size > 0 ? std::min(chunk_size, component_tokens) : component_tokens; + if (effective_chunk != component_tokens) effective_chunk = component_tokens; + if (tokens.size() < effective_chunk) return 0; + + size_t encoder_chunk = 0; + if (prefill_encoder_ && input_index(*prefill_encoder_, "input_ids") >= 0 && input_index(*prefill_encoder_, "position_ids") >= 0) { + encoder_chunk = component_chunk_tokens(*prefill_encoder_, "input_ids"); + if (encoder_chunk == 0 || effective_chunk % encoder_chunk != 0) { + encoder_chunk = 0; + } + } + + size_t processed = 0; + while (processed + effective_chunk <= tokens.size()) { + for (size_t i = 0; i < decoder_prefill_->input_buffers.size(); ++i) { + std::fill(decoder_prefill_->input_buffers[i].begin(), decoder_prefill_->input_buffers[i].end(), 0); + } + if (encoder_chunk > 0) { + for (size_t chunk_offset = 0; chunk_offset < effective_chunk; chunk_offset += encoder_chunk) { + for (size_t i = 0; i < prefill_encoder_->input_buffers.size(); ++i) { + std::fill(prefill_encoder_->input_buffers[i].begin(), prefill_encoder_->input_buffers[i].end(), 0); + } + for (size_t i = 0; i < encoder_chunk; ++i) { + write_int_input_at(*prefill_encoder_, "input_ids", i, static_cast(tokens[processed + chunk_offset + i])); + write_int_input_at(*prefill_encoder_, "position_ids", i, static_cast(start_position + processed + chunk_offset + i)); + } + prefill_encoder_->graph->execute(); + copy_component_outputs_to_chunk_inputs_range(*prefill_encoder_, *decoder_prefill_, chunk_offset); + } + } else { + for (size_t i = 0; i < effective_chunk; ++i) { + run_encoder_step(tokens[processed + i], start_position + processed + i); + copy_component_outputs_to_chunk_inputs(*encoder_, *decoder_prefill_, i); + } + } + decoder_prefill_->graph->execute(); + processed += effective_chunk; + } + if (processed > 0 && prepare_decode) { + for (size_t i = 0; i < decoder_->input_buffers.size(); ++i) { + std::fill(decoder_->input_buffers[i].begin(), decoder_->input_buffers[i].end(), 0); + } + copy_cache_states(*decoder_prefill_, *decoder_); + } + return processed; +} + +void Model::run_full_context_text() { + if (!encoder_ || !decoder_ || context_tokens_.empty()) return; + int input_ids_idx = input_index(*encoder_, "input_ids"); + int attention_mask_idx = input_index(*encoder_, "attention_mask"); + if (input_ids_idx < 0 || attention_mask_idx < 0) { + throw std::runtime_error("text_lm_encoder requires input_ids and attention_mask inputs"); + } + size_t input_node = static_cast(encoder_->runtime_input_node_ids[input_ids_idx]); + const auto& input_desc = encoder_->graph->get_output_buffer(input_node); + if (context_tokens_.size() > input_desc.total_size) { + throw std::runtime_error("context exceeds transpiled text_lm_encoder capacity"); + } + std::fill(encoder_->input_buffers[input_ids_idx].begin(), encoder_->input_buffers[input_ids_idx].end(), 0); + std::fill(encoder_->input_buffers[attention_mask_idx].begin(), encoder_->input_buffers[attention_mask_idx].end(), 0); + for (size_t i = 0; i < context_tokens_.size(); ++i) { + write_int_input_at(*encoder_, "input_ids", i, static_cast(context_tokens_[i])); + write_int_input_at(*encoder_, "attention_mask", i, 1); + } + encoder_->graph->execute(); for (size_t i = 0; i < encoder_->output_node_ids.size() && i < encoder_->logical_outputs.size(); ++i) { const std::string& out_name = encoder_->logical_outputs[i]; int dst_idx = input_index(*decoder_, out_name); @@ -295,6 +773,7 @@ void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { void* src_ptr = encoder_->graph->get_output(src_node); std::memcpy(decoder_->input_buffers[dst_idx].data(), src_ptr, src_desc.byte_size); } + last_logit_position_ = context_tokens_.empty() ? 0 : context_tokens_.size() - 1; decoder_->graph->execute(); } @@ -304,7 +783,8 @@ uint32_t Model::argmax_last_logits() { void* ptr = decoder_->graph->get_output(out_node); size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; - size_t row_off = (seq > 0 ? (seq - 1) * vocab : 0); + size_t row = decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT ? std::min(last_logit_position_, seq > 0 ? seq - 1 : 0) : (seq > 0 ? seq - 1 : 0); + size_t row_off = row * vocab; uint32_t best = 0; float best_v = -std::numeric_limits::infinity(); if (desc.precision == Precision::FP32) { @@ -323,11 +803,19 @@ uint32_t Model::argmax_last_logits() { return best; } -void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/) { - for (size_t i = 0; i < tokens.size(); ++i) { - run_step(tokens[i], cache_total_seq_len_ + i, /*read_logits=*/false); +void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& /*profile_file*/, bool prepare_decode) { + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + if (!context_tokens_.empty()) run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + return; + } + size_t processed = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); + cache_total_seq_len_ += processed; + for (size_t i = processed; i < tokens.size(); ++i) { + run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false); + ++cache_total_seq_len_; } - cache_total_seq_len_ += tokens.size(); } void Model::prefill_with_images(const std::vector& tokens, @@ -340,6 +828,15 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* size_t /*top_k*/, const std::string& /*profile_file*/, float* out_entropy, float /*min_p*/, float /*repetition_penalty*/) { if (tokens.empty()) return 0; + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); + run_full_context_text(); + cache_total_seq_len_ = context_tokens_.size(); + if (out_entropy) *out_entropy = 0.0f; + uint32_t result = argmax_last_logits(); + record_sampled_token(result); + return result; + } for (size_t i = 0; i + 1 < tokens.size(); ++i) { run_step(tokens[i], cache_total_seq_len_ + i, /*read_logits=*/false); } @@ -374,7 +871,11 @@ std::vector Model::get_audio_embeddings(const std::vector& /*mel_b void Model::reset_cache() { cache_total_seq_len_ = 0; + last_logit_position_ = 0; + context_tokens_.clear(); token_history_.clear(); + if (decoder_) reset_component_cache_states(*decoder_); + if (decoder_prefill_) reset_component_cache_states(*decoder_prefill_); } void Model::set_cache_window(size_t /*window_size*/, size_t /*sink_size*/) {} diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index 9a5eeb1b1..594c75de2 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -283,6 +283,32 @@ bool file_exists(const std::string& path) { return f.good(); } +std::vector parse_token_ids(const std::string& text) { + std::vector tokens; + std::stringstream ss(text); + std::string part; + while (std::getline(ss, part, ',')) { + size_t start = part.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) continue; + size_t end = part.find_last_not_of(" \t\r\n"); + std::string value = part.substr(start, end - start + 1); + char* parsed_end = nullptr; + unsigned long parsed = std::strtoul(value.c_str(), &parsed_end, 10); + if (!parsed_end || *parsed_end != '\0') { + throw std::runtime_error("Invalid token id: " + value); + } + tokens.push_back(static_cast(parsed)); + } + return tokens; +} + +bool write_text_file(const std::string& path, const std::string& text) { + std::ofstream out(path); + if (!out) return false; + out << text; + return out.good(); +} + std::string json_string_value(const std::string& json, const std::string& key) { std::string needle = "\"" + key + "\":\""; size_t start = json.find(needle); @@ -338,7 +364,8 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--system ] [--image ] [--audio ]" - << " [--prompt ] [--thinking]\n"; + << " [--prompt ] [--input-ids ] [--max-new-tokens ]" + << " [--result-json ] [--thinking]\n"; } } // namespace @@ -354,6 +381,9 @@ int main(int argc, char** argv) { std::string current_image; std::string current_audio; std::string initial_prompt; + std::string input_ids; + std::string result_json; + int max_new_tokens = kMaxTokens; bool thinking = false; for (int i = 2; i < argc; ++i) { @@ -366,6 +396,12 @@ int main(int argc, char** argv) { current_audio = expand_tilde(argv[++i]); } else if (arg == "--prompt" && i + 1 < argc) { initial_prompt = argv[++i]; + } else if (arg == "--input-ids" && i + 1 < argc) { + input_ids = argv[++i]; + } else if (arg == "--max-new-tokens" && i + 1 < argc) { + max_new_tokens = std::max(0, std::atoi(argv[++i])); + } else if (arg == "--result-json" && i + 1 < argc) { + result_json = argv[++i]; } else if (arg == "--thinking") { thinking = true; } @@ -387,6 +423,39 @@ int main(int argc, char** argv) { return 1; } + if (!input_ids.empty()) { + std::vector tokens; + try { + tokens = parse_token_ids(input_ids); + } catch (const std::exception& e) { + std::cerr << e.what() << "\n"; + cactus_destroy(model); + return 1; + } + if (tokens.empty()) { + std::cerr << "--input-ids did not contain any token ids\n"; + cactus_destroy(model); + return 1; + } + std::vector response(kResponseBufferSize, 0); + int rc = cactus_benchmark_tokens( + model, + tokens.data(), + tokens.size(), + static_cast(max_new_tokens), + response.data(), + response.size()); + std::string response_json(response.data()); + if (!result_json.empty() && !write_text_file(result_json, response_json)) { + std::cerr << "Failed to write result JSON: " << result_json << "\n"; + cactus_destroy(model); + return 1; + } + std::cout << response_json << "\n"; + cactus_destroy(model); + return rc < 0 ? 1 : 0; + } + std::cout << "Model loaded.\n"; std::cout << "Commands: /image [prompt], /audio [prompt], "; #ifdef HAVE_SDL2 @@ -485,7 +554,7 @@ int main(int argc, char** argv) { history.push_back({"user", input}); std::string messages = build_messages(system_prompt, history, current_image, current_audio, attach_media); std::string options = "{\"temperature\":0.7,\"top_p\":0.95,\"top_k\":40,\"max_tokens\":" - + std::to_string(kMaxTokens) + + std::to_string(max_new_tokens) + ",\"enable_thinking_if_supported\":" + (thinking ? "true" : "false") + ",\"auto_handoff\":false,\"confidence_threshold\":0.0" + ",\"stop_sequences\":[\"<|im_end|>\",\"\"]}"; @@ -512,6 +581,9 @@ int main(int argc, char** argv) { current_pcm.size()); std::string response_json(response.data()); + if (!result_json.empty() && !write_text_file(result_json, response_json)) { + std::cerr << "Failed to write result JSON: " << result_json << "\n"; + } double ram_mb = json_number_value(response_json, "ram_usage_mb"); printer.print_stats(ram_mb); std::cout << "\n"; diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index e05832e7d..5491f541a 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -239,9 +239,15 @@ void compute_attention_cached_node( size_t new_seq_len = key_new_buf.total_size / (kv_heads * hdim); size_t history_len = (cache_len >= new_seq_len) ? cache_len - new_seq_len : 0; + bool cache_only_attention = false; size_t position_offset = node.params.position_offset; if (position_offset == std::numeric_limits::max()) { position_offset = history_len; + } else if (position_offset == std::numeric_limits::max() - 1) { + position_offset = (cache_len >= seq_len) ? cache_len - seq_len : 0; + history_len = cache_len; + new_seq_len = 0; + cache_only_attention = true; } if (k_cache_buf.precision == Precision::FP16 || v_cache_buf.precision == Precision::FP16) { @@ -263,19 +269,19 @@ void compute_attention_cached_node( return; } - cactus_attention_hybrid_int8_fp16( - query_buf.data_as<__fp16>(), - cached_keys, - cached_values, - k_scales, - v_scales, - key_new_buf.data_as<__fp16>(), - val_new_buf.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, history_len, seq_len, - num_q_heads, kv_heads, hdim, - node.params.scale, - position_offset, + cactus_attention_hybrid_int8_fp16( + query_buf.data_as<__fp16>(), + cached_keys, + cached_values, + k_scales, + v_scales, + key_new_buf.data_as<__fp16>(), + val_new_buf.data_as<__fp16>(), + node.output_buffer.data_as<__fp16>(), + batch_size, seq_len, history_len, cache_only_attention ? 0 : seq_len, + num_q_heads, kv_heads, hdim, + node.params.scale, + position_offset, true, node.params.window_size, KV_QUANT_GROUP_SIZE, diff --git a/cactus-kernels/src/threading.h b/cactus-kernels/src/threading.h index 665fecc69..8d5c7cdcc 100644 --- a/cactus-kernels/src/threading.h +++ b/cactus-kernels/src/threading.h @@ -240,11 +240,18 @@ namespace CactusThreading { inline bool pin_current_thread_to_cores(const std::vector& cores) { if (cores.empty()) return false; + cpu_set_t current_mask; + const bool has_current_mask = sched_getaffinity(0, sizeof(current_mask), ¤t_mask) == 0; cpu_set_t mask; CPU_ZERO(&mask); + bool selected = false; for (int core : cores) { - CPU_SET(core, &mask); + if (!has_current_mask || CPU_ISSET(core, ¤t_mask)) { + CPU_SET(core, &mask); + selected = true; + } } + if (!selected && has_current_mask) return true; return sched_setaffinity(0, sizeof(mask), &mask) == 0; } #endif diff --git a/cactus/build.sh b/cactus/build.sh index b139b23e0..dc41a98d3 100755 --- a/cactus/build.sh +++ b/cactus/build.sh @@ -32,7 +32,16 @@ echo "Building Cactus library..." cd "$(dirname "$0")/../cactus" -rm -rf build +LOCK_DIR="${TMPDIR:-/tmp}/cactus-build.lock" +while ! mkdir "$LOCK_DIR" 2>/dev/null; do + if [ -f "$LOCK_DIR/pid" ] && ! kill -0 "$(cat "$LOCK_DIR/pid" 2>/dev/null)" 2>/dev/null; then + rm -rf "$LOCK_DIR" + continue + fi + sleep 2 +done +echo "$$" > "$LOCK_DIR/pid" +trap 'rm -rf "$LOCK_DIR"' EXIT mkdir -p build cd build diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index 9681600ab..423f8ab2b 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -75,6 +75,14 @@ def cmd_build(args): if result.returncode != 0: print_color(RED, "Failed to build cactus library") return 1 + for _ in range(50): + if lib_path.exists(): + break + import time + time.sleep(0.1) + if not lib_path.exists(): + print_color(RED, f"Build did not produce {lib_path}") + return 1 tests_dir = PROJECT_ROOT / "cactus-engine" / "tests" build_dir = tests_dir / "build" build_dir.mkdir(parents=True, exist_ok=True) diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 3f4b519ba..ce4981ef5 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -2,26 +2,12 @@ import platform import subprocess import sys -import json from pathlib import Path -<<<<<<< HEAD -from .common import ( - PROJECT_ROOT, - _ensure_chat_binary, - get_effective_weights_dir, - print_color, - RED, GREEN, YELLOW, -) -from .download import cmd_download -from cactus.transpile.model_profiles import profile_for_model_type -======= from .common import PROJECT_ROOT, print_color, RED, GREEN ->>>>>>> origin/v2 def _resolve_bundle_dir(model_id: str) -> Path | None: - """Treat model_id as a local bundle dir; return its root if it is a transpiled bundle.""" path = Path(model_id).expanduser() if not path.exists() or not path.is_dir(): return None @@ -32,159 +18,56 @@ def _resolve_bundle_dir(model_id: str) -> Path | None: return None -<<<<<<< HEAD -def _validate_audio_path(audio_path: str) -> str | None: - resolved = str(Path(audio_path).expanduser().resolve()) - if not Path(resolved).exists(): - print_color(RED, f"Error: Audio file not found: {resolved}") - return None - return resolved - - -def _model_type_from_weights_dir(weights_dir: Path) -> str: - config_json = weights_dir / "config.json" - if config_json.exists(): - try: - payload = json.loads(config_json.read_text()) - model_type = str(payload.get("model_type", "") or "").strip().lower() - if model_type: - return model_type - except Exception: - pass - config_txt = weights_dir / "config.txt" - if config_txt.exists(): - for line in config_txt.read_text(errors="ignore").splitlines(): - key, sep, value = line.partition("=") - if sep and key.strip() == "model_type": - return value.strip().lower() - return "" - - -def _should_avoid_native_loader(weights_dir: Path) -> bool: - model_type = _model_type_from_weights_dir(weights_dir) - profile = profile_for_model_type(model_type) - return bool(profile and profile.avoid_native_loader) - - -def _prepare_transpiled_run_args(args, *, manifest_path: Path) -> int: - args.bundle_dir = str(_transpiled_bundle_root_from_manifest(manifest_path)) - args._transpiled_from_run = True - - image_path = getattr(args, 'image', None) - if image_path: - resolved = _validate_image_path(image_path) - if resolved is None: - return 1 - args.image = resolved - - image_files = [] - for image_file in getattr(args, 'image_file', []) or []: - if not image_file: - continue - resolved = _validate_image_path(str(image_file)) - if resolved is None: - return 1 - image_files.append(resolved) - args.image_file = image_files - - audio_path = getattr(args, 'audio', None) or getattr(args, 'audio_file', None) - if audio_path: - resolved = _validate_audio_path(str(audio_path)) - if resolved is None: - return 1 - args.audio = resolved - args.audio_file = resolved - - return 0 -======= def _ensure_chat_binary() -> Path | None: chat = PROJECT_ROOT / "cactus-engine" / "tests" / "build" / "chat" if chat.exists(): return chat print_color(RED, "Error: chat binary not found. Run `cactus build` first.") return None ->>>>>>> origin/v2 def cmd_run(args): """Run a transpiled Cactus bundle through the libcactus-backed chat binary.""" - if getattr(args, 'no_cloud_tele', False): + if getattr(args, "no_cloud_tele", False): os.environ["CACTUS_NO_CLOUD_TELE"] = "1" bundle_dir = _resolve_bundle_dir(args.model_id) if bundle_dir is None: - print_color(RED, - f"Error: {args.model_id} is not a transpiled bundle. " - "Run `cactus convert ` to produce one.") - return 1 - -<<<<<<< HEAD - local_path = Path(model_id) - if local_path.exists() and (local_path / "config.txt").exists(): - weights_dir = local_path - print_color(GREEN, f"Using local model: {weights_dir}") - else: - download_result = cmd_download(args) - if download_result != 0: - return download_result - weights_dir = get_effective_weights_dir(model_id, args) - manifest_path = _resolve_transpiled_manifest(weights_dir) - if manifest_path is not None: - if _prepare_transpiled_run_args(args, manifest_path=manifest_path) != 0: - return 1 - _clear_terminal_for_chat() - print_color(GREEN, f"Starting Cactus Chat with model: {model_id}") - print() - return cmd_run_transpiled(args) - - weights_path = Path(weights_dir) - if _should_avoid_native_loader(weights_path): print_color( RED, - "This weights folder contains CQ weights but no transpiled Cactus component bundle.", - ) - print_color( - YELLOW, - "Native C++ model subclasses are not available in this build, so this model must run through " - "the transpiled graph path. Build the bundle first, then rerun:\n" - f" cactus convert {model_id} {weights_path} --bits 4\n" - f" cactus run {weights_path}", + f"Error: {args.model_id} is not a transpiled bundle. " + "Run `cactus convert ` to produce one.", ) return 1 - image_path = getattr(args, 'image', None) - if image_path: - image_path = str(Path(image_path).resolve()) - if not Path(image_path).exists(): - print_color(RED, f"Error: Image file not found: {image_path}") - return 1 - valid_exts = {'.png', '.jpg', '.jpeg', '.bmp'} - if Path(image_path).suffix.lower() not in valid_exts: - print_color(RED, f"Error: Unsupported image format. Supported: {', '.join(valid_exts)}") - return 1 - - try: - chat_binary = _ensure_chat_binary(PROJECT_ROOT, lib_path) - except RuntimeError as exc: - print_color(RED, f"Error: {exc}") -======= chat = _ensure_chat_binary() if chat is None: ->>>>>>> origin/v2 return 1 cmd = [str(chat), str(bundle_dir)] - for flag, value in (("--system", getattr(args, 'system', None)), - ("--prompt", getattr(args, 'prompt', None)), - ("--image", getattr(args, 'image', None)), - ("--audio", getattr(args, 'audio', None) or getattr(args, 'audio_file', None))): + for flag, value in ( + ("--system", getattr(args, "system", None)), + ("--prompt", getattr(args, "prompt", None)), + ("--image", getattr(args, "image", None)), + ("--audio", getattr(args, "audio", None) or getattr(args, "audio_file", None)), + ): if value: - cmd.extend([flag, str(Path(value).expanduser().resolve()) if flag in ("--image", "--audio") else str(value)]) - if getattr(args, 'thinking', False): + cmd.extend([ + flag, + str(Path(value).expanduser().resolve()) if flag in ("--image", "--audio") else str(value), + ]) + for flag, value in ( + ("--input-ids", getattr(args, "input_ids", None)), + ("--max-new-tokens", getattr(args, "max_new_tokens", None)), + ("--result-json", getattr(args, "result_json", None)), + ): + if value is not None: + cmd.extend([flag, str(value)]) + if getattr(args, "thinking", False): cmd.append("--thinking") if sys.stdout.isatty(): - os.system('clear' if platform.system() != 'Windows' else 'cls') + os.system("clear" if platform.system() != "Windows" else "cls") print_color(GREEN, f"Starting Cactus Chat with model: {bundle_dir}") print() diff --git a/python/cactus/cli/transpile.py b/python/cactus/cli/transpile.py index c03bf4737..330d837e3 100644 --- a/python/cactus/cli/transpile.py +++ b/python/cactus/cli/transpile.py @@ -1,14 +1,17 @@ from __future__ import annotations +import json import os import platform import shutil import subprocess import sys +import time from pathlib import Path from .common import ( PROJECT_ROOT, + GREEN, RED, YELLOW, print_color, @@ -171,6 +174,23 @@ def _extra_args_has_option(extra_args: list[str], option: str) -> bool: return any(arg == option or arg.startswith(prefix) for arg in extra_args) +def _resolve_transpiled_manifest(path_value: str | os.PathLike[str] | None) -> Path | None: + if not path_value: + return None + candidate = Path(path_value).expanduser().resolve() + if not candidate.exists(): + return None + if candidate.is_file() and candidate.name == "manifest.json": + return candidate + for manifest in ( + candidate / "components" / "manifest.json", + candidate / "manifest.json", + ): + if manifest.exists(): + return manifest + return None + + def _prepend_python_path(env: dict[str, str]) -> None: python_root = str(PROJECT_ROOT / "python") existing = env.get("PYTHONPATH") @@ -220,7 +240,6 @@ def cmd_transpile(args) -> int: _prepend_python_path(env) result = subprocess.run(command, cwd=PROJECT_ROOT, env=env) return result.returncode -<<<<<<< HEAD def cmd_run_transpiled(args) -> int: @@ -560,5 +579,3 @@ def _print_transpiled_run_result(result: dict[str, object]) -> None: return print(json.dumps(result, indent=2, sort_keys=True)) -======= ->>>>>>> origin/v2 diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index d3daf70f3..0402ee77d 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -303,6 +303,7 @@ def _write_component_bundle( "audio_encoder", "vision_encoder", "lm_encoder", + "lm_encoder_text_chunk", "decoder_prefill_chunk", "decoder", "lm_encoder_step", @@ -2471,6 +2472,14 @@ def _load_transformers_bundle( ): config = _load_config_json(model_id) config_model_type = str(config.get("model_type", "") or "").lower() + if not config_model_type: + normalized_model_id = model_id.lower().replace("_", "-") + if "qwen3-vl" in normalized_model_id: + config_model_type = "qwen3_vl" + elif "lfm2.5-vl" in normalized_model_id or "lfm2-vl" in normalized_model_id: + config_model_type = "lfm2_vl" + if config_model_type: + config = {"model_type": config_model_type} external_transformers_site_packages = _ensure_transformers_supports_model_type(config_model_type) patch_note = _patch_transformers_torchvision_probe() if patch_note: @@ -2524,7 +2533,7 @@ def _load_transformers_bundle( f"Could not load tokenizer for {model_id}:\n" + "\n".join(tokenizer_errors) ) - if config_model_type in {"lfm2_vl", "qwen3_5"}: + if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}: model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2535,7 +2544,7 @@ def _load_transformers_bundle( tie_note = _tie_lfm2_vl_lm_head_if_needed(model) if tie_note: print(f"note={tie_note}") - elif config_model_type == "qwen3_5" and isinstance(config.get("text_config"), dict): + elif config_model_type in {"qwen3_5", "qwen3_vl"} and isinstance(config.get("text_config"), dict): model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -2584,7 +2593,7 @@ def _load_transformers_bundle( + processor_config_hint ) - if config_model_type in {"lfm2_vl", "qwen3_5"}: + if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}: model = AutoModelForImageTextToText.from_pretrained( model_source, dtype=torch_dtype, @@ -3017,7 +3026,7 @@ def main() -> int: system_prompt=args.system_prompt, enable_thinking_if_supported=args.enable_thinking, ) - elif config_model_type == "qwen3_5": + elif config_model_type in {"qwen3_5", "qwen3_vl"}: prepared = _prepare_qwen3_5_multimodal_inputs( processor_or_tokenizer, prompt=args.prompt, diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index 4925e8e47..c6639295a 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -23,6 +23,16 @@ _DYNAMIC_KV_POSITION_OFFSET = (1 << 64) - 1 +_DYNAMIC_KV_QUERY_POSITION_OFFSET = (1 << 64) - 2 +_KV_CACHE_TRANSPARENT_OPS = { + "expand", + "precision_cast", + "reshape", + "slice", + "transpose", + "unsqueeze", + "view", +} @dataclass @@ -254,6 +264,13 @@ def _lower_attention_with_internal_kv_cache( sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) g.kv_cache_append(key, k_cache, window_size=window_size, sink_size=sink_size) g.kv_cache_append(value, v_cache, window_size=window_size, sink_size=sink_size) + kv_input_cache_states: dict[tuple[str, str], tuple[Tensor, Tensor, int, int]] = env.setdefault( + "__internal_kv_cache_by_kv_inputs", + {}, + ) # type: ignore[assignment] + cache_entry = (k_cache, v_cache, window_size, int(value_shape[3])) + kv_input_cache_states[(node.inputs[1], node.inputs[2])] = cache_entry + kv_input_cache_states[_canonical_kv_cache_input_key(ir, node)] = cache_entry out = g.attention_cached( query, key, @@ -270,6 +287,66 @@ def _lower_attention_with_internal_kv_cache( return [g.permute(out, (0, 2, 1, 3))] +def _lower_attention_with_shared_internal_kv_cache( + g: Graph, + ir: IRGraph, + env: dict[str, Any], + node: IRNode, + *, + query: Tensor, + key: Tensor, + value: Tensor, + output_layout: str, +) -> list[Tensor] | None: + kv_input_cache_states: dict[tuple[str, str], tuple[Tensor, Tensor, int, int]] = env.get( + "__internal_kv_cache_by_kv_inputs", + {}, + ) + cache_state = kv_input_cache_states.get((node.inputs[1], node.inputs[2])) + if cache_state is None: + cache_state = kv_input_cache_states.get(_canonical_kv_cache_input_key(ir, node)) + if cache_state is None: + return None + k_cache, v_cache, window_size, v_head_dim = cache_state + out = g.attention_cached( + query, + key, + value, + k_cache, + v_cache, + scale=_resolve_attention_scale(node, query), + position_offset=_DYNAMIC_KV_QUERY_POSITION_OFFSET, + window_size=window_size, + v_head_dim=v_head_dim, + ) + if output_layout == "bthd": + return [out] + return [g.permute(out, (0, 2, 1, 3))] + + +def _canonical_kv_cache_input_key(ir: IRGraph, node: IRNode) -> tuple[str, str]: + return ( + _canonical_kv_cache_value_id(ir, node.inputs[1]), + _canonical_kv_cache_value_id(ir, node.inputs[2]), + ) + + +def _canonical_kv_cache_value_id(ir: IRGraph, value_id: str) -> str: + current = value_id + seen: set[str] = set() + while current not in seen: + seen.add(current) + value = ir.values.get(current) + producer_id = None if value is None else value.producer + if producer_id is None: + break + producer = ir.nodes.get(producer_id) + if producer is None or producer.op not in _KV_CACHE_TRANSPARENT_OPS or len(producer.inputs) != 1: + break + current = producer.inputs[0] + return current + + def _should_lower_conv1d_with_internal_conv_cache(ir: IRGraph, node: IRNode, x: Tensor, weight: Tensor) -> bool: if not bool(ir.meta.get("use_internal_conv_cache", False)): return False @@ -1090,6 +1167,18 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) -> if mask_tensor is not None: mask_tensor = _ensure_fp16_tensor(g, mask_tensor) mask_tensor = _normalize_attention_mask_for_cactus(g, mask_tensor, query) + shared_cached = _lower_attention_with_shared_internal_kv_cache( + g, + ir, + env, + node, + query=query, + key=key, + value=value, + output_layout=output_layout, + ) + if shared_cached is not None: + return shared_cached if _should_lower_attention_with_internal_kv_cache(ir, node): return _lower_attention_with_internal_kv_cache( g, diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 42e91222c..acd578d59 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -18,6 +18,7 @@ _GEMMA4_SAFE_TEXT_MLP_PRODUCT_SCALE = 1.0 / 64.0 +_UNSET = object() try: from transformers.models.gemma4.modeling_gemma4 import create_bidirectional_mask as _GEMMA4_CREATE_BIDIRECTIONAL_MASK # type: ignore except Exception: @@ -330,6 +331,10 @@ def _torch_is_compiling() -> bool: def _gemma4_cpu_safe_text_mlp_enabled(layer: torch.nn.Module, hidden_states: torch.Tensor) -> bool: if hidden_states.device.type != "cpu" or hidden_states.dtype != torch.float16: return False + return _gemma4_text_mlp_manual_available(layer) + + +def _gemma4_text_mlp_manual_available(layer: torch.nn.Module) -> bool: if bool(getattr(layer, "enable_moe_block", False)): return False mlp = getattr(layer, "mlp", None) @@ -417,8 +422,12 @@ def _gemma4_text_attention_forward( value_states = attn.v_norm(value_states) value_states = value_states.transpose(1, 2) - if bool(getattr(attn, "store_full_length_kv", False)): - shared_kv_states[int(attn.layer_idx)] = (key_states, value_states) + layer_idx = getattr(attn, "layer_idx", None) + if layer_idx is not None and ( + bool(getattr(attn, "store_full_length_kv", False)) + or not bool(getattr(attn, "is_kv_shared_layer", False)) + ): + shared_kv_states[int(layer_idx)] = (key_states, value_states) attention_interface = eager_attention_forward if getattr(attn.config, "_attn_implementation", "eager") != "eager": @@ -450,7 +459,8 @@ def _gemma4_text_decoder_layer_forward( use_cache: bool = False, shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, ) -> torch.Tensor: - if not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): + manual_attention_required = shared_kv_states is not None + if not manual_attention_required and not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states): return layer( hidden_states, position_embeddings=position_embeddings, @@ -460,6 +470,8 @@ def _gemma4_text_decoder_layer_forward( past_key_values=past_key_values, use_cache=use_cache, ) + if not _gemma4_text_mlp_manual_available(layer): + raise RuntimeError("Gemma4 KV sharing requires a supported text MLP structure") residual = hidden_states @@ -506,6 +518,10 @@ def _gemma4_text_backbone_forward( per_layer_inputs: torch.Tensor | None, causal_mask_mapping: dict[str, torch.Tensor], position_ids: torch.LongTensor, + layer_start: int = 0, + layer_end: int | None = None, + apply_norm: bool = True, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None | object = _UNSET, ) -> torch.Tensor: hidden_states = inputs_embeds layer_types = tuple(dict.fromkeys(getattr(backbone.config, "layer_types", ()))) @@ -513,10 +529,14 @@ def _gemma4_text_backbone_forward( layer_type: backbone.rotary_emb(hidden_states, position_ids, layer_type) for layer_type in layer_types } - shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + if shared_kv_states is _UNSET: + num_shared_layers = int(getattr(backbone.config, "num_kv_shared_layers", 0) or 0) + shared_kv_states = {} if num_shared_layers > 0 else None config_layer_types = tuple(getattr(backbone.config, "layer_types", ())) - for layer_index, decoder_layer in enumerate(backbone.layers[: backbone.config.num_hidden_layers]): + end = backbone.config.num_hidden_layers if layer_end is None else min(int(layer_end), int(backbone.config.num_hidden_layers)) + for layer_index in range(max(0, int(layer_start)), end): + decoder_layer = backbone.layers[layer_index] layer_per_input = None if per_layer_inputs is not None: layer_per_input = per_layer_inputs[:, :, decoder_layer.layer_idx, :] @@ -537,7 +557,7 @@ def _gemma4_text_backbone_forward( shared_kv_states=shared_kv_states, ) - return backbone.norm(hidden_states) + return backbone.norm(hidden_states) if apply_norm else hidden_states def _gemma4_strip_audio_padding(audio_output: object) -> torch.Tensor: @@ -1688,6 +1708,10 @@ def get_transpile_metadata(self): } +class Lfm2VlLMEncoderTextChunkAdapter(Lfm2VlLMEncoderStepAdapter): + pass + + class Lfm2VlDecoderAdapter(torch.nn.Module): def __init__( self, @@ -3040,6 +3064,75 @@ class Gemma4DecoderStepAdapter(Gemma4DecoderAdapter): pass +class Gemma4DecoderPrefillChunkAdapter(Gemma4DecoderAdapter): + def forward( + self, + inputs_embeds: torch.Tensor, + per_layer_inputs: torch.Tensor, + position_ids: torch.LongTensor, + ) -> torch.Tensor: + normalized_per_layer_inputs = per_layer_inputs + if normalized_per_layer_inputs.numel() == 0: + normalized_per_layer_inputs = None + attention_mask = torch.ones( + position_ids.shape, + dtype=torch.long, + device=position_ids.device, + ) + causal_mask_mapping = _gemma4_build_standard_causal_mask_mapping( + create_causal_mask=self._create_causal_mask, + create_sliding_window_causal_mask=self._create_sliding_window_causal_mask, + config=self.backbone.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + ) + num_layers = int(getattr(self.backbone.config, "num_hidden_layers", len(self.backbone.layers))) + num_shared_layers = int(getattr(self.backbone.config, "num_kv_shared_layers", 0) or 0) + first_shared_layer = max(0, num_layers - num_shared_layers) + if first_shared_layer <= 0 or first_shared_layer >= num_layers or int(inputs_embeds.shape[1]) <= 1: + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + ) + else: + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=inputs_embeds, + per_layer_inputs=normalized_per_layer_inputs, + causal_mask_mapping=causal_mask_mapping, + position_ids=position_ids, + layer_end=first_shared_layer, + apply_norm=False, + shared_kv_states=shared_kv_states, + ) + tail_masks = { + key: value[:, :, -1:, :] if value.ndim == 4 else value + for key, value in causal_mask_mapping.items() + } + tail_per_layer_inputs = ( + None + if normalized_per_layer_inputs is None + else normalized_per_layer_inputs[:, -1:, :, :] + ) + hidden_states = _gemma4_text_backbone_forward( + self.backbone, + inputs_embeds=hidden_states[:, -1:, :], + per_layer_inputs=tail_per_layer_inputs, + causal_mask_mapping=tail_masks, + position_ids=position_ids[:, -1:], + layer_start=first_shared_layer, + apply_norm=True, + shared_kv_states=shared_kv_states, + ) + logits = self.model.lm_head(hidden_states[:, -1:, :]) + return _gemma4_apply_final_logit_softcapping(self.model, logits) + + def _build_gemma4_multimodal_component_specs( model: torch.nn.Module, *, @@ -3076,32 +3169,27 @@ def _require(component: str) -> None: _require("lm_encoder_step") _require("lm_encoder_media_step") _require("decoder_step") - if "decoder_prefill_chunk" in requested_set: + if "lm_encoder" in requested_set: _require("vision_encoder") _require("audio_encoder") _require("lm_encoder") + if "decoder_prefill_chunk" in requested_set: _require("decoder_prefill_chunk") - elif "lm_encoder" in requested_set: + if "vision_encoder" in requested_set: _require("vision_encoder") + if "audio_encoder" in requested_set: _require("audio_encoder") - _require("lm_encoder") - else: - if "vision_encoder" in requested_set: - _require("vision_encoder") - if "audio_encoder" in requested_set: - _require("audio_encoder") - if "lm_encoder_step" in requested_set: - _require("lm_encoder_step") - if "lm_encoder_text_chunk" in requested_set: - _require("lm_encoder_text_chunk") - if "lm_encoder_media_step" in requested_set: - _require("lm_encoder_media_step") - if "lm_encoder_media_chunk" in requested_set: - _require("lm_encoder_media_chunk") - if "decoder_step" in requested_set: - _require("lm_encoder_step") - _require("lm_encoder_media_step") - _require("decoder_step") + if "lm_encoder_step" in requested_set: + _require("lm_encoder_step") + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "lm_encoder_media_step" in requested_set: + _require("lm_encoder_media_step") + if "lm_encoder_media_chunk" in requested_set: + _require("lm_encoder_media_chunk") + if "decoder_step" in requested_set: + _require("lm_encoder_step") + _require("decoder_step") vision_tower = getattr(getattr(model, "model", model), "vision_tower", None) pooling_kernel_size = int(_module_or_config_attr(vision_tower, "pooling_kernel_size", 3) or 3) @@ -3125,6 +3213,7 @@ def _require(component: str) -> None: image_features: torch.Tensor | None = None audio_features: torch.Tensor | None = None decoder_inputs: tuple[torch.Tensor, ...] | None = None + prefill_decoder_inputs: tuple[torch.Tensor, ...] | None = None decoder_step_inputs: tuple[torch.Tensor, ...] | None = None step_input_ids: torch.Tensor | None = None step_position_ids: torch.Tensor | None = None @@ -3135,16 +3224,14 @@ def _require(component: str) -> None: if "vision_encoder" in expanded_components and ( "lm_encoder" in expanded_components or "decoder" in expanded_components - or "decoder_prefill_chunk" in expanded_components ): image_features = vision_encoder(pixel_values, pixel_position_ids) if "audio_encoder" in expanded_components and ( "lm_encoder" in expanded_components or "decoder" in expanded_components - or "decoder_prefill_chunk" in expanded_components ): audio_features = audio_encoder(input_features, input_features_mask) - if "lm_encoder" in expanded_components or "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if "lm_encoder" in expanded_components or "decoder" in expanded_components: if image_features is None: image_features = vision_encoder(pixel_values, pixel_position_ids) if audio_features is None: @@ -3160,17 +3247,18 @@ def _require(component: str) -> None: weights_dir=weights_dir, native_merge_plan=native_merge_plan, ).eval() - encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "32") or "32")) + encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "128") or "128")) encoder_chunk_size = min(encoder_chunk_size, int(input_ids.shape[1])) lm_encoder_step = Gemma4LMEncoderStepAdapter(model, weights_dir=weights_dir).eval() lm_encoder_text_chunk = Gemma4LMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() lm_encoder_media_step = Gemma4LMEncoderMediaStepAdapter(model, weights_dir=weights_dir).eval() lm_encoder_media_chunk = Gemma4LMEncoderMediaChunkAdapter(model, weights_dir=weights_dir).eval() decoder = Gemma4DecoderAdapter(model, weights_dir=weights_dir).eval() + decoder_prefill = Gemma4DecoderPrefillChunkAdapter(model, weights_dir=weights_dir).eval() decoder_step = Gemma4DecoderStepAdapter(model, weights_dir=weights_dir).eval() with torch.no_grad(): - if "decoder" in expanded_components or "decoder_prefill_chunk" in expanded_components: + if "decoder" in expanded_components: decoder_inputs = lm_encoder( input_ids, attention_mask, @@ -3209,6 +3297,8 @@ def _require(component: str) -> None: dtype=torch.long, device=input_ids.device, ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + if "decoder_prefill_chunk" in expanded_components and decoder_inputs is None: + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) if media_step_embeds is not None: hidden_size = int(media_step_embeds.shape[-1]) media_chunk_embeds = torch.zeros( @@ -3268,7 +3358,7 @@ def _require(component: str) -> None: metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) cache_seq_len = max(1024, int(input_ids.shape[1]) + 256) - prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "64") or "64")) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) if "decoder" in expanded_components: if decoder_inputs is None: @@ -3283,17 +3373,21 @@ def _require(component: str) -> None: metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) if "decoder_prefill_chunk" in expanded_components: - if decoder_inputs is None: + if decoder_inputs is not None: + chunk_inputs = tuple( + value[:, :prefill_chunk_size, ...].contiguous() + if value.ndim >= 2 + else value + for value in decoder_inputs + ) + elif prefill_decoder_inputs is not None: + prefill_chunk_size = int(prefill_decoder_inputs[0].shape[1]) + chunk_inputs = tuple(prefill_decoder_inputs) + else: raise RuntimeError("Gemma4 decoder_prefill_chunk spec requires precomputed decoder inputs") - chunk_inputs = tuple( - value[:, :prefill_chunk_size, ...].contiguous() - if value.ndim >= 2 - else value - for value in decoder_inputs - ) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", - module=decoder, + module=decoder_prefill, example_inputs=chunk_inputs, input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, output_keys=("logits",), @@ -3422,11 +3516,15 @@ def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[tor position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=linear_attn_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3443,7 +3541,7 @@ def debug_forward(self, input_ids: torch.Tensor) -> tuple[torch.Tensor, list[tor return self.model.lm_head(hidden_states), checkpoints def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3488,11 +3586,15 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = linear_attn_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=linear_attn_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3502,7 +3604,7 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. return self.model.lm_head(hidden_states[:, -1:, :]) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3534,6 +3636,26 @@ def _resolve_qwen35_multimodal_backbone(model: torch.nn.Module) -> torch.nn.Modu raise AttributeError(f"{type(model).__name__} does not expose a Qwen3.5 multimodal backbone") +def _qwen35_layer_types(config) -> tuple[str, ...]: + layer_types = tuple(str(value) for value in (getattr(config, "layer_types", ()) or ())) + if layer_types: + return layer_types + num_layers = int(getattr(config, "num_hidden_layers", 0) or 0) + return tuple("full_attention" for _ in range(num_layers)) + + +def _qwen35_attention_mask_for_layer( + config, + layer_index: int, + *, + causal_mask, + linear_attention_mask, +): + layer_types = _qwen35_layer_types(config) + layer_type = layer_types[layer_index] if layer_index < len(layer_types) else "full_attention" + return linear_attention_mask if layer_type == "linear_attention" else causal_mask + + class Qwen35VisionEncoderAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module, *, image_grid_thw: torch.Tensor): super().__init__() @@ -3677,11 +3799,15 @@ def forward( hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = attention_mask if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=attention_mask, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3690,7 +3816,7 @@ def forward( return self.model.lm_head(hidden_states) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3732,6 +3858,10 @@ def get_transpile_metadata(self): } +class Qwen35LMEncoderTextChunkAdapter(Qwen35LMEncoderStepAdapter): + pass + + class Qwen35EmbedsCausalLMStepAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module): super().__init__() @@ -3762,11 +3892,15 @@ def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> to hidden_states = inputs_embeds position_embeddings = self.backbone.rotary_emb(hidden_states, multimodal_position_ids) for i, decoder_layer in enumerate(self.backbone.layers[: self.backbone.config.num_hidden_layers]): - layer_mask = None if self.backbone.config.layer_types[i] == "linear_attention" else causal_mask hidden_states = decoder_layer( hidden_states, position_embeddings=position_embeddings, - attention_mask=layer_mask, + attention_mask=_qwen35_attention_mask_for_layer( + self.backbone.config, + i, + causal_mask=causal_mask, + linear_attention_mask=None, + ), position_ids=text_position_ids, past_key_values=None, use_cache=False, @@ -3775,7 +3909,7 @@ def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> to return self.model.lm_head(hidden_states[:, -1:, :]) def get_transpile_metadata(self): - layer_types = tuple(getattr(self.backbone.config, "layer_types", ())) + layer_types = _qwen35_layer_types(self.backbone.config) return { "graph": { **_transpile_graph_meta( @@ -3790,6 +3924,10 @@ def get_transpile_metadata(self): } +class Qwen35EmbedsCausalLMPrefillChunkAdapter(Qwen35EmbedsCausalLMStepAdapter): + pass + + class Qwen3CausalLMLogitsAdapter(torch.nn.Module): def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None): super().__init__() @@ -4011,6 +4149,8 @@ def _build_qwen3_5_multimodal_component_specs( "vision_encoder", "lm_encoder", "decoder", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", "lm_encoder_step", "decoder_media_step", "decoder_step", @@ -4026,9 +4166,16 @@ def _require(component: str) -> None: _require("vision_encoder") _require("lm_encoder") _require("decoder") + _require("lm_encoder_text_chunk") + _require("decoder_prefill_chunk") if "lm_encoder" in requested_set: _require("vision_encoder") _require("lm_encoder") + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "decoder_prefill_chunk" in requested_set: + _require("lm_encoder_text_chunk") + _require("decoder_prefill_chunk") if "vision_encoder" in requested_set: _require("vision_encoder") if "decoder_step" in requested_set: @@ -4042,11 +4189,14 @@ def _require(component: str) -> None: lm_encoder = Qwen35LMEncoderAdapter(model, input_ids=input_ids).eval() decoder = Qwen35EmbedsCausalLMLogitsAdapter(model).eval() lm_encoder_step = Qwen35LMEncoderStepAdapter(model).eval() + lm_encoder_text_chunk = Qwen35LMEncoderTextChunkAdapter(model).eval() decoder_media_step = Qwen35EmbedsCausalLMStepAdapter(model).eval() + decoder_prefill_chunk = Qwen35EmbedsCausalLMPrefillChunkAdapter(model).eval() decoder_step = Qwen35CausalLMStepAdapter(model, pad_token_id=_resolve_model_pad_token_id(model)).eval() image_features: torch.Tensor | None = None decoder_inputs: tuple[torch.Tensor, ...] | None = None + prefill_decoder_inputs: tuple[torch.Tensor, ...] | None = None with torch.no_grad(): if "vision_encoder" in expanded_components and ("lm_encoder" in expanded_components or "decoder" in expanded_components): image_features = vision_encoder(pixel_values) @@ -4054,6 +4204,16 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values) decoder_inputs = lm_encoder(input_ids, attention_mask, position_ids, image_features) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_QWEN_PREFILL_CHUNK", "128") or "128")) + prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) + chunk_input_ids = input_ids[:, :prefill_chunk_size].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk_size, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + if "decoder_prefill_chunk" in expanded_components: + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) common_graph_meta = { "weights_dir": weights_dir, @@ -4107,6 +4267,38 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, )) + if "lm_encoder_text_chunk" in expanded_components: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "position_ids"), + graph_meta={**common_graph_meta, "component": "lm_encoder_text_chunk"}, + metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, + )) + if "decoder_prefill_chunk" in expanded_components: + if prefill_decoder_inputs is None: + raise RuntimeError("Qwen3.5 decoder_prefill_chunk spec requires precomputed text chunk decoder inputs") + max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + specs.append(ComponentModuleSpec( + component="decoder_prefill_chunk", + module=decoder_prefill_chunk, + example_inputs=prefill_decoder_inputs, + input_keys=("inputs_embeds", "position_ids"), + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_prefill_chunk", + "use_internal_kv_cache": True, + "use_internal_conv_cache": True, + "use_internal_gated_deltanet_cache": True, + "max_cache_seq_len": max_cache_seq_len, + "cache_sink_size": 4, + "prefill_chunk_size": prefill_chunk_size, + }, + metadata={"family": "qwen3_5", "task": "multimodal_causal_lm_logits"}, + )) if "decoder_media_step" in expanded_components: if decoder_inputs is None: raise RuntimeError("Qwen3.5 decoder_media_step spec requires precomputed decoder inputs") @@ -4388,6 +4580,8 @@ def _family_key(model: torch.nn.Module) -> str: return "gemma" if module_name.startswith("transformers.models.qwen3_5."): return "qwen3_5" + if module_name.startswith("transformers.models.qwen3_vl."): + return "qwen3_5" if module_name.startswith("transformers.models.qwen3."): return "qwen3" if module_name.startswith("transformers.models.lfm2_vl."): @@ -4567,7 +4761,7 @@ def _build_lfm2_vl_multimodal_component_specs( dtype=attention_mask.dtype, device=attention_mask.device, ) - prefill_chunk = max(2, int(os.environ.get("CACTUS_LFM2_PREFILL_CHUNK", "64") or "64")) + prefill_chunk = max(2, int(os.environ.get("CACTUS_LFM2_PREFILL_CHUNK", "128") or "128")) prefill_chunk = min(prefill_chunk, int(input_ids.shape[1])) requested_components = tuple(components or ("vision_encoder", "lm_encoder", "text_lm_encoder", "decoder")) @@ -4588,28 +4782,28 @@ def _require(component: str) -> None: _require("decoder") _require("text_decoder") _require("lm_encoder_step") + _require("lm_encoder_text_chunk") _require("decoder_prefill_chunk") _require("decoder_step") - elif "lm_encoder" in requested_set: + if "lm_encoder" in requested_set: _require("vision_encoder") _require("lm_encoder") - elif "lm_encoder_step" in requested_set: + if "lm_encoder_step" in requested_set: _require("lm_encoder_step") - if "decoder_step" in requested_set: - _require("decoder_step") - elif "decoder_step" in requested_set: + if "lm_encoder_text_chunk" in requested_set: + _require("lm_encoder_text_chunk") + if "decoder_step" in requested_set: _require("lm_encoder_step") _require("decoder_step") - elif "decoder_prefill_chunk" in requested_set: + if "decoder_prefill_chunk" in requested_set: + _require("lm_encoder_text_chunk") _require("decoder_prefill_chunk") - elif "text_lm_encoder" in requested_set: + if "text_lm_encoder" in requested_set: _require("text_lm_encoder") - if "text_decoder" in requested_set: - _require("text_decoder") - elif "text_decoder" in requested_set: + if "text_decoder" in requested_set: _require("text_lm_encoder") _require("text_decoder") - elif "vision_encoder" in requested_set: + if "vision_encoder" in requested_set: _require("vision_encoder") vision_encoder = Lfm2VlVisionEncoderAdapter( @@ -4621,6 +4815,7 @@ def _require(component: str) -> None: lm_encoder = Lfm2VlLMEncoderAdapter(model, input_ids=input_ids, weights_dir=weights_dir).eval() text_lm_encoder = Lfm2VlTextLMEncoderAdapter(model, weights_dir=weights_dir).eval() lm_encoder_step = Lfm2VlLMEncoderStepAdapter(model, weights_dir=weights_dir).eval() + lm_encoder_text_chunk = Lfm2VlLMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() decoder = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir, last_token_only=False).eval() decoder_last_token = Lfm2VlDecoderAdapter(model, weights_dir=weights_dir, last_token_only=True).eval() @@ -4637,6 +4832,13 @@ def _require(component: str) -> None: if "text_lm_encoder" in expanded_components: text_decoder_inputs = text_lm_encoder(text_input_ids, text_attention_mask) + chunk_input_ids = input_ids[:, :prefill_chunk].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + common_graph_meta = { "weights_dir": weights_dir, "task": "multimodal_causal_lm_logits", @@ -4715,6 +4917,20 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, )) + if "lm_encoder_text_chunk" in expanded_components: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "attention_mask", "position_ids"), + graph_meta={ + **common_graph_meta, + "component": "lm_encoder_text_chunk", + "encoder_chunk_size": prefill_chunk, + }, + metadata={"family": "lfm2_vl", "task": "multimodal_causal_lm_logits"}, + )) if "decoder_prefill_chunk" in expanded_components: if decoder_inputs is None: if image_features is None: diff --git a/python/cactus/transpile/model_profiles.py b/python/cactus/transpile/model_profiles.py index 7e84e4c69..6a07d9c7c 100644 --- a/python/cactus/transpile/model_profiles.py +++ b/python/cactus/transpile/model_profiles.py @@ -166,7 +166,7 @@ class ModelProfile: QWEN_PROFILE = ModelProfile( family="qwen", - model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5"), + model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"), multimodal_context_tokens=512, input_combinations=((), ("image",)), model_id_aliases=( @@ -177,16 +177,25 @@ class ModelProfile: ("qwen3", "Qwen/Qwen3-1.7B"), ), model_id_markers=("qwen",), - family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5"), + family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"), stop_tokens=("<|im_end|>",), avoid_native_loader=True, - cached_step_components=("decoder_step",), + cached_step_components=("decoder_media_step",), cached_step_skip_components=("decoder",), - fp16_kv_cache_components=("decoder_step",), + fp16_kv_cache_components=("decoder_prefill_chunk", "decoder_media_step", "decoder_step"), prompt_style="qwen_chat", multimodal_preprocessor="qwen3_5", default_task="multimodal_causal_lm_logits", - default_components=("vision_encoder", "lm_encoder", "decoder", "lm_encoder_step", "decoder_media_step", "decoder_step"), + default_components=( + "vision_encoder", + "lm_encoder", + "decoder", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + "lm_encoder_step", + "decoder_media_step", + "decoder_step", + ), needs_image=True, force_component_pipeline=True, ) From fb4cc3964d7871d3d7b15cc5e72ef1cebb213dce Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Tue, 19 May 2026 16:27:00 -0700 Subject: [PATCH 02/12] Add aggregated benchmark result CSVs Signed-off-by: Noah Cylich --- .../matrix_galaxy_s24_or_s25_full_core.csv | 61 ++++++++ ...matrix_galaxy_s24_or_s25_single_thread.csv | 126 ++++++++++++++++ .../results/matrix_mac_m4pro_full_core.csv | 76 ++++++++++ .../matrix_mac_m4pro_single_thread.csv | 141 ++++++++++++++++++ .../results/matrix_pixel_10a_full_core.csv | 61 ++++++++ .../matrix_pixel_10a_single_thread.csv | 126 ++++++++++++++++ 6 files changed, 591 insertions(+) create mode 100644 experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv create mode 100644 experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv create mode 100644 experiments/matrix/results/matrix_mac_m4pro_full_core.csv create mode 100644 experiments/matrix/results/matrix_mac_m4pro_single_thread.csv create mode 100644 experiments/matrix/results/matrix_pixel_10a_full_core.csv create mode 100644 experiments/matrix/results/matrix_pixel_10a_single_thread.csv diff --git a/experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv b/experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv new file mode 100644 index 000000000..027f84846 --- /dev/null +++ b/experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv @@ -0,0 +1,61 @@ +device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,256,0,134.290000,1613.160,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,512,0,119.260000,1592.599,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,1024,0,90.890000,1593.312,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,2048,0,87.180000,1596.804,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,256,0,58.380000,1224.530,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,512,0,58.860000,1224.609,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,58.210000,1226.157,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,57.470000,1225.903,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,256,0,52.400000,1976.993,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,512,0,52.090000,1977.999,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,1024,0,48.280000,1977.347,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,2048,0,43.520000,1976.959,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,256,0,69.638400,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,512,0,119.321000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,181.958000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,234.322000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,256,0,57.920000,3499.900,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,512,0,127.760000,3497.240,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,1024,0,195.370000,3495.300,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,2048,0,223.510000,3490.750,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,256,0,13.453600,3116.622,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,512,0,12.083600,3179.782,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,1024,0,16.327400,3192.851,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,2048,0,19.124900,3215.312,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,20.150500,724.741,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,21.435900,759.907,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,31.473600,768.084,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,42.476600,778.535,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,256,0,18.710500,1190.149,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,512,0,15.800400,1252.039,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,1024,0,23.243800,1310.097,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,2048,0,32.747000,1416.436,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,256,0,29.693100,1627.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,512,0,26.433400,1796.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,1024,0,33.878900,2225.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,2048,0,35.691600,3402.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,112.955000,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,166.089000,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,146.154000,2450.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,156.402000,3504.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,256,0,161.673000,1281.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,512,0,169.303000,1570.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,1024,0,168.010000,2194.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,2048,0,102.698000,3874.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" diff --git a/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv b/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv new file mode 100644 index 000000000..bb0344844 --- /dev/null +++ b/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv @@ -0,0 +1,126 @@ +device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,256,0,33.520000,1626.336,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,256,100,12.680000,1626.336,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,512,0,43.150000,1626.239,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,512,100,12.060000,1626.239,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,1024,0,54.070000,1626.288,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,1024,100,11.170000,1626.288,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,2048,0,56.020000,1626.360,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,2048,100,10.340000,1626.360,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,256,0,20.200000,666.358,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,256,100,19.350000,666.358,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,512,0,20.060000,666.320,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,512,100,18.800000,666.320,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,16.370000,666.346,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,1024,100,14.760000,666.346,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,15.070000,666.470,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,2048,100,13.720000,666.470,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,parakeet_tdt_v3,parakeet_transcribe,,,0.200471,2475.970,777.969,778.112,ok,WER=0.000000; runner=android_transpiled_tdt; core_affinity=cpu7; taskset_mask=80; serial=RFGL42B1VLW; android=16; thermal_status=1; decoder_steps=152; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey; thread_count=1 +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,256,0,11.620000,1224.077,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,256,100,11.060000,1224.077,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,512,0,11.250000,1223.396,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,512,100,10.620000,1223.396,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,1024,0,10.550000,1223.269,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,1024,100,9.160000,1223.269,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,2048,0,9.420000,1224.050,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,2048,100,7.520000,1224.050,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,2048,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,256,0,94.300200,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,256,100,3.173630,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,512,0,137.059000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,512,100,3.258580,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,149.938000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,1024,100,3.055200,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,122.384000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; prefill_only=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,2048,100,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export has KV cache capacity 2048; cannot decode 100 tokens from context 2048 because required cache positions extend to 2147 +galaxy_s24_or_s25,executorch,parakeet_tdt_v3,parakeet_transcribe,,,0.215333,1506.785,719.224,719.224,ok,WER=0.000000; runner=android_executorch_parakeet; format=Q4 8da4w XNNPACK; core_affinity=cpu7; taskset_mask=80; serial=RFGL42B1VLW; android=16; thermal_status=1; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,2048,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,256,0,33.570000,3502.820,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,256,100,10.930000,3502.820,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,512,0,69.420000,3497.600,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,512,100,11.480000,3497.600,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,1024,0,142.190000,3489.960,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,1024,100,11.750000,3489.960,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,2048,0,144.750000,3490.270,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,2048,100,11.530000,3490.270,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,0.071528,1285.852,667.588,667.590,ok,WER=0.806452; runner=android_litert_parakeet_asr; chunks=6; encoder_frames=378; decoder_steps=177; core_affinity=cpu7; taskset_mask=80; serial=RFGL42B1VLW; android=16; thermal_status=1; transcript=con returned to its place amidst the tents the english foaded two the french congratulations +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,256,0,7.636660,3118.732,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,256,100,5.812560,3118.732,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,512,0,8.088930,3184.755,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,512,100,5.944650,3184.755,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,1024,0,8.150660,3195.753,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,1024,100,5.935540,3195.753,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,2048,0,8.062440,3218.219,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,2048,100,5.518090,3218.219,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,16.236700,723.342,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,256,100,13.149400,723.342,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,17.256000,760.725,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,512,100,13.279900,760.725,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,17.409700,766.848,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,1024,100,13.038800,766.848,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,17.474600,781.181,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,2048,100,12.794700,781.181,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,llama.cpp does not have a configured Parakeet TDT runner or artifact +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,256,0,12.016400,1191.867,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,256,100,8.270410,1191.867,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,512,0,12.504600,1253.624,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,512,100,8.192990,1253.624,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,1024,0,12.650800,1311.758,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,1024,100,7.561010,1311.758,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,2048,0,12.040200,1417.365,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,2048,100,5.878280,1417.365,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,256,0,18.213300,1617.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,256,100,1.663560,1617.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,512,0,17.986700,1782.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,512,100,1.631470,1782.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,1024,0,17.626400,2182.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,1024,100,1.591130,2182.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,2048,0,16.546700,3291.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,2048,100,1.471690,3291.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,92.588500,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,256,100,22.606100,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,93.125200,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,512,100,21.820600,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,89.599600,2402.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,1024,100,20.428700,2402.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,89.010700,3277.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,2048,100,17.192900,3277.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,ONNX Runtime Parakeet TDT Android artifact and ASR runner are not configured +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,256,0,57.884400,1303.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,256,100,15.374200,1303.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,512,0,58.630400,1596.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,512,100,13.419100,1596.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,1024,0,57.203700,2181.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,1024,100,9.827840,2181.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,2048,0,54.185400,3654.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,2048,100,6.592570,3654.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" diff --git a/experiments/matrix/results/matrix_mac_m4pro_full_core.csv b/experiments/matrix/results/matrix_mac_m4pro_full_core.csv new file mode 100644 index 000000000..a149f03e8 --- /dev/null +++ b/experiments/matrix/results/matrix_mac_m4pro_full_core.csv @@ -0,0 +1,76 @@ +device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes +mac_m4pro,cactus,gemma_4_e2b,prefill,256,0,442.380000,585.770,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,512,0,463.330000,586.780,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,1024,0,459.850000,586.300,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,2048,0,437.370000,586.350,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,4096,0,415.940000,586.020,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,256,0,62.900000,75.770,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,512,0,61.810000,75.660,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,61.720000,75.810,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,60.300000,75.310,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,57.020000,74.830,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,256,0,35.740000,381.700,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,512,0,35.150000,381.470,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,1024,0,33.580000,381.100,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,2048,0,30.860000,381.420,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,4096,0,26.450000,381.660,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,256,0,172.346606,2826.660,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; format=Q4 8da4w XNNPACK" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,512,0,172.190570,2834.456,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; format=Q4 8da4w XNNPACK" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,171.029009,2840.613,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; format=Q4 8da4w XNNPACK" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,169.352145,2847.160,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; format=Q4 8da4w XNNPACK" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,4096,0,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export supports at most 2048 input tokens for token_embedding; requested prefill seqlen=4096 +mac_m4pro,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,256,0,223.950000,2606.676,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,512,0,446.980000,2607.020,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,1024,0,894.970000,2608.129,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,2048,0,948.450000,2605.239,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,4096,0,980.730000,2605.145,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,4096,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,256,0,26.328900,1544.145,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,512,0,24.621900,1605.566,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,1024,0,22.588400,1620.285,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,2048,0,20.365000,1642.067,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,4096,0,17.536500,1684.770,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,138.438000,740.611,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,130.636000,778.392,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,117.733000,784.549,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,98.284200,799.814,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,4096,0,73.697800,827.861,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,256,0,35.528400,1211.721,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,512,0,33.307200,1273.174,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,1024,0,29.568300,1330.846,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,2048,0,24.105000,1437.799,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,4096,0,17.570000,1677.581,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,256,0,199.357507,1427.222,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,512,0,186.217956,1631.519,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,1024,0,171.094057,2126.191,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,2048,0,155.352941,3367.770,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,4096,0,145.326047,7367.398,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,802.337017,6456.881,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,865.978363,6465.740,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,862.162204,6594.131,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,774.924446,6839.162,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,4096,0,705.188699,8493.491,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,256,0,576.476548,7458.866,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4 block=32; synthetic exact-length token ids" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,512,0,566.069027,7467.319,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4 block=32; synthetic exact-length token ids" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,1024,0,529.783736,7529.819,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4 block=32; synthetic exact-length token ids" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,2048,0,415.214815,6704.819,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4 block=32; synthetic exact-length token ids" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,4096,0,389.554473,9128.398,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; gpu=disabled; threads=8; format=INT4/Q4 block=32; synthetic exact-length token ids" diff --git a/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv b/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv new file mode 100644 index 000000000..5af0f1590 --- /dev/null +++ b/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv @@ -0,0 +1,141 @@ +device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes +mac_m4pro,cactus,gemma_4_e2b,prefill,256,0,119.090000,585.440,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,decode,256,100,37.040000,586.000,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,512,0,187.990000,585.830,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,decode,512,100,35.940000,586.770,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,1024,0,262.570000,586.970,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,decode,1024,100,34.140000,586.580,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,2048,0,321.490000,586.380,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,decode,2048,100,32.920000,586.280,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,4096,0,356.670000,586.390,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,256,0,63.130000,75.470,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,decode,256,100,63.350000,75.470,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,512,0,62.680000,74.880,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,decode,512,100,62.240000,74.880,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,62.440000,75.580,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,decode,1024,100,60.640000,75.580,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,61.160000,75.190,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,decode,2048,100,58.050000,75.190,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,57.300000,76.380,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; uses fixed image fixture; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,parakeet_tdt_v3,parakeet_transcribe,,,0.049118,1196.127,777.969,778.112,ok,WER=0.000000; runner=native; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey +mac_m4pro,cactus,qwen3_vl_2b,prefill,256,0,36.350000,381.280,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,decode,256,100,35.280000,381.280,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,512,0,34.790000,381.300,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,decode,512,100,32.800000,381.300,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,1024,0,33.550000,381.170,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,decode,1024,100,30.620000,381.170,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,2048,0,30.830000,381.580,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,decode,2048,100,25.850000,381.580,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,4096,0,26.400000,381.850,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" +mac_m4pro,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,decode,256,100,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,decode,512,100,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,decode,1024,100,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,decode,2048,100,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,gemma_4_e2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,256,0,166.214015,2829.081,2427.657,2427.657,ok,"warmup_runs=2,measurement_runs=3; runner=executorch; threads=1; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,decode,256,100,51.452337,2829.081,2427.657,2427.657,ok,"warmup_runs=2,measurement_runs=3; runner=executorch; threads=1; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,512,0,168.461314,2835.378,2427.657,2427.657,ok,"warmup_runs=2,measurement_runs=3; runner=executorch; threads=1; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,decode,512,100,51.786664,2835.378,2427.657,2427.657,ok,"warmup_runs=2,measurement_runs=3; runner=executorch; threads=1; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,166.713309,2838.738,2427.657,2427.657,ok,"warmup_runs=2,measurement_runs=3; runner=executorch; threads=1; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,decode,1024,100,49.704966,2838.738,2427.657,2427.657,ok,"warmup_runs=2,measurement_runs=3; runner=executorch; threads=1; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,165.246954,2846.816,2427.657,2427.657,ok,"warmup_runs=2,measurement_runs=3; runner=executorch; threads=1; format=Q4 8da4w XNNPACK; prefill_only=true; shared_context=2048; shared_decode_tokens=100" +mac_m4pro,executorch,lfm_2_5_vl_1_6b,decode,2048,100,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export has KV cache capacity 2048; cannot decode 100 tokens from context 2048 because required cache positions extend to 2147 +mac_m4pro,executorch,lfm_2_5_vl_1_6b,prefill,4096,0,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export supports at most 2048 input tokens for token_embedding; requested prefill seqlen=4096 +mac_m4pro,executorch,parakeet_tdt_v3,parakeet_transcribe,,,0.279906,1664.158,719.224,719.224,ok,WER=0.000000; runner=executorch_parakeet; format=Q4 8da4w XNNPACK; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey +mac_m4pro,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,decode,256,100,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,decode,512,100,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,decode,1024,100,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,decode,2048,100,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,executorch,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Mac Qwen3 VL 2B is unsupported: ExecuTorch Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found no .pte or .ptd artifacts; available ExecuTorch Qwen repos are Qwen3 text-only 0.6B/1.7B/4B, not Qwen3-VL-2B; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/onnxruntime/Qwen3-VL-2B-Instruct-ONNX/onnxruntime/cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json)" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,256,0,72.410000,2606.333,2588.148,2588.148,ok,"warmup_runs=2,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; threads=1; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,decode,256,100,19.720000,2606.333,2588.148,2588.148,ok,"warmup_runs=2,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; threads=1; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,512,0,141.250000,2608.379,2588.148,2588.148,ok,"warmup_runs=2,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; threads=1; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,decode,512,100,19.290000,2608.379,2588.148,2588.148,ok,"warmup_runs=2,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; threads=1; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,1024,0,282.090000,2605.723,2588.148,2588.148,ok,"warmup_runs=2,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; threads=1; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,decode,1024,100,19.390000,2605.723,2588.148,2588.148,ok,"warmup_runs=2,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; threads=1; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,2048,0,287.840000,2605.864,2588.148,2588.148,ok,"warmup_runs=2,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; threads=1; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,decode,2048,100,19.160000,2605.864,2588.148,2588.148,ok,"warmup_runs=2,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; threads=1; quantization=int4_per_output_channel" +mac_m4pro,litert_lm,gemma_4_e2b,prefill,4096,0,,,2588.148,2588.148,error,"F0000 00:00:1779185326.775354 82584826 litert_lm_advanced_main.cc:246] Check failed: MainHelper(argc, argv) is OK (INVALID_ARGUMENT: Input token ids are too long. Exceeding the maximum number of tokens allowed: 4096 >= 4096)" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,decode,2048,100,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,4096,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" +mac_m4pro,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,0.045189,1016.705,667.588,667.590,ok,"WER=0.806452; runner=normal_litert_parakeet; chunks=6; encoder_frames=63,63,63,63,63,63; transcript=con returned to its place amidst the tents the english foaded two the french congratulations" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,decode,256,100,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,decode,512,100,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,decode,1024,100,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,decode,2048,100,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,litert_lm,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,256,0,26.134900,1544.176,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +mac_m4pro,llama_cpp,gemma_4_e2b,decode,256,100,12.704300,1544.176,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,512,0,24.820200,1604.598,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +mac_m4pro,llama_cpp,gemma_4_e2b,decode,512,100,12.582000,1604.598,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,1024,0,22.607500,1619.004,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +mac_m4pro,llama_cpp,gemma_4_e2b,decode,1024,100,12.041600,1619.004,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,2048,0,20.427700,1643.223,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +mac_m4pro,llama_cpp,gemma_4_e2b,decode,2048,100,11.489600,1643.223,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,gemma_4_e2b,prefill,4096,0,16.724600,1688.254,3106.736,3106.736,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,135.705000,742.611,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,decode,256,100,50.426600,742.611,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,129.094000,777.142,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,decode,512,100,48.927300,777.142,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,107.156000,785.924,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,decode,1024,100,39.812700,785.924,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,85.748600,800.033,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,decode,2048,100,40.581800,800.033,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,lfm_2_5_vl_1_6b,prefill,4096,0,73.122000,828.002,695.752,695.753,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_0" +mac_m4pro,llama_cpp,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,llama.cpp does not have a configured Parakeet TDT runner or artifact +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,256,0,35.395200,1214.393,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +mac_m4pro,llama_cpp,qwen3_vl_2b,decode,256,100,18.888400,1214.393,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,512,0,33.213900,1273.987,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +mac_m4pro,llama_cpp,qwen3_vl_2b,decode,512,100,17.749700,1273.987,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,1024,0,29.229300,1331.721,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +mac_m4pro,llama_cpp,qwen3_vl_2b,decode,1024,100,15.062900,1331.721,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,2048,0,24.137500,1439.674,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +mac_m4pro,llama_cpp,qwen3_vl_2b,decode,2048,100,10.687700,1439.674,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +mac_m4pro,llama_cpp,qwen3_vl_2b,prefill,4096,0,16.600000,1677.268,1107.410,1107.410,ok,"warmup_runs=2,measurement_runs=3; runner=native_llama_cpp_json; threads=1; mmap=on; n_gpu_layers=0; synthetic exact-length token ids; quantization=Q4_K_M" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,256,0,28.380413,1408.003,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +mac_m4pro,onnxruntime,gemma_4_e2b,decode,256,100,3.805683,1408.003,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,512,0,28.538463,1710.222,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +mac_m4pro,onnxruntime,gemma_4_e2b,decode,512,100,3.732831,1710.222,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,1024,0,28.333752,2268.128,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +mac_m4pro,onnxruntime,gemma_4_e2b,decode,1024,100,3.698919,2268.128,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,2048,0,26.969070,4359.177,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +mac_m4pro,onnxruntime,gemma_4_e2b,decode,2048,100,3.504154,4359.177,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,gemma_4_e2b,prefill,4096,0,26.131633,9827.462,3627.412,3627.412,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,147.337391,3026.081,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,decode,256,100,48.660199,3026.081,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,146.773416,2175.299,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,decode,512,100,46.493205,2175.299,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,144.077422,2565.956,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,decode,1024,100,42.986674,2565.956,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,140.196273,3228.394,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,decode,2048,100,38.713559,3228.394,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,lfm_2_5_vl_1_6b,prefill,4096,0,135.896664,5706.786,1754.691,1754.691,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4; synthetic exact-length token ids" +mac_m4pro,onnxruntime,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,"ONNX Runtime Parakeet TDT runner is not configured; available exact-model ONNX exports are split encoder plus decoder/joint ASR graphs, not the LLM paired prefill/decode contract used by this backend" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,256,0,91.784308,1287.205,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +mac_m4pro,onnxruntime,qwen3_vl_2b,decode,256,100,33.615546,1287.205,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,512,0,91.100134,1838.862,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +mac_m4pro,onnxruntime,qwen3_vl_2b,decode,512,100,29.627780,1838.862,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,1024,0,88.772262,3018.316,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +mac_m4pro,onnxruntime,qwen3_vl_2b,decode,1024,100,23.890315,3018.316,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,2048,0,84.573427,5204.630,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +mac_m4pro,onnxruntime,qwen3_vl_2b,decode,2048,100,16.363954,5204.630,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" +mac_m4pro,onnxruntime,qwen3_vl_2b,prefill,4096,0,78.219476,7917.085,1440.083,1440.085,ok,"warmup_runs=2,measurement_runs=3; runner=runtime_onnxruntime.py; provider=CPUExecutionProvider; threads=1; format=INT4/Q4 block=32; synthetic exact-length token ids" diff --git a/experiments/matrix/results/matrix_pixel_10a_full_core.csv b/experiments/matrix/results/matrix_pixel_10a_full_core.csv new file mode 100644 index 000000000..050687869 --- /dev/null +++ b/experiments/matrix/results/matrix_pixel_10a_full_core.csv @@ -0,0 +1,61 @@ +device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes +pixel_10a,cactus,gemma_4_e2b,prefill,256,0,10.510000,1596.372,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,gemma_4_e2b,prefill,512,0,10.030000,1596.384,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,gemma_4_e2b,prefill,1024,0,10.100000,1596.690,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,gemma_4_e2b,prefill,2048,0,9.750000,1597.104,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,256,0,10.650000,1227.073,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,512,0,11.070000,1227.092,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,10.960000,1227.420,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,10.780000,1227.182,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,256,0,5.970000,1976.169,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,512,0,5.890000,1976.275,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,1024,0,5.900000,1976.448,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,2048,0,5.760000,1976.136,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,256,0,55.794200,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,512,0,86.118300,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,123.841000,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,124.219000,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +pixel_10a,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,litert_lm,gemma_4_e2b,prefill,256,0,27.010000,3386.180,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +pixel_10a,litert_lm,gemma_4_e2b,prefill,512,0,57.200000,3378.180,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +pixel_10a,litert_lm,gemma_4_e2b,prefill,1024,0,84.910000,3363.700,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +pixel_10a,litert_lm,gemma_4_e2b,prefill,2048,0,89.450000,3367.510,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,llama_cpp,gemma_4_e2b,prefill,256,0,21.059300,3108.870,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,gemma_4_e2b,prefill,512,0,20.226400,3177.969,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,gemma_4_e2b,prefill,1024,0,15.655200,3190.514,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,gemma_4_e2b,prefill,2048,0,15.181000,3213.866,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,29.738200,722.176,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,30.228800,756.379,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,28.614700,763.805,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,27.632600,777.173,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,256,0,22.897300,1187.265,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,512,0,20.419300,1250.273,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,1024,0,19.414900,1308.021,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,2048,0,19.173800,1415.186,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,256,0,25.808400,1601.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,512,0,23.110300,1761.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,1024,0,18.672300,2136.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,2048,0,17.540100,3135.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,112.184000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,120.816000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,122.030000,2370.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,118.489000,3233.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,256,0,86.648800,1266.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,512,0,88.170000,1542.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,1024,0,92.428900,2117.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,2048,0,77.079800,3604.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" diff --git a/experiments/matrix/results/matrix_pixel_10a_single_thread.csv b/experiments/matrix/results/matrix_pixel_10a_single_thread.csv new file mode 100644 index 000000000..b0ea9821a --- /dev/null +++ b/experiments/matrix/results/matrix_pixel_10a_single_thread.csv @@ -0,0 +1,126 @@ +device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes +pixel_10a,cactus,gemma_4_e2b,prefill,256,0,9.210000,1623.354,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,cactus,gemma_4_e2b,decode,256,100,3.670000,1623.354,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,cactus,gemma_4_e2b,prefill,512,0,11.010000,1623.244,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,cactus,gemma_4_e2b,decode,512,100,3.310000,1623.244,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,cactus,gemma_4_e2b,prefill,1024,0,12.660000,1623.220,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,cactus,gemma_4_e2b,decode,1024,100,3.090000,1623.220,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +pixel_10a,cactus,gemma_4_e2b,prefill,2048,0,13.580000,1623.345,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,cactus,gemma_4_e2b,decode,2048,100,2.660000,1623.345,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,256,0,4.640000,654.183,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,256,100,4.500000,654.183,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,512,0,4.530000,654.177,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,512,100,4.360000,654.177,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,4.440000,654.778,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,1024,100,4.120000,654.778,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,4.180000,654.908,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,2048,100,3.750000,654.908,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +pixel_10a,cactus,parakeet_tdt_v3,parakeet_transcribe,,,0.771500,2463.964,777.969,778.112,ok,WER=0.000000; runner=android_transpiled_tdt; core_affinity=cpu7; taskset_mask=80; serial=5C111JEA320125; android=16; thermal_status=0; decoder_steps=152; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey; thread_count=1 +pixel_10a,cactus,qwen3_vl_2b,prefill,256,0,3.380000,1210.687,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,cactus,qwen3_vl_2b,decode,256,100,3.180000,1210.687,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,cactus,qwen3_vl_2b,prefill,512,0,3.230000,1210.468,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,cactus,qwen3_vl_2b,decode,512,100,3.010000,1210.468,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,cactus,qwen3_vl_2b,prefill,1024,0,3.060000,1210.529,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,cactus,qwen3_vl_2b,decode,1024,100,2.700000,1210.529,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +pixel_10a,cactus,qwen3_vl_2b,prefill,2048,0,2.750000,1210.533,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,cactus,qwen3_vl_2b,decode,2048,100,2.300000,1210.533,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +pixel_10a,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,decode,2048,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,256,0,64.486400,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true" +pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,256,100,3.582790,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,512,0,79.262100,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true" +pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,512,100,3.574530,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,81.952200,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true" +pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,1024,100,3.291510,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,84.928700,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; prefill_only=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,2048,100,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export has KV cache capacity 2048; cannot decode 100 tokens from context 2048 because required cache positions extend to 2147 +pixel_10a,executorch,parakeet_tdt_v3,parakeet_transcribe,,,0.281650,1509.574,719.224,719.224,ok,WER=0.000000; runner=android_executorch_parakeet; format=Q4 8da4w XNNPACK; core_affinity=cpu7; taskset_mask=80; serial=5C111JEA320125; android=16; thermal_status=1; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey +pixel_10a,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,decode,2048,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,litert_lm,gemma_4_e2b,prefill,256,0,17.330000,3390.210,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,decode,256,100,8.410000,3390.210,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,prefill,512,0,39.530000,3383.900,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,decode,512,100,8.010000,3383.900,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,prefill,1024,0,78.300000,3373.320,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,decode,1024,100,8.840000,3373.320,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,prefill,2048,0,61.040000,3373.440,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,decode,2048,100,8.510000,3373.440,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,0.129975,1287.174,667.588,667.590,ok,WER=0.790323; runner=android_litert_parakeet_asr; chunks=6; encoder_frames=378; decoder_steps=179; core_affinity=cpu7; taskset_mask=80; serial=5C111JEA320125; android=16; thermal_status=1; transcript=con returned to its place amidst the tents the english fought at two the french and the congratulations +pixel_10a,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,llama_cpp,gemma_4_e2b,prefill,256,0,7.574750,3117.683,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,llama_cpp,gemma_4_e2b,decode,256,100,5.126740,3117.683,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,llama_cpp,gemma_4_e2b,prefill,512,0,6.500580,3181.424,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,llama_cpp,gemma_4_e2b,decode,512,100,4.069490,3181.424,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,llama_cpp,gemma_4_e2b,prefill,1024,0,6.719770,3193.352,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,llama_cpp,gemma_4_e2b,decode,1024,100,4.177220,3193.352,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +pixel_10a,llama_cpp,gemma_4_e2b,prefill,2048,0,6.400140,3216.722,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,llama_cpp,gemma_4_e2b,decode,2048,100,3.757320,3216.722,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,11.494200,722.785,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,256,100,8.817700,722.785,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,11.139800,757.023,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,512,100,8.640550,757.023,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,11.064900,764.609,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,1024,100,8.369130,764.609,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,10.929200,777.979,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,2048,100,7.819560,777.979,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +pixel_10a,llama_cpp,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,llama.cpp does not have a configured Parakeet TDT runner or artifact +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,256,0,9.659300,1188.873,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,llama_cpp,qwen3_vl_2b,decode,256,100,5.750710,1188.873,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,512,0,9.500390,1251.921,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,llama_cpp,qwen3_vl_2b,decode,512,100,5.510410,1251.921,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,1024,0,8.884470,1309.535,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,llama_cpp,qwen3_vl_2b,decode,1024,100,4.948920,1309.535,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,2048,0,8.311190,1417.267,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,llama_cpp,qwen3_vl_2b,decode,2048,100,4.193310,1417.267,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,256,0,14.488300,1609.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,onnxruntime,gemma_4_e2b,decode,256,100,1.465220,1609.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,512,0,12.055700,1769.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,onnxruntime,gemma_4_e2b,decode,512,100,1.438240,1769.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,1024,0,11.659800,2144.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,onnxruntime,gemma_4_e2b,decode,1024,100,1.338380,2144.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,2048,0,10.320800,3156.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,onnxruntime,gemma_4_e2b,decode,2048,100,1.236880,3156.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,52.833400,2155.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,256,100,13.406000,2155.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,57.754000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,512,100,13.007000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,51.452000,2380.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,1024,100,12.617700,2380.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,47.859300,3235.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,2048,100,10.108500,3235.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,ONNX Runtime Parakeet TDT Android artifact and ASR runner are not configured +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,256,0,48.712800,1296.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" +pixel_10a,onnxruntime,qwen3_vl_2b,decode,256,100,9.224760,1296.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,512,0,46.142200,1585.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" +pixel_10a,onnxruntime,qwen3_vl_2b,decode,512,100,9.544100,1585.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,1024,0,39.708000,2165.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" +pixel_10a,onnxruntime,qwen3_vl_2b,decode,1024,100,6.351120,2165.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,2048,0,31.953900,3612.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,onnxruntime,qwen3_vl_2b,decode,2048,100,4.031710,3612.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" From e89117b312be2f6018c534406ab891377985d52a Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Tue, 19 May 2026 16:46:14 -0700 Subject: [PATCH 03/12] Add Mac Gemma chunk prefill results Signed-off-by: Noah Cylich --- ...rix_mac_m4pro_cactus_gemma4_chunk_prefill.csv | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 experiments/matrix/results/matrix_mac_m4pro_cactus_gemma4_chunk_prefill.csv diff --git a/experiments/matrix/results/matrix_mac_m4pro_cactus_gemma4_chunk_prefill.csv b/experiments/matrix/results/matrix_mac_m4pro_cactus_gemma4_chunk_prefill.csv new file mode 100644 index 000000000..53c7fe406 --- /dev/null +++ b/experiments/matrix/results/matrix_mac_m4pro_cactus_gemma4_chunk_prefill.csv @@ -0,0 +1,16 @@ +device,runtime,model,chunk_size,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes +mac_m4pro,cactus,gemma_4_e2b,c128,prefill,256,0,442.380000,585.770,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c128,prefill,512,0,463.330000,586.780,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c128,prefill,1024,0,459.850000,586.300,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c128,prefill,2048,0,437.370000,586.350,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c128,prefill,4096,0,415.940000,586.020,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c256,prefill,256,0,735.550000,1415.990,1893.472,1893.620,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c256; provider=cpu; gpu=disabled; cactus_chunk_size=c256; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c256,prefill,512,0,794.570000,1423.360,1893.472,1893.620,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c256; provider=cpu; gpu=disabled; cactus_chunk_size=c256; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c256,prefill,1024,0,796.010000,1422.630,1893.472,1893.620,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c256; provider=cpu; gpu=disabled; cactus_chunk_size=c256; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c256,prefill,2048,0,756.270000,1423.820,1893.472,1893.620,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c256; provider=cpu; gpu=disabled; cactus_chunk_size=c256; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c256,prefill,4096,0,715.150000,1422.800,1893.472,1893.620,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c256; provider=cpu; gpu=disabled; cactus_chunk_size=c256; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c512,prefill,256,0,38.800000,247.880,1893.463,1893.610,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c512; provider=cpu; gpu=disabled; cactus_chunk_size=c512; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c512,prefill,512,0,805.890000,2739.520,1893.463,1893.610,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c512; provider=cpu; gpu=disabled; cactus_chunk_size=c512; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c512,prefill,1024,0,878.710000,2759.820,1893.463,1893.610,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c512; provider=cpu; gpu=disabled; cactus_chunk_size=c512; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c512,prefill,2048,0,842.170000,2760.430,1893.463,1893.610,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c512; provider=cpu; gpu=disabled; cactus_chunk_size=c512; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,c512,prefill,4096,0,803.560000,2758.820,1893.463,1893.610,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv-c512; provider=cpu; gpu=disabled; cactus_chunk_size=c512; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" From af613af4a9d2404fda2b0d974666a046f1df60a0 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Tue, 19 May 2026 16:49:42 -0700 Subject: [PATCH 04/12] Fix full-window KV cache eviction Signed-off-by: Noah Cylich --- cactus-graph/src/ops_cache.cpp | 55 ++++++++++++++++++++++++------- cactus-graph/tests/test_cache.cpp | 27 +++++++++++++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 5491f541a..c309f257a 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -143,17 +143,33 @@ void compute_kv_cache_append_node( size_t new_total = current_len + new_seq_len; bool needs_eviction = new_total > window; if (needs_eviction) { - size_t remaining = window - sink - new_seq_len; + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) { + if (tail_capacity > 0) { + size_t source_offset = new_seq_len - tail_capacity; + std::memcpy( + fp16_base + keep_sink * stride, + source + source_offset * stride, + tail_capacity * stride * sizeof(__fp16)); + } + meta->current_seq_len = keep_sink + tail_capacity; + *node.output_buffer.data_as() = static_cast(meta->current_seq_len); + return; + } + + size_t remaining = tail_capacity - new_seq_len; + remaining = std::min(remaining, current_len - keep_sink); size_t shift_src = current_len - remaining; - if (remaining > 0 && shift_src > sink) { + if (remaining > 0 && shift_src > keep_sink) { std::memmove( - fp16_base + sink * stride, + fp16_base + keep_sink * stride, fp16_base + shift_src * stride, remaining * stride * sizeof(__fp16)); } - size_t append_offset = window - new_seq_len; + size_t append_offset = keep_sink + remaining; std::memcpy(fp16_base + append_offset * stride, source, new_seq_len * stride * sizeof(__fp16)); - meta->current_seq_len = window; + meta->current_seq_len = append_offset + new_seq_len; } else { std::memcpy(fp16_base + current_len * stride, source, new_seq_len * stride * sizeof(__fp16)); meta->current_seq_len = new_total; @@ -173,26 +189,43 @@ void compute_kv_cache_append_node( bool needs_eviction = new_total > window; if (needs_eviction) { - size_t remaining = window - sink - new_seq_len; + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) { + if (tail_capacity > 0) { + size_t source_offset = new_seq_len - tail_capacity; + cactus_quantize_kv_fp16_to_int8( + new_kv.data_as<__fp16>() + source_offset * int8_stride, + int8_base + keep_sink * int8_stride, + scale_base + keep_sink * scale_stride, + tail_capacity, kv_heads, hdim); + } + meta->current_seq_len = keep_sink + tail_capacity; + *node.output_buffer.data_as() = static_cast(meta->current_seq_len); + return; + } + + size_t remaining = tail_capacity - new_seq_len; + remaining = std::min(remaining, current_len - keep_sink); size_t shift_src = current_len - remaining; - if (remaining > 0 && shift_src > sink) { - std::memmove(int8_base + sink * int8_stride, + if (remaining > 0 && shift_src > keep_sink) { + std::memmove(int8_base + keep_sink * int8_stride, int8_base + shift_src * int8_stride, remaining * int8_stride); - std::memmove(scale_base + sink * scale_stride, + std::memmove(scale_base + keep_sink * scale_stride, scale_base + shift_src * scale_stride, remaining * scale_stride * sizeof(float)); } - size_t append_offset = window - new_seq_len; + size_t append_offset = keep_sink + remaining; cactus_quantize_kv_fp16_to_int8( new_kv.data_as<__fp16>(), int8_base + append_offset * int8_stride, scale_base + append_offset * scale_stride, new_seq_len, kv_heads, hdim); - meta->current_seq_len = window; + meta->current_seq_len = append_offset + new_seq_len; } else { cactus_quantize_kv_fp16_to_int8( new_kv.data_as<__fp16>(), diff --git a/cactus-graph/tests/test_cache.cpp b/cactus-graph/tests/test_cache.cpp index 38947597f..21cbdca7d 100644 --- a/cactus-graph/tests/test_cache.cpp +++ b/cactus-graph/tests/test_cache.cpp @@ -163,6 +163,32 @@ bool test_kv_cache_append_eviction() { return true; } +bool test_kv_cache_append_full_window_eviction() { + CactusGraph g; + + const size_t kv_heads = 1, head_dim = 16; + const size_t window = 8, sink = 2; + + size_t cache_node = g.kv_cache_state(window, kv_heads, head_dim, window, sink); + + for (int pass = 0; pass < 2; pass++) { + if (pass > 0) g.soft_reset(); + const size_t tokens = window; + const size_t elements = tokens * kv_heads * head_dim; + size_t kv_input = g.input({elements}, Precision::FP16); + std::vector<__fp16> data(elements, static_cast<__fp16>(static_cast(pass + 1))); + g.set_input(kv_input, data.data(), Precision::FP16); + g.kv_cache_append(kv_input, cache_node, window, sink); + g.execute(); + + auto* raw = static_cast(g.get_output(cache_node)); + uint64_t seq = *reinterpret_cast(raw); + if (seq != window) return false; + } + + return true; +} + bool test_attention_cached_basic() { const size_t b = 1, s = 1, h = 2, kv = 2, d = 16; const size_t max_seq = 64; @@ -493,6 +519,7 @@ int main() { runner.run_test("KV Cache Append Basic", test_kv_cache_append_basic()); runner.run_test("KV Cache Append Multiple", test_kv_cache_append_multiple()); runner.run_test("KV Cache Append Eviction", test_kv_cache_append_eviction()); + runner.run_test("KV Cache Append Full Window Eviction", test_kv_cache_append_full_window_eviction()); runner.run_test("Attention Cached Basic", test_attention_cached_basic()); runner.run_test("Attention Cached Multistep", test_attention_cached_multistep()); runner.run_test("KV Cache Invalidate", test_kv_cache_invalidate()); From ea5604a98d113d755ab401aca8ce0004dd386a6b Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Tue, 19 May 2026 21:59:15 -0700 Subject: [PATCH 05/12] Fix Gemma4 vision projection precision --- .../cactus/convert/model_adapters/policy.py | 12 ++- python/cactus/transpile/model_adapters.py | 5 +- python/tests/test_gemma4_vision_precision.py | 90 +++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 python/tests/test_gemma4_vision_precision.py diff --git a/python/cactus/convert/model_adapters/policy.py b/python/cactus/convert/model_adapters/policy.py index 118683495..6c0aebd71 100644 --- a/python/cactus/convert/model_adapters/policy.py +++ b/python/cactus/convert/model_adapters/policy.py @@ -24,6 +24,8 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("ignored", "none", None, component, False, "none", "no output filename") if "norm" in out.lower(): return TensorPolicy("fallback", "FP16", None, component, False, "none", "norm tensor") + if family == "gemma4" and name == "model.embed_vision.embedding_projection.weight": + return TensorPolicy("convert", f"CQ{user_bits}", user_bits, component, False, "hadamard", "vision embedding projection has no stable Hessian target") if family in {"parakeet", "parakeet_tdt"} and out.endswith(".bias") and ( "conv_" in out or out.startswith("subsampling_") or out.startswith("ctc_head_") ): @@ -32,18 +34,20 @@ def policy_for_tensor(match: NameMatch, shape: tuple[int, ...], user_bits: int, return TensorPolicy("fallback", "FP16", None, component, False, "none", "lstm recurrent tensor") if family == "parakeet_tdt" and out.startswith("tdt_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "tdt decoder tensor") + if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") + if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "depthwise conv tensor") if family in {"parakeet", "parakeet_tdt"} and out.startswith("layer_") and ( "conv_pointwise" in out or "conv_depthwise" in out ): return TensorPolicy("fallback", "FP16", None, component, False, "none", "conformer conv tensor") if family == "whisper" and out.startswith("encoder.layer_"): return TensorPolicy("fallback", "FP16", None, component, False, "none", "whisper encoder tensor") + if family == "gemma4" and component == "audio" and (name.endswith(".bias") or out.endswith(".bias") or ".bias." in out): + return TensorPolicy("fallback", "INT8", 8, component, False, "none", "bias tensor") if name.endswith(".bias") or out.endswith(".bias") or ".bias." in out: return TensorPolicy("fallback", "FP16", None, component, False, "none", "bias tensor") - if family in {"parakeet", "parakeet_tdt"} and "conv_pointwise" in out and len(shape) == 3 and shape[2] == 1: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "pointwise conv tensor") - if "conv_depthwise.weights" in out and len(shape) == 3 and shape[1] == 1: - return TensorPolicy("fallback", "INT8", 8, component, False, "none", "depthwise conv tensor") if len(shape) != 2: return TensorPolicy("fallback", "FP16", None, component, False, "none", "non-2d tensor") if "position_embedding" in out.lower() or "pos_embed" in out.lower() or "embed_positions" in out.lower(): diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index acd578d59..914f693f5 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -857,10 +857,11 @@ def _gemma4_compute_native_like_image_features( projection = getattr(embed_vision, "embedding_projection", None) if not isinstance(projection, torch.nn.Linear): raise TypeError("Gemma4 vision embedder is missing embedding_projection") + pooled_hidden = pooled_hidden.to(dtype=projection.weight.dtype) projected = F.linear( pooled_hidden, - projection.weight.float(), - None if projection.bias is None else projection.bias.float(), + projection.weight, + projection.bias, ) if post_proj_norm_weight is not None and post_proj_norm_weight.numel() > 0: return _gemma4_rms_norm( diff --git a/python/tests/test_gemma4_vision_precision.py b/python/tests/test_gemma4_vision_precision.py new file mode 100644 index 000000000..8a13766f5 --- /dev/null +++ b/python/tests/test_gemma4_vision_precision.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import torch + +import cactus.transpile.model_adapters as model_adapters +from cactus.transpile.capture_pytorch import capture_model + + +class _PatchEmbedder(torch.nn.Module): + def forward( + self, + pixel_values: torch.Tensor, + pixel_position_ids: torch.Tensor, + padding_positions: torch.Tensor, + ) -> torch.Tensor: + return pixel_values + + +class _VisionTower(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.patch_embedder = _PatchEmbedder() + self.encoder = torch.nn.Identity() + + +class _EmbedVision(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.embedding_projection = torch.nn.Linear(3, 2).half() + self.eps = 1e-6 + + +class _Backbone(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.vision_tower = _VisionTower() + self.embed_vision = _EmbedVision() + + +class _NativeLikeVisionWrapper(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.backbone = _Backbone() + + def forward(self, pixel_values: torch.Tensor, pixel_position_ids: torch.Tensor) -> torch.Tensor: + return model_adapters._gemma4_compute_native_like_image_features( + self.backbone, + pixel_values, + pixel_position_ids, + ) + + +def test_gemma4_native_like_vision_projection_keeps_weight_precision(monkeypatch) -> None: + def fake_hidden_states( + _vision_encoder: torch.nn.Module, + inputs_embeds: torch.Tensor, + _attention_mask: torch.Tensor, + _pixel_position_ids: torch.Tensor, + ) -> torch.Tensor: + return inputs_embeds + + def fake_pool( + _vision_tower: torch.nn.Module, + vision_hidden: torch.Tensor, + _pixel_position_ids: torch.Tensor, + **_kwargs: object, + ) -> torch.Tensor: + return vision_hidden.reshape(-1, vision_hidden.shape[-1]).float() + + monkeypatch.setattr(model_adapters, "_gemma4_vision_encoder_hidden_states", fake_hidden_states) + monkeypatch.setattr(model_adapters, "_gemma4_pool_vision_hidden_native_like", fake_pool) + + captured = capture_model( + _NativeLikeVisionWrapper().eval(), + ( + torch.randn(1, 2, 3, dtype=torch.float16), + torch.zeros(1, 2, 2, dtype=torch.long), + ), + strict=False, + ) + + bad_casts = [] + for node_id in captured.ir_graph.order: + node = captured.ir_graph.nodes[node_id] + if node.op != "precision_cast": + continue + if any("embed_vision_embedding_projection" in input_id for input_id in node.inputs): + bad_casts.append(node.id) + + assert bad_casts == [] From 71e9f4f1282a7dd3567444280350052b89bd1d4e Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Tue, 19 May 2026 22:40:31 -0700 Subject: [PATCH 06/12] final android, long context = 4096 numbers --- .../matrix_galaxy_s24_or_s25_full_core.csv | 15 +++ ...matrix_galaxy_s24_or_s25_single_thread.csv | 105 ++++++++++-------- .../results/matrix_pixel_10a_full_core.csv | 15 +++ .../matrix_pixel_10a_single_thread.csv | 105 ++++++++++-------- 4 files changed, 150 insertions(+), 90 deletions(-) diff --git a/experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv b/experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv index 027f84846..1bf222289 100644 --- a/experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv +++ b/experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv @@ -3,59 +3,74 @@ galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,256,0,134.290000,1613.160,1893.450, galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,512,0,119.260000,1592.599,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,1024,0,90.890000,1593.312,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,2048,0,87.180000,1596.804,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,4096,0,93.160000,1607.806,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,256,0,58.380000,1224.530,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,512,0,58.860000,1224.609,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,58.210000,1226.157,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,57.470000,1225.903,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,51.860000,1237.481,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,256,0,52.400000,1976.993,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,512,0,52.090000,1977.999,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,1024,0,48.280000,1977.347,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,2048,0,43.520000,1976.959,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,4096,0,44.180000,1986.758,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,256,0,69.638400,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,512,0,119.321000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,181.958000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,234.322000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,4096,0,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export supports at most 2048 input tokens for token_embedding; requested prefill seqlen=4096 galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,256,0,57.920000,3499.900,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,512,0,127.760000,3497.240,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,1024,0,195.370000,3495.300,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,2048,0,223.510000,3490.750,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,4096,0,284.020000,3490.030,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,4096,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,256,0,13.453600,3116.622,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,512,0,12.083600,3179.782,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,1024,0,16.327400,3192.851,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,2048,0,19.124900,3215.312,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,4096,0,22.371600,3257.274,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,20.150500,724.741,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,21.435900,759.907,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,31.473600,768.084,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,42.476600,778.535,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,4096,0,43.791300,810.154,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_0" galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,256,0,18.710500,1190.149,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,512,0,15.800400,1252.039,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,1024,0,23.243800,1310.097,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,2048,0,32.747000,1416.436,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; prefill_only=true; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,4096,0,29.123200,1653.144,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=2; synthetic exact-length token ids; quantization=Q4_K_M" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,256,0,29.693100,1627.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,512,0,26.433400,1796.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,1024,0,33.878900,2225.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,2048,0,35.691600,3402.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,4096,0,29.496700,6574.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,112.955000,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,166.089000,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,146.154000,2450.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,156.402000,3504.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,4096,0,145.372000,7759.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,256,0,161.673000,1281.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,512,0,169.303000,1570.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,1024,0,168.010000,2194.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,2048,0,102.698000,3874.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,4096,0,97.477500,7454.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" diff --git a/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv b/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv index bb0344844..c7ab6307b 100644 --- a/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv +++ b/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv @@ -1,126 +1,141 @@ device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,256,0,33.520000,1626.336,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,256,100,12.680000,1626.336,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,512,0,43.150000,1626.239,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,512,100,12.060000,1626.239,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,1024,0,54.070000,1626.288,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,1024,100,11.170000,1626.288,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,2048,0,56.020000,1626.360,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,gemma_4_e2b,prefill,4096,0,80.360000,1596.133,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,256,100,12.680000,1626.336,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,512,100,12.060000,1626.239,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,1024,100,11.170000,1626.288,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,2048,100,10.340000,1626.360,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,256,0,20.200000,666.358,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,256,100,19.350000,666.358,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,512,0,20.060000,666.320,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,512,100,18.800000,666.320,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,16.370000,666.346,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,1024,100,14.760000,666.346,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,15.070000,666.470,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,38.730000,1226.418,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,256,100,19.350000,666.358,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,512,100,18.800000,666.320,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,1024,100,14.760000,666.346,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,2048,100,13.720000,666.470,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,parakeet_tdt_v3,parakeet_transcribe,,,0.200471,2475.970,777.969,778.112,ok,WER=0.000000; runner=android_transpiled_tdt; core_affinity=cpu7; taskset_mask=80; serial=RFGL42B1VLW; android=16; thermal_status=1; decoder_steps=152; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey; thread_count=1 galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,256,0,11.620000,1224.077,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,256,100,11.060000,1224.077,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,512,0,11.250000,1223.396,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,512,100,10.620000,1223.396,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,1024,0,10.550000,1223.269,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,1024,100,9.160000,1223.269,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,2048,0,9.420000,1224.050,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,4096,0,29.890000,1977.018,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,256,100,11.060000,1224.077,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,512,100,10.620000,1223.396,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,1024,100,9.160000,1223.269,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,2048,100,7.520000,1224.050,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,gemma_4_e2b,decode,2048,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,256,0,94.300200,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true" -galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,256,100,3.173630,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,512,0,137.059000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true" -galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,512,100,3.258580,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,149.938000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true" -galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,1024,100,3.055200,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,122.384000,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; prefill_only=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,prefill,4096,0,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export supports at most 2048 input tokens for token_embedding; requested prefill seqlen=4096 +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,256,100,3.173630,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,512,100,3.258580,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,1024,100,3.055200,1137.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" galaxy_s24_or_s25,executorch,lfm_2_5_vl_1_6b,decode,2048,100,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export has KV cache capacity 2048; cannot decode 100 tokens from context 2048 because required cache positions extend to 2147 galaxy_s24_or_s25,executorch,parakeet_tdt_v3,parakeet_transcribe,,,0.215333,1506.785,719.224,719.224,ok,WER=0.000000; runner=android_executorch_parakeet; format=Q4 8da4w XNNPACK; core_affinity=cpu7; taskset_mask=80; serial=RFGL42B1VLW; android=16; thermal_status=1; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,executorch,qwen3_vl_2b,decode,2048,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,256,0,33.570000,3502.820,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" -galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,256,100,10.930000,3502.820,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,512,0,69.420000,3497.600,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" -galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,512,100,11.480000,3497.600,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,1024,0,142.190000,3489.960,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" -galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,1024,100,11.750000,3489.960,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,2048,0,144.750000,3490.270,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,prefill,4096,0,138.320000,3490.860,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=strict; runner=litert_lm; provider=cpu; gpu=disabled; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,256,100,10.930000,3502.820,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,512,100,11.480000,3497.600,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,1024,100,11.750000,3489.960,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" galaxy_s24_or_s25,litert_lm,gemma_4_e2b,decode,2048,100,11.530000,3490.270,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm -galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm -galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm -galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,prefill,4096,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm galaxy_s24_or_s25,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,0.071528,1285.852,667.588,667.590,ok,WER=0.806452; runner=android_litert_parakeet_asr; chunks=6; encoder_frames=378; decoder_steps=177; core_affinity=cpu7; taskset_mask=80; serial=RFGL42B1VLW; android=16; thermal_status=1; transcript=con returned to its place amidst the tents the english foaded two the french congratulations galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm -galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm -galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm -galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,256,0,7.636660,3118.732,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,256,100,5.812560,3118.732,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,512,0,8.088930,3184.755,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,512,100,5.944650,3184.755,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,1024,0,8.150660,3195.753,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,1024,100,5.935540,3195.753,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,2048,0,8.062440,3218.219,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,prefill,4096,0,7.344820,3257.243,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=strict; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,256,100,5.812560,3118.732,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,512,100,5.944650,3184.755,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,1024,100,5.935540,3195.753,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,gemma_4_e2b,decode,2048,100,5.518090,3218.219,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,16.236700,723.342,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,256,100,13.149400,723.342,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,17.256000,760.725,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,512,100,13.279900,760.725,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,17.409700,766.848,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,1024,100,13.038800,766.848,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,17.474600,781.181,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,prefill,4096,0,15.291500,807.133,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=strict; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_0" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,256,100,13.149400,723.342,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,512,100,13.279900,760.725,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,1024,100,13.038800,766.848,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,lfm_2_5_vl_1_6b,decode,2048,100,12.794700,781.181,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,llama.cpp does not have a configured Parakeet TDT runner or artifact galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,256,0,12.016400,1191.867,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,256,100,8.270410,1191.867,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,512,0,12.504600,1253.624,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,512,100,8.192990,1253.624,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,1024,0,12.650800,1311.758,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,1024,100,7.561010,1311.758,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,2048,0,12.040200,1417.365,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,prefill,4096,0,10.308000,1652.887,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=strict; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,256,100,8.270410,1191.867,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,512,100,8.192990,1253.624,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,1024,100,7.561010,1311.758,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,llama_cpp,qwen3_vl_2b,decode,2048,100,5.878280,1417.365,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=RFGL42B1VLW; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,256,0,18.213300,1617.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,256,100,1.663560,1617.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,512,0,17.986700,1782.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,512,100,1.631470,1782.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,1024,0,17.626400,2182.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,1024,100,1.591130,2182.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,2048,0,16.546700,3291.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,prefill,4096,0,14.316300,6346.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=strict; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,256,100,1.663560,1617.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,512,100,1.631470,1782.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,1024,100,1.591130,2182.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,gemma_4_e2b,decode,2048,100,1.471690,3291.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,92.588500,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,256,100,22.606100,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,93.125200,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,512,100,21.820600,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,89.599600,2402.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,1024,100,20.428700,2402.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,89.010700,3277.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,prefill,4096,0,76.975400,7010.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=strict; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,256,100,22.606100,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,512,100,21.820600,2161.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,1024,100,20.428700,2402.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,lfm_2_5_vl_1_6b,decode,2048,100,17.192900,3277.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,ONNX Runtime Parakeet TDT Android artifact and ASR runner are not configured galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,256,0,57.884400,1303.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,256,100,15.374200,1303.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,512,0,58.630400,1596.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,512,100,13.419100,1596.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,1024,0,57.203700,2181.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,1024,100,9.827840,2181.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,2048,0,54.185400,3654.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,prefill,4096,0,49.628100,7326.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=strict; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,256,100,15.374200,1303.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,512,100,13.419100,1596.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,1024,100,9.827840,2181.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" galaxy_s24_or_s25,onnxruntime,qwen3_vl_2b,decode,2048,100,6.592570,3654.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" diff --git a/experiments/matrix/results/matrix_pixel_10a_full_core.csv b/experiments/matrix/results/matrix_pixel_10a_full_core.csv index 050687869..8c0492d20 100644 --- a/experiments/matrix/results/matrix_pixel_10a_full_core.csv +++ b/experiments/matrix/results/matrix_pixel_10a_full_core.csv @@ -3,59 +3,74 @@ pixel_10a,cactus,gemma_4_e2b,prefill,256,0,10.510000,1596.372,1893.450,1893.591, pixel_10a,cactus,gemma_4_e2b,prefill,512,0,10.030000,1596.384,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,gemma_4_e2b,prefill,1024,0,10.100000,1596.690,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,gemma_4_e2b,prefill,2048,0,9.750000,1597.104,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,gemma_4_e2b,prefill,4096,0,9.240000,1592.893,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,256,0,10.650000,1227.073,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,512,0,11.070000,1227.092,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,10.960000,1227.420,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,10.780000,1227.182,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,10.600000,1223.487,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,qwen3_vl_2b,prefill,256,0,5.970000,1976.169,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,qwen3_vl_2b,prefill,512,0,5.890000,1976.275,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,qwen3_vl_2b,prefill,1024,0,5.900000,1976.448,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,qwen3_vl_2b,prefill,2048,0,5.760000,1976.136,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,4096,0,5.770000,1969.446,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,gemma_4_e2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,256,0,55.794200,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,512,0,86.118300,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,123.841000,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,124.219000,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=executorch; provider=cpu_xnnpack; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=Q4 8da4w XNNPACK" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,4096,0,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export supports at most 2048 input tokens for token_embedding; requested prefill seqlen=4096 pixel_10a,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" +pixel_10a,executorch,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,litert_lm,gemma_4_e2b,prefill,256,0,27.010000,3386.180,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" pixel_10a,litert_lm,gemma_4_e2b,prefill,512,0,57.200000,3378.180,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" pixel_10a,litert_lm,gemma_4_e2b,prefill,1024,0,84.910000,3363.700,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" pixel_10a,litert_lm,gemma_4_e2b,prefill,2048,0,89.450000,3367.510,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" +pixel_10a,litert_lm,gemma_4_e2b,prefill,4096,0,124.860000,3369.260,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; runner=litert_lm; provider=cpu; gpu=disabled; threads=8; thread_count=8; quantization=int4_per_output_channel; affinity=not_set; taskset_mask=none" pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,4096,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none pixel_10a,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none pixel_10a,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none pixel_10a,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none pixel_10a,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none +pixel_10a,litert_lm,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none pixel_10a,llama_cpp,gemma_4_e2b,prefill,256,0,21.059300,3108.870,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" pixel_10a,llama_cpp,gemma_4_e2b,prefill,512,0,20.226400,3177.969,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" pixel_10a,llama_cpp,gemma_4_e2b,prefill,1024,0,15.655200,3190.514,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" pixel_10a,llama_cpp,gemma_4_e2b,prefill,2048,0,15.181000,3213.866,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,gemma_4_e2b,prefill,4096,0,14.338300,3257.068,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,29.738200,722.176,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,30.228800,756.379,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,28.614700,763.805,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,27.632600,777.173,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,4096,0,26.957200,806.471,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" pixel_10a,llama_cpp,qwen3_vl_2b,prefill,256,0,22.897300,1187.265,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" pixel_10a,llama_cpp,qwen3_vl_2b,prefill,512,0,20.419300,1250.273,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" pixel_10a,llama_cpp,qwen3_vl_2b,prefill,1024,0,19.414900,1308.021,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" pixel_10a,llama_cpp,qwen3_vl_2b,prefill,2048,0,19.173800,1415.186,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,4096,0,17.844500,1653.597,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=full_core_prefill; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=8; thread_count=8; taskset_mask=none; affinity=not_set; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" pixel_10a,onnxruntime,gemma_4_e2b,prefill,256,0,25.808400,1601.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" pixel_10a,onnxruntime,gemma_4_e2b,prefill,512,0,23.110300,1761.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" pixel_10a,onnxruntime,gemma_4_e2b,prefill,1024,0,18.672300,2136.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" pixel_10a,onnxruntime,gemma_4_e2b,prefill,2048,0,17.540100,3135.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,4096,0,16.036200,5545.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,112.184000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,120.816000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,122.030000,2370.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,118.489000,3233.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,4096,0,33.642400,5075.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4; synthetic exact-length token ids" pixel_10a,onnxruntime,qwen3_vl_2b,prefill,256,0,86.648800,1266.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" pixel_10a,onnxruntime,qwen3_vl_2b,prefill,512,0,88.170000,1542.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" pixel_10a,onnxruntime,qwen3_vl_2b,prefill,1024,0,92.428900,2117.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" pixel_10a,onnxruntime,qwen3_vl_2b,prefill,2048,0,77.079800,3604.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; prefill_only=true; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,4096,0,30.841600,5182.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none; format=INT4/Q4 block=32; synthetic exact-length token ids" diff --git a/experiments/matrix/results/matrix_pixel_10a_single_thread.csv b/experiments/matrix/results/matrix_pixel_10a_single_thread.csv index b0ea9821a..b91b7ab66 100644 --- a/experiments/matrix/results/matrix_pixel_10a_single_thread.csv +++ b/experiments/matrix/results/matrix_pixel_10a_single_thread.csv @@ -1,126 +1,141 @@ device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes pixel_10a,cactus,gemma_4_e2b,prefill,256,0,9.210000,1623.354,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,cactus,gemma_4_e2b,decode,256,100,3.670000,1623.354,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" pixel_10a,cactus,gemma_4_e2b,prefill,512,0,11.010000,1623.244,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,cactus,gemma_4_e2b,decode,512,100,3.310000,1623.244,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" pixel_10a,cactus,gemma_4_e2b,prefill,1024,0,12.660000,1623.220,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,cactus,gemma_4_e2b,decode,1024,100,3.090000,1623.220,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,cactus,gemma_4_e2b,prefill,2048,0,13.580000,1623.345,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,cactus,gemma_4_e2b,prefill,4096,0,43.610000,1599.065,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,gemma_4_e2b,decode,256,100,3.670000,1623.354,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,cactus,gemma_4_e2b,decode,512,100,3.310000,1623.244,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,cactus,gemma_4_e2b,decode,1024,100,3.090000,1623.220,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,cactus,gemma_4_e2b,decode,2048,100,2.660000,1623.345,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,256,0,4.640000,654.183,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,256,100,4.500000,654.183,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,512,0,4.530000,654.177,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,512,100,4.360000,654.177,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,4.440000,654.778,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,1024,100,4.120000,654.778,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,4.180000,654.908,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,29.400000,1228.633,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,256,100,4.500000,654.183,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,512,100,4.360000,654.177,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,1024,100,4.120000,654.778,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,2048,100,3.750000,654.908,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" pixel_10a,cactus,parakeet_tdt_v3,parakeet_transcribe,,,0.771500,2463.964,777.969,778.112,ok,WER=0.000000; runner=android_transpiled_tdt; core_affinity=cpu7; taskset_mask=80; serial=5C111JEA320125; android=16; thermal_status=0; decoder_steps=152; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey; thread_count=1 pixel_10a,cactus,qwen3_vl_2b,prefill,256,0,3.380000,1210.687,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,cactus,qwen3_vl_2b,decode,256,100,3.180000,1210.687,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" pixel_10a,cactus,qwen3_vl_2b,prefill,512,0,3.230000,1210.468,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,cactus,qwen3_vl_2b,decode,512,100,3.010000,1210.468,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" pixel_10a,cactus,qwen3_vl_2b,prefill,1024,0,3.060000,1210.529,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,cactus,qwen3_vl_2b,decode,1024,100,2.700000,1210.529,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,cactus,qwen3_vl_2b,prefill,2048,0,2.750000,1210.533,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,cactus,qwen3_vl_2b,prefill,4096,0,21.380000,1978.438,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,decode,256,100,3.180000,1210.687,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,cactus,qwen3_vl_2b,decode,512,100,3.010000,1210.468,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,cactus,qwen3_vl_2b,decode,1024,100,2.700000,1210.529,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,cactus,qwen3_vl_2b,decode,2048,100,2.300000,1210.533,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" pixel_10a,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -pixel_10a,executorch,gemma_4_e2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -pixel_10a,executorch,gemma_4_e2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -pixel_10a,executorch,gemma_4_e2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,gemma_4_e2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,gemma_4_e2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,gemma_4_e2b,decode,2048,100,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,256,0,64.486400,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true" -pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,256,100,3.582790,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,512,0,79.262100,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true" -pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,512,100,3.574530,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,1024,0,81.952200,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true" -pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,1024,100,3.291510,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,2048,0,84.928700,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; prefill_only=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,executorch,lfm_2_5_vl_1_6b,prefill,4096,0,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export supports at most 2048 input tokens for token_embedding; requested prefill seqlen=4096 +pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,256,100,3.582790,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=256; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,512,100,3.574530,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=512; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" +pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,1024,100,3.291510,1138.000,2427.657,2427.657,ok,"warmup_runs=1,measurement_runs=3; runner=executorch; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=Q4 8da4w XNNPACK; shared_context=1024; shared_decode_tokens=100; paired_prefill_decode=true; shares measured invocation with prefill row" pixel_10a,executorch,lfm_2_5_vl_1_6b,decode,2048,100,,,2427.657,2427.657,unsupported,configured ExecuTorch .pte export has KV cache capacity 2048; cannot decode 100 tokens from context 2048 because required cache positions extend to 2147 pixel_10a,executorch,parakeet_tdt_v3,parakeet_transcribe,,,0.281650,1509.574,719.224,719.224,ok,WER=0.000000; runner=android_executorch_parakeet; format=Q4 8da4w XNNPACK; core_affinity=cpu7; taskset_mask=80; serial=5C111JEA320125; android=16; thermal_status=1; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey pixel_10a,executorch,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -pixel_10a,executorch,qwen3_vl_2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -pixel_10a,executorch,qwen3_vl_2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" -pixel_10a,executorch,qwen3_vl_2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,decode,256,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,decode,512,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" +pixel_10a,executorch,qwen3_vl_2b,decode,1024,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,executorch,qwen3_vl_2b,decode,2048,100,,,,,unsupported,"ExecuTorch Android Qwen3 VL 2B is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for qwen3_vl_2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a" pixel_10a,litert_lm,gemma_4_e2b,prefill,256,0,17.330000,3390.210,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" -pixel_10a,litert_lm,gemma_4_e2b,decode,256,100,8.410000,3390.210,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" pixel_10a,litert_lm,gemma_4_e2b,prefill,512,0,39.530000,3383.900,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" -pixel_10a,litert_lm,gemma_4_e2b,decode,512,100,8.010000,3383.900,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" pixel_10a,litert_lm,gemma_4_e2b,prefill,1024,0,78.300000,3373.320,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" -pixel_10a,litert_lm,gemma_4_e2b,decode,1024,100,8.840000,3373.320,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" pixel_10a,litert_lm,gemma_4_e2b,prefill,2048,0,61.040000,3373.440,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,prefill,4096,0,69.430000,3373.930,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=strict; runner=litert_lm; provider=cpu; gpu=disabled; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,decode,256,100,8.410000,3390.210,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,decode,512,100,8.010000,3383.900,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" +pixel_10a,litert_lm,gemma_4_e2b,decode,1024,100,8.840000,3373.320,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" pixel_10a,litert_lm,gemma_4_e2b,decode,2048,100,8.510000,3373.440,2588.148,2588.148,ok,"warmup_runs=1,measurement_runs=3; runner=litert_lm; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; threads=1; thread_count=1; quantization=int4_per_output_channel; core_affinity=cpu7; taskset_mask=80" pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm -pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm -pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm -pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,prefill,4096,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm +pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm pixel_10a,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,0.129975,1287.174,667.588,667.590,ok,WER=0.790323; runner=android_litert_parakeet_asr; chunks=6; encoder_frames=378; decoder_steps=179; core_affinity=cpu7; taskset_mask=80; serial=5C111JEA320125; android=16; thermal_status=1; transcript=con returned to its place amidst the tents the english fought at two the french and the congratulations pixel_10a,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm -pixel_10a,litert_lm,qwen3_vl_2b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm pixel_10a,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm -pixel_10a,litert_lm,qwen3_vl_2b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm pixel_10a,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm -pixel_10a,litert_lm,qwen3_vl_2b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm pixel_10a,litert_lm,qwen3_vl_2b,prefill,2048,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,prefill,4096,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,decode,256,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm +pixel_10a,litert_lm,qwen3_vl_2b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm pixel_10a,litert_lm,qwen3_vl_2b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm pixel_10a,llama_cpp,gemma_4_e2b,prefill,256,0,7.574750,3117.683,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,llama_cpp,gemma_4_e2b,decode,256,100,5.126740,3117.683,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" pixel_10a,llama_cpp,gemma_4_e2b,prefill,512,0,6.500580,3181.424,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,llama_cpp,gemma_4_e2b,decode,512,100,4.069490,3181.424,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" pixel_10a,llama_cpp,gemma_4_e2b,prefill,1024,0,6.719770,3193.352,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,llama_cpp,gemma_4_e2b,decode,1024,100,4.177220,3193.352,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,llama_cpp,gemma_4_e2b,prefill,2048,0,6.400140,3216.722,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,llama_cpp,gemma_4_e2b,prefill,4096,0,6.148060,3256.506,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=strict; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,gemma_4_e2b,decode,256,100,5.126740,3117.683,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,llama_cpp,gemma_4_e2b,decode,512,100,4.069490,3181.424,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,llama_cpp,gemma_4_e2b,decode,1024,100,4.177220,3193.352,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,llama_cpp,gemma_4_e2b,decode,2048,100,3.757320,3216.722,3106.736,3106.736,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=0; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,256,0,11.494200,722.785,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,256,100,8.817700,722.785,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,512,0,11.139800,757.023,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,512,100,8.640550,757.023,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,1024,0,11.064900,764.609,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,1024,100,8.369130,764.609,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,2048,0,10.929200,777.979,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,prefill,4096,0,10.888800,806.259,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=strict; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,256,100,8.817700,722.785,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,512,100,8.640550,757.023,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,1024,100,8.369130,764.609,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,llama_cpp,lfm_2_5_vl_1_6b,decode,2048,100,7.819560,777.979,695.752,695.753,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_0; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" pixel_10a,llama_cpp,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,llama.cpp does not have a configured Parakeet TDT runner or artifact pixel_10a,llama_cpp,qwen3_vl_2b,prefill,256,0,9.659300,1188.873,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,llama_cpp,qwen3_vl_2b,decode,256,100,5.750710,1188.873,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" pixel_10a,llama_cpp,qwen3_vl_2b,prefill,512,0,9.500390,1251.921,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,llama_cpp,qwen3_vl_2b,decode,512,100,5.510410,1251.921,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" pixel_10a,llama_cpp,qwen3_vl_2b,prefill,1024,0,8.884470,1309.535,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,llama_cpp,qwen3_vl_2b,decode,1024,100,4.948920,1309.535,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,llama_cpp,qwen3_vl_2b,prefill,2048,0,8.311190,1417.267,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,llama_cpp,qwen3_vl_2b,prefill,4096,0,7.840860,1653.500,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; benchmark_mode=strict; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M" +pixel_10a,llama_cpp,qwen3_vl_2b,decode,256,100,5.750710,1188.873,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" +pixel_10a,llama_cpp,qwen3_vl_2b,decode,512,100,5.510410,1251.921,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" +pixel_10a,llama_cpp,qwen3_vl_2b,decode,1024,100,4.948920,1309.535,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,llama_cpp,qwen3_vl_2b,decode,2048,100,4.193310,1417.267,1107.410,1107.410,ok,"warmup_runs=1,measurement_runs=3; runner=android_llama_cpp_bench; llama_cpp_version=llama-cli 9200 (3e12fbdea); threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; mmap=on; n_gpu_layers=0; serial=5C111JEA320125; android=16; thermal_status=1; synthetic exact-length token ids; quantization=Q4_K_M; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" pixel_10a,onnxruntime,gemma_4_e2b,prefill,256,0,14.488300,1609.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,onnxruntime,gemma_4_e2b,decode,256,100,1.465220,1609.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,gemma_4_e2b,prefill,512,0,12.055700,1769.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,onnxruntime,gemma_4_e2b,decode,512,100,1.438240,1769.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,gemma_4_e2b,prefill,1024,0,11.659800,2144.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,onnxruntime,gemma_4_e2b,decode,1024,100,1.338380,2144.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,gemma_4_e2b,prefill,2048,0,10.320800,3156.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,onnxruntime,gemma_4_e2b,prefill,4096,0,8.036470,5446.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=strict; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,gemma_4_e2b,decode,256,100,1.465220,1609.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,gemma_4_e2b,decode,512,100,1.438240,1769.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,gemma_4_e2b,decode,1024,100,1.338380,2144.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,gemma_4_e2b,decode,2048,100,1.236880,3156.000,3627.412,3627.412,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,256,0,52.833400,2155.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,256,100,13.406000,2155.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,512,0,57.754000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,512,100,13.007000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,1024,0,51.452000,2380.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,1024,100,12.617700,2380.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,2048,0,47.859300,3235.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,prefill,4096,0,22.112000,5580.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=strict; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,256,100,13.406000,2155.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,512,100,13.007000,2151.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,1024,100,12.617700,2380.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,lfm_2_5_vl_1_6b,decode,2048,100,10.108500,3235.000,1754.691,1754.691,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,ONNX Runtime Parakeet TDT Android artifact and ASR runner are not configured pixel_10a,onnxruntime,qwen3_vl_2b,prefill,256,0,48.712800,1296.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,onnxruntime,qwen3_vl_2b,decode,256,100,9.224760,1296.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,qwen3_vl_2b,prefill,512,0,46.142200,1585.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,onnxruntime,qwen3_vl_2b,decode,512,100,9.544100,1585.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,qwen3_vl_2b,prefill,1024,0,39.708000,2165.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,onnxruntime,qwen3_vl_2b,decode,1024,100,6.351120,2165.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,qwen3_vl_2b,prefill,2048,0,31.953900,3612.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,onnxruntime,qwen3_vl_2b,prefill,4096,0,19.746800,5076.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=strict; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; gpu=disabled; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids" +pixel_10a,onnxruntime,qwen3_vl_2b,decode,256,100,9.224760,1296.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,qwen3_vl_2b,decode,512,100,9.544100,1585.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; shares measured invocation with prefill row" +pixel_10a,onnxruntime,qwen3_vl_2b,decode,1024,100,6.351120,2165.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; shares measured invocation with prefill row" pixel_10a,onnxruntime,qwen3_vl_2b,decode,2048,100,4.031710,3612.000,1440.083,1440.085,ok,"warmup_runs=1,measurement_runs=3; runner=onnxruntime_llm_bench; provider=CPUExecutionProvider; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; format=INT4/Q4 block=32; synthetic exact-length token ids; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; shares measured invocation with prefill row" From 86184721cbf2f2f5fc39ae0189dd9454bf06910b Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 01:11:27 -0700 Subject: [PATCH 07/12] updated single thread prefill runs with fixed qwen/lfm --- ...matrix_galaxy_s24_or_s25_single_thread.csv | 16 +++++----- .../matrix_mac_m4pro_single_thread.csv | 30 +++++++++---------- .../matrix_pixel_10a_single_thread.csv | 16 +++++----- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv b/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv index c7ab6307b..0699bd5a4 100644 --- a/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv +++ b/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv @@ -8,20 +8,20 @@ galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,256,100,12.680000,1626.336,1893.450, galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,512,100,12.060000,1626.239,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,1024,100,11.170000,1626.288,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,gemma_4_e2b,decode,2048,100,10.340000,1626.360,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,256,0,20.200000,666.358,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,512,0,20.060000,666.320,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,16.370000,666.346,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,15.070000,666.470,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,256,0,61.560000,1225.110,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,512,0,51.760000,1225.484,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,43.470000,1225.487,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture" +galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,40.930000,1225.914,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,38.730000,1226.418,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,256,100,19.350000,666.358,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,512,100,18.800000,666.320,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,1024,100,14.760000,666.346,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,decode,2048,100,13.720000,666.470,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,parakeet_tdt_v3,parakeet_transcribe,,,0.200471,2475.970,777.969,778.112,ok,WER=0.000000; runner=android_transpiled_tdt; core_affinity=cpu7; taskset_mask=80; serial=RFGL42B1VLW; android=16; thermal_status=1; decoder_steps=152; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey; thread_count=1 -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,256,0,11.620000,1224.077,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,512,0,11.250000,1223.396,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,1024,0,10.550000,1223.269,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,2048,0,9.420000,1224.050,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,256,0,38.170000,1973.848,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,512,0,35.540000,1974.535,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,1024,0,31.440000,1974.963,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture" +galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,2048,0,28.410000,1976.022,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture" galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,4096,0,29.890000,1977.018,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,256,100,11.060000,1224.077,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" galaxy_s24_or_s25,cactus,qwen3_vl_2b,decode,512,100,10.620000,1223.396,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=RFGL42B1VLW; android=16; thermal_status=1; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" diff --git a/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv b/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv index 5af0f1590..1a1f2cc6f 100644 --- a/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv +++ b/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv @@ -1,32 +1,32 @@ device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes -mac_m4pro,cactus,gemma_4_e2b,prefill,256,0,119.090000,585.440,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,256,0,453.960000,585.890,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; prefill_only=true; ram_source=cactus_run_native_peak" mac_m4pro,cactus,gemma_4_e2b,decode,256,100,37.040000,586.000,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,gemma_4_e2b,prefill,512,0,187.990000,585.830,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,512,0,479.610000,586.050,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; prefill_only=true; ram_source=cactus_run_native_peak" mac_m4pro,cactus,gemma_4_e2b,decode,512,100,35.940000,586.770,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,gemma_4_e2b,prefill,1024,0,262.570000,586.970,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,1024,0,466.580000,585.970,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; prefill_only=true; ram_source=cactus_run_native_peak" mac_m4pro,cactus,gemma_4_e2b,decode,1024,100,34.140000,586.580,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,gemma_4_e2b,prefill,2048,0,321.490000,586.380,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,2048,0,446.420000,585.860,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; prefill_only=true; ram_source=cactus_run_native_peak" mac_m4pro,cactus,gemma_4_e2b,decode,2048,100,32.920000,586.280,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,gemma_4_e2b,prefill,4096,0,356.670000,586.390,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread; paired_prefill_decode=true; shared_decode_tokens=100; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,256,0,63.130000,75.470,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,gemma_4_e2b,prefill,4096,0,422.390000,586.690,1893.450,1893.591,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict_single_thread_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=1; thread_count=1; thread_mode=env_single_thread; affinity=not_set; taskset_mask=none; prefill_only=true; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,256,0,108.510000,678.360,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,lfm_2_5_vl_1_6b,decode,256,100,63.350000,75.470,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; uses fixed image fixture; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,512,0,62.680000,74.880,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,512,0,174.910000,685.390,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,lfm_2_5_vl_1_6b,decode,512,100,62.240000,74.880,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; uses fixed image fixture; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,62.440000,75.580,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,251.930000,684.270,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,lfm_2_5_vl_1_6b,decode,1024,100,60.640000,75.580,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; uses fixed image fixture; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,61.160000,75.190,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; uses fixed image fixture; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,306.240000,685.670,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,lfm_2_5_vl_1_6b,decode,2048,100,58.050000,75.190,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; uses fixed image fixture; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,57.300000,76.380,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; uses fixed image fixture; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,314.790000,685.030,1759.526,1759.659,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,parakeet_tdt_v3,parakeet_transcribe,,,0.049118,1196.127,777.969,778.112,ok,WER=0.000000; runner=native; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey -mac_m4pro,cactus,qwen3_vl_2b,prefill,256,0,36.350000,381.280,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,256,0,62.140000,1414.580,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,qwen3_vl_2b,decode,256,100,35.280000,381.280,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,512,0,34.790000,381.300,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,512,0,99.670000,1419.080,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,qwen3_vl_2b,decode,512,100,32.800000,381.300,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,1024,0,33.550000,381.170,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,1024,0,138.600000,1418.640,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,qwen3_vl_2b,decode,1024,100,30.620000,381.170,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,2048,0,30.830000,381.580,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,2048,0,174.910000,1421.220,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,cactus,qwen3_vl_2b,decode,2048,100,25.850000,381.580,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,4096,0,26.400000,381.850,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,4096,0,203.120000,1420.140,1218.608,1218.743,ok,"warmup_runs=2,measurement_runs=3; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; threads=default; affinity=not_set; taskset_mask=none; prefill measured with one generated token for TTFT; ram_source=cactus_run_native_peak" mac_m4pro,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" mac_m4pro,executorch,gemma_4_e2b,decode,256,100,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" mac_m4pro,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" diff --git a/experiments/matrix/results/matrix_pixel_10a_single_thread.csv b/experiments/matrix/results/matrix_pixel_10a_single_thread.csv index b91b7ab66..ac6fc4af3 100644 --- a/experiments/matrix/results/matrix_pixel_10a_single_thread.csv +++ b/experiments/matrix/results/matrix_pixel_10a_single_thread.csv @@ -8,20 +8,20 @@ pixel_10a,cactus,gemma_4_e2b,decode,256,100,3.670000,1623.354,1893.450,1893.591, pixel_10a,cactus,gemma_4_e2b,decode,512,100,3.310000,1623.244,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" pixel_10a,cactus,gemma_4_e2b,decode,1024,100,3.090000,1623.220,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,cactus,gemma_4_e2b,decode,2048,100,2.660000,1623.345,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,256,0,4.640000,654.183,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,512,0,4.530000,654.177,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,4.440000,654.778,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,4.180000,654.908,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,256,0,44.060000,1223.427,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,512,0,43.130000,1223.294,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,38.400000,1223.398,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,34.310000,1224.161,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,29.400000,1228.633,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,256,100,4.500000,654.183,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,512,100,4.360000,654.177,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,1024,100,4.120000,654.778,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shares measured invocation with prefill row" pixel_10a,cactus,lfm_2_5_vl_1_6b,decode,2048,100,3.750000,654.908,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shares measured invocation with prefill row" pixel_10a,cactus,parakeet_tdt_v3,parakeet_transcribe,,,0.771500,2463.964,777.969,778.112,ok,WER=0.000000; runner=android_transpiled_tdt; core_affinity=cpu7; taskset_mask=80; serial=5C111JEA320125; android=16; thermal_status=0; decoder_steps=152; transcript=concord returned to its place amidst the tents the english forwarded to the french baskets of flowers of which they had made a plentiful provision to greet the arrival of the young princess the french in return invited the english to a supper which was to be given the next day congratulations were poured in upon the princess everywhere during her journey; thread_count=1 -pixel_10a,cactus,qwen3_vl_2b,prefill,256,0,3.380000,1210.687,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shared_decode_tokens=100" -pixel_10a,cactus,qwen3_vl_2b,prefill,512,0,3.230000,1210.468,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shared_decode_tokens=100" -pixel_10a,cactus,qwen3_vl_2b,prefill,1024,0,3.060000,1210.529,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=1024; shared_decode_tokens=100" -pixel_10a,cactus,qwen3_vl_2b,prefill,2048,0,2.750000,1210.533,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=2048; shared_decode_tokens=100" +pixel_10a,cactus,qwen3_vl_2b,prefill,256,0,28.800000,1976.316,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,512,0,29.080000,1976.069,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,1024,0,24.110000,1976.200,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,2048,0,21.080000,1976.187,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,qwen3_vl_2b,prefill,4096,0,21.380000,1978.438,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,qwen3_vl_2b,decode,256,100,3.180000,1210.687,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=256; shares measured invocation with prefill row" pixel_10a,cactus,qwen3_vl_2b,decode,512,100,3.010000,1210.468,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=1; thread_count=1; core_affinity=cpu7; taskset_mask=80; benchmark_mode=strict; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture; paired_prefill_decode=true; shared_context=512; shares measured invocation with prefill row" From 40dcd664706d17f754893b2a7760652310642f64 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 01:28:37 -0700 Subject: [PATCH 08/12] listing parakeet as unsupported for litert bc it's setup cheats - it only supports 5s --- .../matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv | 2 +- experiments/matrix/results/matrix_mac_m4pro_single_thread.csv | 2 +- experiments/matrix/results/matrix_pixel_10a_single_thread.csv | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv b/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv index 0699bd5a4..73007b92b 100644 --- a/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv +++ b/experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv @@ -73,7 +73,7 @@ galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,Andro galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm galaxy_s24_or_s25,litert_lm,lfm_2_5_vl_1_6b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm -galaxy_s24_or_s25,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,0.071528,1285.852,667.588,667.590,ok,WER=0.806452; runner=android_litert_parakeet_asr; chunks=6; encoder_frames=378; decoder_steps=177; core_affinity=cpu7; taskset_mask=80; serial=RFGL42B1VLW; android=16; thermal_status=1; transcript=con returned to its place amidst the tents the english foaded two the french congratulations +galaxy_s24_or_s25,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,"unsupported: normal LiteRT Parakeet ASR path is excluded from the LiteRT-LM comparison because it is not an exact Q4/int4 LiteRT-LM artifact and resets decoder state across fixed chunks, producing high WER; no acceptable accuracy-preserving LiteRT-LM Parakeet row is currently supported" galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm galaxy_s24_or_s25,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm diff --git a/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv b/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv index 1a1f2cc6f..3a9249d47 100644 --- a/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv +++ b/experiments/matrix/results/matrix_mac_m4pro_single_thread.csv @@ -73,7 +73,7 @@ mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,"LiteRT-LM L mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,2048,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,decode,2048,100,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" mac_m4pro,litert_lm,lfm_2_5_vl_1_6b,prefill,4096,0,,,,,unsupported,"LiteRT-LM LFM 2.5 VL 1.6B is not supported in this matrix because a fresh Hugging Face file scan found exact ONNX/GGUF/MLX/Cactus artifacts and an exact ExecuTorch .pte, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm" -mac_m4pro,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,0.045189,1016.705,667.588,667.590,ok,"WER=0.806452; runner=normal_litert_parakeet; chunks=6; encoder_frames=63,63,63,63,63,63; transcript=con returned to its place amidst the tents the english foaded two the french congratulations" +mac_m4pro,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,"unsupported: normal LiteRT Parakeet ASR path is excluded from the LiteRT-LM comparison because it is not an exact Q4/int4 LiteRT-LM artifact and resets decoder state across fixed chunks, producing high WER; no acceptable accuracy-preserving LiteRT-LM Parakeet row is currently supported" mac_m4pro,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" mac_m4pro,litert_lm,qwen3_vl_2b,decode,256,100,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" mac_m4pro,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,"LiteRT-LM Qwen3 VL 2B is not supported in this matrix because a fresh Hugging Face file scan over exact Qwen3-VL-2B/Qwen3VL-2B results found ONNX/GGUF/GPTQ/AWQ/OpenVINO/AXERA artifacts, but no exact end-to-end int4/Q4 .litertlm artifact; expected weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm" diff --git a/experiments/matrix/results/matrix_pixel_10a_single_thread.csv b/experiments/matrix/results/matrix_pixel_10a_single_thread.csv index ac6fc4af3..cc88c8fdc 100644 --- a/experiments/matrix/results/matrix_pixel_10a_single_thread.csv +++ b/experiments/matrix/results/matrix_pixel_10a_single_thread.csv @@ -73,7 +73,7 @@ pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,256,100,,,,,unsupported,Android LiteR pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,512,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,1024,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm pixel_10a,litert_lm,lfm_2_5_vl_1_6b,decode,2048,100,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/LFM2.5-VL-1.6B-int4.litertlm -pixel_10a,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,0.129975,1287.174,667.588,667.590,ok,WER=0.790323; runner=android_litert_parakeet_asr; chunks=6; encoder_frames=378; decoder_steps=179; core_affinity=cpu7; taskset_mask=80; serial=5C111JEA320125; android=16; thermal_status=1; transcript=con returned to its place amidst the tents the english fought at two the french and the congratulations +pixel_10a,litert_lm,parakeet_tdt_v3,parakeet_transcribe,,,,,,,unsupported,"unsupported: normal LiteRT Parakeet ASR path is excluded from the LiteRT-LM comparison because it is not an exact Q4/int4 LiteRT-LM artifact and resets decoder state across fixed chunks, producing high WER; no acceptable accuracy-preserving LiteRT-LM Parakeet row is currently supported" pixel_10a,litert_lm,qwen3_vl_2b,prefill,256,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm pixel_10a,litert_lm,qwen3_vl_2b,prefill,512,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm pixel_10a,litert_lm,qwen3_vl_2b,prefill,1024,0,,,,,unsupported,Android LiteRT-LM artifact does not exist: weights/litert_lm/Qwen3VL-2B-Instruct-int4.litertlm From ef9ba7093fddfd40d63f9bf48615048f840f169d Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 11:04:23 -0700 Subject: [PATCH 09/12] test: align v2 merge expectations --- cactus-graph/tests/test_io.cpp | 8 +++---- cactus-graph/tests/test_utils.h | 2 +- python/tests/test_cli_transpile_defaults.py | 25 ++++++++++++++++----- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/cactus-graph/tests/test_io.cpp b/cactus-graph/tests/test_io.cpp index 826bf54dc..298e6c440 100644 --- a/cactus-graph/tests/test_io.cpp +++ b/cactus-graph/tests/test_io.cpp @@ -75,7 +75,7 @@ bool test_graph_save_load() { size_t input_a = graph.input({2, 3}, Precision::FP16); size_t input_b = graph.input({2, 3}, Precision::FP16); size_t sum_id = graph.add(input_a, input_b); - graph.pow(sum_id, 2.0f); + size_t pow_id = graph.pow(sum_id, 2.0f); graph.save(filename); @@ -88,8 +88,8 @@ bool test_graph_save_load() { return false; } - if (sg.graph_inputs.size() != 2 || sg.graph_inputs[0] != 0 || - sg.graph_inputs[1] != 1) { + if (sg.graph_inputs.size() != 2 || sg.graph_inputs[0] != input_a || + sg.graph_inputs[1] != input_b) { std::cout << "[graph_save_load] unexpected graph_inputs:"; for (uint32_t idx : sg.graph_inputs) { std::cout << " " << idx; @@ -99,7 +99,7 @@ bool test_graph_save_load() { return false; } - if (sg.graph_outputs.size() != 1 || sg.graph_outputs[0] != 3) { + if (sg.graph_outputs.size() != 1 || sg.graph_outputs[0] != pow_id) { std::cout << "[graph_save_load] unexpected graph_outputs:"; for (uint32_t idx : sg.graph_outputs) { std::cout << " " << idx; diff --git a/cactus-graph/tests/test_utils.h b/cactus-graph/tests/test_utils.h index 31bd14945..511fd446e 100644 --- a/cactus-graph/tests/test_utils.h +++ b/cactus-graph/tests/test_utils.h @@ -135,7 +135,7 @@ inline void fill_random_fp16(std::vector<__fp16>& data) { } inline size_t runtime_id_from_serialized_index(uint32_t serialized_index) { - return static_cast(serialized_index) + 1; + return static_cast(serialized_index); } inline bool test_basic_operation(const std::string& op_name, diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index 676c2eeb2..6061ddbc9 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -30,7 +30,7 @@ def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[s "--task", "multimodal_causal_lm_logits", "--max-new-tokens", - "32", + "512", "--component-pipeline", "on", "--prompt", @@ -198,14 +198,24 @@ def _fake_cmd_transpile(transpile_args): assert "--audio-file" in extra_args -def test_cmd_convert_infers_text_tasks_for_qwen_and_lfm(monkeypatch, tmp_path: Path) -> None: +def test_cmd_convert_infers_profile_defaults_for_qwen_and_lfm(monkeypatch, tmp_path: Path) -> None: parser = cli.create_parser() import cactus.convert.cli as cq_cli - for alias, model_type, arch in ( - ("qwen", "qwen3", "Qwen3ForCausalLM"), - ("lfm", "lfm2_vl", "Lfm2VlForConditionalGeneration"), + for alias, model_type, arch, expected_components in ( + ( + "qwen", + "qwen3", + "Qwen3ForCausalLM", + "vision_encoder,lm_encoder,decoder,lm_encoder_text_chunk,decoder_prefill_chunk,lm_encoder_step,decoder_media_step,decoder_step", + ), + ( + "lfm", + "lfm2_vl", + "Lfm2VlForConditionalGeneration", + "vision_encoder,lm_encoder,decoder", + ), ): output_dir = tmp_path / alias args = parser.parse_args(["convert", alias, str(output_dir)]) @@ -230,7 +240,10 @@ def _fake_cmd_transpile(transpile_args): assert rc == 0 extra_args = transpile_calls[0].extra_args - assert extra_args[extra_args.index("--task") + 1] == "causal_lm_logits" + assert extra_args[extra_args.index("--task") + 1] == "multimodal_causal_lm_logits" + assert extra_args[extra_args.index("--component-pipeline") + 1] == "on" + assert extra_args[extra_args.index("--components") + 1] == expected_components + assert "--image-file" in extra_args assert "--audio-file" not in extra_args From d7af8ba17b64b765b6d489783c288b8e8a83657f Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 11:11:51 -0700 Subject: [PATCH 10/12] chore: ignore generated benchmark artifacts --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index f89ffb7b0..f36badb1e 100644 --- a/.gitignore +++ b/.gitignore @@ -46,11 +46,16 @@ gemma-4-eap-extras/ .githooks/ experiments/matrix/.build/ experiments/matrix/.matrix_*.yaml +experiments/matrix/.tmp_matrix_*.yaml experiments/matrix/logs/ experiments/matrix/results/ +experiments/matrix/parakeet_litert_export/out*/ +experiments/matrix/parakeet_litert_export/cpu_compare_*.json tests/android/build*/ tests/android/__pycache__/ +tests/android/onnxruntime_llm_bench +tests/build*/ tests/assets/ third_party/ From 9256c767b5245ba56a78e66c9215983317a16d12 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 11:12:36 -0700 Subject: [PATCH 11/12] update mac full core prefill --- .../results/matrix_mac_m4pro_full_core.csv | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/experiments/matrix/results/matrix_mac_m4pro_full_core.csv b/experiments/matrix/results/matrix_mac_m4pro_full_core.csv index a149f03e8..e3276cdd4 100644 --- a/experiments/matrix/results/matrix_mac_m4pro_full_core.csv +++ b/experiments/matrix/results/matrix_mac_m4pro_full_core.csv @@ -4,16 +4,16 @@ mac_m4pro,cactus,gemma_4_e2b,prefill,512,0,463.330000,586.780,1893.450,1893.591, mac_m4pro,cactus,gemma_4_e2b,prefill,1024,0,459.850000,586.300,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" mac_m4pro,cactus,gemma_4_e2b,prefill,2048,0,437.370000,586.350,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" mac_m4pro,cactus,gemma_4_e2b,prefill,4096,0,415.940000,586.020,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,256,0,62.900000,75.770,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,512,0,61.810000,75.660,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,61.720000,75.810,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,60.300000,75.310,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,57.020000,74.830,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,256,0,35.740000,381.700,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,512,0,35.150000,381.470,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,1024,0,33.580000,381.100,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,2048,0,30.860000,381.420,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" -mac_m4pro,cactus,qwen3_vl_2b,prefill,4096,0,26.450000,381.660,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,256,0,468.340000,646.130,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,512,0,486.780000,646.610,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,473.260000,648.110,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,436.810000,647.690,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,390.570000,647.160,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; uses fixed image fixture; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,256,0,277.070000,1125.190,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,512,0,281.020000,1125.190,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,1024,0,267.630000,1125.580,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,2048,0,255.000000,1124.630,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" +mac_m4pro,cactus,qwen3_vl_2b,prefill,4096,0,246.430000,1126.080,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; prefill-only measured with zero decode tokens; ram_source=cactus_run_native_peak" mac_m4pro,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" mac_m4pro,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" mac_m4pro,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Mac Gemma 4 E2B IT is unsupported: ExecuTorch Gemma 4 E2B is not supported in this matrix because a fresh Hugging Face file scan found LiteRT-LM, GGUF, ONNX, MLX, AWQ, Cactus, and source-model artifacts for Gemma 4 E2B, but no exact required-model Q4/int4 ExecuTorch .pte artifact; missing required-model .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer path: artifacts.executorch.tokenizer_path is not configured (local tokenizer candidate: weights/gemma-4-e2b-it/tokenizer.json)" From e645aae315aa4e90be47c35addbab10d44baba90 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 20 May 2026 12:27:28 -0700 Subject: [PATCH 12/12] Update full-core matrix results and runner wiring Replace the Pixel Cactus LFM and Qwen full-core c128 prefill rows with fresh rerun measurements, including the focused Qwen 4096 retry row. Keep the corrected Mac, Samsung, and Pixel full-core rows in the main aggregate CSVs, and remove redundant handoff/per-run artifacts from the tracked branch after verifying the Samsung per-run rows are already present in matrix_galaxy_s24_or_s25_full_core.csv. Add the Android cactus_llm_bench target wiring needed to build the standalone benchmark runner, including the mbedTLS archive links and expected output path. --- android/CMakeLists.txt | 20 ++ experiments/matrix/MAY_19_NIGHT_STATUS.md | 171 ------------------ ...n_full_core_c128_fresh_runner_20260520.csv | 11 -- .../results/matrix_pixel_10a_full_core.csv | 20 +- 4 files changed, 30 insertions(+), 192 deletions(-) delete mode 100644 experiments/matrix/MAY_19_NIGHT_STATUS.md delete mode 100644 experiments/matrix/results/matrix_galaxy_s24_or_s25_cactus_lfm_qwen_full_core_c128_fresh_runner_20260520.csv diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 79b424cab..74115522b 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -73,3 +73,23 @@ add_custom_target(cactus_static ALL ) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +add_executable(cactus_llm_bench + ${PROJECT_ROOT}/tests/android/cactus_llm_bench.cpp +) +target_include_directories(cactus_llm_bench PRIVATE ${PROJECT_ROOT}/cactus ${CACTUS_ENGINE_DIR} ${CACTUS_ENGINE_DIR}/src) +target_compile_definitions(cactus_llm_bench PRIVATE PLATFORM_CPU_ONLY=1) +target_compile_options(cactus_llm_bench PRIVATE -O3 -fvisibility=hidden -fvisibility-inlines-hidden) +target_link_libraries(cactus_llm_bench PRIVATE cactus_engine cactus_graph cactus_kernels ${LOG_LIB} android) +if(EXISTS "${CACTUS_MBEDTLS_LIBRARY}" AND EXISTS "${CACTUS_MBEDX509_LIBRARY}" AND EXISTS "${CACTUS_MBEDCRYPTO_LIBRARY}") + target_link_libraries(cactus_llm_bench PRIVATE + "${CACTUS_MBEDTLS_LIBRARY}" + "${CACTUS_MBEDX509_LIBRARY}" + "${CACTUS_MBEDCRYPTO_LIBRARY}" + ) +endif() +target_link_options(cactus_llm_bench PRIVATE -Wl,-z,max-page-size=16384) +set_target_properties(cactus_llm_bench PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} +) diff --git a/experiments/matrix/MAY_19_NIGHT_STATUS.md b/experiments/matrix/MAY_19_NIGHT_STATUS.md deleted file mode 100644 index bbb47be97..000000000 --- a/experiments/matrix/MAY_19_NIGHT_STATUS.md +++ /dev/null @@ -1,171 +0,0 @@ -# May 19 Night Status - -## Purpose - -Live execution tracker for tonight's work. Use this for handoffs, active runs, completed runs, blockers, and next-agent instructions. - -The main plan stays in `MAY_19_NIGHT_PLAN.md`. Pixel investigation details stay in `PIXEL_OPTIMIZATION_STATUS.md`. - -## Current State - -- Main plan has three top-level sections: - 1. Correct full-core prefill rows - 2. Pixel slowdown investigation - 3. ASR performance notes -- Parakeet full-core rows for Cactus and ExecuTorch across all devices are now included under section 1. -- Pixel optimization details have a separate status doc. - -## Active Runs - -- Subgoal: corrected full-core Cactus LFM/Qwen c128 prefill rerun - Device: pixel_10a (`5C111JEA320125`) - Command: `source ./venv/bin/activate` then `cactus build` then `python experiments/matrix/run_matrix.py --config experiments/matrix/matrix.yaml --device pixel_10a --runtime cactus --model lfm_2_5_vl_1_6b --model qwen3_vl_2b --benchmark-mode full_core_prefill --full-core-threads 8 --warmup-runs 1 --measurement-runs 3 --out experiments/matrix/results/matrix_pixel_10a_cactus_lfm_qwen_full_core_c128_fresh_runner_escalated_20260520.csv` - Log path: `experiments/matrix/results/night_logs/pixel_lfm_qwen_full_core_c128_fresh_runner_escalated_20260520.log` - Output path: `experiments/matrix/results/matrix_pixel_10a_cactus_lfm_qwen_full_core_c128_fresh_runner_escalated_20260520.csv` - Started: 2026-05-20 09:25:11 PDT - Next poll no earlier than: 2026-05-20 10:50:28 PDT - Notes: The previously recorded 09:08 and 09:19 active entries had no matching process and no output/log file as of 2026-05-20 09:25:11 PDT, so the run was restarted. The sandboxed 09:17 rerun wrote `experiments/matrix/results/matrix_pixel_10a_cactus_lfm_qwen_full_core_c128_fresh_runner_20260520.csv` with 10 `unsupported` rows because adb could not connect to its daemon; that artifact is invalid for final CSV patching. `android/build/cactus_llm_bench` was refreshed at 2026-05-20 09:06 after a clean `cactus_llm_bench` target build; `adb devices -l` showed Pixel serial `5C111JEA320125` attached before restart, and run_matrix is expected to redeploy that runner. Polled at 2026-05-20 09:35:35 PDT: session `81878` was still running after completing the local `cactus build` sequence; no final output validation yet. Polled at 2026-05-20 09:46:01 PDT: `run_matrix.py` still active, adb was in `measure_lfm_2_5_vl_1_6b_prefill_2048_1`, output CSV not written yet. Polled at 2026-05-20 09:56:36 PDT: `run_matrix.py` still active, adb was in `measure_lfm_2_5_vl_1_6b_prefill_4096_0`, output CSV not written yet. Polled at 2026-05-20 10:07:09 PDT: `run_matrix.py` still active, adb was in `measure_lfm_2_5_vl_1_6b_prefill_4096_1`, output CSV not written yet. Polled at 2026-05-20 10:18:38 PDT: `run_matrix.py` still active, adb was in `warmup_qwen3_vl_2b_prefill_512_0`, output CSV not written yet. Polled at 2026-05-20 10:29:15 PDT: `run_matrix.py` still active, adb was in `measure_qwen3_vl_2b_prefill_1024_1`, output CSV not written yet. Polled at 2026-05-20 10:40:28 PDT: `run_matrix.py` still active, adb was in `measure_qwen3_vl_2b_prefill_2048_0`, output CSV not written yet. - -When a long-running task starts, record: - -- command -- device -- output CSV or artifact path -- log path -- start time -- expected next poll time - -Poll long-running tasks at most once every 10 minutes unless there is a concrete reason to expect completion or failure sooner. - -Active run template: - -```text -- Subgoal: - Device: - Command: - Log path: - Output path: - Started: - Next poll no earlier than: - Notes: -``` - -Completed result template: - -```text -- Subgoal: - Device: - Output path: - Source raw path: - Command: - Git SHA / build identifier: - Runner: - Device serial: - Model artifact: - Benchmark mode: - Chunk size: - Validation: - Decision: - Next action: -``` - -## Completed Work - -- Subgoal: corrected full-core Cactus LFM/Qwen c128 prefill rerun - Device: galaxy_s24_or_s25 - Output path: `experiments/matrix/results/matrix_galaxy_s24_or_s25_cactus_lfm_qwen_full_core_c128_fresh_runner_20260520.csv` - Source raw path: `experiments/matrix/results/matrix_galaxy_s24_or_s25_cactus_lfm_qwen_full_core_c128_fresh_runner_20260520.csv` - Log path: `experiments/matrix/results/night_logs/samsung_lfm_qwen_full_core_c128_fresh_runner_20260520.log` - Command: `source ./venv/bin/activate` then `cactus build` then `python experiments/matrix/run_matrix.py --config experiments/matrix/matrix.yaml --device galaxy_s24_or_s25 --runtime cactus --model lfm_2_5_vl_1_6b --model qwen3_vl_2b --benchmark-mode full_core_prefill --full-core-threads 8 --warmup-runs 1 --measurement-runs 3 --out experiments/matrix/results/matrix_galaxy_s24_or_s25_cactus_lfm_qwen_full_core_c128_fresh_runner_20260520.csv` - Git SHA / build identifier: `40dcd664706d17f754893b2a7760652310642f64` with dirty worktree; `android/build/cactus_llm_bench` mtime `2026-05-20 09:06:00 PDT`; `cactus build` completed immediately before the run. - Runner: `android_cactus_llm_bench` - Device serial: `RFGL42B1VLW` - Model artifact: LFM `weights/lfm2.5-vl-1.6b-retranspile-4096`; Qwen `weights/qwen3-vl-2b-instruct-reconvert` - Benchmark mode: `full_core_prefill` - Chunk size: `c128` - Validation: 10/10 rows `status=ok`; rows are prefill-only with `decode_tokens=0`; notes include `benchmark_mode=full_core_prefill`, `cactus_chunk_size=c128`, corrected artifact paths, `runner=android_cactus_llm_bench`, `serial=RFGL42B1VLW`, `threads=8`, `thread_count=8`, `thread_mode=full_core_prefill`, `affinity=not_set`, and `taskset_mask=none`. Throughput rows: LFM `78.55,78.37,76.97,74.77,57.57`; Qwen `52.04,55.92,50.27,42.98,39.86`. - Decision: patched `experiments/matrix/results/matrix_galaxy_s24_or_s25_full_core.csv` with these 10 rows; older May 19 Samsung rows are superseded for this goal. - Next action: wait for the active Pixel LFM/Qwen fresh-runner rerun and validate before patching `matrix_pixel_10a_full_core.csv`. -- Subgoal: corrected full-core Cactus LFM/Qwen c128 prefill rerun - Device: mac_m4pro - Output path: `experiments/matrix/results/matrix_mac_m4pro_cactus_lfm_qwen_full_core_c128_rerun_20260520.csv` - Source raw path: `experiments/matrix/results/matrix_mac_m4pro_cactus_lfm_qwen_full_core_c128_rerun_20260520.csv`; backing raw JSON files are the 40 `experiments/matrix/results/.run_json/cactus_matrix_*.json` files modified from 2026-05-20 01:54:53 PDT through 2026-05-20 01:58:46 PDT. - Log path: `experiments/matrix/results/night_logs/mac_lfm_qwen_full_core_c128_20260520.log` - Command: `source ./venv/bin/activate` then `cactus build` then `python experiments/matrix/run_matrix.py --config experiments/matrix/matrix.yaml --device mac_m4pro --runtime cactus --model lfm_2_5_vl_1_6b --model qwen3_vl_2b --benchmark-mode full_core_prefill --full-core-threads 8 --warmup-runs 1 --measurement-runs 3 --out experiments/matrix/results/matrix_mac_m4pro_cactus_lfm_qwen_full_core_c128_rerun_20260520.csv` - Git SHA / build identifier: `40dcd664706d17f754893b2a7760652310642f64` with dirty worktree; `cactus build` completed immediately before the run. - Runner: `run_matrix.py` invoking local `cactus run` - Device serial: n/a (`mac_m4pro` local Mac) - Model artifact: LFM `weights/lfm2.5-vl-1.6b-retranspile-4096`; Qwen `weights/qwen3-vl-2b-instruct-reconvert` - Benchmark mode: `full_core_prefill` - Chunk size: `c128` - Validation: 10/10 rows `status=ok`; rows are prefill-only with `decode_tokens=0`; notes include `benchmark_mode=full_core_prefill`, `cactus_chunk_size=c128`, `prefill_only=true`, `threads=8`, `thread_count=8`, `thread_mode=full_core_prefill`, `affinity=not_set`, and `taskset_mask=none`; corrected artifact paths are present for both models. - Decision: patched `experiments/matrix/results/matrix_mac_m4pro_full_core.csv` with these 10 rows. - Next action: run Android full-core Cactus LFM/Qwen reruns after a fresh Android runner rebuild/redeploy. -- Corrected single-thread CSVs were staged earlier: - - `experiments/matrix/results/matrix_mac_m4pro_single_thread.csv` - - `experiments/matrix/results/matrix_pixel_10a_single_thread.csv` - - `experiments/matrix/results/matrix_galaxy_s24_or_s25_single_thread.csv` -- Created and refined `experiments/matrix/MAY_19_NIGHT_PLAN.md`. -- Added Pixel slowdown investigation to the plan. -- Added Parakeet full-core rows to section 1. -- Ran CQ4 profiling/probe experiments for Pixel diagnosis. -- Pixel full-core Cactus LFM/Qwen rerun attempt notes: - - The first sandboxed attempt returned only `unsupported` rows because `run_matrix.py` could not connect to the adb daemon. - - The 2026-05-20 02:13 PDT rerun produced 10 `ok` rows at `experiments/matrix/results/matrix_pixel_10a_cactus_lfm_qwen_full_core_c128_rerun_20260520.csv`, but it is invalid for final CSV patching because `android/build.sh` did not refresh `android/build/cactus_llm_bench`; the stale runner had mtime 2026-05-20 00:42 while `android/CMakeLists.txt` was newer. - - A clean target build refreshed `android/build/cactus_llm_bench` at 2026-05-20 09:06. The fresh-runner rerun command was interrupted before it started; as of 2026-05-20 09:07:37 PDT there were no active `run_matrix.py`, `cactus_llm_bench`, or device adb benchmark processes, and no fresh-runner output/log file existed. - -## Pending Runs - -Full-core Cactus prefill reruns: - -- Pixel LFM and Qwen -- Samsung LFM and Qwen - -Parakeet full-core confirmation: - -- Cactus and ExecuTorch on Mac -- Cactus and ExecuTorch on Pixel -- Cactus and ExecuTorch on Samsung - -Pixel confirmation: - -- Rerun official Pixel/Samsung Cactus rows with freshly rebuilt Android runners. - -## Blockers / Caveats - -- Old Pixel Cactus numbers may include stale-runner effects. -- Do not patch final full-core CSVs until corrected rows are validated. -- Experimental Pixel kernel probes are diagnostic only unless integrated and rerun in the real model path. -- After C++/CMake/FFI/model-runner changes, Android results require rebuild, redeploy, and runner/version verification before they can be trusted. -- For final CSV edits, keep provenance attached: raw output, command, git SHA or build identifier, runner, serial, model artifact, benchmark mode, and chunk size. -- For Pixel ASR, do not assume the CQ4 LLM diagnosis applies. The rebuilt ASR recheck did not hit CQ4. -- For Pixel kernel work, do not change model files, weight formats, runtime weight layout, threading implementation, or thread choices. - -## Subagent Handoff Template - -```text -- Agent: - Hypothesis: - Worktree / file scope: - Falsification test: - Devices tested: - Real model result: - Probe result: - Regressions: - Conclusion: - Next action: -``` - -## Next Agent Instructions - -1. Read `MAY_19_NIGHT_PLAN.md` first. -2. Read `MAY_19_NIGHT_STATUS.md` for current execution state. -3. Read `PIXEL_OPTIMIZATION_STATUS.md` before doing Pixel optimization work. -4. Work iteratively: pick one subgoal, finish it, validate it, update this status doc, then move to the next subgoal. -5. Before Cactus commands, follow repo setup: - - `source ./venv/bin/activate` - - `cactus build` -6. For kernel changes, test all available target devices: Mac, Pixel, and Samsung. Treat small deltas cautiously because these runs have high variance. -7. Keep final CSV patching separate from diagnostic/probe changes. -8. For Pixel optimization, do not change model files, weight formats, runtime weight layout, threading implementation, or thread choices for a given kernel. -9. Do not let subagents work open-endedly. Give each one a single hypothesis, a scoped worktree or file set, and a concrete falsification test. diff --git a/experiments/matrix/results/matrix_galaxy_s24_or_s25_cactus_lfm_qwen_full_core_c128_fresh_runner_20260520.csv b/experiments/matrix/results/matrix_galaxy_s24_or_s25_cactus_lfm_qwen_full_core_c128_fresh_runner_20260520.csv deleted file mode 100644 index f75333071..000000000 --- a/experiments/matrix/results/matrix_galaxy_s24_or_s25_cactus_lfm_qwen_full_core_c128_fresh_runner_20260520.csv +++ /dev/null @@ -1,11 +0,0 @@ -device,runtime,model,operation,seqlen,decode_tokens,throughput_tok_per_s,peak_ram_mb,disk_size_mb,zipped_size_mb,status,notes -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,256,0,78.550000,1236.780,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,512,0,78.370000,1228.248,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,76.970000,1228.221,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,74.770000,1228.305,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,57.570000,1229.147,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,256,0,52.040000,1978.945,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,512,0,55.920000,1979.331,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,1024,0,50.270000,1979.368,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,2048,0,42.980000,1979.611,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" -galaxy_s24_or_s25,cactus,qwen3_vl_2b,prefill,4096,0,39.860000,1979.835,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=RFGL42B1VLW; android=16; thermal_status=0; fixed token-id fixture" diff --git a/experiments/matrix/results/matrix_pixel_10a_full_core.csv b/experiments/matrix/results/matrix_pixel_10a_full_core.csv index 8c0492d20..951ec93b3 100644 --- a/experiments/matrix/results/matrix_pixel_10a_full_core.csv +++ b/experiments/matrix/results/matrix_pixel_10a_full_core.csv @@ -4,16 +4,16 @@ pixel_10a,cactus,gemma_4_e2b,prefill,512,0,10.030000,1596.384,1893.450,1893.591, pixel_10a,cactus,gemma_4_e2b,prefill,1024,0,10.100000,1596.690,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,gemma_4_e2b,prefill,2048,0,9.750000,1597.104,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,cactus,gemma_4_e2b,prefill,4096,0,9.240000,1592.893,1893.450,1893.591,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/gemma-4-e2b-it-sharedkv; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,256,0,10.650000,1227.073,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,512,0,11.070000,1227.092,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,10.960000,1227.420,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,10.780000,1227.182,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,10.600000,1223.487,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,qwen3_vl_2b,prefill,256,0,5.970000,1976.169,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,qwen3_vl_2b,prefill,512,0,5.890000,1976.275,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,qwen3_vl_2b,prefill,1024,0,5.900000,1976.448,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,qwen3_vl_2b,prefill,2048,0,5.760000,1976.136,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" -pixel_10a,cactus,qwen3_vl_2b,prefill,4096,0,5.770000,1969.446,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,256,0,10.100000,1217.883,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,512,0,10.610000,1216.749,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,1024,0,10.900000,1217.689,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,2048,0,10.930000,1217.860,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,lfm_2_5_vl_1_6b,prefill,4096,0,11.130000,1217.901,1759.526,1759.659,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/lfm2.5-vl-1.6b-retranspile-4096; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,256,0,6.160000,1966.961,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,512,0,6.120000,1966.921,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,1024,0,6.000000,1967.082,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,2048,0,5.870000,1966.979,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 33ced0a6bdf0a535a52fdb31b835d3e8cede6882; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" +pixel_10a,cactus,qwen3_vl_2b,prefill,4096,0,5.910000,1983.548,1218.608,1218.743,ok,"warmup_runs=1,measurement_runs=3; runner=android_cactus_llm_bench; threads=8; thread_count=8; taskset_mask=none; affinity=not_set; benchmark_mode=full_core_prefill; cactus_runtime_version=repo 9256c767b5245ba56a78e66c9215983317a16d12; artifact_path=/Users/noahcylich/Documents/Desert/cactus/weights/qwen3-vl-2b-instruct-reconvert; provider=cpu; gpu=disabled; cactus_chunk_size=c128; prefill_only=true; threads=8; thread_count=8; thread_mode=full_core_prefill; affinity=not_set; taskset_mask=none; serial=5C111JEA320125; android=16; thermal_status=0; fixed token-id fixture" pixel_10a,executorch,gemma_4_e2b,prefill,256,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,gemma_4_e2b,prefill,512,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none" pixel_10a,executorch,gemma_4_e2b,prefill,1024,0,,,,,unsupported,"ExecuTorch Android Gemma 4 E2B IT is unsupported: missing .pte artifact: artifacts.executorch.path is null; no required-model .pte found under weights, ../third_party/executorch; ignored unrelated .pte files: ../third_party/executorch/examples/qualcomm/oss_scripts/llama/artifacts/stories260k_hybrid_llama_qnn.pte, ../third_party/executorch/extension/apple/ExecuTorch/__tests__/resources/add.pte, ../third_party/executorch/test/models/deprecated/ModuleLinear-no-constant-segment.pte; missing tokenizer package: artifacts.executorch.tokenizer_path for gemma_4_e2b is not configured; missing arm64-v8a ExecuTorch runtime libraries (libexecutorch.so, libexecutorch.a, libexecutorch_core.a, libportable_ops_lib.a) under tests/android/executorch/lib/arm64-v8a, tests/android/build_executorch/lib, third_party/executorch/android/arm64-v8a; benchmark_mode=full_core_prefill; provider=cpu; gpu=disabled; threads=8; thread_count=8; affinity=not_set; taskset_mask=none"