Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions android/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,4 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug")
target_compile_definitions(cactus_jni PRIVATE CACTUS_ANDROID_ENABLE_LOGGING)
endif()

target_compile_options(cactus_jni PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden -ffunction-sections -fdata-sections)
target_link_options(cactus_jni PRIVATE -Wl,--gc-sections -flto -Wl,-z,max-page-size=16384)
target_link_options(cactus_jni PRIVATE -Wl,-z,max-page-size=16384)
3 changes: 2 additions & 1 deletion apple/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ if(BUILD_SHARED_LIBS)
set_target_properties(cactus_framework PROPERTIES OUTPUT_NAME cactus)
add_dependencies(cactus_framework ${_DEPS})

target_link_libraries(cactus_framework PRIVATE cactus_flags)

target_link_options(cactus_framework PRIVATE
-force_load $<TARGET_FILE:cactus_engine>
-force_load $<TARGET_FILE:cactus_graph>
-force_load $<TARGET_FILE:cactus_kernels>
-Wl,-dead_strip -flto
)

target_link_libraries(cactus_framework PUBLIC
Expand Down
1 change: 1 addition & 0 deletions cactus-engine/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
add_library(cactus_engine_shared SHARED ${CMAKE_BINARY_DIR}/cactus_engine_shared_stub.cpp)
set_target_properties(cactus_engine_shared PROPERTIES OUTPUT_NAME cactus_engine)
add_dependencies(cactus_engine_shared ${_CACTUS_ENGINE_DEPS})
target_link_libraries(cactus_engine_shared PRIVATE cactus_flags)

if(APPLE)
target_link_options(cactus_engine_shared PRIVATE
Expand Down
23 changes: 2 additions & 21 deletions cactus-engine/src/telemetry_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -362,19 +362,16 @@ static void persist_registered_flag(const std::string& file) {
}
}

static std::string sysctl_string(const char* key) {
#if defined(__APPLE__)
static std::string sysctl_string(const char* key) {
size_t size = 0;
if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) return {};
std::string out(size, '\0');
if (sysctlbyname(key, out.data(), &size, nullptr, 0) != 0) return {};
if (!out.empty() && out.back() == '\0') out.pop_back();
return out;
#else
(void)key;
return {};
#endif
}
#endif

static Event make_event(EventType type, const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message) {
Event e;
Expand Down Expand Up @@ -783,22 +780,6 @@ static std::string format_timestamp(const std::chrono::system_clock::time_point&
return std::string(buf);
}

[[maybe_unused]] static bool is_valid_uuid(const std::string& s) {
// Simple check used only for validation when accepting user-provided IDs.
if (s.size() != 36) return false;
for (size_t i = 0; i < s.size(); ++i) {
char c = s[i];
if (i == 8 || i == 13 || i == 18 || i == 23) {
if (c != '-') return false;
} else {
bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
if (!hex) return false;
}
}
return true;
}


static void collect_device_info() {
struct utsname u;
if (uname(&u) == 0) {
Expand Down
2 changes: 1 addition & 1 deletion cactus-engine/src/transcribe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ using namespace cactus::ffi;

extern "C" {

int cactus_preprocess_audio_features(
CACTUS_FFI_EXPORT int cactus_preprocess_audio_features(
const char* audio_file_path,
const char* model_type,
size_t mel_bins,
Expand Down
4 changes: 2 additions & 2 deletions cactus-graph/src/graph_ffi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1176,7 +1176,7 @@ int cactus_graph_conv1d_pointwise(cactus_graph_t graph, cactus_node_t input, cac
}
}

int cactus_graph_clamp(cactus_graph_t graph, cactus_node_t input, float lo, float hi, cactus_node_t* out) {
CACTUS_FFI_EXPORT int cactus_graph_clamp(cactus_graph_t graph, cactus_node_t input, float lo, float hi, cactus_node_t* out) {
if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_clamp");
try {
*out = static_cast<cactus_node_t>(as_graph(graph)->graph.clamp(static_cast<size_t>(input), lo, hi));
Expand Down Expand Up @@ -1332,7 +1332,7 @@ int cactus_graph_moe_layer_gated(cactus_graph_t graph, cactus_node_t hidden, cac
}
}

int cactus_graph_dense_mlp_tq_fused(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t gate_weight, cactus_node_t up_weight, cactus_node_t down_weight, float product_scale, cactus_node_t* out) {
CACTUS_FFI_EXPORT int cactus_graph_dense_mlp_tq_fused(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t gate_weight, cactus_node_t up_weight, cactus_node_t down_weight, float product_scale, cactus_node_t* out) {
if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_dense_mlp_tq_fused");
try {
*out = static_cast<cactus_node_t>(
Expand Down
23 changes: 22 additions & 1 deletion cactus-kernels/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,31 @@ target_include_directories(cactus_kernels
${CMAKE_CURRENT_SOURCE_DIR}/libs
)

target_compile_options(cactus_kernels PUBLIC
add_library(cactus_flags INTERFACE)

target_compile_options(cactus_flags INTERFACE
-march=armv8.2-a+fp16+simd+dotprod+i8mm -pthread -O3
-fvisibility=hidden -fvisibility-inlines-hidden
-ffunction-sections -fdata-sections -fno-rtti
$<$<CXX_COMPILER_ID:Clang,AppleClang>:-flto=thin>
)

target_link_options(cactus_flags INTERFACE
$<$<CXX_COMPILER_ID:Clang,AppleClang>:-flto=thin>
)

if(APPLE)
target_link_options(cactus_flags INTERFACE -Wl,-dead_strip)
else()
target_link_options(cactus_flags INTERFACE -Wl,--gc-sections)
endif()

if(ANDROID)
target_link_options(cactus_flags INTERFACE -Wl,--exclude-libs,ALL)
endif()

target_link_libraries(cactus_kernels PUBLIC cactus_flags)

target_compile_options(cactus_kernels PRIVATE
-Wall -Wextra -pedantic -Wno-missing-field-initializers
$<$<CXX_COMPILER_ID:AppleClang,Clang>:-Wno-c99-extensions>
Expand Down
2 changes: 2 additions & 0 deletions cactus-kernels/src/conv.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
#endif

constexpr size_t T_TILE_F16 = 2;
#ifdef __APPLE__
constexpr size_t ACCELERATE_K_THRESHOLD = 32;
constexpr size_t ACCELERATE_L_THRESHOLD = 128;
#endif

#ifdef __APPLE__
static void conv1d_causal_depthwise_f16_accelerate(
Expand Down
24 changes: 8 additions & 16 deletions cactus-kernels/src/lstm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -255,14 +255,6 @@ static void lstm_gemv_f16_neon(
}
#endif

static inline void fp16_to_fp32_neon(const __fp16* src, float* dst, size_t n) {
cactus_fp16_to_fp32(src, dst, n);
}

static inline void fp32_to_fp16_neon(const float* src, __fp16* dst, size_t n) {
cactus_fp32_to_fp16(src, dst, n);
}

void cactus_bilstm_sequence_f16(
const __fp16* input,
const __fp16* weight_ih_fwd,
Expand Down Expand Up @@ -294,13 +286,13 @@ void cactus_bilstm_sequence_f16(
std::vector<float> W_fwd_f32(gate_size * combined_K);
std::vector<float> W_bwd_f32(gate_size * combined_K);
for (size_t g = 0; g < gate_size; ++g) {
fp16_to_fp32_neon(weight_ih_fwd + g * input_size,
cactus_fp16_to_fp32(weight_ih_fwd + g * input_size,
W_fwd_f32.data() + g * combined_K, input_size);
fp16_to_fp32_neon(weight_hh_fwd + g * hidden_size,
cactus_fp16_to_fp32(weight_hh_fwd + g * hidden_size,
W_fwd_f32.data() + g * combined_K + input_size, hidden_size);
fp16_to_fp32_neon(weight_ih_bwd + g * input_size,
cactus_fp16_to_fp32(weight_ih_bwd + g * input_size,
W_bwd_f32.data() + g * combined_K, input_size);
fp16_to_fp32_neon(weight_hh_bwd + g * hidden_size,
cactus_fp16_to_fp32(weight_hh_bwd + g * hidden_size,
W_bwd_f32.data() + g * combined_K + input_size, hidden_size);
}

Expand All @@ -317,7 +309,7 @@ void cactus_bilstm_sequence_f16(

for (size_t idx = 0; idx < seq_len; ++idx) {
const size_t t = seq_len - 1 - idx;
fp16_to_fp32_neon(batch_in + t * input_size, xh.data(), input_size);
cactus_fp16_to_fp32(batch_in + t * input_size, xh.data(), input_size);
memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(float));

memcpy(gates.data(), bias_bwd_f32.data(), gate_size * sizeof(float));
Expand All @@ -328,7 +320,7 @@ void cactus_bilstm_sequence_f16(

apply_lstm_gates_f32(gates.data(), c.data(), h.data(), hidden_size);

fp32_to_fp16_neon(h.data(), batch_out + t * output_size + hidden_size, hidden_size);
cactus_fp32_to_fp16(h.data(), batch_out + t * output_size + hidden_size, hidden_size);
}
});

Expand All @@ -339,7 +331,7 @@ void cactus_bilstm_sequence_f16(
std::vector<float> c(hidden_size, 0.0f);

for (size_t t = 0; t < seq_len; ++t) {
fp16_to_fp32_neon(batch_in + t * input_size, xh.data(), input_size);
cactus_fp16_to_fp32(batch_in + t * input_size, xh.data(), input_size);
memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(float));

memcpy(gates.data(), bias_fwd_f32.data(), gate_size * sizeof(float));
Expand All @@ -350,7 +342,7 @@ void cactus_bilstm_sequence_f16(

apply_lstm_gates_f32(gates.data(), c.data(), h.data(), hidden_size);

fp32_to_fp16_neon(h.data(), batch_out + t * output_size, hidden_size);
cactus_fp32_to_fp16(h.data(), batch_out + t * output_size, hidden_size);
}
}

Expand Down
Loading