diff --git a/.gitignore b/.gitignore index 9e5c685d9..1aa29a612 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ apple/*.xcframework apple/libcactus.a *.a !libs/curl/**/*.a +!libs/xgrammar/**/*.a venv __pycache__ @@ -39,4 +40,4 @@ tests/plan.md tests/assets/test_wav_info.txt vision_activations.json gemma-4-eap-extras/ -.githooks/ \ No newline at end of file +.githooks/ diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index f6ed65a65..d621db513 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -6,6 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) set(SOURCE_DIR ${CMAKE_SOURCE_DIR}/../cactus) set(CACTUS_CURL_ROOT "${CMAKE_SOURCE_DIR}/../libs/curl" CACHE PATH "Path to vendored libcurl artifacts") +set(CACTUS_XGRAMMAR_ROOT "${CMAKE_SOURCE_DIR}/../libs/xgrammar" CACHE PATH "Path to vendored xgrammar artifacts") set(_CACTUS_VERSION_FILE "${CMAKE_SOURCE_DIR}/../CACTUS_VERSION") if(EXISTS "${_CACTUS_VERSION_FILE}") @@ -90,6 +91,8 @@ target_link_libraries(cactus ${LOG_LIB} android) set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") set(CACTUS_CURL_LIBRARY_ANDROID "${CACTUS_CURL_ROOT}/android/${ANDROID_ABI}/libcurl.a") +set(CACTUS_XGRAMMAR_INCLUDE_DIR "${CACTUS_XGRAMMAR_ROOT}/include") +set(CACTUS_XGRAMMAR_LIBRARY_ANDROID "${CACTUS_XGRAMMAR_ROOT}/android/${ANDROID_ABI}/libxgrammar.a") set(CACTUS_MBEDTLS_ROOT "${CMAKE_SOURCE_DIR}/mbedtls/${ANDROID_ABI}") if(NOT EXISTS "${CACTUS_MBEDTLS_ROOT}") set(CACTUS_MBEDTLS_ROOT "${CMAKE_SOURCE_DIR}/../libs/mbedtls/android/${ANDROID_ABI}") @@ -105,17 +108,26 @@ if(EXISTS "${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h" AND EXISTS "${CACTUS_CURL_LIB message(FATAL_ERROR "Vendored mbedTLS libraries not found. Expected: ${CACTUS_MBEDTLS_LIBRARY}, ${CACTUS_MBEDX509_LIBRARY}, ${CACTUS_MBEDCRYPTO_LIBRARY}") endif() + if(NOT EXISTS "${CACTUS_XGRAMMAR_INCLUDE_DIR}/xgrammar/xgrammar.h" OR NOT EXISTS "${CACTUS_XGRAMMAR_LIBRARY_ANDROID}") + message(FATAL_ERROR "Vendored Android xgrammar not found. Expected: ${CACTUS_XGRAMMAR_INCLUDE_DIR}/xgrammar/xgrammar.h and ${CACTUS_XGRAMMAR_LIBRARY_ANDROID}") + endif() + message(STATUS "Using vendored xgrammar for Android: ${CACTUS_XGRAMMAR_LIBRARY_ANDROID}") + target_include_directories(cactus PRIVATE "${CACTUS_CURL_INCLUDE_DIR}") target_include_directories(cactus_static PRIVATE "${CACTUS_CURL_INCLUDE_DIR}") target_include_directories(cactus PRIVATE "${CACTUS_MBEDTLS_INCLUDE_DIR}") target_include_directories(cactus_static PRIVATE "${CACTUS_MBEDTLS_INCLUDE_DIR}") - target_compile_definitions(cactus PRIVATE CACTUS_USE_CURL=1) - target_compile_definitions(cactus_static PRIVATE CACTUS_USE_CURL=1) + target_include_directories(cactus PRIVATE "${CACTUS_XGRAMMAR_INCLUDE_DIR}") + target_include_directories(cactus_static PRIVATE "${CACTUS_XGRAMMAR_INCLUDE_DIR}") + target_compile_definitions(cactus PRIVATE CACTUS_USE_CURL=1 CACTUS_USE_XGRAMMAR=1) + target_compile_definitions(cactus_static PRIVATE CACTUS_USE_CURL=1 CACTUS_USE_XGRAMMAR=1) target_link_libraries(cactus "${CACTUS_CURL_LIBRARY_ANDROID}") target_link_libraries(cactus_static "${CACTUS_CURL_LIBRARY_ANDROID}") target_link_libraries(cactus "${CACTUS_MBEDTLS_LIBRARY}" "${CACTUS_MBEDX509_LIBRARY}" "${CACTUS_MBEDCRYPTO_LIBRARY}") target_link_libraries(cactus_static "${CACTUS_MBEDTLS_LIBRARY}" "${CACTUS_MBEDX509_LIBRARY}" "${CACTUS_MBEDCRYPTO_LIBRARY}") + target_link_libraries(cactus "${CACTUS_XGRAMMAR_LIBRARY_ANDROID}") + target_link_libraries(cactus_static "${CACTUS_XGRAMMAR_LIBRARY_ANDROID}") else() message(FATAL_ERROR "Vendored Android libcurl not found. Expected: ${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h and ${CACTUS_CURL_LIBRARY_ANDROID}") endif() diff --git a/apple/CmakeLists.txt b/apple/CmakeLists.txt index 9f04b0cca..69ab42bd1 100644 --- a/apple/CmakeLists.txt +++ b/apple/CmakeLists.txt @@ -9,6 +9,7 @@ set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "16.0" CACHE STRING "Minimu set(SOURCE_DIR ${CMAKE_SOURCE_DIR}/../cactus) set(CACTUS_CURL_ROOT "${CMAKE_SOURCE_DIR}/../libs/curl" CACHE PATH "Path to vendored libcurl artifacts") +set(CACTUS_XGRAMMAR_ROOT "${CMAKE_SOURCE_DIR}/../libs/xgrammar" CACHE PATH "Path to vendored xgrammar artifacts") set(_CACTUS_VERSION_FILE "${CMAKE_SOURCE_DIR}/../CACTUS_VERSION") if(EXISTS "${_CACTUS_VERSION_FILE}") @@ -24,6 +25,12 @@ if(NOT CMAKE_BUILD_TYPE) message(STATUS "CMAKE_BUILD_TYPE was not set, defaulting to Release.") endif() +function(cactus_require_vendored_xgrammar include_dir library_path platform_label) + if(NOT EXISTS "${include_dir}/xgrammar/xgrammar.h" OR NOT EXISTS "${library_path}") + message(FATAL_ERROR "Vendored ${platform_label} xgrammar not found. Expected: ${include_dir}/xgrammar/xgrammar.h and ${library_path}") + endif() +endfunction() + file(GLOB ENGINE_SOURCES "${SOURCE_DIR}/engine/*.cpp") file(GLOB GRAPH_SOURCES "${SOURCE_DIR}/graph/*.cpp") file(GLOB KERNEL_SOURCES "${SOURCE_DIR}/kernel/*.cpp") @@ -113,10 +120,14 @@ find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") + set(CACTUS_XGRAMMAR_INCLUDE_DIR "${CACTUS_XGRAMMAR_ROOT}/include") + set(CACTUS_XGRAMMAR_LIBRARY_MACOS "${CACTUS_XGRAMMAR_ROOT}/macos/libxgrammar.a") if(NOT EXISTS "${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h" OR NOT EXISTS "${CACTUS_CURL_LIBRARY_MACOS}") message(FATAL_ERROR "Vendored macOS libcurl not found. Expected: ${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h and ${CACTUS_CURL_LIBRARY_MACOS}") endif() + cactus_require_vendored_xgrammar("${CACTUS_XGRAMMAR_INCLUDE_DIR}" "${CACTUS_XGRAMMAR_LIBRARY_MACOS}" "macOS") message(STATUS "Using vendored libcurl for macOS: ${CACTUS_CURL_LIBRARY_MACOS}") + message(STATUS "Using vendored xgrammar for macOS: ${CACTUS_XGRAMMAR_LIBRARY_MACOS}") target_link_libraries(cactus PUBLIC ${COREML_FRAMEWORK} ${FOUNDATION_FRAMEWORK} @@ -125,9 +136,11 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") ${SYSTEMCONFIGURATION_FRAMEWORK} ${CFNETWORK_FRAMEWORK} "${CACTUS_CURL_LIBRARY_MACOS}" + "${CACTUS_XGRAMMAR_LIBRARY_MACOS}" ) target_include_directories(cactus PUBLIC "${CACTUS_CURL_INCLUDE_DIR}") - target_compile_definitions(cactus PRIVATE CACTUS_USE_CURL=1) + target_include_directories(cactus PUBLIC "${CACTUS_XGRAMMAR_INCLUDE_DIR}") + target_compile_definitions(cactus PRIVATE CACTUS_USE_CURL=1 CACTUS_USE_XGRAMMAR=1) elseif(CMAKE_SYSTEM_NAME STREQUAL "iOS") target_link_libraries(cactus PUBLIC ${COREML_FRAMEWORK} @@ -138,19 +151,26 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "iOS") ${CFNETWORK_FRAMEWORK} ) set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") + set(CACTUS_XGRAMMAR_INCLUDE_DIR "${CACTUS_XGRAMMAR_ROOT}/include") string(TOLOWER "${CMAKE_OSX_SYSROOT}" CACTUS_SYSROOT_LOWER) if(CACTUS_SYSROOT_LOWER MATCHES "simulator") set(CACTUS_CURL_LIBRARY_IOS "${CACTUS_CURL_ROOT}/ios/simulator/libcurl.a") + set(CACTUS_XGRAMMAR_LIBRARY_IOS "${CACTUS_XGRAMMAR_ROOT}/ios/simulator/libxgrammar.a") else() set(CACTUS_CURL_LIBRARY_IOS "${CACTUS_CURL_ROOT}/ios/device/libcurl.a") + set(CACTUS_XGRAMMAR_LIBRARY_IOS "${CACTUS_XGRAMMAR_ROOT}/ios/device/libxgrammar.a") endif() if(NOT EXISTS "${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h" OR NOT EXISTS "${CACTUS_CURL_LIBRARY_IOS}") message(FATAL_ERROR "Vendored iOS libcurl not found for sysroot ${CMAKE_OSX_SYSROOT}. Expected: ${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h and ${CACTUS_CURL_LIBRARY_IOS}") endif() + cactus_require_vendored_xgrammar("${CACTUS_XGRAMMAR_INCLUDE_DIR}" "${CACTUS_XGRAMMAR_LIBRARY_IOS}" "iOS") message(STATUS "Using vendored libcurl for iOS: ${CACTUS_CURL_LIBRARY_IOS}") + message(STATUS "Using vendored xgrammar for iOS: ${CACTUS_XGRAMMAR_LIBRARY_IOS}") target_link_libraries(cactus PUBLIC "${CACTUS_CURL_LIBRARY_IOS}") + target_link_libraries(cactus PUBLIC "${CACTUS_XGRAMMAR_LIBRARY_IOS}") target_include_directories(cactus PUBLIC "${CACTUS_CURL_INCLUDE_DIR}") - target_compile_definitions(cactus PRIVATE CACTUS_USE_CURL=1) + target_include_directories(cactus PUBLIC "${CACTUS_XGRAMMAR_INCLUDE_DIR}") + target_compile_definitions(cactus PRIVATE CACTUS_USE_CURL=1 CACTUS_USE_XGRAMMAR=1) else() target_link_libraries(cactus PUBLIC ${COREML_FRAMEWORK} ${FOUNDATION_FRAMEWORK} ${ACCELERATE_FRAMEWORK}) endif() diff --git a/cactus/CMakeLists.txt b/cactus/CMakeLists.txt index 59784eeb6..a5041c53a 100644 --- a/cactus/CMakeLists.txt +++ b/cactus/CMakeLists.txt @@ -24,6 +24,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) set(CACTUS_CURL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../libs/curl" CACHE PATH "Path to vendored libcurl artifacts") +set(CACTUS_XGRAMMAR_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../libs/xgrammar" CACHE PATH "Path to vendored xgrammar artifacts") function(cactus_require_vendored_curl include_dir library_path platform_label) if(NOT EXISTS "${include_dir}/curl/curl.h" OR NOT EXISTS "${library_path}") @@ -37,6 +38,18 @@ function(cactus_require_vendored_curl include_dir library_path platform_label) endif() endfunction() +function(cactus_require_vendored_xgrammar include_dir library_path platform_label) + if(NOT EXISTS "${include_dir}/xgrammar/xgrammar.h" OR NOT EXISTS "${library_path}") + message(WARNING "==============================================================================") + message(WARNING " Vendored xgrammar is required for ${platform_label} builds but is missing.") + message(WARNING " Expected header: ${include_dir}/xgrammar/xgrammar.h") + message(WARNING " Expected library: ${library_path}") + message(WARNING " Populate libs/xgrammar from your local xgrammar build before running cactus build.") + message(WARNING "==============================================================================") + message(FATAL_ERROR "Missing vendored xgrammar artifacts for ${platform_label}") + endif() +endfunction() + if(APPLE) set(CMAKE_OSX_ARCHITECTURES "arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64 -march=armv8.2-a+fp16+simd+dotprod+i8mm -pthread -Wall -Wextra -pedantic -O3 -Wno-missing-field-initializers") @@ -52,6 +65,9 @@ if(APPLE) set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") cactus_require_vendored_curl("${CACTUS_CURL_INCLUDE_DIR}" "${CACTUS_CURL_LIBRARY_MACOS}" "macOS") + set(CACTUS_XGRAMMAR_INCLUDE_DIR "${CACTUS_XGRAMMAR_ROOT}/include") + set(CACTUS_XGRAMMAR_LIBRARY_MACOS "${CACTUS_XGRAMMAR_ROOT}/macos/libxgrammar.a") + cactus_require_vendored_xgrammar("${CACTUS_XGRAMMAR_INCLUDE_DIR}" "${CACTUS_XGRAMMAR_LIBRARY_MACOS}" "macOS") find_library(COREML_FRAMEWORK CoreML REQUIRED) find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) @@ -66,6 +82,10 @@ else() __ARM_FEATURE_DOTPROD=1 __ARM_FEATURE_MATMUL_INT8=1 ) + + set(CACTUS_XGRAMMAR_INCLUDE_DIR "${CACTUS_XGRAMMAR_ROOT}/include") + set(CACTUS_XGRAMMAR_LIBRARY_LINUX_AARCH64 "${CACTUS_XGRAMMAR_ROOT}/linux/aarch64/libxgrammar.a") + cactus_require_vendored_xgrammar("${CACTUS_XGRAMMAR_INCLUDE_DIR}" "${CACTUS_XGRAMMAR_LIBRARY_LINUX_AARCH64}" "Linux ARM64") endif() file(GLOB ENGINE_SOURCES CONFIGURE_DEPENDS "engine/*.cpp") @@ -129,9 +149,21 @@ function(configure_cactus_target target_name) ${SYSTEMCONFIGURATION_FRAMEWORK} ${CFNETWORK_FRAMEWORK} ${CACTUS_CURL_LIBRARY_MACOS} + ${CACTUS_XGRAMMAR_LIBRARY_MACOS} ) target_include_directories(${target_name} PUBLIC ${CACTUS_CURL_INCLUDE_DIR}) - target_compile_definitions(${target_name} PRIVATE CACTUS_USE_CURL=1) + target_include_directories(${target_name} PUBLIC ${CACTUS_XGRAMMAR_INCLUDE_DIR}) + target_compile_definitions(${target_name} PRIVATE CACTUS_USE_CURL=1 CACTUS_USE_XGRAMMAR=1) + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_include_directories(${target_name} PUBLIC ${CACTUS_XGRAMMAR_INCLUDE_DIR}) + target_link_libraries(${target_name} PUBLIC ${CACTUS_XGRAMMAR_LIBRARY_LINUX_AARCH64}) + target_compile_definitions(${target_name} PRIVATE CACTUS_USE_XGRAMMAR=1) + + if(CURL_FOUND) + target_include_directories(${target_name} PRIVATE ${CURL_INCLUDE_DIRS}) + target_link_libraries(${target_name} PUBLIC ${CURL_LIBRARIES}) + target_compile_definitions(${target_name} PRIVATE CACTUS_USE_CURL) + endif() elseif(CURL_FOUND) target_include_directories(${target_name} PRIVATE ${CURL_INCLUDE_DIRS}) target_link_libraries(${target_name} PUBLIC ${CURL_LIBRARIES}) diff --git a/cactus/engine/engine.h b/cactus/engine/engine.h index ee611981d..4a8931631 100644 --- a/cactus/engine/engine.h +++ b/cactus/engine/engine.h @@ -5,8 +5,17 @@ #include #include #include +#include #include +#include +#include +#include +#include +#include +#include +#include + #include "../graph/graph.h" #ifdef __clang__ @@ -84,7 +93,7 @@ struct Config { float rescale_factor = 0.00392156862745098f; float image_mean = 0.5f; float image_std = 0.5f; - + uint32_t downsample_factor = 2; uint32_t min_tiles = 2; uint32_t max_tiles = 10; @@ -215,6 +224,84 @@ struct Config { std::string to_json() const; }; +class GrammarVocabulary { +public: + GrammarVocabulary(xgrammar::TokenizerInfo tokenizer_info); + + size_t vocab_size() const; + const std::vector& stop_token_ids() const; + const xgrammar::TokenizerInfo& raw_value() const; + +private: + xgrammar::TokenizerInfo tokenizer_info; + std::vector stop_token_ids_; +}; + +class Grammar { +public: + Grammar(); + ~Grammar() = default; + + static Grammar ebnf(const std::string& ebnf, const std::string& start_symbol = "root"); + static Grammar json(); + static Grammar universal(); + static Grammar json_schema( + const std::string& json_schema, + bool any_whitespace = true, + int indent = 2, + std::pair separators = {",", ":"}, + bool strict_mode = true, + int max_whitespace_count = -1 + ); + static Grammar regex(const std::string& regex); + static Grammar structural_tag(const std::string& structural_tag_json, const GrammarVocabulary* vocab = nullptr); + static Grammar unite(const std::vector& grammars); + static Grammar concatenate(const std::vector& grammars); + + bool is_empty() const; + bool is_universal() const; + std::string ebnf() const; + + const xgrammar::Grammar& raw_value() const; + +private: + Grammar(xgrammar::Grammar raw_grammar); + + xgrammar::Grammar grammar; + bool is_universal_ = false; +}; + +class GrammarMatcher; + +class GrammarEngine { +public: + GrammarEngine(const GrammarVocabulary& vocab); + ~GrammarEngine() = default; + + GrammarMatcher compile_matcher(const Grammar& grammar); + +private: + xgrammar::GrammarCompiler compiler; + xgrammar::TokenizerInfo tokenizer_info; +}; + +class GrammarMatcher { +public: + GrammarMatcher(xgrammar::GrammarMatcher matcher, xgrammar::TokenizerInfo tokenizer_info); + ~GrammarMatcher() = default; + + void rollback(int tokens = 1); + void reset(); + GrammarMatcher fork() const; + bool is_completed() const; + bool is_terminated() const; + bool accept(uint32_t token_id, bool log_rejection = false); + bool next_bitmask(std::vector& bitmask, size_t logits_buffer_size); + +private: + xgrammar::GrammarMatcher matcher; + xgrammar::TokenizerInfo tokenizer_info; +}; struct MergeRule { @@ -222,7 +309,7 @@ struct MergeRule { std::string second; std::string merged; uint32_t priority; - + MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) : first(f), second(s), merged(m), priority(p) {} }; @@ -324,15 +411,17 @@ class Tokenizer { virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = false) const; + virtual GrammarVocabulary get_grammar_vocabulary() const; virtual uint32_t get_vocab_size() const = 0; virtual uint32_t get_unk_token() const = 0; virtual uint32_t get_bos_token() const = 0; virtual uint32_t get_eos_token() const = 0; + virtual const std::vector& get_encoded_vocab() const = 0; virtual bool has_chat_template() const { return has_chat_template_; } std::string get_default_stop_sequence() const; virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; - + uint32_t get_image_token_id() const { return image_token_id_; } uint32_t get_fake_token_id() const { return fake_token_id_; } uint32_t get_global_img_token_id() const { return global_img_token_id_; } @@ -344,7 +433,7 @@ class Tokenizer { ModelVariant model_variant_ = ModelVariant::DEFAULT; bool has_chat_template_ = false; std::string chat_template_; - + uint32_t image_token_id_ = 396; uint32_t fake_token_id_ = 49189; uint32_t global_img_token_id_ = 49152; @@ -382,12 +471,13 @@ class BPETokenizer : public Tokenizer { uint32_t get_unk_token() const override { return unk_token_id_; } uint32_t get_bos_token() const override { return bos_token_id_; } uint32_t get_eos_token() const override { return eos_token_id_; } + const std::vector& get_encoded_vocab() const override { return id_to_token_; } private: std::unordered_map token_to_id_; std::vector id_to_token_; std::vector merge_rules_; - std::unordered_map merge_map_; + std::unordered_map merge_map_; uint32_t vocab_size_; uint32_t unk_token_id_; @@ -402,14 +492,14 @@ class BPETokenizer : public Tokenizer { std::vector apply_bpe(const std::vector& tokens) const; std::pair find_best_merge_fast(const std::vector& tokens) const; - + std::string bytes_to_unicode(const std::string& text) const; std::string unicode_to_bytes(const std::string& text) const; std::vector byte_level_split(const std::string& text) const; std::vector utf8_split(const std::string& text) const; void cleanup_mmap(); - + private: mutable std::unordered_map byte_to_unicode_; mutable std::unordered_map unicode_to_byte_; @@ -420,6 +510,8 @@ class BPETokenizer : public Tokenizer { void load_special_tokens(const std::string& config_file); }; +std::unique_ptr create_tokenizer_from_model_dir(const std::string& model_dir); + class SPTokenizer : public Tokenizer { public: SPTokenizer(); @@ -434,6 +526,7 @@ class SPTokenizer : public Tokenizer { uint32_t get_unk_token() const override { return unk_token_id_; } uint32_t get_bos_token() const override { return bos_token_id_; } uint32_t get_eos_token() const override { return eos_token_id_; } + const std::vector& get_encoded_vocab() const override { return id_to_token_; } private: struct TrieNode { @@ -479,9 +572,9 @@ class ConvCache { struct CircularView { const void* ptr1; size_t len1; - const void* ptr2; + const void* ptr2; size_t len2; - size_t total_len; + size_t total_len; }; void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); @@ -499,9 +592,9 @@ class ConvCache { private: struct LayerState { - std::vector data; - size_t head = 0; - size_t count = 0; + std::vector data; + size_t head = 0; + size_t count = 0; }; std::vector layer_states; @@ -522,8 +615,8 @@ struct KVCache { std::vector layer_caches; - size_t window_size = DEFAULT_WINDOW_SIZE; - size_t sink_size = DEFAULT_SINK_SIZE; + size_t window_size = DEFAULT_WINDOW_SIZE; + size_t sink_size = DEFAULT_SINK_SIZE; size_t current_seq_len = 0; size_t total_seq_len = 0; size_t max_seq_len = 2048; @@ -552,9 +645,9 @@ struct KVCache { struct CircularView { const void* ptr1; - const void* ptr2; + const void* ptr2; size_t len1; - size_t len2; + size_t len2; size_t total_len; }; @@ -573,36 +666,36 @@ struct KVCache { class ToolCallConstrainer { public: enum class State { - DONE, + DONE, - QWEN_START, - QWEN_EXPECT_OPEN_BRACE, - QWEN_EXPECT_NAME_KEY, + QWEN_START, + QWEN_EXPECT_OPEN_BRACE, + QWEN_EXPECT_NAME_KEY, QWEN_EXPECT_NAME_COLON, QWEN_EXPECT_NAME_VALUE, - QWEN_EXPECT_COMMA, - QWEN_EXPECT_ARGS_KEY, - QWEN_EXPECT_ARGS_COLON, + QWEN_EXPECT_COMMA, + QWEN_EXPECT_ARGS_KEY, + QWEN_EXPECT_ARGS_COLON, QWEN_IN_ARGUMENTS, QWEN_EXPECT_CLOSE_BRACE, QWEN_EXPECT_END, NEEDLE_START, - LFM_START, - LFM_EXPECT_BRACKET, + LFM_START, + LFM_EXPECT_BRACKET, LFM_IN_FUNC_NAME, LFM_EXPECT_PAREN, - LFM_IN_ARGUMENTS, - LFM_EXPECT_BRACKET_CLOSE, - LFM_EXPECT_END, - - GEMMA_START, - GEMMA_EXPECT_CALL, - GEMMA_IN_FUNC_NAME, - GEMMA_EXPECT_BRACE, - GEMMA_IN_ARGUMENTS, - GEMMA_EXPECT_END + LFM_IN_ARGUMENTS, + LFM_EXPECT_BRACKET_CLOSE, + LFM_EXPECT_END, + + GEMMA_START, + GEMMA_EXPECT_CALL, + GEMMA_IN_FUNC_NAME, + GEMMA_EXPECT_BRACE, + GEMMA_IN_ARGUMENTS, + GEMMA_EXPECT_END }; void init(Config::ModelType model_type, @@ -647,14 +740,14 @@ class ToolCallConstrainer { std::unordered_set qwen_tool_call_start_tokens_; std::unordered_set qwen_tool_call_end_tokens_; - std::unordered_set open_brace_tokens_; - std::unordered_set close_brace_tokens_; - std::unordered_set colon_tokens_; - std::unordered_set comma_tokens_; - std::unordered_set name_key_tokens_; - std::unordered_set args_key_tokens_; - std::unordered_set quote_tokens_; - std::unordered_set backtick_tokens_; + std::unordered_set open_brace_tokens_; + std::unordered_set close_brace_tokens_; + std::unordered_set colon_tokens_; + std::unordered_set comma_tokens_; + std::unordered_set name_key_tokens_; + std::unordered_set args_key_tokens_; + std::unordered_set quote_tokens_; + std::unordered_set backtick_tokens_; std::unordered_set all_func_name_tokens_; std::unordered_map> func_name_sequences_; NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; @@ -687,13 +780,13 @@ class ToolCallConstrainer { std::string lfm_args_buffer_; std::unordered_set lfm_seen_arg_keys_; std::unordered_map> lfm_required_params_; - std::unordered_map> lfm_all_params_; + std::unordered_map> lfm_all_params_; - std::unordered_set gemma_call_start_tokens_; - std::unordered_set gemma_call_end_tokens_; - std::unordered_set gemma_response_start_tokens_; - std::unordered_set gemma_call_prefix_tokens_; - std::unordered_set escape_tokens_; + std::unordered_set gemma_call_start_tokens_; + std::unordered_set gemma_call_end_tokens_; + std::unordered_set gemma_response_start_tokens_; + std::unordered_set gemma_call_prefix_tokens_; + std::unordered_set escape_tokens_; std::unordered_map current_bias_; @@ -733,7 +826,7 @@ class Model { const std::vector& get_debug_nodes() const; virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); - + virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); @@ -757,9 +850,9 @@ class Model { float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); - + virtual std::vector get_image_embeddings(const std::string& image_path); - + virtual std::vector get_audio_embeddings(const std::vector& audio_features); virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } @@ -814,14 +907,14 @@ class Model { static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; - + virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); - + virtual void load_weights_to_graph(CactusGraph* gb) = 0; - + virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; - + virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, ComputeBackend backend) const = 0; virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, @@ -898,33 +991,33 @@ class Siglip2Preprocessor { }; struct PreprocessedImage { - std::vector pixel_values; - std::vector pixel_attention_mask; - std::vector> spatial_shapes; - std::vector pixel_values_shape; - std::vector pixel_attention_mask_shape; - std::vector spatial_shapes_shape; - int num_patches_height; - int num_patches_width; - int actual_num_patches; - int num_tiles; - int patch_dim; - int max_patches_per_tile; - - int image_rows; - int image_cols; - int image_height; - int image_width; - int tokens_per_tile; - int thumbnail_tokens; - + std::vector pixel_values; + std::vector pixel_attention_mask; + std::vector> spatial_shapes; + std::vector pixel_values_shape; + std::vector pixel_attention_mask_shape; + std::vector spatial_shapes_shape; + int num_patches_height; + int num_patches_width; + int actual_num_patches; + int num_tiles; + int patch_dim; + int max_patches_per_tile; + + int image_rows; + int image_cols; + int image_height; + int image_width; + int tokens_per_tile; + int thumbnail_tokens; + ~PreprocessedImage(); }; struct SpatialShapeResult { - std::vector> shapes; - int grid_rows; - int grid_cols; + std::vector> shapes; + int grid_rows; + int grid_cols; }; explicit Siglip2Preprocessor(const Config& config); diff --git a/cactus/engine/engine_grammar.cpp b/cactus/engine/engine_grammar.cpp new file mode 100644 index 000000000..b6b86cea1 --- /dev/null +++ b/cactus/engine/engine_grammar.cpp @@ -0,0 +1,237 @@ +#include "engine.h" + +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +namespace { + +static xgrammar::VocabType to_xgrammar_vocab_type(const TokenizerRuntimeConfig& runtime_config) { + const auto has_none = runtime_config.decoder == TokenizerRuntimeConfig::Decoder::NONE + && runtime_config.normalizer == TokenizerRuntimeConfig::Normalizer::NONE; + const auto is_byte_level = runtime_config.decoder == TokenizerRuntimeConfig::Decoder::BYTE_LEVEL + || runtime_config.normalizer == TokenizerRuntimeConfig::Normalizer::BYTE_LEVEL; + if (runtime_config.byte_fallback) { + return xgrammar::VocabType::BYTE_FALLBACK; + } + if (is_byte_level || has_none) { + return xgrammar::VocabType::BYTE_LEVEL; + } + return xgrammar::VocabType::RAW; +} + +} // anonymous namespace + +GrammarVocabulary::GrammarVocabulary(xgrammar::TokenizerInfo tokenizer_info) + : tokenizer_info(std::move(tokenizer_info)) { + const auto& stop_token_ids = this->tokenizer_info.GetStopTokenIds(); + stop_token_ids_.assign(stop_token_ids.begin(), stop_token_ids.end()); +} + +size_t GrammarVocabulary::vocab_size() const { + return static_cast(tokenizer_info.GetVocabSize()); +} + +const std::vector& GrammarVocabulary::stop_token_ids() const { + return stop_token_ids_; +} + +const xgrammar::TokenizerInfo& GrammarVocabulary::raw_value() const { + return tokenizer_info; +} + +GrammarVocabulary Tokenizer::get_grammar_vocabulary() const { + std::vector stop_token_ids = {get_eos_token()}; + std::string default_stop = get_default_stop_sequence(); + if (!default_stop.empty()) { + std::vector encoded = encode(default_stop); + if (encoded.size() == 1 && std::find(stop_token_ids.begin(), stop_token_ids.end(), encoded[0]) == stop_token_ids.end()) { + stop_token_ids.push_back(encoded[0]); + } + } + + const auto& encoded_vocab = get_encoded_vocab(); + std::vector stop_token_ids_int32(stop_token_ids.begin(), stop_token_ids.end()); + return GrammarVocabulary(xgrammar::TokenizerInfo( + encoded_vocab, + to_xgrammar_vocab_type(runtime_config_), + static_cast(encoded_vocab.size()), + stop_token_ids_int32, + false + )); +} + +Grammar::Grammar() : grammar(xgrammar::NullObj{}), is_universal_(false) {} + +Grammar::Grammar(xgrammar::Grammar raw_grammar) + : grammar(raw_grammar), is_universal_(false) {} + +Grammar Grammar::ebnf(const std::string& ebnf, const std::string& start_symbol) { + return Grammar(xgrammar::Grammar::FromEBNF(ebnf, start_symbol)); +} + +Grammar Grammar::json() { + return Grammar(xgrammar::Grammar::BuiltinJSONGrammar()); +} + +Grammar Grammar::universal() { + static auto grammar = Grammar::structural_tag(R"({ + "type": "structural_tag", + "format": { + "type": "any_text" + } + })"); + grammar.is_universal_ = true; + return grammar; +} + +Grammar Grammar::json_schema( + const std::string& json_schema, + bool any_whitespace, + int indent, + std::pair separators, + bool strict_mode, + int max_whitespace_count +) { + return Grammar(xgrammar::Grammar::FromJSONSchema( + json_schema, + any_whitespace, + indent, + separators, + strict_mode, + max_whitespace_count >= 0 ? std::optional(max_whitespace_count) : std::nullopt + )); +} + +Grammar Grammar::regex(const std::string& regex) { + return Grammar(xgrammar::Grammar::FromRegex(regex)); +} + +Grammar Grammar::structural_tag(const std::string& structural_tag_json, const GrammarVocabulary* vocab) { + auto result = xgrammar::Grammar::FromStructuralTag( + structural_tag_json, + vocab ? std::optional(vocab->raw_value()) : std::nullopt + ); + if (std::holds_alternative(result)) { + return Grammar(std::get(std::move(result))); + } + throw std::get(result); +} + +Grammar Grammar::unite(const std::vector& grammars) { + std::vector handles; + handles.reserve(grammars.size()); + for (const auto& grammar : grammars) { + if (grammar.is_empty()) continue; + if (grammar.is_universal()) return grammar; + handles.push_back(grammar.raw_value()); + } + return handles.empty() ? Grammar() : Grammar(xgrammar::Grammar::Union(handles)); +} + +Grammar Grammar::concatenate(const std::vector& grammars) { + std::vector handles; + handles.reserve(grammars.size()); + for (const auto& grammar : grammars) { + if (grammar.is_empty()) continue; + handles.push_back(grammar.raw_value()); + } + return handles.empty() ? Grammar() : Grammar(xgrammar::Grammar::Concat(handles)); +} + +bool Grammar::is_empty() const { + return grammar.IsNull(); +} + +bool Grammar::is_universal() const { + return is_universal_; +} + +std::string Grammar::ebnf() const { + if (is_empty()) { + throw std::runtime_error("Cannot get EBNF for empty grammar"); + } + return grammar.ToString(); +} + +const xgrammar::Grammar& Grammar::raw_value() const { + return grammar; +} + +GrammarEngine::GrammarEngine(const GrammarVocabulary& vocab) + : compiler(nullptr), tokenizer_info(vocab.raw_value()) { + compiler = xgrammar::GrammarCompiler( + tokenizer_info, + static_cast(std::max(1u, std::thread::hardware_concurrency())) + ); +} + +GrammarMatcher GrammarEngine::compile_matcher(const Grammar& grammar) { + if (grammar.is_empty()) { + throw std::runtime_error("Cannot create GrammarMatcher with empty grammar"); + } + + auto compiled_grammar = compiler.CompileGrammar(grammar.raw_value()); + return GrammarMatcher(xgrammar::GrammarMatcher(compiled_grammar), tokenizer_info); +} + +GrammarMatcher::GrammarMatcher(xgrammar::GrammarMatcher matcher, xgrammar::TokenizerInfo tokenizer_info) + : matcher(std::move(matcher)), tokenizer_info(std::move(tokenizer_info)) {} + +void GrammarMatcher::rollback(int tokens) { + matcher.Rollback(tokens); +} + +void GrammarMatcher::reset() { + matcher.Reset(); +} + +GrammarMatcher GrammarMatcher::fork() const { + return GrammarMatcher(matcher.Fork(), tokenizer_info); +} + +bool GrammarMatcher::is_completed() const { + return matcher.IsCompleted(); +} + +bool GrammarMatcher::is_terminated() const { + return matcher.IsTerminated(); +} + +bool GrammarMatcher::accept(uint32_t token_id, bool log_rejection) { + const bool accepted = matcher.AcceptToken(token_id); + if (!accepted && log_rejection) { + CACTUS_LOG_WARN("grammar matcher", "Token id: " << token_id << " was not accepted by grammar matcher."); + } + return accepted; +} + +bool GrammarMatcher::next_bitmask(std::vector& bitmask, size_t logits_buffer_size) { + const size_t vocab_bitmask_size = xgrammar::GetBitmaskSize(tokenizer_info.GetVocabSize()); + const size_t logits_bitmask_size = xgrammar::GetBitmaskSize(logits_buffer_size); + bitmask.assign(std::max(vocab_bitmask_size, logits_bitmask_size), 0); + + int64_t bitmask_shape[1]; + int64_t bitmask_strides[1]; + bitmask_shape[0] = vocab_bitmask_size; + bitmask_strides[0] = 1; + + DLTensor bitmask_tensor; + bitmask_tensor.data = bitmask.data(); + bitmask_tensor.device = DLDevice{kDLCPU, 0}; + bitmask_tensor.ndim = 1; + bitmask_tensor.dtype = xgrammar::GetBitmaskDLType(); + bitmask_tensor.shape = bitmask_shape; + bitmask_tensor.strides = bitmask_strides; + bitmask_tensor.byte_offset = 0; + + return matcher.FillNextTokenBitmask(&bitmask_tensor, 0); +} + +} // namespace engine +} // namespace cactus diff --git a/cactus/engine/engine_model.cpp b/cactus/engine/engine_model.cpp index da6b253db..8c80170be 100644 --- a/cactus/engine/engine_model.cpp +++ b/cactus/engine/engine_model.cpp @@ -82,36 +82,8 @@ bool Model::init_internal(CactusGraph* gb, const std::string& model_folder, size return false; } - std::string vocab_file = model_folder + "/vocab.txt"; - std::string merges_file = model_folder + "/merges.txt"; - std::string tokenizer_config_file = model_folder + "/tokenizer_config.txt"; - TokenizerRuntimeConfig tokenizer_runtime_config = load_tokenizer_runtime_config(tokenizer_config_file); - - std::ifstream merges_check(merges_file); - bool has_merges = false; - if (merges_check.is_open()) { - std::string line; - int line_count = 0; - while (std::getline(merges_check, line) && line_count < 10) { - if (!line.empty() && line[0] != '#') { - has_merges = true; - break; - } - line_count++; - } - merges_check.close(); - } - - if (tokenizer_runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::BPE || - (tokenizer_runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::UNKNOWN && has_merges)) { - tokenizer_ = std::make_unique(); - } else { - tokenizer_ = std::make_unique(); - } - - if (!tokenizer_->load_vocabulary_with_config(vocab_file, merges_file, tokenizer_config_file)) { - return false; - } + tokenizer_ = create_tokenizer_from_model_dir(model_folder); + if (!tokenizer_) return false; graph_handle_ = gb; diff --git a/cactus/engine/engine_tokenizer.cpp b/cactus/engine/engine_tokenizer.cpp index d0749714f..094fcdf56 100644 --- a/cactus/engine/engine_tokenizer.cpp +++ b/cactus/engine/engine_tokenizer.cpp @@ -13,6 +13,23 @@ namespace cactus { namespace engine { namespace { + +bool model_uses_bpe_tokenizer(const std::string& merges_file) { + std::ifstream merges_check(merges_file); + if (!merges_check.is_open()) { + return false; + } + + std::string line; + int line_count = 0; + while (std::getline(merges_check, line) && line_count < 10) { + if (!line.empty() && line[0] != '#') { + return true; + } + ++line_count; + } + return false; +} std::string trim_copy(const std::string& value) { size_t start = value.find_first_not_of(" \t\r\n"); @@ -233,6 +250,27 @@ TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_f return config; } +std::unique_ptr create_tokenizer_from_model_dir(const std::string& model_dir) { + const std::string vocab_file = model_dir + "/vocab.txt"; + const std::string merges_file = model_dir + "/merges.txt"; + const std::string config_file = model_dir + "/tokenizer_config.txt"; + + TokenizerRuntimeConfig runtime_config = load_tokenizer_runtime_config(config_file); + + std::unique_ptr tokenizer; + if (runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::BPE || + (runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::UNKNOWN && model_uses_bpe_tokenizer(merges_file))) { + tokenizer = std::make_unique(); + } else { + tokenizer = std::make_unique(); + } + + if (!tokenizer->load_vocabulary_with_config(vocab_file, merges_file, config_file)) { + return nullptr; + } + return tokenizer; +} + void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens) { special_tokens.clear(); diff --git a/cactus/ffi/cactus_ffi.h b/cactus/ffi/cactus_ffi.h index 163e1805c..1a28185e1 100644 --- a/cactus/ffi/cactus_ffi.h +++ b/cactus/ffi/cactus_ffi.h @@ -20,6 +20,10 @@ extern "C" { typedef void* cactus_model_t; typedef void* cactus_index_t; typedef void* cactus_stream_transcribe_t; +typedef void* cactus_grammar_t; +typedef void* cactus_grammar_vocabulary_t; +typedef void* cactus_grammar_engine_t; +typedef void* cactus_grammar_matcher_t; typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); @@ -251,7 +255,7 @@ typedef uint64_t cactus_node_t; typedef struct { int32_t precision; size_t rank; - size_t shape[8]; + size_t shape[8]; size_t num_elements; size_t byte_size; } cactus_tensor_info_t; @@ -447,6 +451,67 @@ cactus_node_t node, void** out_ptr); CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, cactus_node_t node, cactus_tensor_info_t* out_info); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_ebnf(const char* ebnf, const char* start_symbol); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_json(); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_empty(); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_universal(); + +typedef struct cactus_grammar_json_schema_options_t { + bool any_whitespace; // negates indent and separators if true + int indent; + const char* separators[2]; + bool strict_mode; + int max_whitespace_count; // -1 means no limit +} cactus_grammar_json_schema_options_t; + +CACTUS_FFI_EXPORT cactus_grammar_json_schema_options_t cactus_grammar_json_schema_default_options(void); + +CACTUS_FFI_EXPORT cactus_grammar_vocabulary_t cactus_grammar_vocabulary_init(cactus_model_t model); +CACTUS_FFI_EXPORT size_t cactus_grammar_vocabulary_get_size(cactus_grammar_vocabulary_t vocabulary); +CACTUS_FFI_EXPORT int cactus_grammar_vocabulary_get_stop_token_ids( + cactus_grammar_vocabulary_t vocabulary, + uint32_t* buffer, + size_t buffer_size, + size_t* out_token_count +); +CACTUS_FFI_EXPORT void cactus_grammar_vocabulary_destroy(cactus_grammar_vocabulary_t vocabulary); + +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_json_schema( + const char* json_schema, + cactus_grammar_json_schema_options_t options +); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_regex(const char* regex); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_structural_tag( + const char* structural_tag_json, + cactus_grammar_vocabulary_t vocabulary // optional +); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_union(cactus_grammar_t* grammars, size_t num_grammars); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_concatenate(cactus_grammar_t* grammars, size_t num_grammars); +CACTUS_FFI_EXPORT int cactus_grammar_get_ebnf(cactus_grammar_t grammar, char* buffer, size_t buffer_size); +CACTUS_FFI_EXPORT bool cactus_grammar_is_empty(cactus_grammar_t grammar); +CACTUS_FFI_EXPORT bool cactus_grammar_is_universal(cactus_grammar_t grammar); +CACTUS_FFI_EXPORT void cactus_grammar_destroy(cactus_grammar_t grammar); + +CACTUS_FFI_EXPORT cactus_grammar_engine_t cactus_grammar_engine_init(cactus_grammar_vocabulary_t vocabulary); +CACTUS_FFI_EXPORT void cactus_grammar_engine_destroy(cactus_grammar_engine_t engine); +CACTUS_FFI_EXPORT cactus_grammar_matcher_t cactus_grammar_engine_compile_matcher( + cactus_grammar_engine_t engine, + cactus_grammar_t grammar +); + +CACTUS_FFI_EXPORT void cactus_grammar_matcher_destroy(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT void cactus_grammar_matcher_reset(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT void cactus_grammar_matcher_rollback(cactus_grammar_matcher_t matcher, int tokens); +CACTUS_FFI_EXPORT cactus_grammar_matcher_t cactus_grammar_matcher_fork(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT bool cactus_grammar_matcher_is_completed(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT bool cactus_grammar_matcher_is_terminated(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT bool cactus_grammar_matcher_accept(cactus_grammar_matcher_t matcher, uint32_t token_id); +CACTUS_FFI_EXPORT int cactus_grammar_matcher_next_bitmask( + cactus_grammar_matcher_t matcher, + int32_t* bitmask, + size_t logits_buffer_size +); + #ifdef __cplusplus } #endif diff --git a/cactus/ffi/cactus_grammar.cpp b/cactus/ffi/cactus_grammar.cpp new file mode 100644 index 000000000..41cdfcd26 --- /dev/null +++ b/cactus/ffi/cactus_grammar.cpp @@ -0,0 +1,421 @@ +#include "cactus_ffi.h" +#include "cactus_utils.h" + +#include +#include +#include +#include + +using namespace cactus::ffi; +using namespace cactus::engine; + +namespace cactus { +namespace ffi { + +struct CactusGrammarHandle { + std::unique_ptr grammar; +}; + +struct CactusGrammarVocabularyHandle { + std::unique_ptr vocabulary; +}; + +struct CactusGrammarEngineHandle { + std::unique_ptr engine; +}; + +struct CactusGrammarMatcherHandle { + std::unique_ptr matcher; +}; + +cactus_grammar_vocabulary_t make_grammar_vocabulary_from_tokenizer(const cactus::engine::Tokenizer& tokenizer) { + return new CactusGrammarVocabularyHandle{ + std::make_unique(tokenizer.get_grammar_vocabulary()) + }; +} + +} // namespace ffi +} // namespace cactus + +namespace { + +static cactus_grammar_t handle_exception(const char* operation, const std::string& error) { + last_error_message = std::string(operation) + ": " + error; + CACTUS_LOG_ERROR("grammar", last_error_message); + return nullptr; +} + +static bool handle_bool_exception(const char* operation, const std::string& error) { + last_error_message = std::string(operation) + ": " + error; + CACTUS_LOG_ERROR("grammar", last_error_message); + return false; +} + +static int handle_int_exception(const char* operation, const std::string& error) { + last_error_message = std::string(operation) + ": " + error; + CACTUS_LOG_ERROR("grammar", last_error_message); + return -1; +} + +template +static Handle* require_handle(const char* operation, RawHandle handle, const char* label) { + if (!handle) { + handle_exception(operation, std::string(label) + " is null"); + return nullptr; + } + return static_cast(handle); +} + +static std::vector collect_grammars(cactus_grammar_t* grammars, size_t num_grammars) { + std::vector collected; + collected.reserve(num_grammars); + for (size_t i = 0; i < num_grammars; ++i) { + if (!grammars || !grammars[i]) { + continue; + } + + auto* handle = static_cast(grammars[i]); + collected.push_back(*handle->grammar); + } + return collected; +} + +static CactusGrammarVocabularyHandle* require_vocabulary_handle( + const char* operation, + cactus_grammar_vocabulary_t vocabulary +) { + return require_handle(operation, vocabulary, "grammar vocabulary"); +} + +static CactusGrammarEngineHandle* require_engine_handle( + const char* operation, + cactus_grammar_engine_t engine +) { + return require_handle(operation, engine, "grammar engine"); +} + +static CactusGrammarMatcherHandle* require_matcher_handle( + const char* operation, + cactus_grammar_matcher_t matcher +) { + return require_handle(operation, matcher, "grammar matcher"); +} + +static CactusGrammarHandle* require_grammar_handle(const char* operation, cactus_grammar_t grammar) { + return require_handle(operation, grammar, "grammar"); +} + +template +static cactus_grammar_t make_grammar(const char* operation, Factory&& factory) { + try { + return new CactusGrammarHandle{ + std::make_unique(std::forward(factory)()) + }; + } catch (const std::exception& e) { + return handle_exception(operation, e.what()); + } +} + +} // anonymous namespace + +extern "C" { + +cactus_grammar_json_schema_options_t cactus_grammar_json_schema_default_options() { + return { true, 2, {",", ":"}, true, 1, }; +} + +cactus_grammar_vocabulary_t cactus_grammar_vocabulary_init(cactus_model_t model) { + if (!model) { + return handle_exception(__func__, "model is null"); + } + + try { + auto* handle = static_cast(model); + auto* tokenizer = handle->model ? handle->model->get_tokenizer() : nullptr; + if (!tokenizer) { + return handle_exception(__func__, "model tokenizer is null"); + } + return new CactusGrammarVocabularyHandle{ + std::make_unique(tokenizer->get_grammar_vocabulary()) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +size_t cactus_grammar_vocabulary_get_size(cactus_grammar_vocabulary_t vocabulary) { + auto* handle = require_vocabulary_handle(__func__, vocabulary); + if (!handle) return 0; + + return handle->vocabulary->vocab_size(); +} + +int cactus_grammar_vocabulary_get_stop_token_ids( + cactus_grammar_vocabulary_t vocabulary, + uint32_t* buffer, + size_t buffer_size, + size_t* out_token_count +) { + auto* handle = require_vocabulary_handle(__func__, vocabulary); + if (!handle) return -1; + if (!out_token_count) { + return handle_int_exception(__func__, "out_token_count is null"); + } + + const auto& stop_token_ids = handle->vocabulary->stop_token_ids(); + *out_token_count = stop_token_ids.size(); + if (!buffer) { + return handle_int_exception(__func__, "buffer is null"); + } + if (buffer_size < stop_token_ids.size()) { + return handle_int_exception(__func__, "buffer too small"); + } + + std::copy(stop_token_ids.begin(), stop_token_ids.end(), buffer); + return 0; +} + +void cactus_grammar_vocabulary_destroy(cactus_grammar_vocabulary_t vocabulary) { + if (vocabulary) delete static_cast(vocabulary); +} + +cactus_grammar_t cactus_grammar_init_ebnf(const char* ebnf, const char* start_symbol) { + if (!ebnf) { + return handle_exception(__func__, "ebnf is null"); + } + + return make_grammar(__func__, [&] { + return Grammar::ebnf(ebnf, start_symbol ? start_symbol : "root"); + }); +} + +cactus_grammar_t cactus_grammar_init_json() { + return make_grammar(__func__, [] { + return Grammar::json(); + }); +} + +cactus_grammar_t cactus_grammar_init_empty() { + return make_grammar(__func__, [] { + return Grammar(); + }); +} + +cactus_grammar_t cactus_grammar_init_universal() { + return make_grammar(__func__, [] { + return Grammar::universal(); + }); +} + +cactus_grammar_t cactus_grammar_init_json_schema( + const char* json_schema, + cactus_grammar_json_schema_options_t options +) { + if (!json_schema) { + return handle_exception(__func__, "json_schema is null"); + } + + if ((!options.any_whitespace) && (!options.separators[0] || !options.separators[1])) { + return handle_exception( + __func__, + "json schema separators must have 2 strings" + ); + } + + return make_grammar(__func__, [&] { + return Grammar::json_schema( + json_schema, + options.any_whitespace, + options.indent, + { + options.separators[0] ? options.separators[0] : ",", + options.separators[1] ? options.separators[1] : ":" + }, + options.strict_mode, + options.max_whitespace_count + ); + }); +} + +cactus_grammar_t cactus_grammar_init_regex(const char* regex) { + if (!regex) { + return handle_exception(__func__, "regex is null"); + } + + return make_grammar(__func__, [&] { + return Grammar::regex(regex); + }); +} + +cactus_grammar_t cactus_grammar_init_structural_tag( + const char* structural_tag_json, + cactus_grammar_vocabulary_t vocabulary +) { + if (!structural_tag_json) { + return handle_exception(__func__, "structural_tag_json is null"); + } + + return make_grammar(__func__, [&] { + auto* handle = static_cast(vocabulary); + return Grammar::structural_tag( + structural_tag_json, + handle ? handle->vocabulary.get() : nullptr + ); + }); +} + +cactus_grammar_t cactus_grammar_union(cactus_grammar_t* grammars, size_t num_grammars) { + return make_grammar(__func__, [&] { + return Grammar::unite(collect_grammars(grammars, num_grammars)); + }); +} + +cactus_grammar_t cactus_grammar_concatenate(cactus_grammar_t* grammars, size_t num_grammars) { + return make_grammar(__func__, [&] { + return Grammar::concatenate(collect_grammars(grammars, num_grammars)); + }); +} + +int cactus_grammar_get_ebnf(cactus_grammar_t grammar, char* buffer, size_t buffer_size) { + if (!grammar) { + return handle_int_exception(__func__, "grammar is null"); + } + if (!buffer || buffer_size == 0) { + return handle_int_exception(__func__, "buffer is null or buffer_size is 0"); + } + + try { + std::string ebnf = static_cast(grammar)->grammar->ebnf(); + if (ebnf.size() >= buffer_size) { + return handle_int_exception(__func__, "buffer too small"); + } + std::strcpy(buffer, ebnf.c_str()); + return 0; + } catch (const std::exception& e) { + return handle_int_exception(__func__, e.what()); + } +} + +bool cactus_grammar_is_empty(cactus_grammar_t grammar) { + if (!grammar) { + return handle_bool_exception(__func__, "grammar is null"); + } + + return static_cast(grammar)->grammar->is_empty(); +} + +bool cactus_grammar_is_universal(cactus_grammar_t grammar) { + if (!grammar) { + return handle_bool_exception(__func__, "grammar is null"); + } + + return static_cast(grammar)->grammar->is_universal(); +} + +void cactus_grammar_destroy(cactus_grammar_t grammar) { + if (grammar) delete static_cast(grammar); +} + +cactus_grammar_engine_t cactus_grammar_engine_init(cactus_grammar_vocabulary_t vocabulary) { + auto* handle = require_vocabulary_handle(__func__, vocabulary); + if (!handle) return nullptr; + + try { + return new CactusGrammarEngineHandle{ + std::make_unique(*handle->vocabulary) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +void cactus_grammar_engine_destroy(cactus_grammar_engine_t engine) { + if (engine) delete static_cast(engine); +} + +cactus_grammar_matcher_t cactus_grammar_engine_compile_matcher( + cactus_grammar_engine_t engine, + cactus_grammar_t grammar +) { + auto* engine_handle = require_engine_handle(__func__, engine); + if (!engine_handle) return nullptr; + auto* grammar_handle = require_grammar_handle(__func__, grammar); + if (!grammar_handle) return nullptr; + + try { + return new CactusGrammarMatcherHandle{ + std::make_unique( + engine_handle->engine->compile_matcher(*grammar_handle->grammar) + ) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +void cactus_grammar_matcher_destroy(cactus_grammar_matcher_t matcher) { + if (matcher) delete static_cast(matcher); +} + +void cactus_grammar_matcher_reset(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return; + handle->matcher->reset(); +} + +void cactus_grammar_matcher_rollback(cactus_grammar_matcher_t matcher, int tokens) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return; + handle->matcher->rollback(tokens); +} + +cactus_grammar_matcher_t cactus_grammar_matcher_fork(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return nullptr; + + try { + return new CactusGrammarMatcherHandle{ + std::make_unique(handle->matcher->fork()) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +bool cactus_grammar_matcher_is_completed(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + return handle ? handle->matcher->is_completed() : false; +} + +bool cactus_grammar_matcher_is_terminated(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + return handle ? handle->matcher->is_terminated() : false; +} + +bool cactus_grammar_matcher_accept(cactus_grammar_matcher_t matcher, uint32_t token_id) { + auto* handle = require_matcher_handle(__func__, matcher); + return handle ? handle->matcher->accept(token_id) : false; +} + +int cactus_grammar_matcher_next_bitmask( + cactus_grammar_matcher_t matcher, + int32_t* bitmask, + size_t logits_buffer_size +) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return -1; + if (!bitmask) { + return handle_int_exception(__func__, "bitmask is null"); + } + + try { + std::vector next_bitmask; + const bool should_apply = handle->matcher->next_bitmask(next_bitmask, logits_buffer_size); + std::copy(next_bitmask.begin(), next_bitmask.end(), bitmask); + return should_apply ? 1 : 0; + } catch (const std::exception& e) { + return handle_int_exception(__func__, e.what()); + } +} + +} diff --git a/cactus/ffi/cactus_utils.h b/cactus/ffi/cactus_utils.h index aad98d83c..4b5a22d47 100644 --- a/cactus/ffi/cactus_utils.h +++ b/cactus/ffi/cactus_utils.h @@ -1,6 +1,7 @@ #ifndef CACTUS_UTILS_H #define CACTUS_UTILS_H +#include "cactus_ffi.h" #include "../engine/engine.h" #include "../models/model.h" #include @@ -43,7 +44,7 @@ inline size_t get_memory_footprint_bytes() { PROCESS_MEMORY_COUNTERS_EX pmc; if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) return pmc.PrivateUsage; - + #elif defined(__linux__) || defined(__ANDROID__) std::ifstream statm("/proc/self/statm"); if (statm.is_open()) { @@ -730,16 +731,16 @@ inline std::vector parse_messages_json(const std::s std::vector messages; out_image_paths.clear(); if (out_audio_paths) out_audio_paths->clear(); - + size_t pos = json.find('['); if (pos == std::string::npos) { throw std::runtime_error("Invalid JSON: expected array"); } - + pos = json.find('{', pos); while (pos != std::string::npos) { cactus::engine::ChatMessage msg; - + size_t obj_start = pos; int brace_count = 1; size_t obj_end = obj_start + 1; @@ -751,25 +752,25 @@ inline std::vector parse_messages_json(const std::s size_t role_pos = json.find("\"role\"", pos); if (role_pos == std::string::npos || role_pos >= obj_end) break; - + size_t role_start = json.find('"', role_pos + 6) + 1; size_t role_end = json.find('"', role_start); msg.role = json.substr(role_start, role_end - role_start); - + size_t content_pos = json.find("\"content\"", role_end); if (content_pos != std::string::npos && content_pos < obj_end) { size_t content_start = json.find('"', content_pos + 9) + 1; size_t content_end = content_start; - + while (content_end < json.length()) { content_end = json.find('"', content_end); if (content_end == std::string::npos) break; if (json[content_end - 1] != '\\') break; content_end++; } - + msg.content = json.substr(content_start, content_end - content_start); - + size_t escape_pos = 0; while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { msg.content.replace(escape_pos, 2, "\n"); @@ -781,7 +782,7 @@ inline std::vector parse_messages_json(const std::s escape_pos += 1; } } - + auto parse_path_array = [&](const char* key, std::vector& dest, std::vector* out_paths) { size_t key_pos = json.find(key, pos); @@ -892,31 +893,31 @@ inline std::vector parse_messages_json(const std::s inline std::vector parse_tools_json(const std::string& json) { std::vector tools; - + if (json.empty()) return tools; - + size_t pos = json.find('['); if (pos == std::string::npos) return tools; - + pos = json.find("\"function\"", pos); while (pos != std::string::npos) { ToolFunction tool; size_t next_search = pos + 1; - + size_t name_pos = json.find("\"name\"", pos); if (name_pos != std::string::npos) { size_t name_start = json.find('"', name_pos + 6) + 1; size_t name_end = json.find('"', name_start); tool.name = json.substr(name_start, name_end - name_start); } - + size_t desc_pos = json.find("\"description\"", pos); if (desc_pos != std::string::npos) { size_t desc_start = json.find('"', desc_pos + 13) + 1; size_t desc_end = json.find('"', desc_start); tool.description = json.substr(desc_start, desc_end - desc_start); } - + size_t params_pos = json.find("\"parameters\"", pos); if (params_pos != std::string::npos) { size_t params_start = json.find('{', params_pos); @@ -1853,6 +1854,8 @@ inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buf return out; } +cactus_grammar_vocabulary_t make_grammar_vocabulary_from_tokenizer(const cactus::engine::Tokenizer& tokenizer); + } // namespace ffi } // namespace cactus diff --git a/cactus/graph/graph.h b/cactus/graph/graph.h index 9ad6fe389..b43617758 100644 --- a/cactus/graph/graph.h +++ b/cactus/graph/graph.h @@ -149,7 +149,8 @@ enum class OpType { LEAKY_RELU, CONV2D_K3S1P1, STATS_POOL, - WEIGHTED_STATS_POOL + WEIGHTED_STATS_POOL, + DENSE_MLP_INT4_FUSED }; struct PrecisionTraits { @@ -424,6 +425,7 @@ void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_dense_mlp_int4_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); @@ -554,6 +556,8 @@ class CactusGraph { size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); size_t topk(size_t input, size_t k); + size_t dense_mlp_int4_fused(size_t hidden, size_t gate_weight, + size_t up_weight, size_t down_weight); size_t moe_layer(size_t hidden, size_t routing_probs, size_t topk_indices, diff --git a/cactus/graph/graph_builder.cpp b/cactus/graph/graph_builder.cpp index d4deed0d0..ca7b57edc 100644 --- a/cactus/graph/graph_builder.cpp +++ b/cactus/graph/graph_builder.cpp @@ -355,6 +355,31 @@ size_t CactusGraph::topk(size_t input, size_t k) { return add_node(OpType::TOPK, {input}, output_shape, params); } +size_t CactusGraph::dense_mlp_int4_fused(size_t hidden, size_t gate_weight, + size_t up_weight, size_t down_weight) { + const auto& hidden_buf = get_output_buffer(hidden); + if (hidden_buf.shape.size() < 2) { + throw std::runtime_error("dense_mlp_int4_fused requires hidden of rank >= 2"); + } + const auto& down_buf = get_output_buffer(down_weight); + size_t hidden_dim; + if (down_buf.is_interleaved && down_buf.original_N > 0) { + hidden_dim = down_buf.original_N; + } else { + hidden_dim = down_buf.shape[down_buf.shape.size() - 2]; + } + + std::vector output_shape = hidden_buf.shape; + output_shape[output_shape.size() - 1] = hidden_dim; + + OpParams params; + params.output_precision = Precision::FP16; + + return add_node(OpType::DENSE_MLP_INT4_FUSED, + {hidden, gate_weight, up_weight, down_weight}, + output_shape, params); +} + size_t CactusGraph::moe_layer(size_t hidden, size_t routing_probs, size_t topk_indices, diff --git a/cactus/graph/graph_execute.cpp b/cactus/graph/graph_execute.cpp index dd75389fb..f8bbc11a5 100644 --- a/cactus/graph/graph_execute.cpp +++ b/cactus/graph/graph_execute.cpp @@ -68,6 +68,7 @@ DECLARE_COMPUTE(compute_sample_node); DECLARE_COMPUTE(compute_topk_node); DECLARE_COMPUTE(compute_scatter_topk_node); DECLARE_COMPUTE(compute_moe_layer_node); +DECLARE_COMPUTE(compute_dense_mlp_int4_fused_node); DECLARE_COMPUTE(compute_persistent_node); DECLARE_COMPUTE(compute_quantize_activations_node); extern void shrink_thread_local_buffers(); @@ -140,6 +141,7 @@ static const std::unordered_map dispatch_table = { {OpType::TOPK, compute_topk_node}, {OpType::SCATTER_TOPK, compute_scatter_topk_node}, {OpType::MOE_LAYER, compute_moe_layer_node}, + {OpType::DENSE_MLP_INT4_FUSED, compute_dense_mlp_int4_fused_node}, {OpType::PERSISTENT, compute_persistent_node}, {OpType::QUANTIZE_ACTIVATIONS, compute_quantize_activations_node}, {OpType::LSTM_CELL, compute_lstm_cell_node}, @@ -185,7 +187,8 @@ static const char* op_type_names[] = { "LEAKY_RELU", "CONV2D_K3S1P1", "STATS_POOL", - "WEIGHTED_STATS_POOL" + "WEIGHTED_STATS_POOL", + "DENSE_MLP_INT4_FUSED" }; static const char* get_op_name(OpType op) { diff --git a/cactus/graph/graph_io.cpp b/cactus/graph/graph_io.cpp index e100baf96..8eded5e94 100644 --- a/cactus/graph/graph_io.cpp +++ b/cactus/graph/graph_io.cpp @@ -1,4 +1,5 @@ #include "graph.h" +#include "graph_param_io.h" #include #include #include @@ -35,14 +36,6 @@ namespace { out.write(reinterpret_cast(&v), sizeof(v)); } - inline void write_i32(std::ostream& out, int32_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); - } - - inline void write_f32(std::ostream& out, float v) { - out.write(reinterpret_cast(&v), sizeof(v)); - } - void write_size_vector(std::ostream& out, const std::vector& values) { uint32_t size = static_cast(values.size()); write_u32(out, size); @@ -59,118 +52,6 @@ namespace { } } - void write_op_params(std::ostream& out, const GraphFile::NodeEntry& node) { - uint32_t param_flags = 0; - - constexpr uint32_t PARAM_SCALAR = 1u << 0; - constexpr uint32_t PARAM_AXIS = 1u << 1; - constexpr uint32_t PARAM_NEW_SHAPE = 1u << 2; - constexpr uint32_t PARAM_PRETRANSPOSED_RHS = 1u << 3; - constexpr uint32_t PARAM_BACKEND = 1u << 4; - constexpr uint32_t PARAM_SLICE = 1u << 5; - constexpr uint32_t PARAM_EPSILON = 1u << 6; - constexpr uint32_t PARAM_NUM_GROUPS = 1u << 7; - constexpr uint32_t PARAM_INDEX_VALUE = 1u << 8; - - switch (node.op_type) { - case OpType::POW: - case OpType::SCALAR_ADD: - case OpType::SCALAR_SUBTRACT: - case OpType::SCALAR_MULTIPLY: - case OpType::SCALAR_DIVIDE: - param_flags |= PARAM_SCALAR; - break; - default: - break; - } - - switch (node.op_type) { - case OpType::SOFTMAX: - case OpType::SUM: - case OpType::MEAN: - case OpType::VARIANCE: - case OpType::MIN: - case OpType::MAX: - case OpType::INDEX: - case OpType::CONCAT: - case OpType::CAT: - param_flags |= PARAM_AXIS; - break; - default: - break; - } - - if (node.op_type == OpType::INDEX) { - param_flags |= PARAM_INDEX_VALUE; - } - - switch (node.op_type) { - case OpType::VIEW: - case OpType::RESHAPE: - case OpType::FLATTEN: - param_flags |= PARAM_NEW_SHAPE; - break; - default: - break; - } - - if (node.op_type == OpType::MATMUL) { - param_flags |= PARAM_PRETRANSPOSED_RHS; - param_flags |= PARAM_BACKEND; - } - - if (node.op_type == OpType::SLICE) { - param_flags |= PARAM_AXIS; - param_flags |= PARAM_SLICE; - } - - switch (node.op_type) { - case OpType::RMS_NORM: - case OpType::LAYERNORM: - case OpType::GROUPNORM: - case OpType::BATCHNORM: - param_flags |= PARAM_EPSILON; - break; - default: - break; - } - - if (node.op_type == OpType::GROUPNORM) { - param_flags |= PARAM_NUM_GROUPS; - } - - write_u32(out, param_flags); - - if (param_flags & PARAM_SCALAR) { - write_f32(out, node.params.scalar); - } - if (param_flags & PARAM_AXIS) { - write_i32(out, static_cast(node.params.axis)); - } - if (param_flags & PARAM_NEW_SHAPE) { - write_size_vector(out, node.params.new_shape); - } - if (param_flags & PARAM_PRETRANSPOSED_RHS) { - write_u32(out, node.params.pretransposed_rhs ? 1u : 0u); - } - if (param_flags & PARAM_BACKEND) { - write_u32(out, static_cast(node.params.backend)); - } - if (param_flags & PARAM_SLICE) { - write_u64(out, static_cast(node.params.slice_start)); - write_u64(out, static_cast(node.params.slice_length)); - } - if (param_flags & PARAM_EPSILON) { - write_f32(out, node.params.epsilon); - } - if (param_flags & PARAM_NUM_GROUPS) { - write_u64(out, static_cast(node.params.num_groups)); - } - if (param_flags & PARAM_INDEX_VALUE) { - write_u64(out, static_cast(node.params.index_value)); - } - } - std::vector compute_leaf_outputs(const GraphFile::SerializedGraph& sg) { std::vector referenced(sg.nodes.size(), false); @@ -210,7 +91,7 @@ namespace { write_u32_vector(out, node.inputs); write_size_vector(out, node.output_shape); write_u32(out, static_cast(node.precision)); - write_op_params(out, node); + GraphParamIO::write_op_params(out, node.op_type, node.params); } if (!out) { @@ -237,24 +118,6 @@ namespace { return v; } - int32_t read_i32(std::istream& in) { - int32_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) { - throw std::runtime_error("Unexpected EOF while reading int32"); - } - return v; - } - - float read_f32(std::istream& in) { - float v = 0.0f; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) { - throw std::runtime_error("Unexpected EOF while reading float"); - } - return v; - } - std::vector read_u32_vector(std::istream& in) { uint32_t count = read_u32(in); std::vector values; @@ -275,67 +138,6 @@ namespace { return values; } - void read_op_params(std::istream& in, GraphFile::NodeEntry& node) { - uint32_t param_flags = read_u32(in); - - constexpr uint32_t PARAM_SCALAR = 1u << 0; - constexpr uint32_t PARAM_AXIS = 1u << 1; - constexpr uint32_t PARAM_NEW_SHAPE = 1u << 2; - constexpr uint32_t PARAM_PRETRANSPOSED_RHS = 1u << 3; - constexpr uint32_t PARAM_BACKEND = 1u << 4; - constexpr uint32_t PARAM_SLICE = 1u << 5; - constexpr uint32_t PARAM_EPSILON = 1u << 6; - constexpr uint32_t PARAM_NUM_GROUPS = 1u << 7; - constexpr uint32_t PARAM_INDEX_VALUE = 1u << 8; - constexpr uint32_t PARAM_KNOWN_MASK = - PARAM_SCALAR | - PARAM_AXIS | - PARAM_NEW_SHAPE | - PARAM_PRETRANSPOSED_RHS | - PARAM_BACKEND | - PARAM_SLICE | - PARAM_EPSILON | - PARAM_NUM_GROUPS | - PARAM_INDEX_VALUE; - - if ((param_flags & ~PARAM_KNOWN_MASK) != 0) { - throw std::runtime_error("Graph file corrupted: unknown param flags"); - } - - if (param_flags & PARAM_SCALAR) { - node.params.scalar = read_f32(in); - } - if (param_flags & PARAM_AXIS) { - node.params.axis = static_cast(read_i32(in)); - } - if (param_flags & PARAM_NEW_SHAPE) { - node.params.new_shape = read_size_vector(in); - } - if (param_flags & PARAM_PRETRANSPOSED_RHS) { - node.params.pretransposed_rhs = (read_u32(in) != 0); - } - if (param_flags & PARAM_BACKEND) { - uint32_t backend_val = read_u32(in); - if (backend_val > static_cast(ComputeBackend::NPU)) { - throw std::runtime_error("Graph file corrupted: invalid backend"); - } - node.params.backend = static_cast(backend_val); - } - if (param_flags & PARAM_SLICE) { - node.params.slice_start = static_cast(read_u64(in)); - node.params.slice_length = static_cast(read_u64(in)); - } - if (param_flags & PARAM_EPSILON) { - node.params.epsilon = read_f32(in); - } - if (param_flags & PARAM_NUM_GROUPS) { - node.params.num_groups = static_cast(read_u64(in)); - } - if (param_flags & PARAM_INDEX_VALUE) { - node.params.index_value = static_cast(read_u64(in)); - } - } - bool is_binary_broadcast_op(OpType op_type) { switch (op_type) { case OpType::ADD: @@ -373,7 +175,7 @@ namespace { if (header.magic != CACTUS_GRAPH_MAGIC) { throw std::runtime_error("Invalid graph file: bad magic"); } - if (header.version != 1) { + if (header.version != 4) { throw std::runtime_error("Unsupported graph file version: " + std::to_string(header.version)); } @@ -396,7 +198,7 @@ namespace { throw std::runtime_error("Graph file corrupted: invalid precision"); } node.precision = static_cast(precision_val); - read_op_params(in, node); + GraphParamIO::read_op_params(in, node.op_type, node.params); return node; } @@ -607,7 +409,7 @@ void save_graph(const CactusGraph& graph, SerializedGraph sg; sg.header.magic = CACTUS_GRAPH_MAGIC; - sg.header.version = 1; + sg.header.version = 4; sg.header.node_count = static_cast(graph.nodes_.size()); sg.header.flags = 0; diff --git a/cactus/graph/graph_ops_nn.cpp b/cactus/graph/graph_ops_nn.cpp index f840c1c14..ae6f92b0e 100644 --- a/cactus/graph/graph_ops_nn.cpp +++ b/cactus/graph/graph_ops_nn.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace { thread_local std::vector<__fp16> transpose_buffer_fp16; @@ -511,6 +512,182 @@ void compute_moe_layer_node(GraphNode& node, const std::vector gate_tile; + std::vector<__fp16> up_tile; + }; + + // Inline to skip cactus_fp16_to_int8's parallel_for orchestration, which + // dispatches ~5 threads at d_ffn=12288 and dominates the SIMD work. + static inline void fp16_to_int8_inplace(const __fp16* src, int8_t* dst, size_t n, float scale) { + const float inv_scale = 1.0f / scale; + const float32x4_t inv_vec = vdupq_n_f32(inv_scale); + const float32x4_t min_vec = vdupq_n_f32(-128.0f); + const float32x4_t max_vec = vdupq_n_f32(127.0f); + size_t i = 0; + for (; i + 8 <= n; i += 8) { + float16x8_t in = vld1q_f16(src + i); + float32x4_t lo = vmulq_f32(vcvt_f32_f16(vget_low_f16(in)), inv_vec); + float32x4_t hi = vmulq_f32(vcvt_f32_f16(vget_high_f16(in)), inv_vec); + lo = vmaxq_f32(vminq_f32(lo, max_vec), min_vec); + hi = vmaxq_f32(vminq_f32(hi, max_vec), min_vec); + int16x8_t s = vcombine_s16(vqmovn_s32(vcvtnq_s32_f32(lo)), + vqmovn_s32(vcvtnq_s32_f32(hi))); + vst1_s8(dst + i, vqmovn_s16(s)); + } + for (; i < n; ++i) { + float v = static_cast(src[i]) * inv_scale; + if (v > 127.0f) v = 127.0f; else if (v < -128.0f) v = -128.0f; + dst[i] = static_cast(std::lround(v)); + } + } +} + +void compute_dense_mlp_int4_fused_node( + GraphNode& node, + const std::vector>& nodes, + const std::unordered_map& node_index_map) { + const auto& hidden_buf = get_input(node, 0, nodes, node_index_map); + const auto& gate_buf = get_input(node, 1, nodes, node_index_map); + const auto& up_buf = get_input(node, 2, nodes, node_index_map); + const auto& down_buf = get_input(node, 3, nodes, node_index_map); + + if (hidden_buf.precision != Precision::FP16) { + throw std::runtime_error("dense_mlp_int4_fused: hidden must be FP16"); + } + if (gate_buf.precision != Precision::INT4 || gate_buf.group_size == 0 || + up_buf.precision != Precision::INT4 || up_buf.group_size == 0 || + down_buf.precision != Precision::INT4 || down_buf.group_size == 0) { + throw std::runtime_error("dense_mlp_int4_fused: weights must be INT4 group-quantized"); + } + const size_t group_size = gate_buf.group_size; + if (up_buf.group_size != group_size || down_buf.group_size != group_size) { + throw std::runtime_error("dense_mlp_int4_fused: gate/up/down group_size must match"); + } + + const auto& shape = hidden_buf.shape; + const size_t hidden_dim_in = shape.back(); + size_t M = 1; + for (size_t i = 0; i + 1 < shape.size(); ++i) M *= shape[i]; + + const size_t d_ffn = gate_buf.is_interleaved ? gate_buf.original_N : gate_buf.shape[0]; + const size_t hidden_dim = down_buf.is_interleaved ? down_buf.original_N : down_buf.shape[0]; + if (hidden_dim_in != hidden_dim) { + throw std::runtime_error("dense_mlp_int4_fused: hidden dim mismatch with down weight"); + } + if (d_ffn % 4 != 0 || d_ffn % group_size != 0 || hidden_dim % 4 != 0) { + throw std::runtime_error("dense_mlp_int4_fused: d_ffn must be a multiple of 4 and of group_size, hidden_dim a multiple of 4"); + } + const size_t blocks_d_ffn = d_ffn / 4; + const size_t blocks_hidden = hidden_dim / 4; + + constexpr size_t tile_n_blocks = 32; + constexpr size_t tile_rows = tile_n_blocks * 4; + if (tile_rows % group_size != 0) { + throw std::runtime_error("dense_mlp_int4_fused: tile_rows must be a multiple of group_size"); + } + + const __fp16* hidden_fp16 = hidden_buf.data_as<__fp16>(); + __fp16* output = node.output_buffer.data_as<__fp16>(); + + const int8_t* gate_w = gate_buf.data_as(); + const __fp16* gate_s = gate_buf.scales_as_fp16(); + const int8_t* up_w = up_buf.data_as(); + const __fp16* up_s = up_buf.scales_as_fp16(); + const int8_t* down_w = down_buf.data_as(); + const __fp16* down_s = down_buf.scales_as_fp16(); + + thread_local std::vector dense_x_int8; + thread_local std::vector dense_x_scales; + thread_local std::vector<__fp16> dense_h_fp16; + thread_local std::vector dense_h_int8; + if (dense_x_int8.size() < M * hidden_dim) dense_x_int8.resize(M * hidden_dim); + if (dense_x_scales.size() < M) dense_x_scales.resize(M); + if (dense_h_fp16.size() < d_ffn) dense_h_fp16.resize(d_ffn); + if (dense_h_int8.size() < d_ffn) dense_h_int8.resize(d_ffn); + + auto& pool = CactusThreading::get_thread_pool(); + const size_t num_workers = pool.num_workers(); + std::vector dense_fused_scratch(num_workers); + for (auto& sc : dense_fused_scratch) { + sc.gate_tile.resize(tile_rows); + sc.up_tile.resize(tile_rows); + } + + for (size_t t = 0; t < M; ++t) { + const __fp16* x = hidden_fp16 + t * hidden_dim; + float maxabs = cactus_fp16_max_abs(x, hidden_dim); + float xs = std::max(maxabs / 127.0f, 1e-10f); + dense_x_scales[t] = xs; + cactus_fp16_to_int8(x, dense_x_int8.data() + t * hidden_dim, hidden_dim, xs); + } + + const size_t tiles_d_ffn = (blocks_d_ffn + tile_n_blocks - 1) / tile_n_blocks; + const size_t tiles_hidden = (blocks_hidden + tile_n_blocks - 1) / tile_n_blocks; + const size_t num_threads_p1 = std::min(num_workers, tiles_d_ffn); + const size_t num_threads_p2 = std::min(num_workers, tiles_hidden); + + for (size_t t = 0; t < M; ++t) { + const int8_t* x_int8 = dense_x_int8.data() + t * hidden_dim; + const float x_scale = dense_x_scales[t]; + __fp16* y = output + t * hidden_dim; + __fp16* h_full = dense_h_fp16.data(); + int8_t* h_int8 = dense_h_int8.data(); + + std::atomic p1_next{0}; + std::atomic p2_next{0}; + + auto run = [&](size_t nt, auto&& task) { + if (nt <= 1) { task(0); return; } + pool.enqueue_n_threads(nt - 1, nt - 1, [&](size_t s, size_t e) { + for (size_t w = s; w < e; ++w) task(w + 1); + }); + task(0); + pool.wait_all(); + }; + + run(num_threads_p1, [&](size_t worker_id) { + auto& sc = dense_fused_scratch[worker_id]; + __fp16* gate_tile_buf = sc.gate_tile.data(); + __fp16* up_tile_buf = sc.up_tile.data(); + while (true) { + size_t tile_id = p1_next.fetch_add(1, std::memory_order_relaxed); + if (tile_id >= tiles_d_ffn) break; + const size_t blk_start = tile_id * tile_n_blocks; + const size_t blk_end = std::min(blk_start + tile_n_blocks, blocks_d_ffn); + const size_t k_lo = blk_start * 4; + const size_t k_count = (blk_end - blk_start) * 4; + + cactus_gemv_int4_block_range( + x_int8, x_scale, gate_w, gate_s, gate_tile_buf - k_lo, + hidden_dim, d_ffn, group_size, blk_start, blk_end); + cactus_gelu_f16(gate_tile_buf, gate_tile_buf, k_count); + cactus_gemv_int4_block_range( + x_int8, x_scale, up_w, up_s, up_tile_buf - k_lo, + hidden_dim, d_ffn, group_size, blk_start, blk_end); + cactus_multiply_f16(gate_tile_buf, up_tile_buf, h_full + k_lo, k_count); + } + }); + + const float h_maxabs = cactus_fp16_max_abs(h_full, d_ffn); + const float h_scale = std::max(h_maxabs / 127.0f, 1e-10f); + fp16_to_int8_inplace(h_full, h_int8, d_ffn, h_scale); + + run(num_threads_p2, [&](size_t /*worker_id*/) { + while (true) { + size_t tile_id = p2_next.fetch_add(1, std::memory_order_relaxed); + if (tile_id >= tiles_hidden) break; + const size_t blk_start = tile_id * tile_n_blocks; + const size_t blk_end = std::min(blk_start + tile_n_blocks, blocks_hidden); + cactus_gemv_int4_block_range( + h_int8, h_scale, down_w, down_s, y, + d_ffn, hidden_dim, group_size, blk_start, blk_end); + } + }); + } +} + void compute_rms_norm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { const auto& input_buffer = get_input(node, 0, nodes, node_index_map); const auto& weight_buffer = get_input(node, 1, nodes, node_index_map); diff --git a/cactus/graph/graph_param_io.cpp b/cactus/graph/graph_param_io.cpp new file mode 100644 index 000000000..6d49c9ad6 --- /dev/null +++ b/cactus/graph/graph_param_io.cpp @@ -0,0 +1,380 @@ +#include "graph_param_io.h" + +#include +#include +#include +#include + +namespace { + +void write_u32(std::ostream& out, uint32_t v) { + out.write(reinterpret_cast(&v), sizeof(v)); +} + +void write_u64(std::ostream& out, uint64_t v) { + out.write(reinterpret_cast(&v), sizeof(v)); +} + +void write_i32(std::ostream& out, int32_t v) { + out.write(reinterpret_cast(&v), sizeof(v)); +} + +void write_f32(std::ostream& out, float v) { + out.write(reinterpret_cast(&v), sizeof(v)); +} + +uint32_t read_u32(std::istream& in) { + uint32_t v = 0; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) throw std::runtime_error("Unexpected EOF while reading uint32"); + return v; +} + +uint64_t read_u64(std::istream& in) { + uint64_t v = 0; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) throw std::runtime_error("Unexpected EOF while reading uint64"); + return v; +} + +int32_t read_i32(std::istream& in) { + int32_t v = 0; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) throw std::runtime_error("Unexpected EOF while reading int32"); + return v; +} + +float read_f32(std::istream& in) { + float v = 0.0f; + in.read(reinterpret_cast(&v), sizeof(v)); + if (!in) throw std::runtime_error("Unexpected EOF while reading float"); + return v; +} + +void write_size_vector(std::ostream& out, const std::vector& values) { + write_u32(out, static_cast(values.size())); + for (size_t v : values) write_u64(out, static_cast(v)); +} + +std::vector read_size_vector(std::istream& in) { + uint32_t count = read_u32(in); + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; ++i) values.push_back(static_cast(read_u64(in))); + return values; +} + +void write_u32_vector(std::ostream& out, const std::vector& values) { + write_u32(out, static_cast(values.size())); + for (uint32_t v : values) write_u32(out, v); +} + +std::vector read_u32_vector(std::istream& in) { + uint32_t count = read_u32(in); + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; ++i) values.push_back(read_u32(in)); + return values; +} + +void write_f32_vector(std::ostream& out, const std::vector& values) { + write_u32(out, static_cast(values.size())); + for (float v : values) write_f32(out, v); +} + +std::vector read_f32_vector(std::istream& in) { + uint32_t count = read_u32(in); + std::vector values; + values.reserve(count); + for (uint32_t i = 0; i < count; ++i) values.push_back(read_f32(in)); + return values; +} + +enum class ParamField : uint32_t { + Scalar = 1, + Axis, + NewShape, + PretransposedRhs, + Backend, + SliceStart, + SliceLength, + Epsilon, + NumGroups, + IndexValue, + Permutation, + Scale, + Theta, + PositionOffset, + WindowSize, + IsCausal, + AttentionMaskIsAdditive, + LogitCap, + TopK, + Temperature, + TopP, + MinP, + RepetitionPenalty, + RandomSeed, + BiasIndices, + BiasValues, + NormalizeRouting, + NumExperts, + NumExpertsPerTok, + MoeGated, + Activation, + NumKvHeads, + ChunkSize, + DstHeight, + DstWidth, + AlignCorners, + NumClasses, + NumAltupInputs, + KernelSize, + Stride, + Dilation, + NumFftBins, + CacheSeqLen, + HeadDim, + VHeadDim, + CachedKeysInt8Ptr, + CachedValuesInt8Ptr, + CachedKScalesPtr, + CachedVScalesPtr, +}; + +enum class FieldPersistence { + Persistent, + RuntimeOnly, + Derived, +}; + +struct FieldSpec { + ParamField field; + FieldPersistence persistence; +}; + +using Schema = std::vector; + +const Schema& op_schema(OpType op_type) { + static const Schema kEmpty{}; + static const std::unordered_map kSchemas = { + {OpType::POW, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_ADD, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_SUBTRACT, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_MULTIPLY, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SCALAR_DIVIDE, {{ParamField::Scalar, FieldPersistence::Persistent}}}, + {OpType::SOFTMAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::SUM, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::MEAN, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::VARIANCE, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::MIN, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::MAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::INDEX, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::IndexValue, FieldPersistence::Persistent}}}, + {OpType::CONCAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::CAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::VIEW, {{ParamField::NewShape, FieldPersistence::Persistent}}}, + {OpType::RESHAPE, {{ParamField::NewShape, FieldPersistence::Persistent}}}, + {OpType::FLATTEN, {{ParamField::NewShape, FieldPersistence::Persistent}}}, + {OpType::MATMUL, {{ParamField::PretransposedRhs, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::TRANSPOSE, {{ParamField::Permutation, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::SLICE, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::SliceStart, FieldPersistence::Persistent}, {ParamField::SliceLength, FieldPersistence::Persistent}}}, + {OpType::RMS_NORM, {{ParamField::Epsilon, FieldPersistence::Persistent}}}, + {OpType::LAYERNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}}}, + {OpType::GROUPNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::NumGroups, FieldPersistence::Persistent}}}, + {OpType::BATCHNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::Axis, FieldPersistence::Persistent}}}, + {OpType::ROPE, {{ParamField::Theta, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::ROPE_GPTJ, {{ParamField::Theta, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::Scalar, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::TOPK, {{ParamField::TopK, FieldPersistence::Persistent}}}, + {OpType::ATTENTION, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::WindowSize, FieldPersistence::Persistent}, {ParamField::IsCausal, FieldPersistence::Persistent}, {ParamField::AttentionMaskIsAdditive, FieldPersistence::Persistent}, {ParamField::LogitCap, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, + {OpType::REL_POS_BIAS, {{ParamField::Scale, FieldPersistence::Persistent}}}, + {OpType::ATTENTION_INT8_HYBRID, { + {ParamField::Scale, FieldPersistence::Persistent}, + {ParamField::PositionOffset, FieldPersistence::Persistent}, + {ParamField::WindowSize, FieldPersistence::Persistent}, + {ParamField::NumKvHeads, FieldPersistence::Persistent}, + {ParamField::CacheSeqLen, FieldPersistence::Persistent}, + {ParamField::HeadDim, FieldPersistence::Persistent}, + {ParamField::VHeadDim, FieldPersistence::Persistent}, + {ParamField::CachedKeysInt8Ptr, FieldPersistence::RuntimeOnly}, + {ParamField::CachedValuesInt8Ptr, FieldPersistence::RuntimeOnly}, + {ParamField::CachedKScalesPtr, FieldPersistence::RuntimeOnly}, + {ParamField::CachedVScalesPtr, FieldPersistence::RuntimeOnly}, + }}, + {OpType::MOE_LAYER, {{ParamField::Scalar, FieldPersistence::Persistent}, {ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::NormalizeRouting, FieldPersistence::Persistent}, {ParamField::NumExperts, FieldPersistence::Persistent}, {ParamField::NumExpertsPerTok, FieldPersistence::Persistent}, {ParamField::MoeGated, FieldPersistence::Persistent}, {ParamField::Activation, FieldPersistence::Persistent}}}, + {OpType::GATED_DELTANET_DECODE, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::NumKvHeads, FieldPersistence::Persistent}}}, + {OpType::GATED_DELTANET_PREFILL, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::NumKvHeads, FieldPersistence::Persistent}, {ParamField::ChunkSize, FieldPersistence::Persistent}}}, + {OpType::STFT, {{ParamField::Stride, FieldPersistence::Persistent}, {ParamField::NumFftBins, FieldPersistence::Persistent}}}, + {OpType::BILINEAR_INTERPOLATION, {{ParamField::DstHeight, FieldPersistence::Persistent}, {ParamField::DstWidth, FieldPersistence::Persistent}, {ParamField::AlignCorners, FieldPersistence::Persistent}}}, + {OpType::SCATTER_TOPK, {{ParamField::NumClasses, FieldPersistence::Persistent}}}, + {OpType::ALTUP_PREDICT, {{ParamField::NumAltupInputs, FieldPersistence::Persistent}}}, + {OpType::ALTUP_CORRECT, {{ParamField::NumAltupInputs, FieldPersistence::Persistent}}}, + {OpType::MAXPOOL1D, {{ParamField::KernelSize, FieldPersistence::Persistent}, {ParamField::Stride, FieldPersistence::Persistent}}}, + {OpType::CONV1D_CAUSAL, {{ParamField::Dilation, FieldPersistence::Persistent}}}, + {OpType::CONV1D_K3, {{ParamField::Stride, FieldPersistence::Persistent}}}, + {OpType::CONV1D_K7S3, {{ParamField::Stride, FieldPersistence::Persistent}}}, + {OpType::CONV1D, {{ParamField::Stride, FieldPersistence::Persistent}}}, + {OpType::SAMPLE, {{ParamField::Temperature, FieldPersistence::Persistent}, {ParamField::TopP, FieldPersistence::Persistent}, {ParamField::MinP, FieldPersistence::Persistent}, {ParamField::RepetitionPenalty, FieldPersistence::Persistent}, {ParamField::TopK, FieldPersistence::Persistent}, {ParamField::RandomSeed, FieldPersistence::Persistent}, {ParamField::BiasIndices, FieldPersistence::Persistent}, {ParamField::BiasValues, FieldPersistence::Persistent}}}, + }; + + auto it = kSchemas.find(op_type); + return it == kSchemas.end() ? kEmpty : it->second; +} + +void write_field(std::ostream& out, ParamField field, const OpParams& params) { + switch (field) { + case ParamField::Scalar: write_f32(out, params.scalar); break; + case ParamField::Axis: write_i32(out, static_cast(params.axis)); break; + case ParamField::NewShape: write_size_vector(out, params.new_shape); break; + case ParamField::PretransposedRhs: write_u32(out, params.pretransposed_rhs ? 1u : 0u); break; + case ParamField::Backend: write_u32(out, static_cast(params.backend)); break; + case ParamField::SliceStart: write_u64(out, static_cast(params.slice_start)); break; + case ParamField::SliceLength: write_u64(out, static_cast(params.slice_length)); break; + case ParamField::Epsilon: write_f32(out, params.epsilon); break; + case ParamField::NumGroups: write_u64(out, static_cast(params.num_groups)); break; + case ParamField::IndexValue: write_u64(out, static_cast(params.index_value)); break; + case ParamField::Permutation: write_size_vector(out, params.permutation); break; + case ParamField::Scale: write_f32(out, params.scale); break; + case ParamField::Theta: write_f32(out, params.theta); break; + case ParamField::PositionOffset: write_u64(out, static_cast(params.position_offset)); break; + case ParamField::WindowSize: write_u64(out, static_cast(params.window_size)); break; + case ParamField::IsCausal: write_u32(out, params.is_causal ? 1u : 0u); break; + case ParamField::AttentionMaskIsAdditive: write_u32(out, params.attention_mask_is_additive ? 1u : 0u); break; + case ParamField::LogitCap: write_f32(out, params.logit_cap); break; + case ParamField::TopK: write_u64(out, static_cast(params.top_k)); break; + case ParamField::Temperature: write_f32(out, params.temperature); break; + case ParamField::TopP: write_f32(out, params.top_p); break; + case ParamField::MinP: write_f32(out, params.min_p); break; + case ParamField::RepetitionPenalty: write_f32(out, params.repetition_penalty); break; + case ParamField::RandomSeed: write_u64(out, static_cast(params.random_seed)); break; + case ParamField::BiasIndices: write_u32_vector(out, params.bias_indices); break; + case ParamField::BiasValues: write_f32_vector(out, params.bias_values); break; + case ParamField::NormalizeRouting: write_u32(out, params.normalize_routing ? 1u : 0u); break; + case ParamField::NumExperts: write_u64(out, static_cast(params.num_experts)); break; + case ParamField::NumExpertsPerTok: write_u64(out, static_cast(params.num_experts_per_tok)); break; + case ParamField::MoeGated: write_u32(out, params.moe_gated ? 1u : 0u); break; + case ParamField::Activation: write_u32(out, static_cast(params.activation)); break; + case ParamField::NumKvHeads: write_u64(out, static_cast(params.num_kv_heads)); break; + case ParamField::ChunkSize: write_u64(out, static_cast(params.chunk_size)); break; + case ParamField::DstHeight: write_u64(out, static_cast(params.dst_height)); break; + case ParamField::DstWidth: write_u64(out, static_cast(params.dst_width)); break; + case ParamField::AlignCorners: write_u32(out, params.align_corners ? 1u : 0u); break; + case ParamField::NumClasses: write_u64(out, static_cast(params.num_classes)); break; + case ParamField::NumAltupInputs: write_u64(out, static_cast(params.num_altup_inputs)); break; + case ParamField::KernelSize: write_u64(out, static_cast(params.kernel_size)); break; + case ParamField::Stride: write_u64(out, static_cast(params.stride)); break; + case ParamField::Dilation: write_u64(out, static_cast(params.dilation)); break; + case ParamField::NumFftBins: write_u64(out, static_cast(params.num_fft_bins)); break; + case ParamField::CacheSeqLen: write_u64(out, static_cast(params.cache_seq_len)); break; + case ParamField::HeadDim: write_u64(out, static_cast(params.head_dim)); break; + case ParamField::VHeadDim: write_u64(out, static_cast(params.v_head_dim)); break; + case ParamField::CachedKeysInt8Ptr: + case ParamField::CachedValuesInt8Ptr: + case ParamField::CachedKScalesPtr: + case ParamField::CachedVScalesPtr: + throw std::runtime_error("Attempted to serialize runtime-only pointer field"); + } +} + +void read_field(std::istream& in, ParamField field, OpParams& params) { + switch (field) { + case ParamField::Scalar: params.scalar = read_f32(in); break; + case ParamField::Axis: params.axis = static_cast(read_i32(in)); break; + case ParamField::NewShape: params.new_shape = read_size_vector(in); break; + case ParamField::PretransposedRhs: params.pretransposed_rhs = (read_u32(in) != 0); break; + case ParamField::Backend: { + uint32_t backend_val = read_u32(in); + if (backend_val > static_cast(ComputeBackend::NPU)) { + throw std::runtime_error("Graph file corrupted: invalid backend"); + } + params.backend = static_cast(backend_val); + break; + } + case ParamField::SliceStart: params.slice_start = static_cast(read_u64(in)); break; + case ParamField::SliceLength: params.slice_length = static_cast(read_u64(in)); break; + case ParamField::Epsilon: params.epsilon = read_f32(in); break; + case ParamField::NumGroups: params.num_groups = static_cast(read_u64(in)); break; + case ParamField::IndexValue: params.index_value = static_cast(read_u64(in)); break; + case ParamField::Permutation: params.permutation = read_size_vector(in); break; + case ParamField::Scale: params.scale = read_f32(in); break; + case ParamField::Theta: params.theta = read_f32(in); break; + case ParamField::PositionOffset: params.position_offset = static_cast(read_u64(in)); break; + case ParamField::WindowSize: params.window_size = static_cast(read_u64(in)); break; + case ParamField::IsCausal: params.is_causal = (read_u32(in) != 0); break; + case ParamField::AttentionMaskIsAdditive: params.attention_mask_is_additive = (read_u32(in) != 0); break; + case ParamField::LogitCap: params.logit_cap = read_f32(in); break; + case ParamField::TopK: params.top_k = static_cast(read_u64(in)); break; + case ParamField::Temperature: params.temperature = read_f32(in); break; + case ParamField::TopP: params.top_p = read_f32(in); break; + case ParamField::MinP: params.min_p = read_f32(in); break; + case ParamField::RepetitionPenalty: params.repetition_penalty = read_f32(in); break; + case ParamField::RandomSeed: params.random_seed = static_cast(read_u64(in)); break; + case ParamField::BiasIndices: params.bias_indices = read_u32_vector(in); break; + case ParamField::BiasValues: params.bias_values = read_f32_vector(in); break; + case ParamField::NormalizeRouting: params.normalize_routing = (read_u32(in) != 0); break; + case ParamField::NumExperts: params.num_experts = static_cast(read_u64(in)); break; + case ParamField::NumExpertsPerTok: params.num_experts_per_tok = static_cast(read_u64(in)); break; + case ParamField::MoeGated: params.moe_gated = (read_u32(in) != 0); break; + case ParamField::Activation: params.activation = static_cast(read_u32(in)); break; + case ParamField::NumKvHeads: params.num_kv_heads = static_cast(read_u64(in)); break; + case ParamField::ChunkSize: params.chunk_size = static_cast(read_u64(in)); break; + case ParamField::DstHeight: params.dst_height = static_cast(read_u64(in)); break; + case ParamField::DstWidth: params.dst_width = static_cast(read_u64(in)); break; + case ParamField::AlignCorners: params.align_corners = (read_u32(in) != 0); break; + case ParamField::NumClasses: params.num_classes = static_cast(read_u64(in)); break; + case ParamField::NumAltupInputs: params.num_altup_inputs = static_cast(read_u64(in)); break; + case ParamField::KernelSize: params.kernel_size = static_cast(read_u64(in)); break; + case ParamField::Stride: params.stride = static_cast(read_u64(in)); break; + case ParamField::Dilation: params.dilation = static_cast(read_u64(in)); break; + case ParamField::NumFftBins: params.num_fft_bins = static_cast(read_u64(in)); break; + case ParamField::CacheSeqLen: params.cache_seq_len = static_cast(read_u64(in)); break; + case ParamField::HeadDim: params.head_dim = static_cast(read_u64(in)); break; + case ParamField::VHeadDim: params.v_head_dim = static_cast(read_u64(in)); break; + case ParamField::CachedKeysInt8Ptr: + case ParamField::CachedValuesInt8Ptr: + case ParamField::CachedKScalesPtr: + case ParamField::CachedVScalesPtr: + throw std::runtime_error("Graph file corrupted: runtime-only field serialized"); + } +} + +} // namespace + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params) { + const auto& schema = op_schema(op_type); + std::vector fields; + fields.reserve(schema.size()); + for (const auto& spec : schema) { + if (spec.persistence == FieldPersistence::Persistent) { + fields.push_back(spec.field); + } + } + + write_u32(out, static_cast(fields.size())); + for (ParamField field : fields) { + write_u32(out, static_cast(field)); + write_field(out, field, params); + } +} + +void read_op_params(std::istream& in, OpType op_type, OpParams& params) { + uint32_t field_count = read_u32(in); + const auto& schema = op_schema(op_type); + std::unordered_set allowed; + for (const auto& spec : schema) { + if (spec.persistence == FieldPersistence::Persistent) { + allowed.insert(static_cast(spec.field)); + } + } + + for (uint32_t i = 0; i < field_count; ++i) { + uint32_t raw_field = read_u32(in); + if (allowed.count(raw_field) == 0) { + throw std::runtime_error("Graph file corrupted: field not allowed for op"); + } + read_field(in, static_cast(raw_field), params); + } +} + +} // namespace GraphParamIO diff --git a/cactus/graph/graph_param_io.h b/cactus/graph/graph_param_io.h new file mode 100644 index 000000000..0362ea8c5 --- /dev/null +++ b/cactus/graph/graph_param_io.h @@ -0,0 +1,11 @@ +#pragma once + +#include "graph.h" +#include + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); +void read_op_params(std::istream& in, OpType op_type, OpParams& params); + +} // namespace GraphParamIO diff --git a/cactus/kernel/kernel.h b/cactus/kernel/kernel.h index 2c0fdfc31..13f4a00be 100644 --- a/cactus/kernel/kernel.h +++ b/cactus/kernel/kernel.h @@ -68,6 +68,11 @@ void cactus_gemv_int4(const int8_t* A, float A_scale, const int8_t* B_packed, const __fp16* B_scales, __fp16* C, size_t K, size_t N, size_t group_size); +void cactus_gemv_int4_block_range(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size, + size_t block_start, size_t block_end); + void cactus_gemm_int4(const int8_t* A, const float* A_scales, const int8_t* B_packed, const __fp16* B_scales, __fp16* C, size_t M, size_t K, size_t N, size_t group_size); diff --git a/cactus/kernel/kernel_matmul.cpp b/cactus/kernel/kernel_matmul.cpp index 58b57587e..929663aa2 100644 --- a/cactus/kernel/kernel_matmul.cpp +++ b/cactus/kernel/kernel_matmul.cpp @@ -644,6 +644,105 @@ void cactus_gemv_int4( } } +void cactus_gemv_int4_block_range( + const int8_t* A, float A_scale, + const int8_t* B_packed_raw, const __fp16* B_scales, + __fp16* C, + size_t K, size_t N, size_t group_size, + size_t block_start, size_t block_end +) { + const uint8_t* B_packed = reinterpret_cast(B_packed_raw); + if (K == 0 || N == 0 || block_end <= block_start) return; + const size_t num_groups = K / group_size; + + size_t n_block = block_start; + for (; n_block + 1 < block_end; n_block += 2) { + const size_t n_start_a = n_block * 4; + const size_t n_start_b = (n_block + 1) * 4; + float32x4_t sum_a = vdupq_n_f32(0.0f); + float32x4_t sum_b = vdupq_n_f32(0.0f); + + for (size_t g = 0; g < num_groups; ++g) { + const size_t k_base = g * group_size; + const int8_t* a_ptr = A + k_base; + const uint8_t* ba = B_packed + (n_block * K + k_base) * 2; + const uint8_t* bb = B_packed + ((n_block + 1) * K + k_base) * 2; + + int32x4_t acc_a = vdupq_n_s32(0); + int32x4_t acc_b = vdupq_n_s32(0); + int8x16_t a_lo = vld1q_s8(a_ptr); + int8x16_t a_hi = vld1q_s8(a_ptr + 16); + { + int8x16_t b0, b1, b2, b3; + unpack_int4_as_int8x16x2(ba, b1, b0); + unpack_int4_as_int8x16x2(ba + 16, b3, b2); + acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_lo, 0); + acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_lo, 1); + acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_lo, 2); + acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_lo, 3); + unpack_int4_as_int8x16x2(ba + 32, b1, b0); + unpack_int4_as_int8x16x2(ba + 48, b3, b2); + acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_hi, 0); + acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_hi, 1); + acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_hi, 2); + acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_hi, 3); + } + { + int8x16_t b0, b1, b2, b3; + unpack_int4_as_int8x16x2(bb, b1, b0); + unpack_int4_as_int8x16x2(bb + 16, b3, b2); + acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_lo, 0); + acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_lo, 1); + acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_lo, 2); + acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_lo, 3); + unpack_int4_as_int8x16x2(bb + 32, b1, b0); + unpack_int4_as_int8x16x2(bb + 48, b3, b2); + acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_hi, 0); + acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_hi, 1); + acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_hi, 2); + acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_hi, 3); + } + + const __fp16* spa = B_scales + (n_block * num_groups + g) * 4; + const __fp16* spb = B_scales + ((n_block + 1) * num_groups + g) * 4; + float32x4_t sa = vcvt_f32_f16(vld1_f16(spa)); + float32x4_t sb = vcvt_f32_f16(vld1_f16(spb)); + sum_a = vmlaq_f32(sum_a, vcvtq_f32_s32(acc_a), sa); + sum_b = vmlaq_f32(sum_b, vcvtq_f32_s32(acc_b), sb); + } + vst1_f16(C + n_start_a, vcvt_f16_f32(vmulq_n_f32(sum_a, A_scale))); + vst1_f16(C + n_start_b, vcvt_f16_f32(vmulq_n_f32(sum_b, A_scale))); + } + + for (; n_block < block_end; ++n_block) { + const size_t n_start = n_block * 4; + float32x4_t running_sum = vdupq_n_f32(0.0f); + for (size_t g = 0; g < num_groups; ++g) { + const size_t k_base = g * group_size; + const int8_t* a_ptr = A + k_base; + const uint8_t* b_base = B_packed + (n_block * K + k_base) * 2; + int32x4_t acc = vdupq_n_s32(0); + int8x16_t a_lo = vld1q_s8(a_ptr); + int8x16_t a_hi = vld1q_s8(a_ptr + 16); + int8x16_t b0, b1, b2, b3; + unpack_int4_as_int8x16x2(b_base, b1, b0); + unpack_int4_as_int8x16x2(b_base + 16, b3, b2); + acc = CACTUS_DOTQ_LANE(acc, b0, a_lo, 0); + acc = CACTUS_DOTQ_LANE(acc, b1, a_lo, 1); + acc = CACTUS_DOTQ_LANE(acc, b2, a_lo, 2); + acc = CACTUS_DOTQ_LANE(acc, b3, a_lo, 3); + unpack_int4_as_int8x16x2(b_base + 32, b1, b0); + unpack_int4_as_int8x16x2(b_base + 48, b3, b2); + acc = CACTUS_DOTQ_LANE(acc, b0, a_hi, 0); + acc = CACTUS_DOTQ_LANE(acc, b1, a_hi, 1); + acc = CACTUS_DOTQ_LANE(acc, b2, a_hi, 2); + acc = CACTUS_DOTQ_LANE(acc, b3, a_hi, 3); + float32x4_t scales = vcvt_f32_f16(vld1_f16(B_scales + (n_block * num_groups + g) * 4)); + running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc), scales); + } + vst1_f16(C + n_start, vcvt_f16_f32(vmulq_n_f32(running_sum, A_scale))); + } +} void cactus_gemm_int4( const int8_t* A, diff --git a/cactus/kernel/kernel_reduce.cpp b/cactus/kernel/kernel_reduce.cpp index 8f00b4bd8..e0f3d19e8 100644 --- a/cactus/kernel/kernel_reduce.cpp +++ b/cactus/kernel/kernel_reduce.cpp @@ -7,6 +7,35 @@ template static void axis_reduce_f32_impl(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size, FinalizeFn finalize) { + if (inner_size == 1) { + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t start_outer, size_t end_outer) { + constexpr size_t W = SIMD_F16_WIDTH; + const size_t vec_axis = simd_align(axis_size); + + for (size_t outer = start_outer; outer < end_outer; ++outer) { + const __fp16* row = input + outer * axis_size; + float32x4_t sum_lo = vdupq_n_f32(0.0f); + float32x4_t sum_hi = vdupq_n_f32(0.0f); + + for (size_t a = 0; a < vec_axis; a += W) { + float32x4_t lo, hi; + f16x8_split_f32(vld1q_f16(row + a), lo, hi); + sum_lo = vaddq_f32(sum_lo, lo); + sum_hi = vaddq_f32(sum_hi, hi); + } + + float total = vaddvq_f32(vaddq_f32(sum_lo, sum_hi)); + for (size_t a = vec_axis; a < axis_size; ++a) { + total += static_cast(row[a]); + } + + output[outer] = static_cast<__fp16>(finalize(total, axis_size)); + } + }); + return; + } + CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t outer, size_t inner) { constexpr size_t W = SIMD_F16_WIDTH; @@ -41,6 +70,37 @@ static void axis_reduce_f16_impl(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size, float16x8_t init_vec, __fp16 init_scalar, SimdReduceOp simd_reduce, ScalarReduceOp scalar_reduce) { + if (inner_size == 1) { + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t start_outer, size_t end_outer) { + constexpr size_t W = SIMD_F16_WIDTH; + const size_t vec_axis = simd_align(axis_size); + + for (size_t outer = start_outer; outer < end_outer; ++outer) { + const __fp16* row = input + outer * axis_size; + float16x8_t acc = init_vec; + + for (size_t a = 0; a < vec_axis; a += W) { + acc = simd_reduce(acc, vld1q_f16(row + a)); + } + + __fp16 result = init_scalar; + __fp16 arr[W]; + vst1q_f16(arr, acc); + for (size_t j = 0; j < W; ++j) { + result = scalar_reduce(result, arr[j]); + } + + for (size_t a = vec_axis; a < axis_size; ++a) { + result = scalar_reduce(result, row[a]); + } + + output[outer] = result; + } + }); + return; + } + CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t outer, size_t inner) { constexpr size_t W = SIMD_F16_WIDTH; @@ -154,6 +214,44 @@ double cactus_variance_all_f16(const __fp16* data, size_t num_elements) { } void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { + if (inner_size == 1) { + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::AXIS_REDUCE, + [&](size_t start_outer, size_t end_outer) { + constexpr size_t W = SIMD_F16_WIDTH; + const size_t vec_axis = simd_align(axis_size); + + for (size_t outer = start_outer; outer < end_outer; ++outer) { + const __fp16* row = input + outer * axis_size; + float32x4_t sum_lo = vdupq_n_f32(0.0f); + float32x4_t sum_hi = vdupq_n_f32(0.0f); + float32x4_t sq_lo = vdupq_n_f32(0.0f); + float32x4_t sq_hi = vdupq_n_f32(0.0f); + + for (size_t a = 0; a < vec_axis; a += W) { + float32x4_t lo, hi; + f16x8_split_f32(vld1q_f16(row + a), lo, hi); + sum_lo = vaddq_f32(sum_lo, lo); + sum_hi = vaddq_f32(sum_hi, hi); + sq_lo = vfmaq_f32(sq_lo, lo, lo); + sq_hi = vfmaq_f32(sq_hi, hi, hi); + } + + float sum = vaddvq_f32(vaddq_f32(sum_lo, sum_hi)); + float sum_sq = vaddvq_f32(vaddq_f32(sq_lo, sq_hi)); + for (size_t a = vec_axis; a < axis_size; ++a) { + float x = static_cast(row[a]); + sum += x; + sum_sq += x * x; + } + + float mean = sum / static_cast(axis_size); + float mean_sq = sum_sq / static_cast(axis_size); + output[outer] = static_cast<__fp16>(mean_sq - mean * mean); + } + }); + return; + } + CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t outer, size_t inner) { float sum = 0.0f, sum_sq = 0.0f; diff --git a/cactus/models/gemma4/model_gemma4.cpp b/cactus/models/gemma4/model_gemma4.cpp index fe965dfe3..c99917878 100644 --- a/cactus/models/gemma4/model_gemma4.cpp +++ b/cactus/models/gemma4/model_gemma4.cpp @@ -313,6 +313,23 @@ size_t Gemma4Model::build_attention(CactusGraph* gb, size_t input, uint32_t laye size_t Gemma4Model::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend) const { const auto& layer = weight_nodes_.layers[layer_idx]; + static const bool dense_mlp_fused_disabled = []() { + const char* env = std::getenv("CACTUS_DISABLE_DENSE_MLP_FUSED"); + return env && env[0] && env[0] != '0'; + }(); + if (!dense_mlp_fused_disabled) { + const auto& gate_buf = gb->get_output_buffer(layer.ffn_gate_weight); + const auto& up_buf = gb->get_output_buffer(layer.ffn_up_weight); + const auto& down_buf = gb->get_output_buffer(layer.ffn_down_weight); + if (gate_buf.precision == Precision::INT4 && gate_buf.group_size > 0 && + up_buf.precision == Precision::INT4 && up_buf.group_size > 0 && + down_buf.precision == Precision::INT4 && down_buf.group_size > 0) { + return gb->dense_mlp_int4_fused(input, + layer.ffn_gate_weight, + layer.ffn_up_weight, + layer.ffn_down_weight); + } + } auto gate = gb->gelu(gb->matmul(input, layer.ffn_gate_weight, true, backend)); auto up = gb->matmul(input, layer.ffn_up_weight, true, backend); return gb->matmul(gb->multiply(gate, up), layer.ffn_down_weight, true, backend); diff --git a/docs/cactus_graph.md b/docs/cactus_graph.md index 1c2673436..5e586b328 100644 --- a/docs/cactus_graph.md +++ b/docs/cactus_graph.md @@ -490,6 +490,66 @@ size_t similarity = fixture.graph().divide(dot_product, fixture.graph().multiply 3. **Test edge cases**: Include tests for broadcasting, empty tensors, and large inputs 4. **Check precision**: Test operations with different precision types +### Contributing Graph Operations +1. ** Define the op in core graph types ** +Add the new OpType in `cactus/graph/graph.h` +If the op needs additional parameters, add the fields to OpParams in the same file + +2. ** Add a graph builder API ** +Add a builder method in `cactus/graph/graph_builder.cpp` and its declaration in +`cactus/graph/graph.h` +Follow the pattern of existing builder methods, e.g. for a new "relu" op: +```cpp +size_t CactusGraph::relu(size_t input) { + OpParams params; + return add_node(OpType::RELU, {input}, params); +} +``` +3. ** Implement the op in the execution engine ** +Implement the krnel or graph op code in the relevant file, usually in `cactus/kernel/` +Register the new op in the dispatch table in `cactus/graph/graph_execute.cpp` for the supported backends (CPU, NPU) + +4. ** Export op in FFI bindings ** +- header: `cactus/ffi/cactus_ffi.h` +- implementation: `cactus/ffi/cactus_ffi.cpp` + +5. ** Add python ctypes declaration ** +Add `_lib.cactus_graph_my_new_op.argtypes/restype` in `python/src/cactus.py` + +6. ** Add python graph wrapper ** +Add `Graph.my_new_op(...)` in `python/src/graph.py`, and optionally a Tensor + convenience method. + +7. ** Add serialization schema entry if needed ** +If your op has extra parameters that need to be saved/loaded that are not in the +default node, add new ParamField enum values. + +If the op has any graph-persistent params: + +- add any new ParamField enum values in cactus/graph/graph_param_io.cpp +- add read/write logic for those fields +- add the op’s schema entry in `op_schema(...)` + +If the op has no params, you may not need to touch schema beyond maybe adding an +empty schema entry. + +The syntax pattern there is: +``` +{OpType::MY_NEW_OP, { + {ParamField::Alpha, FieldPersistence::Persistent}, + {ParamField::Mode, FieldPersistence::Persistent}, +}}, +``` +8. ** Add test coverage ** +Add unit tests to `tests/test_graph.cpp` covering the native graph function, and +add python tests in `python/tests/test_graph.py` covering the Python API and end-to-end execution. + +and then support those fields in `write_field(...)` / `read_field(...)`. + +If a field is runtime-only, mark it RuntimeOnly instead of Persistent. + + + ### Error Handling ```cpp try { diff --git a/libs/xgrammar/android/arm64-v8a/libxgrammar.a b/libs/xgrammar/android/arm64-v8a/libxgrammar.a new file mode 100644 index 000000000..f4a1eb97b Binary files /dev/null and b/libs/xgrammar/android/arm64-v8a/libxgrammar.a differ diff --git a/libs/xgrammar/include/dlpack/dlpack.h b/libs/xgrammar/include/dlpack/dlpack.h new file mode 100644 index 000000000..bcb77949a --- /dev/null +++ b/libs/xgrammar/include/dlpack/dlpack.h @@ -0,0 +1,332 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file dlpack.h + * \brief The common header of DLPack. + */ +#ifndef DLPACK_DLPACK_H_ +#define DLPACK_DLPACK_H_ + +/** + * \brief Compatibility with C++ + */ +#ifdef __cplusplus +#define DLPACK_EXTERN_C extern "C" +#else +#define DLPACK_EXTERN_C +#endif + +/*! \brief The current major version of dlpack */ +#define DLPACK_MAJOR_VERSION 1 + +/*! \brief The current minor version of dlpack */ +#define DLPACK_MINOR_VERSION 0 + +/*! \brief DLPACK_DLL prefix for windows */ +#ifdef _WIN32 +#ifdef DLPACK_EXPORTS +#define DLPACK_DLL __declspec(dllexport) +#else +#define DLPACK_DLL __declspec(dllimport) +#endif +#else +#define DLPACK_DLL +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * \brief The DLPack version. + * + * A change in major version indicates that we have changed the + * data layout of the ABI - DLManagedTensorVersioned. + * + * A change in minor version indicates that we have added new + * code, such as a new device type, but the ABI is kept the same. + * + * If an obtained DLPack tensor has a major version that disagrees + * with the version number specified in this header file + * (i.e. major != DLPACK_MAJOR_VERSION), the consumer must call the deleter + * (and it is safe to do so). It is not safe to access any other fields + * as the memory layout will have changed. + * + * In the case of a minor version mismatch, the tensor can be safely used as + * long as the consumer knows how to interpret all fields. Minor version + * updates indicate the addition of enumeration values. + */ +typedef struct { + /*! \brief DLPack major version. */ + uint32_t major; + /*! \brief DLPack minor version. */ + uint32_t minor; +} DLPackVersion; + +/*! + * \brief The device type in DLDevice. + */ +#ifdef __cplusplus +typedef enum : int32_t { +#else +typedef enum { +#endif + /*! \brief CPU device */ + kDLCPU = 1, + /*! \brief CUDA GPU device */ + kDLCUDA = 2, + /*! + * \brief Pinned CUDA CPU memory by cudaMallocHost + */ + kDLCUDAHost = 3, + /*! \brief OpenCL devices. */ + kDLOpenCL = 4, + /*! \brief Vulkan buffer for next generation graphics. */ + kDLVulkan = 7, + /*! \brief Metal for Apple GPU. */ + kDLMetal = 8, + /*! \brief Verilog simulator buffer */ + kDLVPI = 9, + /*! \brief ROCm GPUs for AMD GPUs */ + kDLROCM = 10, + /*! + * \brief Pinned ROCm CPU memory allocated by hipMallocHost + */ + kDLROCMHost = 11, + /*! + * \brief Reserved extension device type, + * used for quickly test extension device + * The semantics can differ depending on the implementation. + */ + kDLExtDev = 12, + /*! + * \brief CUDA managed/unified memory allocated by cudaMallocManaged + */ + kDLCUDAManaged = 13, + /*! + * \brief Unified shared memory allocated on a oneAPI non-partititioned + * device. Call to oneAPI runtime is required to determine the device + * type, the USM allocation type and the sycl context it is bound to. + * + */ + kDLOneAPI = 14, + /*! \brief GPU support for next generation WebGPU standard. */ + kDLWebGPU = 15, + /*! \brief Qualcomm Hexagon DSP */ + kDLHexagon = 16, + /*! \brief Microsoft MAIA devices */ + kDLMAIA = 17, +} DLDeviceType; + +/*! + * \brief A Device for Tensor and operator. + */ +typedef struct { + /*! \brief The device type used in the device. */ + DLDeviceType device_type; + /*! + * \brief The device index. + * For vanilla CPU memory, pinned memory, or managed memory, this is set to 0. + */ + int32_t device_id; +} DLDevice; + +/*! + * \brief The type code options DLDataType. + */ +typedef enum { + /*! \brief signed integer */ + kDLInt = 0U, + /*! \brief unsigned integer */ + kDLUInt = 1U, + /*! \brief IEEE floating point */ + kDLFloat = 2U, + /*! + * \brief Opaque handle type, reserved for testing purposes. + * Frameworks need to agree on the handle data type for the exchange to be well-defined. + */ + kDLOpaqueHandle = 3U, + /*! \brief bfloat16 */ + kDLBfloat = 4U, + /*! + * \brief complex number + * (C/C++/Python layout: compact struct per complex number) + */ + kDLComplex = 5U, + /*! \brief boolean */ + kDLBool = 6U, +} DLDataTypeCode; + +/*! + * \brief The data type the tensor can hold. The data type is assumed to follow the + * native endian-ness. An explicit error message should be raised when attempting to + * export an array with non-native endianness + * + * Examples + * - float: type_code = 2, bits = 32, lanes = 1 + * - float4(vectorized 4 float): type_code = 2, bits = 32, lanes = 4 + * - int8: type_code = 0, bits = 8, lanes = 1 + * - std::complex: type_code = 5, bits = 64, lanes = 1 + * - bool: type_code = 6, bits = 8, lanes = 1 (as per common array library convention, the underlying storage size of bool is 8 bits) + */ +typedef struct { + /*! + * \brief Type code of base types. + * We keep it uint8_t instead of DLDataTypeCode for minimal memory + * footprint, but the value should be one of DLDataTypeCode enum values. + * */ + uint8_t code; + /*! + * \brief Number of bits, common choices are 8, 16, 32. + */ + uint8_t bits; + /*! \brief Number of lanes in the type, used for vector types. */ + uint16_t lanes; +} DLDataType; + +/*! + * \brief Plain C Tensor object, does not manage memory. + */ +typedef struct { + /*! + * \brief The data pointer points to the allocated data. This will be CUDA + * device pointer or cl_mem handle in OpenCL. It may be opaque on some device + * types. This pointer is always aligned to 256 bytes as in CUDA. The + * `byte_offset` field should be used to point to the beginning of the data. + * + * Note that as of Nov 2021, multiply libraries (CuPy, PyTorch, TensorFlow, + * TVM, perhaps others) do not adhere to this 256 byte aligment requirement + * on CPU/CUDA/ROCm, and always use `byte_offset=0`. This must be fixed + * (after which this note will be updated); at the moment it is recommended + * to not rely on the data pointer being correctly aligned. + * + * For given DLTensor, the size of memory required to store the contents of + * data is calculated as follows: + * + * \code{.c} + * static inline size_t GetDataSize(const DLTensor* t) { + * size_t size = 1; + * for (tvm_index_t i = 0; i < t->ndim; ++i) { + * size *= t->shape[i]; + * } + * size *= (t->dtype.bits * t->dtype.lanes + 7) / 8; + * return size; + * } + * \endcode + * + * Note that if the tensor is of size zero, then the data pointer should be + * set to `NULL`. + */ + void* data; + /*! \brief The device of the tensor */ + DLDevice device; + /*! \brief Number of dimensions */ + int32_t ndim; + /*! \brief The data type of the pointer*/ + DLDataType dtype; + /*! \brief The shape of the tensor */ + int64_t* shape; + /*! + * \brief strides of the tensor (in number of elements, not bytes) + * can be NULL, indicating tensor is compact and row-majored. + */ + int64_t* strides; + /*! \brief The offset in bytes to the beginning pointer to data */ + uint64_t byte_offset; +} DLTensor; + +/*! + * \brief C Tensor object, manage memory of DLTensor. This data structure is + * intended to facilitate the borrowing of DLTensor by another framework. It is + * not meant to transfer the tensor. When the borrowing framework doesn't need + * the tensor, it should call the deleter to notify the host that the resource + * is no longer needed. + * + * \note This data structure is used as Legacy DLManagedTensor + * in DLPack exchange and is deprecated after DLPack v0.8 + * Use DLManagedTensorVersioned instead. + * This data structure may get renamed or deleted in future versions. + * + * \sa DLManagedTensorVersioned + */ +typedef struct DLManagedTensor { + /*! \brief DLTensor which is being memory managed */ + DLTensor dl_tensor; + /*! \brief the context of the original host framework of DLManagedTensor in + * which DLManagedTensor is used in the framework. It can also be NULL. + */ + void * manager_ctx; + /*! + * \brief Destructor - this should be called + * to destruct the manager_ctx which backs the DLManagedTensor. It can be + * NULL if there is no way for the caller to provide a reasonable destructor. + * The destructor deletes the argument self as well. + */ + void (*deleter)(struct DLManagedTensor * self); +} DLManagedTensor; + +// bit masks used in in the DLManagedTensorVersioned + +/*! \brief bit mask to indicate that the tensor is read only. */ +#define DLPACK_FLAG_BITMASK_READ_ONLY (1UL << 0UL) + +/*! + * \brief bit mask to indicate that the tensor is a copy made by the producer. + * + * If set, the tensor is considered solely owned throughout its lifetime by the + * consumer, until the producer-provided deleter is invoked. + */ +#define DLPACK_FLAG_BITMASK_IS_COPIED (1UL << 1UL) + +/*! + * \brief A versioned and managed C Tensor object, manage memory of DLTensor. + * + * This data structure is intended to facilitate the borrowing of DLTensor by + * another framework. It is not meant to transfer the tensor. When the borrowing + * framework doesn't need the tensor, it should call the deleter to notify the + * host that the resource is no longer needed. + * + * \note This is the current standard DLPack exchange data structure. + */ +struct DLManagedTensorVersioned { + /*! + * \brief The API and ABI version of the current managed Tensor + */ + DLPackVersion version; + /*! + * \brief the context of the original host framework. + * + * Stores DLManagedTensorVersioned is used in the + * framework. It can also be NULL. + */ + void *manager_ctx; + /*! + * \brief Destructor. + * + * This should be called to destruct manager_ctx which holds the DLManagedTensorVersioned. + * It can be NULL if there is no way for the caller to provide a reasonable + * destructor. The destructor deletes the argument self as well. + */ + void (*deleter)(struct DLManagedTensorVersioned *self); + /*! + * \brief Additional bitmask flags information about the tensor. + * + * By default the flags should be set to 0. + * + * \note Future ABI changes should keep everything until this field + * stable, to ensure that deleter can be correctly called. + * + * \sa DLPACK_FLAG_BITMASK_READ_ONLY + * \sa DLPACK_FLAG_BITMASK_IS_COPIED + */ + uint64_t flags; + /*! \brief DLTensor which is being memory managed */ + DLTensor dl_tensor; +}; + +#ifdef __cplusplus +} // DLPACK_EXTERN_C +#endif +#endif // DLPACK_DLPACK_H_ diff --git a/libs/xgrammar/include/picojson/picojson.h b/libs/xgrammar/include/picojson/picojson.h new file mode 100644 index 000000000..5dcd86840 --- /dev/null +++ b/libs/xgrammar/include/picojson/picojson.h @@ -0,0 +1,1319 @@ +/* + * Copyright 2009-2010 Cybozu Labs, Inc. + * Copyright 2011-2014 Kazuho Oku + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#pragma once + +#ifndef PICOJSON_USE_INT64 +#define PICOJSON_USE_INT64 +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS 1 +#endif +#endif + +// If PICOJSON_USE_ORDERED_OBJECT is set, picojson uses object_with_ordered_keys, which maintains +// the insertion order of keys, i.e. the order of keys in the json string. +// This macro is set by default. +#ifndef PICOJSON_USE_ORDERED_OBJECT +#define PICOJSON_USE_ORDERED_OBJECT 1 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// for isnan/isinf +#if __cplusplus >= 201103L +#include +#else +extern "C" { +#ifdef _MSC_VER +#include +#elif defined(__INTEL_COMPILER) +#include +#else +#include +#endif +} +#endif + +#ifndef PICOJSON_USE_RVALUE_REFERENCE +#if (defined(__cpp_rvalue_references) && __cpp_rvalue_references >= 200610) || \ + (defined(_MSC_VER) && _MSC_VER >= 1600) +#define PICOJSON_USE_RVALUE_REFERENCE 1 +#else +#define PICOJSON_USE_RVALUE_REFERENCE 0 +#endif +#endif // PICOJSON_USE_RVALUE_REFERENCE + +#ifndef PICOJSON_NOEXCEPT +#if PICOJSON_USE_RVALUE_REFERENCE +#define PICOJSON_NOEXCEPT noexcept +#else +#define PICOJSON_NOEXCEPT throw() +#endif +#endif + +// experimental support for int64_t (see README.mkdn for detail) +#ifdef PICOJSON_USE_INT64 +#include +#include +#endif + +// to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 +#ifndef PICOJSON_USE_LOCALE +#define PICOJSON_USE_LOCALE 1 +#endif +#if PICOJSON_USE_LOCALE +extern "C" { +#include +} +#endif + +#ifndef PICOJSON_ASSERT +#ifndef PICOJSON_DISABLE_EXCEPTION +#define PICOJSON_ASSERT(e) \ + do { \ + if (!(e)) throw std::runtime_error(#e); \ + } while (0) +#else +#define PICOJSON_ASSERT(e) \ + do { \ + if (!(e)) std::abort(); \ + } while (0) +#endif // PICOJSON_DISABLE_EXCEPTION +#endif + +#ifdef _MSC_VER +#define SNPRINTF _snprintf_s +#pragma warning(push) +#pragma warning(disable : 4244) // conversion from int to char +#pragma warning(disable : 4127) // conditional expression is constant +#pragma warning(disable : 4702) // unreachable code +#else +#define SNPRINTF snprintf +#endif + +namespace picojson { + +enum { + null_type, + boolean_type, + number_type, + string_type, + array_type, + object_type +#ifdef PICOJSON_USE_INT64 + , + int64_type +#endif +}; + +enum { INDENT_WIDTH = 2 }; + +struct null {}; + +class object_with_ordered_keys; + +class value { + public: + typedef std::vector array; +#ifdef PICOJSON_USE_ORDERED_OBJECT + typedef object_with_ordered_keys object; +#else + typedef std::unordered_map object; +#endif + + union _storage { + bool boolean_; + double number_; +#ifdef PICOJSON_USE_INT64 + int64_t int64_; +#endif + std::string* string_; + array* array_; + object* object_; + }; + + protected: + int type_; + _storage u_; + + public: + value(); + value(int type, bool); + explicit value(bool b); +#ifdef PICOJSON_USE_INT64 + explicit value(int64_t i); +#endif + explicit value(double n); + explicit value(const std::string& s); + explicit value(const array& a); + explicit value(const object& o); +#if PICOJSON_USE_RVALUE_REFERENCE + explicit value(std::string&& s); + explicit value(array&& a); + explicit value(object&& o); +#endif + explicit value(const char* s); + value(const char* s, size_t len); + ~value(); + value(const value& x); + value& operator=(const value& x); +#if PICOJSON_USE_RVALUE_REFERENCE + value(value&& x) PICOJSON_NOEXCEPT; + value& operator=(value&& x) PICOJSON_NOEXCEPT; +#endif + void swap(value& x) PICOJSON_NOEXCEPT; + template + bool is() const; + template + const T& get() const; + template + T& get(); + template + void set(const T&); +#if PICOJSON_USE_RVALUE_REFERENCE + template + void set(T&&); +#endif + bool evaluate_as_boolean() const; + const value& get(const size_t idx) const; + const value& get(const std::string& key) const; + value& get(const size_t idx); + value& get(const std::string& key); + + bool contains(const size_t idx) const; + bool contains(const std::string& key) const; + std::string to_str() const; + template + void serialize(Iter os, bool prettify = false) const; + std::string serialize(bool prettify = false) const; + + private: + template + // NOLINTNEXTLINE(runtime/explicit) + value(const T*); // intentionally defined to block implicit conversion of + // pointer to bool + template + static void _indent(Iter os, int indent); + template + void _serialize(Iter os, int indent) const; + std::string _serialize(int indent) const; + void clear(); +}; + +// The ordered version of hashmap. It has the same interface as std::unordered_map, but provides +// ordered_keys() to return the keys in the order they were inserted. +class object_with_ordered_keys : private std::unordered_map { + public: + using typename std::unordered_map::value_type; + using typename std::unordered_map::iterator; + using typename std::unordered_map::const_iterator; + + object_with_ordered_keys() = default; + object_with_ordered_keys(const object_with_ordered_keys&) = default; + object_with_ordered_keys(object_with_ordered_keys&&) = default; + object_with_ordered_keys(std::initializer_list init) + : std::unordered_map(init) { + for (const auto& pair : init) { + ordered_keys_.push_back(pair.first); + } + } + object_with_ordered_keys& operator=(const object_with_ordered_keys&) = default; + object_with_ordered_keys& operator=(object_with_ordered_keys&&) = default; + + using std::unordered_map::begin; + using std::unordered_map::end; + using std::unordered_map::cbegin; + using std::unordered_map::cend; + using std::unordered_map::empty; + using std::unordered_map::size; + using std::unordered_map::at; + using std::unordered_map::count; + using std::unordered_map::find; + using std::unordered_map::reserve; + + value& operator[](const std::string& key) { + if (count(key) == 0) { + ordered_keys_.push_back(key); + } + return std::unordered_map::operator[](key); + } + + const value& operator[](const std::string& key) const { + return std::unordered_map::at(key); + } + + void clear() { + std::unordered_map::clear(); + ordered_keys_.clear(); + } + + std::pair insert(const value_type& kv) { + if (!count(kv.first)) { + ordered_keys_.push_back(kv.first); + } + return std::unordered_map::insert(kv); + } + + template + std::pair emplace(Args&&... args) { + return insert(value_type(std::forward(args)...)); + } + + iterator erase(const_iterator it) { + ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), it->first)); + return std::unordered_map::erase(it); + } + + iterator erase(iterator it) { + ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), it->first)); + return std::unordered_map::erase(it); + } + + size_t erase(const std::string& key) { + if (std::unordered_map::erase(key)) { + ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), key)); + return 1; + } else { + return 0; + } + } + + const std::vector& ordered_keys() const { return ordered_keys_; } + + friend bool operator==(const object_with_ordered_keys& lhs, const object_with_ordered_keys& rhs); + + private: + std::vector ordered_keys_; +}; + +inline bool operator==(const object_with_ordered_keys& lhs, const object_with_ordered_keys& rhs) { + return static_cast&>(lhs) == + static_cast&>(rhs); +} + +typedef value::array array; +typedef value::object object; + +inline value::value() : type_(null_type), u_() {} + +inline value::value(int type, bool) : type_(type), u_() { + switch (type) { +#define INIT(p, v) \ + case p##type: \ + u_.p = v; \ + break + INIT(boolean_, false); + INIT(number_, 0.0); +#ifdef PICOJSON_USE_INT64 + INIT(int64_, 0); +#endif + INIT(string_, new std::string()); + INIT(array_, new array()); + INIT(object_, new object()); +#undef INIT + default: + break; + } +} + +inline value::value(bool b) : type_(boolean_type), u_() { u_.boolean_ = b; } + +#ifdef PICOJSON_USE_INT64 +inline value::value(int64_t i) : type_(int64_type), u_() { u_.int64_ = i; } +#endif + +inline value::value(double n) : type_(number_type), u_() { + if ( +#ifdef _MSC_VER + !_finite(n) +#elif __cplusplus >= 201103L + std::isnan(n) || std::isinf(n) +#else + isnan(n) || isinf(n) +#endif + ) { +#ifndef PICOJSON_DISABLE_EXCEPTION + throw std::overflow_error(""); +#else + std::abort(); +#endif + } + u_.number_ = n; +} + +inline value::value(const std::string& s) : type_(string_type), u_() { + u_.string_ = new std::string(s); +} + +inline value::value(const array& a) : type_(array_type), u_() { u_.array_ = new array(a); } + +inline value::value(const object& o) : type_(object_type), u_() { u_.object_ = new object(o); } + +#if PICOJSON_USE_RVALUE_REFERENCE +inline value::value(std::string&& s) : type_(string_type), u_() { + u_.string_ = new std::string(std::move(s)); +} + +inline value::value(array&& a) : type_(array_type), u_() { u_.array_ = new array(std::move(a)); } + +inline value::value(object&& o) : type_(object_type), u_() { + u_.object_ = new object(std::move(o)); +} +#endif + +inline value::value(const char* s) : type_(string_type), u_() { u_.string_ = new std::string(s); } + +inline value::value(const char* s, size_t len) : type_(string_type), u_() { + u_.string_ = new std::string(s, len); +} + +inline void value::clear() { + switch (type_) { +#define DEINIT(p) \ + case p##type: \ + delete u_.p; \ + break + DEINIT(string_); + DEINIT(array_); + DEINIT(object_); +#undef DEINIT + default: + break; + } +} + +inline value::~value() { clear(); } + +inline value::value(const value& x) : type_(x.type_), u_() { + switch (type_) { +#define INIT(p, v) \ + case p##type: \ + u_.p = v; \ + break + INIT(string_, new std::string(*x.u_.string_)); + INIT(array_, new array(*x.u_.array_)); + INIT(object_, new object(*x.u_.object_)); +#undef INIT + default: + u_ = x.u_; + break; + } +} + +inline value& value::operator=(const value& x) { + if (this != &x) { + value t(x); + swap(t); + } + return *this; +} + +#if PICOJSON_USE_RVALUE_REFERENCE +inline value::value(value&& x) PICOJSON_NOEXCEPT : type_(null_type), u_() { swap(x); } +inline value& value::operator=(value&& x) PICOJSON_NOEXCEPT { + swap(x); + return *this; +} +#endif +inline void value::swap(value& x) PICOJSON_NOEXCEPT { + std::swap(type_, x.type_); + std::swap(u_, x.u_); +} + +#define IS(ctype, jtype) \ + template <> \ + inline bool value::is() const { \ + return type_ == jtype##_type; \ + } +IS(null, null) +IS(bool, boolean) +#ifdef PICOJSON_USE_INT64 +IS(int64_t, int64) +#endif +IS(std::string, string) +IS(array, array) +IS(object, object) +#undef IS +template <> +inline bool value::is() const { + return type_ == number_type +#ifdef PICOJSON_USE_INT64 + || type_ == int64_type +#endif + // NOLINTNEXTLINE(whitespace/semicolon) + ; +} + +#define GET(ctype, var) \ + template <> \ + inline const ctype& value::get() const { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ + return var; \ + } \ + template <> \ + inline ctype& value::get() { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ + return var; \ + } +GET(bool, u_.boolean_) +GET(std::string, *u_.string_) +GET(array, *u_.array_) +GET(object, *u_.object_) +#ifdef PICOJSON_USE_INT64 +GET(double, + (type_ == int64_type && (const_cast(this)->type_ = number_type, + (const_cast(this)->u_.number_ = u_.int64_)), + u_.number_)) +GET(int64_t, u_.int64_) +#else +GET(double, u_.number_) +#endif +#undef GET + +#define SET(ctype, jtype, setter) \ + template <> \ + inline void value::set(const ctype& _val) { \ + clear(); \ + type_ = jtype##_type; \ + setter \ + } +SET(bool, boolean, u_.boolean_ = _val;) +SET(std::string, string, u_.string_ = new std::string(_val);) +SET(array, array, u_.array_ = new array(_val);) +SET(object, object, u_.object_ = new object(_val);) +SET(double, number, u_.number_ = _val;) +#ifdef PICOJSON_USE_INT64 +SET(int64_t, int64, u_.int64_ = _val;) +#endif +#undef SET + +#if PICOJSON_USE_RVALUE_REFERENCE +#define MOVESET(ctype, jtype, setter) \ + template <> \ + inline void value::set(ctype && _val) { \ + clear(); \ + type_ = jtype##_type; \ + setter \ + } +MOVESET(std::string, string, u_.string_ = new std::string(std::move(_val));) +MOVESET(array, array, u_.array_ = new array(std::move(_val));) +MOVESET(object, object, u_.object_ = new object(std::move(_val));) +#undef MOVESET +#endif + +inline bool value::evaluate_as_boolean() const { + switch (type_) { + case null_type: + return false; + case boolean_type: + return u_.boolean_; + case number_type: + return u_.number_ != 0; +#ifdef PICOJSON_USE_INT64 + case int64_type: + return u_.int64_ != 0; +#endif + case string_type: + return !u_.string_->empty(); + default: + return true; + } +} + +inline const value& value::get(const size_t idx) const { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; +} + +inline value& value::get(const size_t idx) { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; +} + +inline const value& value::get(const std::string& key) const { + static value s_null; + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; +} + +inline value& value::get(const std::string& key) { + static value s_null; + PICOJSON_ASSERT(is()); + object::iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; +} + +inline bool value::contains(const size_t idx) const { + PICOJSON_ASSERT(is()); + return idx < u_.array_->size(); +} + +inline bool value::contains(const std::string& key) const { + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end(); +} + +inline std::string value::to_str() const { + switch (type_) { + case null_type: + return "null"; + case boolean_type: + return u_.boolean_ ? "true" : "false"; +#ifdef PICOJSON_USE_INT64 + case int64_type: { + char buf[sizeof("-9223372036854775808")]; + SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); + return buf; + } +#endif + case number_type: { + char buf[256]; + double tmp; + SNPRINTF( + buf, + sizeof(buf), + fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", + u_.number_ + ); +#if PICOJSON_USE_LOCALE + char* decimal_point = localeconv()->decimal_point; + if (strcmp(decimal_point, ".") != 0) { + size_t decimal_point_len = strlen(decimal_point); + for (char* p = buf; *p != '\0'; ++p) { + if (strncmp(p, decimal_point, decimal_point_len) == 0) { + return std::string(buf, p) + "." + (p + decimal_point_len); + } + } + } +#endif + return buf; + } + case string_type: + return *u_.string_; + case array_type: + return "array"; + case object_type: + return "object"; + default: + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + } + return std::string(); +} + +template +void copy(const std::string& s, Iter oi) { + std::copy(s.begin(), s.end(), oi); +} + +template +struct serialize_str_char { + Iter oi; + void operator()(char c) { + switch (c) { +#define MAP(val, sym) \ + case val: \ + copy(sym, oi); \ + break + MAP('"', "\\\""); + MAP('\\', "\\\\"); + MAP('/', "\\/"); + MAP('\b', "\\b"); + MAP('\f', "\\f"); + MAP('\n', "\\n"); + MAP('\r', "\\r"); + MAP('\t', "\\t"); +#undef MAP + default: + if (static_cast(c) < 0x20 || c == 0x7f) { + char buf[7]; + SNPRINTF(buf, sizeof(buf), "\\u%04x", c & 0xff); + copy(buf, buf + 6, oi); + } else { + *oi++ = c; + } + break; + } + } +}; + +template +void serialize_str(const std::string& s, Iter oi) { + *oi++ = '"'; + serialize_str_char process_char = {oi}; + std::for_each(s.begin(), s.end(), process_char); + *oi++ = '"'; +} + +template +void value::serialize(Iter oi, bool prettify) const { + return _serialize(oi, prettify ? 0 : -1); +} + +inline std::string value::serialize(bool prettify) const { return _serialize(prettify ? 0 : -1); } + +template +void value::_indent(Iter oi, int indent) { + *oi++ = '\n'; + for (int i = 0; i < indent * INDENT_WIDTH; ++i) { + *oi++ = ' '; + } +} + +template +void value::_serialize(Iter oi, int indent) const { + switch (type_) { + case string_type: + serialize_str(*u_.string_, oi); + break; + case array_type: { + *oi++ = '['; + if (indent != -1) { + ++indent; + } + for (array::const_iterator i = u_.array_->begin(); i != u_.array_->end(); ++i) { + if (i != u_.array_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + i->_serialize(oi, indent); + } + if (indent != -1) { + --indent; + if (!u_.array_->empty()) { + _indent(oi, indent); + } + } + *oi++ = ']'; + break; + } + case object_type: { + *oi++ = '{'; + if (indent != -1) { + ++indent; + } + +#if PICOJSON_USE_ORDERED_OBJECT + for (auto i = u_.object_->ordered_keys().begin(); i != u_.object_->ordered_keys().end(); + ++i) { + if (i != u_.object_->ordered_keys().begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + serialize_str(*i, oi); + *oi++ = ':'; + if (indent != -1) { + *oi++ = ' '; + } + u_.object_->at(*i)._serialize(oi, indent); + } +#else + for (object::const_iterator i = u_.object_->begin(); i != u_.object_->end(); ++i) { + if (i != u_.object_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + serialize_str(i->first, oi); + *oi++ = ':'; + if (indent != -1) { + *oi++ = ' '; + } + i->second._serialize(oi, indent); + } +#endif + if (indent != -1) { + --indent; + if (!u_.object_->empty()) { + _indent(oi, indent); + } + } + *oi++ = '}'; + break; + } + default: + copy(to_str(), oi); + break; + } + if (indent == 0) { + *oi++ = '\n'; + } +} + +inline std::string value::_serialize(int indent) const { + std::string s; + _serialize(std::back_inserter(s), indent); + return s; +} + +template +class input { + protected: + Iter cur_, end_; + bool consumed_; + int line_; + + public: + input(const Iter& first, const Iter& last) + : cur_(first), end_(last), consumed_(false), line_(1) {} + int getc() { + if (consumed_) { + if (*cur_ == '\n') { + ++line_; + } + ++cur_; + } + if (cur_ == end_) { + consumed_ = false; + return -1; + } + consumed_ = true; + return *cur_ & 0xff; + } + void ungetc() { consumed_ = false; } + Iter cur() const { + if (consumed_) { + input* self = const_cast*>(this); + self->consumed_ = false; + ++self->cur_; + } + return cur_; + } + int line() const { return line_; } + void skip_ws() { + while (1) { + int ch = getc(); + if (!(ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { + ungetc(); + break; + } + } + } + bool expect(const int expected) { + skip_ws(); + if (getc() != expected) { + ungetc(); + return false; + } + return true; + } + bool match(const std::string& pattern) { + for (std::string::const_iterator pi(pattern.begin()); pi != pattern.end(); ++pi) { + if (getc() != *pi) { + ungetc(); + return false; + } + } + return true; + } +}; + +template +// NOLINTNEXTLINE(runtime/references) +inline int _parse_quadhex(input& in) { + int uni_ch = 0, hex; + for (int i = 0; i < 4; i++) { + if ((hex = in.getc()) == -1) { + return -1; + } + if ('0' <= hex && hex <= '9') { + hex -= '0'; + } else if ('A' <= hex && hex <= 'F') { + hex -= 'A' - 0xa; + } else if ('a' <= hex && hex <= 'f') { + hex -= 'a' - 0xa; + } else { + in.ungetc(); + return -1; + } + uni_ch = uni_ch * 16 + hex; + } + return uni_ch; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse_codepoint(String& out, input& in) { + int uni_ch; + if ((uni_ch = _parse_quadhex(in)) == -1) { + return false; + } + if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { + if (0xdc00 <= uni_ch) { + // a second 16-bit of a surrogate pair appeared + return false; + } + // first 16-bit of surrogate pair, get the next one + if (in.getc() != '\\' || in.getc() != 'u') { + in.ungetc(); + return false; + } + int second = _parse_quadhex(in); + if (!(0xdc00 <= second && second <= 0xdfff)) { + return false; + } + uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); + uni_ch += 0x10000; + } + if (uni_ch < 0x80) { + out.push_back(static_cast(uni_ch)); + } else { + if (uni_ch < 0x800) { + out.push_back(static_cast(0xc0 | (uni_ch >> 6))); + } else { + if (uni_ch < 0x10000) { + out.push_back(static_cast(0xe0 | (uni_ch >> 12))); + } else { + out.push_back(static_cast(0xf0 | (uni_ch >> 18))); + out.push_back(static_cast(0x80 | ((uni_ch >> 12) & 0x3f))); + } + out.push_back(static_cast(0x80 | ((uni_ch >> 6) & 0x3f))); + } + out.push_back(static_cast(0x80 | (uni_ch & 0x3f))); + } + return true; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse_string(String& out, input& in) { + while (1) { + int ch = in.getc(); + if (ch < ' ') { + in.ungetc(); + return false; + } else if (ch == '"') { + return true; + } else if (ch == '\\') { + if ((ch = in.getc()) == -1) { + return false; + } + switch (ch) { +#define MAP(sym, val) \ + case sym: \ + out.push_back(val); \ + break + MAP('"', '\"'); + MAP('\\', '\\'); + MAP('/', '/'); + MAP('b', '\b'); + MAP('f', '\f'); + MAP('n', '\n'); + MAP('r', '\r'); + MAP('t', '\t'); +#undef MAP + case 'u': + if (!_parse_codepoint(out, in)) { + return false; + } + break; + default: + return false; + } + } else { + out.push_back(static_cast(ch)); + } + } + return false; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse_array(Context& ctx, input& in) { + if (!ctx.parse_array_start()) { + return false; + } + size_t idx = 0; + if (in.expect(']')) { + return ctx.parse_array_stop(idx); + } + do { + if (!ctx.parse_array_item(in, idx)) { + return false; + } + idx++; + } while (in.expect(',')); + return in.expect(']') && ctx.parse_array_stop(idx); +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse_object(Context& ctx, input& in) { + if (!ctx.parse_object_start()) { + return false; + } + if (in.expect('}')) { + return true; + } + do { + std::string key; + if (!in.expect('"') || !_parse_string(key, in) || !in.expect(':')) { + return false; + } + if (!ctx.parse_object_item(in, key)) { + return false; + } + } while (in.expect(',')); + return in.expect('}'); +} + +template +// NOLINTNEXTLINE(runtime/references) +inline std::string _parse_number(input& in) { + std::string num_str; + while (1) { + int ch = in.getc(); + if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' || ch == 'e' || ch == 'E') { + num_str.push_back(static_cast(ch)); + } else if (ch == '.') { +#if PICOJSON_USE_LOCALE + num_str += localeconv()->decimal_point; +#else + num_str.push_back('.'); +#endif + } else { + in.ungetc(); + break; + } + } + return num_str; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse(Context& ctx, input& in) { + in.skip_ws(); + int ch = in.getc(); + switch (ch) { +#define IS(ch, text, op) \ + case ch: \ + if (in.match(text) && op) { \ + return true; \ + } else { \ + return false; \ + } + IS('n', "ull", ctx.set_null()); + IS('f', "alse", ctx.set_bool(false)); + IS('t', "rue", ctx.set_bool(true)); +#undef IS + case '"': + return ctx.parse_string(in); + case '[': + return _parse_array(ctx, in); + case '{': + return _parse_object(ctx, in); + default: + if (('0' <= ch && ch <= '9') || ch == '-') { + double f; + char* endp; + in.ungetc(); + std::string num_str(_parse_number(in)); + if (num_str.empty()) { + return false; + } +#ifdef PICOJSON_USE_INT64 + { + errno = 0; + intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); + if (errno == 0 && std::numeric_limits::min() <= ival && + ival <= std::numeric_limits::max() && + endp == num_str.c_str() + num_str.size()) { + ctx.set_int64(ival); + return true; + } + } +#endif + f = strtod(num_str.c_str(), &endp); + if (endp == num_str.c_str() + num_str.size()) { + ctx.set_number(f); + return true; + } + return false; + } + break; + } + in.ungetc(); + return false; +} + +class deny_parse_context { + public: + bool set_null() { return false; } + bool set_bool(bool) { return false; } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { return false; } +#endif + bool set_number(double) { return false; } + template + bool parse_string(input&) { + return false; + } + bool parse_array_start() { return false; } + template + bool parse_array_item(input&, size_t) { + return false; + } + bool parse_array_stop(size_t) { return false; } + bool parse_object_start() { return false; } + template + bool parse_object_item(input&, const std::string&) { + return false; + } +}; + +class default_parse_context { + protected: + value* out_; + + public: + // NOLINTNEXTLINE(runtime/explicit) + default_parse_context(value* out) : out_(out) {} + bool set_null() { + *out_ = value(); + return true; + } + bool set_bool(bool b) { + *out_ = value(b); + return true; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t i) { + *out_ = value(i); + return true; + } +#endif + bool set_number(double f) { + *out_ = value(f); + return true; + } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_string(input& in) { + *out_ = value(string_type, false); + return _parse_string(out_->get(), in); + } + bool parse_array_start() { + *out_ = value(array_type, false); + return true; + } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_array_item(input& in, size_t) { + array& a = out_->get(); + a.push_back(value()); + default_parse_context ctx(&a.back()); + return _parse(ctx, in); + } + bool parse_array_stop(size_t) { return true; } + bool parse_object_start() { + *out_ = value(object_type, false); + return true; + } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_object_item(input& in, const std::string& key) { + object& o = out_->get(); + default_parse_context ctx(&o[key]); + return _parse(ctx, in); + } + + private: + default_parse_context(const default_parse_context&); + default_parse_context& operator=(const default_parse_context&); +}; + +class null_parse_context { + public: + struct dummy_str { + void push_back(int) {} + }; + + public: + null_parse_context() {} + bool set_null() { return true; } + bool set_bool(bool) { return true; } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { return true; } +#endif + bool set_number(double) { return true; } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_string(input& in) { + dummy_str s; + return _parse_string(s, in); + } + bool parse_array_start() { return true; } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_array_item(input& in, size_t) { + return _parse(*this, in); + } + bool parse_array_stop(size_t) { return true; } + bool parse_object_start() { return true; } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_object_item(input& in, const std::string&) { + return _parse(*this, in); + } + + private: + null_parse_context(const null_parse_context&); + null_parse_context& operator=(const null_parse_context&); +}; + +// obsolete, use the version below +template +// NOLINTNEXTLINE(runtime/references) +inline std::string parse(value& out, Iter& pos, const Iter& last) { + std::string err; + pos = parse(out, pos, last, &err); + return err; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline Iter _parse(Context& ctx, const Iter& first, const Iter& last, std::string* err) { + input in(first, last); + if (!_parse(ctx, in) && err != NULL) { + char buf[64]; + SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); + *err = buf; + while (1) { + int ch = in.getc(); + if (ch == -1 || ch == '\n') { + break; + } else if (ch >= ' ') { + err->push_back(static_cast(ch)); + } + } + } + return in.cur(); +} + +template +// NOLINTNEXTLINE(runtime/references) +inline Iter parse(value& out, const Iter& first, const Iter& last, std::string* err) { + default_parse_context ctx(&out); + return _parse(ctx, first, last, err); +} + +// NOLINTNEXTLINE(runtime/references) +inline std::string parse(value& out, const std::string& s) { + std::string err; + parse(out, s.begin(), s.end(), &err); + return err; +} + +// NOLINTNEXTLINE(runtime/references) +inline std::string parse(value& out, std::istream& is) { + std::string err; + parse(out, std::istreambuf_iterator(is.rdbuf()), std::istreambuf_iterator(), &err); + return err; +} + +template +struct last_error_t { + static std::string s; +}; +template +// NOLINTNEXTLINE(runtime/string) +std::string last_error_t::s; + +inline void set_last_error(const std::string& s) { last_error_t::s = s; } + +inline const std::string& get_last_error() { return last_error_t::s; } + +inline bool operator==(const value& x, const value& y) { + if (x.is()) return y.is(); +#define PICOJSON_CMP(type) \ + if (x.is()) return y.is() && x.get() == y.get() + PICOJSON_CMP(bool); + PICOJSON_CMP(double); + PICOJSON_CMP(std::string); + PICOJSON_CMP(array); + PICOJSON_CMP(object); +#undef PICOJSON_CMP + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + return false; +} + +inline bool operator!=(const value& x, const value& y) { return !(x == y); } +} // namespace picojson + +#if !PICOJSON_USE_RVALUE_REFERENCE +namespace std { +template <> +inline void swap(picojson::value& x, picojson::value& y) { + x.swap(y); +} +} // namespace std +#endif + +inline std::istream& operator>>(std::istream& is, picojson::value& x) { + picojson::set_last_error(std::string()); + const std::string err(picojson::parse(x, is)); + if (!err.empty()) { + picojson::set_last_error(err); + is.setstate(std::ios::failbit); + } + return is; +} + +inline std::ostream& operator<<(std::ostream& os, const picojson::value& x) { + x.serialize(std::ostream_iterator(os)); + return os; +} +#ifdef _MSC_VER +#pragma warning(pop) +#endif diff --git a/libs/xgrammar/include/xgrammar/compiler.h b/libs/xgrammar/include/xgrammar/compiler.h new file mode 100644 index 000000000..a2f0138dc --- /dev/null +++ b/libs/xgrammar/include/xgrammar/compiler.h @@ -0,0 +1,115 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/compiler.h + * \brief The header for the compiler. + */ + +#ifndef XGRAMMAR_COMPILER_H_ +#define XGRAMMAR_COMPILER_H_ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "xgrammar/exception.h" + +namespace xgrammar { + +/*! + * \brief The compiled grammar of a GrammarMatcher. It contains the preprocessing results of the + * grammar and tokenizer. + */ +class CompiledGrammar { + public: + /*! \brief Get the associated grammar. */ + Grammar GetGrammar() const; + + /*! \brief Get the associated tokenizer info. */ + TokenizerInfo GetTokenizerInfo() const; + + /*! \brief Return the approximate memory usage of the grammar in bytes. */ + std::size_t MemorySizeBytes() const; + + /*! \brief Return the serialized JSON string of the compiled grammar. */ + std::string SerializeJSON() const; + + /*! \brief Deserialize a compiled grammar from a JSON string and tokenizer info. */ + static std::variant DeserializeJSON( + const std::string& json_string, const TokenizerInfo& tokenizer_info + ); + + XGRAMMAR_DEFINE_PIMPL_METHODS(CompiledGrammar); +}; + +/*! + * \brief A cache to get the compiled grammar for grammar or schema. This class avoids + * redundant preprocessing of the grammar or schema when constructing a CompiledGrammar. + * \note This class is associated with a vocabulary when constructed. The vocabulary is used to + * create every compiled grammar. If multiple toke tables are used to create init + * contexts, an instance of this class for each vocabulary should be created. + */ +class GrammarCompiler { + public: + /*! + * \brief Construct a GrammarCompiler with a vocabulary. This class will always + * create compiled grammars with this vocabulary. + * \param tokenizer_info The tokenizer info. + * \param max_threads The maximum number of threads to use for compiling grammars. + * \param cache_enabled Whether to enable the cache. + * \param max_memory_bytes The maximum memory usage in bytes. + */ + GrammarCompiler( + const TokenizerInfo& tokenizer_info, + int max_threads = 8, + bool cache_enabled = true, + int64_t max_memory_bytes = -1 // unlimited + ); + + /*! \brief Get the compiled grammar for a JSON schema string. */ + CompiledGrammar CompileJSONSchema( + const std::string& schema, + bool any_whitespace = true, + std::optional indent = std::nullopt, + std::optional> separators = std::nullopt, + bool strict_mode = true, + std::optional max_whitespace_cnt = std::nullopt + ); + + /*! \brief Get the compiled grammar for pure JSON. */ + CompiledGrammar CompileBuiltinJSONGrammar(); + + /*! \brief Get the compiled grammar for a grammar. */ + CompiledGrammar CompileGrammar(const Grammar& grammar); + + /*! \brief Get the compiled grammar for a grammar. */ + CompiledGrammar CompileGrammar( + const std::string& ebnf_str, const std::string& root_rule_name = "root" + ); + + /*! \brief Get the compiled grammar for a structural tag. */ + CompiledGrammar CompileStructuralTag(const std::string& structural_tag_json); + + /*! \brief Get the compiled grammar for a regex. */ + CompiledGrammar CompileRegex(const std::string& regex); + + /*! \brief Clear the internal cache of compiled grammars. */ + void ClearCache(); + + /*! \brief Return the approximate memory usage of the compiler in bytes. */ + int64_t GetCacheSizeBytes() const; + + /*! \brief Return the approximate memory usage of the compiler in bytes. -1 means unlimited. */ + int64_t CacheLimitBytes() const; + + XGRAMMAR_DEFINE_PIMPL_METHODS(GrammarCompiler); +}; + +} // namespace xgrammar + +#endif // XGRAMMAR_COMPILER_H_ diff --git a/libs/xgrammar/include/xgrammar/config.h b/libs/xgrammar/include/xgrammar/config.h new file mode 100644 index 000000000..4e5de5e84 --- /dev/null +++ b/libs/xgrammar/include/xgrammar/config.h @@ -0,0 +1,35 @@ +/*! + * Copyright (c) 2025 by Contributors + * \file xgrammar/config.h + * \brief Global configuration for XGrammar. + */ + +#ifndef XGRAMMAR_CONFIG_H_ +#define XGRAMMAR_CONFIG_H_ + +#include + +namespace xgrammar { + +/*! + * \brief Set the maximum recursion depth for the grammar. + * \param max_recursion_depth The maximum recursion depth. + */ +void SetMaxRecursionDepth(int max_recursion_depth); + +/*! + * \brief Get the maximum recursion depth for the grammar. + * \return The maximum recursion depth. + */ +int GetMaxRecursionDepth(); + +/*! + * \brief Get the serialization version for the grammar. + * \return The serialization version. + * \note This is used to check the compatibility of the serialized grammar. + */ +std::string GetSerializationVersion(); + +} // namespace xgrammar + +#endif // XGRAMMAR_CONFIG_H_ diff --git a/libs/xgrammar/include/xgrammar/exception.h b/libs/xgrammar/include/xgrammar/exception.h new file mode 100644 index 000000000..7efb3b000 --- /dev/null +++ b/libs/xgrammar/include/xgrammar/exception.h @@ -0,0 +1,80 @@ +#ifndef XGRAMMAR_EXCEPTION_H +#define XGRAMMAR_EXCEPTION_H + +#include +#include +#include + +namespace xgrammar { + +/************** Exception Definitions **************/ + +/*! + * \brief Exception thrown when the version in the serialized data does not follow the current + * serialization version. + */ + +struct XGrammarError : std::runtime_error { + XGrammarError(const std::string& message) : std::runtime_error(message) {} + virtual std::string GetType() const { return "XGrammarError"; } +}; + +struct DeserializeVersionError : XGrammarError { + DeserializeVersionError(const std::string& message) + : XGrammarError(std::string("Deserialize version error: ") + message) {} + std::string GetType() const override { return "DeserializeVersionError"; } +}; + +/*! + * \brief Exception thrown when the JSON is invalid. + */ +struct InvalidJSONError : XGrammarError { + InvalidJSONError(const std::string& message) + : XGrammarError(std::string("Invalid JSON error: ") + message) {} + std::string GetType() const override { return "InvalidJSONError"; } +}; + +/*! + * \brief Exception thrown when the serialized data does not follow the expected format. + */ +struct DeserializeFormatError : XGrammarError { + DeserializeFormatError(const std::string& message) + : XGrammarError(std::string("Deserialize format error: ") + message) {} + std::string GetType() const override { return "DeserializeFormatError"; } +}; + +/*! + * \brief Exception thrown when the JSON schema is invalid or not satisfiable. + */ +struct InvalidJSONSchemaError : XGrammarError { + InvalidJSONSchemaError(const std::string& message) + : XGrammarError(std::string("Invalid JSON schema error: ") + message) {} + std::string GetType() const override { return "InvalidJSONSchemaError"; } +}; + +/*! + * \brief Exception thrown when the structural tag is invalid. + */ +struct InvalidStructuralTagError : XGrammarError { + InvalidStructuralTagError(const std::string& message) + : XGrammarError(std::string("Invalid structural tag error: ") + message) {} + std::string GetType() const override { return "InvalidStructuralTagError"; } +}; + +/************** Union Exceptions **************/ + +/*! + * \brief Represents a serialization error. + */ +using SerializationError = + std::variant; + +/*! + * \brief Represents an error from the structural tag conversion. + */ +using StructuralTagError = + std::variant; + +} // namespace xgrammar + +#endif // XGRAMMAR_EXCEPTION_H diff --git a/libs/xgrammar/include/xgrammar/grammar.h b/libs/xgrammar/include/xgrammar/grammar.h new file mode 100644 index 000000000..17f7d6540 --- /dev/null +++ b/libs/xgrammar/include/xgrammar/grammar.h @@ -0,0 +1,191 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/grammar.h + * \brief The header for the definition and construction of BNF grammar. + */ + +#ifndef XGRAMMAR_GRAMMAR_H_ +#define XGRAMMAR_GRAMMAR_H_ + +#include +#include + +#include +#include +#include +#include +#include + +#include "xgrammar/exception.h" + +namespace xgrammar { + +struct StructuralTagItem { + std::string begin; + std::string schema; + std::string end; + + bool operator==(const StructuralTagItem& other) const { + return begin == other.begin && schema == other.schema && end == other.end; + } +}; + +/*! + * \brief This class stores the abstract syntax tree (AST) of the Backus-Naur Form (BNF) grammar. + * The BNF definition here is standard BNF, and the characters are represented using regex-style + * character classes (e.g. [a-z], [^a-z]). + * + * \details + * ### Rules + * The BNF grammar AST consists of a set of rules. Each rule contains a name and a definition, and + * corresponds to a production in the grammar. The definition of a rule is a GrammarExpr. Each rule + * has a rule_id for reference. + * + * ### GrammarExprs + * GrammarExpr is the definition of a rule or part of the definition of a rule. It can contain + * elements, empty string, reference to other GrammarExprs, or reference to other rules. Each + * GrammarExpr corresponds to a grammar_expr_id for reference. + * + * For example, in the following rule: rule ::= ("a" "b") | "c" + * ("a" "b"), "c", ("a" "b") | "c" are all GrammarExprs. + * + * #### Types of GrammarExprs + * Every GrammarExpr is represented by a type as well as a variable-length array containing its + * data. GrammarExpr has several types: + * - Byte string: a string of bytes (0~255). Supports UTF-8 strings. + * - Character class: a range of characters (each character is a unicode codepoint), e.g. [a-z], + * [ac-z]. Can be negated: [^a-z], [^ac-z]. Now only ascii chars is allowed in [], but this + * expression can accept/reject unicode chars. + * - Character class star: a star quantifier of a character class. e.g. [a-z]*, [^a-z]*. + * - EmptyStr: an empty string, i.e. "" + * - Rule reference: a reference to another rule + * - Sequence: a sequence of grammar_exprs, e.g. ("a" "b"). These grammar_exprs are concatenated + * together. + * - Choices: a choice of grammar_exprs, e.g. ("a" "b") | "c". Each grammar_expr can be matched. + * + * #### Storage of GrammarExprs + * Each type of GrammarExpr has a different data format. For the format of each type of GrammarExpr, + * see docs in Grammar::Impl::GrammarExprType. + * + * We store all GrammarExprs in csr_matrix style. That is, they are stored consecutively in one + * vector (data vector) and the starting position of each GrammarExpr is recorded in the indptr + * vector. + * + * \remark The character class star GrammarExpr is for the special support for elements like [a-z]* + * in the grammar. We add it to make the matching more efficient, as we can avoid recursion into + * rules when matching a sequence of characters. It should be used like: + * rule1 ::= ((element1 element2 rule2 ...) | ...) + * rule2 ::= character_class_star_grammar_expr(id_of_a_character_class_grammar_expr) + */ +class Grammar { + public: + /*! + * \brief Get the EBNF string of the grammar. + */ + std::string ToString() const; + + /*! + * \brief Construct a BNF grammar with a EBNF-formatted string. The grammar will be normalized + * (simplified) by default. + * \param ebnf_string The EBNF-formatted string. + * \param root_rule_name The name of the root rule. + */ + static Grammar FromEBNF( + const std::string& ebnf_string, const std::string& root_rule_name = "root" + ); + + /*! + * \brief Construct a BNF grammar from the json schema string. The schema string should be in the + * format of the schema of a JSON file. We will parse the schema and generate a BNF grammar. + * \param schema The schema string. + * \param indent The number of spaces for indentation. If set to std::nullopt, the output will be + * in one line. Default: 2. + * \param separators Two separators used in the schema: comma and colon. Examples: {",", ":"}, + * {", ", ": "}. If std::nullopt, the default separators will be used: {",", ": "} when the + * indent is not nullopt, and {", ", ": "} otherwise. This follows the convention in python + * json.dumps(). Default: std::nullopt. + * \param strict_mode Whether to use strict mode. In strict mode, the generated grammar will not + * allow properties and items that is not specified in the schema. This is equivalent to + * setting unevaluatedProperties and unevaluatedItems to false. + * + * This helps LLM to generate accurate output in the grammar-guided generation with JSON + * schema. Default: true. + */ + static Grammar FromJSONSchema( + const std::string& schema, + bool any_whitespace = true, + std::optional indent = std::nullopt, + std::optional> separators = std::nullopt, + bool strict_mode = true, + std::optional max_whitespace_cnt = std::nullopt, + bool print_converted_ebnf = false + ); + + /*! + * \brief Construct a grammar from a regular expression string. + * \param regex The regular expression string. + * \param print_converted_ebnf This method will convert the regex to EBNF first. If this is true, + * the converted EBNF string will be printed. For debugging purpose. Default: false. + */ + static Grammar FromRegex(const std::string& regex, bool print_converted_ebnf = false); + + /*! + * \brief Construct a grammar from a structural tag string. + * \param structural_tag_json The structural tag string. + * \param tokenizer_info Optional tokenizer info for resolving string token references. + */ + static std::variant FromStructuralTag( + const std::string& structural_tag_json, + const std::optional& tokenizer_info = std::nullopt + ); + + /*! + * \brief Get the grammar of standard JSON format. We have built-in support for JSON. + * \return The grammar of standard JSON format. + */ + static Grammar BuiltinJSONGrammar(); + + /*! + * \brief Create a grammar that matches any of the grammars in the list. That is equivalent to + * using the `|` operator to concatenate the grammars in the list. + * \param grammars The grammars to create the union of. + * \returns The union of the grammars. + */ + static Grammar Union(const std::vector& grammars); + + /*! + * \brief Create a grammar that matches the concatenation of the grammars in the list. That is + * equivalent to using the `+` operator to concatenate the grammars in the list. + * \param grammars The grammars to create the concatenation of. + * \returns The concatenation of the grammars. + */ + static Grammar Concat(const std::vector& grammars); + + /*! + * \brief Print a BNF grammar. + * \param os The output stream. + * \param grammar The grammar to print. + * \return The output stream. + */ + friend std::ostream& operator<<(std::ostream& os, const Grammar& grammar); + + /*! + * \brief Return the serialized JSON string of the grammar. + * \return The serialized JSON string. + */ + std::string SerializeJSON() const; + + /*! + * \brief Deserialize a grammar from a JSON string. + * \param json_string The JSON string to deserialize. + * \return If the deserialization is successful, return the grammar. Otherwise, return a runtime + * error with the error message. + */ + static std::variant DeserializeJSON(const std::string& json_string); + + XGRAMMAR_DEFINE_PIMPL_METHODS(Grammar); +}; + +} // namespace xgrammar + +#endif // XGRAMMAR_GRAMMAR_H_ diff --git a/libs/xgrammar/include/xgrammar/matcher.h b/libs/xgrammar/include/xgrammar/matcher.h new file mode 100644 index 000000000..654e76014 --- /dev/null +++ b/libs/xgrammar/include/xgrammar/matcher.h @@ -0,0 +1,256 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/matcher.h + * \brief The header for the matcher. + */ + +#ifndef XGRAMMAR_MATCHER_H_ +#define XGRAMMAR_MATCHER_H_ + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace xgrammar { + +int32_t GetBitmaskSize(int vocab_size); + +DLDataType GetBitmaskDLType(); + +void _DebugGetMaskedTokensFromBitmask( + std::vector* rejected_tokens, const DLTensor& token_bitmask, int vocab_size, int index = 0 +); + +std::pair _IsSingleTokenBitmask(const DLTensor& bitmask, int vocab_size, int index); + +void ApplyTokenBitmaskInplaceCPU( + DLTensor* logits, + const DLTensor& bitmask, + int vocab_size = -1, + std::optional> indices = std::nullopt +); + +/*! + * \brief A stateful matcher to match tokens to the specified BNF grammar. This class is the core + * logic of the grammar-guided generation. + * + * \details This class implements the non-deterministic pushdown automaton (NPDA) matching algorithm + * to match characters to a BNF grammar. It keep track of the current state of the matching process + * by maintaining several stacks internally as possible paths in the NPDA. It also supports + * backtracking. + * + * It is particularly capable of finding the set of tokens that are acceptable for the next step + * and storing them in a bitmask. This aids in grammar-guided generation. + * + * \example + * \code + * Tokenizer tokenizer = ...; + * auto compiled_grammar = GrammarMatcher::CreateCompiledGrammar(grammar, + * tokenizer->PostProcessedVocab()); + * GrammarMatcher matcher(compiled_grammar, 10); + * matcher->AcceptToken(67); + * + * // Construct a DLTensor with shape (tokenizer.GetVocabSize() + 31) / 32, and dtype int32. + * DLTensor next_token_bitmask = ...; + * matcher->FillNextTokenBitmask(&next_token_bitmask); + * + * // Rollback is supported + * matcher->Rollback(1); + * \endcode + */ +class GrammarMatcher { + public: + /*! + * \brief Construct a GrammarMatcher from the preprocessing result of type + * CompiledGrammar. + * \param compiled_grammar The compiled grammar. It is obtained through + * CreateCompiledGrammar as a result of preprocessing the grammar and tokenizer. + */ + GrammarMatcher( + const CompiledGrammar& compiled_grammar, + std::optional> override_stop_tokens = std::nullopt, + bool terminate_without_stop_token = false, + int max_rollback_tokens = -1 + ); + + /*! + * \brief Accept one token and update the state of the matcher. + * \param token_id The id of the token to accept. + * \return Whether the token is accepted. + * \note Termination state. + * When the end of the root rule is reached, the matcher can only accept the stop token. + * The matcher is terminated after accepting the stop token, i.e. no AcceptToken or + * FindNextTokenMask operations can be performed. The termination state can be canceled + * using Rollback(). + */ + bool AcceptToken(int32_t token_id, bool debug_print = false); + + /*! + * \brief Accept a string and update the state of the matcher. The whole string is considered + * as one step in rollback. It is used to complement the functionality of AcceptToken, and + * AcceptToken should always be used to accept tokens. + * \param input_str The string to be accepted. + * \param debug_print Whether to print information about the internal state of the matcher. + * \return Whether the string is accepted. + */ + bool AcceptString(const std::string& input_str, bool debug_print = false); + + /*! + * \brief Get the set of tokens that are acceptable for the next step and store them in a + * bitmask. + * \param next_token_bitmask The bitmask to store the result. The bitmask must be pre-allocated + * and with shape (GetBitmaskSize(),) and dtype int32. + * \return Whether the bitmask need to be applied (not all-true). + */ + bool FillNextTokenBitmask(DLTensor* next_token_bitmask, int index = 0, bool debug_print = false); + + /*! + * \brief Traverse a draft token tree and fill the token bitmask for each node. + * + * This function performs a DFS traversal of the speculative decoding tree and fills + * the token bitmask for each node based on grammar constraints. + * + * \param retrieve_next_token DLTensor where retrieve_next_token[i] gives the index of + * the child node of node i, or -1 if no child exists. + * \param retrieve_next_sibling DLTensor where retrieve_next_sibling[i] gives the index of + * the sibling node of node i, or -1 if no sibling exists. + * \param draft_tokens DLTensor of draft token ids at each node. + * \param token_bitmask DLTensor to store the bitmask (2D: num_nodes x bitmask_size). + * \param time_threshold Maximum allowed time in seconds for the DFS traversal. + * If the traversal exceeds this threshold, it returns false. + * A value <= 0 disables the timeout (default: -1.0). + * \return true if the traversal completed successfully, false if it timed out. + */ + bool TraverseDraftTree( + const DLTensor* retrieve_next_token, + const DLTensor* retrieve_next_sibling, + const DLTensor* draft_tokens, + DLTensor* token_bitmask, + double time_threshold = -1.0 + ); + + /*! + * \brief Find the jump-forward string for jump-forward decoding. This is the longest string that + will be valid according to the current syntax. + * \note This method does not change the grammar state. + */ + std::string FindJumpForwardString(); + + /*! + * \brief Rollback the matcher to a previous state. + * \param num_tokens The number of tokens to rollback. It cannot exceed the current number of + * steps, nor can it exceed the specified maximum number of rollback tokens. + */ + void Rollback(int num_tokens = 1); + + /*! + * \brief Check if the matcher has accepted the stop token and terminated. + * \sa AcceptToken + */ + bool IsTerminated() const; + + /*! + * \brief Check if the grammar's root rule has been fully matched by the input accepted so far. + * Unlike IsTerminated(), this does not require the stop token to have been accepted. + * \sa IsTerminated, AcceptToken + */ + bool IsCompleted() const; + + /*! \brief Reset the matcher to the initial state. */ + void Reset(); + + /*! + * \brief Fork the matcher. Returns a new GrammarMatcher with a deep copy of all state except + * compiled_grammar and tokenizer_info, which are shared with this matcher. + */ + GrammarMatcher Fork() const; + + /*! \brief Get the maximum number of rollback tokens allowed. */ + int GetMaxRollbackTokens() const; + + const std::vector& GetStopTokenIds() const; + + /*! \brief Print the internal state of the matcher. This is only used for debugging. The + * representation of the internal state is subject to change. + */ + std::string _DebugPrintInternalState() const; + + XGRAMMAR_DEFINE_PIMPL_METHODS(GrammarMatcher); +}; + +/*! + * \brief A batched version of GrammarMatcher for better efficiency. It supports batch processing + * of multiple GrammarMatcher objects in parallel. + * + * \details This class provides batched versions of the core methods of GrammarMatcher, including + * FillNextTokenBitmask, AcceptString, and AcceptToken. It utilizes multi-threading to process + * multiple GrammarMatcher objects simultaneously, significantly improving efficiency when dealing + * with a large number of matchers. + */ +class BatchGrammarMatcher { + public: + BatchGrammarMatcher(std::variant max_threads = "auto"); + + /*! + \brief A batched version of FillNextTokenBitmask for better efficiency. + \param matchers The array of GrammarMatcher objects. + \param next_token_bitmask The pre-allocated DLTensor to store the result bitmasks. + \param indices The optional array of indices to specify which matcher corresponds to which slice + of the bitmask tensor. If not provided, all matchers will write to the corresponding + indices(matchers[i] to next_token_bitmask[i]). + \param debug_print Whether to print debug information. Default is false. + */ + void BatchFillNextTokenBitmask( + std::vector* matchers, + DLTensor* next_token_bitmask, + const std::optional>& indices = std::nullopt, + bool debug_print = false + ); + + /*! + * \brief A batched version of AcceptString for better efficiency. + * \param matchers The array of GrammarMatcher objects. + * \param input_strs The array of input strings to be accepted. + * \param debug_print Whether to print debug information. Default is false. + * \return A vector of bytes indicating whether each string is accepted. + */ + static std::vector BatchAcceptString( + std::vector* matchers, + const std::vector& input_strs, + bool debug_print = false + ); + + /*! + * \brief A batched version of AcceptToken for better efficiency. + * \param matchers The array of GrammarMatcher objects. + * \param token_ids The array of token ids to be accepted. + * \param debug_print Whether to print debug information. Default is false. + * \return A vector of bytes indicating whether each token is accepted. + */ + static std::vector BatchAcceptToken( + std::vector* matchers, + const std::vector& token_ids, + bool debug_print = false + ); + + /*! + * \brief A batched version of Rollback for better efficiency. + * \param matchers The array of GrammarMatcher objects. + * \param num_tokens The array of the number of tokens to rollback for each matcher. + */ + static void BatchRollback( + std::vector* matchers, const std::vector& num_tokens + ); + + XGRAMMAR_DEFINE_PIMPL_METHODS(BatchGrammarMatcher); +}; + +} // namespace xgrammar + +#endif // XGRAMMAR_MATCHER_H_ diff --git a/libs/xgrammar/include/xgrammar/object.h b/libs/xgrammar/include/xgrammar/object.h new file mode 100644 index 000000000..641b5bc52 --- /dev/null +++ b/libs/xgrammar/include/xgrammar/object.h @@ -0,0 +1,51 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/object.h + * \brief Utilities for creating objects. + */ + +#ifndef XGRAMMAR_OBJECT_H_ +#define XGRAMMAR_OBJECT_H_ + +#include // IWYU pragma: keep +#include // IWYU pragma: keep + +namespace xgrammar { + +/*! + * \brief A tag type for creating a null object. + */ +struct NullObj {}; + +/*! + * \brief This macro defines the methods for the PImpl classes. + * \details Many classes in xgrammar are PImpl classes. PImpl classes only stores a shared pointer + * to the implementation. This allows reference-counter-based memory management and efficient + * object copy and passing. We always expose PImpl classes to Python to control over object sharing + * and memory management. Note simple and critical classes should not be defined as PImpl classes, + * but as normal classes for better efficiency. + */ +#define XGRAMMAR_DEFINE_PIMPL_METHODS(TypeName) \ + public: \ + class Impl; \ + /* Construct a null object. Note operating on a null object will fail. */ \ + explicit TypeName(NullObj) : pimpl_(nullptr) {} \ + /* Construct object with a shared pointer to impl. */ \ + explicit TypeName(std::shared_ptr pimpl) : pimpl_(std::move(pimpl)) {} \ + TypeName(const TypeName& other) = default; \ + TypeName(TypeName&& other) noexcept = default; \ + TypeName& operator=(const TypeName& other) = default; \ + TypeName& operator=(TypeName&& other) noexcept = default; \ + bool IsNull() const { return pimpl_ == nullptr; } \ + /* Access the impl pointer. Useful in implementation. */ \ + Impl* ImplPtr() { return pimpl_.get(); } \ + const Impl* ImplPtr() const { return pimpl_.get(); } \ + Impl* operator->() { return pimpl_.get(); } \ + const Impl* operator->() const { return pimpl_.get(); } \ + \ + private: \ + std::shared_ptr pimpl_ + +} // namespace xgrammar + +#endif // XGRAMMAR_OBJECT_H_ diff --git a/libs/xgrammar/include/xgrammar/tokenizer_info.h b/libs/xgrammar/include/xgrammar/tokenizer_info.h new file mode 100644 index 000000000..6a8e67444 --- /dev/null +++ b/libs/xgrammar/include/xgrammar/tokenizer_info.h @@ -0,0 +1,86 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/tokenizer_info.h + * \brief The header for the tokenizer info. + */ + +#ifndef XGRAMMAR_TOKENIZER_INFO_H_ +#define XGRAMMAR_TOKENIZER_INFO_H_ + +#include + +#include +#include +#include +#include +#include + +#include "xgrammar/exception.h" + +namespace xgrammar { + +enum class VocabType : int { + RAW = 0, + BYTE_FALLBACK = 1, + BYTE_LEVEL = 2, +}; + +class TokenizerInfo { + public: + TokenizerInfo( + const std::vector& encoded_vocab, + VocabType vocab_type = VocabType::RAW, + std::optional vocab_size = std::nullopt, + std::optional> stop_token_ids = std::nullopt, + bool add_prefix_space = false + ); + + VocabType GetVocabType() const; + bool GetAddPrefixSpace() const; + int GetVocabSize() const; + const std::vector& GetDecodedVocab() const; + const std::vector& GetStopTokenIds() const; + const std::vector& GetSpecialTokenIds() const; + const std::vector>& GetSortedDecodedVocab() const; + const std::vector& GetTrieSubtreeNodesRange() const; + std::string DumpMetadata() const; + + /*! + * \brief Create a tokenizer info from a vocabulary and metadata. + * \param encoded_vocab The encoded vocabulary. + * \param metadata The metadata. + * \return The tokenizer info. + */ + static TokenizerInfo FromVocabAndMetadata( + const std::vector& encoded_vocab, const std::string& metadata + ); + + /*! + * \brief Detect the metadata from a Hugging Face backend string. + * \param backend_str The Hugging Face backend string. + * \return The metadata. + */ + static std::string DetectMetadataFromHF(const std::string& backend_str); + + /*! + * \brief Return the serialized JSON string of the tokenizer info. + * \return The serialized JSON string. + */ + std::string SerializeJSON() const; + + /*! + * \brief Deserialize a tokenizer info from a JSON string. + * \param json_string The JSON string to deserialize. + * \return If the deserialization is successful, return the tokenizer info. Otherwise, return a + * runtime error with the error message. + */ + static std::variant DeserializeJSON( + const std::string& json_string + ); + + XGRAMMAR_DEFINE_PIMPL_METHODS(TokenizerInfo); +}; + +} // namespace xgrammar + +#endif // XGRAMMAR_TOKENIZER_INFO_H_ diff --git a/libs/xgrammar/include/xgrammar/xgrammar.h b/libs/xgrammar/include/xgrammar/xgrammar.h new file mode 100644 index 000000000..8513b3a9c --- /dev/null +++ b/libs/xgrammar/include/xgrammar/xgrammar.h @@ -0,0 +1,17 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/xgrammar.h + * \brief The header for the support of grammar-guided generation. + */ + +#ifndef XGRAMMAR_XGRAMMAR_H_ +#define XGRAMMAR_XGRAMMAR_H_ + +#include +#include +#include +#include +#include +#include + +#endif // XGRAMMAR_XGRAMMAR_H_ diff --git a/libs/xgrammar/ios/device/libxgrammar.a b/libs/xgrammar/ios/device/libxgrammar.a new file mode 100644 index 000000000..3017e901d Binary files /dev/null and b/libs/xgrammar/ios/device/libxgrammar.a differ diff --git a/libs/xgrammar/ios/simulator/libxgrammar.a b/libs/xgrammar/ios/simulator/libxgrammar.a new file mode 100644 index 000000000..c9f7a5257 Binary files /dev/null and b/libs/xgrammar/ios/simulator/libxgrammar.a differ diff --git a/libs/xgrammar/linux/aarch64/libxgrammar.a b/libs/xgrammar/linux/aarch64/libxgrammar.a new file mode 100644 index 000000000..eaaf62679 Binary files /dev/null and b/libs/xgrammar/linux/aarch64/libxgrammar.a differ diff --git a/libs/xgrammar/macos/libxgrammar.a b/libs/xgrammar/macos/libxgrammar.a new file mode 100644 index 000000000..3fa1d61dc Binary files /dev/null and b/libs/xgrammar/macos/libxgrammar.a differ diff --git a/python/src/cactus.py b/python/src/cactus.py index 5e9eba632..d338ba853 100644 --- a/python/src/cactus.py +++ b/python/src/cactus.py @@ -190,6 +190,44 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_matmul.restype = ctypes.c_int +_lib.cactus_graph_gather.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_gather.restype = ctypes.c_int +_lib.cactus_graph_embedding_from_tensor.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_embedding_from_tensor.restype = ctypes.c_int +_lib.cactus_graph_embedding_from_file.argtypes = [ + cactus_graph_t, ctypes.c_char_p, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_embedding_from_file.restype = ctypes.c_int +_lib.cactus_graph_mmap_embeddings.argtypes = [ + cactus_graph_t, ctypes.c_char_p, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_mmap_embeddings.restype = ctypes.c_int +_lib.cactus_graph_mmap_weights.argtypes = [ + cactus_graph_t, ctypes.c_char_p, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_mmap_weights.restype = ctypes.c_int +_lib.cactus_graph_bilinear_interpolation.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_bilinear_interpolation.restype = ctypes.c_int +_lib.cactus_graph_set_grouped_scales.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_void_p +] +_lib.cactus_graph_set_grouped_scales.restype = ctypes.c_int +_lib.cactus_graph_set_interleaved.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_bool, ctypes.c_size_t +] +_lib.cactus_graph_set_interleaved.restype = ctypes.c_int +_lib.cactus_graph_release_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] +_lib.cactus_graph_release_weight_pages.restype = ctypes.c_int +_lib.cactus_graph_prefetch_weight_pages.argtypes = [cactus_graph_t, cactus_node_t] +_lib.cactus_graph_prefetch_weight_pages.restype = ctypes.c_int +_lib.cactus_graph_release_all_weight_pages.argtypes = [cactus_graph_t] +_lib.cactus_graph_release_all_weight_pages.restype = ctypes.c_int _lib.cactus_graph_sum.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) @@ -257,10 +295,133 @@ class cactus_tensor_info_t(ctypes.Structure): cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_rms_norm.restype = ctypes.c_int +_lib.cactus_graph_topk.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_topk.restype = ctypes.c_int +_lib.cactus_graph_rope.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.c_size_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_rope.restype = ctypes.c_int +_lib.cactus_graph_rope_gptj.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_rope_gptj.restype = ctypes.c_int _lib.cactus_graph_softmax.argtypes = [ cactus_graph_t, cactus_node_t, ctypes.c_int32, ctypes.POINTER(cactus_node_t) ] _lib.cactus_graph_softmax.restype = ctypes.c_int +_lib.cactus_graph_attention.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.c_float, ctypes.c_bool, + ctypes.c_size_t, ctypes.c_size_t, ctypes.c_int32, ctypes.c_bool, cactus_node_t, ctypes.c_bool, + ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_attention.restype = ctypes.c_int +_lib.cactus_graph_rel_pos_bias.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_rel_pos_bias.restype = ctypes.c_int +_lib.cactus_graph_attention_int8_hybrid.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.c_float, ctypes.c_size_t, + ctypes.POINTER(ctypes.c_int8), ctypes.POINTER(ctypes.c_int8), + ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_float), + ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_attention_int8_hybrid.restype = ctypes.c_int +_lib.cactus_graph_conv1d_causal.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv1d_causal.restype = ctypes.c_int +_lib.cactus_graph_conv1d_k3.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv1d_k3.restype = ctypes.c_int +_lib.cactus_graph_conv1d_k7s3.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv1d_k7s3.restype = ctypes.c_int +_lib.cactus_graph_conv1d.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv1d.restype = ctypes.c_int +_lib.cactus_graph_conv1d_same_depthwise_k9.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv1d_same_depthwise_k9.restype = ctypes.c_int +_lib.cactus_graph_conv1d_pointwise.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv1d_pointwise.restype = ctypes.c_int +_lib.cactus_graph_conv2d_k3s2p1.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv2d_k3s2p1.restype = ctypes.c_int +_lib.cactus_graph_conv2d_depthwise_k3s2p1.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv2d_depthwise_k3s2p1.restype = ctypes.c_int +_lib.cactus_graph_conv2d_pointwise_1x1.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_bool, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_conv2d_pointwise_1x1.restype = ctypes.c_int +_lib.cactus_graph_lstm_cell.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_lstm_cell.restype = ctypes.c_int +_lib.cactus_graph_gated_deltanet_decode.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_gated_deltanet_decode.restype = ctypes.c_int +_lib.cactus_graph_gated_deltanet_prefill.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, cactus_node_t, ctypes.c_size_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_gated_deltanet_prefill.restype = ctypes.c_int +_lib.cactus_graph_stft.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_stft.restype = ctypes.c_int +_lib.cactus_graph_altup_predict.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_altup_predict.restype = ctypes.c_int +_lib.cactus_graph_altup_correct.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.POINTER(cactus_node_t), ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_altup_correct.restype = ctypes.c_int +_lib.cactus_graph_gaussian_topk.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_gaussian_topk.restype = ctypes.c_int +_lib.cactus_graph_moe_layer_gated.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, + ctypes.POINTER(cactus_node_t), ctypes.POINTER(cactus_node_t), ctypes.POINTER(cactus_node_t), + ctypes.c_size_t, ctypes.c_size_t, ctypes.c_bool, ctypes.c_float, ctypes.c_float, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_moe_layer_gated.restype = ctypes.c_int +_lib.cactus_graph_moe_layer_ungated.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t, + ctypes.POINTER(cactus_node_t), ctypes.POINTER(cactus_node_t), + ctypes.c_size_t, ctypes.c_size_t, ctypes.c_bool, ctypes.c_float, ctypes.c_float, ctypes.c_int32, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_moe_layer_ungated.restype = ctypes.c_int +_lib.cactus_graph_sample.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.c_float, ctypes.c_float, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_sample.restype = ctypes.c_int +_lib.cactus_graph_scatter_topk.argtypes = [ + cactus_graph_t, cactus_node_t, cactus_node_t, ctypes.c_size_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_scatter_topk.restype = ctypes.c_int +_lib.cactus_graph_persistent.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.POINTER(cactus_node_t) +] +_lib.cactus_graph_persistent.restype = ctypes.c_int +_lib.cactus_graph_is_populated.argtypes = [ + cactus_graph_t, cactus_node_t, ctypes.POINTER(ctypes.c_int32) +] +_lib.cactus_graph_is_populated.restype = ctypes.c_int +_lib.cactus_graph_invalidate_persistent.argtypes = [cactus_graph_t, cactus_node_t] +_lib.cactus_graph_invalidate_persistent.restype = ctypes.c_int _lib.cactus_graph_execute.argtypes = [cactus_graph_t] _lib.cactus_graph_execute.restype = ctypes.c_int diff --git a/python/src/cli.py b/python/src/cli.py index 4b23bbeba..6360bf3f3 100644 --- a/python/src/cli.py +++ b/python/src/cli.py @@ -398,12 +398,13 @@ def _load_raw_hf_state_dict(repo_id, cast_to_bf16=True): except ImportError: Lfm2VlForConditionalGeneration = None - is_vlm = 'vl' in model_id.lower() or 'vlm' in model_id.lower() - is_whisper = 'whisper' in model_id.lower() - is_parakeet = 'parakeet' in model_id.lower() - is_vad = 'silero-vad' in model_id.lower() - is_pyannote = 'segmentation-3.0' in model_id.lower() - is_wespeaker = 'wespeaker' in model_id.lower() + model_name = str(model_id) + is_vlm = 'vl' in model_name.lower() or 'vlm' in model_name.lower() + is_whisper = 'whisper' in model_name.lower() + is_parakeet = 'parakeet' in model_name.lower() + is_vad = 'silero-vad' in model_name.lower() + is_pyannote = 'segmentation-3.0' in model_name.lower() + is_wespeaker = 'wespeaker' in model_name.lower() try: if is_vlm: @@ -748,6 +749,11 @@ def cmd_build(args): cactus_dir = PROJECT_ROOT / "cactus" lib_path = cactus_dir / "build" / "libcactus.a" vendored_curl = PROJECT_ROOT / "libs" / "curl" / "macos" / "libcurl.a" + xgrammar_include = PROJECT_ROOT / "libs" / "xgrammar" / "include" + if platform.system() == "Darwin": + vendored_xgrammar = PROJECT_ROOT / "libs" / "xgrammar" / "macos" / "libxgrammar.a" + else: + vendored_xgrammar = PROJECT_ROOT / "libs" / "xgrammar" / "linux" / "aarch64" / "libxgrammar.a" print_color(YELLOW, "Building Cactus library...") build_script = cactus_dir / "build.sh" @@ -806,14 +812,20 @@ def cmd_build(args): print_color(RED, f"Error: vendored libcurl not found at {vendored_curl}") print("Build it first and place it in libs/curl/macos/libcurl.a") return 1 + if not vendored_xgrammar.exists(): + print_color(RED, f"Error: vendored xgrammar not found at {vendored_xgrammar}") + print("Build it first and place it in libs/xgrammar/macos/libxgrammar.a") + return 1 compiler = "clang++" cmd = [ compiler, "-std=c++20", "-O3", "-DACCELERATE_NEW_LAPACK", f"-I{PROJECT_ROOT}", + f"-I{PROJECT_ROOT / 'libs' / 'xgrammar' / 'include'}", *sdl2_flags, str(chat_cpp), str(lib_path), + str(vendored_xgrammar), "-o", "chat", str(vendored_curl), "-framework", "Accelerate", @@ -825,13 +837,19 @@ def cmd_build(args): *sdl2_link, ] else: + if not vendored_xgrammar.exists(): + print_color(RED, f"Error: vendored xgrammar not found at {vendored_xgrammar}") + print("Build it first and place it in libs/xgrammar/linux/aarch64/libxgrammar.a") + return 1 compiler = "g++" cmd = [ compiler, "-std=c++20", "-O3", f"-I{PROJECT_ROOT}", + f"-I{xgrammar_include}", *sdl2_flags, str(chat_cpp), str(lib_path), + str(vendored_xgrammar), "-o", "chat", "-lcurl", "-pthread", @@ -858,9 +876,11 @@ def cmd_build(args): compiler, "-std=c++20", "-O3", "-DACCELERATE_NEW_LAPACK", f"-I{PROJECT_ROOT}", + f"-I{PROJECT_ROOT / 'libs' / 'xgrammar' / 'include'}", *sdl2_flags, str(asr_cpp), str(lib_path), + str(vendored_xgrammar), "-o", "asr", str(vendored_curl), "-framework", "Accelerate", @@ -875,9 +895,11 @@ def cmd_build(args): cmd = [ compiler, "-std=c++20", "-O3", f"-I{PROJECT_ROOT}", + f"-I{xgrammar_include}", *sdl2_flags, str(asr_cpp), str(lib_path), + str(vendored_xgrammar), "-o", "asr", "-lcurl", "-pthread", @@ -1549,7 +1571,7 @@ class DownloadArgs: if getattr(args, 'exhaustive', False): cmd.append("--exhaustive") test_filter = args.only - for _test_name in ['llm', 'vlm', 'stt', 'embed', 'rag', 'graph', 'index', 'kernel', 'kv_cache', 'performance']: + for _test_name in ['llm', 'vlm', 'stt', 'embed', 'rag', 'graph', 'index', 'kernel', 'kv_cache', 'performance', 'grammar']: if getattr(args, _test_name, False): test_filter = _test_name break @@ -2173,7 +2195,7 @@ def create_parser(): test_parser.add_argument('--exhaustive', action='store_true', help='Run exhaustive golden tests for all model families and precisions') test_parser.add_argument('--only', help='(deprecated, use -- instead) Only run the specified test') - for _test_name in ['llm', 'vlm', 'stt', 'embed', 'rag', 'graph', 'index', 'kernel', 'kv_cache', 'performance']: + for _test_name in ['llm', 'vlm', 'stt', 'embed', 'rag', 'graph', 'index', 'kernel', 'kv_cache', 'performance', 'grammar']: test_parser.add_argument(f'--{_test_name}', action='store_true', help=f'Only run the {_test_name} tests') test_parser.add_argument('--enable-telemetry', action='store_true', diff --git a/python/src/graph.py b/python/src/graph.py index 97bba5559..b7eb3b03b 100644 --- a/python/src/graph.py +++ b/python/src/graph.py @@ -9,6 +9,14 @@ class Graph: FP16 = 1 FP32 = 2 INT4 = 3 + CPU = 0 + NPU = 1 + ACT_SILU = 0 + ACT_GELU = 1 + ACT_GELU_ERF = 2 + ACT_RELU = 3 + ACT_SIGMOID = 4 + ACT_TANH = 5 def __init__(self): self.h = _lib.cactus_graph_create() @@ -60,6 +68,22 @@ def set_input(self, tensor, data, dtype=None): if rc != 0: raise RuntimeError("graph_set_input failed") + def set_external_input(self, tensor, data_ptr, dtype=None): + if not isinstance(tensor, Tensor): + raise TypeError("tensor must be a Tensor") + if tensor.g is not self: + raise ValueError("tensor belongs to a different graph") + target_dtype = int(tensor.dtype if dtype is None else dtype) + ptr = ctypes.c_void_p(data_ptr if isinstance(data_ptr, int) else int(data_ptr)) + rc = _lib.cactus_graph_set_external_input( + self.h, + cactus_node_t(tensor.id), + ptr, + target_dtype, + ) + if rc != 0: + raise RuntimeError("graph_set_external_input failed") + def hard_reset(self): rc = _lib.cactus_graph_hard_reset(self.h) if rc != 0: @@ -73,6 +97,9 @@ def execute(self): def add(self, a, b): return self._binary("cactus_graph_add", a, b) + def add_clipped(self, a, b): + return self._binary("cactus_graph_add_clipped", a, b) + def subtract(self, a, b): return self._binary("cactus_graph_subtract", a, b) @@ -98,6 +125,61 @@ def pow(self, x, exponent): raise RuntimeError("graph_pow failed") return self._tensor_from_node(out.value) + def precision_cast(self, x, dtype): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_precision_cast(self.h, cactus_node_t(x.id), int(dtype), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_precision_cast failed") + return self._tensor_from_node(out.value) + + def quantize_activations(self, x): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_quantize_activations(self.h, cactus_node_t(x.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_quantize_activations failed") + return self._tensor_from_node(out.value) + + def _scalar(self, fn_name, x, value=None): + x = self._ensure_tensor(x) + out = cactus_node_t() + fn = getattr(_lib, fn_name) + if value is None: + rc = fn(self.h, cactus_node_t(x.id), ctypes.byref(out)) + else: + rc = fn(self.h, cactus_node_t(x.id), ctypes.c_float(float(value)), ctypes.byref(out)) + if rc != 0: + raise RuntimeError(f"{fn_name} failed") + return self._tensor_from_node(out.value) + + def scalar_add(self, x, value): + return self._scalar("cactus_graph_scalar_add", x, value) + + def scalar_subtract(self, x, value): + return self._scalar("cactus_graph_scalar_subtract", x, value) + + def scalar_multiply(self, x, value): + return self._scalar("cactus_graph_scalar_multiply", x, value) + + def scalar_divide(self, x, value): + return self._scalar("cactus_graph_scalar_divide", x, value) + + def scalar_exp(self, x): + return self._scalar("cactus_graph_scalar_exp", x) + + def scalar_sqrt(self, x): + return self._scalar("cactus_graph_scalar_sqrt", x) + + def scalar_cos(self, x): + return self._scalar("cactus_graph_scalar_cos", x) + + def scalar_sin(self, x): + return self._scalar("cactus_graph_scalar_sin", x) + + def scalar_log(self, x): + return self._scalar("cactus_graph_scalar_log", x) + def view(self, x, shape): x = self._ensure_tensor(x) shape = tuple(int(v) for v in shape) @@ -108,6 +190,16 @@ def view(self, x, shape): raise RuntimeError("graph_view failed") return self._tensor_from_node(out.value) + def reshape(self, x, shape): + x = self._ensure_tensor(x) + shape = tuple(int(v) for v in shape) + arr = (ctypes.c_size_t * len(shape))(*shape) + out = cactus_node_t() + rc = _lib.cactus_graph_reshape(self.h, cactus_node_t(x.id), arr, len(shape), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_reshape failed") + return self._tensor_from_node(out.value) + def flatten(self, x, start_dim=0, end_dim=-1): x = self._ensure_tensor(x) out = cactus_node_t() @@ -122,6 +214,171 @@ def flatten(self, x, start_dim=0, end_dim=-1): raise RuntimeError("graph_flatten failed") return self._tensor_from_node(out.value) + def slice(self, x, axis, start, length=0): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_slice( + self.h, + cactus_node_t(x.id), + ctypes.c_int32(int(axis)), + ctypes.c_size_t(int(start)), + ctypes.c_size_t(int(length)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_slice failed") + return self._tensor_from_node(out.value) + + def index(self, x, index_value, axis=0): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_index( + self.h, + cactus_node_t(x.id), + ctypes.c_size_t(int(index_value)), + ctypes.c_int32(int(axis)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_index failed") + return self._tensor_from_node(out.value) + + def transpose(self, x, backend=CPU): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_transpose( + self.h, + cactus_node_t(x.id), + ctypes.c_int32(int(backend)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_transpose failed") + return self._tensor_from_node(out.value) + + def permute(self, x, permutation, backend=CPU): + x = self._ensure_tensor(x) + permutation = tuple(int(v) for v in permutation) + arr = (ctypes.c_size_t * len(permutation))(*permutation) + out = cactus_node_t() + rc = _lib.cactus_graph_transpose_n( + self.h, + cactus_node_t(x.id), + arr, + len(permutation), + ctypes.c_int32(int(backend)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_transpose_n failed") + return self._tensor_from_node(out.value) + + def matmul(self, a, b, pretransposed_rhs=False, backend=CPU): + a = self._ensure_tensor(a) + b = self._ensure_tensor(b) + out = cactus_node_t() + rc = _lib.cactus_graph_matmul( + self.h, + cactus_node_t(a.id), + cactus_node_t(b.id), + ctypes.c_bool(bool(pretransposed_rhs)), + ctypes.c_int32(int(backend)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_matmul failed") + return self._tensor_from_node(out.value) + + def gather(self, tensor, indices): + tensor = self._ensure_tensor(tensor) + indices = self._ensure_tensor(indices) + out = cactus_node_t() + rc = _lib.cactus_graph_gather(self.h, cactus_node_t(tensor.id), cactus_node_t(indices.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_gather failed") + return self._tensor_from_node(out.value) + + def embedding_from_tensor(self, embedding_tensor, indices): + embedding_tensor = self._ensure_tensor(embedding_tensor) + indices = self._ensure_tensor(indices) + out = cactus_node_t() + rc = _lib.cactus_graph_embedding_from_tensor( + self.h, cactus_node_t(embedding_tensor.id), cactus_node_t(indices.id), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_embedding_from_tensor failed") + return self._tensor_from_node(out.value) + + def embedding_from_file(self, filename, indices): + indices = self._ensure_tensor(indices) + out = cactus_node_t() + rc = _lib.cactus_graph_embedding_from_file(self.h, str(filename).encode(), cactus_node_t(indices.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_embedding_from_file failed") + return self._tensor_from_node(out.value) + + def mmap_embeddings(self, filename): + out = cactus_node_t() + rc = _lib.cactus_graph_mmap_embeddings(self.h, str(filename).encode(), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_mmap_embeddings failed") + return self._tensor_from_node(out.value) + + def mmap_weights(self, filename): + out = cactus_node_t() + rc = _lib.cactus_graph_mmap_weights(self.h, str(filename).encode(), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_mmap_weights failed") + return self._tensor_from_node(out.value) + + def bilinear_interpolation(self, pos_embeds, dst_height, dst_width): + pos_embeds = self._ensure_tensor(pos_embeds) + out = cactus_node_t() + rc = _lib.cactus_graph_bilinear_interpolation( + self.h, cactus_node_t(pos_embeds.id), ctypes.c_size_t(int(dst_height)), ctypes.c_size_t(int(dst_width)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_bilinear_interpolation failed") + return self._tensor_from_node(out.value) + + def set_grouped_scales(self, tensor, group_size, num_groups, scales): + tensor = self._ensure_tensor(tensor) + arr = np.ascontiguousarray(scales, dtype=np.float16) + rc = _lib.cactus_graph_set_grouped_scales( + self.h, + cactus_node_t(tensor.id), + ctypes.c_size_t(int(group_size)), + ctypes.c_size_t(int(num_groups)), + arr.ctypes.data_as(ctypes.c_void_p), + ) + if rc != 0: + raise RuntimeError("graph_set_grouped_scales failed") + + def set_interleaved(self, tensor, interleaved=True, original_n=0): + tensor = self._ensure_tensor(tensor) + rc = _lib.cactus_graph_set_interleaved( + self.h, cactus_node_t(tensor.id), ctypes.c_bool(bool(interleaved)), ctypes.c_size_t(int(original_n)) + ) + if rc != 0: + raise RuntimeError("graph_set_interleaved failed") + + def release_weight_pages(self, tensor): + tensor = self._ensure_tensor(tensor) + rc = _lib.cactus_graph_release_weight_pages(self.h, cactus_node_t(tensor.id)) + if rc != 0: + raise RuntimeError("graph_release_weight_pages failed") + + def prefetch_weight_pages(self, tensor): + tensor = self._ensure_tensor(tensor) + rc = _lib.cactus_graph_prefetch_weight_pages(self.h, cactus_node_t(tensor.id)) + if rc != 0: + raise RuntimeError("graph_prefetch_weight_pages failed") + + def release_all_weight_pages(self): + rc = _lib.cactus_graph_release_all_weight_pages(self.h) + if rc != 0: + raise RuntimeError("graph_release_all_weight_pages failed") + def concat(self, a, b, axis=0): a = self._ensure_tensor(a) b = self._ensure_tensor(b) @@ -154,16 +411,17 @@ def cat(self, tensors, axis=0): raise RuntimeError("graph_cat failed") return self._tensor_from_node(out.value) - def group_norm(self, x, normalized_shape, eps=1e-5): + def groupnorm(self, x, weight, bias, num_groups, eps=1e-5): x = self._ensure_tensor(x) - normalized_shape = tuple(int(v) for v in normalized_shape) - shape_arr = (ctypes.c_size_t * len(normalized_shape))(*normalized_shape) + weight = self._ensure_tensor(weight) + bias = self._ensure_tensor(bias) out = cactus_node_t() - rc = _lib.cactus_graph_group_norm( + rc = _lib.cactus_graph_groupnorm( self.h, cactus_node_t(x.id), - shape_arr, - len(normalized_shape), + cactus_node_t(weight.id), + cactus_node_t(bias.id), + ctypes.c_size_t(int(num_groups)), ctypes.c_float(float(eps)), ctypes.byref(out), ) @@ -171,22 +429,126 @@ def group_norm(self, x, normalized_shape, eps=1e-5): raise RuntimeError("graph_group_norm failed") return self._tensor_from_node(out.value) - def layer_norm(self, x, normalized_shape, eps=1e-5): + def group_norm(self, x, weight, bias, num_groups, eps=1e-5): + return self.groupnorm(x, weight, bias, num_groups, eps=eps) + + def layernorm(self, x, weight, bias=None, eps=1e-5): x = self._ensure_tensor(x) - normalized_shape = tuple(int(v) for v in normalized_shape) - shape_arr = (ctypes.c_size_t * len(normalized_shape))(*normalized_shape) + weight = self._ensure_tensor(weight) + has_bias = bias is not None + bias_node = cactus_node_t(0) + if has_bias: + bias = self._ensure_tensor(bias) + bias_node = cactus_node_t(bias.id) out = cactus_node_t() - rc = _lib.cactus_graph_layer_norm( + rc = _lib.cactus_graph_layernorm( self.h, cactus_node_t(x.id), - shape_arr, - len(normalized_shape), + cactus_node_t(weight.id), + bias_node, ctypes.c_float(float(eps)), + ctypes.c_bool(has_bias), ctypes.byref(out), ) if rc != 0: raise RuntimeError("graph_layer_norm failed") return self._tensor_from_node(out.value) + + def layer_norm(self, x, weight, bias=None, eps=1e-5): + return self.layernorm(x, weight, bias=bias, eps=eps) + + def batchnorm(self, x, weight, bias, running_mean, running_var, axis=1, eps=1e-5): + x = self._ensure_tensor(x) + weight = self._ensure_tensor(weight) + bias = self._ensure_tensor(bias) + running_mean = self._ensure_tensor(running_mean) + running_var = self._ensure_tensor(running_var) + out = cactus_node_t() + rc = _lib.cactus_graph_batchnorm( + self.h, + cactus_node_t(x.id), + cactus_node_t(weight.id), + cactus_node_t(bias.id), + cactus_node_t(running_mean.id), + cactus_node_t(running_var.id), + ctypes.c_int32(int(axis)), + ctypes.c_float(float(eps)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_batchnorm failed") + return self._tensor_from_node(out.value) + + def batch_norm(self, x, weight, bias, running_mean, running_var, axis=1, eps=1e-5): + return self.batchnorm(x, weight, bias, running_mean, running_var, axis=axis, eps=eps) + + def rms_norm(self, x, weight, eps=1e-5): + x = self._ensure_tensor(x) + weight = self._ensure_tensor(weight) + out = cactus_node_t() + rc = _lib.cactus_graph_rms_norm( + self.h, + cactus_node_t(x.id), + cactus_node_t(weight.id), + ctypes.c_float(float(eps)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_rms_norm failed") + return self._tensor_from_node(out.value) + + def topk(self, x, k): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_topk(self.h, cactus_node_t(x.id), ctypes.c_size_t(int(k)), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_topk failed") + return self._tensor_from_node(out.value) + + def rope(self, x, theta, position_offset=0, backend=CPU): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_rope( + self.h, cactus_node_t(x.id), ctypes.c_float(float(theta)), ctypes.c_size_t(int(position_offset)), + ctypes.c_int32(int(backend)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_rope failed") + return self._tensor_from_node(out.value) + + def rope_gptj(self, x, theta, position_offset=0, rot_dim=0, backend=CPU): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_rope_gptj( + self.h, cactus_node_t(x.id), ctypes.c_float(float(theta)), ctypes.c_size_t(int(position_offset)), + ctypes.c_size_t(int(rot_dim)), ctypes.c_int32(int(backend)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_rope_gptj failed") + return self._tensor_from_node(out.value) + + def _reduce(self, fn_name, x, axis): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = getattr(_lib, fn_name)(self.h, cactus_node_t(x.id), ctypes.c_int32(int(axis)), ctypes.byref(out)) + if rc != 0: + raise RuntimeError(f"{fn_name} failed") + return self._tensor_from_node(out.value) + + def sum(self, x, axis): + return self._reduce("cactus_graph_sum", x, axis) + + def mean(self, x, axis): + return self._reduce("cactus_graph_mean", x, axis) + + def variance(self, x, axis): + return self._reduce("cactus_graph_variance", x, axis) + + def min(self, x, axis): + return self._reduce("cactus_graph_min", x, axis) + + def max(self, x, axis): + return self._reduce("cactus_graph_max", x, axis) def softmax(self, x, axis=-1): x = self._ensure_tensor(x) @@ -200,6 +562,78 @@ def softmax(self, x, axis=-1): if rc != 0: raise RuntimeError("graph_softmax failed") return self._tensor_from_node(out.value) + + def attention(self, query, key, value, scale, is_causal=True, position_offset=0, window_size=0, + backend=CPU, mask=None, additive_mask=False): + query = self._ensure_tensor(query) + key = self._ensure_tensor(key) + value = self._ensure_tensor(value) + mask_node = cactus_node_t(0) + use_mask = mask is not None + if use_mask: + mask_node = cactus_node_t(self._ensure_tensor(mask).id) + out = cactus_node_t() + rc = _lib.cactus_graph_attention( + self.h, + cactus_node_t(query.id), + cactus_node_t(key.id), + cactus_node_t(value.id), + ctypes.c_float(float(scale)), + ctypes.c_bool(bool(is_causal)), + ctypes.c_size_t(int(position_offset)), + ctypes.c_size_t(int(window_size)), + ctypes.c_int32(int(backend)), + ctypes.c_bool(use_mask), + mask_node, + ctypes.c_bool(bool(additive_mask)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_attention failed") + return self._tensor_from_node(out.value) + + def rel_pos_bias(self, query, relative_key, scale): + query = self._ensure_tensor(query) + relative_key = self._ensure_tensor(relative_key) + out = cactus_node_t() + rc = _lib.cactus_graph_rel_pos_bias( + self.h, cactus_node_t(query.id), cactus_node_t(relative_key.id), ctypes.c_float(float(scale)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_rel_pos_bias failed") + return self._tensor_from_node(out.value) + + def attention_int8_hybrid(self, query, key_new, value_new, scale, position_offset, + cached_keys, cached_values, k_scales, v_scales, + cache_len, num_kv_heads, head_dim, window_size=0): + query = self._ensure_tensor(query) + key_new = self._ensure_tensor(key_new) + value_new = self._ensure_tensor(value_new) + ck = np.ascontiguousarray(cached_keys, dtype=np.int8) + cv = np.ascontiguousarray(cached_values, dtype=np.int8) + ks = np.ascontiguousarray(k_scales, dtype=np.float32) + vs = np.ascontiguousarray(v_scales, dtype=np.float32) + out = cactus_node_t() + rc = _lib.cactus_graph_attention_int8_hybrid( + self.h, + cactus_node_t(query.id), + cactus_node_t(key_new.id), + cactus_node_t(value_new.id), + ctypes.c_float(float(scale)), + ctypes.c_size_t(int(position_offset)), + ck.ctypes.data_as(ctypes.POINTER(ctypes.c_int8)), + cv.ctypes.data_as(ctypes.POINTER(ctypes.c_int8)), + ks.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + vs.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + ctypes.c_size_t(int(cache_len)), + ctypes.c_size_t(int(num_kv_heads)), + ctypes.c_size_t(int(head_dim)), + ctypes.c_size_t(int(window_size)), + ctypes.byref(out), + ) + if rc != 0: + raise RuntimeError("graph_attention_int8_hybrid failed") + return self._tensor_from_node(out.value) def relu(self, x): x = self._ensure_tensor(x) @@ -209,6 +643,14 @@ def relu(self, x): raise RuntimeError("graph_relu failed") return self._tensor_from_node(out.value) + def silu(self, x): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_silu(self.h, cactus_node_t(x.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_silu failed") + return self._tensor_from_node(out.value) + def gelu(self, x): x = self._ensure_tensor(x) out = cactus_node_t() @@ -217,6 +659,14 @@ def gelu(self, x): raise RuntimeError("graph_gelu failed") return self._tensor_from_node(out.value) + def gelu_erf(self, x): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_gelu_erf(self.h, cactus_node_t(x.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_gelu_erf failed") + return self._tensor_from_node(out.value) + def sigmoid(self, x): x = self._ensure_tensor(x) out = cactus_node_t() @@ -233,6 +683,233 @@ def tanh(self, x): raise RuntimeError("graph_tanh failed") return self._tensor_from_node(out.value) + def glu(self, x, axis=-1): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_glu(self.h, cactus_node_t(x.id), ctypes.c_int32(int(axis)), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_glu failed") + return self._tensor_from_node(out.value) + + def conv1d_causal(self, x, weight, kernel_size, dilation): + x = self._ensure_tensor(x) + weight = self._ensure_tensor(weight) + out = cactus_node_t() + rc = _lib.cactus_graph_conv1d_causal( + self.h, cactus_node_t(x.id), cactus_node_t(weight.id), + ctypes.c_size_t(int(kernel_size)), ctypes.c_size_t(int(dilation)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_conv1d_causal failed") + return self._tensor_from_node(out.value) + + def conv1d_k3(self, x, weight, stride=1): + x = self._ensure_tensor(x) + weight = self._ensure_tensor(weight) + out = cactus_node_t() + rc = _lib.cactus_graph_conv1d_k3( + self.h, cactus_node_t(x.id), cactus_node_t(weight.id), ctypes.c_size_t(int(stride)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_conv1d_k3 failed") + return self._tensor_from_node(out.value) + + def conv1d_k7s3(self, x, weight, bias): + x = self._ensure_tensor(x) + weight = self._ensure_tensor(weight) + bias = self._ensure_tensor(bias) + out = cactus_node_t() + rc = _lib.cactus_graph_conv1d_k7s3( + self.h, cactus_node_t(x.id), cactus_node_t(weight.id), cactus_node_t(bias.id), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_conv1d_k7s3 failed") + return self._tensor_from_node(out.value) + + def conv1d(self, x, weight, bias=None, stride=1): + return self._conv_with_optional_bias("cactus_graph_conv1d", x, weight, bias, ctypes.c_size_t(int(stride))) + + def conv1d_same_depthwise_k9(self, x, weight, bias=None): + return self._conv_with_optional_bias("cactus_graph_conv1d_same_depthwise_k9", x, weight, bias) + + def conv1d_pointwise(self, x, weight, bias=None): + return self._conv_with_optional_bias("cactus_graph_conv1d_pointwise", x, weight, bias) + + def conv2d_k3s2p1(self, x, weight, bias=None): + return self._conv_with_optional_bias("cactus_graph_conv2d_k3s2p1", x, weight, bias) + + def conv2d_depthwise_k3s2p1(self, x, weight, bias=None): + return self._conv_with_optional_bias("cactus_graph_conv2d_depthwise_k3s2p1", x, weight, bias) + + def conv2d_pointwise_1x1(self, x, weight, bias=None): + return self._conv_with_optional_bias("cactus_graph_conv2d_pointwise_1x1", x, weight, bias) + + def _conv_with_optional_bias(self, fn_name, x, weight, bias=None, *extra): + x = self._ensure_tensor(x) + weight = self._ensure_tensor(weight) + has_bias = bias is not None + bias_node = cactus_node_t(0 if bias is None else self._ensure_tensor(bias).id) + out = cactus_node_t() + rc = getattr(_lib, fn_name)( + self.h, cactus_node_t(x.id), cactus_node_t(weight.id), ctypes.c_bool(has_bias), bias_node, *extra, ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError(f"{fn_name} failed") + return self._tensor_from_node(out.value) + + def lstm_cell(self, input, h_prev, c_prev, weight_ih, weight_hh, bias_ih, bias_hh): + tensors = [self._ensure_tensor(t) for t in (input, h_prev, c_prev, weight_ih, weight_hh, bias_ih, bias_hh)] + out = cactus_node_t() + rc = _lib.cactus_graph_lstm_cell(self.h, *(cactus_node_t(t.id) for t in tensors), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_lstm_cell failed") + return self._tensor_from_node(out.value) + + def gated_deltanet_decode(self, query, key, value, gate_log, beta, initial_state, scale): + tensors = [self._ensure_tensor(t) for t in (query, key, value, gate_log, beta, initial_state)] + out = cactus_node_t() + rc = _lib.cactus_graph_gated_deltanet_decode( + self.h, *(cactus_node_t(t.id) for t in tensors), ctypes.c_float(float(scale)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_gated_deltanet_decode failed") + return self._tensor_from_node(out.value) + + def gated_deltanet_prefill(self, query, key, value, gate_log, beta, initial_state, chunk_size, scale): + tensors = [self._ensure_tensor(t) for t in (query, key, value, gate_log, beta, initial_state)] + out = cactus_node_t() + rc = _lib.cactus_graph_gated_deltanet_prefill( + self.h, *(cactus_node_t(t.id) for t in tensors), ctypes.c_size_t(int(chunk_size)), ctypes.c_float(float(scale)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_gated_deltanet_prefill failed") + return self._tensor_from_node(out.value) + + def stft(self, x, weight, stride, num_fft_bins): + x = self._ensure_tensor(x) + weight = self._ensure_tensor(weight) + out = cactus_node_t() + rc = _lib.cactus_graph_stft( + self.h, cactus_node_t(x.id), cactus_node_t(weight.id), + ctypes.c_size_t(int(stride)), ctypes.c_size_t(int(num_fft_bins)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_stft failed") + return self._tensor_from_node(out.value) + + def altup_predict(self, coefs, streams): + coefs = self._ensure_tensor(coefs) + streams = [self._ensure_tensor(t) for t in streams] + ids = (cactus_node_t * len(streams))(*(cactus_node_t(t.id) for t in streams)) + out = cactus_node_t() + rc = _lib.cactus_graph_altup_predict(self.h, cactus_node_t(coefs.id), ids, ctypes.c_size_t(len(streams)), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_altup_predict failed") + return self._tensor_from_node(out.value) + + def altup_correct(self, coefs, innovation, predictions): + coefs = self._ensure_tensor(coefs) + innovation = self._ensure_tensor(innovation) + predictions = [self._ensure_tensor(t) for t in predictions] + ids = (cactus_node_t * len(predictions))(*(cactus_node_t(t.id) for t in predictions)) + out = cactus_node_t() + rc = _lib.cactus_graph_altup_correct( + self.h, cactus_node_t(coefs.id), cactus_node_t(innovation.id), ids, ctypes.c_size_t(len(predictions)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_altup_correct failed") + return self._tensor_from_node(out.value) + + def gaussian_topk(self, x, ppf): + x = self._ensure_tensor(x) + out = cactus_node_t() + rc = _lib.cactus_graph_gaussian_topk(self.h, cactus_node_t(x.id), ctypes.c_float(float(ppf)), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_gaussian_topk failed") + return self._tensor_from_node(out.value) + + def moe_layer_gated(self, hidden, routing_probs, topk_indices, w1_weights, w3_weights, w2_weights, + num_experts, num_experts_per_tok, normalize_routing=True, epsilon=1e-6, routed_scaling_factor=1.0): + hidden = self._ensure_tensor(hidden) + routing_probs = self._ensure_tensor(routing_probs) + topk_indices = self._ensure_tensor(topk_indices) + w1 = (cactus_node_t * len(w1_weights))(*(cactus_node_t(self._ensure_tensor(t).id) for t in w1_weights)) + w3 = (cactus_node_t * len(w3_weights))(*(cactus_node_t(self._ensure_tensor(t).id) for t in w3_weights)) + w2 = (cactus_node_t * len(w2_weights))(*(cactus_node_t(self._ensure_tensor(t).id) for t in w2_weights)) + out = cactus_node_t() + rc = _lib.cactus_graph_moe_layer_gated( + self.h, cactus_node_t(hidden.id), cactus_node_t(routing_probs.id), cactus_node_t(topk_indices.id), + w1, w3, w2, ctypes.c_size_t(int(num_experts)), ctypes.c_size_t(int(num_experts_per_tok)), + ctypes.c_bool(bool(normalize_routing)), ctypes.c_float(float(epsilon)), + ctypes.c_float(float(routed_scaling_factor)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_moe_layer_gated failed") + return self._tensor_from_node(out.value) + + def moe_layer_ungated(self, hidden, routing_probs, topk_indices, w1_weights, w2_weights, + num_experts, num_experts_per_tok, normalize_routing=True, epsilon=1e-6, + routed_scaling_factor=1.0, activation=ACT_GELU): + hidden = self._ensure_tensor(hidden) + routing_probs = self._ensure_tensor(routing_probs) + topk_indices = self._ensure_tensor(topk_indices) + w1 = (cactus_node_t * len(w1_weights))(*(cactus_node_t(self._ensure_tensor(t).id) for t in w1_weights)) + w2 = (cactus_node_t * len(w2_weights))(*(cactus_node_t(self._ensure_tensor(t).id) for t in w2_weights)) + out = cactus_node_t() + rc = _lib.cactus_graph_moe_layer_ungated( + self.h, cactus_node_t(hidden.id), cactus_node_t(routing_probs.id), cactus_node_t(topk_indices.id), + w1, w2, ctypes.c_size_t(int(num_experts)), ctypes.c_size_t(int(num_experts_per_tok)), + ctypes.c_bool(bool(normalize_routing)), ctypes.c_float(float(epsilon)), + ctypes.c_float(float(routed_scaling_factor)), ctypes.c_int32(int(activation)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_moe_layer_ungated failed") + return self._tensor_from_node(out.value) + + def sample(self, logits, temperature=1.0, top_p=1.0, top_k=0): + logits = self._ensure_tensor(logits) + out = cactus_node_t() + rc = _lib.cactus_graph_sample( + self.h, cactus_node_t(logits.id), ctypes.c_float(float(temperature)), + ctypes.c_float(float(top_p)), ctypes.c_size_t(int(top_k)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_sample failed") + return self._tensor_from_node(out.value) + + def scatter_topk(self, indices, values, num_classes): + indices = self._ensure_tensor(indices) + values = self._ensure_tensor(values) + out = cactus_node_t() + rc = _lib.cactus_graph_scatter_topk( + self.h, cactus_node_t(indices.id), cactus_node_t(values.id), ctypes.c_size_t(int(num_classes)), ctypes.byref(out) + ) + if rc != 0: + raise RuntimeError("graph_scatter_topk failed") + return self._tensor_from_node(out.value) + + def persistent(self, source_node): + source_node = self._ensure_tensor(source_node) + out = cactus_node_t() + rc = _lib.cactus_graph_persistent(self.h, cactus_node_t(source_node.id), ctypes.byref(out)) + if rc != 0: + raise RuntimeError("graph_persistent failed") + return self._tensor_from_node(out.value) + + def is_populated(self, persistent_node): + persistent_node = self._ensure_tensor(persistent_node) + out_is_populated = ctypes.c_int32() + rc = _lib.cactus_graph_is_populated(self.h, cactus_node_t(persistent_node.id), ctypes.byref(out_is_populated)) + if rc != 0: + raise RuntimeError("graph_is_populated failed") + return bool(out_is_populated.value) + + def invalidate_persistent(self, persistent_node): + persistent_node = self._ensure_tensor(persistent_node) + rc = _lib.cactus_graph_invalidate_persistent(self.h, cactus_node_t(persistent_node.id)) + if rc != 0: + raise RuntimeError("graph_invalidate_persistent failed") + def output_info(self, x): x = self._ensure_tensor(x) return self._get_output_info(x.id) @@ -314,6 +991,39 @@ def abs(self): def pow(self, exponent): return self.g.pow(self, exponent) + def precision_cast(self, dtype): + return self.g.precision_cast(self, dtype) + + def quantize_activations(self): + return self.g.quantize_activations(self) + + def scalar_add(self, value): + return self.g.scalar_add(self, value) + + def scalar_subtract(self, value): + return self.g.scalar_subtract(self, value) + + def scalar_multiply(self, value): + return self.g.scalar_multiply(self, value) + + def scalar_divide(self, value): + return self.g.scalar_divide(self, value) + + def scalar_exp(self): + return self.g.scalar_exp(self) + + def scalar_sqrt(self): + return self.g.scalar_sqrt(self) + + def scalar_cos(self): + return self.g.scalar_cos(self) + + def scalar_sin(self): + return self.g.scalar_sin(self) + + def scalar_log(self): + return self.g.scalar_log(self) + def relu(self): return self.g.relu(self) @@ -326,27 +1036,84 @@ def tanh(self): def gelu(self): return self.g.gelu(self) + def gelu_erf(self): + return self.g.gelu_erf(self) + + def silu(self): + return self.g.silu(self) + def view(self, shape): return self.g.view(self, shape) + def reshape(self, shape): + return self.g.reshape(self, shape) + def flatten(self, start_dim=0, end_dim=-1): return self.g.flatten(self, start_dim=start_dim, end_dim=end_dim) + def slice(self, axis, start, length=0): + return self.g.slice(self, axis, start, length=length) + + def index(self, index_value, axis=0): + return self.g.index(self, index_value, axis=axis) + + def transpose(self, backend=Graph.CPU): + return self.g.transpose(self, backend=backend) + + def permute(self, permutation, backend=Graph.CPU): + return self.g.permute(self, permutation, backend=backend) + def concat(self, other, axis=0): return self.g.concat(self, other, axis=axis) def cat(self, tensors, axis=0): return self.g.cat([self] + tensors, axis=axis) - def group_norm(self, normalized_shape, eps=1e-5): - return self.g.group_norm(self, normalized_shape, eps) + def groupnorm(self, weight, bias, num_groups, eps=1e-5): + return self.g.groupnorm(self, weight, bias, num_groups, eps=eps) + + def layernorm(self, weight, bias=None, eps=1e-5): + return self.g.layernorm(self, weight, bias=bias, eps=eps) + + def batchnorm(self, weight, bias, running_mean, running_var, axis=1, eps=1e-5): + return self.g.batchnorm(self, weight, bias, running_mean, running_var, axis=axis, eps=eps) - def layer_norm(self, normalized_shape, eps=1e-5): - return self.g.layer_norm(self, normalized_shape, eps) + def group_norm(self, weight, bias, num_groups, eps=1e-5): + return self.groupnorm(weight, bias, num_groups, eps=eps) + + def layer_norm(self, weight, bias=None, eps=1e-5): + return self.layernorm(weight, bias=bias, eps=eps) + + def batch_norm(self, weight, bias, running_mean, running_var, axis=1, eps=1e-5): + return self.batchnorm(weight, bias, running_mean, running_var, axis=axis, eps=eps) + + def rms_norm(self, weight, eps=1e-5): + return self.g.rms_norm(self, weight, eps=eps) def softmax(self, axis=-1): return self.g.softmax(self, axis) + def glu(self, axis=-1): + return self.g.glu(self, axis=axis) + + def matmul(self, other, pretransposed_rhs=False, backend=Graph.CPU): + return self.g.matmul(self, other, pretransposed_rhs=pretransposed_rhs, backend=backend) + + def sum(self, axis): + return self.g.sum(self, axis) + + def mean(self, axis): + return self.g.mean(self, axis) + + def variance(self, axis): + return self.g.variance(self, axis) + + def min(self, axis): + return self.g.min(self, axis) + + def max(self, axis): + return self.g.max(self, axis) + def numpy(self): info = cactus_tensor_info_t() rc = _lib.cactus_graph_get_output_info(self.g.h, cactus_node_t(self.id), ctypes.byref(info)) diff --git a/python/tests/test_graph.py b/python/tests/test_graph.py index c4c787cb7..e83b9e90d 100644 --- a/python/tests/test_graph.py +++ b/python/tests/test_graph.py @@ -1,4 +1,5 @@ import unittest + import numpy as np import sys import tempfile @@ -135,6 +136,44 @@ def test_flatten(self): expected = np.array([1, 2, 3, 4, 5, 6], dtype=np.float16) np.testing.assert_allclose(out, expected, atol=1e-2) + def test_reshape(self): + g = Graph() + a = g.input((2, 3)) + y = a.reshape((3, 2)) + + g.set_input(a, np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float16)) + g.execute() + + out = y.numpy() + expected = np.array([[1, 2], [3, 4], [5, 6]], dtype=np.float16) + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_transpose(self): + g = Graph() + a = g.input((2, 3)) + y = a.transpose() + + data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float16) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = data.T.astype(np.float16) + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_permute(self): + g = Graph() + a = g.input((2, 3, 4)) + y = a.permute((1, 0, 2)) + + data = np.arange(24, dtype=np.float16).reshape(2, 3, 4) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = np.transpose(data, (1, 0, 2)).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=1e-2) + def test_concat(self): g = Graph() a = g.input((2, 2)) @@ -177,6 +216,182 @@ def test_cat_2d_axis1(self): expected = np.array([[1, 2, 5, 6, 7], [3, 4, 8, 9, 10]], dtype=np.float16) np.testing.assert_allclose(out, expected, atol=1e-2) + def test_matmul(self): + g = Graph() + a = g.input((2, 3)) + b = g.input((3, 2)) + y = a.matmul(b) + + a_data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float16) + b_data = np.array([[7, 8], [9, 10], [11, 12]], dtype=np.float16) + g.set_input(a, a_data) + g.set_input(b, b_data) + g.execute() + + out = y.numpy() + expected = (a_data.astype(np.float32) @ b_data.astype(np.float32)).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_gather(self): + g = Graph() + embeddings = g.input((4, 2)) + indices = g.input((3,), dtype=Graph.FP32) + y = g.gather(embeddings, indices) + + embedding_data = np.array( + [[10, 11], [20, 21], [30, 31], [40, 41]], + dtype=np.float16, + ) + index_data = np.array([2, 0, 3], dtype=np.float32) + g.set_input(embeddings, embedding_data) + g.set_input(indices, index_data, dtype=Graph.FP32) + g.execute() + + out = y.numpy() + expected = embedding_data[[2, 0, 3]] + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_slice(self): + g = Graph() + a = g.input((4, 3)) + y = a.slice(axis=0, start=1, length=2) + + data = np.arange(12, dtype=np.float16).reshape(4, 3) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = data[1:3] + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_index(self): + g = Graph() + a = g.input((4, 3)) + y = a.index(2, axis=0) + + data = np.arange(12, dtype=np.float16).reshape(4, 3) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = data[2] + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_mean(self): + g = Graph() + a = g.input((2, 3)) + y = a.mean(axis=1) + + data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float16) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = data.astype(np.float32).mean(axis=1).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_variance(self): + g = Graph() + a = g.input((2, 3)) + y = a.variance(axis=1) + + data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float16) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = data.astype(np.float32).var(axis=1).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_min(self): + g = Graph() + a = g.input((2, 3)) + y = a.min(axis=1) + + data = np.array([[3, 1, 2], [6, 4, 5]], dtype=np.float16) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = data.min(axis=1).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_max(self): + g = Graph() + a = g.input((2, 3)) + y = a.max(axis=1) + + data = np.array([[3, 1, 2], [6, 4, 5]], dtype=np.float16) + g.set_input(a, data) + g.execute() + + out = y.numpy() + expected = data.max(axis=1).astype(np.float16) + np.testing.assert_allclose(out, expected, atol=1e-2) + + +class TestGraphNorms(unittest.TestCase): + + def test_layernorm_matches_alias(self): + g = Graph() + x = g.input((2, 3)) + weight = g.input((3,)) + bias = g.input((3,)) + y_primary = g.layernorm(x, weight, bias, eps=1e-5) + y_alias = g.layer_norm(x, weight, bias, eps=1e-5) + + x_data = np.array([[1.0, 3.0, 5.0], [2.0, 4.0, 8.0]], dtype=np.float16) + weight_data = np.array([1.0, 1.5, 0.5], dtype=np.float16) + bias_data = np.array([0.25, -0.5, 1.0], dtype=np.float16) + g.set_input(x, x_data) + g.set_input(weight, weight_data) + g.set_input(bias, bias_data) + g.execute() + + np.testing.assert_allclose(y_primary.numpy(), y_alias.numpy(), atol=1e-2) + + def test_groupnorm_matches_alias(self): + g = Graph() + x = g.input((1, 4)) + weight = g.input((4,)) + bias = g.input((4,)) + y_primary = g.groupnorm(x, weight, bias, num_groups=2, eps=1e-5) + y_alias = g.group_norm(x, weight, bias, num_groups=2, eps=1e-5) + + x_data = np.array([[1.0, 3.0, 2.0, 4.0]], dtype=np.float16) + weight_data = np.array([1.0, 1.0, 0.5, 0.5], dtype=np.float16) + bias_data = np.array([0.0, 0.5, 1.0, -1.0], dtype=np.float16) + g.set_input(x, x_data) + g.set_input(weight, weight_data) + g.set_input(bias, bias_data) + g.execute() + + np.testing.assert_allclose(y_primary.numpy(), y_alias.numpy(), atol=1e-2) + + def test_batchnorm_matches_alias(self): + g = Graph() + x = g.input((1, 3)) + weight = g.input((3,)) + bias = g.input((3,)) + running_mean = g.input((3,)) + running_var = g.input((3,)) + y_primary = g.batchnorm(x, weight, bias, running_mean, running_var, axis=1, eps=1e-5) + y_alias = g.batch_norm(x, weight, bias, running_mean, running_var, axis=1, eps=1e-5) + + x_data = np.array([[1.0, 2.0, 3.0]], dtype=np.float16) + weight_data = np.array([1.0, 1.5, 2.0], dtype=np.float16) + bias_data = np.array([0.0, -1.0, 0.5], dtype=np.float16) + mean_data = np.array([0.5, 1.0, 1.5], dtype=np.float16) + var_data = np.array([1.0, 4.0, 9.0], dtype=np.float16) + g.set_input(x, x_data) + g.set_input(weight, weight_data) + g.set_input(bias, bias_data) + g.set_input(running_mean, mean_data) + g.set_input(running_var, var_data) + g.execute() + + np.testing.assert_allclose(y_primary.numpy(), y_alias.numpy(), atol=1e-2) + class TestGraphActivations(unittest.TestCase): @@ -316,6 +531,148 @@ def test_save_load_roundtrip_softmax_cat(self): self.assertEqual(info["precision"], expected_info["precision"]) np.testing.assert_allclose(out, expected, atol=5e-2) + def test_save_load_roundtrip_matmul(self): + g = Graph() + + a = g.input((2, 3)) + b = g.input((3, 2)) + out_node = a.matmul(b) + + a_data = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float16) + b_data = np.array([[7.0, 8.0], [9.0, 10.0], [11.0, 12.0]], dtype=np.float16) + + g.set_input(a, a_data) + g.set_input(b, b_data) + g.execute() + expected = out_node.numpy() + expected_info = g.output_info(out_node) + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "matmul_graph.cg" + g.save(path) + + loaded = Graph.load(path) + loaded_a = self._rebind_tensor(loaded, a) + loaded_b = self._rebind_tensor(loaded, b) + loaded_out = self._rebind_tensor(loaded, out_node) + + loaded.set_input(loaded_a, a_data) + loaded.set_input(loaded_b, b_data) + loaded.execute() + + out = loaded_out.numpy() + info = loaded.output_info(loaded_out) + + self.assertEqual(info["shape"], expected_info["shape"]) + self.assertEqual(info["precision"], expected_info["precision"]) + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_save_load_roundtrip_permute(self): + g = Graph() + + a = g.input((2, 1, 2)) + out_node = a.permute((1, 0, 2)) + + data = np.array([[[58.0, 64.0]], [[139.0, 154.0]]], dtype=np.float16) + + g.set_input(a, data) + g.execute() + expected = out_node.numpy() + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "permute_graph.cg" + g.save(path) + + loaded = Graph.load(path) + loaded_a = self._rebind_tensor(loaded, a) + loaded_out = self._rebind_tensor(loaded, out_node) + + loaded.set_input(loaded_a, data) + loaded.execute() + + out = loaded_out.numpy() + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_save_load_roundtrip_gather(self): + g = Graph() + + embeddings = g.input((4, 2)) + indices = g.input((3,), dtype=Graph.FP32) + out_node = g.gather(embeddings, indices) + + embedding_data = np.array( + [[10, 11], [20, 21], [30, 31], [40, 41]], + dtype=np.float16, + ) + index_data = np.array([2, 0, 3], dtype=np.float32) + + g.set_input(embeddings, embedding_data) + g.set_input(indices, index_data, dtype=Graph.FP32) + g.execute() + expected = out_node.numpy() + expected_info = g.output_info(out_node) + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "gather_graph.cg" + g.save(path) + + loaded = Graph.load(path) + loaded_embeddings = self._rebind_tensor(loaded, embeddings) + loaded_indices = self._rebind_tensor(loaded, indices) + loaded_out = self._rebind_tensor(loaded, out_node) + + loaded.set_input(loaded_embeddings, embedding_data) + loaded.set_input(loaded_indices, index_data, dtype=Graph.FP32) + loaded.execute() + + out = loaded_out.numpy() + info = loaded.output_info(loaded_out) + + self.assertEqual(info["shape"], expected_info["shape"]) + self.assertEqual(info["precision"], expected_info["precision"]) + np.testing.assert_allclose(out, expected, atol=1e-2) + + def test_save_load_roundtrip_layernorm(self): + g = Graph() + + x = g.input((2, 3)) + weight = g.input((3,)) + bias = g.input((3,)) + out_node = g.layernorm(x, weight, bias, eps=1e-5) + + x_data = np.array([[1.0, 3.0, 5.0], [2.0, 4.0, 8.0]], dtype=np.float16) + weight_data = np.array([1.0, 1.5, 0.5], dtype=np.float16) + bias_data = np.array([0.25, -0.5, 1.0], dtype=np.float16) + + g.set_input(x, x_data) + g.set_input(weight, weight_data) + g.set_input(bias, bias_data) + g.execute() + expected = out_node.numpy() + expected_info = g.output_info(out_node) + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "layernorm_graph.cg" + g.save(path) + + loaded = Graph.load(path) + loaded_x = self._rebind_tensor(loaded, x) + loaded_weight = self._rebind_tensor(loaded, weight) + loaded_bias = self._rebind_tensor(loaded, bias) + loaded_out = self._rebind_tensor(loaded, out_node) + + loaded.set_input(loaded_x, x_data) + loaded.set_input(loaded_weight, weight_data) + loaded.set_input(loaded_bias, bias_data) + loaded.execute() + + out = loaded_out.numpy() + info = loaded.output_info(loaded_out) + + self.assertEqual(info["shape"], expected_info["shape"]) + self.assertEqual(info["precision"], expected_info["precision"]) + np.testing.assert_allclose(out, expected, atol=1e-2) + if __name__ == "__main__": unittest.main() diff --git a/rust/cactus-sys/build.rs b/rust/cactus-sys/build.rs index 59256f995..519d2bf54 100644 --- a/rust/cactus-sys/build.rs +++ b/rust/cactus-sys/build.rs @@ -84,6 +84,7 @@ fn link_platform_dependencies() { } println!("cargo:rustc-link-lib=curl"); println!("cargo:rustc-link-lib=c++"); + link_vendored_xgrammar(); } #[cfg(target_os = "linux")] @@ -92,11 +93,37 @@ fn link_platform_dependencies() { println!("cargo:rustc-link-lib=m"); println!("cargo:rustc-link-lib=pthread"); println!("cargo:rustc-link-lib=curl"); + link_vendored_xgrammar(); } #[cfg(not(any(target_os = "macos", target_os = "linux")))] fn link_platform_dependencies() {} +fn link_vendored_xgrammar() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let repo_root = manifest_dir.ancestors().nth(2).unwrap(); + let xgrammar_root = repo_root.join("libs/xgrammar"); + + let (xgrammar_dir, xgrammar_lib) = if cfg!(target_os = "linux") { + let dir = xgrammar_root.join("linux/aarch64"); + let lib = dir.join("libxgrammar.a"); + (dir, lib) + } else { + let dir = xgrammar_root.join("macos"); + let lib = dir.join("libxgrammar.a"); + (dir, lib) + }; + + assert!( + xgrammar_lib.exists(), + "Vendored xgrammar archive not found at {}", + xgrammar_lib.display() + ); + + println!("cargo:rustc-link-search=native={}", xgrammar_dir.display()); + println!("cargo:rustc-link-lib=static=xgrammar"); +} + fn generate_bindings(cactus_src: &Path) { let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 23b9ae586..f85d7a0f7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,18 @@ function(cactus_require_vendored_curl include_dir library_path platform_label) endif() endfunction() +function(cactus_require_vendored_xgrammar include_dir library_path platform_label) + if(NOT EXISTS "${include_dir}/xgrammar/xgrammar.h" OR NOT EXISTS "${library_path}") + message(WARNING "==============================================================================") + message(WARNING " Vendored xgrammar is required for ${platform_label} tests but is missing.") + message(WARNING " Expected header: ${include_dir}/xgrammar/xgrammar.h") + message(WARNING " Expected library: ${library_path}") + message(WARNING " Populate libs/xgrammar from your local xgrammar build before running cactus test.") + message(WARNING "==============================================================================") + message(FATAL_ERROR "Missing vendored xgrammar artifacts for ${platform_label} tests") + endif() +endfunction() + if(APPLE) set(CMAKE_OSX_ARCHITECTURES "arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64 -pthread -Wall -Wextra -pedantic -O3") @@ -23,7 +35,15 @@ if(APPLE) set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") cactus_require_vendored_curl("${CACTUS_CURL_INCLUDE_DIR}" "${CACTUS_CURL_LIBRARY_MACOS}" "macOS") + set(CACTUS_XGRAMMAR_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../libs/xgrammar" CACHE PATH "Path to vendored xgrammar artifacts") + set(CACTUS_XGRAMMAR_INCLUDE_DIR "${CACTUS_XGRAMMAR_ROOT}/include") + set(CACTUS_XGRAMMAR_LIBRARY_MACOS "${CACTUS_XGRAMMAR_ROOT}/macos/libxgrammar.a") + cactus_require_vendored_xgrammar("${CACTUS_XGRAMMAR_INCLUDE_DIR}" "${CACTUS_XGRAMMAR_LIBRARY_MACOS}" "macOS") else() + set(CACTUS_XGRAMMAR_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../libs/xgrammar" CACHE PATH "Path to vendored xgrammar artifacts") + set(CACTUS_XGRAMMAR_INCLUDE_DIR "${CACTUS_XGRAMMAR_ROOT}/include") + set(CACTUS_XGRAMMAR_LIBRARY_LINUX_AARCH64 "${CACTUS_XGRAMMAR_ROOT}/linux/aarch64/libxgrammar.a") + cactus_require_vendored_xgrammar("${CACTUS_XGRAMMAR_INCLUDE_DIR}" "${CACTUS_XGRAMMAR_LIBRARY_LINUX_AARCH64}" "Linux ARM64") find_package(CURL) endif() @@ -85,7 +105,18 @@ foreach(TEST_FILE ${TEST_SOURCES}) if(APPLE) target_link_libraries(${TEST_NAME} PRIVATE ${CACTUS_CURL_LIBRARY_MACOS}) target_include_directories(${TEST_NAME} PRIVATE ${CACTUS_CURL_INCLUDE_DIR}) + target_link_libraries(${TEST_NAME} PRIVATE ${CACTUS_XGRAMMAR_LIBRARY_MACOS}) + target_include_directories(${TEST_NAME} PRIVATE ${CACTUS_XGRAMMAR_INCLUDE_DIR}) message(STATUS "Test ${TEST_NAME} will link with libcurl") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_libraries(${TEST_NAME} PRIVATE ${CACTUS_XGRAMMAR_LIBRARY_LINUX_AARCH64}) + target_include_directories(${TEST_NAME} PRIVATE ${CACTUS_XGRAMMAR_INCLUDE_DIR}) + + if(CURL_FOUND) + target_link_libraries(${TEST_NAME} PRIVATE ${CURL_LIBRARIES}) + target_include_directories(${TEST_NAME} PRIVATE ${CURL_INCLUDE_DIRS}) + message(STATUS "Test ${TEST_NAME} will link with system libcurl") + endif() elseif(CURL_FOUND) target_link_libraries(${TEST_NAME} PRIVATE ${CURL_LIBRARIES}) target_include_directories(${TEST_NAME} PRIVATE ${CURL_INCLUDE_DIRS}) @@ -121,6 +152,16 @@ foreach(APP_NAME chat asr) if(APPLE) target_link_libraries(${APP_NAME} PRIVATE ${CACTUS_CURL_LIBRARY_MACOS}) target_include_directories(${APP_NAME} PRIVATE ${CACTUS_CURL_INCLUDE_DIR}) + target_link_libraries(${APP_NAME} PRIVATE ${CACTUS_XGRAMMAR_LIBRARY_MACOS}) + target_include_directories(${APP_NAME} PRIVATE ${CACTUS_XGRAMMAR_INCLUDE_DIR}) + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_libraries(${APP_NAME} PRIVATE ${CACTUS_XGRAMMAR_LIBRARY_LINUX_AARCH64}) + target_include_directories(${APP_NAME} PRIVATE ${CACTUS_XGRAMMAR_INCLUDE_DIR}) + + if(CURL_FOUND) + target_link_libraries(${APP_NAME} PRIVATE ${CURL_LIBRARIES}) + target_include_directories(${APP_NAME} PRIVATE ${CURL_INCLUDE_DIRS}) + endif() elseif(CURL_FOUND) target_link_libraries(${APP_NAME} PRIVATE ${CURL_LIBRARIES}) target_include_directories(${APP_NAME} PRIVATE ${CURL_INCLUDE_DIRS}) diff --git a/tests/run.sh b/tests/run.sh index 50ecf62f8..d6250ae52 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -90,7 +90,7 @@ while [[ $# -gt 0 ]]; do echo " --ios Run tests on iOS device or simulator" echo " --no-rebuild Skip building cactus library and tests" echo " --exhaustive Run exhaustive golden tests for all model families and precisions" - echo " --only Only run the specified test (llm, vlm, stt, embed, rag, graph, index, kernel, kv_cache, performance)" + echo " --only Only run the specified test (llm, vlm, stt, embed, rag, graph, index, kernel, kv_cache, performance, grammar)" echo " --help, -h Show this help message" exit 0 ;; diff --git a/tests/test_grammar.cpp b/tests/test_grammar.cpp new file mode 100644 index 000000000..bc92269f5 --- /dev/null +++ b/tests/test_grammar.cpp @@ -0,0 +1,530 @@ +#include "test_utils.h" + +#include "../cactus/ffi/cactus_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; + +namespace { + +using GrammarHandle = std::unique_ptr; +using GrammarVocabularyHandle = std::unique_ptr; +using GrammarEngineHandle = std::unique_ptr; +using GrammarMatcherHandle = std::unique_ptr; + +static std::unique_ptr create_test_tokenizer() { + const char* model_path = std::getenv("CACTUS_TEST_MODEL"); + if (!model_path) { + throw std::runtime_error("CACTUS_TEST_MODEL is not set"); + } + + auto tokenizer = create_tokenizer_from_model_dir(model_path); + if (!tokenizer) { + throw std::runtime_error("Failed to load tokenizer from test model files"); + } + return tokenizer; +} + +static cactus_grammar_t make_grammar_handle(cactus_grammar_t grammar) { + if (!grammar) { + throw std::runtime_error(cactus_get_last_error()); + } + return grammar; +} + +static cactus_grammar_t make_grammar_ebnf(const std::string& ebnf, const char* start_symbol = "root") { + return make_grammar_handle(cactus_grammar_init_ebnf(ebnf.c_str(), start_symbol)); +} + +static cactus_grammar_t make_grammar_empty() { + return make_grammar_handle(cactus_grammar_init_empty()); +} + +static cactus_grammar_t make_grammar_universal() { + return make_grammar_handle(cactus_grammar_init_universal()); +} + +static cactus_grammar_t make_grammar_json_schema(const std::string& json_schema) { + return make_grammar_handle( + cactus_grammar_init_json_schema( + json_schema.c_str(), + cactus_grammar_json_schema_default_options() + ) + ); +} + +static cactus_grammar_t make_grammar_regex(const std::string& regex) { + return make_grammar_handle(cactus_grammar_init_regex(regex.c_str())); +} + +static cactus_grammar_t make_grammar_structural_tag( + const std::string& structural_tag_json, + cactus_grammar_vocabulary_t vocabulary = nullptr +) { + return make_grammar_handle( + cactus_grammar_init_structural_tag(structural_tag_json.c_str(), vocabulary) + ); +} + +static cactus_grammar_vocabulary_t make_vocab_handle_from_tokenizer(const Tokenizer& tokenizer) { + return make_grammar_vocabulary_from_tokenizer(tokenizer); +} + +static cactus_grammar_matcher_t make_matcher(cactus_grammar_t grammar, cactus_grammar_engine_t engine) { + auto matcher = cactus_grammar_engine_compile_matcher(engine, grammar); + if (!matcher) { + throw std::runtime_error(cactus_get_last_error()); + } + return matcher; +} + +struct GrammarFixture { + std::unique_ptr tokenizer; + GrammarVocabularyHandle vocab; + GrammarEngineHandle engine; + size_t vocab_size = 0; + std::vector stop_token_ids; + + GrammarFixture() + : tokenizer(create_test_tokenizer()), + vocab(make_vocab_handle_from_tokenizer(*tokenizer), &cactus_grammar_vocabulary_destroy), + engine(cactus_grammar_engine_init(vocab.get()), &cactus_grammar_engine_destroy) { + if (!engine) { + throw std::runtime_error(cactus_get_last_error()); + } + vocab_size = cactus_grammar_vocabulary_get_size(vocab.get()); + if (vocab_size == 0) { + throw std::runtime_error(cactus_get_last_error()); + } + + stop_token_ids = tokenizer->get_grammar_vocabulary().stop_token_ids(); + std::vector actual_stop_token_ids(stop_token_ids.size()); + size_t out_token_count = 0; + if (cactus_grammar_vocabulary_get_stop_token_ids( + vocab.get(), + actual_stop_token_ids.data(), + actual_stop_token_ids.size(), + &out_token_count + ) != 0) { + throw std::runtime_error(cactus_get_last_error()); + } + actual_stop_token_ids.resize(out_token_count); + stop_token_ids = std::move(actual_stop_token_ids); + } +}; + +static bool accept_text(cactus_grammar_matcher_t matcher, const GrammarFixture& fixture, const std::string& text) { + const std::vector tokens = fixture.tokenizer->encode(text); + if (tokens.empty() && !text.empty()) return false; + for (uint32_t token : tokens) { + if (!cactus_grammar_matcher_accept(matcher, token)) return false; + } + return true; +} + +static bool accepts_complete_text(cactus_grammar_t grammar, const GrammarFixture& fixture, const std::string& text) { + auto matcher = GrammarMatcherHandle(make_matcher(grammar, fixture.engine.get()), &cactus_grammar_matcher_destroy); + if (!accept_text(matcher.get(), fixture, text)) return false; + return cactus_grammar_matcher_accept(matcher.get(), fixture.tokenizer->get_eos_token()); +} + +static bool rejects_text(cactus_grammar_t grammar, const GrammarFixture& fixture, const std::string& text) { + auto matcher = GrammarMatcherHandle(make_matcher(grammar, fixture.engine.get()), &cactus_grammar_matcher_destroy); + return !accept_text(matcher.get(), fixture, text); +} + +static bool rejects_eos_after_text(cactus_grammar_t grammar, const GrammarFixture& fixture, const std::string& text) { + auto matcher = GrammarMatcherHandle(make_matcher(grammar, fixture.engine.get()), &cactus_grammar_matcher_destroy); + if (!accept_text(matcher.get(), fixture, text)) return true; + return !cactus_grammar_matcher_accept(matcher.get(), fixture.tokenizer->get_eos_token()); +} + +static bool bitmask_allows_token(const std::vector& bitmask, uint32_t token_id) { + const size_t word_index = token_id / 32; + const uint32_t bit_index = token_id % 32; + if (word_index >= bitmask.size()) return false; + return ((bitmask[word_index]) & (uint32_t{1} << bit_index)) != 0; +} + +static size_t bitmask_size(size_t logits_buffer_size) { + return (logits_buffer_size + 31) / 32; +} + +static std::string grammar_ebnf(cactus_grammar_t grammar) { + std::vector buffer(4096, '\0'); + if (cactus_grammar_get_ebnf(grammar, buffer.data(), buffer.size()) != 0) { + throw std::runtime_error(cactus_get_last_error()); + } + return std::string(buffer.data()); +} + +static std::string tool_call_structural_tag_json() { + return R"({ + "type": "structural_tag", + "format": { + "type": "triggered_tags", + "triggers": ["<|tool▁calls▁begin|>"], + "tags": [ + { + "begin": "<|tool▁calls▁begin|>", + "end": "<|tool▁calls▁end|>", + "content": { + "type": "tags_with_separator", + "separator": "\n", + "tags": [ + { + "begin": "<|tool▁call▁begin|>function<|tool▁sep|>function_name_1\n```jsonc\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "city": {"type": "string"} + }, + "required": ["city"], + "additionalProperties": false + } + }, + "end": "\n```<|tool▁call▁end|>" + }, + { + "begin": "<|tool▁call▁begin|>function<|tool▁sep|>function_name_2\n```jsonc\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "count": {"type": "integer"} + }, + "required": ["count"], + "additionalProperties": false + } + }, + "end": "\n```<|tool▁call▁end|>" + } + ] + } + } + ], + "stop_after_first": true + } + })"; +} + +static bool test_vocab_accessors(const GrammarFixture& fixture) { + return fixture.vocab_size == fixture.tokenizer->get_vocab_size() + && !fixture.stop_token_ids.empty() + && fixture.stop_token_ids.front() == fixture.tokenizer->get_eos_token(); +} + +static bool test_empty_grammar_properties() { + auto empty = GrammarHandle(make_grammar_empty(), &cactus_grammar_destroy); + auto empty2 = GrammarHandle(make_grammar_empty(), &cactus_grammar_destroy); + auto simple = GrammarHandle(make_grammar_ebnf("root ::= \"hello\""), &cactus_grammar_destroy); + + if (!cactus_grammar_is_empty(empty.get()) || !cactus_grammar_is_empty(empty2.get()) || cactus_grammar_is_empty(simple.get())) { + return false; + } + + if (cactus_grammar_is_universal(empty.get()) || cactus_grammar_is_universal(empty2.get()) || cactus_grammar_is_universal(simple.get())) { + return false; + } + + cactus_grammar_t empty_union_inputs[] = {empty.get(), empty2.get()}; + auto empty_union = GrammarHandle(make_grammar_handle(cactus_grammar_union(empty_union_inputs, 2)), &cactus_grammar_destroy); + auto empty_concat = GrammarHandle(make_grammar_handle(cactus_grammar_concatenate(empty_union_inputs, 2)), &cactus_grammar_destroy); + + cactus_grammar_t union_with_simple_inputs[] = {empty.get(), simple.get()}; + auto union_with_simple = GrammarHandle(make_grammar_handle(cactus_grammar_union(union_with_simple_inputs, 2)), &cactus_grammar_destroy); + auto concat_with_simple = GrammarHandle(make_grammar_handle(cactus_grammar_concatenate(union_with_simple_inputs, 2)), &cactus_grammar_destroy); + + return cactus_grammar_is_empty(empty_union.get()) + && cactus_grammar_is_empty(empty_concat.get()) + && !cactus_grammar_is_empty(union_with_simple.get()) + && !cactus_grammar_is_empty(concat_with_simple.get()); +} + +static bool test_ebnf_string_export_matches_parenthesized_input_ebnf() { + const std::string source = "root ::= ((\"hello\") | (\"hi\"))\n"; + auto grammar = GrammarHandle(make_grammar_ebnf(source), &cactus_grammar_destroy); + return grammar_ebnf(grammar.get()) == source; +} + +static bool test_concat_accepts_expected_language(const GrammarFixture& fixture) { + auto left = GrammarHandle(make_grammar_ebnf("root ::= \"hello\""), &cactus_grammar_destroy); + auto right = GrammarHandle(make_grammar_ebnf("root ::= \" world\""), &cactus_grammar_destroy); + cactus_grammar_t handles[] = {left.get(), right.get()}; + auto combined = GrammarHandle(make_grammar_handle(cactus_grammar_concatenate(handles, 2)), &cactus_grammar_destroy); + + return accepts_complete_text(combined.get(), fixture, "hello world") + && rejects_text(combined.get(), fixture, "world hello") + && rejects_eos_after_text(combined.get(), fixture, "hello"); +} + +static bool test_union_accepts_expected_language(const GrammarFixture& fixture) { + auto left = GrammarHandle(make_grammar_ebnf("root ::= \"hello\""), &cactus_grammar_destroy); + auto right = GrammarHandle(make_grammar_ebnf("root ::= \"goodbye\""), &cactus_grammar_destroy); + cactus_grammar_t handles[] = {left.get(), right.get()}; + auto combined = GrammarHandle(make_grammar_handle(cactus_grammar_union(handles, 2)), &cactus_grammar_destroy); + + return accepts_complete_text(combined.get(), fixture, "hello") + && accepts_complete_text(combined.get(), fixture, "goodbye") + && rejects_text(combined.get(), fixture, "hello goodbye"); +} + +static bool test_three_way_concat(const GrammarFixture& fixture) { + auto alpha = GrammarHandle(make_grammar_ebnf("root ::= \"alpha\""), &cactus_grammar_destroy); + auto dash = GrammarHandle(make_grammar_ebnf("root ::= \"-\""), &cactus_grammar_destroy); + auto beta = GrammarHandle(make_grammar_ebnf("root ::= \"beta\""), &cactus_grammar_destroy); + cactus_grammar_t handles[] = {alpha.get(), dash.get(), beta.get()}; + auto combined = GrammarHandle(make_grammar_handle(cactus_grammar_concatenate(handles, 3)), &cactus_grammar_destroy); + + return accepts_complete_text(combined.get(), fixture, "alpha-beta") + && rejects_text(combined.get(), fixture, "alphabeta") + && rejects_text(combined.get(), fixture, "beta-alpha"); +} + +static bool test_three_way_union(const GrammarFixture& fixture) { + auto red = GrammarHandle(make_grammar_ebnf("root ::= \"red\""), &cactus_grammar_destroy); + auto green = GrammarHandle(make_grammar_ebnf("root ::= \"green\""), &cactus_grammar_destroy); + auto blue = GrammarHandle(make_grammar_ebnf("root ::= \"blue\""), &cactus_grammar_destroy); + cactus_grammar_t handles[] = {red.get(), green.get(), blue.get()}; + auto combined = GrammarHandle(make_grammar_handle(cactus_grammar_union(handles, 3)), &cactus_grammar_destroy); + + return accepts_complete_text(combined.get(), fixture, "red") + && accepts_complete_text(combined.get(), fixture, "green") + && accepts_complete_text(combined.get(), fixture, "blue") + && rejects_text(combined.get(), fixture, "orange"); +} + +static bool test_unordered_choice(const GrammarFixture& fixture) { + auto combined = GrammarHandle(make_grammar_ebnf("root ::= \"a\" | \"ab\""), &cactus_grammar_destroy); + + return accepts_complete_text(combined.get(), fixture, "a") + && accepts_complete_text(combined.get(), fixture, "ab") + && rejects_text(combined.get(), fixture, "ac"); +} + +static bool test_regex_and_json_schema_construction() { + auto regex = GrammarHandle(make_grammar_regex("(cat|dog)"), &cactus_grammar_destroy); + auto json_schema = GrammarHandle(make_grammar_json_schema(R"({"type":"object","properties":{"name":{"type":"string"}},"required":["name"]})"), &cactus_grammar_destroy); + return !cactus_grammar_is_empty(regex.get()) && !cactus_grammar_is_empty(json_schema.get()); +} + +static bool test_universal_grammar_accepts_anything(const GrammarFixture& fixture) { + auto grammar = GrammarHandle(make_grammar_universal(), &cactus_grammar_destroy); + return cactus_grammar_is_universal(grammar.get()) + && accepts_complete_text(grammar.get(), fixture, "") + && accepts_complete_text(grammar.get(), fixture, "blob says hello from cactus") + && accepts_complete_text(grammar.get(), fixture, "line one\nline two\nline three"); +} + +static bool test_union_with_universal_returns_universal() { + auto universal = GrammarHandle(make_grammar_universal(), &cactus_grammar_destroy); + auto specific = GrammarHandle(make_grammar_ebnf("root ::= \"hello\""), &cactus_grammar_destroy); + cactus_grammar_t handles[] = {specific.get(), universal.get()}; + auto combined = GrammarHandle(make_grammar_handle(cactus_grammar_union(handles, 2)), &cactus_grammar_destroy); + return cactus_grammar_is_universal(combined.get()); +} + +static bool test_structural_tag_accepts_and_rejects_expected_text(const GrammarFixture& fixture) { + auto grammar = GrammarHandle(make_grammar_structural_tag(tool_call_structural_tag_json(), fixture.vocab.get()), &cactus_grammar_destroy); + + return accepts_complete_text( + grammar.get(), + fixture, + "<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>function_name_1\n```jsonc\n{\"city\":\"Oakland\"}\n```<|tool▁call▁end|><|tool▁calls▁end|>" + ) + && accepts_complete_text( + grammar.get(), + fixture, + "<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>function_name_1\n```jsonc\n{\"city\":\"Oakland\"}\n```<|tool▁call▁end|>\n<|tool▁call▁begin|>function<|tool▁sep|>function_name_2\n```jsonc\n{\"count\":2}\n```<|tool▁call▁end|><|tool▁calls▁end|>" + ) + && rejects_text( + grammar.get(), + fixture, + "<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>function_name_1\n```jsonc\n{\"city\":\"Oakland\"}\n```<|tool▁calls▁end|>" + ) + && rejects_text( + grammar.get(), + fixture, + "<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>function_name_3\n```jsonc\n{\"city\":\"Oakland\"}\n```<|tool▁call▁end|><|tool▁calls▁end|>" + ) + && rejects_eos_after_text( + grammar.get(), + fixture, + "<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>function_name_1\n```jsonc\n{\"city\":1}\n```<|tool▁call▁end|><|tool▁calls▁end|>" + ) + && rejects_text( + grammar.get(), + fixture, + "<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>function_name_1\n```jsonc\n{\"city\":\"Oakland\"}\n```<|tool▁call▁end|><|tool▁call▁begin|>function<|tool▁sep|>function_name_2\n```jsonc\n{\"count\":2}\n```<|tool▁call▁end|><|tool▁calls▁end|>" + ); +} + +static bool test_regex_accepts_expected_text(const GrammarFixture& fixture) { + auto regex = GrammarHandle(make_grammar_regex("(cat|dog)"), &cactus_grammar_destroy); + + return accepts_complete_text(regex.get(), fixture, "cat") + && accepts_complete_text(regex.get(), fixture, "dog") + && rejects_text(regex.get(), fixture, "cow"); +} + +static bool test_json_schema_accepts_expected_text(const GrammarFixture& fixture) { + auto json_schema = GrammarHandle(make_grammar_json_schema( + R"({"type":"object","properties":{"name":{"type":"string"}},"required":["name"]})" + ), &cactus_grammar_destroy); + + return accepts_complete_text(json_schema.get(), fixture, R"({"name":"cactus"})") + && rejects_text(json_schema.get(), fixture, R"({"age":1})"); +} + +static bool test_grammar_matcher_reset_restores_initial_state(const GrammarFixture& fixture) { + auto grammar = GrammarHandle(make_grammar_ebnf("root ::= \"hello\""), &cactus_grammar_destroy); + auto matcher = GrammarMatcherHandle(make_matcher(grammar.get(), fixture.engine.get()), &cactus_grammar_matcher_destroy); + + if (!accept_text(matcher.get(), fixture, "hello")) return false; + if (!cactus_grammar_matcher_accept(matcher.get(), fixture.tokenizer->get_eos_token())) return false; + + cactus_grammar_matcher_reset(matcher.get()); + return rejects_text(grammar.get(), fixture, "goodbye") + && accept_text(matcher.get(), fixture, "hello") + && cactus_grammar_matcher_accept(matcher.get(), fixture.tokenizer->get_eos_token()); +} + +static bool test_grammar_matcher_rollback_restores_previous_state(const GrammarFixture& fixture) { + auto grammar = GrammarHandle(make_grammar_ebnf("root ::= \"hello\" | \"hi\""), &cactus_grammar_destroy); + auto matcher = GrammarMatcherHandle(make_matcher(grammar.get(), fixture.engine.get()), &cactus_grammar_matcher_destroy); + const std::vector hello_tokens = fixture.tokenizer->encode("hello"); + const std::vector hi_tokens = fixture.tokenizer->encode("hi"); + const uint32_t eos_token = fixture.tokenizer->get_eos_token(); + + for (uint32_t token : hello_tokens) { + if (!cactus_grammar_matcher_accept(matcher.get(), token)) return false; + } + if (!cactus_grammar_matcher_accept(matcher.get(), eos_token)) return false; + + cactus_grammar_matcher_rollback(matcher.get(), static_cast(hello_tokens.size()) + 1); + + for (uint32_t token : hi_tokens) { + if (!cactus_grammar_matcher_accept(matcher.get(), token)) return false; + } + return cactus_grammar_matcher_accept(matcher.get(), eos_token); +} + +static bool test_grammar_matcher_completion_state(const GrammarFixture& fixture) { + auto grammar = GrammarHandle(make_grammar_ebnf("root ::= \"hello\""), &cactus_grammar_destroy); + auto matcher = GrammarMatcherHandle(make_matcher(grammar.get(), fixture.engine.get()), &cactus_grammar_matcher_destroy); + const uint32_t eos_token = fixture.tokenizer->get_eos_token(); + + if (cactus_grammar_matcher_is_completed(matcher.get()) || cactus_grammar_matcher_is_terminated(matcher.get())) return false; + if (!accept_text(matcher.get(), fixture, "hello")) return false; + if (!cactus_grammar_matcher_is_completed(matcher.get()) || cactus_grammar_matcher_is_terminated(matcher.get())) return false; + if (!cactus_grammar_matcher_accept(matcher.get(), eos_token)) return false; + return cactus_grammar_matcher_is_completed(matcher.get()) && cactus_grammar_matcher_is_terminated(matcher.get()); +} + +static bool test_grammar_matcher_fork_preserves_accept_state(const GrammarFixture& fixture) { + auto grammar = GrammarHandle(make_grammar_ebnf("root ::= \"hello world\""), &cactus_grammar_destroy); + auto matcher = GrammarMatcherHandle(make_matcher(grammar.get(), fixture.engine.get()), &cactus_grammar_matcher_destroy); + const std::vector tokens = fixture.tokenizer->encode("hello world"); + const uint32_t eos_token = fixture.tokenizer->get_eos_token(); + + if (tokens.size() < 2) return false; + + const size_t fork_point = tokens.size() / 2; + + for (size_t i = 0; i < fork_point; ++i) { + if (!cactus_grammar_matcher_accept(matcher.get(), tokens[i])) return false; + } + + auto forked = GrammarMatcherHandle(cactus_grammar_matcher_fork(matcher.get()), &cactus_grammar_matcher_destroy); + if (!forked) return false; + + for (size_t i = fork_point; i < tokens.size(); ++i) { + if (!cactus_grammar_matcher_accept(matcher.get(), tokens[i])) return false; + } + if (!cactus_grammar_matcher_accept(matcher.get(), eos_token)) return false; + + for (size_t i = fork_point; i < tokens.size(); ++i) { + if (!cactus_grammar_matcher_accept(forked.get(), tokens[i])) return false; + } + return cactus_grammar_matcher_accept(forked.get(), eos_token); +} + +static bool test_grammar_matcher_next_bitmask_tracks_simple_grammar(const GrammarFixture& fixture) { + auto grammar = GrammarHandle(make_grammar_ebnf("root ::= \"hello\""), &cactus_grammar_destroy); + auto matcher = GrammarMatcherHandle(make_matcher(grammar.get(), fixture.engine.get()), &cactus_grammar_matcher_destroy); + const std::vector hello_tokens = fixture.tokenizer->encode("hello"); + const uint32_t eos_token = fixture.tokenizer->get_eos_token(); + + if (hello_tokens.empty()) return false; + + std::vector bitmask(bitmask_size(fixture.vocab_size)); + if (cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size) != 1) return false; + if (!bitmask_allows_token(bitmask, hello_tokens.front())) return false; + if (bitmask_allows_token(bitmask, eos_token)) return false; + + for (size_t i = 0; i < hello_tokens.size(); ++i) { + if (cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size) != 1) return false; + if (!bitmask_allows_token(bitmask, hello_tokens[i])) return false; + if (!cactus_grammar_matcher_accept(matcher.get(), hello_tokens[i])) return false; + } + + if (cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size) != 1) return false; + if (!bitmask_allows_token(bitmask, eos_token)) return false; + return !bitmask_allows_token(bitmask, hello_tokens.front()); +} + +static bool test_grammar_matcher_next_bitmask_zeroes_overallocated_tail(const GrammarFixture& fixture) { + auto grammar = GrammarHandle(make_grammar_ebnf("root ::= \"hello\""), &cactus_grammar_destroy); + auto matcher = GrammarMatcherHandle(make_matcher(grammar.get(), fixture.engine.get()), &cactus_grammar_matcher_destroy); + std::vector bitmask(bitmask_size(fixture.vocab_size + 1), -1); + + if (cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size + 1) != 1) return false; + return (bitmask.back() & 0xFF000000u) == 0; +} + +} // anonymous namespace + +int main() { + TestUtils::TestRunner runner("Grammar Tests"); + + try { + GrammarFixture fixture; + runner.run_test("vocab_accessors", test_vocab_accessors(fixture)); + runner.run_test("empty_properties", test_empty_grammar_properties()); + runner.run_test("ebnf_export_matches", test_ebnf_string_export_matches_parenthesized_input_ebnf()); + runner.run_test("regex_json_schema_init", test_regex_and_json_schema_construction()); + runner.run_test("concat_language", test_concat_accepts_expected_language(fixture)); + runner.run_test("union_language", test_union_accepts_expected_language(fixture)); + runner.run_test("three_way_concat", test_three_way_concat(fixture)); + runner.run_test("three_way_union", test_three_way_union(fixture)); + runner.run_test("unordred_choice", test_unordered_choice(fixture)); + runner.run_test("regex_language", test_regex_accepts_expected_text(fixture)); + runner.run_test("json_schema_language", test_json_schema_accepts_expected_text(fixture)); + runner.run_test("universal", test_universal_grammar_accepts_anything(fixture)); + runner.run_test("union_with_universal_returns_universal", test_union_with_universal_returns_universal()); + runner.run_test("structural_tag_language", test_structural_tag_accepts_and_rejects_expected_text(fixture)); + runner.run_test("grammar_matcher_reset", test_grammar_matcher_reset_restores_initial_state(fixture)); + runner.run_test("grammar_matcher_rollback", test_grammar_matcher_rollback_restores_previous_state(fixture)); + runner.run_test("grammar_matcher_completion_state", test_grammar_matcher_completion_state(fixture)); + runner.run_test("grammar_matcher_fork", test_grammar_matcher_fork_preserves_accept_state(fixture)); + runner.run_test("grammar_matcher_next_bitmask", test_grammar_matcher_next_bitmask_tracks_simple_grammar(fixture)); + runner.run_test("grammar_matcher_next_bitmask_overallocated_tail", test_grammar_matcher_next_bitmask_zeroes_overallocated_tail(fixture)); + } catch (const std::exception& e) { + std::cerr << "[✗] Grammar test setup failed: " << e.what() << "\n"; + } + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/tests/test_kernel.cpp b/tests/test_kernel.cpp index 8b05b3652..3eaf176bb 100644 --- a/tests/test_kernel.cpp +++ b/tests/test_kernel.cpp @@ -121,6 +121,117 @@ bool test_neon_reduction_correctness() { return true; } +bool test_neon_sum_axis_inner1_correctness() { + const size_t outer_size = 2; + const size_t axis_size = 9; + const size_t inner_size = 1; + + std::vector<__fp16> input = { + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, + 2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f, 14.0f, 16.0f, 18.0f + }; + + std::vector<__fp16> output(outer_size * inner_size); + std::vector<__fp16> expected = {45.0f, 90.0f}; + + cactus_sum_axis_f16(input.data(), output.data(), outer_size, axis_size, inner_size); + return TestUtils::compare_arrays(output.data(), expected.data(), expected.size(), 1e-2f); +} + +bool test_neon_mean_axis_inner1_correctness() { + const size_t outer_size = 2; + const size_t axis_size = 9; + const size_t inner_size = 1; + + std::vector<__fp16> input = { + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, + 2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f, 14.0f, 16.0f, 18.0f + }; + std::vector<__fp16> output(outer_size * inner_size); + std::vector<__fp16> expected = {5.0f, 10.0f}; + + cactus_mean_axis_f16(input.data(), output.data(), outer_size, axis_size, inner_size); + return TestUtils::compare_arrays(output.data(), expected.data(), expected.size(), 1e-2f); +} + +bool test_neon_variance_axis_inner1_correctness() { + const size_t outer_size = 2; + const size_t axis_size = 9; + const size_t inner_size = 1; + + std::vector<__fp16> input = { + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, + 2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f, 14.0f, 16.0f, 18.0f + }; + std::vector<__fp16> output(outer_size * inner_size); + std::vector<__fp16> expected = {6.6667f, 26.6667f}; + + cactus_variance_axis_f16(input.data(), output.data(), outer_size, axis_size, inner_size); + return TestUtils::compare_arrays(output.data(), expected.data(), expected.size(), 0.05f); +} + +bool test_neon_variance_axis_non_inner1_correctness() { + const size_t outer_size = 2; + const size_t axis_size = 9; + const size_t inner_size = 3; + + std::vector<__fp16> input(outer_size * axis_size * inner_size); + auto idx = [&](size_t outer, size_t axis, size_t inner) { + return outer * axis_size * inner_size + axis * inner_size + inner; + }; + + for (size_t a = 0; a < axis_size; ++a) { + input[idx(0, a, 0)] = static_cast<__fp16>(1.0f + static_cast(a)); + input[idx(0, a, 1)] = static_cast<__fp16>(2.0f + static_cast(a)); + input[idx(0, a, 2)] = static_cast<__fp16>(-4.0f + static_cast(a)); + + input[idx(1, a, 0)] = static_cast<__fp16>(2.0f + 2.0f * static_cast(a)); + input[idx(1, a, 1)] = static_cast<__fp16>(3.0f + 2.0f * static_cast(a)); + input[idx(1, a, 2)] = static_cast<__fp16>(-8.0f + 2.0f * static_cast(a)); + } + + std::vector<__fp16> output(outer_size * inner_size); + std::vector<__fp16> expected = { + 6.6667f, 6.6667f, 6.6667f, + 26.6667f, 26.6667f, 26.6667f + }; + + cactus_variance_axis_f16(input.data(), output.data(), outer_size, axis_size, inner_size); + return TestUtils::compare_arrays(output.data(), expected.data(), expected.size(), 0.05f); +} + +bool test_neon_min_axis_inner1_correctness() { + const size_t outer_size = 2; + const size_t axis_size = 9; + const size_t inner_size = 1; + + std::vector<__fp16> input = { + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, + 2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f, 14.0f, 16.0f, 18.0f + }; + std::vector<__fp16> output(outer_size * inner_size); + std::vector<__fp16> expected = {1.0f, 2.0f}; + + cactus_min_axis_f16(input.data(), output.data(), outer_size, axis_size, inner_size); + return TestUtils::compare_arrays(output.data(), expected.data(), expected.size(), 1e-2f); +} + +bool test_neon_max_axis_inner1_correctness() { + const size_t outer_size = 2; + const size_t axis_size = 9; + const size_t inner_size = 1; + + std::vector<__fp16> input = { + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, + 2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f, 14.0f, 16.0f, 18.0f + }; + std::vector<__fp16> output(outer_size * inner_size); + std::vector<__fp16> expected = {9.0f, 18.0f}; + + cactus_max_axis_f16(input.data(), output.data(), outer_size, axis_size, inner_size); + return TestUtils::compare_arrays(output.data(), expected.data(), expected.size(), 1e-2f); +} + bool test_neon_transpose_fp16_correctness() { const size_t M = 3, N = 4; std::vector<__fp16> input = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; @@ -499,6 +610,12 @@ int main() { runner.run_test("Kernel Multiply FP16 Correctness", test_neon_multiply_fp16_correctness()); runner.run_test("Kernel Scalar Ops FP16 Correctness", test_neon_scalar_operations_fp16_correctness()); runner.run_test("Kernel Reduction Correctness", test_neon_reduction_correctness()); + runner.run_test("Kernel Sum Axis Inner1 FP16 Correctness", test_neon_sum_axis_inner1_correctness()); + runner.run_test("Kernel Mean Axis Inner1 FP16 Correctness", test_neon_mean_axis_inner1_correctness()); + runner.run_test("Kernel Variance Axis Inner1 FP16 Correctness", test_neon_variance_axis_inner1_correctness()); + runner.run_test("Kernel Variance Axis Non-Inner1 FP16 Correctness", test_neon_variance_axis_non_inner1_correctness()); + runner.run_test("Kernel Min Axis Inner1 FP16 Correctness", test_neon_min_axis_inner1_correctness()); + runner.run_test("Kernel Max Axis Inner1 FP16 Correctness", test_neon_max_axis_inner1_correctness()); runner.run_test("Kernel Transpose FP16 Correctness", test_neon_transpose_fp16_correctness()); runner.run_test("Kernel Softmax Correctness", test_neon_softmax_correctness()); runner.run_test("Kernel RoPE Correctness", test_neon_rope_correctness()); diff --git a/tests/test_performance.cpp b/tests/test_performance.cpp index c78e85d45..c88cd9d02 100644 --- a/tests/test_performance.cpp +++ b/tests/test_performance.cpp @@ -429,6 +429,73 @@ void benchmark_reduction_ops(TestUtils::TestRunner& runner, const BenchmarkConfi } } +template +void benchmark_axis_reduction_inner1_ops(TestUtils::TestRunner& runner, const BenchmarkConfig& config) { + const std::vector>> ops = { + {"Axis Sum", [](CactusGraph& b, size_t a) { return b.sum(a, 1); }}, + {"Axis Mean", [](CactusGraph& b, size_t a) { return b.mean(a, 1); }}, + {"Axis Variance", [](CactusGraph& b, size_t a) { return b.variance(a, 1); }}, + {"Axis Min", [](CactusGraph& b, size_t a) { return b.min(a, 1); }}, + {"Axis Max", [](CactusGraph& b, size_t a) { return b.max(a, 1); }} + }; + + Precision precision = TestUtils::default_precision(); + + for (const auto& [op_name, op_func] : ops) { + for (size_t dim : config.dimensions) { + size_t total_elements = dim * dim; + + TestUtils::TestFixture fixture(op_name); + size_t input_a = fixture.create_input({dim, dim}, precision); + + std::vector data_a(total_elements); + setup_random_data(data_a); + fixture.set_input_data(input_a, data_a, precision); + + op_func(fixture.graph(), input_a); + + double time_ms = time_operation([&]() { + fixture.execute(); + }, config.iterations); + + double gb_per_sec = (total_elements * PrecisionTraits::size_of(precision)) / (time_ms * 1e6); + + std::ostringstream details; + details << std::fixed << std::setprecision(3) << time_ms << "ms, " + << std::setprecision(2) << gb_per_sec << " GB/s"; + runner.log_performance(op_name + " " + std::to_string(dim) + "x" + std::to_string(dim), + details.str()); + } + } + + for (size_t dim : config.dimensions) { + const size_t channels = 4; + size_t total_elements = dim * dim * channels; + + TestUtils::TestFixture fixture("Axis Variance Non-Inner1"); + size_t input_a = fixture.create_input({dim, dim, channels}, precision); + + std::vector data_a(total_elements); + setup_random_data(data_a); + fixture.set_input_data(input_a, data_a, precision); + + fixture.graph().variance(input_a, 1); + + double time_ms = time_operation([&]() { + fixture.execute(); + }, config.iterations); + + double gb_per_sec = (total_elements * PrecisionTraits::size_of(precision)) / (time_ms * 1e6); + + std::ostringstream details; + details << std::fixed << std::setprecision(3) << time_ms << "ms, " + << std::setprecision(2) << gb_per_sec << " GB/s"; + runner.log_performance("Axis Variance Non-Inner1 " + std::to_string(dim) + "x" + + std::to_string(dim) + "x" + std::to_string(channels), + details.str()); + } +} + template void benchmark_advanced_ops(TestUtils::TestRunner& runner, const BenchmarkConfig& config) { Precision precision = TestUtils::default_precision(); @@ -1027,6 +1094,12 @@ bool test_reduction_operations_performance(TestUtils::TestRunner& runner) { return true; } +bool test_axis_reduction_inner1_operations_performance(TestUtils::TestRunner& runner) { + BenchmarkConfig config; + benchmark_axis_reduction_inner1_ops<__fp16>(runner, config); + return true; +} + bool test_advanced_operations_performance(TestUtils::TestRunner& runner) { BenchmarkConfig config; benchmark_advanced_ops<__fp16>(runner, config); @@ -1195,6 +1268,7 @@ int main() { runner.run_test("Grouped INT8 MatMul", test_grouped_int8_matmul_performance(runner)); runner.run_test("Unary Operations", test_unary_operations_performance(runner)); runner.run_test("Reduction Operations", test_reduction_operations_performance(runner)); + runner.run_test("Axis Reduction Inner1 Operations", test_axis_reduction_inner1_operations_performance(runner)); runner.run_test("Advanced Operations", test_advanced_operations_performance(runner)); runner.run_test("Engine Operations", test_engine_operations_performance(runner)); runner.run_test("Gather Operations", test_gather_operations_performance(runner)); @@ -1204,4 +1278,4 @@ int main() { runner.print_summary(); return runner.all_passed() ? 0 : 1; -} \ No newline at end of file +}