diff --git a/.gitignore b/.gitignore index 99271876e..8053eb26c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,7 @@ tests/assets/test_wav_info.txt vision_activations.json gemma-4-eap-extras/ .githooks/ +*.cg + +**/*/compile_commands.json ruy diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index d04de1768..5b67a0558 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -12,6 +12,7 @@ set(CACTUS_MBEDTLS_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/mbedtls/${ANDROID_ABI}") if(NOT EXISTS "${CACTUS_MBEDTLS_ROOT}") set(CACTUS_MBEDTLS_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../libs/mbedtls/android/${ANDROID_ABI}") endif() +set(CACTUS_XGRAMMAR_LIBRARY "${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine/libs/xgrammar/android/${ANDROID_ABI}/libxgrammar.a") if(NOT TARGET cactus_engine) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus ${CMAKE_CURRENT_BINARY_DIR}/cactus) @@ -38,6 +39,10 @@ target_link_libraries(cactus_jni PRIVATE ${_DEPS}) find_library(LOG_LIB log) target_link_libraries(cactus_jni PRIVATE ${LOG_LIB} android) +if(EXISTS "${CACTUS_XGRAMMAR_LIBRARY}") + target_link_libraries(cactus_jni PRIVATE "${CACTUS_XGRAMMAR_LIBRARY}") +endif() + set(_MBEDTLS "${CACTUS_MBEDTLS_ROOT}/lib/libmbedtls.a") set(_MBEDX509 "${CACTUS_MBEDTLS_ROOT}/lib/libmbedx509.a") set(_MBEDCRYPTO "${CACTUS_MBEDTLS_ROOT}/lib/libmbedcrypto.a") diff --git a/apple/CMakeLists.txt b/apple/CMakeLists.txt index 510dfa64a..46625b291 100644 --- a/apple/CMakeLists.txt +++ b/apple/CMakeLists.txt @@ -5,6 +5,7 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0" CACHE STRING "") set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "13.0" CACHE STRING "") set(CMAKE_BUILD_TYPE Release CACHE STRING "") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CACTUS_XGRAMMAR_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine/libs/xgrammar") if(NOT TARGET cactus_engine) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus ${CMAKE_CURRENT_BINARY_DIR}/cactus) @@ -33,16 +34,22 @@ if(BUILD_SHARED_LIBS) if(CMAKE_SYSTEM_NAME STREQUAL "iOS") string(TOLOWER "${CMAKE_OSX_SYSROOT}" _sysroot_lower) if(_sysroot_lower MATCHES "simulator") + set(CACTUS_XGRAMMAR_LIBRARY "${CACTUS_XGRAMMAR_ROOT}/ios/simulator/libxgrammar.a") set(_CURL "${CACTUS_CURL_ROOT}/ios/simulator/libcurl.a") else() + set(CACTUS_XGRAMMAR_LIBRARY "${CACTUS_XGRAMMAR_ROOT}/ios/device/libxgrammar.a") set(_CURL "${CACTUS_CURL_ROOT}/ios/device/libcurl.a") endif() else() + set(CACTUS_XGRAMMAR_LIBRARY "${CACTUS_XGRAMMAR_ROOT}/macos/libxgrammar.a") set(_CURL "${CACTUS_CURL_ROOT}/macos/libcurl.a") endif() if(EXISTS "${_CURL}") target_link_libraries(cactus_framework PUBLIC "${_CURL}") endif() + if(EXISTS "${CACTUS_XGRAMMAR_LIBRARY}") + target_link_libraries(cactus_framework PUBLIC "${CACTUS_XGRAMMAR_LIBRARY}") + endif() set_target_properties(cactus_framework PROPERTIES FRAMEWORK TRUE diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index d8fe682b3..fd07c92da 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -10,6 +10,14 @@ endif() # --- libcurl --- set(CACTUS_CURL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/libs/curl" CACHE PATH "Path to vendored libcurl") +set(CACTUS_XGRAMMAR_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/libs/xgrammar" CACHE PATH "Path to vendored xgrammar") + +set(CACTUS_XGRAMMAR_INCLUDE_DIR "${CACTUS_XGRAMMAR_ROOT}/include") +if(APPLE) + set(CACTUS_XGRAMMAR_LIBRARY "${CACTUS_XGRAMMAR_ROOT}/macos/libxgrammar.a") +else() + set(CACTUS_XGRAMMAR_LIBRARY "${CACTUS_XGRAMMAR_ROOT}/linux/aarch64/libxgrammar.a") +endif() set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") if(APPLE) @@ -47,6 +55,8 @@ set(ENGINE_SOURCES src/bpe.cpp src/sp.cpp src/constraints.cpp + src/grammar.cpp + src/grammar_ffi.cpp src/model.cpp src/engine_image.cpp src/index.cpp @@ -80,12 +90,13 @@ set_target_properties(cactus_engine PROPERTIES target_include_directories(cactus_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CACTUS_XGRAMMAR_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/libs ${CMAKE_CURRENT_SOURCE_DIR}/../cactus-kernels/src ${CMAKE_CURRENT_SOURCE_DIR}/../cactus-kernels/libs ) - target_link_libraries(cactus_engine PUBLIC cactus_graph) +target_link_libraries(cactus_engine PRIVATE ${CACTUS_XGRAMMAR_LIBRARY}) # --- curl linking --- if(CACTUS_HAS_CURL) diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index dbb9d5138..ecff015aa 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -19,6 +19,11 @@ extern "C" { typedef void* cactus_model_t; typedef void* cactus_index_t; +typedef void* cactus_stream_transcribe_t; +typedef void* cactus_grammar_t; +typedef void* cactus_grammar_vocabulary_t; +typedef void* cactus_grammar_engine_t; +typedef void* cactus_grammar_matcher_t; typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); CACTUS_FFI_EXPORT cactus_model_t cactus_init( @@ -188,7 +193,7 @@ typedef uint64_t cactus_node_t; typedef struct { int32_t precision; size_t rank; - size_t shape[8]; + size_t shape[8]; size_t num_elements; size_t byte_size; } cactus_tensor_info_t; @@ -421,6 +426,78 @@ cactus_node_t node, void** out_ptr); CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, cactus_node_t node, cactus_tensor_info_t* out_info); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_ebnf(const char* ebnf, const char* start_symbol); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_json(); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_empty(); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_epsilon(); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_universal(); + +typedef struct cactus_grammar_json_schema_options_t { + bool any_whitespace; // negates indent and separators if true + int indent; + const char* separators[2]; + bool strict_mode; + int max_whitespace_count; // -1 means no limit +} cactus_grammar_json_schema_options_t; + +CACTUS_FFI_EXPORT cactus_grammar_json_schema_options_t cactus_grammar_json_schema_default_options(void); + +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_json_schema( + const char* json_schema, + cactus_grammar_json_schema_options_t options +); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_regex(const char* regex); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_init_structural_tag( + const char* structural_tag_json, + cactus_grammar_vocabulary_t vocabulary // optional +); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_union(cactus_grammar_t* grammars, size_t num_grammars); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_concatenate(cactus_grammar_t* grammars, size_t num_grammars); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_optional(cactus_grammar_t grammar); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_star(cactus_grammar_t grammar); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_repeat(cactus_grammar_t grammar, int count); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_repeat_range( + cactus_grammar_t grammar, + int min_count, + int max_count // -1 means unlimited +); +CACTUS_FFI_EXPORT int cactus_grammar_get_ebnf(cactus_grammar_t grammar, char* buffer, size_t buffer_size); +CACTUS_FFI_EXPORT int cactus_grammar_is_empty(cactus_grammar_t grammar); +CACTUS_FFI_EXPORT void cactus_grammar_destroy(cactus_grammar_t grammar); + +CACTUS_FFI_EXPORT cactus_grammar_engine_t cactus_grammar_engine_init(cactus_grammar_vocabulary_t vocabulary); +CACTUS_FFI_EXPORT void cactus_grammar_engine_destroy(cactus_grammar_engine_t engine); +CACTUS_FFI_EXPORT cactus_grammar_matcher_t cactus_grammar_engine_compile_matcher( + cactus_grammar_engine_t engine, + cactus_grammar_t grammar +); + +CACTUS_FFI_EXPORT cactus_grammar_vocabulary_t cactus_grammar_vocabulary_init(const char* model_path); +CACTUS_FFI_EXPORT cactus_grammar_vocabulary_t cactus_grammar_vocabulary_init_from_model(cactus_model_t model); +CACTUS_FFI_EXPORT int cactus_grammar_vocabulary_get_add_prefix_space(cactus_grammar_vocabulary_t vocabulary); +CACTUS_FFI_EXPORT size_t cactus_grammar_vocabulary_get_size(cactus_grammar_vocabulary_t vocabulary); +CACTUS_FFI_EXPORT int cactus_grammar_vocabulary_get_stop_token_ids( + cactus_grammar_vocabulary_t vocabulary, + uint32_t* buffer, + size_t buffer_size, + size_t* out_token_count +); +CACTUS_FFI_EXPORT void cactus_grammar_vocabulary_destroy(cactus_grammar_vocabulary_t vocabulary); + +CACTUS_FFI_EXPORT void cactus_grammar_matcher_destroy(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT void cactus_grammar_matcher_reset(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT void cactus_grammar_matcher_rollback(cactus_grammar_matcher_t matcher, int tokens); +CACTUS_FFI_EXPORT cactus_grammar_matcher_t cactus_grammar_matcher_fork(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT cactus_grammar_t cactus_grammar_matcher_get_grammar(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT int cactus_grammar_matcher_is_completed(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT int cactus_grammar_matcher_is_terminated(cactus_grammar_matcher_t matcher); +CACTUS_FFI_EXPORT int cactus_grammar_matcher_accept(cactus_grammar_matcher_t matcher, uint32_t token_id); +CACTUS_FFI_EXPORT int cactus_grammar_matcher_next_bitmask( + cactus_grammar_matcher_t matcher, + int32_t* bitmask, + size_t logits_buffer_size +); + #ifdef __cplusplus } #endif diff --git a/cactus-engine/libs/picojson.h b/cactus-engine/libs/picojson.h deleted file mode 100644 index 76742fe06..000000000 --- a/cactus-engine/libs/picojson.h +++ /dev/null @@ -1,1200 +0,0 @@ -/* - * Copyright 2009-2010 Cybozu Labs, Inc. - * Copyright 2011-2014 Kazuho Oku - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef picojson_h -#define picojson_h - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// for isnan/isinf -#if __cplusplus >= 201103L -#include -#else -extern "C" { -#ifdef _MSC_VER -#include -#elif defined(__INTEL_COMPILER) -#include -#else -#include -#endif -} -#endif - -#ifndef PICOJSON_USE_RVALUE_REFERENCE -#if (defined(__cpp_rvalue_references) && __cpp_rvalue_references >= 200610) || (defined(_MSC_VER) && _MSC_VER >= 1600) -#define PICOJSON_USE_RVALUE_REFERENCE 1 -#else -#define PICOJSON_USE_RVALUE_REFERENCE 0 -#endif -#endif // PICOJSON_USE_RVALUE_REFERENCE - -#ifndef PICOJSON_NOEXCEPT -#if PICOJSON_USE_RVALUE_REFERENCE -#define PICOJSON_NOEXCEPT noexcept -#else -#define PICOJSON_NOEXCEPT throw() -#endif -#endif - -// experimental support for int64_t (see README.mkdn for detail) -#ifdef PICOJSON_USE_INT64 -#define __STDC_FORMAT_MACROS -#include -#if __cplusplus >= 201103L -#include -#else -extern "C" { -#include -} -#endif -#endif - -// to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 -#ifndef PICOJSON_USE_LOCALE -#define PICOJSON_USE_LOCALE 1 -#endif -#if PICOJSON_USE_LOCALE -extern "C" { -#include -} -#endif - -#ifndef PICOJSON_ASSERT -#define PICOJSON_ASSERT(e) \ - do { \ - if (!(e)) \ - throw std::runtime_error(#e); \ - } while (0) -#endif - -#ifdef _MSC_VER -#define SNPRINTF _snprintf_s -#pragma warning(push) -#pragma warning(disable : 4244) // conversion from int to char -#pragma warning(disable : 4127) // conditional expression is constant -#pragma warning(disable : 4702) // unreachable code -#pragma warning(disable : 4706) // assignment within conditional expression -#else -#define SNPRINTF snprintf -#endif - -namespace picojson { - -enum { - null_type, - boolean_type, - number_type, - string_type, - array_type, - object_type -#ifdef PICOJSON_USE_INT64 - , - int64_type -#endif -}; - -enum { INDENT_WIDTH = 2, DEFAULT_MAX_DEPTHS = 100 }; - -struct null {}; - -class value { -public: - typedef std::vector array; - typedef std::map object; - union _storage { - bool boolean_; - double number_; -#ifdef PICOJSON_USE_INT64 - int64_t int64_; -#endif - std::string *string_; - array *array_; - object *object_; - }; - -protected: - int type_; - _storage u_; - -public: - value(); - value(int type, bool); - explicit value(bool b); -#ifdef PICOJSON_USE_INT64 - explicit value(int64_t i); -#endif - explicit value(double n); - explicit value(const std::string &s); - explicit value(const array &a); - explicit value(const object &o); -#if PICOJSON_USE_RVALUE_REFERENCE - explicit value(std::string &&s); - explicit value(array &&a); - explicit value(object &&o); -#endif - explicit value(const char *s); - value(const char *s, size_t len); - ~value(); - value(const value &x); - value &operator=(const value &x); -#if PICOJSON_USE_RVALUE_REFERENCE - value(value &&x) PICOJSON_NOEXCEPT; - value &operator=(value &&x) PICOJSON_NOEXCEPT; -#endif - void swap(value &x) PICOJSON_NOEXCEPT; - template bool is() const; - template const T &get() const; - template T &get(); - template void set(const T &); -#if PICOJSON_USE_RVALUE_REFERENCE - template void set(T &&); -#endif - bool evaluate_as_boolean() const; - const value &get(const size_t idx) const; - const value &get(const std::string &key) const; - value &get(const size_t idx); - value &get(const std::string &key); - - bool contains(const size_t idx) const; - bool contains(const std::string &key) const; - std::string to_str() const; - template void serialize(Iter os, bool prettify = false) const; - std::string serialize(bool prettify = false) const; - -private: - template value(const T *); // intentionally defined to block implicit conversion of pointer to bool - template static void _indent(Iter os, int indent); - template void _serialize(Iter os, int indent) const; - std::string _serialize(int indent) const; - void clear(); -}; - -typedef value::array array; -typedef value::object object; - -inline value::value() : type_(null_type), u_() { -} - -inline value::value(int type, bool) : type_(type), u_() { - switch (type) { -#define INIT(p, v) \ - case p##type: \ - u_.p = v; \ - break - INIT(boolean_, false); - INIT(number_, 0.0); -#ifdef PICOJSON_USE_INT64 - INIT(int64_, 0); -#endif - INIT(string_, new std::string()); - INIT(array_, new array()); - INIT(object_, new object()); -#undef INIT - default: - break; - } -} - -inline value::value(bool b) : type_(boolean_type), u_() { - u_.boolean_ = b; -} - -#ifdef PICOJSON_USE_INT64 -inline value::value(int64_t i) : type_(int64_type), u_() { - u_.int64_ = i; -} -#endif - -inline value::value(double n) : type_(number_type), u_() { - if ( -#ifdef _MSC_VER - !_finite(n) -#elif __cplusplus >= 201103L - std::isnan(n) || std::isinf(n) -#else - isnan(n) || isinf(n) -#endif - ) { - throw std::overflow_error(""); - } - u_.number_ = n; -} - -inline value::value(const std::string &s) : type_(string_type), u_() { - u_.string_ = new std::string(s); -} - -inline value::value(const array &a) : type_(array_type), u_() { - u_.array_ = new array(a); -} - -inline value::value(const object &o) : type_(object_type), u_() { - u_.object_ = new object(o); -} - -#if PICOJSON_USE_RVALUE_REFERENCE -inline value::value(std::string &&s) : type_(string_type), u_() { - u_.string_ = new std::string(std::move(s)); -} - -inline value::value(array &&a) : type_(array_type), u_() { - u_.array_ = new array(std::move(a)); -} - -inline value::value(object &&o) : type_(object_type), u_() { - u_.object_ = new object(std::move(o)); -} -#endif - -inline value::value(const char *s) : type_(string_type), u_() { - u_.string_ = new std::string(s); -} - -inline value::value(const char *s, size_t len) : type_(string_type), u_() { - u_.string_ = new std::string(s, len); -} - -inline void value::clear() { - switch (type_) { -#define DEINIT(p) \ - case p##type: \ - delete u_.p; \ - break - DEINIT(string_); - DEINIT(array_); - DEINIT(object_); -#undef DEINIT - default: - break; - } -} - -inline value::~value() { - clear(); -} - -inline value::value(const value &x) : type_(x.type_), u_() { - switch (type_) { -#define INIT(p, v) \ - case p##type: \ - u_.p = v; \ - break - INIT(string_, new std::string(*x.u_.string_)); - INIT(array_, new array(*x.u_.array_)); - INIT(object_, new object(*x.u_.object_)); -#undef INIT - default: - u_ = x.u_; - break; - } -} - -inline value &value::operator=(const value &x) { - if (this != &x) { - value t(x); - swap(t); - } - return *this; -} - -#if PICOJSON_USE_RVALUE_REFERENCE -inline value::value(value &&x) PICOJSON_NOEXCEPT : type_(null_type), u_() { - swap(x); -} -inline value &value::operator=(value &&x) PICOJSON_NOEXCEPT { - swap(x); - return *this; -} -#endif -inline void value::swap(value &x) PICOJSON_NOEXCEPT { - std::swap(type_, x.type_); - std::swap(u_, x.u_); -} - -#define IS(ctype, jtype) \ - template <> inline bool value::is() const { \ - return type_ == jtype##_type; \ - } -IS(null, null) -IS(bool, boolean) -#ifdef PICOJSON_USE_INT64 -IS(int64_t, int64) -#endif -IS(std::string, string) -IS(array, array) -IS(object, object) -#undef IS -template <> inline bool value::is() const { - return type_ == number_type -#ifdef PICOJSON_USE_INT64 - || type_ == int64_type -#endif - ; -} - -#define GET(ctype, var) \ - template <> inline const ctype &value::get() const { \ - PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ - return var; \ - } \ - template <> inline ctype &value::get() { \ - PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ - return var; \ - } -GET(bool, u_.boolean_) -GET(std::string, *u_.string_) -GET(array, *u_.array_) -GET(object, *u_.object_) -#ifdef PICOJSON_USE_INT64 -GET(double, - (type_ == int64_type && (const_cast(this)->type_ = number_type, (const_cast(this)->u_.number_ = u_.int64_)), - u_.number_)) -GET(int64_t, u_.int64_) -#else -GET(double, u_.number_) -#endif -#undef GET - -#define SET(ctype, jtype, setter) \ - template <> inline void value::set(const ctype &_val) { \ - clear(); \ - type_ = jtype##_type; \ - setter \ - } -SET(bool, boolean, u_.boolean_ = _val;) -SET(std::string, string, u_.string_ = new std::string(_val);) -SET(array, array, u_.array_ = new array(_val);) -SET(object, object, u_.object_ = new object(_val);) -SET(double, number, u_.number_ = _val;) -#ifdef PICOJSON_USE_INT64 -SET(int64_t, int64, u_.int64_ = _val;) -#endif -#undef SET - -#if PICOJSON_USE_RVALUE_REFERENCE -#define MOVESET(ctype, jtype, setter) \ - template <> inline void value::set(ctype && _val) { \ - clear(); \ - type_ = jtype##_type; \ - setter \ - } -MOVESET(std::string, string, u_.string_ = new std::string(std::move(_val));) -MOVESET(array, array, u_.array_ = new array(std::move(_val));) -MOVESET(object, object, u_.object_ = new object(std::move(_val));) -#undef MOVESET -#endif - -inline bool value::evaluate_as_boolean() const { - switch (type_) { - case null_type: - return false; - case boolean_type: - return u_.boolean_; - case number_type: - return u_.number_ != 0; -#ifdef PICOJSON_USE_INT64 - case int64_type: - return u_.int64_ != 0; -#endif - case string_type: - return !u_.string_->empty(); - default: - return true; - } -} - -inline const value &value::get(const size_t idx) const { - static value s_null; - PICOJSON_ASSERT(is()); - return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; -} - -inline value &value::get(const size_t idx) { - static value s_null; - PICOJSON_ASSERT(is()); - return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; -} - -inline const value &value::get(const std::string &key) const { - static value s_null; - PICOJSON_ASSERT(is()); - object::const_iterator i = u_.object_->find(key); - return i != u_.object_->end() ? i->second : s_null; -} - -inline value &value::get(const std::string &key) { - static value s_null; - PICOJSON_ASSERT(is()); - object::iterator i = u_.object_->find(key); - return i != u_.object_->end() ? i->second : s_null; -} - -inline bool value::contains(const size_t idx) const { - PICOJSON_ASSERT(is()); - return idx < u_.array_->size(); -} - -inline bool value::contains(const std::string &key) const { - PICOJSON_ASSERT(is()); - object::const_iterator i = u_.object_->find(key); - return i != u_.object_->end(); -} - -inline std::string value::to_str() const { - switch (type_) { - case null_type: - return "null"; - case boolean_type: - return u_.boolean_ ? "true" : "false"; -#ifdef PICOJSON_USE_INT64 - case int64_type: { - char buf[sizeof("-9223372036854775808")]; - SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); - return buf; - } -#endif - case number_type: { - char buf[256]; - double tmp; - SNPRINTF(buf, sizeof(buf), fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", u_.number_); -#if PICOJSON_USE_LOCALE - char *decimal_point = localeconv()->decimal_point; - if (strcmp(decimal_point, ".") != 0) { - size_t decimal_point_len = strlen(decimal_point); - for (char *p = buf; *p != '\0'; ++p) { - if (strncmp(p, decimal_point, decimal_point_len) == 0) { - return std::string(buf, p) + "." + (p + decimal_point_len); - } - } - } -#endif - return buf; - } - case string_type: - return *u_.string_; - case array_type: - return "array"; - case object_type: - return "object"; - default: - PICOJSON_ASSERT(0); -#ifdef _MSC_VER - __assume(0); -#endif - } - return std::string(); -} - -template void copy(const std::string &s, Iter oi) { - std::copy(s.begin(), s.end(), oi); -} - -template struct serialize_str_char { - Iter oi; - void operator()(char c) { - switch (c) { -#define MAP(val, sym) \ - case val: \ - copy(sym, oi); \ - break - MAP('"', "\\\""); - MAP('\\', "\\\\"); - MAP('/', "\\/"); - MAP('\b', "\\b"); - MAP('\f', "\\f"); - MAP('\n', "\\n"); - MAP('\r', "\\r"); - MAP('\t', "\\t"); -#undef MAP - default: - if (static_cast(c) < 0x20 || c == 0x7f) { - char buf[7]; - SNPRINTF(buf, sizeof(buf), "\\u%04x", c & 0xff); - copy(buf, buf + 6, oi); - } else { - *oi++ = c; - } - break; - } - } -}; - -template void serialize_str(const std::string &s, Iter oi) { - *oi++ = '"'; - serialize_str_char process_char = {oi}; - std::for_each(s.begin(), s.end(), process_char); - *oi++ = '"'; -} - -template void value::serialize(Iter oi, bool prettify) const { - return _serialize(oi, prettify ? 0 : -1); -} - -inline std::string value::serialize(bool prettify) const { - return _serialize(prettify ? 0 : -1); -} - -template void value::_indent(Iter oi, int indent) { - *oi++ = '\n'; - for (int i = 0; i < indent * INDENT_WIDTH; ++i) { - *oi++ = ' '; - } -} - -template void value::_serialize(Iter oi, int indent) const { - switch (type_) { - case string_type: - serialize_str(*u_.string_, oi); - break; - case array_type: { - *oi++ = '['; - if (indent != -1) { - ++indent; - } - for (array::const_iterator i = u_.array_->begin(); i != u_.array_->end(); ++i) { - if (i != u_.array_->begin()) { - *oi++ = ','; - } - if (indent != -1) { - _indent(oi, indent); - } - i->_serialize(oi, indent); - } - if (indent != -1) { - --indent; - if (!u_.array_->empty()) { - _indent(oi, indent); - } - } - *oi++ = ']'; - break; - } - case object_type: { - *oi++ = '{'; - if (indent != -1) { - ++indent; - } - for (object::const_iterator i = u_.object_->begin(); i != u_.object_->end(); ++i) { - if (i != u_.object_->begin()) { - *oi++ = ','; - } - if (indent != -1) { - _indent(oi, indent); - } - serialize_str(i->first, oi); - *oi++ = ':'; - if (indent != -1) { - *oi++ = ' '; - } - i->second._serialize(oi, indent); - } - if (indent != -1) { - --indent; - if (!u_.object_->empty()) { - _indent(oi, indent); - } - } - *oi++ = '}'; - break; - } - default: - copy(to_str(), oi); - break; - } - if (indent == 0) { - *oi++ = '\n'; - } -} - -inline std::string value::_serialize(int indent) const { - std::string s; - _serialize(std::back_inserter(s), indent); - return s; -} - -template class input { -protected: - Iter cur_, end_; - bool consumed_; - int line_; - -public: - input(const Iter &first, const Iter &last) : cur_(first), end_(last), consumed_(false), line_(1) { - } - int getc() { - if (consumed_) { - if (*cur_ == '\n') { - ++line_; - } - ++cur_; - } - if (cur_ == end_) { - consumed_ = false; - return -1; - } - consumed_ = true; - return *cur_ & 0xff; - } - void ungetc() { - consumed_ = false; - } - Iter cur() const { - if (consumed_) { - input *self = const_cast *>(this); - self->consumed_ = false; - ++self->cur_; - } - return cur_; - } - int line() const { - return line_; - } - void skip_ws() { - while (1) { - int ch = getc(); - if (!(ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { - ungetc(); - break; - } - } - } - bool expect(const int expected) { - skip_ws(); - if (getc() != expected) { - ungetc(); - return false; - } - return true; - } - bool match(const std::string &pattern) { - for (std::string::const_iterator pi(pattern.begin()); pi != pattern.end(); ++pi) { - if (getc() != *pi) { - ungetc(); - return false; - } - } - return true; - } -}; - -template inline int _parse_quadhex(input &in) { - int uni_ch = 0, hex; - for (int i = 0; i < 4; i++) { - if ((hex = in.getc()) == -1) { - return -1; - } - if ('0' <= hex && hex <= '9') { - hex -= '0'; - } else if ('A' <= hex && hex <= 'F') { - hex -= 'A' - 0xa; - } else if ('a' <= hex && hex <= 'f') { - hex -= 'a' - 0xa; - } else { - in.ungetc(); - return -1; - } - uni_ch = uni_ch * 16 + hex; - } - return uni_ch; -} - -template inline bool _parse_codepoint(String &out, input &in) { - int uni_ch; - if ((uni_ch = _parse_quadhex(in)) == -1) { - return false; - } - if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { - if (0xdc00 <= uni_ch) { - // a second 16-bit of a surrogate pair appeared - return false; - } - // first 16-bit of surrogate pair, get the next one - if (in.getc() != '\\' || in.getc() != 'u') { - in.ungetc(); - return false; - } - int second = _parse_quadhex(in); - if (!(0xdc00 <= second && second <= 0xdfff)) { - return false; - } - uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); - uni_ch += 0x10000; - } - if (uni_ch < 0x80) { - out.push_back(static_cast(uni_ch)); - } else { - if (uni_ch < 0x800) { - out.push_back(static_cast(0xc0 | (uni_ch >> 6))); - } else { - if (uni_ch < 0x10000) { - out.push_back(static_cast(0xe0 | (uni_ch >> 12))); - } else { - out.push_back(static_cast(0xf0 | (uni_ch >> 18))); - out.push_back(static_cast(0x80 | ((uni_ch >> 12) & 0x3f))); - } - out.push_back(static_cast(0x80 | ((uni_ch >> 6) & 0x3f))); - } - out.push_back(static_cast(0x80 | (uni_ch & 0x3f))); - } - return true; -} - -template inline bool _parse_string(String &out, input &in) { - while (1) { - int ch = in.getc(); - if (ch < ' ') { - in.ungetc(); - return false; - } else if (ch == '"') { - return true; - } else if (ch == '\\') { - if ((ch = in.getc()) == -1) { - return false; - } - switch (ch) { -#define MAP(sym, val) \ - case sym: \ - out.push_back(val); \ - break - MAP('"', '\"'); - MAP('\\', '\\'); - MAP('/', '/'); - MAP('b', '\b'); - MAP('f', '\f'); - MAP('n', '\n'); - MAP('r', '\r'); - MAP('t', '\t'); -#undef MAP - case 'u': - if (!_parse_codepoint(out, in)) { - return false; - } - break; - default: - return false; - } - } else { - out.push_back(static_cast(ch)); - } - } - return false; -} - -template inline bool _parse_array(Context &ctx, input &in) { - if (!ctx.parse_array_start()) { - return false; - } - size_t idx = 0; - if (in.expect(']')) { - return ctx.parse_array_stop(idx); - } - do { - if (!ctx.parse_array_item(in, idx)) { - return false; - } - idx++; - } while (in.expect(',')); - return in.expect(']') && ctx.parse_array_stop(idx); -} - -template inline bool _parse_object(Context &ctx, input &in) { - if (!ctx.parse_object_start()) { - return false; - } - if (in.expect('}')) { - return ctx.parse_object_stop(); - } - do { - std::string key; - if (!in.expect('"') || !_parse_string(key, in) || !in.expect(':')) { - return false; - } - if (!ctx.parse_object_item(in, key)) { - return false; - } - } while (in.expect(',')); - return in.expect('}') && ctx.parse_object_stop(); -} - -template inline std::string _parse_number(input &in) { - std::string num_str; - while (1) { - int ch = in.getc(); - if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' || ch == 'e' || ch == 'E') { - num_str.push_back(static_cast(ch)); - } else if (ch == '.') { -#if PICOJSON_USE_LOCALE - num_str += localeconv()->decimal_point; -#else - num_str.push_back('.'); -#endif - } else { - in.ungetc(); - break; - } - } - return num_str; -} - -template inline bool _parse(Context &ctx, input &in) { - in.skip_ws(); - int ch = in.getc(); - switch (ch) { -#define IS(ch, text, op) \ - case ch: \ - if (in.match(text) && op) { \ - return true; \ - } else { \ - return false; \ - } - IS('n', "ull", ctx.set_null()); - IS('f', "alse", ctx.set_bool(false)); - IS('t', "rue", ctx.set_bool(true)); -#undef IS - case '"': - return ctx.parse_string(in); - case '[': - return _parse_array(ctx, in); - case '{': - return _parse_object(ctx, in); - default: - if (('0' <= ch && ch <= '9') || ch == '-') { - double f; - char *endp; - in.ungetc(); - std::string num_str(_parse_number(in)); - if (num_str.empty()) { - return false; - } -#ifdef PICOJSON_USE_INT64 - { - errno = 0; - intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); - if (errno == 0 && std::numeric_limits::min() <= ival && ival <= std::numeric_limits::max() && - endp == num_str.c_str() + num_str.size()) { - ctx.set_int64(ival); - return true; - } - } -#endif - f = strtod(num_str.c_str(), &endp); - if (endp == num_str.c_str() + num_str.size()) { - ctx.set_number(f); - return true; - } - return false; - } - break; - } - in.ungetc(); - return false; -} - -class deny_parse_context { -public: - bool set_null() { - return false; - } - bool set_bool(bool) { - return false; - } -#ifdef PICOJSON_USE_INT64 - bool set_int64(int64_t) { - return false; - } -#endif - bool set_number(double) { - return false; - } - template bool parse_string(input &) { - return false; - } - bool parse_array_start() { - return false; - } - template bool parse_array_item(input &, size_t) { - return false; - } - bool parse_array_stop(size_t) { - return false; - } - bool parse_object_start() { - return false; - } - template bool parse_object_item(input &, const std::string &) { - return false; - } -}; - -class default_parse_context { -protected: - value *out_; - size_t depths_; - -public: - default_parse_context(value *out, size_t depths = DEFAULT_MAX_DEPTHS) : out_(out), depths_(depths) { - } - bool set_null() { - *out_ = value(); - return true; - } - bool set_bool(bool b) { - *out_ = value(b); - return true; - } -#ifdef PICOJSON_USE_INT64 - bool set_int64(int64_t i) { - *out_ = value(i); - return true; - } -#endif - bool set_number(double f) { - *out_ = value(f); - return true; - } - template bool parse_string(input &in) { - *out_ = value(string_type, false); - return _parse_string(out_->get(), in); - } - bool parse_array_start() { - if (depths_ == 0) - return false; - --depths_; - *out_ = value(array_type, false); - return true; - } - template bool parse_array_item(input &in, size_t) { - array &a = out_->get(); - a.push_back(value()); - default_parse_context ctx(&a.back(), depths_); - return _parse(ctx, in); - } - bool parse_array_stop(size_t) { - ++depths_; - return true; - } - bool parse_object_start() { - if (depths_ == 0) - return false; - *out_ = value(object_type, false); - return true; - } - template bool parse_object_item(input &in, const std::string &key) { - object &o = out_->get(); - default_parse_context ctx(&o[key], depths_); - return _parse(ctx, in); - } - bool parse_object_stop() { - ++depths_; - return true; - } - -private: - default_parse_context(const default_parse_context &); - default_parse_context &operator=(const default_parse_context &); -}; - -class null_parse_context { -protected: - size_t depths_; - -public: - struct dummy_str { - void push_back(int) { - } - }; - -public: - null_parse_context(size_t depths = DEFAULT_MAX_DEPTHS) : depths_(depths) { - } - bool set_null() { - return true; - } - bool set_bool(bool) { - return true; - } -#ifdef PICOJSON_USE_INT64 - bool set_int64(int64_t) { - return true; - } -#endif - bool set_number(double) { - return true; - } - template bool parse_string(input &in) { - dummy_str s; - return _parse_string(s, in); - } - bool parse_array_start() { - if (depths_ == 0) - return false; - --depths_; - return true; - } - template bool parse_array_item(input &in, size_t) { - return _parse(*this, in); - } - bool parse_array_stop(size_t) { - ++depths_; - return true; - } - bool parse_object_start() { - if (depths_ == 0) - return false; - --depths_; - return true; - } - template bool parse_object_item(input &in, const std::string &) { - ++depths_; - return _parse(*this, in); - } - bool parse_object_stop() { - return true; - } - -private: - null_parse_context(const null_parse_context &); - null_parse_context &operator=(const null_parse_context &); -}; - -// obsolete, use the version below -template inline std::string parse(value &out, Iter &pos, const Iter &last) { - std::string err; - pos = parse(out, pos, last, &err); - return err; -} - -template inline Iter _parse(Context &ctx, const Iter &first, const Iter &last, std::string *err) { - input in(first, last); - if (!_parse(ctx, in) && err != NULL) { - char buf[64]; - SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); - *err = buf; - while (1) { - int ch = in.getc(); - if (ch == -1 || ch == '\n') { - break; - } else if (ch >= ' ') { - err->push_back(static_cast(ch)); - } - } - } - return in.cur(); -} - -template inline Iter parse(value &out, const Iter &first, const Iter &last, std::string *err) { - default_parse_context ctx(&out); - return _parse(ctx, first, last, err); -} - -inline std::string parse(value &out, const std::string &s) { - std::string err; - parse(out, s.begin(), s.end(), &err); - return err; -} - -inline std::string parse(value &out, std::istream &is) { - std::string err; - parse(out, std::istreambuf_iterator(is.rdbuf()), std::istreambuf_iterator(), &err); - return err; -} - -template struct last_error_t { static std::string s; }; -template std::string last_error_t::s; - -inline void set_last_error(const std::string &s) { - last_error_t::s = s; -} - -inline const std::string &get_last_error() { - return last_error_t::s; -} - -inline bool operator==(const value &x, const value &y) { - if (x.is()) - return y.is(); -#define PICOJSON_CMP(type) \ - if (x.is()) \ - return y.is() && x.get() == y.get() - PICOJSON_CMP(bool); - PICOJSON_CMP(double); - PICOJSON_CMP(std::string); - PICOJSON_CMP(array); - PICOJSON_CMP(object); -#undef PICOJSON_CMP - PICOJSON_ASSERT(0); -#ifdef _MSC_VER - __assume(0); -#endif - return false; -} - -inline bool operator!=(const value &x, const value &y) { - return !(x == y); -} -} - -#if !PICOJSON_USE_RVALUE_REFERENCE -namespace std { -template <> inline void swap(picojson::value &x, picojson::value &y) { - x.swap(y); -} -} -#endif - -inline std::istream &operator>>(std::istream &is, picojson::value &x) { - picojson::set_last_error(std::string()); - const std::string err(picojson::parse(x, is)); - if (!err.empty()) { - picojson::set_last_error(err); - is.setstate(std::ios::failbit); - } - return is; -} - -inline std::ostream &operator<<(std::ostream &os, const picojson::value &x) { - x.serialize(std::ostream_iterator(os)); - return os; -} -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#endif diff --git a/cactus-engine/libs/xgrammar/android/arm64-v8a/libxgrammar.a b/cactus-engine/libs/xgrammar/android/arm64-v8a/libxgrammar.a new file mode 100644 index 000000000..f4a1eb97b Binary files /dev/null and b/cactus-engine/libs/xgrammar/android/arm64-v8a/libxgrammar.a differ diff --git a/cactus-engine/libs/xgrammar/include/dlpack/dlpack.h b/cactus-engine/libs/xgrammar/include/dlpack/dlpack.h new file mode 100644 index 000000000..bcb77949a --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/dlpack/dlpack.h @@ -0,0 +1,332 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file dlpack.h + * \brief The common header of DLPack. + */ +#ifndef DLPACK_DLPACK_H_ +#define DLPACK_DLPACK_H_ + +/** + * \brief Compatibility with C++ + */ +#ifdef __cplusplus +#define DLPACK_EXTERN_C extern "C" +#else +#define DLPACK_EXTERN_C +#endif + +/*! \brief The current major version of dlpack */ +#define DLPACK_MAJOR_VERSION 1 + +/*! \brief The current minor version of dlpack */ +#define DLPACK_MINOR_VERSION 0 + +/*! \brief DLPACK_DLL prefix for windows */ +#ifdef _WIN32 +#ifdef DLPACK_EXPORTS +#define DLPACK_DLL __declspec(dllexport) +#else +#define DLPACK_DLL __declspec(dllimport) +#endif +#else +#define DLPACK_DLL +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * \brief The DLPack version. + * + * A change in major version indicates that we have changed the + * data layout of the ABI - DLManagedTensorVersioned. + * + * A change in minor version indicates that we have added new + * code, such as a new device type, but the ABI is kept the same. + * + * If an obtained DLPack tensor has a major version that disagrees + * with the version number specified in this header file + * (i.e. major != DLPACK_MAJOR_VERSION), the consumer must call the deleter + * (and it is safe to do so). It is not safe to access any other fields + * as the memory layout will have changed. + * + * In the case of a minor version mismatch, the tensor can be safely used as + * long as the consumer knows how to interpret all fields. Minor version + * updates indicate the addition of enumeration values. + */ +typedef struct { + /*! \brief DLPack major version. */ + uint32_t major; + /*! \brief DLPack minor version. */ + uint32_t minor; +} DLPackVersion; + +/*! + * \brief The device type in DLDevice. + */ +#ifdef __cplusplus +typedef enum : int32_t { +#else +typedef enum { +#endif + /*! \brief CPU device */ + kDLCPU = 1, + /*! \brief CUDA GPU device */ + kDLCUDA = 2, + /*! + * \brief Pinned CUDA CPU memory by cudaMallocHost + */ + kDLCUDAHost = 3, + /*! \brief OpenCL devices. */ + kDLOpenCL = 4, + /*! \brief Vulkan buffer for next generation graphics. */ + kDLVulkan = 7, + /*! \brief Metal for Apple GPU. */ + kDLMetal = 8, + /*! \brief Verilog simulator buffer */ + kDLVPI = 9, + /*! \brief ROCm GPUs for AMD GPUs */ + kDLROCM = 10, + /*! + * \brief Pinned ROCm CPU memory allocated by hipMallocHost + */ + kDLROCMHost = 11, + /*! + * \brief Reserved extension device type, + * used for quickly test extension device + * The semantics can differ depending on the implementation. + */ + kDLExtDev = 12, + /*! + * \brief CUDA managed/unified memory allocated by cudaMallocManaged + */ + kDLCUDAManaged = 13, + /*! + * \brief Unified shared memory allocated on a oneAPI non-partititioned + * device. Call to oneAPI runtime is required to determine the device + * type, the USM allocation type and the sycl context it is bound to. + * + */ + kDLOneAPI = 14, + /*! \brief GPU support for next generation WebGPU standard. */ + kDLWebGPU = 15, + /*! \brief Qualcomm Hexagon DSP */ + kDLHexagon = 16, + /*! \brief Microsoft MAIA devices */ + kDLMAIA = 17, +} DLDeviceType; + +/*! + * \brief A Device for Tensor and operator. + */ +typedef struct { + /*! \brief The device type used in the device. */ + DLDeviceType device_type; + /*! + * \brief The device index. + * For vanilla CPU memory, pinned memory, or managed memory, this is set to 0. + */ + int32_t device_id; +} DLDevice; + +/*! + * \brief The type code options DLDataType. + */ +typedef enum { + /*! \brief signed integer */ + kDLInt = 0U, + /*! \brief unsigned integer */ + kDLUInt = 1U, + /*! \brief IEEE floating point */ + kDLFloat = 2U, + /*! + * \brief Opaque handle type, reserved for testing purposes. + * Frameworks need to agree on the handle data type for the exchange to be well-defined. + */ + kDLOpaqueHandle = 3U, + /*! \brief bfloat16 */ + kDLBfloat = 4U, + /*! + * \brief complex number + * (C/C++/Python layout: compact struct per complex number) + */ + kDLComplex = 5U, + /*! \brief boolean */ + kDLBool = 6U, +} DLDataTypeCode; + +/*! + * \brief The data type the tensor can hold. The data type is assumed to follow the + * native endian-ness. An explicit error message should be raised when attempting to + * export an array with non-native endianness + * + * Examples + * - float: type_code = 2, bits = 32, lanes = 1 + * - float4(vectorized 4 float): type_code = 2, bits = 32, lanes = 4 + * - int8: type_code = 0, bits = 8, lanes = 1 + * - std::complex: type_code = 5, bits = 64, lanes = 1 + * - bool: type_code = 6, bits = 8, lanes = 1 (as per common array library convention, the underlying storage size of bool is 8 bits) + */ +typedef struct { + /*! + * \brief Type code of base types. + * We keep it uint8_t instead of DLDataTypeCode for minimal memory + * footprint, but the value should be one of DLDataTypeCode enum values. + * */ + uint8_t code; + /*! + * \brief Number of bits, common choices are 8, 16, 32. + */ + uint8_t bits; + /*! \brief Number of lanes in the type, used for vector types. */ + uint16_t lanes; +} DLDataType; + +/*! + * \brief Plain C Tensor object, does not manage memory. + */ +typedef struct { + /*! + * \brief The data pointer points to the allocated data. This will be CUDA + * device pointer or cl_mem handle in OpenCL. It may be opaque on some device + * types. This pointer is always aligned to 256 bytes as in CUDA. The + * `byte_offset` field should be used to point to the beginning of the data. + * + * Note that as of Nov 2021, multiply libraries (CuPy, PyTorch, TensorFlow, + * TVM, perhaps others) do not adhere to this 256 byte aligment requirement + * on CPU/CUDA/ROCm, and always use `byte_offset=0`. This must be fixed + * (after which this note will be updated); at the moment it is recommended + * to not rely on the data pointer being correctly aligned. + * + * For given DLTensor, the size of memory required to store the contents of + * data is calculated as follows: + * + * \code{.c} + * static inline size_t GetDataSize(const DLTensor* t) { + * size_t size = 1; + * for (tvm_index_t i = 0; i < t->ndim; ++i) { + * size *= t->shape[i]; + * } + * size *= (t->dtype.bits * t->dtype.lanes + 7) / 8; + * return size; + * } + * \endcode + * + * Note that if the tensor is of size zero, then the data pointer should be + * set to `NULL`. + */ + void* data; + /*! \brief The device of the tensor */ + DLDevice device; + /*! \brief Number of dimensions */ + int32_t ndim; + /*! \brief The data type of the pointer*/ + DLDataType dtype; + /*! \brief The shape of the tensor */ + int64_t* shape; + /*! + * \brief strides of the tensor (in number of elements, not bytes) + * can be NULL, indicating tensor is compact and row-majored. + */ + int64_t* strides; + /*! \brief The offset in bytes to the beginning pointer to data */ + uint64_t byte_offset; +} DLTensor; + +/*! + * \brief C Tensor object, manage memory of DLTensor. This data structure is + * intended to facilitate the borrowing of DLTensor by another framework. It is + * not meant to transfer the tensor. When the borrowing framework doesn't need + * the tensor, it should call the deleter to notify the host that the resource + * is no longer needed. + * + * \note This data structure is used as Legacy DLManagedTensor + * in DLPack exchange and is deprecated after DLPack v0.8 + * Use DLManagedTensorVersioned instead. + * This data structure may get renamed or deleted in future versions. + * + * \sa DLManagedTensorVersioned + */ +typedef struct DLManagedTensor { + /*! \brief DLTensor which is being memory managed */ + DLTensor dl_tensor; + /*! \brief the context of the original host framework of DLManagedTensor in + * which DLManagedTensor is used in the framework. It can also be NULL. + */ + void * manager_ctx; + /*! + * \brief Destructor - this should be called + * to destruct the manager_ctx which backs the DLManagedTensor. It can be + * NULL if there is no way for the caller to provide a reasonable destructor. + * The destructor deletes the argument self as well. + */ + void (*deleter)(struct DLManagedTensor * self); +} DLManagedTensor; + +// bit masks used in in the DLManagedTensorVersioned + +/*! \brief bit mask to indicate that the tensor is read only. */ +#define DLPACK_FLAG_BITMASK_READ_ONLY (1UL << 0UL) + +/*! + * \brief bit mask to indicate that the tensor is a copy made by the producer. + * + * If set, the tensor is considered solely owned throughout its lifetime by the + * consumer, until the producer-provided deleter is invoked. + */ +#define DLPACK_FLAG_BITMASK_IS_COPIED (1UL << 1UL) + +/*! + * \brief A versioned and managed C Tensor object, manage memory of DLTensor. + * + * This data structure is intended to facilitate the borrowing of DLTensor by + * another framework. It is not meant to transfer the tensor. When the borrowing + * framework doesn't need the tensor, it should call the deleter to notify the + * host that the resource is no longer needed. + * + * \note This is the current standard DLPack exchange data structure. + */ +struct DLManagedTensorVersioned { + /*! + * \brief The API and ABI version of the current managed Tensor + */ + DLPackVersion version; + /*! + * \brief the context of the original host framework. + * + * Stores DLManagedTensorVersioned is used in the + * framework. It can also be NULL. + */ + void *manager_ctx; + /*! + * \brief Destructor. + * + * This should be called to destruct manager_ctx which holds the DLManagedTensorVersioned. + * It can be NULL if there is no way for the caller to provide a reasonable + * destructor. The destructor deletes the argument self as well. + */ + void (*deleter)(struct DLManagedTensorVersioned *self); + /*! + * \brief Additional bitmask flags information about the tensor. + * + * By default the flags should be set to 0. + * + * \note Future ABI changes should keep everything until this field + * stable, to ensure that deleter can be correctly called. + * + * \sa DLPACK_FLAG_BITMASK_READ_ONLY + * \sa DLPACK_FLAG_BITMASK_IS_COPIED + */ + uint64_t flags; + /*! \brief DLTensor which is being memory managed */ + DLTensor dl_tensor; +}; + +#ifdef __cplusplus +} // DLPACK_EXTERN_C +#endif +#endif // DLPACK_DLPACK_H_ diff --git a/cactus-engine/libs/xgrammar/include/picojson/picojson.h b/cactus-engine/libs/xgrammar/include/picojson/picojson.h new file mode 100644 index 000000000..5dcd86840 --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/picojson/picojson.h @@ -0,0 +1,1319 @@ +/* + * Copyright 2009-2010 Cybozu Labs, Inc. + * Copyright 2011-2014 Kazuho Oku + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#pragma once + +#ifndef PICOJSON_USE_INT64 +#define PICOJSON_USE_INT64 +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS 1 +#endif +#endif + +// If PICOJSON_USE_ORDERED_OBJECT is set, picojson uses object_with_ordered_keys, which maintains +// the insertion order of keys, i.e. the order of keys in the json string. +// This macro is set by default. +#ifndef PICOJSON_USE_ORDERED_OBJECT +#define PICOJSON_USE_ORDERED_OBJECT 1 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// for isnan/isinf +#if __cplusplus >= 201103L +#include +#else +extern "C" { +#ifdef _MSC_VER +#include +#elif defined(__INTEL_COMPILER) +#include +#else +#include +#endif +} +#endif + +#ifndef PICOJSON_USE_RVALUE_REFERENCE +#if (defined(__cpp_rvalue_references) && __cpp_rvalue_references >= 200610) || \ + (defined(_MSC_VER) && _MSC_VER >= 1600) +#define PICOJSON_USE_RVALUE_REFERENCE 1 +#else +#define PICOJSON_USE_RVALUE_REFERENCE 0 +#endif +#endif // PICOJSON_USE_RVALUE_REFERENCE + +#ifndef PICOJSON_NOEXCEPT +#if PICOJSON_USE_RVALUE_REFERENCE +#define PICOJSON_NOEXCEPT noexcept +#else +#define PICOJSON_NOEXCEPT throw() +#endif +#endif + +// experimental support for int64_t (see README.mkdn for detail) +#ifdef PICOJSON_USE_INT64 +#include +#include +#endif + +// to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 +#ifndef PICOJSON_USE_LOCALE +#define PICOJSON_USE_LOCALE 1 +#endif +#if PICOJSON_USE_LOCALE +extern "C" { +#include +} +#endif + +#ifndef PICOJSON_ASSERT +#ifndef PICOJSON_DISABLE_EXCEPTION +#define PICOJSON_ASSERT(e) \ + do { \ + if (!(e)) throw std::runtime_error(#e); \ + } while (0) +#else +#define PICOJSON_ASSERT(e) \ + do { \ + if (!(e)) std::abort(); \ + } while (0) +#endif // PICOJSON_DISABLE_EXCEPTION +#endif + +#ifdef _MSC_VER +#define SNPRINTF _snprintf_s +#pragma warning(push) +#pragma warning(disable : 4244) // conversion from int to char +#pragma warning(disable : 4127) // conditional expression is constant +#pragma warning(disable : 4702) // unreachable code +#else +#define SNPRINTF snprintf +#endif + +namespace picojson { + +enum { + null_type, + boolean_type, + number_type, + string_type, + array_type, + object_type +#ifdef PICOJSON_USE_INT64 + , + int64_type +#endif +}; + +enum { INDENT_WIDTH = 2 }; + +struct null {}; + +class object_with_ordered_keys; + +class value { + public: + typedef std::vector array; +#ifdef PICOJSON_USE_ORDERED_OBJECT + typedef object_with_ordered_keys object; +#else + typedef std::unordered_map object; +#endif + + union _storage { + bool boolean_; + double number_; +#ifdef PICOJSON_USE_INT64 + int64_t int64_; +#endif + std::string* string_; + array* array_; + object* object_; + }; + + protected: + int type_; + _storage u_; + + public: + value(); + value(int type, bool); + explicit value(bool b); +#ifdef PICOJSON_USE_INT64 + explicit value(int64_t i); +#endif + explicit value(double n); + explicit value(const std::string& s); + explicit value(const array& a); + explicit value(const object& o); +#if PICOJSON_USE_RVALUE_REFERENCE + explicit value(std::string&& s); + explicit value(array&& a); + explicit value(object&& o); +#endif + explicit value(const char* s); + value(const char* s, size_t len); + ~value(); + value(const value& x); + value& operator=(const value& x); +#if PICOJSON_USE_RVALUE_REFERENCE + value(value&& x) PICOJSON_NOEXCEPT; + value& operator=(value&& x) PICOJSON_NOEXCEPT; +#endif + void swap(value& x) PICOJSON_NOEXCEPT; + template + bool is() const; + template + const T& get() const; + template + T& get(); + template + void set(const T&); +#if PICOJSON_USE_RVALUE_REFERENCE + template + void set(T&&); +#endif + bool evaluate_as_boolean() const; + const value& get(const size_t idx) const; + const value& get(const std::string& key) const; + value& get(const size_t idx); + value& get(const std::string& key); + + bool contains(const size_t idx) const; + bool contains(const std::string& key) const; + std::string to_str() const; + template + void serialize(Iter os, bool prettify = false) const; + std::string serialize(bool prettify = false) const; + + private: + template + // NOLINTNEXTLINE(runtime/explicit) + value(const T*); // intentionally defined to block implicit conversion of + // pointer to bool + template + static void _indent(Iter os, int indent); + template + void _serialize(Iter os, int indent) const; + std::string _serialize(int indent) const; + void clear(); +}; + +// The ordered version of hashmap. It has the same interface as std::unordered_map, but provides +// ordered_keys() to return the keys in the order they were inserted. +class object_with_ordered_keys : private std::unordered_map { + public: + using typename std::unordered_map::value_type; + using typename std::unordered_map::iterator; + using typename std::unordered_map::const_iterator; + + object_with_ordered_keys() = default; + object_with_ordered_keys(const object_with_ordered_keys&) = default; + object_with_ordered_keys(object_with_ordered_keys&&) = default; + object_with_ordered_keys(std::initializer_list init) + : std::unordered_map(init) { + for (const auto& pair : init) { + ordered_keys_.push_back(pair.first); + } + } + object_with_ordered_keys& operator=(const object_with_ordered_keys&) = default; + object_with_ordered_keys& operator=(object_with_ordered_keys&&) = default; + + using std::unordered_map::begin; + using std::unordered_map::end; + using std::unordered_map::cbegin; + using std::unordered_map::cend; + using std::unordered_map::empty; + using std::unordered_map::size; + using std::unordered_map::at; + using std::unordered_map::count; + using std::unordered_map::find; + using std::unordered_map::reserve; + + value& operator[](const std::string& key) { + if (count(key) == 0) { + ordered_keys_.push_back(key); + } + return std::unordered_map::operator[](key); + } + + const value& operator[](const std::string& key) const { + return std::unordered_map::at(key); + } + + void clear() { + std::unordered_map::clear(); + ordered_keys_.clear(); + } + + std::pair insert(const value_type& kv) { + if (!count(kv.first)) { + ordered_keys_.push_back(kv.first); + } + return std::unordered_map::insert(kv); + } + + template + std::pair emplace(Args&&... args) { + return insert(value_type(std::forward(args)...)); + } + + iterator erase(const_iterator it) { + ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), it->first)); + return std::unordered_map::erase(it); + } + + iterator erase(iterator it) { + ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), it->first)); + return std::unordered_map::erase(it); + } + + size_t erase(const std::string& key) { + if (std::unordered_map::erase(key)) { + ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), key)); + return 1; + } else { + return 0; + } + } + + const std::vector& ordered_keys() const { return ordered_keys_; } + + friend bool operator==(const object_with_ordered_keys& lhs, const object_with_ordered_keys& rhs); + + private: + std::vector ordered_keys_; +}; + +inline bool operator==(const object_with_ordered_keys& lhs, const object_with_ordered_keys& rhs) { + return static_cast&>(lhs) == + static_cast&>(rhs); +} + +typedef value::array array; +typedef value::object object; + +inline value::value() : type_(null_type), u_() {} + +inline value::value(int type, bool) : type_(type), u_() { + switch (type) { +#define INIT(p, v) \ + case p##type: \ + u_.p = v; \ + break + INIT(boolean_, false); + INIT(number_, 0.0); +#ifdef PICOJSON_USE_INT64 + INIT(int64_, 0); +#endif + INIT(string_, new std::string()); + INIT(array_, new array()); + INIT(object_, new object()); +#undef INIT + default: + break; + } +} + +inline value::value(bool b) : type_(boolean_type), u_() { u_.boolean_ = b; } + +#ifdef PICOJSON_USE_INT64 +inline value::value(int64_t i) : type_(int64_type), u_() { u_.int64_ = i; } +#endif + +inline value::value(double n) : type_(number_type), u_() { + if ( +#ifdef _MSC_VER + !_finite(n) +#elif __cplusplus >= 201103L + std::isnan(n) || std::isinf(n) +#else + isnan(n) || isinf(n) +#endif + ) { +#ifndef PICOJSON_DISABLE_EXCEPTION + throw std::overflow_error(""); +#else + std::abort(); +#endif + } + u_.number_ = n; +} + +inline value::value(const std::string& s) : type_(string_type), u_() { + u_.string_ = new std::string(s); +} + +inline value::value(const array& a) : type_(array_type), u_() { u_.array_ = new array(a); } + +inline value::value(const object& o) : type_(object_type), u_() { u_.object_ = new object(o); } + +#if PICOJSON_USE_RVALUE_REFERENCE +inline value::value(std::string&& s) : type_(string_type), u_() { + u_.string_ = new std::string(std::move(s)); +} + +inline value::value(array&& a) : type_(array_type), u_() { u_.array_ = new array(std::move(a)); } + +inline value::value(object&& o) : type_(object_type), u_() { + u_.object_ = new object(std::move(o)); +} +#endif + +inline value::value(const char* s) : type_(string_type), u_() { u_.string_ = new std::string(s); } + +inline value::value(const char* s, size_t len) : type_(string_type), u_() { + u_.string_ = new std::string(s, len); +} + +inline void value::clear() { + switch (type_) { +#define DEINIT(p) \ + case p##type: \ + delete u_.p; \ + break + DEINIT(string_); + DEINIT(array_); + DEINIT(object_); +#undef DEINIT + default: + break; + } +} + +inline value::~value() { clear(); } + +inline value::value(const value& x) : type_(x.type_), u_() { + switch (type_) { +#define INIT(p, v) \ + case p##type: \ + u_.p = v; \ + break + INIT(string_, new std::string(*x.u_.string_)); + INIT(array_, new array(*x.u_.array_)); + INIT(object_, new object(*x.u_.object_)); +#undef INIT + default: + u_ = x.u_; + break; + } +} + +inline value& value::operator=(const value& x) { + if (this != &x) { + value t(x); + swap(t); + } + return *this; +} + +#if PICOJSON_USE_RVALUE_REFERENCE +inline value::value(value&& x) PICOJSON_NOEXCEPT : type_(null_type), u_() { swap(x); } +inline value& value::operator=(value&& x) PICOJSON_NOEXCEPT { + swap(x); + return *this; +} +#endif +inline void value::swap(value& x) PICOJSON_NOEXCEPT { + std::swap(type_, x.type_); + std::swap(u_, x.u_); +} + +#define IS(ctype, jtype) \ + template <> \ + inline bool value::is() const { \ + return type_ == jtype##_type; \ + } +IS(null, null) +IS(bool, boolean) +#ifdef PICOJSON_USE_INT64 +IS(int64_t, int64) +#endif +IS(std::string, string) +IS(array, array) +IS(object, object) +#undef IS +template <> +inline bool value::is() const { + return type_ == number_type +#ifdef PICOJSON_USE_INT64 + || type_ == int64_type +#endif + // NOLINTNEXTLINE(whitespace/semicolon) + ; +} + +#define GET(ctype, var) \ + template <> \ + inline const ctype& value::get() const { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ + return var; \ + } \ + template <> \ + inline ctype& value::get() { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ + return var; \ + } +GET(bool, u_.boolean_) +GET(std::string, *u_.string_) +GET(array, *u_.array_) +GET(object, *u_.object_) +#ifdef PICOJSON_USE_INT64 +GET(double, + (type_ == int64_type && (const_cast(this)->type_ = number_type, + (const_cast(this)->u_.number_ = u_.int64_)), + u_.number_)) +GET(int64_t, u_.int64_) +#else +GET(double, u_.number_) +#endif +#undef GET + +#define SET(ctype, jtype, setter) \ + template <> \ + inline void value::set(const ctype& _val) { \ + clear(); \ + type_ = jtype##_type; \ + setter \ + } +SET(bool, boolean, u_.boolean_ = _val;) +SET(std::string, string, u_.string_ = new std::string(_val);) +SET(array, array, u_.array_ = new array(_val);) +SET(object, object, u_.object_ = new object(_val);) +SET(double, number, u_.number_ = _val;) +#ifdef PICOJSON_USE_INT64 +SET(int64_t, int64, u_.int64_ = _val;) +#endif +#undef SET + +#if PICOJSON_USE_RVALUE_REFERENCE +#define MOVESET(ctype, jtype, setter) \ + template <> \ + inline void value::set(ctype && _val) { \ + clear(); \ + type_ = jtype##_type; \ + setter \ + } +MOVESET(std::string, string, u_.string_ = new std::string(std::move(_val));) +MOVESET(array, array, u_.array_ = new array(std::move(_val));) +MOVESET(object, object, u_.object_ = new object(std::move(_val));) +#undef MOVESET +#endif + +inline bool value::evaluate_as_boolean() const { + switch (type_) { + case null_type: + return false; + case boolean_type: + return u_.boolean_; + case number_type: + return u_.number_ != 0; +#ifdef PICOJSON_USE_INT64 + case int64_type: + return u_.int64_ != 0; +#endif + case string_type: + return !u_.string_->empty(); + default: + return true; + } +} + +inline const value& value::get(const size_t idx) const { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; +} + +inline value& value::get(const size_t idx) { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; +} + +inline const value& value::get(const std::string& key) const { + static value s_null; + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; +} + +inline value& value::get(const std::string& key) { + static value s_null; + PICOJSON_ASSERT(is()); + object::iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; +} + +inline bool value::contains(const size_t idx) const { + PICOJSON_ASSERT(is()); + return idx < u_.array_->size(); +} + +inline bool value::contains(const std::string& key) const { + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end(); +} + +inline std::string value::to_str() const { + switch (type_) { + case null_type: + return "null"; + case boolean_type: + return u_.boolean_ ? "true" : "false"; +#ifdef PICOJSON_USE_INT64 + case int64_type: { + char buf[sizeof("-9223372036854775808")]; + SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); + return buf; + } +#endif + case number_type: { + char buf[256]; + double tmp; + SNPRINTF( + buf, + sizeof(buf), + fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", + u_.number_ + ); +#if PICOJSON_USE_LOCALE + char* decimal_point = localeconv()->decimal_point; + if (strcmp(decimal_point, ".") != 0) { + size_t decimal_point_len = strlen(decimal_point); + for (char* p = buf; *p != '\0'; ++p) { + if (strncmp(p, decimal_point, decimal_point_len) == 0) { + return std::string(buf, p) + "." + (p + decimal_point_len); + } + } + } +#endif + return buf; + } + case string_type: + return *u_.string_; + case array_type: + return "array"; + case object_type: + return "object"; + default: + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + } + return std::string(); +} + +template +void copy(const std::string& s, Iter oi) { + std::copy(s.begin(), s.end(), oi); +} + +template +struct serialize_str_char { + Iter oi; + void operator()(char c) { + switch (c) { +#define MAP(val, sym) \ + case val: \ + copy(sym, oi); \ + break + MAP('"', "\\\""); + MAP('\\', "\\\\"); + MAP('/', "\\/"); + MAP('\b', "\\b"); + MAP('\f', "\\f"); + MAP('\n', "\\n"); + MAP('\r', "\\r"); + MAP('\t', "\\t"); +#undef MAP + default: + if (static_cast(c) < 0x20 || c == 0x7f) { + char buf[7]; + SNPRINTF(buf, sizeof(buf), "\\u%04x", c & 0xff); + copy(buf, buf + 6, oi); + } else { + *oi++ = c; + } + break; + } + } +}; + +template +void serialize_str(const std::string& s, Iter oi) { + *oi++ = '"'; + serialize_str_char process_char = {oi}; + std::for_each(s.begin(), s.end(), process_char); + *oi++ = '"'; +} + +template +void value::serialize(Iter oi, bool prettify) const { + return _serialize(oi, prettify ? 0 : -1); +} + +inline std::string value::serialize(bool prettify) const { return _serialize(prettify ? 0 : -1); } + +template +void value::_indent(Iter oi, int indent) { + *oi++ = '\n'; + for (int i = 0; i < indent * INDENT_WIDTH; ++i) { + *oi++ = ' '; + } +} + +template +void value::_serialize(Iter oi, int indent) const { + switch (type_) { + case string_type: + serialize_str(*u_.string_, oi); + break; + case array_type: { + *oi++ = '['; + if (indent != -1) { + ++indent; + } + for (array::const_iterator i = u_.array_->begin(); i != u_.array_->end(); ++i) { + if (i != u_.array_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + i->_serialize(oi, indent); + } + if (indent != -1) { + --indent; + if (!u_.array_->empty()) { + _indent(oi, indent); + } + } + *oi++ = ']'; + break; + } + case object_type: { + *oi++ = '{'; + if (indent != -1) { + ++indent; + } + +#if PICOJSON_USE_ORDERED_OBJECT + for (auto i = u_.object_->ordered_keys().begin(); i != u_.object_->ordered_keys().end(); + ++i) { + if (i != u_.object_->ordered_keys().begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + serialize_str(*i, oi); + *oi++ = ':'; + if (indent != -1) { + *oi++ = ' '; + } + u_.object_->at(*i)._serialize(oi, indent); + } +#else + for (object::const_iterator i = u_.object_->begin(); i != u_.object_->end(); ++i) { + if (i != u_.object_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + serialize_str(i->first, oi); + *oi++ = ':'; + if (indent != -1) { + *oi++ = ' '; + } + i->second._serialize(oi, indent); + } +#endif + if (indent != -1) { + --indent; + if (!u_.object_->empty()) { + _indent(oi, indent); + } + } + *oi++ = '}'; + break; + } + default: + copy(to_str(), oi); + break; + } + if (indent == 0) { + *oi++ = '\n'; + } +} + +inline std::string value::_serialize(int indent) const { + std::string s; + _serialize(std::back_inserter(s), indent); + return s; +} + +template +class input { + protected: + Iter cur_, end_; + bool consumed_; + int line_; + + public: + input(const Iter& first, const Iter& last) + : cur_(first), end_(last), consumed_(false), line_(1) {} + int getc() { + if (consumed_) { + if (*cur_ == '\n') { + ++line_; + } + ++cur_; + } + if (cur_ == end_) { + consumed_ = false; + return -1; + } + consumed_ = true; + return *cur_ & 0xff; + } + void ungetc() { consumed_ = false; } + Iter cur() const { + if (consumed_) { + input* self = const_cast*>(this); + self->consumed_ = false; + ++self->cur_; + } + return cur_; + } + int line() const { return line_; } + void skip_ws() { + while (1) { + int ch = getc(); + if (!(ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { + ungetc(); + break; + } + } + } + bool expect(const int expected) { + skip_ws(); + if (getc() != expected) { + ungetc(); + return false; + } + return true; + } + bool match(const std::string& pattern) { + for (std::string::const_iterator pi(pattern.begin()); pi != pattern.end(); ++pi) { + if (getc() != *pi) { + ungetc(); + return false; + } + } + return true; + } +}; + +template +// NOLINTNEXTLINE(runtime/references) +inline int _parse_quadhex(input& in) { + int uni_ch = 0, hex; + for (int i = 0; i < 4; i++) { + if ((hex = in.getc()) == -1) { + return -1; + } + if ('0' <= hex && hex <= '9') { + hex -= '0'; + } else if ('A' <= hex && hex <= 'F') { + hex -= 'A' - 0xa; + } else if ('a' <= hex && hex <= 'f') { + hex -= 'a' - 0xa; + } else { + in.ungetc(); + return -1; + } + uni_ch = uni_ch * 16 + hex; + } + return uni_ch; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse_codepoint(String& out, input& in) { + int uni_ch; + if ((uni_ch = _parse_quadhex(in)) == -1) { + return false; + } + if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { + if (0xdc00 <= uni_ch) { + // a second 16-bit of a surrogate pair appeared + return false; + } + // first 16-bit of surrogate pair, get the next one + if (in.getc() != '\\' || in.getc() != 'u') { + in.ungetc(); + return false; + } + int second = _parse_quadhex(in); + if (!(0xdc00 <= second && second <= 0xdfff)) { + return false; + } + uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); + uni_ch += 0x10000; + } + if (uni_ch < 0x80) { + out.push_back(static_cast(uni_ch)); + } else { + if (uni_ch < 0x800) { + out.push_back(static_cast(0xc0 | (uni_ch >> 6))); + } else { + if (uni_ch < 0x10000) { + out.push_back(static_cast(0xe0 | (uni_ch >> 12))); + } else { + out.push_back(static_cast(0xf0 | (uni_ch >> 18))); + out.push_back(static_cast(0x80 | ((uni_ch >> 12) & 0x3f))); + } + out.push_back(static_cast(0x80 | ((uni_ch >> 6) & 0x3f))); + } + out.push_back(static_cast(0x80 | (uni_ch & 0x3f))); + } + return true; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse_string(String& out, input& in) { + while (1) { + int ch = in.getc(); + if (ch < ' ') { + in.ungetc(); + return false; + } else if (ch == '"') { + return true; + } else if (ch == '\\') { + if ((ch = in.getc()) == -1) { + return false; + } + switch (ch) { +#define MAP(sym, val) \ + case sym: \ + out.push_back(val); \ + break + MAP('"', '\"'); + MAP('\\', '\\'); + MAP('/', '/'); + MAP('b', '\b'); + MAP('f', '\f'); + MAP('n', '\n'); + MAP('r', '\r'); + MAP('t', '\t'); +#undef MAP + case 'u': + if (!_parse_codepoint(out, in)) { + return false; + } + break; + default: + return false; + } + } else { + out.push_back(static_cast(ch)); + } + } + return false; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse_array(Context& ctx, input& in) { + if (!ctx.parse_array_start()) { + return false; + } + size_t idx = 0; + if (in.expect(']')) { + return ctx.parse_array_stop(idx); + } + do { + if (!ctx.parse_array_item(in, idx)) { + return false; + } + idx++; + } while (in.expect(',')); + return in.expect(']') && ctx.parse_array_stop(idx); +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse_object(Context& ctx, input& in) { + if (!ctx.parse_object_start()) { + return false; + } + if (in.expect('}')) { + return true; + } + do { + std::string key; + if (!in.expect('"') || !_parse_string(key, in) || !in.expect(':')) { + return false; + } + if (!ctx.parse_object_item(in, key)) { + return false; + } + } while (in.expect(',')); + return in.expect('}'); +} + +template +// NOLINTNEXTLINE(runtime/references) +inline std::string _parse_number(input& in) { + std::string num_str; + while (1) { + int ch = in.getc(); + if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' || ch == 'e' || ch == 'E') { + num_str.push_back(static_cast(ch)); + } else if (ch == '.') { +#if PICOJSON_USE_LOCALE + num_str += localeconv()->decimal_point; +#else + num_str.push_back('.'); +#endif + } else { + in.ungetc(); + break; + } + } + return num_str; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline bool _parse(Context& ctx, input& in) { + in.skip_ws(); + int ch = in.getc(); + switch (ch) { +#define IS(ch, text, op) \ + case ch: \ + if (in.match(text) && op) { \ + return true; \ + } else { \ + return false; \ + } + IS('n', "ull", ctx.set_null()); + IS('f', "alse", ctx.set_bool(false)); + IS('t', "rue", ctx.set_bool(true)); +#undef IS + case '"': + return ctx.parse_string(in); + case '[': + return _parse_array(ctx, in); + case '{': + return _parse_object(ctx, in); + default: + if (('0' <= ch && ch <= '9') || ch == '-') { + double f; + char* endp; + in.ungetc(); + std::string num_str(_parse_number(in)); + if (num_str.empty()) { + return false; + } +#ifdef PICOJSON_USE_INT64 + { + errno = 0; + intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); + if (errno == 0 && std::numeric_limits::min() <= ival && + ival <= std::numeric_limits::max() && + endp == num_str.c_str() + num_str.size()) { + ctx.set_int64(ival); + return true; + } + } +#endif + f = strtod(num_str.c_str(), &endp); + if (endp == num_str.c_str() + num_str.size()) { + ctx.set_number(f); + return true; + } + return false; + } + break; + } + in.ungetc(); + return false; +} + +class deny_parse_context { + public: + bool set_null() { return false; } + bool set_bool(bool) { return false; } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { return false; } +#endif + bool set_number(double) { return false; } + template + bool parse_string(input&) { + return false; + } + bool parse_array_start() { return false; } + template + bool parse_array_item(input&, size_t) { + return false; + } + bool parse_array_stop(size_t) { return false; } + bool parse_object_start() { return false; } + template + bool parse_object_item(input&, const std::string&) { + return false; + } +}; + +class default_parse_context { + protected: + value* out_; + + public: + // NOLINTNEXTLINE(runtime/explicit) + default_parse_context(value* out) : out_(out) {} + bool set_null() { + *out_ = value(); + return true; + } + bool set_bool(bool b) { + *out_ = value(b); + return true; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t i) { + *out_ = value(i); + return true; + } +#endif + bool set_number(double f) { + *out_ = value(f); + return true; + } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_string(input& in) { + *out_ = value(string_type, false); + return _parse_string(out_->get(), in); + } + bool parse_array_start() { + *out_ = value(array_type, false); + return true; + } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_array_item(input& in, size_t) { + array& a = out_->get(); + a.push_back(value()); + default_parse_context ctx(&a.back()); + return _parse(ctx, in); + } + bool parse_array_stop(size_t) { return true; } + bool parse_object_start() { + *out_ = value(object_type, false); + return true; + } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_object_item(input& in, const std::string& key) { + object& o = out_->get(); + default_parse_context ctx(&o[key]); + return _parse(ctx, in); + } + + private: + default_parse_context(const default_parse_context&); + default_parse_context& operator=(const default_parse_context&); +}; + +class null_parse_context { + public: + struct dummy_str { + void push_back(int) {} + }; + + public: + null_parse_context() {} + bool set_null() { return true; } + bool set_bool(bool) { return true; } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { return true; } +#endif + bool set_number(double) { return true; } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_string(input& in) { + dummy_str s; + return _parse_string(s, in); + } + bool parse_array_start() { return true; } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_array_item(input& in, size_t) { + return _parse(*this, in); + } + bool parse_array_stop(size_t) { return true; } + bool parse_object_start() { return true; } + template + // NOLINTNEXTLINE(runtime/references) + bool parse_object_item(input& in, const std::string&) { + return _parse(*this, in); + } + + private: + null_parse_context(const null_parse_context&); + null_parse_context& operator=(const null_parse_context&); +}; + +// obsolete, use the version below +template +// NOLINTNEXTLINE(runtime/references) +inline std::string parse(value& out, Iter& pos, const Iter& last) { + std::string err; + pos = parse(out, pos, last, &err); + return err; +} + +template +// NOLINTNEXTLINE(runtime/references) +inline Iter _parse(Context& ctx, const Iter& first, const Iter& last, std::string* err) { + input in(first, last); + if (!_parse(ctx, in) && err != NULL) { + char buf[64]; + SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); + *err = buf; + while (1) { + int ch = in.getc(); + if (ch == -1 || ch == '\n') { + break; + } else if (ch >= ' ') { + err->push_back(static_cast(ch)); + } + } + } + return in.cur(); +} + +template +// NOLINTNEXTLINE(runtime/references) +inline Iter parse(value& out, const Iter& first, const Iter& last, std::string* err) { + default_parse_context ctx(&out); + return _parse(ctx, first, last, err); +} + +// NOLINTNEXTLINE(runtime/references) +inline std::string parse(value& out, const std::string& s) { + std::string err; + parse(out, s.begin(), s.end(), &err); + return err; +} + +// NOLINTNEXTLINE(runtime/references) +inline std::string parse(value& out, std::istream& is) { + std::string err; + parse(out, std::istreambuf_iterator(is.rdbuf()), std::istreambuf_iterator(), &err); + return err; +} + +template +struct last_error_t { + static std::string s; +}; +template +// NOLINTNEXTLINE(runtime/string) +std::string last_error_t::s; + +inline void set_last_error(const std::string& s) { last_error_t::s = s; } + +inline const std::string& get_last_error() { return last_error_t::s; } + +inline bool operator==(const value& x, const value& y) { + if (x.is()) return y.is(); +#define PICOJSON_CMP(type) \ + if (x.is()) return y.is() && x.get() == y.get() + PICOJSON_CMP(bool); + PICOJSON_CMP(double); + PICOJSON_CMP(std::string); + PICOJSON_CMP(array); + PICOJSON_CMP(object); +#undef PICOJSON_CMP + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + return false; +} + +inline bool operator!=(const value& x, const value& y) { return !(x == y); } +} // namespace picojson + +#if !PICOJSON_USE_RVALUE_REFERENCE +namespace std { +template <> +inline void swap(picojson::value& x, picojson::value& y) { + x.swap(y); +} +} // namespace std +#endif + +inline std::istream& operator>>(std::istream& is, picojson::value& x) { + picojson::set_last_error(std::string()); + const std::string err(picojson::parse(x, is)); + if (!err.empty()) { + picojson::set_last_error(err); + is.setstate(std::ios::failbit); + } + return is; +} + +inline std::ostream& operator<<(std::ostream& os, const picojson::value& x) { + x.serialize(std::ostream_iterator(os)); + return os; +} +#ifdef _MSC_VER +#pragma warning(pop) +#endif diff --git a/cactus-engine/libs/xgrammar/include/xgrammar/compiler.h b/cactus-engine/libs/xgrammar/include/xgrammar/compiler.h new file mode 100644 index 000000000..a2f0138dc --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/xgrammar/compiler.h @@ -0,0 +1,115 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/compiler.h + * \brief The header for the compiler. + */ + +#ifndef XGRAMMAR_COMPILER_H_ +#define XGRAMMAR_COMPILER_H_ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "xgrammar/exception.h" + +namespace xgrammar { + +/*! + * \brief The compiled grammar of a GrammarMatcher. It contains the preprocessing results of the + * grammar and tokenizer. + */ +class CompiledGrammar { + public: + /*! \brief Get the associated grammar. */ + Grammar GetGrammar() const; + + /*! \brief Get the associated tokenizer info. */ + TokenizerInfo GetTokenizerInfo() const; + + /*! \brief Return the approximate memory usage of the grammar in bytes. */ + std::size_t MemorySizeBytes() const; + + /*! \brief Return the serialized JSON string of the compiled grammar. */ + std::string SerializeJSON() const; + + /*! \brief Deserialize a compiled grammar from a JSON string and tokenizer info. */ + static std::variant DeserializeJSON( + const std::string& json_string, const TokenizerInfo& tokenizer_info + ); + + XGRAMMAR_DEFINE_PIMPL_METHODS(CompiledGrammar); +}; + +/*! + * \brief A cache to get the compiled grammar for grammar or schema. This class avoids + * redundant preprocessing of the grammar or schema when constructing a CompiledGrammar. + * \note This class is associated with a vocabulary when constructed. The vocabulary is used to + * create every compiled grammar. If multiple toke tables are used to create init + * contexts, an instance of this class for each vocabulary should be created. + */ +class GrammarCompiler { + public: + /*! + * \brief Construct a GrammarCompiler with a vocabulary. This class will always + * create compiled grammars with this vocabulary. + * \param tokenizer_info The tokenizer info. + * \param max_threads The maximum number of threads to use for compiling grammars. + * \param cache_enabled Whether to enable the cache. + * \param max_memory_bytes The maximum memory usage in bytes. + */ + GrammarCompiler( + const TokenizerInfo& tokenizer_info, + int max_threads = 8, + bool cache_enabled = true, + int64_t max_memory_bytes = -1 // unlimited + ); + + /*! \brief Get the compiled grammar for a JSON schema string. */ + CompiledGrammar CompileJSONSchema( + const std::string& schema, + bool any_whitespace = true, + std::optional indent = std::nullopt, + std::optional> separators = std::nullopt, + bool strict_mode = true, + std::optional max_whitespace_cnt = std::nullopt + ); + + /*! \brief Get the compiled grammar for pure JSON. */ + CompiledGrammar CompileBuiltinJSONGrammar(); + + /*! \brief Get the compiled grammar for a grammar. */ + CompiledGrammar CompileGrammar(const Grammar& grammar); + + /*! \brief Get the compiled grammar for a grammar. */ + CompiledGrammar CompileGrammar( + const std::string& ebnf_str, const std::string& root_rule_name = "root" + ); + + /*! \brief Get the compiled grammar for a structural tag. */ + CompiledGrammar CompileStructuralTag(const std::string& structural_tag_json); + + /*! \brief Get the compiled grammar for a regex. */ + CompiledGrammar CompileRegex(const std::string& regex); + + /*! \brief Clear the internal cache of compiled grammars. */ + void ClearCache(); + + /*! \brief Return the approximate memory usage of the compiler in bytes. */ + int64_t GetCacheSizeBytes() const; + + /*! \brief Return the approximate memory usage of the compiler in bytes. -1 means unlimited. */ + int64_t CacheLimitBytes() const; + + XGRAMMAR_DEFINE_PIMPL_METHODS(GrammarCompiler); +}; + +} // namespace xgrammar + +#endif // XGRAMMAR_COMPILER_H_ diff --git a/cactus-engine/libs/xgrammar/include/xgrammar/config.h b/cactus-engine/libs/xgrammar/include/xgrammar/config.h new file mode 100644 index 000000000..4e5de5e84 --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/xgrammar/config.h @@ -0,0 +1,35 @@ +/*! + * Copyright (c) 2025 by Contributors + * \file xgrammar/config.h + * \brief Global configuration for XGrammar. + */ + +#ifndef XGRAMMAR_CONFIG_H_ +#define XGRAMMAR_CONFIG_H_ + +#include + +namespace xgrammar { + +/*! + * \brief Set the maximum recursion depth for the grammar. + * \param max_recursion_depth The maximum recursion depth. + */ +void SetMaxRecursionDepth(int max_recursion_depth); + +/*! + * \brief Get the maximum recursion depth for the grammar. + * \return The maximum recursion depth. + */ +int GetMaxRecursionDepth(); + +/*! + * \brief Get the serialization version for the grammar. + * \return The serialization version. + * \note This is used to check the compatibility of the serialized grammar. + */ +std::string GetSerializationVersion(); + +} // namespace xgrammar + +#endif // XGRAMMAR_CONFIG_H_ diff --git a/cactus-engine/libs/xgrammar/include/xgrammar/exception.h b/cactus-engine/libs/xgrammar/include/xgrammar/exception.h new file mode 100644 index 000000000..7efb3b000 --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/xgrammar/exception.h @@ -0,0 +1,80 @@ +#ifndef XGRAMMAR_EXCEPTION_H +#define XGRAMMAR_EXCEPTION_H + +#include +#include +#include + +namespace xgrammar { + +/************** Exception Definitions **************/ + +/*! + * \brief Exception thrown when the version in the serialized data does not follow the current + * serialization version. + */ + +struct XGrammarError : std::runtime_error { + XGrammarError(const std::string& message) : std::runtime_error(message) {} + virtual std::string GetType() const { return "XGrammarError"; } +}; + +struct DeserializeVersionError : XGrammarError { + DeserializeVersionError(const std::string& message) + : XGrammarError(std::string("Deserialize version error: ") + message) {} + std::string GetType() const override { return "DeserializeVersionError"; } +}; + +/*! + * \brief Exception thrown when the JSON is invalid. + */ +struct InvalidJSONError : XGrammarError { + InvalidJSONError(const std::string& message) + : XGrammarError(std::string("Invalid JSON error: ") + message) {} + std::string GetType() const override { return "InvalidJSONError"; } +}; + +/*! + * \brief Exception thrown when the serialized data does not follow the expected format. + */ +struct DeserializeFormatError : XGrammarError { + DeserializeFormatError(const std::string& message) + : XGrammarError(std::string("Deserialize format error: ") + message) {} + std::string GetType() const override { return "DeserializeFormatError"; } +}; + +/*! + * \brief Exception thrown when the JSON schema is invalid or not satisfiable. + */ +struct InvalidJSONSchemaError : XGrammarError { + InvalidJSONSchemaError(const std::string& message) + : XGrammarError(std::string("Invalid JSON schema error: ") + message) {} + std::string GetType() const override { return "InvalidJSONSchemaError"; } +}; + +/*! + * \brief Exception thrown when the structural tag is invalid. + */ +struct InvalidStructuralTagError : XGrammarError { + InvalidStructuralTagError(const std::string& message) + : XGrammarError(std::string("Invalid structural tag error: ") + message) {} + std::string GetType() const override { return "InvalidStructuralTagError"; } +}; + +/************** Union Exceptions **************/ + +/*! + * \brief Represents a serialization error. + */ +using SerializationError = + std::variant; + +/*! + * \brief Represents an error from the structural tag conversion. + */ +using StructuralTagError = + std::variant; + +} // namespace xgrammar + +#endif // XGRAMMAR_EXCEPTION_H diff --git a/cactus-engine/libs/xgrammar/include/xgrammar/grammar.h b/cactus-engine/libs/xgrammar/include/xgrammar/grammar.h new file mode 100644 index 000000000..17f7d6540 --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/xgrammar/grammar.h @@ -0,0 +1,191 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/grammar.h + * \brief The header for the definition and construction of BNF grammar. + */ + +#ifndef XGRAMMAR_GRAMMAR_H_ +#define XGRAMMAR_GRAMMAR_H_ + +#include +#include + +#include +#include +#include +#include +#include + +#include "xgrammar/exception.h" + +namespace xgrammar { + +struct StructuralTagItem { + std::string begin; + std::string schema; + std::string end; + + bool operator==(const StructuralTagItem& other) const { + return begin == other.begin && schema == other.schema && end == other.end; + } +}; + +/*! + * \brief This class stores the abstract syntax tree (AST) of the Backus-Naur Form (BNF) grammar. + * The BNF definition here is standard BNF, and the characters are represented using regex-style + * character classes (e.g. [a-z], [^a-z]). + * + * \details + * ### Rules + * The BNF grammar AST consists of a set of rules. Each rule contains a name and a definition, and + * corresponds to a production in the grammar. The definition of a rule is a GrammarExpr. Each rule + * has a rule_id for reference. + * + * ### GrammarExprs + * GrammarExpr is the definition of a rule or part of the definition of a rule. It can contain + * elements, empty string, reference to other GrammarExprs, or reference to other rules. Each + * GrammarExpr corresponds to a grammar_expr_id for reference. + * + * For example, in the following rule: rule ::= ("a" "b") | "c" + * ("a" "b"), "c", ("a" "b") | "c" are all GrammarExprs. + * + * #### Types of GrammarExprs + * Every GrammarExpr is represented by a type as well as a variable-length array containing its + * data. GrammarExpr has several types: + * - Byte string: a string of bytes (0~255). Supports UTF-8 strings. + * - Character class: a range of characters (each character is a unicode codepoint), e.g. [a-z], + * [ac-z]. Can be negated: [^a-z], [^ac-z]. Now only ascii chars is allowed in [], but this + * expression can accept/reject unicode chars. + * - Character class star: a star quantifier of a character class. e.g. [a-z]*, [^a-z]*. + * - EmptyStr: an empty string, i.e. "" + * - Rule reference: a reference to another rule + * - Sequence: a sequence of grammar_exprs, e.g. ("a" "b"). These grammar_exprs are concatenated + * together. + * - Choices: a choice of grammar_exprs, e.g. ("a" "b") | "c". Each grammar_expr can be matched. + * + * #### Storage of GrammarExprs + * Each type of GrammarExpr has a different data format. For the format of each type of GrammarExpr, + * see docs in Grammar::Impl::GrammarExprType. + * + * We store all GrammarExprs in csr_matrix style. That is, they are stored consecutively in one + * vector (data vector) and the starting position of each GrammarExpr is recorded in the indptr + * vector. + * + * \remark The character class star GrammarExpr is for the special support for elements like [a-z]* + * in the grammar. We add it to make the matching more efficient, as we can avoid recursion into + * rules when matching a sequence of characters. It should be used like: + * rule1 ::= ((element1 element2 rule2 ...) | ...) + * rule2 ::= character_class_star_grammar_expr(id_of_a_character_class_grammar_expr) + */ +class Grammar { + public: + /*! + * \brief Get the EBNF string of the grammar. + */ + std::string ToString() const; + + /*! + * \brief Construct a BNF grammar with a EBNF-formatted string. The grammar will be normalized + * (simplified) by default. + * \param ebnf_string The EBNF-formatted string. + * \param root_rule_name The name of the root rule. + */ + static Grammar FromEBNF( + const std::string& ebnf_string, const std::string& root_rule_name = "root" + ); + + /*! + * \brief Construct a BNF grammar from the json schema string. The schema string should be in the + * format of the schema of a JSON file. We will parse the schema and generate a BNF grammar. + * \param schema The schema string. + * \param indent The number of spaces for indentation. If set to std::nullopt, the output will be + * in one line. Default: 2. + * \param separators Two separators used in the schema: comma and colon. Examples: {",", ":"}, + * {", ", ": "}. If std::nullopt, the default separators will be used: {",", ": "} when the + * indent is not nullopt, and {", ", ": "} otherwise. This follows the convention in python + * json.dumps(). Default: std::nullopt. + * \param strict_mode Whether to use strict mode. In strict mode, the generated grammar will not + * allow properties and items that is not specified in the schema. This is equivalent to + * setting unevaluatedProperties and unevaluatedItems to false. + * + * This helps LLM to generate accurate output in the grammar-guided generation with JSON + * schema. Default: true. + */ + static Grammar FromJSONSchema( + const std::string& schema, + bool any_whitespace = true, + std::optional indent = std::nullopt, + std::optional> separators = std::nullopt, + bool strict_mode = true, + std::optional max_whitespace_cnt = std::nullopt, + bool print_converted_ebnf = false + ); + + /*! + * \brief Construct a grammar from a regular expression string. + * \param regex The regular expression string. + * \param print_converted_ebnf This method will convert the regex to EBNF first. If this is true, + * the converted EBNF string will be printed. For debugging purpose. Default: false. + */ + static Grammar FromRegex(const std::string& regex, bool print_converted_ebnf = false); + + /*! + * \brief Construct a grammar from a structural tag string. + * \param structural_tag_json The structural tag string. + * \param tokenizer_info Optional tokenizer info for resolving string token references. + */ + static std::variant FromStructuralTag( + const std::string& structural_tag_json, + const std::optional& tokenizer_info = std::nullopt + ); + + /*! + * \brief Get the grammar of standard JSON format. We have built-in support for JSON. + * \return The grammar of standard JSON format. + */ + static Grammar BuiltinJSONGrammar(); + + /*! + * \brief Create a grammar that matches any of the grammars in the list. That is equivalent to + * using the `|` operator to concatenate the grammars in the list. + * \param grammars The grammars to create the union of. + * \returns The union of the grammars. + */ + static Grammar Union(const std::vector& grammars); + + /*! + * \brief Create a grammar that matches the concatenation of the grammars in the list. That is + * equivalent to using the `+` operator to concatenate the grammars in the list. + * \param grammars The grammars to create the concatenation of. + * \returns The concatenation of the grammars. + */ + static Grammar Concat(const std::vector& grammars); + + /*! + * \brief Print a BNF grammar. + * \param os The output stream. + * \param grammar The grammar to print. + * \return The output stream. + */ + friend std::ostream& operator<<(std::ostream& os, const Grammar& grammar); + + /*! + * \brief Return the serialized JSON string of the grammar. + * \return The serialized JSON string. + */ + std::string SerializeJSON() const; + + /*! + * \brief Deserialize a grammar from a JSON string. + * \param json_string The JSON string to deserialize. + * \return If the deserialization is successful, return the grammar. Otherwise, return a runtime + * error with the error message. + */ + static std::variant DeserializeJSON(const std::string& json_string); + + XGRAMMAR_DEFINE_PIMPL_METHODS(Grammar); +}; + +} // namespace xgrammar + +#endif // XGRAMMAR_GRAMMAR_H_ diff --git a/cactus-engine/libs/xgrammar/include/xgrammar/matcher.h b/cactus-engine/libs/xgrammar/include/xgrammar/matcher.h new file mode 100644 index 000000000..654e76014 --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/xgrammar/matcher.h @@ -0,0 +1,256 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/matcher.h + * \brief The header for the matcher. + */ + +#ifndef XGRAMMAR_MATCHER_H_ +#define XGRAMMAR_MATCHER_H_ + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace xgrammar { + +int32_t GetBitmaskSize(int vocab_size); + +DLDataType GetBitmaskDLType(); + +void _DebugGetMaskedTokensFromBitmask( + std::vector* rejected_tokens, const DLTensor& token_bitmask, int vocab_size, int index = 0 +); + +std::pair _IsSingleTokenBitmask(const DLTensor& bitmask, int vocab_size, int index); + +void ApplyTokenBitmaskInplaceCPU( + DLTensor* logits, + const DLTensor& bitmask, + int vocab_size = -1, + std::optional> indices = std::nullopt +); + +/*! + * \brief A stateful matcher to match tokens to the specified BNF grammar. This class is the core + * logic of the grammar-guided generation. + * + * \details This class implements the non-deterministic pushdown automaton (NPDA) matching algorithm + * to match characters to a BNF grammar. It keep track of the current state of the matching process + * by maintaining several stacks internally as possible paths in the NPDA. It also supports + * backtracking. + * + * It is particularly capable of finding the set of tokens that are acceptable for the next step + * and storing them in a bitmask. This aids in grammar-guided generation. + * + * \example + * \code + * Tokenizer tokenizer = ...; + * auto compiled_grammar = GrammarMatcher::CreateCompiledGrammar(grammar, + * tokenizer->PostProcessedVocab()); + * GrammarMatcher matcher(compiled_grammar, 10); + * matcher->AcceptToken(67); + * + * // Construct a DLTensor with shape (tokenizer.GetVocabSize() + 31) / 32, and dtype int32. + * DLTensor next_token_bitmask = ...; + * matcher->FillNextTokenBitmask(&next_token_bitmask); + * + * // Rollback is supported + * matcher->Rollback(1); + * \endcode + */ +class GrammarMatcher { + public: + /*! + * \brief Construct a GrammarMatcher from the preprocessing result of type + * CompiledGrammar. + * \param compiled_grammar The compiled grammar. It is obtained through + * CreateCompiledGrammar as a result of preprocessing the grammar and tokenizer. + */ + GrammarMatcher( + const CompiledGrammar& compiled_grammar, + std::optional> override_stop_tokens = std::nullopt, + bool terminate_without_stop_token = false, + int max_rollback_tokens = -1 + ); + + /*! + * \brief Accept one token and update the state of the matcher. + * \param token_id The id of the token to accept. + * \return Whether the token is accepted. + * \note Termination state. + * When the end of the root rule is reached, the matcher can only accept the stop token. + * The matcher is terminated after accepting the stop token, i.e. no AcceptToken or + * FindNextTokenMask operations can be performed. The termination state can be canceled + * using Rollback(). + */ + bool AcceptToken(int32_t token_id, bool debug_print = false); + + /*! + * \brief Accept a string and update the state of the matcher. The whole string is considered + * as one step in rollback. It is used to complement the functionality of AcceptToken, and + * AcceptToken should always be used to accept tokens. + * \param input_str The string to be accepted. + * \param debug_print Whether to print information about the internal state of the matcher. + * \return Whether the string is accepted. + */ + bool AcceptString(const std::string& input_str, bool debug_print = false); + + /*! + * \brief Get the set of tokens that are acceptable for the next step and store them in a + * bitmask. + * \param next_token_bitmask The bitmask to store the result. The bitmask must be pre-allocated + * and with shape (GetBitmaskSize(),) and dtype int32. + * \return Whether the bitmask need to be applied (not all-true). + */ + bool FillNextTokenBitmask(DLTensor* next_token_bitmask, int index = 0, bool debug_print = false); + + /*! + * \brief Traverse a draft token tree and fill the token bitmask for each node. + * + * This function performs a DFS traversal of the speculative decoding tree and fills + * the token bitmask for each node based on grammar constraints. + * + * \param retrieve_next_token DLTensor where retrieve_next_token[i] gives the index of + * the child node of node i, or -1 if no child exists. + * \param retrieve_next_sibling DLTensor where retrieve_next_sibling[i] gives the index of + * the sibling node of node i, or -1 if no sibling exists. + * \param draft_tokens DLTensor of draft token ids at each node. + * \param token_bitmask DLTensor to store the bitmask (2D: num_nodes x bitmask_size). + * \param time_threshold Maximum allowed time in seconds for the DFS traversal. + * If the traversal exceeds this threshold, it returns false. + * A value <= 0 disables the timeout (default: -1.0). + * \return true if the traversal completed successfully, false if it timed out. + */ + bool TraverseDraftTree( + const DLTensor* retrieve_next_token, + const DLTensor* retrieve_next_sibling, + const DLTensor* draft_tokens, + DLTensor* token_bitmask, + double time_threshold = -1.0 + ); + + /*! + * \brief Find the jump-forward string for jump-forward decoding. This is the longest string that + will be valid according to the current syntax. + * \note This method does not change the grammar state. + */ + std::string FindJumpForwardString(); + + /*! + * \brief Rollback the matcher to a previous state. + * \param num_tokens The number of tokens to rollback. It cannot exceed the current number of + * steps, nor can it exceed the specified maximum number of rollback tokens. + */ + void Rollback(int num_tokens = 1); + + /*! + * \brief Check if the matcher has accepted the stop token and terminated. + * \sa AcceptToken + */ + bool IsTerminated() const; + + /*! + * \brief Check if the grammar's root rule has been fully matched by the input accepted so far. + * Unlike IsTerminated(), this does not require the stop token to have been accepted. + * \sa IsTerminated, AcceptToken + */ + bool IsCompleted() const; + + /*! \brief Reset the matcher to the initial state. */ + void Reset(); + + /*! + * \brief Fork the matcher. Returns a new GrammarMatcher with a deep copy of all state except + * compiled_grammar and tokenizer_info, which are shared with this matcher. + */ + GrammarMatcher Fork() const; + + /*! \brief Get the maximum number of rollback tokens allowed. */ + int GetMaxRollbackTokens() const; + + const std::vector& GetStopTokenIds() const; + + /*! \brief Print the internal state of the matcher. This is only used for debugging. The + * representation of the internal state is subject to change. + */ + std::string _DebugPrintInternalState() const; + + XGRAMMAR_DEFINE_PIMPL_METHODS(GrammarMatcher); +}; + +/*! + * \brief A batched version of GrammarMatcher for better efficiency. It supports batch processing + * of multiple GrammarMatcher objects in parallel. + * + * \details This class provides batched versions of the core methods of GrammarMatcher, including + * FillNextTokenBitmask, AcceptString, and AcceptToken. It utilizes multi-threading to process + * multiple GrammarMatcher objects simultaneously, significantly improving efficiency when dealing + * with a large number of matchers. + */ +class BatchGrammarMatcher { + public: + BatchGrammarMatcher(std::variant max_threads = "auto"); + + /*! + \brief A batched version of FillNextTokenBitmask for better efficiency. + \param matchers The array of GrammarMatcher objects. + \param next_token_bitmask The pre-allocated DLTensor to store the result bitmasks. + \param indices The optional array of indices to specify which matcher corresponds to which slice + of the bitmask tensor. If not provided, all matchers will write to the corresponding + indices(matchers[i] to next_token_bitmask[i]). + \param debug_print Whether to print debug information. Default is false. + */ + void BatchFillNextTokenBitmask( + std::vector* matchers, + DLTensor* next_token_bitmask, + const std::optional>& indices = std::nullopt, + bool debug_print = false + ); + + /*! + * \brief A batched version of AcceptString for better efficiency. + * \param matchers The array of GrammarMatcher objects. + * \param input_strs The array of input strings to be accepted. + * \param debug_print Whether to print debug information. Default is false. + * \return A vector of bytes indicating whether each string is accepted. + */ + static std::vector BatchAcceptString( + std::vector* matchers, + const std::vector& input_strs, + bool debug_print = false + ); + + /*! + * \brief A batched version of AcceptToken for better efficiency. + * \param matchers The array of GrammarMatcher objects. + * \param token_ids The array of token ids to be accepted. + * \param debug_print Whether to print debug information. Default is false. + * \return A vector of bytes indicating whether each token is accepted. + */ + static std::vector BatchAcceptToken( + std::vector* matchers, + const std::vector& token_ids, + bool debug_print = false + ); + + /*! + * \brief A batched version of Rollback for better efficiency. + * \param matchers The array of GrammarMatcher objects. + * \param num_tokens The array of the number of tokens to rollback for each matcher. + */ + static void BatchRollback( + std::vector* matchers, const std::vector& num_tokens + ); + + XGRAMMAR_DEFINE_PIMPL_METHODS(BatchGrammarMatcher); +}; + +} // namespace xgrammar + +#endif // XGRAMMAR_MATCHER_H_ diff --git a/cactus-engine/libs/xgrammar/include/xgrammar/object.h b/cactus-engine/libs/xgrammar/include/xgrammar/object.h new file mode 100644 index 000000000..641b5bc52 --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/xgrammar/object.h @@ -0,0 +1,51 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/object.h + * \brief Utilities for creating objects. + */ + +#ifndef XGRAMMAR_OBJECT_H_ +#define XGRAMMAR_OBJECT_H_ + +#include // IWYU pragma: keep +#include // IWYU pragma: keep + +namespace xgrammar { + +/*! + * \brief A tag type for creating a null object. + */ +struct NullObj {}; + +/*! + * \brief This macro defines the methods for the PImpl classes. + * \details Many classes in xgrammar are PImpl classes. PImpl classes only stores a shared pointer + * to the implementation. This allows reference-counter-based memory management and efficient + * object copy and passing. We always expose PImpl classes to Python to control over object sharing + * and memory management. Note simple and critical classes should not be defined as PImpl classes, + * but as normal classes for better efficiency. + */ +#define XGRAMMAR_DEFINE_PIMPL_METHODS(TypeName) \ + public: \ + class Impl; \ + /* Construct a null object. Note operating on a null object will fail. */ \ + explicit TypeName(NullObj) : pimpl_(nullptr) {} \ + /* Construct object with a shared pointer to impl. */ \ + explicit TypeName(std::shared_ptr pimpl) : pimpl_(std::move(pimpl)) {} \ + TypeName(const TypeName& other) = default; \ + TypeName(TypeName&& other) noexcept = default; \ + TypeName& operator=(const TypeName& other) = default; \ + TypeName& operator=(TypeName&& other) noexcept = default; \ + bool IsNull() const { return pimpl_ == nullptr; } \ + /* Access the impl pointer. Useful in implementation. */ \ + Impl* ImplPtr() { return pimpl_.get(); } \ + const Impl* ImplPtr() const { return pimpl_.get(); } \ + Impl* operator->() { return pimpl_.get(); } \ + const Impl* operator->() const { return pimpl_.get(); } \ + \ + private: \ + std::shared_ptr pimpl_ + +} // namespace xgrammar + +#endif // XGRAMMAR_OBJECT_H_ diff --git a/cactus-engine/libs/xgrammar/include/xgrammar/tokenizer_info.h b/cactus-engine/libs/xgrammar/include/xgrammar/tokenizer_info.h new file mode 100644 index 000000000..6a8e67444 --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/xgrammar/tokenizer_info.h @@ -0,0 +1,86 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/tokenizer_info.h + * \brief The header for the tokenizer info. + */ + +#ifndef XGRAMMAR_TOKENIZER_INFO_H_ +#define XGRAMMAR_TOKENIZER_INFO_H_ + +#include + +#include +#include +#include +#include +#include + +#include "xgrammar/exception.h" + +namespace xgrammar { + +enum class VocabType : int { + RAW = 0, + BYTE_FALLBACK = 1, + BYTE_LEVEL = 2, +}; + +class TokenizerInfo { + public: + TokenizerInfo( + const std::vector& encoded_vocab, + VocabType vocab_type = VocabType::RAW, + std::optional vocab_size = std::nullopt, + std::optional> stop_token_ids = std::nullopt, + bool add_prefix_space = false + ); + + VocabType GetVocabType() const; + bool GetAddPrefixSpace() const; + int GetVocabSize() const; + const std::vector& GetDecodedVocab() const; + const std::vector& GetStopTokenIds() const; + const std::vector& GetSpecialTokenIds() const; + const std::vector>& GetSortedDecodedVocab() const; + const std::vector& GetTrieSubtreeNodesRange() const; + std::string DumpMetadata() const; + + /*! + * \brief Create a tokenizer info from a vocabulary and metadata. + * \param encoded_vocab The encoded vocabulary. + * \param metadata The metadata. + * \return The tokenizer info. + */ + static TokenizerInfo FromVocabAndMetadata( + const std::vector& encoded_vocab, const std::string& metadata + ); + + /*! + * \brief Detect the metadata from a Hugging Face backend string. + * \param backend_str The Hugging Face backend string. + * \return The metadata. + */ + static std::string DetectMetadataFromHF(const std::string& backend_str); + + /*! + * \brief Return the serialized JSON string of the tokenizer info. + * \return The serialized JSON string. + */ + std::string SerializeJSON() const; + + /*! + * \brief Deserialize a tokenizer info from a JSON string. + * \param json_string The JSON string to deserialize. + * \return If the deserialization is successful, return the tokenizer info. Otherwise, return a + * runtime error with the error message. + */ + static std::variant DeserializeJSON( + const std::string& json_string + ); + + XGRAMMAR_DEFINE_PIMPL_METHODS(TokenizerInfo); +}; + +} // namespace xgrammar + +#endif // XGRAMMAR_TOKENIZER_INFO_H_ diff --git a/cactus-engine/libs/xgrammar/include/xgrammar/xgrammar.h b/cactus-engine/libs/xgrammar/include/xgrammar/xgrammar.h new file mode 100644 index 000000000..8513b3a9c --- /dev/null +++ b/cactus-engine/libs/xgrammar/include/xgrammar/xgrammar.h @@ -0,0 +1,17 @@ +/*! + * Copyright (c) 2024 by Contributors + * \file xgrammar/xgrammar.h + * \brief The header for the support of grammar-guided generation. + */ + +#ifndef XGRAMMAR_XGRAMMAR_H_ +#define XGRAMMAR_XGRAMMAR_H_ + +#include +#include +#include +#include +#include +#include + +#endif // XGRAMMAR_XGRAMMAR_H_ diff --git a/cactus-engine/libs/xgrammar/ios/device/libxgrammar.a b/cactus-engine/libs/xgrammar/ios/device/libxgrammar.a new file mode 100644 index 000000000..3017e901d Binary files /dev/null and b/cactus-engine/libs/xgrammar/ios/device/libxgrammar.a differ diff --git a/cactus-engine/libs/xgrammar/ios/simulator/libxgrammar.a b/cactus-engine/libs/xgrammar/ios/simulator/libxgrammar.a new file mode 100644 index 000000000..c9f7a5257 Binary files /dev/null and b/cactus-engine/libs/xgrammar/ios/simulator/libxgrammar.a differ diff --git a/cactus-engine/libs/xgrammar/linux/aarch64/libxgrammar.a b/cactus-engine/libs/xgrammar/linux/aarch64/libxgrammar.a new file mode 100644 index 000000000..d1c8de538 Binary files /dev/null and b/cactus-engine/libs/xgrammar/linux/aarch64/libxgrammar.a differ diff --git a/cactus-engine/libs/xgrammar/macos/libxgrammar.a b/cactus-engine/libs/xgrammar/macos/libxgrammar.a new file mode 100644 index 000000000..f4f1d152b Binary files /dev/null and b/cactus-engine/libs/xgrammar/macos/libxgrammar.a differ diff --git a/cactus-engine/src/bpe.cpp b/cactus-engine/src/bpe.cpp index 58785735d..2764f7968 100644 --- a/cactus-engine/src/bpe.cpp +++ b/cactus-engine/src/bpe.cpp @@ -63,7 +63,6 @@ bool BPETokenizer::load_vocabulary_mmap(const std::string& vocab_file, const std std::string line; token_to_id_.clear(); id_to_token_.clear(); - special_tokens_.clear(); const bool use_id_tab_vocab = runtime_config_.vocab_format == TokenizerRuntimeConfig::VocabFormat::ID_TAB_TOKEN; @@ -156,7 +155,7 @@ bool BPETokenizer::load_vocabulary_mmap(const std::string& vocab_file, const std priority++; } } - + auto is_whitespace_only = [](const std::string& s) { if (s.empty()) return false; for (char c : s) { diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 0899634b2..4c525ddfd 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -9,6 +9,15 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include + #include "cactus_graph.h" class CactusGraph; @@ -127,7 +136,7 @@ struct Config { float rescale_factor = 0.00392156862745098f; float image_mean = 0.5f; float image_std = 0.5f; - + uint32_t downsample_factor = 2; uint32_t min_tiles = 2; uint32_t max_tiles = 10; @@ -260,6 +269,97 @@ struct Config { std::string to_json() const; }; +class Tokenizer; + +class GrammarVocabulary { +public: + static GrammarVocabulary from_model_dir(const std::string& model_dir); + static GrammarVocabulary from_tokenizer(const Tokenizer& tokenizer); + GrammarVocabulary(xgrammar::TokenizerInfo tokenizer_info); + + bool add_prefix_space() const; + size_t vocab_size() const; + const std::vector& stop_token_ids() const; + const xgrammar::TokenizerInfo& raw_value() const; + +private: + xgrammar::TokenizerInfo tokenizer_info; + std::vector stop_token_ids_; +}; + +class Grammar { +public: + Grammar(); + Grammar(xgrammar::Grammar raw_grammar); + ~Grammar() = default; + + static Grammar ebnf(const std::string& ebnf, const std::string& start_symbol = "root"); + static Grammar epsilon(); + static Grammar json(); + static Grammar universal(); + static Grammar json_schema( + const std::string& json_schema, + bool any_whitespace = true, + int indent = 2, + std::pair separators = {",", ":"}, + bool strict_mode = true, + int max_whitespace_count = -1 + ); + static Grammar regex(const std::string& regex); + static Grammar structural_tag(const std::string& structural_tag_json, const GrammarVocabulary* vocab = nullptr); + static Grammar unite(const std::vector& grammars); + static Grammar concatenate(const std::vector& grammars); + static Grammar optional(const Grammar& grammar); + static Grammar star(const Grammar& grammar); + static Grammar repeat(const Grammar& grammar, int count); + static Grammar repeat_range(const Grammar& grammar, int min_count, int max_count); + + bool is_empty() const; + std::string ebnf() const; + + const xgrammar::Grammar& raw_value() const; + +private: + xgrammar::Grammar grammar; +}; + +class GrammarMatcher; + +class GrammarEngine { +public: + GrammarEngine(const GrammarVocabulary& vocab); + ~GrammarEngine() = default; + + GrammarMatcher compile_matcher(const Grammar& grammar); + +private: + xgrammar::GrammarCompiler compiler; + xgrammar::TokenizerInfo tokenizer_info; +}; + +class GrammarMatcher { +public: + GrammarMatcher( + xgrammar::GrammarMatcher matcher, + xgrammar::CompiledGrammar compiled_grammar, + xgrammar::TokenizerInfo tokenizer_info + ); + ~GrammarMatcher() = default; + + Grammar grammar() const; + void rollback(int tokens = 1); + void reset(); + GrammarMatcher fork() const; + bool is_completed() const; + bool is_terminated() const; + bool accept(uint32_t token_id, bool log_rejection = false); + bool next_bitmask(std::vector& bitmask, size_t logits_buffer_size); + +private: + xgrammar::GrammarMatcher matcher; + xgrammar::CompiledGrammar compiled_grammar; + xgrammar::TokenizerInfo tokenizer_info; +}; struct MergeRule { @@ -267,7 +367,7 @@ struct MergeRule { std::string second; std::string merged; uint32_t priority; - + MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) : first(f), second(s), merged(m), priority(p) {} }; @@ -308,6 +408,12 @@ struct TokenizerRuntimeConfig { bool has_chat_template = false; }; +struct TokenizerJsonMetadata { + picojson::value decoder; + picojson::value normalizer; + picojson::value pre_tokenizer; +}; + TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); @@ -334,6 +440,8 @@ inline std::string extract_json_string(const std::string& json, size_t& pos) { class Tokenizer { public: + static std::unique_ptr from_model_dir(const std::string& model_dir); + virtual ~Tokenizer() = default; virtual std::vector encode(const std::string& text) const = 0; @@ -346,11 +454,13 @@ class Tokenizer { virtual uint32_t get_unk_token() const = 0; virtual uint32_t get_bos_token() const = 0; virtual uint32_t get_eos_token() const = 0; + virtual const std::vector& get_encoded_vocab() const = 0; virtual bool has_chat_template() const { return has_chat_template_; } std::string get_default_stop_sequence() const; + const TokenizerJsonMetadata& tokenizer_json_metadata() const { return tokenizer_json_metadata_; } virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; - + uint32_t get_image_token_id() const { return image_token_id_; } uint32_t get_fake_token_id() const { return fake_token_id_; } uint32_t get_global_img_token_id() const { return global_img_token_id_; } @@ -365,7 +475,7 @@ class Tokenizer { ModelVariant model_variant_ = ModelVariant::DEFAULT; bool has_chat_template_ = false; std::string chat_template_; - + uint32_t image_token_id_ = 396; uint32_t fake_token_id_ = 49189; uint32_t global_img_token_id_ = 49152; @@ -377,9 +487,12 @@ class Tokenizer { uint32_t vision_image_size_ = 768; size_t image_soft_token_count_ = 0; TokenizerRuntimeConfig runtime_config_; + TokenizerJsonMetadata tokenizer_json_metadata_; + std::unordered_map special_tokens_; void detect_model_type(const std::string& config_path); void load_chat_template(const std::string& template_file); + void load_tokenizer_json_data(const std::string& tokenizer_json_path); std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; @@ -400,12 +513,13 @@ class BPETokenizer : public Tokenizer { uint32_t get_unk_token() const override { return unk_token_id_; } uint32_t get_bos_token() const override { return bos_token_id_; } uint32_t get_eos_token() const override { return eos_token_id_; } + const std::vector& get_encoded_vocab() const override { return id_to_token_; } private: std::unordered_map token_to_id_; std::vector id_to_token_; std::vector merge_rules_; - std::unordered_map merge_map_; + std::unordered_map merge_map_; uint32_t vocab_size_; uint32_t unk_token_id_; @@ -420,20 +534,19 @@ class BPETokenizer : public Tokenizer { std::vector apply_bpe(const std::vector& tokens) const; std::pair find_best_merge_fast(const std::vector& tokens) const; - + std::string bytes_to_unicode(const std::string& text) const; std::string unicode_to_bytes(const std::string& text) const; std::vector byte_level_split(const std::string& text) const; std::vector utf8_split(const std::string& text) const; void cleanup_mmap(); - + private: mutable std::unordered_map byte_to_unicode_; mutable std::unordered_map unicode_to_byte_; void init_byte_mappings() const; - std::unordered_map special_tokens_; std::vector split_with_special_tokens(const std::string& text) const; void load_special_tokens(const std::string& config_file); }; @@ -452,6 +565,7 @@ class SPTokenizer : public Tokenizer { uint32_t get_unk_token() const override { return unk_token_id_; } uint32_t get_bos_token() const override { return bos_token_id_; } uint32_t get_eos_token() const override { return eos_token_id_; } + const std::vector& get_encoded_vocab() const override { return id_to_token_; } private: struct TrieNode { @@ -487,13 +601,10 @@ class SPTokenizer : public Tokenizer { void cleanup_mmap(); - std::unordered_map special_tokens_; std::vector split_with_special_tokens(const std::string& text) const; void load_special_tokens(const std::string& config_file); }; - - class ToolCallConstrainer { public: enum class State { @@ -613,7 +724,7 @@ class Model { const std::atomic* should_stop = nullptr); std::vector get_embeddings(const std::vector& tokens, bool pooled = true, - bool normalize = false, const std::string& profile_file = ""); + bool normalize = false, const std::string& profile_file = ""); std::vector get_image_embeddings(const std::string& image_path); @@ -628,7 +739,7 @@ class Model { } double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, - size_t context, size_t* tokens_scored); + size_t context, size_t* tokens_scored); void set_cache_window(size_t window_size, size_t sink_size = 4); size_t get_cache_size() const { return cache_total_seq_len_; } diff --git a/cactus-engine/src/grammar.cpp b/cactus-engine/src/grammar.cpp new file mode 100644 index 000000000..7d561107b --- /dev/null +++ b/cactus-engine/src/grammar.cpp @@ -0,0 +1,321 @@ +#include "engine.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { +namespace engine { + +namespace { + +std::string xgrammar_tokenizer_metadata(const Tokenizer& tokenizer) { + const auto& tokenizer_json_metadata = tokenizer.tokenizer_json_metadata(); + picojson::object minimal_tokenizer; + minimal_tokenizer["decoder"] = tokenizer_json_metadata.decoder; + minimal_tokenizer["normalizer"] = tokenizer_json_metadata.normalizer; + minimal_tokenizer["pre_tokenizer"] = tokenizer_json_metadata.pre_tokenizer; + + const std::string detected_metadata = + xgrammar::TokenizerInfo::DetectMetadataFromHF(picojson::value(minimal_tokenizer).serialize(false)); + + picojson::value metadata_value; + picojson::parse(metadata_value, detected_metadata); + + picojson::object metadata = metadata_value.get(); + metadata["vocab_size"] = picojson::value(static_cast(tokenizer.get_encoded_vocab().size())); + + picojson::array stop_token_ids { picojson::value(static_cast(tokenizer.get_eos_token())) }; + const auto default_stop_sequence = tokenizer.get_default_stop_sequence(); + if (!default_stop_sequence.empty()) { + const auto token = tokenizer.encode(default_stop_sequence)[0]; + stop_token_ids.emplace_back(static_cast(token)); + } + metadata["stop_token_ids"] = picojson::value(std::move(stop_token_ids)); + + return picojson::value(metadata).serialize(false); +} + +} // namespace + +GrammarVocabulary GrammarVocabulary::from_model_dir(const std::string& model_dir) { + auto tokenizer = Tokenizer::from_model_dir(model_dir); + if (!tokenizer) throw std::runtime_error("Unable to load tokenizer"); + return GrammarVocabulary::from_tokenizer(*tokenizer); +} + +GrammarVocabulary GrammarVocabulary::from_tokenizer(const Tokenizer& tokenizer) { + auto metadata = xgrammar_tokenizer_metadata(tokenizer); + return GrammarVocabulary( + xgrammar::TokenizerInfo::FromVocabAndMetadata(tokenizer.get_encoded_vocab(), metadata) + ); +} + +GrammarVocabulary::GrammarVocabulary(xgrammar::TokenizerInfo tokenizer_info) + : tokenizer_info(std::move(tokenizer_info)) { + const auto& stop_token_ids = this->tokenizer_info.GetStopTokenIds(); + stop_token_ids_.assign(stop_token_ids.begin(), stop_token_ids.end()); +} + +bool GrammarVocabulary::add_prefix_space() const { + return tokenizer_info.GetAddPrefixSpace(); +} + +size_t GrammarVocabulary::vocab_size() const { + return static_cast(tokenizer_info.GetVocabSize()); +} + +const std::vector& GrammarVocabulary::stop_token_ids() const { + return stop_token_ids_; +} + +const xgrammar::TokenizerInfo& GrammarVocabulary::raw_value() const { + return tokenizer_info; +} + +Grammar::Grammar() : grammar(xgrammar::NullObj{}) {} + +Grammar::Grammar(xgrammar::Grammar raw_grammar) + : grammar(std::move(raw_grammar)) {} + +Grammar Grammar::ebnf(const std::string& ebnf, const std::string& start_symbol) { + return Grammar(xgrammar::Grammar::FromEBNF(ebnf, start_symbol)); +} + +Grammar Grammar::epsilon() { + static auto grammar = Grammar::ebnf("root ::= \"\""); + return grammar; +} + +Grammar Grammar::json() { + return Grammar(xgrammar::Grammar::BuiltinJSONGrammar()); +} + +Grammar Grammar::universal() { + static auto grammar = Grammar::structural_tag(R"({ + "type": "structural_tag", + "format": { + "type": "any_text" + } + })"); + return grammar; +} + +Grammar Grammar::json_schema( + const std::string& json_schema, + bool any_whitespace, + int indent, + std::pair separators, + bool strict_mode, + int max_whitespace_count +) { + return Grammar(xgrammar::Grammar::FromJSONSchema( + json_schema, + any_whitespace, + indent, + separators, + strict_mode, + max_whitespace_count >= 0 ? std::optional(max_whitespace_count) : std::nullopt + )); +} + +Grammar Grammar::regex(const std::string& regex) { + return Grammar(xgrammar::Grammar::FromRegex(regex)); +} + +Grammar Grammar::structural_tag(const std::string& structural_tag_json, const GrammarVocabulary* vocab) { + auto result = xgrammar::Grammar::FromStructuralTag( + structural_tag_json, + vocab ? std::optional(vocab->raw_value()) : std::nullopt + ); + if (std::holds_alternative(result)) { + return Grammar(std::get(std::move(result))); + } + throw std::get(result); +} + +Grammar Grammar::unite(const std::vector& grammars) { + std::vector handles; + handles.reserve(grammars.size()); + for (const auto& grammar : grammars) { + if (grammar.is_empty()) continue; + handles.push_back(grammar.raw_value()); + } + return handles.empty() ? Grammar() : Grammar(xgrammar::Grammar::Union(handles)); +} + +Grammar Grammar::concatenate(const std::vector& grammars) { + std::vector handles; + handles.reserve(grammars.size()); + for (const auto& grammar : grammars) { + if (grammar.is_empty()) continue; + handles.push_back(grammar.raw_value()); + } + if (handles.empty()) return Grammar(); + return Grammar(xgrammar::Grammar::Concat(handles)); +} + +Grammar Grammar::optional(const Grammar& grammar) { + if (grammar.is_empty()) return grammar; + return Grammar::unite({Grammar::epsilon(), grammar}); +} + +Grammar Grammar::star(const Grammar& grammar) { + if (grammar.is_empty()) return grammar; + + picojson::object structural_tag; + structural_tag["type"] = picojson::value("structural_tag"); + + picojson::object star; + star["type"] = picojson::value("star"); + + picojson::object grammar_obj; + grammar_obj["type"] = picojson::value("grammar"); + grammar_obj["grammar"] = picojson::value(grammar.ebnf()); + + star["content"] = picojson::value(grammar_obj); + structural_tag["format"] = picojson::value(star); + return Grammar::structural_tag(picojson::value(structural_tag).serialize()); +} + +Grammar Grammar::repeat(const Grammar& grammar, int count) { + if (grammar.is_empty()) return grammar; + if (count == 0) return Grammar::epsilon(); + std::vector grammars(count, grammar); + return Grammar::concatenate(grammars); +} + +Grammar Grammar::repeat_range(const Grammar& grammar, int min_count, int max_count) { + if (grammar.is_empty()) return Grammar(); + if (max_count != -1 && max_count < min_count) { + throw std::runtime_error("repeat_range max_count must be >= min_count or size_t(-1)"); + } + if (min_count == 0 && max_count == 0) return Grammar::epsilon(); + if (min_count == max_count) return Grammar::repeat(grammar, min_count); + + picojson::value format_json; + + picojson::object structural_tag; + structural_tag["type"] = picojson::value("structural_tag"); + + picojson::object repeat; + repeat["type"] = picojson::value("repeat"); + repeat["min"] = picojson::value(static_cast(min_count)); + repeat["max"] = picojson::value(static_cast(max_count)); + + picojson::object grammar_obj; + grammar_obj["type"] = picojson::value("grammar"); + grammar_obj["grammar"] = picojson::value(grammar.ebnf()); + + repeat["content"] = picojson::value(grammar_obj); + + structural_tag["format"] = picojson::value(repeat); + return Grammar::structural_tag(picojson::value(structural_tag).serialize()); +} + +bool Grammar::is_empty() const { + return grammar.IsNull(); +} + +std::string Grammar::ebnf() const { + if (is_empty()) { + throw std::runtime_error("Cannot get EBNF for empty grammar"); + } + return grammar.ToString(); +} + +const xgrammar::Grammar& Grammar::raw_value() const { + return grammar; +} + +GrammarEngine::GrammarEngine(const GrammarVocabulary& vocab) + : compiler(nullptr), tokenizer_info(std::move(vocab.raw_value())) { + compiler = xgrammar::GrammarCompiler( + tokenizer_info, + static_cast(std::max(1u, std::thread::hardware_concurrency())) + ); +} + +GrammarMatcher GrammarEngine::compile_matcher(const Grammar& grammar) { + if (grammar.is_empty()) { + throw std::runtime_error("Cannot create GrammarMatcher with empty grammar"); + } + + auto compiled_grammar = compiler.CompileGrammar(grammar.raw_value()); + return GrammarMatcher( + xgrammar::GrammarMatcher(compiled_grammar), + std::move(compiled_grammar), + tokenizer_info + ); +} + +GrammarMatcher::GrammarMatcher( + xgrammar::GrammarMatcher matcher, + xgrammar::CompiledGrammar compiled_grammar, + xgrammar::TokenizerInfo tokenizer_info +) + : matcher(std::move(matcher)), + compiled_grammar(std::move(compiled_grammar)), + tokenizer_info(std::move(tokenizer_info)) {} + +Grammar GrammarMatcher::grammar() const { + return Grammar(compiled_grammar.GetGrammar()); +} + +void GrammarMatcher::rollback(int tokens) { + matcher.Rollback(tokens); +} + +void GrammarMatcher::reset() { + matcher.Reset(); +} + +GrammarMatcher GrammarMatcher::fork() const { + return GrammarMatcher(matcher.Fork(), compiled_grammar, tokenizer_info); +} + +bool GrammarMatcher::is_completed() const { + return matcher.IsCompleted(); +} + +bool GrammarMatcher::is_terminated() const { + return matcher.IsTerminated(); +} + +bool GrammarMatcher::accept(uint32_t token_id, bool log_rejection) { + const bool accepted = matcher.AcceptToken(token_id); + if (!accepted && log_rejection) { + CACTUS_LOG_WARN("grammar matcher", "Token id: " << token_id << " was not accepted by grammar matcher."); + } + return accepted; +} + +bool GrammarMatcher::next_bitmask(std::vector& bitmask, size_t logits_buffer_size) { + const size_t vocab_bitmask_size = xgrammar::GetBitmaskSize(tokenizer_info.GetVocabSize()); + const size_t logits_bitmask_size = xgrammar::GetBitmaskSize(logits_buffer_size); + bitmask.assign(std::max(vocab_bitmask_size, logits_bitmask_size), 0); + + int64_t bitmask_shape[1]; + int64_t bitmask_strides[1]; + bitmask_shape[0] = vocab_bitmask_size; + bitmask_strides[0] = 1; + + DLTensor bitmask_tensor; + bitmask_tensor.data = bitmask.data(); + bitmask_tensor.device = DLDevice{kDLCPU, 0}; + bitmask_tensor.ndim = 1; + bitmask_tensor.dtype = xgrammar::GetBitmaskDLType(); + bitmask_tensor.shape = bitmask_shape; + bitmask_tensor.strides = bitmask_strides; + bitmask_tensor.byte_offset = 0; + + return matcher.FillNextTokenBitmask(&bitmask_tensor, 0); +} + +} // namespace engine +} // namespace cactus diff --git a/cactus-engine/src/grammar_ffi.cpp b/cactus-engine/src/grammar_ffi.cpp new file mode 100644 index 000000000..363d8183b --- /dev/null +++ b/cactus-engine/src/grammar_ffi.cpp @@ -0,0 +1,449 @@ +#include "../cactus_engine.h" +#include "utils.h" +#include "engine.h" + +#include +#include +#include +#include + +using namespace cactus::ffi; +using namespace cactus::engine; + +namespace cactus { +namespace ffi { + +struct CactusGrammarHandle { + std::unique_ptr grammar; +}; + +struct CactusGrammarVocabularyHandle { + std::unique_ptr vocabulary; +}; + +struct CactusGrammarEngineHandle { + std::unique_ptr engine; +}; + +struct CactusGrammarMatcherHandle { + std::unique_ptr matcher; +}; + +} // namespace ffi +} // namespace cactus + +namespace { + +static cactus_grammar_t handle_exception(const char* operation, const std::string& error) { + last_error_message = std::string(operation) + ": " + error; + CACTUS_LOG_ERROR("grammar", last_error_message); + return nullptr; +} + +static int handle_int_exception(const char* operation, const std::string& error) { + last_error_message = std::string(operation) + ": " + error; + CACTUS_LOG_ERROR("grammar", last_error_message); + return -1; +} + +template +static Handle* require_handle(const char* operation, RawHandle handle, const char* label) { + if (!handle) { + handle_exception(operation, std::string(label) + " is null"); + return nullptr; + } + return static_cast(handle); +} + +static std::vector collect_grammars(cactus_grammar_t* grammars, size_t num_grammars) { + std::vector collected; + collected.reserve(num_grammars); + for (size_t i = 0; i < num_grammars; ++i) { + if (!grammars || !grammars[i]) continue; + collected.push_back(*static_cast(grammars[i])->grammar); + } + return collected; +} + +static CactusGrammarVocabularyHandle* require_vocabulary_handle( + const char* operation, + cactus_grammar_vocabulary_t vocabulary +) { + return require_handle(operation, vocabulary, "grammar vocabulary"); +} + +static CactusGrammarEngineHandle* require_engine_handle( + const char* operation, + cactus_grammar_engine_t engine +) { + return require_handle(operation, engine, "grammar engine"); +} + +static CactusGrammarMatcherHandle* require_matcher_handle( + const char* operation, + cactus_grammar_matcher_t matcher +) { + return require_handle(operation, matcher, "grammar matcher"); +} + +static CactusGrammarHandle* require_grammar_handle(const char* operation, cactus_grammar_t grammar) { + return require_handle(operation, grammar, "grammar"); +} + +template +static cactus_grammar_t make_grammar(const char* operation, Factory&& factory) { + try { + return new CactusGrammarHandle{ + std::make_unique(std::forward(factory)()) + }; + } catch (const std::exception& e) { + return handle_exception(operation, e.what()); + } +} + +} // anonymous namespace + +extern "C" { + +cactus_grammar_json_schema_options_t cactus_grammar_json_schema_default_options() { + return { true, 2, {",", ":"}, true, 1, }; +} + +cactus_grammar_vocabulary_t cactus_grammar_vocabulary_init(const char* model_path) { + if (!model_path) return handle_exception(__func__, "model path is null"); + + try { + const auto vocabulary = GrammarVocabulary::from_model_dir(model_path); + return new CactusGrammarVocabularyHandle{ + std::make_unique(vocabulary) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +cactus_grammar_vocabulary_t cactus_grammar_vocabulary_init_from_model(cactus_model_t model) { + if (!model) return handle_exception(__func__, "model is null"); + + try { + auto handle = static_cast(model); + auto tokenizer = handle->model ? handle->model->get_tokenizer() : nullptr; + if (!tokenizer) return handle_exception(__func__, "model tokenizer is null"); + const auto vocabulary = GrammarVocabulary::from_tokenizer(*tokenizer); + return new CactusGrammarVocabularyHandle{ + std::make_unique(vocabulary) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +int cactus_grammar_vocabulary_get_add_prefix_space(cactus_grammar_vocabulary_t vocabulary) { + auto* handle = require_vocabulary_handle(__func__, vocabulary); + if (!handle) return -1; + + return handle->vocabulary->add_prefix_space() ? 1 : 0; +} + +size_t cactus_grammar_vocabulary_get_size(cactus_grammar_vocabulary_t vocabulary) { + auto* handle = require_vocabulary_handle(__func__, vocabulary); + if (!handle) return 0; + + return handle->vocabulary->vocab_size(); +} + +int cactus_grammar_vocabulary_get_stop_token_ids( + cactus_grammar_vocabulary_t vocabulary, + uint32_t* buffer, + size_t buffer_size, + size_t* out_token_count +) { + auto* handle = require_vocabulary_handle(__func__, vocabulary); + if (!handle) return -1; + if (!out_token_count) return handle_int_exception(__func__, "out_token_count is null"); + + const auto& stop_token_ids = handle->vocabulary->stop_token_ids(); + *out_token_count = stop_token_ids.size(); + if (!buffer) return handle_int_exception(__func__, "buffer is null"); + if (buffer_size < stop_token_ids.size()) { + return handle_int_exception(__func__, "buffer too small"); + } + + std::copy(stop_token_ids.begin(), stop_token_ids.end(), buffer); + return 0; +} + +void cactus_grammar_vocabulary_destroy(cactus_grammar_vocabulary_t vocabulary) { + if (vocabulary) delete static_cast(vocabulary); +} + +cactus_grammar_t cactus_grammar_init_ebnf(const char* ebnf, const char* start_symbol) { + if (!ebnf) return handle_exception(__func__, "ebnf is null"); + + return make_grammar(__func__, [&] { + return Grammar::ebnf(ebnf, start_symbol ? start_symbol : "root"); + }); +} + +cactus_grammar_t cactus_grammar_init_json() { + return make_grammar(__func__, [] { + return Grammar::json(); + }); +} + +cactus_grammar_t cactus_grammar_init_empty() { + return make_grammar(__func__, [] { + return Grammar(); + }); +} + +cactus_grammar_t cactus_grammar_init_epsilon() { + return make_grammar(__func__, [] { + return Grammar::epsilon(); + }); +} + +cactus_grammar_t cactus_grammar_init_universal() { + return make_grammar(__func__, [] { + return Grammar::universal(); + }); +} + +cactus_grammar_t cactus_grammar_init_json_schema( + const char* json_schema, + cactus_grammar_json_schema_options_t options +) { + if (!json_schema) return handle_exception(__func__, "json_schema is null"); + + if ((!options.any_whitespace) && (!options.separators[0] || !options.separators[1])) { + return handle_exception( + __func__, + "json schema separators must have 2 strings" + ); + } + + return make_grammar(__func__, [&] { + return Grammar::json_schema( + json_schema, + options.any_whitespace, + options.indent, + { + options.separators[0] ? options.separators[0] : ",", + options.separators[1] ? options.separators[1] : ":" + }, + options.strict_mode, + options.max_whitespace_count + ); + }); +} + +cactus_grammar_t cactus_grammar_init_regex(const char* regex) { + if (!regex) return handle_exception(__func__, "regex is null"); + + return make_grammar(__func__, [&] { + return Grammar::regex(regex); + }); +} + +cactus_grammar_t cactus_grammar_init_structural_tag( + const char* structural_tag_json, + cactus_grammar_vocabulary_t vocabulary +) { + if (!structural_tag_json) return handle_exception(__func__, "structural_tag_json is null"); + + return make_grammar(__func__, [&] { + auto* handle = static_cast(vocabulary); + return Grammar::structural_tag( + structural_tag_json, + handle ? handle->vocabulary.get() : nullptr + ); + }); +} + +cactus_grammar_t cactus_grammar_union(cactus_grammar_t* grammars, size_t num_grammars) { + return make_grammar(__func__, [&] { + return Grammar::unite(collect_grammars(grammars, num_grammars)); + }); +} + +cactus_grammar_t cactus_grammar_concatenate(cactus_grammar_t* grammars, size_t num_grammars) { + return make_grammar(__func__, [&] { + return Grammar::concatenate(collect_grammars(grammars, num_grammars)); + }); +} + +cactus_grammar_t cactus_grammar_optional(cactus_grammar_t grammar) { + auto* handle = require_grammar_handle(__func__, grammar); + if (!handle) return nullptr; + + return make_grammar(__func__, [&] { + return Grammar::optional(*handle->grammar); + }); +} + +cactus_grammar_t cactus_grammar_star(cactus_grammar_t grammar) { + auto* handle = require_grammar_handle(__func__, grammar); + if (!handle) return nullptr; + + return make_grammar(__func__, [&] { + return Grammar::star(*handle->grammar); + }); +} + +cactus_grammar_t cactus_grammar_repeat(cactus_grammar_t grammar, int count) { + auto* handle = require_grammar_handle(__func__, grammar); + if (!handle) return nullptr; + + return make_grammar(__func__, [&] { + return Grammar::repeat(*handle->grammar, count); + }); +} + +cactus_grammar_t cactus_grammar_repeat_range(cactus_grammar_t grammar, int min_count, int max_count) { + auto* handle = require_grammar_handle(__func__, grammar); + if (!handle) return nullptr; + + return make_grammar(__func__, [&] { + return Grammar::repeat_range(*handle->grammar, min_count, max_count); + }); +} + +int cactus_grammar_get_ebnf(cactus_grammar_t grammar, char* buffer, size_t buffer_size) { + if (!grammar) return handle_int_exception(__func__, "grammar is null"); + if (!buffer || buffer_size == 0) { + return handle_int_exception(__func__, "buffer is null or buffer_size is 0"); + } + + try { + std::string ebnf = static_cast(grammar)->grammar->ebnf(); + if (ebnf.size() >= buffer_size) return handle_int_exception(__func__, "buffer too small"); + std::strcpy(buffer, ebnf.c_str()); + return 0; + } catch (const std::exception& e) { + return handle_int_exception(__func__, e.what()); + } +} + +int cactus_grammar_is_empty(cactus_grammar_t grammar) { + if (!grammar) return handle_int_exception(__func__, "grammar is null"); + return static_cast(grammar)->grammar->is_empty() ? 1 : 0; +} + +void cactus_grammar_destroy(cactus_grammar_t grammar) { + if (grammar) delete static_cast(grammar); +} + +cactus_grammar_engine_t cactus_grammar_engine_init(cactus_grammar_vocabulary_t vocabulary) { + auto* handle = require_vocabulary_handle(__func__, vocabulary); + if (!handle) return nullptr; + + try { + return new CactusGrammarEngineHandle{ + std::make_unique(*handle->vocabulary) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +void cactus_grammar_engine_destroy(cactus_grammar_engine_t engine) { + if (engine) delete static_cast(engine); +} + +cactus_grammar_matcher_t cactus_grammar_engine_compile_matcher( + cactus_grammar_engine_t engine, + cactus_grammar_t grammar +) { + auto* engine_handle = require_engine_handle(__func__, engine); + if (!engine_handle) return nullptr; + auto* grammar_handle = require_grammar_handle(__func__, grammar); + if (!grammar_handle) return nullptr; + + try { + return new CactusGrammarMatcherHandle{ + std::make_unique( + engine_handle->engine->compile_matcher(*grammar_handle->grammar) + ) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +void cactus_grammar_matcher_destroy(cactus_grammar_matcher_t matcher) { + if (matcher) delete static_cast(matcher); +} + +void cactus_grammar_matcher_reset(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return; + handle->matcher->reset(); +} + +void cactus_grammar_matcher_rollback(cactus_grammar_matcher_t matcher, int tokens) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return; + handle->matcher->rollback(tokens); +} + +cactus_grammar_matcher_t cactus_grammar_matcher_fork(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return nullptr; + + try { + return new CactusGrammarMatcherHandle{ + std::make_unique(handle->matcher->fork()) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +cactus_grammar_t cactus_grammar_matcher_get_grammar(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return nullptr; + + try { + return new CactusGrammarHandle{ + std::make_unique(handle->matcher->grammar()) + }; + } catch (const std::exception& e) { + return handle_exception(__func__, e.what()); + } +} + +int cactus_grammar_matcher_is_completed(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + return handle ? (handle->matcher->is_completed() ? 1 : 0) : -1; +} + +int cactus_grammar_matcher_is_terminated(cactus_grammar_matcher_t matcher) { + auto* handle = require_matcher_handle(__func__, matcher); + return handle ? (handle->matcher->is_terminated() ? 1 : 0) : -1; +} + +int cactus_grammar_matcher_accept(cactus_grammar_matcher_t matcher, uint32_t token_id) { + auto* handle = require_matcher_handle(__func__, matcher); + return handle ? (handle->matcher->accept(token_id) ? 1 : 0) : -1; +} + +int cactus_grammar_matcher_next_bitmask( + cactus_grammar_matcher_t matcher, + int32_t* bitmask, + size_t logits_buffer_size +) { + auto* handle = require_matcher_handle(__func__, matcher); + if (!handle) return -1; + if (!bitmask) return handle_int_exception(__func__, "bitmask is null"); + + try { + std::vector next_bitmask; + const bool should_apply = handle->matcher->next_bitmask(next_bitmask, logits_buffer_size); + std::copy(next_bitmask.begin(), next_bitmask.end(), bitmask); + return should_apply ? 1 : 0; + } catch (const std::exception& e) { + return handle_int_exception(__func__, e.what()); + } +} + +} diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index fec6f94f5..22c3b5cc0 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2,9 +2,6 @@ #include "cactus_graph.h" #include "cactus_kernels.h" -#define PICOJSON_USE_INT64 -#include "picojson.h" - #include #include #include diff --git a/cactus-engine/src/tokenizer.cpp b/cactus-engine/src/tokenizer.cpp index 9cd05edaa..679dcaf67 100644 --- a/cactus-engine/src/tokenizer.cpp +++ b/cactus-engine/src/tokenizer.cpp @@ -11,6 +11,23 @@ namespace engine { namespace { +bool has_merges(const std::string& merges_file) { + std::ifstream merges_check(merges_file); + if (!merges_check.is_open()) { + return false; + } + + std::string line; + int line_count = 0; + while (std::getline(merges_check, line) && line_count < 10) { + if (!line.empty() && line[0] != '#') { + return true; + } + ++line_count; + } + return false; +} + std::string format_tool_call_for_prompt(const std::string& name, const std::string& arguments, bool gemma4) { if (gemma4) { return "\n<|tool_call>\ncall:" + name + "(" + arguments + ")\n\n"; @@ -153,6 +170,59 @@ bool parse_added_token_entry(const std::string& object, std::string& token_conte return true; } +void parse_tokenizer_json_data( + const std::string& content, + TokenizerJsonMetadata& metadata, + std::unordered_map& special_tokens +) { + special_tokens.clear(); + + picojson::value root; + const std::string err = picojson::parse(root, content); + if (!err.empty() || !root.is()) { + return; + } + + const auto& tokenizer_obj = root.get(); + auto load_tokenizer_obj_field = [&](const char* key, picojson::value& out) { + auto it = tokenizer_obj.find(key); + if (it != tokenizer_obj.end()) { + out = it->second; + } + }; + + load_tokenizer_obj_field("decoder", metadata.decoder); + load_tokenizer_obj_field("normalizer", metadata.normalizer); + load_tokenizer_obj_field("pre_tokenizer", metadata.pre_tokenizer); + + auto added_tokens_it = tokenizer_obj.find("added_tokens"); + if (added_tokens_it == tokenizer_obj.end() || !added_tokens_it->second.is()) { + return; + } + + for (const auto& added_token : added_tokens_it->second.get()) { + if (!added_token.is()) continue; + const auto& added_token_obj = added_token.get(); + + auto special_it = added_token_obj.find("special"); + if (special_it == added_token_obj.end() || !special_it->second.is() || !special_it->second.get()) { + continue; + } + + auto content_it = added_token_obj.find("content"); + auto id_it = added_token_obj.find("id"); + if (content_it == added_token_obj.end() || id_it == added_token_obj.end()) { + continue; + } + if (!content_it->second.is() || !id_it->second.is()) { + continue; + } + + special_tokens[content_it->second.get()] = + static_cast(id_it->second.get()); + } +} + void load_tokenizer_json_added_special_tokens( const std::string& tokenizer_json_path, std::unordered_map& special_tokens) { @@ -237,6 +307,40 @@ TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_f return config; } +std::unique_ptr Tokenizer::from_model_dir(const std::string& model_dir) { + const std::string vocab_file = model_dir + "/vocab.txt"; + const std::string merges_file = model_dir + "/merges.txt"; + const std::string config_file = model_dir + "/tokenizer_config.txt"; + + TokenizerRuntimeConfig runtime_config = load_tokenizer_runtime_config(config_file); + + std::unique_ptr tokenizer; + if (runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::BPE || + (runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::UNKNOWN && has_merges(merges_file))) { + tokenizer = std::make_unique(); + } else { + tokenizer = std::make_unique(); + } + + tokenizer->load_tokenizer_json_data(model_dir + "/tokenizer.json"); + + if (!tokenizer->load_vocabulary_with_config(vocab_file, merges_file, config_file)) { + return nullptr; + } + return tokenizer; +} + +void Tokenizer::load_tokenizer_json_data(const std::string& tokenizer_json_path) { + tokenizer_json_metadata_ = {}; + special_tokens_.clear(); + + std::ifstream file(tokenizer_json_path); + if (!file.is_open()) return; + + const std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + parse_tokenizer_json_data(content, tokenizer_json_metadata_, special_tokens_); +} + void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens) { special_tokens.clear(); diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 1e18fa899..af4d8ce2e 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -1,6 +1,7 @@ #ifndef CACTUS_UTILS_H #define CACTUS_UTILS_H +#include "../cactus_engine.h" #include "engine.h" #include "cactus_kernels.h" #include @@ -43,7 +44,7 @@ inline size_t get_memory_footprint_bytes() { PROCESS_MEMORY_COUNTERS_EX pmc; if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) return pmc.PrivateUsage; - + #elif defined(__linux__) || defined(__ANDROID__) std::ifstream statm("/proc/self/statm"); if (statm.is_open()) { @@ -837,16 +838,16 @@ inline std::vector parse_messages_json(const std::s std::vector messages; out_image_paths.clear(); if (out_audio_paths) out_audio_paths->clear(); - + size_t pos = json.find('['); if (pos == std::string::npos) { throw std::runtime_error("Invalid JSON: expected array"); } - + pos = json.find('{', pos); while (pos != std::string::npos) { cactus::engine::ChatMessage msg; - + size_t obj_start = pos; int brace_count = 1; size_t obj_end = obj_start + 1; @@ -858,25 +859,25 @@ inline std::vector parse_messages_json(const std::s size_t role_pos = json.find("\"role\"", pos); if (role_pos == std::string::npos || role_pos >= obj_end) break; - + size_t role_start = json.find('"', role_pos + 6) + 1; size_t role_end = json.find('"', role_start); msg.role = json.substr(role_start, role_end - role_start); - + size_t content_pos = json.find("\"content\"", role_end); if (content_pos != std::string::npos && content_pos < obj_end) { size_t content_start = json.find('"', content_pos + 9) + 1; size_t content_end = content_start; - + while (content_end < json.length()) { content_end = json.find('"', content_end); if (content_end == std::string::npos) break; if (json[content_end - 1] != '\\') break; content_end++; } - + msg.content = json.substr(content_start, content_end - content_start); - + size_t escape_pos = 0; while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { msg.content.replace(escape_pos, 2, "\n"); @@ -888,7 +889,7 @@ inline std::vector parse_messages_json(const std::s escape_pos += 1; } } - + auto parse_path_array = [&](const char* key, std::vector& dest, std::vector* out_paths) { size_t key_pos = json.find(key, pos); @@ -999,31 +1000,31 @@ inline std::vector parse_messages_json(const std::s inline std::vector parse_tools_json(const std::string& json) { std::vector tools; - + if (json.empty()) return tools; - + size_t pos = json.find('['); if (pos == std::string::npos) return tools; - + pos = json.find("\"function\"", pos); while (pos != std::string::npos) { ToolFunction tool; size_t next_search = pos + 1; - + size_t name_pos = json.find("\"name\"", pos); if (name_pos != std::string::npos) { size_t name_start = json.find('"', name_pos + 6) + 1; size_t name_end = json.find('"', name_start); tool.name = json.substr(name_start, name_end - name_start); } - + size_t desc_pos = json.find("\"description\"", pos); if (desc_pos != std::string::npos) { size_t desc_start = json.find('"', desc_pos + 13) + 1; size_t desc_end = json.find('"', desc_start); tool.description = json.substr(desc_start, desc_end - desc_start); } - + size_t params_pos = json.find("\"parameters\"", pos); if (params_pos != std::string::npos) { size_t params_start = json.find('{', params_pos); @@ -1740,6 +1741,8 @@ inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buf return out; } +cactus_grammar_vocabulary_t make_grammar_vocabulary_from_tokenizer(const cactus::engine::Tokenizer& tokenizer); + } // namespace ffi } // namespace cactus diff --git a/cactus-engine/test.sh b/cactus-engine/test.sh index 50954e0e6..a8f3cc251 100755 --- a/cactus-engine/test.sh +++ b/cactus-engine/test.sh @@ -5,10 +5,11 @@ cd "$(dirname "$0")" PROJECT_ROOT="$(pwd)/.." ASSETS_DIR="$(pwd)/tests/assets" +ONLY_TEST="" +NO_REBUILD=0 IOS_MODE=false ANDROID_MODE=false -ONLY_EXEC="" MODEL_ARG="" while [[ $# -gt 0 ]]; do @@ -23,9 +24,13 @@ while [[ $# -gt 0 ]]; do ;; --only) [ -z "${2:-}" ] && echo "Error: --only requires an argument" && exit 1 - ONLY_EXEC="$2" + ONLY_TEST="$2" shift 2 ;; + --no-rebuild) + NO_REBUILD=1 + shift + ;; --model) [ -z "${2:-}" ] && echo "Error: --model requires an argument" && exit 1 MODEL_ARG="$2" @@ -40,12 +45,12 @@ done MODEL_DIR="${CACTUS_TEST_MODEL:-${MODEL_ARG:-$PROJECT_ROOT/weights/gemma-4-e2b-it}}" if [ "$IOS_MODE" = true ]; then - export CACTUS_TEST_ONLY="$ONLY_EXEC" + export CACTUS_TEST_ONLY="$ONLY_TEST" exec "$(pwd)/tests/ios/run.sh" "$MODEL_DIR" fi if [ "$ANDROID_MODE" = true ]; then - export CACTUS_TEST_ONLY="$ONLY_EXEC" + export CACTUS_TEST_ONLY="$ONLY_TEST" exec "$(pwd)/tests/android/run.sh" "$MODEL_DIR" fi @@ -58,19 +63,23 @@ fi echo "Building and testing cactus-engine..." echo "Model: $MODEL_DIR" -cd "$PROJECT_ROOT/cactus" -rm -rf build -mkdir -p build -cd build -cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 -make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) +if [ "$NO_REBUILD" -eq 0 ]; then + cd "$PROJECT_ROOT/cactus" + rm -rf build + mkdir -p build + cd build + cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 + make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) -cd "$PROJECT_ROOT/cactus-engine/tests" -rm -rf build -mkdir -p build -cd build -cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 -make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) + cd "$PROJECT_ROOT/cactus-engine/tests" + rm -rf build + mkdir -p build + cd build + cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 + make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) +else + cd "$PROJECT_ROOT/cactus-engine/tests/build" +fi echo "" export CACTUS_TEST_MODEL="$MODEL_DIR" @@ -78,15 +87,13 @@ export CACTUS_TEST_ASSETS="$ASSETS_DIR" export CACTUS_INDEX_PATH="$ASSETS_DIR" FAILED=0 - -if [ -n "$ONLY_EXEC" ]; then - target="./test_$ONLY_EXEC" - if [ -x "$target" ]; then - ./"$target" || FAILED=1 - else - echo "Test not found: $target" - FAILED=1 +if [ -n "$ONLY_TEST" ]; then + TEST_BIN="test_${ONLY_TEST}" + if [ ! -x "$TEST_BIN" ]; then + echo "Test binary not found: $TEST_BIN" + exit 1 fi + ./"$TEST_BIN" || FAILED=1 else for test_bin in test_*; do [ -x "$test_bin" ] && ./"$test_bin" || FAILED=1 diff --git a/cactus-engine/tests/CMakeLists.txt b/cactus-engine/tests/CMakeLists.txt index ebd22f639..6cea66a87 100644 --- a/cactus-engine/tests/CMakeLists.txt +++ b/cactus-engine/tests/CMakeLists.txt @@ -6,6 +6,12 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) set(CACTUS_ENGINE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..) set(CACTUS_PROJECT_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..) +set(CACTUS_XGRAMMAR_INCLUDE_DIR ${CACTUS_ENGINE_ROOT}/libs/xgrammar/include) +if(APPLE) + set(CACTUS_XGRAMMAR_LIBRARY ${CACTUS_ENGINE_ROOT}/libs/xgrammar/macos/libxgrammar.a) +else() + set(CACTUS_XGRAMMAR_LIBRARY ${CACTUS_ENGINE_ROOT}/libs/xgrammar/linux/aarch64/libxgrammar.a) +endif() if(APPLE) set(CMAKE_OSX_ARCHITECTURES "arm64") @@ -38,9 +44,10 @@ else() endif() function(cactus_link_target target_name) - target_link_libraries(${target_name} PRIVATE ${CACTUS_LIB}) + target_link_libraries(${target_name} PRIVATE ${CACTUS_LIB} ${CACTUS_XGRAMMAR_LIBRARY}) target_include_directories(${target_name} PRIVATE ${CACTUS_ENGINE_ROOT} + ${CACTUS_XGRAMMAR_INCLUDE_DIR} ${CACTUS_PROJECT_ROOT}/cactus-graph ${CACTUS_PROJECT_ROOT}/cactus-kernels ${CACTUS_PROJECT_ROOT}/cactus-kernels/src diff --git a/cactus-engine/tests/test_grammar.cpp b/cactus-engine/tests/test_grammar.cpp new file mode 100644 index 000000000..18a5c3b92 --- /dev/null +++ b/cactus-engine/tests/test_grammar.cpp @@ -0,0 +1,583 @@ +#include "test_utils.h" + +#include "../src/utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace cactus::engine; +using namespace cactus::ffi; + +namespace { + +using GrammarHandle = std::unique_ptr; +using GrammarVocabularyHandle = std::unique_ptr; +using GrammarEngineHandle = std::unique_ptr; +using GrammarMatcherHandle = std::unique_ptr; + +static GrammarHandle grammar_handle(cactus_grammar_t grammar) { + if (!grammar) { + throw std::runtime_error(cactus_get_last_error()); + } + return GrammarHandle(grammar, &cactus_grammar_destroy); +} + +static GrammarVocabularyHandle vocab_handle(cactus_grammar_vocabulary_t vocabulary) { + return GrammarVocabularyHandle(vocabulary, &cactus_grammar_vocabulary_destroy); +} + +static GrammarEngineHandle engine_handle(cactus_grammar_engine_t engine) { + return GrammarEngineHandle(engine, &cactus_grammar_engine_destroy); +} + +static GrammarMatcherHandle matcher_handle(cactus_grammar_matcher_t matcher) { + if (!matcher) { + throw std::runtime_error(cactus_get_last_error()); + } + return GrammarMatcherHandle(matcher, &cactus_grammar_matcher_destroy); +} + +static const std::string require_model_path() { + const char* model_path = std::getenv("CACTUS_TEST_MODEL"); + if (!model_path) { + throw std::runtime_error("CACTUS_TEST_MODEL is not set"); + } + return model_path; +} + +static std::unique_ptr create_test_tokenizer() { + auto tokenizer = Tokenizer::from_model_dir(require_model_path()); + if (!tokenizer) { + throw std::runtime_error("Failed to load tokenizer from test model files"); + } + return tokenizer; +} + +static cactus_grammar_t json_schema_grammar(const std::string& json_schema) { + return cactus_grammar_init_json_schema( + json_schema.c_str(), + cactus_grammar_json_schema_default_options() + ); +} + +static cactus_grammar_matcher_t make_matcher(cactus_grammar_t grammar, cactus_grammar_engine_t engine) { + auto matcher = cactus_grammar_engine_compile_matcher(engine, grammar); + if (!matcher) { + throw std::runtime_error(cactus_get_last_error()); + } + return matcher; +} + +struct GrammarFixture { + std::unique_ptr tokenizer; + GrammarVocabularyHandle vocab; + GrammarEngineHandle engine; + bool add_prefix_space = false; + size_t vocab_size = 0; + std::vector stop_token_ids; + + GrammarFixture() + : tokenizer(create_test_tokenizer()), + vocab(vocab_handle(cactus_grammar_vocabulary_init(require_model_path().c_str()))), + engine(engine_handle(cactus_grammar_engine_init(vocab.get()))) { + if (!engine) { + throw std::runtime_error(cactus_get_last_error()); + } + add_prefix_space = cactus_grammar_vocabulary_get_add_prefix_space(vocab.get()); + vocab_size = cactus_grammar_vocabulary_get_size(vocab.get()); + if (vocab_size == 0) { + throw std::runtime_error(cactus_get_last_error()); + } + std::vector actual_stop_token_ids(16); + size_t out_token_count = 0; + auto result = cactus_grammar_vocabulary_get_stop_token_ids( + vocab.get(), + actual_stop_token_ids.data(), + actual_stop_token_ids.size(), + &out_token_count + ); + if (result != 0) throw std::runtime_error(cactus_get_last_error()); + actual_stop_token_ids.resize(out_token_count); + stop_token_ids = std::move(actual_stop_token_ids); + } +}; + +static bool accept_text(cactus_grammar_matcher_t matcher, const GrammarFixture& fixture, const std::string& text) { + const std::vector tokens = fixture.tokenizer->encode(text); + if (tokens.empty() && !text.empty()) return false; + for (uint32_t token : tokens) { + if (!cactus_grammar_matcher_accept(matcher, token)) return false; + } + return true; +} + +static bool accepts_complete_text(cactus_grammar_t grammar, const GrammarFixture& fixture, const std::string& text) { + auto matcher = matcher_handle(make_matcher(grammar, fixture.engine.get())); + if (!accept_text(matcher.get(), fixture, text)) return false; + return cactus_grammar_matcher_accept(matcher.get(), fixture.tokenizer->get_eos_token()); +} + +static bool rejects_text(cactus_grammar_t grammar, const GrammarFixture& fixture, const std::string& text) { + auto matcher = matcher_handle(make_matcher(grammar, fixture.engine.get())); + return !accept_text(matcher.get(), fixture, text); +} + +static bool rejects_eos_after_text(cactus_grammar_t grammar, const GrammarFixture& fixture, const std::string& text) { + auto matcher = matcher_handle(make_matcher(grammar, fixture.engine.get())); + if (!accept_text(matcher.get(), fixture, text)) return true; + return !cactus_grammar_matcher_accept(matcher.get(), fixture.tokenizer->get_eos_token()); +} + +static bool bitmask_allows_token(const std::vector& bitmask, uint32_t token_id) { + const size_t word_index = token_id / 32; + const uint32_t bit_index = token_id % 32; + if (word_index >= bitmask.size()) return false; + return ((bitmask[word_index]) & (uint32_t{1} << bit_index)) != 0; +} + +static size_t bitmask_size(size_t logits_buffer_size) { + return (logits_buffer_size + 31) / 32; +} + +static std::string grammar_ebnf(cactus_grammar_t grammar) { + std::vector buffer(4096, '\0'); + if (cactus_grammar_get_ebnf(grammar, buffer.data(), buffer.size()) != 0) { + throw std::runtime_error(cactus_get_last_error()); + } + return std::string(buffer.data()); +} + +static std::string tool_call_structural_tag_json() { + return R"({ + "type": "structural_tag", + "format": { + "type": "tags_with_separator", + "separator": "\n", + "tags": [ + { + "begin": "<|tool_call>\ncall:function_name_1(", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "city": {"type": "string"} + }, + "required": ["city"], + "additionalProperties": false + } + }, + "end": ")\n" + }, + { + "begin": "<|tool_call>\ncall:function_name_2(", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "count": {"type": "integer"} + }, + "required": ["count"], + "additionalProperties": false + } + }, + "end": ")\n" + } + ] + } + })"; +} + +static bool test_vocab_accessors(const GrammarFixture& fixture) { + const auto vocabulary = GrammarVocabulary::from_tokenizer(*fixture.tokenizer); + return fixture.vocab_size == fixture.tokenizer->get_vocab_size() + && fixture.add_prefix_space == vocabulary.add_prefix_space() + && !fixture.stop_token_ids.empty() + && fixture.stop_token_ids.front() == fixture.tokenizer->get_eos_token(); +} + +static bool test_empty_grammar_properties() { + auto empty = grammar_handle(cactus_grammar_init_empty()); + auto empty2 = grammar_handle(cactus_grammar_init_empty()); + auto simple = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\"", "root")); + + if (!cactus_grammar_is_empty(empty.get()) + || !cactus_grammar_is_empty(empty2.get()) + || cactus_grammar_is_empty(simple.get())) { + return false; + } + + cactus_grammar_t empty_union_inputs[] = {empty.get(), empty2.get()}; + auto empty_union = grammar_handle(cactus_grammar_union(empty_union_inputs, 2)); + auto empty_concat = grammar_handle(cactus_grammar_concatenate(empty_union_inputs, 2)); + + cactus_grammar_t union_with_simple_inputs[] = {empty.get(), simple.get()}; + auto union_with_simple = grammar_handle(cactus_grammar_union(union_with_simple_inputs, 2)); + auto concat_with_simple = grammar_handle(cactus_grammar_concatenate(union_with_simple_inputs, 2)); + + return cactus_grammar_is_empty(empty_union.get()) + && cactus_grammar_is_empty(empty_concat.get()) + && !cactus_grammar_is_empty(union_with_simple.get()) + && !cactus_grammar_is_empty(concat_with_simple.get()); +} + +static bool test_epsilon_grammar_accepts_only_empty_string(const GrammarFixture& fixture) { + auto epsilon = grammar_handle(cactus_grammar_init_epsilon()); + return accepts_complete_text(epsilon.get(), fixture, "") + && rejects_text(epsilon.get(), fixture, "hello"); +} + +static bool test_optional_grammar_accepts_zero_or_one_occurrence(const GrammarFixture& fixture) { + auto hello = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\"", "root")); + auto optional = grammar_handle(cactus_grammar_optional(hello.get())); + + return accepts_complete_text(optional.get(), fixture, "") + && accepts_complete_text(optional.get(), fixture, "hello") + && rejects_text(optional.get(), fixture, "goodbye") + && rejects_text(optional.get(), fixture, "hellohello"); +} + +static bool test_optional_empty_grammar_stays_empty() { + auto empty = grammar_handle(cactus_grammar_init_empty()); + auto optional = grammar_handle(cactus_grammar_optional(empty.get())); + return cactus_grammar_is_empty(optional.get()); +} + +static bool test_star_grammar_accepts_zero_or_more_occurrences(const GrammarFixture& fixture) { + auto ha = grammar_handle(cactus_grammar_init_ebnf("root ::= \"ha\"", "root")); + auto star = grammar_handle(cactus_grammar_star(ha.get())); + + return accepts_complete_text(star.get(), fixture, "") + && accepts_complete_text(star.get(), fixture, "ha") + && accepts_complete_text(star.get(), fixture, "hahaha") + && rejects_text(star.get(), fixture, "hello"); +} + +static bool test_star_empty_grammar_stays_empty() { + auto empty = grammar_handle(cactus_grammar_init_empty()); + auto star = grammar_handle(cactus_grammar_star(empty.get())); + return cactus_grammar_is_empty(star.get()); +} + +static bool test_repeat_exact_language(const GrammarFixture& fixture) { + auto ha = grammar_handle(cactus_grammar_init_ebnf("root ::= \"ha\"", "root")); + auto repeated = grammar_handle(cactus_grammar_repeat(ha.get(), 3)); + + return accepts_complete_text(repeated.get(), fixture, "hahaha") + && rejects_eos_after_text(repeated.get(), fixture, "") + && rejects_eos_after_text(repeated.get(), fixture, "ha") + && rejects_text(repeated.get(), fixture, "hahahaha") + && rejects_text(repeated.get(), fixture, "hello"); +} + +static bool test_repeat_range_language(const GrammarFixture& fixture) { + auto ha = grammar_handle(cactus_grammar_init_ebnf("root ::= \"ha\"", "root")); + auto repeated = grammar_handle(cactus_grammar_repeat_range(ha.get(), 2, 4)); + + return accepts_complete_text(repeated.get(), fixture, "haha") + && accepts_complete_text(repeated.get(), fixture, "hahaha") + && accepts_complete_text(repeated.get(), fixture, "hahahaha") + && rejects_eos_after_text(repeated.get(), fixture, "") + && rejects_eos_after_text(repeated.get(), fixture, "ha") + && rejects_text(repeated.get(), fixture, "hahahahaha"); +} + +static bool test_repeat_range_unbounded_language(const GrammarFixture& fixture) { + auto ha = grammar_handle(cactus_grammar_init_ebnf("root ::= \"ha\"", "root")); + auto repeated = grammar_handle(cactus_grammar_repeat_range(ha.get(), 2, -1)); + + return accepts_complete_text(repeated.get(), fixture, "haha") + && accepts_complete_text(repeated.get(), fixture, "hahaha") + && accepts_complete_text(repeated.get(), fixture, "hahahaha") + && rejects_eos_after_text(repeated.get(), fixture, "") + && rejects_eos_after_text(repeated.get(), fixture, "ha"); +} + +static bool test_repeat_empty_grammar_stays_empty() { + auto empty = grammar_handle(cactus_grammar_init_empty()); + auto exact = grammar_handle(cactus_grammar_repeat(empty.get(), 3)); + auto range = grammar_handle(cactus_grammar_repeat_range(empty.get(), 1, 3)); + return cactus_grammar_is_empty(exact.get()) && cactus_grammar_is_empty(range.get()); +} + +static bool test_ebnf_string_export_matches_parenthesized_input_ebnf() { + const std::string source = "root ::= ((\"hello\") | (\"hi\"))\n"; + auto grammar = grammar_handle(cactus_grammar_init_ebnf(source.c_str(), "root")); + return grammar_ebnf(grammar.get()) == source; +} + +static bool test_concat_accepts_expected_language(const GrammarFixture& fixture) { + auto left = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\"", "root")); + auto right = grammar_handle(cactus_grammar_init_ebnf("root ::= \" world\"", "root")); + cactus_grammar_t handles[] = {left.get(), right.get()}; + auto combined = grammar_handle(cactus_grammar_concatenate(handles, 2)); + + return accepts_complete_text(combined.get(), fixture, "hello world") + && rejects_text(combined.get(), fixture, "world hello") + && rejects_eos_after_text(combined.get(), fixture, "hello"); +} + +static bool test_union_accepts_expected_language(const GrammarFixture& fixture) { + auto left = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\"", "root")); + auto right = grammar_handle(cactus_grammar_init_ebnf("root ::= \"goodbye\"", "root")); + cactus_grammar_t handles[] = {left.get(), right.get()}; + auto combined = grammar_handle(cactus_grammar_union(handles, 2)); + + return accepts_complete_text(combined.get(), fixture, "hello") + && accepts_complete_text(combined.get(), fixture, "goodbye") + && rejects_text(combined.get(), fixture, "hello goodbye"); +} + +static bool test_three_way_concat(const GrammarFixture& fixture) { + auto alpha = grammar_handle(cactus_grammar_init_ebnf("root ::= \"alpha\"", "root")); + auto dash = grammar_handle(cactus_grammar_init_ebnf("root ::= \"-\"", "root")); + auto beta = grammar_handle(cactus_grammar_init_ebnf("root ::= \"beta\"", "root")); + cactus_grammar_t handles[] = {alpha.get(), dash.get(), beta.get()}; + auto combined = grammar_handle(cactus_grammar_concatenate(handles, 3)); + + return accepts_complete_text(combined.get(), fixture, "alpha-beta") + && rejects_text(combined.get(), fixture, "alphabeta") + && rejects_text(combined.get(), fixture, "beta-alpha"); +} + +static bool test_three_way_union(const GrammarFixture& fixture) { + auto red = grammar_handle(cactus_grammar_init_ebnf("root ::= \"red\"", "root")); + auto green = grammar_handle(cactus_grammar_init_ebnf("root ::= \"green\"", "root")); + auto blue = grammar_handle(cactus_grammar_init_ebnf("root ::= \"blue\"", "root")); + cactus_grammar_t handles[] = {red.get(), green.get(), blue.get()}; + auto combined = grammar_handle(cactus_grammar_union(handles, 3)); + + return accepts_complete_text(combined.get(), fixture, "red") + && accepts_complete_text(combined.get(), fixture, "green") + && accepts_complete_text(combined.get(), fixture, "blue") + && rejects_text(combined.get(), fixture, "orange"); +} + +static bool test_unordered_choice(const GrammarFixture& fixture) { + auto combined = grammar_handle(cactus_grammar_init_ebnf("root ::= \"a\" | \"ab\"", "root")); + + return accepts_complete_text(combined.get(), fixture, "a") + && accepts_complete_text(combined.get(), fixture, "ab") + && rejects_text(combined.get(), fixture, "ac"); +} + +static bool test_regex_and_json_schema_construction() { + auto regex = grammar_handle(cactus_grammar_init_regex("(cat|dog)")); + auto json_schema = grammar_handle(json_schema_grammar(R"({"type":"object","properties":{"name":{"type":"string"}},"required":["name"]})")); + return !cactus_grammar_is_empty(regex.get()) && !cactus_grammar_is_empty(json_schema.get()); +} + +static bool test_universal_grammar_accepts_anything(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_universal()); + return accepts_complete_text(grammar.get(), fixture, "") + && accepts_complete_text(grammar.get(), fixture, "blob says hello from cactus") + && accepts_complete_text(grammar.get(), fixture, "line one\nline two\nline three"); +} + +static bool test_structural_tag_accepts_and_rejects_expected_text(const GrammarFixture& fixture) { + auto grammar = grammar_handle( + cactus_grammar_init_structural_tag(tool_call_structural_tag_json().c_str(), fixture.vocab.get()) + ); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + std::vector bitmask(bitmask_size(fixture.vocab_size)); + + return !cactus_grammar_is_empty(grammar.get()) + && matcher != nullptr + && cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size) == 1; +} + +static bool test_regex_accepts_expected_text(const GrammarFixture& fixture) { + auto regex = grammar_handle(cactus_grammar_init_regex("(cat|dog)")); + + return accepts_complete_text(regex.get(), fixture, "cat") + && accepts_complete_text(regex.get(), fixture, "dog") + && rejects_text(regex.get(), fixture, "cow"); +} + +static bool test_json_schema_accepts_expected_text(const GrammarFixture& fixture) { + auto json_schema = grammar_handle(json_schema_grammar( + R"({"type":"object","properties":{"name":{"type":"string"}},"required":["name"]})" + )); + + return accepts_complete_text(json_schema.get(), fixture, R"({"name":"cactus"})") + && rejects_text(json_schema.get(), fixture, R"({"age":1})"); +} + +static bool test_grammar_matcher_reset_restores_initial_state(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\"", "root")); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + + if (!accept_text(matcher.get(), fixture, "hello")) return false; + if (!cactus_grammar_matcher_accept(matcher.get(), fixture.tokenizer->get_eos_token())) return false; + + cactus_grammar_matcher_reset(matcher.get()); + return rejects_text(grammar.get(), fixture, "goodbye") + && accept_text(matcher.get(), fixture, "hello") + && cactus_grammar_matcher_accept(matcher.get(), fixture.tokenizer->get_eos_token()); +} + +static bool test_grammar_matcher_rollback_restores_previous_state(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\" | \"hi\"", "root")); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + const std::vector hello_tokens = fixture.tokenizer->encode("hello"); + const std::vector hi_tokens = fixture.tokenizer->encode("hi"); + const uint32_t eos_token = fixture.tokenizer->get_eos_token(); + + for (uint32_t token : hello_tokens) { + if (!cactus_grammar_matcher_accept(matcher.get(), token)) return false; + } + if (!cactus_grammar_matcher_accept(matcher.get(), eos_token)) return false; + + cactus_grammar_matcher_rollback(matcher.get(), static_cast(hello_tokens.size()) + 1); + + for (uint32_t token : hi_tokens) { + if (!cactus_grammar_matcher_accept(matcher.get(), token)) return false; + } + return cactus_grammar_matcher_accept(matcher.get(), eos_token); +} + +static bool test_grammar_matcher_completion_state(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\"", "root")); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + const uint32_t eos_token = fixture.tokenizer->get_eos_token(); + + if (cactus_grammar_matcher_is_completed(matcher.get()) || cactus_grammar_matcher_is_terminated(matcher.get())) return false; + if (!accept_text(matcher.get(), fixture, "hello")) return false; + if (!cactus_grammar_matcher_is_completed(matcher.get()) || cactus_grammar_matcher_is_terminated(matcher.get())) return false; + if (!cactus_grammar_matcher_accept(matcher.get(), eos_token)) return false; + return cactus_grammar_matcher_is_completed(matcher.get()) && cactus_grammar_matcher_is_terminated(matcher.get()); +} + +static bool test_grammar_matcher_fork_preserves_accept_state(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello world\"", "root")); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + const std::vector tokens = fixture.tokenizer->encode("hello world"); + const uint32_t eos_token = fixture.tokenizer->get_eos_token(); + + if (tokens.size() < 2) return false; + + const size_t fork_point = tokens.size() / 2; + + for (size_t i = 0; i < fork_point; ++i) { + if (!cactus_grammar_matcher_accept(matcher.get(), tokens[i])) return false; + } + + auto forked = matcher_handle(cactus_grammar_matcher_fork(matcher.get())); + if (!forked) return false; + + for (size_t i = fork_point; i < tokens.size(); ++i) { + if (!cactus_grammar_matcher_accept(matcher.get(), tokens[i])) return false; + } + if (!cactus_grammar_matcher_accept(matcher.get(), eos_token)) return false; + + for (size_t i = fork_point; i < tokens.size(); ++i) { + if (!cactus_grammar_matcher_accept(forked.get(), tokens[i])) return false; + } + return cactus_grammar_matcher_accept(forked.get(), eos_token); +} + +static bool test_grammar_matcher_get_grammar_round_trips_source(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello world\"", "root")); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + auto retrieved = grammar_handle(cactus_grammar_matcher_get_grammar(matcher.get())); + if (!retrieved) return false; + + return grammar_ebnf(retrieved.get()) == grammar_ebnf(grammar.get()); +} + +static bool test_grammar_matcher_fork_preserves_source_grammar(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello world\"", "root")); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + auto forked = matcher_handle(cactus_grammar_matcher_fork(matcher.get())); + if (!forked) return false; + + auto retrieved = grammar_handle(cactus_grammar_matcher_get_grammar(forked.get())); + if (!retrieved) return false; + + return grammar_ebnf(retrieved.get()) == grammar_ebnf(grammar.get()); +} + +static bool test_grammar_matcher_next_bitmask_tracks_simple_grammar(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\"", "root")); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + const std::vector hello_tokens = fixture.tokenizer->encode("hello"); + const uint32_t eos_token = fixture.tokenizer->get_eos_token(); + + if (hello_tokens.empty()) return false; + + std::vector bitmask(bitmask_size(fixture.vocab_size)); + if (cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size) != 1) return false; + if (!bitmask_allows_token(bitmask, hello_tokens.front())) return false; + if (bitmask_allows_token(bitmask, eos_token)) return false; + + for (size_t i = 0; i < hello_tokens.size(); ++i) { + if (cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size) != 1) return false; + if (!bitmask_allows_token(bitmask, hello_tokens[i])) return false; + if (!cactus_grammar_matcher_accept(matcher.get(), hello_tokens[i])) return false; + } + + if (cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size) != 1) return false; + if (!bitmask_allows_token(bitmask, eos_token)) return false; + return !bitmask_allows_token(bitmask, hello_tokens.front()); +} + +static bool test_grammar_matcher_next_bitmask_zeroes_overallocated_tail(const GrammarFixture& fixture) { + auto grammar = grammar_handle(cactus_grammar_init_ebnf("root ::= \"hello\"", "root")); + auto matcher = matcher_handle(make_matcher(grammar.get(), fixture.engine.get())); + std::vector bitmask(bitmask_size(fixture.vocab_size + 1), -1); + + if (cactus_grammar_matcher_next_bitmask(matcher.get(), bitmask.data(), fixture.vocab_size + 1) != 1) return false; + return (bitmask.back() & 0xFF000000u) == 0; +} + +} // anonymous namespace + +int main() { + TestUtils::TestRunner runner("Grammar Tests"); + + try { + GrammarFixture fixture; + runner.run_test("vocab_accessors", test_vocab_accessors(fixture)); + runner.run_test("empty_properties", test_empty_grammar_properties()); + runner.run_test("epsilon_language", test_epsilon_grammar_accepts_only_empty_string(fixture)); + runner.run_test("optional_language", test_optional_grammar_accepts_zero_or_one_occurrence(fixture)); + runner.run_test("optional_empty_stays_empty", test_optional_empty_grammar_stays_empty()); + runner.run_test("star_language", test_star_grammar_accepts_zero_or_more_occurrences(fixture)); + runner.run_test("star_empty_stays_empty", test_star_empty_grammar_stays_empty()); + runner.run_test("repeat_exact_language", test_repeat_exact_language(fixture)); + runner.run_test("repeat_range_language", test_repeat_range_language(fixture)); + runner.run_test("repeat_range_unbounded_language", test_repeat_range_unbounded_language(fixture)); + runner.run_test("repeat_empty_stays_empty", test_repeat_empty_grammar_stays_empty()); + runner.run_test("ebnf_export_matches", test_ebnf_string_export_matches_parenthesized_input_ebnf()); + runner.run_test("regex_json_schema_init", test_regex_and_json_schema_construction()); + runner.run_test("concat_language", test_concat_accepts_expected_language(fixture)); + runner.run_test("union_language", test_union_accepts_expected_language(fixture)); + runner.run_test("three_way_concat", test_three_way_concat(fixture)); + runner.run_test("three_way_union", test_three_way_union(fixture)); + runner.run_test("unordred_choice", test_unordered_choice(fixture)); + runner.run_test("regex_language", test_regex_accepts_expected_text(fixture)); + runner.run_test("json_schema_language", test_json_schema_accepts_expected_text(fixture)); + runner.run_test("universal", test_universal_grammar_accepts_anything(fixture)); + runner.run_test("structural_tag_language", test_structural_tag_accepts_and_rejects_expected_text(fixture)); + runner.run_test("grammar_matcher_reset", test_grammar_matcher_reset_restores_initial_state(fixture)); + runner.run_test("grammar_matcher_rollback", test_grammar_matcher_rollback_restores_previous_state(fixture)); + runner.run_test("grammar_matcher_completion_state", test_grammar_matcher_completion_state(fixture)); + runner.run_test("grammar_matcher_fork", test_grammar_matcher_fork_preserves_accept_state(fixture)); + runner.run_test("grammar_matcher_get_grammar", test_grammar_matcher_get_grammar_round_trips_source(fixture)); + runner.run_test("grammar_matcher_fork_preserves_grammar", test_grammar_matcher_fork_preserves_source_grammar(fixture)); + runner.run_test("grammar_matcher_next_bitmask", test_grammar_matcher_next_bitmask_tracks_simple_grammar(fixture)); + runner.run_test("grammar_matcher_next_bitmask_overallocated_tail", test_grammar_matcher_next_bitmask_zeroes_overallocated_tail(fixture)); + } catch (const std::exception& e) { + std::cerr << "[✗] Grammar test setup failed: " << e.what() << "\n"; + } + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus/CMakeLists.txt b/cactus/CMakeLists.txt index 41167424c..2d8d1f575 100644 --- a/cactus/CMakeLists.txt +++ b/cactus/CMakeLists.txt @@ -1,6 +1,9 @@ cmake_minimum_required(VERSION 3.10) project(Cactus LANGUAGES CXX) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_BUILD_TYPE Release CACHE STRING "") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) @@ -11,6 +14,7 @@ endif() set(_ARCHIVES $ $ $) set(_DEPS cactus_engine cactus_graph cactus_kernels) +set(_CACTUS_XGRAMMAR_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine/libs/xgrammar") if(APPLE) add_custom_target(cactus ALL @@ -39,6 +43,10 @@ if(APPLE) -Wl,-force_load,$ -Wl,-force_load,$ ) + set(_CACTUS_XGRAMMAR_LIBRARY "${_CACTUS_XGRAMMAR_ROOT}/macos/libxgrammar.a") + if(EXISTS "${_CACTUS_XGRAMMAR_LIBRARY}") + target_link_libraries(cactus_shared PRIVATE "${_CACTUS_XGRAMMAR_LIBRARY}") + endif() set(_CURL "${CACTUS_CURL_ROOT}/macos/libcurl.a") if(EXISTS "${_CURL}") target_link_libraries(cactus_shared PRIVATE "${_CURL}") @@ -49,6 +57,10 @@ if(APPLE) ) else() target_link_options(cactus_shared PRIVATE -Wl,--whole-archive ${_ARCHIVES} -Wl,--no-whole-archive) + set(_CACTUS_XGRAMMAR_LIBRARY "${_CACTUS_XGRAMMAR_ROOT}/linux/aarch64/libxgrammar.a") + if(EXISTS "${_CACTUS_XGRAMMAR_LIBRARY}") + target_link_libraries(cactus_shared PRIVATE "${_CACTUS_XGRAMMAR_LIBRARY}") + endif() find_package(CURL QUIET) if(CURL_FOUND) target_link_libraries(cactus_shared PRIVATE ${CURL_LIBRARIES}) diff --git a/cactus/build.sh b/cactus/build.sh index 051de97c2..207ea6540 100755 --- a/cactus/build.sh +++ b/cactus/build.sh @@ -26,6 +26,11 @@ mkdir -p build cd build cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 + +if [ -f compile_commands.json ]; then + ln -sfn ../cactus/build/compile_commands.json ../../cactus-engine/compile_commands.json +fi + make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) echo "Cactus library built successfully!"