Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions cactus-engine/tests/chat.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <iostream>
#include <mutex>
#include <sstream>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>
Expand Down Expand Up @@ -285,7 +286,11 @@ bool file_exists(const std::string& path) {

std::vector<uint32_t> parse_token_ids(const std::string& text) {
std::vector<uint32_t> tokens;
std::stringstream ss(text);
std::string normalized = text;
for (char& ch : normalized) {
if (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n') ch = ',';
}
std::stringstream ss(normalized);
std::string part;
while (std::getline(ss, part, ',')) {
size_t start = part.find_first_not_of(" \t\r\n");
Expand All @@ -302,6 +307,16 @@ std::vector<uint32_t> parse_token_ids(const std::string& text) {
return tokens;
}

bool read_text_file(const std::string& path, std::string& text) {
std::ifstream in(path);
if (!in) return false;
std::ostringstream buffer;
buffer << in.rdbuf();
if (!in.good() && !in.eof()) return false;
text = buffer.str();
return true;
}

bool write_text_file(const std::string& path, const std::string& text) {
std::ofstream out(path);
if (!out) return false;
Expand Down Expand Up @@ -364,7 +379,7 @@ std::string build_messages(const std::string& system_prompt,
void print_usage(const char* argv0) {
std::cerr << "Usage: " << argv0
<< " <model_path> [--system <prompt>] [--image <path>] [--audio <path>]"
<< " [--prompt <text>] [--input-ids <ids>] [--max-new-tokens <n>]"
<< " [--prompt <text>] [--input-ids <ids>] [--input-ids-file <path>] [--max-new-tokens <n>]"
<< " [--result-json <path>] [--thinking]\n";
}

Expand All @@ -382,6 +397,7 @@ int main(int argc, char** argv) {
std::string current_audio;
std::string initial_prompt;
std::string input_ids;
std::string input_ids_file;
std::string result_json;
int max_new_tokens = kMaxTokens;
bool thinking = false;
Expand All @@ -398,6 +414,8 @@ int main(int argc, char** argv) {
initial_prompt = argv[++i];
} else if (arg == "--input-ids" && i + 1 < argc) {
input_ids = argv[++i];
} else if (arg == "--input-ids-file" && i + 1 < argc) {
input_ids_file = expand_tilde(argv[++i]);
} else if (arg == "--max-new-tokens" && i + 1 < argc) {
max_new_tokens = std::max(0, std::atoi(argv[++i]));
} else if (arg == "--result-json" && i + 1 < argc) {
Expand All @@ -415,6 +433,20 @@ int main(int argc, char** argv) {
std::cerr << "Audio file not found: " << current_audio << "\n";
return 1;
}
if (!input_ids.empty() && !input_ids_file.empty()) {
std::cerr << "Use either --input-ids or --input-ids-file, not both\n";
return 1;
}
if (!input_ids_file.empty()) {
if (!file_exists(input_ids_file)) {
std::cerr << "Input ids file not found: " << input_ids_file << "\n";
return 1;
}
if (!read_text_file(input_ids_file, input_ids)) {
std::cerr << "Failed to read input ids file: " << input_ids_file << "\n";
return 1;
}
}

std::cout << "Loading model from " << model_path << "...\n";
cactus_model_t model = cactus_init(model_path.c_str(), nullptr, false);
Expand Down
6 changes: 6 additions & 0 deletions python/cactus/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ def create_parser():
help="Initial prompt to send immediately")
run_parser.add_argument("--input-ids", default=None,
help="Comma-separated token ids for transpiled causal-LM bundles")
run_parser.add_argument("--input-ids-file", default=None,
help="File containing token ids for transpiled causal-LM bundles")
run_parser.add_argument("--max-new-tokens", type=int, default=None,
help="Maximum tokens to generate for transpiled causal-LM bundles")
run_parser.add_argument("--result-json", default=None,
Expand Down Expand Up @@ -277,8 +279,12 @@ def create_parser():
help="Allow HF remote code during the transpile phase")
convert_parser.add_argument("--local-files-only", action="store_true",
help="Require HF model/processor files to already be local during transpile")
convert_parser.add_argument("--cache-context-length", default=None,
help="KV cache context length for cached decode graphs (default: model config)")
convert_parser.add_argument("--reconvert", action="store_true",
help="Force conversion from source")
convert_parser.add_argument("--retranspile", action="store_true",
help="Force transpilation while reusing existing converted weights")
convert_parser.add_argument("--npu", action="store_true",
help="Also emit CoreML .mlpackage(s) for NPU (Apple Neural Engine) audio + vision encoders")
convert_parser.add_argument("--npu-quantize", type=int, choices=[0, 4, 8], default=None,
Expand Down
2 changes: 2 additions & 0 deletions python/cactus/cli/convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ def cmd_convert(args):
bits=args.bits or 4,
token=args.token,
reconvert=args.reconvert,
retranspile=args.retranspile,
output_dir=output_dir,
transpile=TranspileOptions(
task=args.task or "auto",
Expand All @@ -26,6 +27,7 @@ def cmd_convert(args):
system_prompt=args.system_prompt,
trust_remote_code=args.trust_remote_code,
local_files_only=args.local_files_only,
cache_context_length=args.cache_context_length,
npu=getattr(args, "npu", False),
npu_quantize=getattr(args, "npu_quantize", None),
npu_audio_quantize=getattr(args, "npu_audio_quantize", None),
Expand Down
7 changes: 5 additions & 2 deletions python/cactus/cli/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,14 +200,15 @@ class TranspileOptions:
system_prompt: str | None = None
trust_remote_code: bool = False
local_files_only: bool = False
cache_context_length: str | int | None = None
npu: bool = False
npu_quantize: int | None = None
npu_audio_quantize: int | None = None
npu_vision_quantize: int | None = None


def ensure_bundle(model_id, *, bits=4, token=None,
reconvert=False, output_dir=None, transpile=None):
reconvert=False, retranspile=False, output_dir=None, transpile=None):
"""Return path to transpiled bundle, creating it if needed.
"""
from .download import get_weights_dir
Expand All @@ -228,7 +229,7 @@ def ensure_bundle(model_id, *, bits=4, token=None,
)

# Step 2: skip if already transpiled
if _has_transpiled_bundle(output_dir):
if _has_transpiled_bundle(output_dir) and not retranspile:
return output_dir

# Step 3: infer transpile spec from converted output
Expand Down Expand Up @@ -316,6 +317,8 @@ def ensure_bundle(model_id, *, bits=4, token=None,
extra_args.append("--trust-remote-code")
if opts.local_files_only:
extra_args.append("--local-files-only")
if opts.cache_context_length is not None:
extra_args.extend(["--cache-context-length", str(opts.cache_context_length)])
if opts.npu:
extra_args.append("--npu")
if opts.npu_quantize is not None:
Expand Down
1 change: 1 addition & 0 deletions python/cactus/cli/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ def cmd_run(args):
("--image", args.image),
("--audio", args.audio),
("--input-ids", args.input_ids),
("--input-ids-file", args.input_ids_file),
("--max-new-tokens", args.max_new_tokens),
("--result-json", args.result_json),
):
Expand Down
6 changes: 6 additions & 0 deletions python/cactus/transpile/hf_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -2892,6 +2892,11 @@ def main() -> int:
"so run-transpiled can do greedy autoregressive decoding."
),
)
parser.add_argument(
"--cache-context-length",
default="auto",
help="KV cache context length for cached decode graphs. Use auto to read the model config.",
)
parser.add_argument(
"--audio-file",
default="",
Expand Down Expand Up @@ -3184,6 +3189,7 @@ def main() -> int:
weights_dir=weights_dir,
inputs_metadata=prepared.metadata,
components=requested_components,
cache_context_length=args.cache_context_length,
)
use_component_pipeline = False
if args.component_pipeline == "on":
Expand Down
3 changes: 2 additions & 1 deletion python/cactus/transpile/lower.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,8 @@ def _lower_attention_with_internal_kv_cache(
)
if layer_key not in cache_states:
default_cache_len = max(512, int(key_shape[1]))
max_cache_seq_len = int(ir.meta.get("max_cache_seq_len", default_cache_len) or default_cache_len)
requested_cache_len = node.meta.get("max_cache_seq_len", ir.meta.get("max_cache_seq_len", default_cache_len))
max_cache_seq_len = max(default_cache_len, int(requested_cache_len or default_cache_len))
sink_size = int(ir.meta.get("cache_sink_size", 4) or 4)
window_size = int(node.attrs.get("window_size", 0) or 0)
k_cache = g.kv_cache_state(
Expand Down
Loading
Loading