Repository navigation
Expand file tree
/
Copy pathmain.cpp
More file actions
67 lines (51 loc) · 2.08 KB
/
Copy pathmain.cpp
File metadata and controls
67 lines (51 loc) · 2.08 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
#include <iostream>
#include <string>
#include "inference_pipeline.h"
// Example function to print tokens as they're generated
void printToken(const std::string& token) {
std::cout << token << std::flush;
}
int main(int argc, char** argv) {
if (argc < 2) {
std::cerr << "Usage: " << argv[0] << " <model_path> [quantization_mode]" << std::endl;
return 1;
}
std::string model_path = argv[1];
int quantization_mode = 0; // Default to no quantization
if (argc >= 3) {
quantization_mode = std::stoi(argv[2]);
}
try {
// Set up quantization options
mlx_transformer::QuantizationOptions quant_options;
quant_options.mode = static_cast<mlx_transformer::QuantizationMode>(quantization_mode);
std::cout << "Loading model from: " << model_path << std::endl;
// Create inference pipeline
mlx_transformer::InferencePipeline pipeline(model_path, quant_options);
// Example 1: Basic text generation
std::string prompt = "Once upon a time in a galaxy far, far away";
std::cout << "\nGenerating text with prompt: " << prompt << std::endl;
std::string generated_text = pipeline.generate(
prompt,
100, // max_length
0.7, // temperature
50 // top_k
);
std::cout << "Generated: " << generated_text << std::endl;
// Example 2: Streaming generation
std::cout << "\nStreaming generation with the same prompt:" << std::endl;
std::cout << prompt; // Print the prompt first
pipeline.generate_stream(
prompt,
printToken, // Callback function to process each token
100, // max_length
0.7, // temperature
50 // top_k
);
std::cout << "\n\nGeneration complete!" << std::endl;
} catch (const std::exception& e) {
std::cerr << "Error: " << e.what() << std::endl;
return 1;
}
return 0;
}