diff --git a/Gemfile b/Gemfile index ca993a7e..f868a679 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ gem "geocoder" gem "rubyzip", require: "zip" # AI/LLM integration for experience generation [https://github.com/crmne/ruby_llm] -gem "ruby_llm" +gem "ruby_llm", "~> 2.1" # CLI framework for Platform [https://github.com/rails/thor] diff --git a/Gemfile.lock b/Gemfile.lock index b862fc93..4974df86 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -173,7 +173,6 @@ GEM erubi (1.13.1) et-orbi (1.4.0) tzinfo - event_stream_parser (1.0.0) faraday (2.14.0) faraday-net_http (>= 2.0, < 3.5) json @@ -490,19 +489,19 @@ GEM ruby-vips (2.3.0) ffi (~> 1.12) logger - ruby_llm (1.9.1) + ruby_llm (2.1.1) base64 - event_stream_parser (~> 1) - faraday (>= 1.10.0) + faraday (>= 1.10.0, != 2.0.0) faraday-multipart (>= 1) faraday-net_http (>= 1) faraday-retry (>= 1) - marcel (~> 1.0) - ruby_llm-schema (~> 0.2.1) + json (< 4) + marcel (>= 1.0, < 3) + schematist (~> 1.1) zeitwerk (~> 2) - ruby_llm-schema (0.2.5) rubyzip (3.2.2) rugged (1.9.0) + schematist (1.1.0) securerandom (0.4.1) selenium-webdriver (4.39.0) base64 (~> 0.2) @@ -656,7 +655,7 @@ DEPENDENCIES rails (= 8.1.3.1) rollbar rubocop-rails-omakase - ruby_llm + ruby_llm (~> 2.1) rubyzip selenium-webdriver simplecov diff --git a/app/services/ai/openai_queue.rb b/app/services/ai/openai_queue.rb index 71627470..d19b3a42 100644 --- a/app/services/ai/openai_queue.rb +++ b/app/services/ai/openai_queue.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true module Ai - # Thin wrapper around RubyLLM for OpenAI API requests - # Provides centralized error handling and logging + # Thin wrapper around RubyLLM (2.x) for OpenAI API requests + # Provides centralized error handling and logging, and logs each call's + # model, tokens and cost # # Rate limiting is handled by RubyLLM's built-in Faraday retry middleware: # - Automatically retries on 429 (rate limit) status codes @@ -95,6 +96,7 @@ def execute_request(prompt:, schema: nil, context: "OpenaiQueue") # Check for gateway errors in the response content check_for_gateway_error_in_response(response) + log_usage(response, context) parse_response(response, schema) rescue GatewayError => e @@ -155,17 +157,30 @@ def execute_request(prompt:, schema: nil, context: "OpenaiQueue") private + # RubyLLM 2 returns structured output as a JSON string in `content` and + # parsed in `parsed`. Without a schema the reply is plain text. def parse_response(response, schema) return nil if response.nil? || response.content.nil? + return response.content if schema.nil? - if response.content.is_a?(Hash) - response.content.deep_symbolize_keys - elsif schema.nil? - # No schema means we expect plain text, return as-is - response.content - else - parse_ai_json_response(response.content) - end + parsed = response.parsed + parsed.is_a?(Hash) ? parsed.deep_symbolize_keys : parse_ai_json_response(response.content) + rescue JSON::ParserError + # A model that wraps its JSON in prose or a code fence + parse_ai_json_response(response.content) + end + + # One line per call, so cost per service shows in the log until tracing + # (Langfuse) records it. Never lets a reporting problem fail the request. + def log_usage(response, context) + return if response.nil? || !response.respond_to?(:tokens) + + tokens = response.tokens + cost = response.cost.total + Rails.logger.info "[#{context}] #{response.model} tokens in=#{tokens&.input || 0} " \ + "out=#{tokens&.output || 0} cost=$#{format("%.6f", cost || 0)}" + rescue StandardError => e + Rails.logger.debug "[#{context}] usage not logged: #{e.class}" end def parse_ai_json_response(content) @@ -183,8 +198,8 @@ def parse_ai_json_response(content) def sanitize_ai_json(json_str) json_str = json_str.dup - json_str.gsub!(/[""]/, '"') - json_str.gsub!(/['']/, "'") + json_str.gsub!(/[\u201C\u201D]/, '"') + json_str.gsub!(/[\u2018\u2019]/, "'") json_str.gsub!(/,(\s*[\}\]])/, '\1') json_str end diff --git a/config/initializers/ruby_llm.rb b/config/initializers/ruby_llm.rb index 3a696902..40c898ec 100644 --- a/config/initializers/ruby_llm.rb +++ b/config/initializers/ruby_llm.rb @@ -2,18 +2,16 @@ # https://rubyllm.com/ RubyLLM.configure do |config| - # OpenAI API key for GPT models + # Content generation runs on OpenAI (decided 2026-09-30). The other keys stay + # optional so a model from another provider can be named per call. config.openai_api_key = ENV.fetch("OPENAI_API_KEY", nil) - - # Anthropic API key for Claude models (preferred) config.anthropic_api_key = ENV.fetch("ANTHROPIC_API_KEY", nil) - - # Google AI API key for Gemini models config.gemini_api_key = ENV.fetch("GEMINI_API_KEY", nil) - # Default model to use - # Options: "gpt-4o-mini", "claude-sonnet-4-20250514", "gemini-2.0-flash", etc. - config.default_model = ENV.fetch("LLM_DEFAULT_MODEL", "gpt-4o-mini") + # GPT-6 Luna, OpenAI's low-cost tier ($0.10 in, $0.50 out per million tokens + # on 2026-10-14; it replaced gpt-4o-mini at $0.15 and $0.60). RubyLLM's model + # registry carries its prices, so every reply reports its cost. + config.default_model = ENV.fetch("LLM_DEFAULT_MODEL", "gpt-6-luna") # Request timeout in seconds # Increased to 300s (5 min) for complex AI proposals with structured output @@ -30,8 +28,7 @@ # - Attempt 2: wait ~5s (5 * 2^0 + randomness) # - Attempt 3: wait ~10s (5 * 2^1 + randomness) # - Attempt 4: wait ~20s (5 * 2^2 + randomness) - # - Attempt 5: wait ~40s (5 * 2^3 + randomness) - # Total max wait: ~75s before giving up + # - Attempt 5: wait ~30s (capped by retry_max_interval) config.max_retries = ENV.fetch("LLM_MAX_RETRIES", 5).to_i config.retry_interval = ENV.fetch("LLM_RETRY_INTERVAL", 5.0).to_f config.retry_backoff_factor = ENV.fetch("LLM_RETRY_BACKOFF_FACTOR", 2.0).to_f diff --git a/test/services/ai/openai_queue_test.rb b/test/services/ai/openai_queue_test.rb index 50004436..6c5c26b6 100644 --- a/test/services/ai/openai_queue_test.rb +++ b/test/services/ai/openai_queue_test.rb @@ -4,14 +4,18 @@ module Ai class OpenaiQueueTest < ActiveJob::TestCase + # An assistant reply as RubyLLM 2 returns it: structured output arrives as + # a JSON string in content, read back through #parsed. + def reply(content) + RubyLLM::Message.new(role: :assistant, content: content, model: "gpt-6-luna") + end + # === Request method tests === test "request returns parsed hash response when schema is provided" do response_content = { name: "Test", value: 123 } - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { response_content } + mock_response = reply(response_content.to_json) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -29,9 +33,7 @@ class OpenaiQueueTest < ActiveJob::TestCase end test "request returns string response when no schema is provided" do - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { "Plain text response" } + mock_response = reply("Plain text response") mock_chat = Object.new mock_chat.define_singleton_method(:ask) { |_prompt| mock_response } @@ -61,9 +63,7 @@ class OpenaiQueueTest < ActiveJob::TestCase end test "request returns nil when response content is nil" do - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { nil } + mock_response = reply(nil) mock_chat = Object.new mock_chat.define_singleton_method(:ask) { |_prompt| mock_response } @@ -78,16 +78,53 @@ class OpenaiQueueTest < ActiveJob::TestCase end end + # === Model and cost (RubyLLM 2) === + + test "GPT-6 Luna is in RubyLLM's registry with its prices" do + model = RubyLLM.models.find("gpt-6-luna") + tokens = RubyLLM::Tokens.new(input: 100_000, output: 100_000) + + # $0.10 in and $0.50 out per million tokens (OpenAI, 2026-10-14) + assert_in_delta 0.06, RubyLLM::Cost.new(tokens: tokens, model: model).total, 0.0001 + end + + test "each call logs its model, tokens and cost" do + response = reply("Plain text response") + response.define_singleton_method(:tokens) { RubyLLM::Tokens.new(input: 1200, output: 300) } + mock_chat = Object.new + mock_chat.define_singleton_method(:ask) { |_prompt| response } + logged = [] + + Rails.logger.stub :info, ->(message) { logged << message } do + RubyLLM.stub :chat, mock_chat do + Ai::OpenaiQueue.request(prompt: "Test", context: "Describer") + end + end + + assert_includes logged, "[Describer] gpt-6-luna tokens in=1200 out=300 cost=$0.000270" + end + + test "a reply without usage still returns its content" do + response = reply("Plain text response") + response.define_singleton_method(:cost) { raise "no pricing" } + mock_chat = Object.new + mock_chat.define_singleton_method(:ask) { |_prompt| response } + + RubyLLM.stub :chat, mock_chat do + assert_equal "Plain text response", Ai::OpenaiQueue.request(prompt: "Test", context: "Test") + end + end + # === Error handling tests === test "request raises RateLimitError on RubyLLM::RateLimitError" do - # RubyLLM errors expect a response object with a body method + # RubyLLM 2 errors take the response as a keyword; its body is the message mock_response = Object.new mock_response.define_singleton_method(:body) { '{"error": {"message": "Rate limit exceeded"}}' } mock_chat = Object.new mock_chat.define_singleton_method(:ask) do |_prompt| - raise RubyLLM::RateLimitError.new(mock_response) + raise RubyLLM::RateLimitError.new(response: mock_response) end RubyLLM.stub :chat, mock_chat do @@ -100,13 +137,13 @@ class OpenaiQueueTest < ActiveJob::TestCase end test "request raises RequestError on RubyLLM::Error" do - # RubyLLM errors expect a response object with a body method + # RubyLLM 2 errors take the response as a keyword; its body is the message mock_response = Object.new mock_response.define_singleton_method(:body) { '{"error": {"message": "API error"}}' } mock_chat = Object.new mock_chat.define_singleton_method(:ask) do |_prompt| - raise RubyLLM::Error.new(mock_response) + raise RubyLLM::Error.new(response: mock_response) end RubyLLM.stub :chat, mock_chat do @@ -138,9 +175,7 @@ class OpenaiQueueTest < ActiveJob::TestCase test "parses JSON from markdown code block" do json_content = "```json\n{\"name\": \"Test\", \"value\": 123}\n```" - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -160,9 +195,7 @@ class OpenaiQueueTest < ActiveJob::TestCase test "parses raw JSON object" do json_content = '{"name": "Test", "items": ["a", "b"]}' - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -180,12 +213,10 @@ class OpenaiQueueTest < ActiveJob::TestCase end test "sanitizes smart quotes in JSON" do - # Using smart quotes that should be converted to regular quotes - json_content = '{"name": "Test"}' + # Curly quotes, as a model writing prose sometimes returns + json_content = "{\u201Cname\u201D: \u201CTest\u201D}" - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -205,9 +236,7 @@ class OpenaiQueueTest < ActiveJob::TestCase test "handles trailing commas in JSON" do json_content = '{"name": "Test", "value": 1,}' - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -227,9 +256,7 @@ class OpenaiQueueTest < ActiveJob::TestCase test "returns empty hash on invalid JSON" do json_content = "not valid json at all" - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -306,13 +333,9 @@ class OpenaiQueueTest < ActiveJob::TestCase gateway_html = "