diff --git a/Gemfile b/Gemfile index ca993a7e..f868a679 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ gem "geocoder" gem "rubyzip", require: "zip" # AI/LLM integration for experience generation [https://github.com/crmne/ruby_llm] -gem "ruby_llm" +gem "ruby_llm", "~> 2.1" # CLI framework for Platform [https://github.com/rails/thor] diff --git a/Gemfile.lock b/Gemfile.lock index b862fc93..4974df86 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -173,7 +173,6 @@ GEM erubi (1.13.1) et-orbi (1.4.0) tzinfo - event_stream_parser (1.0.0) faraday (2.14.0) faraday-net_http (>= 2.0, < 3.5) json @@ -490,19 +489,19 @@ GEM ruby-vips (2.3.0) ffi (~> 1.12) logger - ruby_llm (1.9.1) + ruby_llm (2.1.1) base64 - event_stream_parser (~> 1) - faraday (>= 1.10.0) + faraday (>= 1.10.0, != 2.0.0) faraday-multipart (>= 1) faraday-net_http (>= 1) faraday-retry (>= 1) - marcel (~> 1.0) - ruby_llm-schema (~> 0.2.1) + json (< 4) + marcel (>= 1.0, < 3) + schematist (~> 1.1) zeitwerk (~> 2) - ruby_llm-schema (0.2.5) rubyzip (3.2.2) rugged (1.9.0) + schematist (1.1.0) securerandom (0.4.1) selenium-webdriver (4.39.0) base64 (~> 0.2) @@ -656,7 +655,7 @@ DEPENDENCIES rails (= 8.1.3.1) rollbar rubocop-rails-omakase - ruby_llm + ruby_llm (~> 2.1) rubyzip selenium-webdriver simplecov diff --git a/app/services/ai/openai_queue.rb b/app/services/ai/openai_queue.rb index 71627470..d19b3a42 100644 --- a/app/services/ai/openai_queue.rb +++ b/app/services/ai/openai_queue.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true module Ai - # Thin wrapper around RubyLLM for OpenAI API requests - # Provides centralized error handling and logging + # Thin wrapper around RubyLLM (2.x) for OpenAI API requests + # Provides centralized error handling and logging, and logs each call's + # model, tokens and cost # # Rate limiting is handled by RubyLLM's built-in Faraday retry middleware: # - Automatically retries on 429 (rate limit) status codes @@ -95,6 +96,7 @@ def execute_request(prompt:, schema: nil, context: "OpenaiQueue") # Check for gateway errors in the response content check_for_gateway_error_in_response(response) + log_usage(response, context) parse_response(response, schema) rescue GatewayError => e @@ -155,17 +157,30 @@ def execute_request(prompt:, schema: nil, context: "OpenaiQueue") private + # RubyLLM 2 returns structured output as a JSON string in `content` and + # parsed in `parsed`. Without a schema the reply is plain text. def parse_response(response, schema) return nil if response.nil? || response.content.nil? + return response.content if schema.nil? - if response.content.is_a?(Hash) - response.content.deep_symbolize_keys - elsif schema.nil? - # No schema means we expect plain text, return as-is - response.content - else - parse_ai_json_response(response.content) - end + parsed = response.parsed + parsed.is_a?(Hash) ? parsed.deep_symbolize_keys : parse_ai_json_response(response.content) + rescue JSON::ParserError + # A model that wraps its JSON in prose or a code fence + parse_ai_json_response(response.content) + end + + # One line per call, so cost per service shows in the log until tracing + # (Langfuse) records it. Never lets a reporting problem fail the request. + def log_usage(response, context) + return if response.nil? || !response.respond_to?(:tokens) + + tokens = response.tokens + cost = response.cost.total + Rails.logger.info "[#{context}] #{response.model} tokens in=#{tokens&.input || 0} " \ + "out=#{tokens&.output || 0} cost=$#{format("%.6f", cost || 0)}" + rescue StandardError => e + Rails.logger.debug "[#{context}] usage not logged: #{e.class}" end def parse_ai_json_response(content) @@ -183,8 +198,8 @@ def parse_ai_json_response(content) def sanitize_ai_json(json_str) json_str = json_str.dup - json_str.gsub!(/[""]/, '"') - json_str.gsub!(/['']/, "'") + json_str.gsub!(/[\u201C\u201D]/, '"') + json_str.gsub!(/[\u2018\u2019]/, "'") json_str.gsub!(/,(\s*[\}\]])/, '\1') json_str end diff --git a/config/initializers/ruby_llm.rb b/config/initializers/ruby_llm.rb index 3a696902..40c898ec 100644 --- a/config/initializers/ruby_llm.rb +++ b/config/initializers/ruby_llm.rb @@ -2,18 +2,16 @@ # https://rubyllm.com/ RubyLLM.configure do |config| - # OpenAI API key for GPT models + # Content generation runs on OpenAI (decided 2026-09-30). The other keys stay + # optional so a model from another provider can be named per call. config.openai_api_key = ENV.fetch("OPENAI_API_KEY", nil) - - # Anthropic API key for Claude models (preferred) config.anthropic_api_key = ENV.fetch("ANTHROPIC_API_KEY", nil) - - # Google AI API key for Gemini models config.gemini_api_key = ENV.fetch("GEMINI_API_KEY", nil) - # Default model to use - # Options: "gpt-4o-mini", "claude-sonnet-4-20250514", "gemini-2.0-flash", etc. - config.default_model = ENV.fetch("LLM_DEFAULT_MODEL", "gpt-4o-mini") + # GPT-6 Luna, OpenAI's low-cost tier ($0.10 in, $0.50 out per million tokens + # on 2026-10-14; it replaced gpt-4o-mini at $0.15 and $0.60). RubyLLM's model + # registry carries its prices, so every reply reports its cost. + config.default_model = ENV.fetch("LLM_DEFAULT_MODEL", "gpt-6-luna") # Request timeout in seconds # Increased to 300s (5 min) for complex AI proposals with structured output @@ -30,8 +28,7 @@ # - Attempt 2: wait ~5s (5 * 2^0 + randomness) # - Attempt 3: wait ~10s (5 * 2^1 + randomness) # - Attempt 4: wait ~20s (5 * 2^2 + randomness) - # - Attempt 5: wait ~40s (5 * 2^3 + randomness) - # Total max wait: ~75s before giving up + # - Attempt 5: wait ~30s (capped by retry_max_interval) config.max_retries = ENV.fetch("LLM_MAX_RETRIES", 5).to_i config.retry_interval = ENV.fetch("LLM_RETRY_INTERVAL", 5.0).to_f config.retry_backoff_factor = ENV.fetch("LLM_RETRY_BACKOFF_FACTOR", 2.0).to_f diff --git a/test/services/ai/openai_queue_test.rb b/test/services/ai/openai_queue_test.rb index 50004436..6c5c26b6 100644 --- a/test/services/ai/openai_queue_test.rb +++ b/test/services/ai/openai_queue_test.rb @@ -4,14 +4,18 @@ module Ai class OpenaiQueueTest < ActiveJob::TestCase + # An assistant reply as RubyLLM 2 returns it: structured output arrives as + # a JSON string in content, read back through #parsed. + def reply(content) + RubyLLM::Message.new(role: :assistant, content: content, model: "gpt-6-luna") + end + # === Request method tests === test "request returns parsed hash response when schema is provided" do response_content = { name: "Test", value: 123 } - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { response_content } + mock_response = reply(response_content.to_json) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -29,9 +33,7 @@ class OpenaiQueueTest < ActiveJob::TestCase end test "request returns string response when no schema is provided" do - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { "Plain text response" } + mock_response = reply("Plain text response") mock_chat = Object.new mock_chat.define_singleton_method(:ask) { |_prompt| mock_response } @@ -61,9 +63,7 @@ class OpenaiQueueTest < ActiveJob::TestCase end test "request returns nil when response content is nil" do - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { nil } + mock_response = reply(nil) mock_chat = Object.new mock_chat.define_singleton_method(:ask) { |_prompt| mock_response } @@ -78,16 +78,53 @@ class OpenaiQueueTest < ActiveJob::TestCase end end + # === Model and cost (RubyLLM 2) === + + test "GPT-6 Luna is in RubyLLM's registry with its prices" do + model = RubyLLM.models.find("gpt-6-luna") + tokens = RubyLLM::Tokens.new(input: 100_000, output: 100_000) + + # $0.10 in and $0.50 out per million tokens (OpenAI, 2026-10-14) + assert_in_delta 0.06, RubyLLM::Cost.new(tokens: tokens, model: model).total, 0.0001 + end + + test "each call logs its model, tokens and cost" do + response = reply("Plain text response") + response.define_singleton_method(:tokens) { RubyLLM::Tokens.new(input: 1200, output: 300) } + mock_chat = Object.new + mock_chat.define_singleton_method(:ask) { |_prompt| response } + logged = [] + + Rails.logger.stub :info, ->(message) { logged << message } do + RubyLLM.stub :chat, mock_chat do + Ai::OpenaiQueue.request(prompt: "Test", context: "Describer") + end + end + + assert_includes logged, "[Describer] gpt-6-luna tokens in=1200 out=300 cost=$0.000270" + end + + test "a reply without usage still returns its content" do + response = reply("Plain text response") + response.define_singleton_method(:cost) { raise "no pricing" } + mock_chat = Object.new + mock_chat.define_singleton_method(:ask) { |_prompt| response } + + RubyLLM.stub :chat, mock_chat do + assert_equal "Plain text response", Ai::OpenaiQueue.request(prompt: "Test", context: "Test") + end + end + # === Error handling tests === test "request raises RateLimitError on RubyLLM::RateLimitError" do - # RubyLLM errors expect a response object with a body method + # RubyLLM 2 errors take the response as a keyword; its body is the message mock_response = Object.new mock_response.define_singleton_method(:body) { '{"error": {"message": "Rate limit exceeded"}}' } mock_chat = Object.new mock_chat.define_singleton_method(:ask) do |_prompt| - raise RubyLLM::RateLimitError.new(mock_response) + raise RubyLLM::RateLimitError.new(response: mock_response) end RubyLLM.stub :chat, mock_chat do @@ -100,13 +137,13 @@ class OpenaiQueueTest < ActiveJob::TestCase end test "request raises RequestError on RubyLLM::Error" do - # RubyLLM errors expect a response object with a body method + # RubyLLM 2 errors take the response as a keyword; its body is the message mock_response = Object.new mock_response.define_singleton_method(:body) { '{"error": {"message": "API error"}}' } mock_chat = Object.new mock_chat.define_singleton_method(:ask) do |_prompt| - raise RubyLLM::Error.new(mock_response) + raise RubyLLM::Error.new(response: mock_response) end RubyLLM.stub :chat, mock_chat do @@ -138,9 +175,7 @@ class OpenaiQueueTest < ActiveJob::TestCase test "parses JSON from markdown code block" do json_content = "```json\n{\"name\": \"Test\", \"value\": 123}\n```" - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -160,9 +195,7 @@ class OpenaiQueueTest < ActiveJob::TestCase test "parses raw JSON object" do json_content = '{"name": "Test", "items": ["a", "b"]}' - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -180,12 +213,10 @@ class OpenaiQueueTest < ActiveJob::TestCase end test "sanitizes smart quotes in JSON" do - # Using smart quotes that should be converted to regular quotes - json_content = '{"name": "Test"}' + # Curly quotes, as a model writing prose sometimes returns + json_content = "{\u201Cname\u201D: \u201CTest\u201D}" - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -205,9 +236,7 @@ class OpenaiQueueTest < ActiveJob::TestCase test "handles trailing commas in JSON" do json_content = '{"name": "Test", "value": 1,}' - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -227,9 +256,7 @@ class OpenaiQueueTest < ActiveJob::TestCase test "returns empty hash on invalid JSON" do json_content = "not valid json at all" - mock_response = Object.new - mock_response.define_singleton_method(:nil?) { false } - mock_response.define_singleton_method(:content) { json_content } + mock_response = reply(json_content) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -306,13 +333,9 @@ class OpenaiQueueTest < ActiveJob::TestCase gateway_html = "502 Bad Gateway" good_response = { result: "success" } - mock_response_bad = Object.new - mock_response_bad.define_singleton_method(:nil?) { false } - mock_response_bad.define_singleton_method(:content) { gateway_html } + mock_response_bad = reply(gateway_html) - mock_response_good = Object.new - mock_response_good.define_singleton_method(:nil?) { false } - mock_response_good.define_singleton_method(:content) { good_response } + mock_response_good = reply(good_response.to_json) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } @@ -378,9 +401,7 @@ class OpenaiQueueTest < ActiveJob::TestCase call_count = 0 good_response = { result: "success" } - mock_response_good = Object.new - mock_response_good.define_singleton_method(:nil?) { false } - mock_response_good.define_singleton_method(:content) { good_response } + mock_response_good = reply(good_response.to_json) mock_chat = Object.new mock_chat.define_singleton_method(:with_schema) { |_schema| self } diff --git a/wiki/architecture/ai-content-pipeline.md b/wiki/architecture/ai-content-pipeline.md index f155008e..9951838e 100644 --- a/wiki/architecture/ai-content-pipeline.md +++ b/wiki/architecture/ai-content-pipeline.md @@ -2,11 +2,12 @@ title: AI content pipeline kind: reference status: living -updated: 2026-09-29 +updated: 2026-10-14 repos: - usput.ba confidence: medium sources: +- sources/conversations/2026-10-14--usput--tooling-models-langfuse-jev.md - app/services/ai/openai_queue.rb - app/services/ai/rate_limiter.rb - app/services/ai/bih_context.rb @@ -114,12 +115,14 @@ for wording, not live code. `Ai::OpenaiQueue.request(prompt:, schema:, context:)` is the synchronous path every service uses. Despite its name it calls whatever model RubyLLM -is configured for: `LLM_DEFAULT_MODEL`, default `gpt-4o-mini`, with OpenAI, -Anthropic and Gemini keys all read from the environment -(`config/initializers/ruby_llm.rb`). With a JSON schema it uses -`with_schema` and parses the result into a symbol-keyed hash, falling back -to extracting JSON from fenced or bare text and repairing smart quotes and -trailing commas. +is configured for: `LLM_DEFAULT_MODEL`, default `gpt-6-luna` since +2026-10-14 (it was `gpt-4o-mini`), with OpenAI, Anthropic and Gemini keys all +read from the environment (`config/initializers/ruby_llm.rb`). RubyLLM is +2.1. With a JSON schema it uses `with_schema` and reads the reply through +`parsed` into a symbol-keyed hash, falling back to extracting JSON from +fenced or bare text and repairing curly quotes and trailing commas. Every +call logs one line with its model, input and output tokens and cost in US +dollars, priced from RubyLLM's model registry. Retries come in two layers. RubyLLM's Faraday middleware retries 429, 5xx and network failures up to `LLM_MAX_RETRIES` (default 5) with exponential diff --git a/wiki/architecture/overview.md b/wiki/architecture/overview.md index 58f7f8a6..01fb4189 100644 --- a/wiki/architecture/overview.md +++ b/wiki/architecture/overview.md @@ -7,6 +7,7 @@ repos: - usput.ba confidence: medium sources: +- sources/conversations/2026-10-14--usput--tooling-models-langfuse-jev.md - Gemfile - .herb.yml - .ruby-version @@ -82,7 +83,7 @@ other. The deeper pages are [Platform DSL](platform-dsl.md), | Web server | Puma behind Thruster | `Gemfile`, `Dockerfile` | | Frontend | Turbo, Stimulus, importmap, vendored Leaflet 1.9.4 and markercluster 1.5.3 | `config/importmap.rb` | | CSS | Tailwind CSS 4 through `tailwindcss-rails` and the standalone CLI | `Gemfile`, `Dockerfile`, `package.json` | -| LLM | RubyLLM, default model `gpt-4o-mini` unless `LLM_DEFAULT_MODEL` is set | `config/initializers/ruby_llm.rb` | +| LLM | RubyLLM 2.1, default model `gpt-6-luna` unless `LLM_DEFAULT_MODEL` is set | `config/initializers/ruby_llm.rb` | | Text to speech | ElevenLabs by default, OpenAI TTS as an alternative | `app/services/ai/audio_tour_generator.rb` | | Files | Active Storage, S3 in production when `AWS_BUCKET` is set, disk otherwise | `config/storage.yml`, `config/environments/production.rb` | | Images | `image_processing` over libvips (`libvips42` in the image, `libvips` in CI) | `Gemfile`, `Dockerfile`, `.github/workflows/ci.yml` | diff --git a/wiki/initiatives/rubyllm-2-upgrade.md b/wiki/initiatives/rubyllm-2-upgrade.md index d9c64113..4d682581 100644 --- a/wiki/initiatives/rubyllm-2-upgrade.md +++ b/wiki/initiatives/rubyllm-2-upgrade.md @@ -2,7 +2,7 @@ title: RubyLLM 2 in the rebuild kind: initiative status: proposed -updated: 2026-09-30 +updated: 2026-10-14 repos: - usput.ba confidence: medium @@ -10,6 +10,8 @@ depends_on: - architecture/ai-content-pipeline.md - decisions/avo-now-compile-later.md sources: +- sources/conversations/2026-10-14--usput--tooling-models-langfuse-jev.md +- https://developers.openai.com/api/docs/pricing - sources/conversations/2026-09-30--usput--platform-direction.md - Gemfile - Gemfile.lock @@ -114,6 +116,18 @@ Small to medium: slices 1, 2 and 4 are small, slices 3, 5 and 6 are medium, slic **Decided on 2026-09-30** ([answers](../../sources/conversations/2026-09-30--usput--platform-direction.md)): Content generation uses OpenAI. Embeddings use OpenAI `text-embedding-3-small`. Audio tours are in version 1. The questions below that these answers settle are closed; the rest stay open. -- **Default provider and model** for content generation. Today the default is `gpt-4o-mini` with keys for three providers configured; the rebuild can name one provider and one model per task. +- **Default provider and model** for content generation. Decided on 2026-10-14: `gpt-6-luna`, OpenAI's low-cost tier ([answers](../../sources/conversations/2026-10-14--usput--tooling-models-langfuse-jev.md)). A task that needs stronger prose can name another model per call. - **OpenAI protocol**: the 2.0 default (Responses API) or `chat_completions`. - **Whether audio tours come back** in the rebuild, and if so after version 1. + +## Build notes + +On 2026-10-14 the operator asked for the latest RubyLLM and OpenAI's Luna models now, on the current code, rather than waiting for the rebuild ([answers](../../sources/conversations/2026-10-14--usput--tooling-models-langfuse-jev.md)). The first slice landed on `Ai::OpenaiQueue` as it is: + +- **`ruby_llm` 2.1.1**, pinned `~> 2.1`. The legacy `acts_as` warning is gone with the API it warned about. +- **`gpt-6-luna` is the default model.** OpenAI lists it at $0.10 input, $0.01 cached input and $0.50 output per million tokens, half that on Batch and Flex; `gpt-4o-mini` was $0.15 and $0.60 (https://developers.openai.com/api/docs/pricing, read 2026-10-14). Above 272K input tokens the long-context price applies. `LLM_DEFAULT_MODEL` still overrides it, so an environment that sets it keeps its own model. +- **Structured output is read through `parsed`**, with the regex extraction kept only as a fallback for a reply that wraps its JSON in prose. The curly-quote repair now matches real curly quotes; before, its character classes held plain ASCII quotes and repaired nothing. +- **Cost on every call.** The wrapper logs the model, tokens and dollar cost of each reply, priced by RubyLLM's model registry, until Langfuse traces carry them. +- **Tests describe 2.x**: replies are `RubyLLM::Message` objects with JSON string content, errors take `response:` as a keyword. + +Still open from the scope above: the one seam beyond `OpenaiQueue` (the Platform DSL's `LLMHelper` still calls `RubyLLM.chat` itself), schema classes, prompts only from files, and generation jobs. diff --git a/wiki/state.md b/wiki/state.md index 533d83ec..5faf2194 100644 --- a/wiki/state.md +++ b/wiki/state.md @@ -2,11 +2,12 @@ title: Product state kind: reference status: living -updated: 2026-10-08 +updated: 2026-10-14 repos: - usput.ba confidence: medium sources: +- sources/conversations/2026-10-14--usput--tooling-models-langfuse-jev.md - sources/conversations/2026-09-30--usput--platform-direction.md - sources/planning/README.md - sources/planning/IMPLEMENTATION.md @@ -69,7 +70,7 @@ Where Usput.ba has been, what is true in the code today, what the older planning - **Moments need a check-in.** Since 2026-10-08 a moment is saved only at a place the traveller checked in at, on any of their plans, and the upload tile appears only after that check-in; guests keep the sign-in tile ([decision](decisions/capture-needs-a-check-in.md); `app/models/moment.rb`). - **Reviews are unmoderated.** `Review` has no status column; any visitor can post (the controller has no login requirement and still permits `author_name`), and every review counts toward `average_rating` (`app/models/review.rb`, `app/controllers/reviews_controller.rb`). - **Platform CLI.** `bin/platform` offers `status`, `version`, `query` and `exec` for DSL queries; there is no `chat` command (`lib/platform/cli.rb`). Executors cover schema, table queries, content, curators, external data, infrastructure and quality. The MCP server exists (`lib/platform/mcp_server.rb`) and refuses to run in production. -- **AI content.** `Ai::LocationEnricher`, `Ai::ExperienceTypeClassifier`, `Ai::AudioTourGenerator` and `Ai::ExperienceLocationSyncer` call the LLM directly with prompts under `app/prompts/`. Audio tours are generated from rake tasks and DSL, not from the dashboard. +- **AI content.** `Ai::LocationEnricher`, `Ai::ExperienceTypeClassifier`, `Ai::AudioTourGenerator` and `Ai::ExperienceLocationSyncer` call the LLM through `Ai::OpenaiQueue` with prompts under `app/prompts/`, on RubyLLM 2.1 with `gpt-6-luna` as the default model since 2026-10-14; each call logs its tokens and cost. Audio tours are generated from rake tasks and DSL, not from the dashboard. - **Accessibility.** Locations carry an `accessibility` JSONB (wheelchair level plus five features and notes); Browse rows carry `wheelchair_accessible`. See [location accessibility](initiatives/location-accessibility.md). ## Perceived