From d7875c79317b0983b98eaf91bfedb3d3e55eaf46 Mon Sep 17 00:00:00 2001 From: ZTCS Date: Fri, 12 Jun 2026 11:35:24 +0200 Subject: [PATCH 1/9] cut TTFT, startup, and streaming-render cost (keep_alive, prompt-cache-stable time, MCP catalogue cache, lazy hydration, chunked streaming markdown, windowed transcript) --- src-tauri/src/commands.rs | 35 ++++++-- src-tauri/src/db.rs | 41 +++++++++ src-tauri/src/lib.rs | 5 ++ src-tauri/src/mcp/mod.rs | 74 +++++++++++++++++ src-tauri/src/preload.rs | 9 +- src-tauri/src/providers/ollama.rs | 58 +++++++++++-- src/components/ChatCanvas.tsx | 83 +++++++++++++++++- src/components/Markdown.test.ts | 45 +++++++++- src/components/Markdown.tsx | 134 ++++++++++++++++++++++++------ src/components/Message.tsx | 12 ++- src/components/SettingsDialog.tsx | 78 ++++++++++++++++- src/lib/tauri.ts | 8 ++ src/lib/temporal.ts | 21 +++-- src/stores/chatStore.ts | 75 ++++++++++++----- src/types.ts | 15 ++++ 15 files changed, 620 insertions(+), 73 deletions(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index b4a929d..7f8348d 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -243,6 +243,15 @@ pub async fn list_messages( state.db.list_messages(&session_id).map_err(err) } +/// Per-session message counts (`{ session_id: count }`). Lets the frontend +/// cull empty sessions at startup without loading every transcript over IPC. +#[tauri::command] +pub async fn session_message_counts( + state: State<'_, AppState>, +) -> Result, String> { + state.db.session_message_counts().map_err(err) +} + #[derive(Debug, Deserialize)] pub struct AppendMessageArgs { pub session_id: String, @@ -624,7 +633,14 @@ pub async fn ollama_preload_model( base_url: String, model: String, ) -> Result<(), String> { - providers::ollama::preload_model(&state.http, &base_url, &model) + let keep_alive = state + .db + .get_setting("ollama_keep_alive") + .ok() + .flatten() + .as_deref() + .and_then(providers::ollama::keep_alive_value); + providers::ollama::preload_model(&state.http, &base_url, &model, keep_alive) .await .map_err(err) } @@ -1398,7 +1414,7 @@ pub async fn mcp_save( .map(|m| serde_json::to_string(m).map_err(err)) .transpose()?; - state + let saved = state .db .upsert_mcp_server( input.id.as_deref(), @@ -1407,12 +1423,18 @@ pub async fn mcp_save( headers_json.as_deref(), input.enabled.unwrap_or(true), ) - .map_err(err) + .map_err(err)?; + // The cached tool catalogue (and the slugs derived from server names) is + // now stale — drop it so the next send re-aggregates with this change. + crate::mcp::invalidate_tools_cache(&state.mcp_tools_cache).await; + Ok(saved) } #[tauri::command] pub async fn mcp_delete(state: State<'_, AppState>, id: String) -> Result<(), String> { - state.db.delete_mcp_server(&id).map_err(err) + state.db.delete_mcp_server(&id).map_err(err)?; + crate::mcp::invalidate_tools_cache(&state.mcp_tools_cache).await; + Ok(()) } /// Probe the given MCP server config (handshake + list tools). Accepts the @@ -1456,6 +1478,7 @@ pub async fn chat_stream( let registry = state.streams.clone(); let provider = request.provider.clone(); let db = state.db.clone(); + let mcp_cache = state.mcp_tools_cache.clone(); // Register the cancel Notify SYNCHRONOUSLY here — before we spawn the // worker — so a `chat_cancel(stream_id)` arriving while we're still @@ -1481,7 +1504,7 @@ pub async fn chat_stream( let (tools, errors) = if request.private { (Vec::new(), Vec::new()) } else { - let agg_fut = crate::mcp::aggregate_tools(&db); + let agg_fut = crate::mcp::aggregate_tools_cached(&db, &mcp_cache); tokio::select! { biased; _ = cancel.notified() => { @@ -1882,6 +1905,8 @@ pub async fn import_data_with_dialog( .await .map_err(|e| format!("import restore task panicked: {e}"))??; + // The restore rewrote the mcp_servers table — drop any cached catalogue. + crate::mcp::invalidate_tools_cache(&state.mcp_tools_cache).await; Ok(Some(stats)) } diff --git a/src-tauri/src/db.rs b/src-tauri/src/db.rs index 8c3cc09..a0a346b 100644 --- a/src-tauri/src/db.rs +++ b/src-tauri/src/db.rs @@ -913,6 +913,24 @@ impl Database { }) } + /// Per-session message counts in one indexed pass — `SELECT session_id, + /// COUNT(*) ... GROUP BY session_id`, covered by `idx_messages_session`. + /// Lets the frontend's startup empty-session cull learn which chats are + /// empty without loading every session's full transcript (with inlined + /// attachments) over IPC. Sessions with zero messages don't appear in the + /// map (GROUP BY emits only sessions that have rows), so a missing key + /// means count 0. + pub fn session_message_counts(&self) -> Result> { + self.with_read(|conn| { + let mut stmt = + conn.prepare("SELECT session_id, COUNT(*) FROM messages GROUP BY session_id")?; + let map = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? + .collect::, _>>()?; + Ok(map) + }) + } + pub fn append_message( &self, session_id: &str, @@ -2388,6 +2406,29 @@ mod tests { assert!(db.list_messages(&s.id).unwrap().is_empty()); } + #[test] + fn session_message_counts_groups_by_session_and_omits_empty() { + let (db, _dir) = fresh_db(); + let a = db.create_session("a", "ollama", "llama3", None, None).unwrap(); + let b = db.create_session("b", "ollama", "llama3", None, None).unwrap(); + // `c` stays empty — it must be ABSENT from the map (GROUP BY emits only + // sessions that have rows). The startup cull relies on a missing key + // meaning "0 messages"; if this query ever started emitting a 0 row for + // empty sessions the cull would still work, but the absence is the + // contract the frontend's `counts[id] ?? 0` is written against. + let c = db.create_session("c", "ollama", "llama3", None, None).unwrap(); + + db.append_message(&a.id, "user", "hi", None).unwrap(); + db.append_message(&a.id, "assistant", "yo", None).unwrap(); + db.append_message(&b.id, "user", "solo", None).unwrap(); + + let counts = db.session_message_counts().unwrap(); + assert_eq!(counts.get(&a.id), Some(&2)); + assert_eq!(counts.get(&b.id), Some(&1)); + assert_eq!(counts.get(&c.id), None); + assert_eq!(counts.len(), 2); + } + #[test] fn space_crud_and_session_link() { let (db, _dir) = fresh_db(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 5b24897..b24a8e4 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -48,6 +48,9 @@ pub struct AppState { pub db: Arc, pub http: reqwest::Client, pub streams: StreamRegistry, + /// Cached MCP tool catalogue so each chat send doesn't re-handshake + /// every enabled server. Invalidated on MCP config changes and restores. + pub mcp_tools_cache: crate::mcp::ToolsCache, } #[cfg_attr(mobile, tauri::mobile_entry_point)] @@ -174,6 +177,7 @@ pub fn run() { db, http, streams: StreamRegistry::new(), + mcp_tools_cache: crate::mcp::new_tools_cache(), }; app.manage(state); @@ -266,6 +270,7 @@ pub fn run() { commands::update_session_params, commands::export_session, commands::list_messages, + commands::session_message_counts, commands::append_message, commands::import_messages, commands::delete_import_group, diff --git a/src-tauri/src/mcp/mod.rs b/src-tauri/src/mcp/mod.rs index 32a9da9..e1ab0e9 100644 --- a/src-tauri/src/mcp/mod.rs +++ b/src-tauri/src/mcp/mod.rs @@ -35,6 +35,8 @@ pub use types::{Attachment, McpCallResult, McpTestResult, McpToolDef}; use anyhow::{anyhow, bail, Context, Result}; use reqwest::Url; use serde_json::Value; +use std::sync::Arc; +use std::time::{Duration, Instant}; use crate::db::{Database, McpServer}; @@ -123,6 +125,78 @@ pub async fn aggregate_tools( (defs, errors) } +/// How long a cached tool catalogue stays fresh before the next chat send +/// re-probes every enabled server. Config edits (add / edit / delete / +/// enable-toggle) and snapshot restores invalidate the cache eagerly via +/// [`invalidate_tools_cache`], so this TTL only bounds how long a server's +/// *own* tool-list change (rare) — or a transient reachability change — +/// stays reflected. Chosen long enough to cover a normal read-then-reply +/// chat cadence (where a 60 s TTL would expire between messages and defeat +/// the purpose) while keeping staleness modest. +const TOOLS_CACHE_TTL: Duration = Duration::from_secs(300); + +/// One cached [`aggregate_tools`] result plus the wall-clock instant it was +/// filled, so callers can age it out against [`TOOLS_CACHE_TTL`]. +#[derive(Clone)] +pub struct CachedTools { + cached_at: Instant, + tools: Vec, + errors: Vec<(String, String)>, +} + +/// Process-lifetime cache of the aggregated MCP tool catalogue, parked on +/// `AppState`. `None` = empty/invalidated, so the next read repopulates it. +pub type ToolsCache = Arc>>; + +/// Construct a fresh, empty tool cache for `AppState`. +pub fn new_tools_cache() -> ToolsCache { + Arc::new(tokio::sync::Mutex::new(None)) +} + +/// [`aggregate_tools`] for the chat hot path, served from `cache` when the +/// entry is still within [`TOOLS_CACHE_TTL`]. On a miss (or expiry) it runs +/// the live probe — DNS-pin + initialize + tools/list per enabled server — +/// and stores the result so subsequent sends in the same conversation skip +/// that whole round-trip. Returned shape is identical to `aggregate_tools` +/// so the caller (and its cancel-aware `select!`) is unchanged; a cache hit +/// simply resolves near-instantly. +/// +/// No single-flight guard: two simultaneous misses may both probe and the +/// last writer wins — acceptable for a desktop app where concurrent sends to +/// the same MCP set are rare, and far simpler than coordinating a barrier. +pub async fn aggregate_tools_cached( + db: &Database, + cache: &ToolsCache, +) -> (Vec, Vec<(String, String)>) { + { + let guard = cache.lock().await; + if let Some(entry) = guard.as_ref() { + if entry.cached_at.elapsed() < TOOLS_CACHE_TTL { + return (entry.tools.clone(), entry.errors.clone()); + } + } + } + let (tools, errors) = aggregate_tools(db).await; + { + let mut guard = cache.lock().await; + *guard = Some(CachedTools { + cached_at: Instant::now(), + tools: tools.clone(), + errors: errors.clone(), + }); + } + (tools, errors) +} + +/// Drop any cached catalogue so the next chat send re-probes the servers. +/// Must be called after every mutation of the `mcp_servers` table (add / +/// edit / delete / enable-toggle) and after a snapshot restore, or the model +/// would keep seeing the pre-change tool set (including stale slugs, which +/// derive from server names) for up to [`TOOLS_CACHE_TTL`]. +pub async fn invalidate_tools_cache(cache: &ToolsCache) { + *cache.lock().await = None; +} + async fn collect_one(server: &McpServer, slug: &str) -> Result> { let (http, _) = pin_client_for(server).await?; let mut session = McpSession::new(http, server)?; diff --git a/src-tauri/src/preload.rs b/src-tauri/src/preload.rs index 5e7f3e7..a9bfabb 100644 --- a/src-tauri/src/preload.rs +++ b/src-tauri/src/preload.rs @@ -148,10 +148,17 @@ pub fn try_warm_default_model(db: Arc, http: reqwest::Client) { } tracing::info!("preload: warming {model} via {base_url}"); + // Warm with the user's configured keep_alive so the model doesn't + // evaporate before their first message if they picked a longer + // residency than Ollama's 5-minute default. + let keep_alive = settings + .get("ollama_keep_alive") + .map(String::as_str) + .and_then(crate::providers::ollama::keep_alive_value); // `preload_model` swallows its own transport errors (timeout, // connection refused, 4xx model-not-found) and returns Ok(()). // The `let _ =` is belt-and-braces for any future signature change. - let _ = crate::providers::ollama::preload_model(&http, &base_url, &model).await; + let _ = crate::providers::ollama::preload_model(&http, &base_url, &model, keep_alive).await; }); } diff --git a/src-tauri/src/providers/ollama.rs b/src-tauri/src/providers/ollama.rs index f7765a2..925c972 100644 --- a/src-tauri/src/providers/ollama.rs +++ b/src-tauri/src/providers/ollama.rs @@ -449,21 +449,31 @@ pub async fn unload_model(http: &Client, base_url: &str, model: &str) -> Result< Ok(()) } -/// Preload a model into VRAM by sending an empty chat. The default Ollama -/// keep_alive (5m) takes over after the load completes, so the model stays -/// resident long enough for the user's first real request to skip the cold -/// load. Errors are swallowed — preload is best-effort and must never block -/// app startup if Ollama is unreachable or the model is missing. -pub async fn preload_model(http: &Client, base_url: &str, model: &str) -> Result<()> { +/// Preload a model into VRAM by sending an empty chat. `keep_alive` controls +/// how long Ollama keeps it resident after the warm completes — pass the +/// user's configured value so the model survives until their first real +/// request even if that's more than Ollama's built-in 5-minute default away +/// (`None` falls back to that default). Errors are swallowed — preload is +/// best-effort and must never block app startup if Ollama is unreachable or +/// the model is missing. +pub async fn preload_model( + http: &Client, + base_url: &str, + model: &str, + keep_alive: Option, +) -> Result<()> { super::refuse_link_local_host(base_url) .await .map_err(|e| anyhow!(e))?; let url = format!("{}/api/chat", base_url.trim_end_matches('/')); - let body = serde_json::json!({ + let mut body = serde_json::json!({ "model": model, "messages": [], "stream": false, }); + if let Some(ka) = keep_alive { + body["keep_alive"] = ka; + } // Preload can legitimately take longer than ADMIN_TIMEOUT for large // models (a 70 B model cold-loading from disk easily exceeds 30 s), // so give it a more generous ceiling. Still bounded so a wedged @@ -582,6 +592,17 @@ pub async fn chat_stream( .collect::>())) }; + // How long Ollama should keep this model resident after the request. + // Read once per stream from the global setting and sent on every turn so + // each round-trip resets the idle timer. Unset → omit the field and let + // Ollama apply its 5-minute default (Loach's pre-setting behaviour). + let keep_alive: Option = db + .get_setting("ollama_keep_alive") + .ok() + .flatten() + .as_deref() + .and_then(keep_alive_value); + let start = Instant::now(); let mut total_tokens: u32 = 0; let mut think_already_drop: bool = false; // sticky retry guard across turns @@ -600,6 +621,9 @@ pub async fn chat_stream( "stream": true, "options": build_options(&req), }); + if let Some(ka) = keep_alive.as_ref() { + body["keep_alive"] = ka.clone(); + } if let Some(t) = req.params.think { if !think_already_drop { body["think"] = json!(t); @@ -1009,6 +1033,26 @@ fn emit_metrics(app: &AppHandle, channel: &str, tokens: u32, start: Instant) { ); } +/// Translate the stored `ollama_keep_alive` setting string into the JSON +/// value Ollama's `keep_alive` field expects. Ollama accepts either a Go +/// duration string ("5m", "30m", "1h") or an integer number of seconds, +/// where a negative value means "keep the model resident until it is +/// explicitly unloaded". We send the until-unloaded sentinel as a JSON +/// number (`-1`) rather than the string `"-1"` because Go's +/// `time.ParseDuration` rejects a unit-less string. An empty / unset value +/// yields `None`, letting Ollama apply its built-in 5-minute idle default — +/// exactly the behaviour Loach had before this setting existed. +pub fn keep_alive_value(setting: &str) -> Option { + let s = setting.trim(); + if s.is_empty() { + return None; + } + match s.parse::() { + Ok(n) => Some(json!(n)), + Err(_) => Some(json!(s)), + } +} + fn build_options(req: &ChatRequest) -> Value { let mut o = serde_json::Map::new(); let p = &req.params; diff --git a/src/components/ChatCanvas.tsx b/src/components/ChatCanvas.tsx index d408cd6..78eb49c 100644 --- a/src/components/ChatCanvas.tsx +++ b/src/components/ChatCanvas.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useRef, useState } from "react"; +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { ArrowDown, ChevronDown, ChevronUp, Hourglass, Import, Search, Sparkles, Trash2, Zap, X } from "lucide-react"; import { MessageItem } from "./Message"; import { Button } from "@/components/ui/button"; @@ -10,6 +10,13 @@ import type { Message } from "@/types"; const EMPTY_MESSAGES: Message[] = []; +/** How many trailing render items mount when a chat opens. Older items reveal + * on demand via the "Show earlier messages" control. Opening a long chat used + * to mount (and markdown-parse + highlight) every message synchronously; + * windowing bounds that to the last screenful-plus. Counted in render items + * (a folded import batch is one item), which is what actually mounts. */ +const WINDOW_SIZE = 50; + /** A transcript row to render: either a single normal message or a *hidden* * imported batch (a run of rows sharing one `import_group`) folded into one * collapsible card. Visible imports are NOT folded — they render as ordinary @@ -136,6 +143,14 @@ export function ChatCanvas() { // sync via the same scroll handler that updates `stickToBottom`. const [showScrollButton, setShowScrollButton] = useState(false); + // Transcript windowing (see WINDOW_SIZE). Only the last `visibleCount` render + // items mount on open; `visibleCount` only ever GROWS within a session view + // (so revealing older rows never re-collapses and jolts the scroll) and + // resets on chat switch via the effect below. `prependAnchor` lets the layout + // effect keep the viewport pinned when a reveal prepends rows above the fold. + const [visibleCount, setVisibleCount] = useState(WINDOW_SIZE); + const prependAnchor = useRef<{ height: number; top: number } | null>(null); + // ---------------- Search-in-chat ---------------- // A browser-style "find on page" overlay scoped to the current chat's // messages. Triggered by the chat header's "Search in chat" menu item via @@ -331,6 +346,32 @@ export function ChatCanvas() { if (el) el.scrollTop = el.scrollHeight; }, [messages, isStreaming, waitingHere]); + // Reset the window when switching chats — the new transcript opens at its own + // tail, not the depth the previous chat happened to be revealed to. + useEffect(() => { + setVisibleCount(WINDOW_SIZE); + }, [sessionId]); + + // Searching needs every matching row mounted: jump-to-match and the inline + // highlighter both reach messages through per-message DOM refs, which only + // exist for mounted rows. Expand fully while the finder is open. The + // expansion sticks (visibleCount only grows) so closing search never + // re-collapses the list and yanks the scroll position. + useEffect(() => { + if (searchOpen) setVisibleCount(Number.MAX_SAFE_INTEGER); + }, [searchOpen]); + + // After a reveal prepends older rows above the fold, shift scrollTop by the + // height those rows added so the rows the user was reading stay put. + useLayoutEffect(() => { + const anchor = prependAnchor.current; + if (!anchor) return; + prependAnchor.current = null; + const el = scrollerRef.current; + if (!el) return; + el.scrollTop = anchor.top + (el.scrollHeight - anchor.height); + }, [visibleCount]); + const scrollToBottom = () => { const el = scrollerRef.current; if (!el) return; @@ -339,6 +380,14 @@ export function ChatCanvas() { setShowScrollButton(false); }; + // Reveal the next batch of older rows. Snapshot the scroller geometry first + // so the layout effect above can re-anchor once the prepended rows mount. + const showEarlierMessages = () => { + const el = scrollerRef.current; + if (el) prependAnchor.current = { height: el.scrollHeight, top: el.scrollTop }; + setVisibleCount((c) => c + WINDOW_SIZE); + }; + if (!sessionId) { return (
@@ -354,11 +403,41 @@ export function ChatCanvas() { ); } + // Only the trailing window mounts. Items keep their original `index` / + // `startIndex` (computed against the full `messages` array), so the + // compaction-divider placement and `isLast` checks below stay correct on the + // slice. `hiddenBefore` is how many messages sit above the window. + const visibleItems = + renderItems.length > visibleCount + ? renderItems.slice(renderItems.length - visibleCount) + : renderItems; + const firstVisible = visibleItems[0]; + const hiddenBefore = firstVisible + ? firstVisible.kind === "message" + ? firstVisible.index + : firstVisible.startIndex + : 0; + return (
- {renderItems.map((item) => { + {hiddenBefore > 0 && ( +
+ +
+ )} + {visibleItems.map((item) => { // The compaction divider sits before whichever render item holds // the first still-active (uncompacted) message. For an import // card that's a range check, since the card spans several rows. diff --git a/src/components/Markdown.test.ts b/src/components/Markdown.test.ts index e741f90..7f1dce0 100644 --- a/src/components/Markdown.test.ts +++ b/src/components/Markdown.test.ts @@ -8,7 +8,7 @@ // schemes; javascript:/file:/data:/relative hrefs are refused. import { describe, it, expect } from "vitest"; -import { __testing } from "./Markdown"; +import { __testing, stableSplit } from "./Markdown"; const { texReplace, rehypeTexFallback, shouldOpenExternally } = __testing; @@ -82,6 +82,49 @@ describe("rehypeTexFallback", () => { }); }); +describe("stableSplit (streaming prefix/tail boundary)", () => { + // Round-trip helper — the split must never lose or reorder a byte. + const split = (s: string) => { + const at = stableSplit(s); + const stable = s.slice(0, at); + const tail = s.slice(at); + expect(stable + tail).toBe(s); + return [stable, tail] as const; + }; + + it("cuts at the last top-level blank line", () => { + const [stable, tail] = split("para one\n\npara two\n\npara three"); + expect(stable).toBe("para one\n\npara two\n\n"); + expect(tail).toBe("para three"); + }); + + it("returns 0 for a single still-growing block (no boundary yet)", () => { + expect(stableSplit("still typing the first paragraph")).toBe(0); + }); + + it("never splits inside an open code fence", () => { + // The blank line is INSIDE the unterminated fence, so the whole block must + // stay in the tail (rendered without highlight) until the fence closes. + const [stable, tail] = split("intro\n\n```py\na = 1\n\nb = 2"); + expect(stable).toBe("intro\n\n"); + expect(tail).toBe("```py\na = 1\n\nb = 2"); + }); + + it("treats a blank line after a CLOSED fence as a boundary", () => { + const [stable, tail] = split("```py\nx = 1\n```\n\nnext para"); + expect(stable).toBe("```py\nx = 1\n```\n\n"); + expect(tail).toBe("next para"); + }); + + it("puts everything in the stable prefix when content ends at a boundary", () => { + // A pause at a block boundary (content ending in a blank line) renders + // everything as the highlighted, memoised prefix and an empty tail. + const [stable, tail] = split("a\n\nb\n\n"); + expect(stable).toBe("a\n\nb\n\n"); + expect(tail).toBe(""); + }); +}); + describe("shouldOpenExternally (link scheme allow-list)", () => { it("allows absolute http / https / mailto", () => { expect(shouldOpenExternally("https://example.com")).toBe(true); diff --git a/src/components/Markdown.tsx b/src/components/Markdown.tsx index 30f573a..500fccf 100644 --- a/src/components/Markdown.tsx +++ b/src/components/Markdown.tsx @@ -1,4 +1,4 @@ -import { memo, type ReactNode } from "react"; +import { memo, useMemo, type ReactNode } from "react"; import ReactMarkdown, { type Components } from "react-markdown"; import remarkGfm from "remark-gfm"; import rehypeHighlight from "rehype-highlight"; @@ -212,6 +212,14 @@ function extractRaw(node: ReactNode): string { // bubble where this component re-renders thousands of times. const MARKDOWN_PLUGINS_REMARK = [remarkGfm]; const MARKDOWN_PLUGINS_REHYPE = [rehypeTexFallback, rehypeHighlight]; +// Same pipeline minus syntax highlighting, used for the still-streaming tail +// of a message. Re-highlighting a growing block on every animation-frame flush +// is the dominant per-token render cost (worst case `highlightAuto` runs every +// grammar for an as-yet-unknown language); skipping it while the block is live +// — then re-rendering the whole message *with* highlighting once streaming +// stops — keeps streaming cheap without changing the settled view. The TeX +// fallback stays (cheap, and keeps inline symbols consistent across the split). +const MARKDOWN_PLUGINS_REHYPE_NOHL = [rehypeTexFallback]; // Open a link from rendered (untrusted) markdown through the OS browser / mail // client. Mirrors the helper used by Settings / Onboarding / Updates; the @@ -278,36 +286,110 @@ const MARKDOWN_COMPONENTS: Components = { }, }; -// `memo` so non-streaming messages skip the entire react-markdown -// re-parse when their parent re-renders for unrelated reasons. The -// streaming bubble still re-renders per token (content does change), -// but at least the components-object allocation no longer happens each -// tick. +// `prose-sm` is our base — chat reads in chunks, not articles, so we want +// compact rhythm. The custom theme in tailwind.config.ts dials line-height +// back up and rewires colours / borders to fit the glass surface. The +// first/last resets stop a leading/trailing block adding stray margin (the +// bubble already provides padding). +const PROSE_CLASS = cn( + "prose prose-sm prose-invert max-w-none", + "prose-p:first:mt-0 prose-p:last:mb-0", +); + +// The bare react-markdown invocation, memoised so an unchanged `content` (+ +// `highlight`) skips the whole remark/rehype re-parse when a parent re-renders. +// Renders no wrapper element, so several can sit inside one `prose` container +// and have block margins collapse normally across the boundary — that's what +// lets `StreamingMarkdown` split a message without a visible spacing seam. +const MarkdownBody = memo(function MarkdownBody({ + content, + highlight, +}: { + content: string; + highlight: boolean; +}) { + return ( + + {content} + + ); +}); + +// `memo` so non-streaming messages skip the entire react-markdown re-parse when +// their parent re-renders for unrelated reasons. export const Markdown = memo(function Markdown({ content, className, }: MarkdownProps) { return ( -
- - {content} - +
+ +
+ ); +}); + +// Where to split a still-streaming message into a stable prefix (already- +// complete blocks) and the live tail (the block currently being typed). We cut +// at the last blank line sitting at the TOP level — not inside an open code +// fence — because a blank line there is always a CommonMark block boundary, so +// the prefix renders identically whether or not the tail is appended. Returns +// the prefix length; 0 when there's no safe split yet. +// +// Fence tracking matters two ways: a blank line *inside* a ``` block isn't a +// boundary, and a long code block streams as one growing tail — which the +// no-highlight tail renderer keeps cheap until its closing fence lands and it +// folds into the highlighted, memoised prefix. +export function stableSplit(content: string): number { + const lines = content.split("\n"); + let inFence = false; + let fenceChar = ""; + let offset = 0; + let boundary = 0; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const fence = /^\s{0,3}(`{3,}|~{3,})/.exec(line); + if (fence) { + const ch = fence[1][0]; + if (!inFence) { + inFence = true; + fenceChar = ch; + } else if (ch === fenceChar) { + inFence = false; + } + } + offset += line.length + 1; // +1 for the "\n" that split() consumed + // A top-level blank line closes the preceding block. Never treat the very + // last line as a boundary — the tail must keep the trailing block so a + // message ending in "\n\n" doesn't render an empty tail. + if (!inFence && line.trim() === "" && i < lines.length - 1) { + boundary = offset; + } + } + return boundary; +} + +// Streaming variant of `Markdown`: only the trailing, still-growing block is +// re-parsed per flush; completed blocks render once through the memoised +// `MarkdownBody`, and the live tail skips syntax highlighting. Both halves +// render inside ONE prose container so block spacing across the split matches a +// single document. Once streaming ends, callers swap back to `Markdown`, which +// re-renders the whole message with highlighting — so the settled view is +// identical to what it was before this optimisation. +export const StreamingMarkdown = memo(function StreamingMarkdown({ + content, + className, +}: MarkdownProps) { + const splitAt = useMemo(() => stableSplit(content), [content]); + const stable = content.slice(0, splitAt); + const tail = content.slice(splitAt); + return ( +
+ {stable.length > 0 && } +
); }); diff --git a/src/components/Message.tsx b/src/components/Message.tsx index 0ea7602..90fafbd 100644 --- a/src/components/Message.tsx +++ b/src/components/Message.tsx @@ -16,7 +16,7 @@ import { Wrench, } from "lucide-react"; import { AttachmentActions } from "./AttachmentActions"; -import { Markdown } from "./Markdown"; +import { Markdown, StreamingMarkdown } from "./Markdown"; import { MarkdownSourceProvider } from "./markdownSource"; import { lastCodeBlock } from "@/lib/codeBlocks"; import { @@ -690,7 +690,15 @@ function MessageItemImpl({ message, isStreaming, metrics, canRegenerate }: Messa lastBlockRaw, }} > - + {/* While streaming, only the trailing block re-parses per flush + (and skips highlighting) — completed blocks render once. Once + done, fall back to the plain whole-message render so the + settled view (and its full highlighting) is unchanged. */} + {isStreaming ? ( + + ) : ( + + )}
)} diff --git a/src/components/SettingsDialog.tsx b/src/components/SettingsDialog.tsx index 6e4a559..a547591 100644 --- a/src/components/SettingsDialog.tsx +++ b/src/components/SettingsDialog.tsx @@ -90,7 +90,14 @@ import { import { useSecurityStore } from "@/stores/securityStore"; import { cn } from "@/lib/utils"; import { DEFAULT_TONE_ID, TONES } from "@/lib/tones"; -import type { FontSize, ImportStats, ModelInfo, ProviderId, Session } from "@/types"; +import type { + FontSize, + ImportStats, + ModelInfo, + OllamaKeepAlive, + ProviderId, + Session, +} from "@/types"; import pkg from "../../package.json"; type ConnTestState = @@ -670,6 +677,31 @@ export function SettingsDialog() { />
+ + + +
+ +

+ How long Ollama keeps the model in VRAM after a reply. + Longer means the next message skips the multi-second cold + reload, at the cost of pinning VRAM while idle.{" "} + + Always + {" "} + keeps it resident until you unload it or quit Ollama. + Ignored by OpenAI API providers. +

+ + settings.update("ollama_keep_alive", next) + } + /> +
@@ -1604,6 +1636,50 @@ const FONT_SIZE_OPTIONS: { value: FontSize; label: string; previewPx: number }[] { value: "large", label: "Large", previewPx: 16 }, ]; +const KEEP_ALIVE_OPTIONS: { value: OllamaKeepAlive; label: string }[] = [ + { value: "5m", label: "5 min" }, + { value: "30m", label: "30 min" }, + { value: "1h", label: "1 hour" }, + { value: "-1", label: "Always" }, +]; + +function KeepAliveSwitch({ + value, + onChange, +}: { + value: OllamaKeepAlive; + onChange: (next: OllamaKeepAlive) => void; +}) { + return ( +
+ {KEEP_ALIVE_OPTIONS.map((opt) => { + const selected = value === opt.value; + return ( + + ); + })} +
+ ); +} + function FontSizeSwitch({ value, onChange, diff --git a/src/lib/tauri.ts b/src/lib/tauri.ts index e42f7be..60054a4 100644 --- a/src/lib/tauri.ts +++ b/src/lib/tauri.ts @@ -169,6 +169,14 @@ export function listMessages(sessionId: string): Promise { return invoke("list_messages", { sessionId }); } +/** Per-session message counts (`{ sessionId: count }`). A session with zero + * messages is absent from the map — callers treat a missing key as 0. Used + * for the startup empty-session cull without loading every transcript. */ +export function sessionMessageCounts(): Promise> { + if (!isTauri) return notInTauri>({}); + return invoke("session_message_counts", {}); +} + export function appendMessage(args: { session_id: string; role: "user" | "assistant" | "system"; diff --git a/src/lib/temporal.ts b/src/lib/temporal.ts index eb1a166..f8351f2 100644 --- a/src/lib/temporal.ts +++ b/src/lib/temporal.ts @@ -1,10 +1,12 @@ /** * Temporal awareness helpers. * - * Models have no clock of their own, so we inject the current date / time - * into the system prompt of every request. Compatible with Open WebUI's - * `{{CURRENT_DATE}}` / `{{CURRENT_TIME}}` / `{{CURRENT_WEEKDAY}}` template - * syntax — authors who want precise placement can drop the placeholders + * Models have no clock of their own, so we inject the current date / + * weekday / timezone into the system prompt of every request. Compatible + * with Open WebUI's `{{CURRENT_DATE}}` / `{{CURRENT_TIME}}` / + * `{{CURRENT_WEEKDAY}}` template syntax — authors who want precise + * placement (including the minute-precision time, which the auto preamble + * deliberately omits — see `temporalPreamble`) can drop the placeholders * directly into their system prompt. Everyone else gets a short preamble * auto-prepended. * @@ -95,9 +97,18 @@ export function promptUsesTemporalVars(prompt: string): boolean { /** * Short stand-alone preamble suitable for prepending to any system prompt. * Kept deliberately terse so it doesn't dominate small context windows. + * + * Carries only date / weekday / timezone — values that change at most once + * a day. Minute-precision time is intentionally NOT included: this preamble + * sits at the very front of the system prompt, i.e. the head of the model's + * KV-cache prefix, so a per-minute timestamp here would invalidate Ollama's + * cached prefix on nearly every follow-up message and force a full + * re-evaluation of the whole conversation. Users who need the exact wall + * time can place `{{CURRENT_TIME}}` explicitly (accepting that trade-off) or + * enable the datetime built-in tool, which the model can call on demand. */ export function temporalPreamble(vars: TemporalVars): string { - return `Current date: ${vars.CURRENT_DATE} (${vars.CURRENT_WEEKDAY}). Current local time: ${vars.CURRENT_TIME} (${vars.CURRENT_TIMEZONE}).`; + return `Current date: ${vars.CURRENT_DATE} (${vars.CURRENT_WEEKDAY}, ${vars.CURRENT_TIMEZONE}).`; } /** diff --git a/src/stores/chatStore.ts b/src/stores/chatStore.ts index c134460..b59e595 100644 --- a/src/stores/chatStore.ts +++ b/src/stores/chatStore.ts @@ -13,6 +13,7 @@ import { getSpaceContext, listMessages, listSessions, + sessionMessageCounts, makeRequestId, markMessagesCompacted, ollamaUnloadModel, @@ -1320,24 +1321,22 @@ export const useChatStore = create((set, get) => ({ // Archived sessions are left alone even if empty — the user explicitly // moved them to the archive and shouldn't have them silently culled. // - // Fan out the per-session message reads in parallel — they're - // independent IPC calls and the prior sequential await was the dominant - // cost of `hydrate()` for users with many chats. - const loaded = await Promise.all( - sessions.map((s) => listMessages(s.id).then((msgs) => ({ s, msgs }))), + // We only need to know which sessions are EMPTY, not their contents, so + // fetch per-session counts in one indexed query instead of loading every + // transcript. The prior approach eagerly pulled every message of every + // session (with inlined base64 attachments) across IPC before first + // paint — the dominant cost of `hydrate()`, scaling unboundedly with the + // chat corpus. Transcripts now load lazily: the landing session via + // `selectSession` below, every other session on first open or first send + // (see the guard in `sendUserMessage`). + const counts = await sessionMessageCounts(); + const emptySessions: Session[] = sessions.filter( + (s) => !s.archived_at && (counts[s.id] ?? 0) === 0, ); - const messageMap: Record = {}; - const emptySessions: Session[] = []; - for (const { s, msgs } of loaded) { - messageMap[s.id] = msgs; - if (msgs.length === 0 && !s.archived_at) emptySessions.push(s); - } - set((st) => ({ messages: { ...st.messages, ...messageMap } })); // Delete all empty sessions except the first (most recent). - // `emptySessions` preserves the iteration order of `loaded`, which - // preserves the order of `sessions`, which is what `list_sessions` - // returned. The Rust query is + // `emptySessions` preserves the order of `sessions`, which is what + // `list_sessions` returned. The Rust query is // SELECT … FROM sessions ORDER BY updated_at DESC // (see `db::list_sessions`) so `emptySessions[0]` is the most- // recently-touched empty chat. If that ORDER BY ever changes, @@ -1355,11 +1354,12 @@ export const useChatStore = create((set, get) => ({ // Pick a session to land on: prefer the surviving empty chat (so a // half-typed welcome screen is preserved across restarts), otherwise - // fall back to the most recent non-archived chat. If neither exists - // we leave activeSessionId null and let the NoChatState CTA take - // over — the user explicitly emptied their sidebar and we shouldn't - // silently re-create a chat behind their back. Onboarding still - // creates the very first chat from `onboardingStore.complete()`. + // fall back to the most recent non-archived chat. `selectSession` loads + // that one session's transcript lazily. If neither exists we leave + // activeSessionId null and let the NoChatState CTA take over — the user + // explicitly emptied their sidebar and we shouldn't silently re-create a + // chat behind their back. Onboarding still creates the very first chat + // from `onboardingStore.complete()`. if (emptySessions.length > 0) { await get().selectSession(emptySessions[0].id); } else { @@ -1409,13 +1409,19 @@ export const useChatStore = create((set, get) => ({ }, newSession: async (opts) => { - // Remove all existing empty sessions (no messages) before creating a new one. + // Remove all existing empty sessions (no messages) before creating a new + // one. Use per-session counts rather than N sequential transcript loads — + // under lazy hydration most sessions' messages aren't resident, so the old + // `messages[s.id] ?? await listMessages(s.id)` would re-import the exact + // cost hydrate was changed to avoid. In-memory length wins when present + // (it reflects just-sent messages the counts query might predate). const { sessions, messages } = get(); + const counts = await sessionMessageCounts(); const emptyIds: string[] = []; for (const s of sessions) { if (s.archived_at) continue; // leave archived chats untouched - const msgs = messages[s.id] ?? await listMessages(s.id); - if (msgs.length === 0) emptyIds.push(s.id); + const count = messages[s.id]?.length ?? counts[s.id] ?? 0; + if (count === 0) emptyIds.push(s.id); } for (const id of emptyIds) { await deleteSession(id); @@ -1743,6 +1749,29 @@ export const useChatStore = create((set, get) => ({ throw new Error("No model selected. Pick one from the model dropdown."); } + // Lazy hydration: a session other than the one loaded at startup only + // fetches its transcript when opened. A send can race that load (the user + // hits Enter the instant they switch chats), so make sure the full history + // is resident before appending the user message below — otherwise the + // `?? []` fallbacks when building the request would start from an empty + // transcript and silently drop the conversation's context. A freshly + // created session is already seeded with `[]` in newSession, so this only + // fetches for existing chats opened-then-sent before their load settled. + if (!get().messages[sessionId!]) { + try { + const msgs = await listMessages(sessionId!); + set((s) => ({ messages: { ...s.messages, [sessionId!]: msgs } })); + } catch (e) { + logger.error("failed to load chat history before send", e); + useToastStore.getState().push({ + kind: "error", + title: "Couldn't open chat", + body: e instanceof Error ? e.message : String(e), + }); + return; + } + } + // Hard cap: at most one in-flight task per chat. If this session is // already the runner or already has a waiter, drop this submit. The UI // disables the send button in this state too — belt-and-suspenders. diff --git a/src/types.ts b/src/types.ts index b4764b3..d8e792e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -309,6 +309,13 @@ export type ThemeChoice = "light" | "dark" | "system"; export type BackgroundStyle = "gradient" | "solid"; export type FontSize = "small" | "normal" | "large"; +/** How long Ollama keeps a model resident in VRAM after a request. Maps to + * the `keep_alive` field on `/api/chat`: a Go duration string, or `"-1"` + * (sent as the integer -1) to keep it loaded until explicitly unloaded. + * `"5m"` matches Ollama's own default — the value Loach effectively used + * before this setting existed. */ +export type OllamaKeepAlive = "5m" | "30m" | "1h" | "-1"; + export interface Settings { theme: ThemeChoice; background_style: BackgroundStyle; @@ -388,6 +395,13 @@ export interface Settings { * hardware where you'd otherwise have to remember to flip the per-chat * toggle. Off by default. Ignored by OpenAI providers. */ low_vram_global: boolean; + /** How long Ollama keeps the resolved model resident in VRAM after each + * request (the `keep_alive` field). The default `"5m"` matches Ollama's + * own idle-unload timeout; longer values (or `"-1"` = until unloaded) + * stop the model being evicted between messages, so a reply after a pause + * skips the multi-second cold reload — at the cost of pinned VRAM. + * Ignored by OpenAI providers. */ + ollama_keep_alive: OllamaKeepAlive; /** Default value for the per-chat Thinking toggle. Applied as a baseline * in `readSessionParams` so new chats (and chats that haven't touched * the slider) inherit it. The per-chat Thinking switch in the parameter @@ -433,6 +447,7 @@ export const DEFAULT_SETTINGS: Settings = { ip_tool_enabled: false, pdf_tool_enabled: false, low_vram_global: false, + ollama_keep_alive: "5m", thinking_default: true, default_tone_id: "default", onboarding_completed: false, From 1ecbcba19a654fb64769eebbc6ea3a8700a4e645 Mon Sep 17 00:00:00 2001 From: ZTCS Date: Fri, 12 Jun 2026 15:52:52 +0200 Subject: [PATCH 2/9] land MEDIUM optimizations (release profile, preload options, OpenAI keyring hoist, render-subscription + JSON-parse memoization, Private Chat batching, Code Canvas highlight gating, slider commit, space-context cache, cancellable memory extraction, cold-launch flash, code-splitting) --- index.html | 7 ++ src-tauri/Cargo.toml | 17 +++++ src-tauri/src/commands.rs | 6 +- src-tauri/src/preload.rs | 16 ++++- src-tauri/src/providers/ollama.rs | 43 ++++++++++++ src-tauri/src/providers/openai.rs | 20 +++++- src-tauri/tauri.conf.json | 3 +- src/App.tsx | 102 ++++++++++++++++++++------- src/components/CodeCanvas.tsx | 17 ++++- src/components/CodeView.tsx | 13 +++- src/components/Message.tsx | 19 ++++- src/components/ParameterPanel.tsx | 29 ++++++-- src/lib/memory.ts | 30 +++++++- src/lib/spaceContextCache.ts | 38 ++++++++++ src/lib/tauri.ts | 18 ++++- src/stores/chatStore.ts | 37 ++++++++-- src/stores/privateChatStore.ts | 112 ++++++++++++++++++++++++++++-- src/stores/spaceStore.ts | 14 ++++ 18 files changed, 487 insertions(+), 54 deletions(-) create mode 100644 src/lib/spaceContextCache.ts diff --git a/index.html b/index.html index b587343..f89ada5 100644 --- a/index.html +++ b/index.html @@ -4,6 +4,13 @@ Loach + +
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 6eb20de..72604eb 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -91,3 +91,20 @@ tempfile = "3" # this feature is used for production builds or when a dev server is not specified # DO NOT REMOVE!! custom-protocol = ["tauri/custom-protocol"] + +[profile.release] +# Smaller binary → smaller NSIS installer + updater `.nsis.zip` download for +# every user on every release, and a faster cold start (Windows cold-start cost +# is roughly proportional to bytes read + Defender-scanned). `lto = "thin"` +# captures most of full LTO's size win for a fraction of the link time; +# `codegen-units = 1` lets the optimizer see the whole crate; `strip` drops the +# symbol table (a real win on the Linux/macOS artifacts — on Windows MSVC the +# symbols already live in a separate, unshipped PDB). +# +# Intentionally NOT set: `panic = "abort"` (Tauri commands rely on unwinding a +# panic and catching it — abort would turn any command panic into a full-app +# crash) and `opt-level = "s"` (would measurably slow the token-streaming serde +# hot path; the default opt-level = 3 stays). +lto = "thin" +codegen-units = 1 +strip = "symbols" diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 7f8348d..db748f2 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -632,6 +632,9 @@ pub async fn ollama_preload_model( state: State<'_, AppState>, base_url: String, model: String, + num_ctx: Option, + low_vram: Option, + num_gpu: Option, ) -> Result<(), String> { let keep_alive = state .db @@ -640,7 +643,8 @@ pub async fn ollama_preload_model( .flatten() .as_deref() .and_then(providers::ollama::keep_alive_value); - providers::ollama::preload_model(&state.http, &base_url, &model, keep_alive) + let options = providers::ollama::preload_options(num_ctx, low_vram, num_gpu); + providers::ollama::preload_model(&state.http, &base_url, &model, keep_alive, options) .await .map_err(err) } diff --git a/src-tauri/src/preload.rs b/src-tauri/src/preload.rs index a9bfabb..6bd202a 100644 --- a/src-tauri/src/preload.rs +++ b/src-tauri/src/preload.rs @@ -155,10 +155,24 @@ pub fn try_warm_default_model(db: Arc, http: reqwest::Client) { .get("ollama_keep_alive") .map(String::as_str) .and_then(crate::providers::ollama::keep_alive_value); + // Size the warmed runner like the first real request will. We can't + // read the model's Modelfile defaults from here (no /api/show plumbing + // on the startup path), so we use the app default num_ctx + the global + // low-VRAM pin; the post-unlock JS preload re-fires with the precisely + // resolved params and corrects any mismatch before the user sends. + let low_vram_global = settings.get("low_vram_global").map(String::as_str) == Some("true"); + let options = crate::providers::ollama::preload_options( + Some(crate::providers::ollama::DEFAULT_NUM_CTX), + if low_vram_global { Some(true) } else { None }, + None, + ); // `preload_model` swallows its own transport errors (timeout, // connection refused, 4xx model-not-found) and returns Ok(()). // The `let _ =` is belt-and-braces for any future signature change. - let _ = crate::providers::ollama::preload_model(&http, &base_url, &model, keep_alive).await; + let _ = crate::providers::ollama::preload_model( + &http, &base_url, &model, keep_alive, options, + ) + .await; }); } diff --git a/src-tauri/src/providers/ollama.rs b/src-tauri/src/providers/ollama.rs index 925c972..c71d553 100644 --- a/src-tauri/src/providers/ollama.rs +++ b/src-tauri/src/providers/ollama.rs @@ -461,6 +461,7 @@ pub async fn preload_model( base_url: &str, model: &str, keep_alive: Option, + options: Option, ) -> Result<()> { super::refuse_link_local_host(base_url) .await @@ -474,6 +475,12 @@ pub async fn preload_model( if let Some(ka) = keep_alive { body["keep_alive"] = ka; } + // Warm the runner with the same options the first real request will send + // (num_ctx above all) so Ollama doesn't have to reallocate the KV cache — + // i.e. reload the model — on that first message. + if let Some(opts) = options { + body["options"] = opts; + } // Preload can legitimately take longer than ADMIN_TIMEOUT for large // models (a 70 B model cold-loading from disk easily exceeds 30 s), // so give it a more generous ceiling. Still bounded so a wedged @@ -1033,6 +1040,42 @@ fn emit_metrics(app: &AppHandle, channel: &str, tokens: u32, start: Instant) { ); } +/// App-default context window, mirrored from `DEFAULT_PARAMS.num_ctx` in +/// `src/types.ts`. The startup preload (in `preload.rs`) can't resolve a +/// model's Modelfile defaults the way the frontend does, so it warms with this +/// value — the size the first real message most commonly asks for. The +/// post-unlock JS preload re-fires with the precisely-resolved params, so any +/// mismatch here is corrected before the user's first send. Keep in sync with +/// the TS default. +pub const DEFAULT_NUM_CTX: u32 = 8192; + +/// Build the Ollama `options` object for a preload so the warmed runner is +/// sized like the one the first real chat request will ask for. `num_ctx` is +/// the load-bearing one (a mismatch forces a KV-cache realloc, i.e. a reload); +/// `low_vram` / `num_gpu` matter on memory-constrained setups. Returns `None` +/// when nothing is set so the request body omits `options` entirely. +pub fn preload_options( + num_ctx: Option, + low_vram: Option, + num_gpu: Option, +) -> Option { + let mut o = serde_json::Map::new(); + if let Some(v) = num_ctx { + o.insert("num_ctx".into(), json!(v)); + } + if let Some(v) = low_vram { + o.insert("low_vram".into(), json!(v)); + } + if let Some(v) = num_gpu { + o.insert("num_gpu".into(), json!(v)); + } + if o.is_empty() { + None + } else { + Some(Value::Object(o)) + } +} + /// Translate the stored `ollama_keep_alive` setting string into the JSON /// value Ollama's `keep_alive` field expects. Ollama accepts either a Go /// duration string ("5m", "30m", "1h") or an integer number of seconds, diff --git a/src-tauri/src/providers/openai.rs b/src-tauri/src/providers/openai.rs index 8ac1529..91f2253 100644 --- a/src-tauri/src/providers/openai.rs +++ b/src-tauri/src/providers/openai.rs @@ -80,7 +80,8 @@ pub async fn list_models(http: &Client, base_url: &str) -> Result // provider at an `http://` LAN host — or a corrupted / imported // `openai_base_url` — would ship the key in cleartext on every // model-list refresh, the exact leak the chat path already prevents. - if let Some(key) = secrets::get_openai_key()? { + // Off the runtime — blocking OS-keyring read (see chat_stream's note). + if let Some(key) = tokio::task::spawn_blocking(secrets::get_openai_key).await?? { if !key.is_empty() && is_safe_for_bearer(base_url) { req = req.bearer_auth(key); } @@ -250,6 +251,19 @@ pub async fn chat_stream( .collect::>())) }; + // Read the OpenAI key ONCE for the whole stream, off the runtime. + // `get_openai_key` is a blocking OS-keyring call (the commands.rs call + // sites all wrap it in spawn_blocking); previously the provider read it + // inline inside `run_one_turn`, i.e. once per tool-loop turn, on a tokio + // worker that was concurrently pumping other streams' events. The key + // can't legitimately change mid-conversation, so a single read is correct + // and keeps that blocking call off the per-turn TTFT path. + let api_key: Option = + tokio::task::spawn_blocking(|| secrets::get_openai_key().ok().flatten()) + .await + .ok() + .flatten(); + let start = Instant::now(); let mut total_tokens: u32 = 0; // Authoritative token count from `usage.completion_tokens` if the @@ -303,6 +317,7 @@ pub async fn chat_stream( &req.base_url, &channel, &cancel, + api_key.as_deref(), body, &mut total_tokens, &mut reported_tokens, @@ -464,6 +479,7 @@ async fn run_one_turn( base_url: &str, channel: &str, cancel: &Arc, + api_key: Option<&str>, body: Value, total_tokens: &mut u32, reported_tokens: &mut Option, @@ -480,7 +496,7 @@ async fn run_one_turn( // so we keep firing the request unauthenticated; only the // genuinely-auth-required path benefits from the augmented message. let mut bearer_withheld = false; - if let Some(key) = secrets::get_openai_key().ok().flatten() { + if let Some(key) = api_key { if !key.is_empty() { if is_safe_for_bearer(base_url) { http_req = http_req.bearer_auth(key); diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index ceb3c54..9e41a5a 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -26,7 +26,8 @@ "center": true, "visible": true, "shadow": true, - "dragDropEnabled": false + "dragDropEnabled": false, + "backgroundColor": "#06070f" } ], "security": { diff --git a/src/App.tsx b/src/App.tsx index 27ef818..1f23405 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { lazy, Suspense, useEffect, useState } from "react"; import { TitleBar } from "@/components/TitleBar"; import { Sidebar } from "@/components/Sidebar"; import { ChatHeader } from "@/components/ChatHeader"; @@ -10,17 +10,36 @@ import { HelpDialog } from "@/components/HelpDialog"; import { TooltipProvider } from "@/components/ui/tooltip"; import { SwitchVariantProvider } from "@/components/ui/switch"; import { SpaceForm } from "@/components/SpaceForm"; -import { SpaceView } from "@/components/SpaceView"; -import { SpacesLibrary } from "@/components/SpacesLibrary"; +// Lazily loaded surfaces — each renders only when the user navigates to it +// (a space/model view, a sidebar library, the code canvas, onboarding), so +// splitting them out keeps their code (and deps) out of the initial bundle +// that the chat path pays to parse on every cold start. They each already sit +// behind a render condition below, so the chunk loads exactly when first shown. +const SpaceView = lazy(() => + import("@/components/SpaceView").then((m) => ({ default: m.SpaceView })), +); +const SpacesLibrary = lazy(() => + import("@/components/SpacesLibrary").then((m) => ({ default: m.SpacesLibrary })), +); import { SnippetDialog } from "@/components/SnippetDialog"; import { SnippetVariableDialog } from "@/components/SnippetVariableDialog"; import { SnippetVariableFillDialog } from "@/components/SnippetVariableFillDialog"; -import { SnippetsLibrary } from "@/components/SnippetsLibrary"; -import { ModelsView } from "@/components/ModelsView"; -import { ModelsLibrary } from "@/components/ModelsLibrary"; +const SnippetsLibrary = lazy(() => + import("@/components/SnippetsLibrary").then((m) => ({ default: m.SnippetsLibrary })), +); +const ModelsView = lazy(() => + import("@/components/ModelsView").then((m) => ({ default: m.ModelsView })), +); +const ModelsLibrary = lazy(() => + import("@/components/ModelsLibrary").then((m) => ({ default: m.ModelsLibrary })), +); import { LockScreen } from "@/components/LockScreen"; -import { Onboarding } from "@/components/Onboarding"; -import { CodeCanvas } from "@/components/CodeCanvas"; +const Onboarding = lazy(() => + import("@/components/Onboarding").then((m) => ({ default: m.Onboarding })), +); +const CodeCanvas = lazy(() => + import("@/components/CodeCanvas").then((m) => ({ default: m.CodeCanvas })), +); import { SearchBar } from "@/components/SearchBar"; import { PrivateChat } from "@/components/PrivateChat"; import { KeyboardShortcuts } from "@/components/KeyboardShortcuts"; @@ -40,9 +59,7 @@ import { useSecurityStore, lockUntilHydrated } from "@/stores/securityStore"; import { useUIStore } from "@/stores/uiStore"; import { useCanvasStore } from "@/stores/canvasStore"; import { cn } from "@/lib/utils"; -import type { Message } from "@/types"; - -const EMPTY_MESSAGES: Message[] = []; +import { DEFAULT_PARAMS } from "@/types"; export default function App() { const hydrateSettings = useSettingsStore((s) => s.hydrate); @@ -67,10 +84,15 @@ export default function App() { const session = useChatStore((s) => s.activeSessionId ? s.sessions.find((x) => x.id === s.activeSessionId) : undefined, ); - const messages = useChatStore((s) => - s.activeSessionId ? s.messages[s.activeSessionId] ?? EMPTY_MESSAGES : EMPTY_MESSAGES, + // Subscribe to a boolean, NOT the messages array: the array's identity + // changes on every streaming flush, so selecting the array here re-rendered + // the whole App shell ~60×/s mid-stream even though only `.length > 0` is + // ever read. Returning the primitive lets zustand's Object.is bail. + const hasMessages = useChatStore((s) => + s.activeSessionId + ? (s.messages[s.activeSessionId]?.length ?? 0) > 0 + : false, ); - const hasMessages = messages.length > 0; // True only during the post-unlock hydrate window. Lets us swap the // "No chat open" CTA for a loading skeleton so users with chats on disk // don't briefly see an empty-state message that tells them to create one. @@ -137,9 +159,23 @@ export default function App() { useChatStore.getState().sessions, ); if (resolved.provider === "ollama" && resolved.model) { - void ollamaPreloadModel(s.ollama_base_url, resolved.model).catch( - () => {}, - ); + // Resolve the runner-affecting params a fresh chat with this model + // would send (num_ctx from the Modelfile defaults, else the app + // default; plus the global low-VRAM pin) so the warmed model isn't + // reloaded the instant the user sends their first message. + void useModelsStore + .getState() + .loadModelDefaults(resolved.model) + .then((md) => { + void ollamaPreloadModel( + s.ollama_base_url, + resolved.model, + md.num_ctx ?? DEFAULT_PARAMS.num_ctx, + s.low_vram_global ? true : md.low_vram, + md.num_gpu, + ).catch(() => {}); + }) + .catch(() => {}); } }); }, [ @@ -213,23 +249,33 @@ export default function App() { {viewingSpaceId ? ( - + + + ) : viewingModel ? ( - + + + ) : sidebarTab === "spaces" ? ( - + + + ) : sidebarTab === "snippets" ? ( - + + + ) : sidebarTab === "models" ? ( - + + + ) : ( <> @@ -263,7 +309,13 @@ export default function App() { this swap, so the user's snippet is still there if they toggle params back. */} - {canvasOpen ? : } + {canvasOpen ? ( + + + + ) : ( + + )} )} @@ -291,7 +343,9 @@ export default function App() { {showOnboarding && ( - + + + )}
diff --git a/src/components/CodeCanvas.tsx b/src/components/CodeCanvas.tsx index ff96c54..ee54d08 100644 --- a/src/components/CodeCanvas.tsx +++ b/src/components/CodeCanvas.tsx @@ -68,6 +68,17 @@ export function CodeCanvas() { const displayCode = live ? live.code : code; const displayLanguage = live ? live.language : language; + // True only while the bound message is ACTIVELY streaming (not merely still + // bound after it finished). Gates CodeView's syntax highlighting: we render + // plain text while the block grows and apply the full highlight once it + // settles — re-highlighting every flush is the canvas's main streaming cost. + const bindingStreaming = useChatStore((s) => { + if (!binding || s.streamingSessionId !== binding.sessionId) return false; + const msgs = s.messages[binding.sessionId]; + const last = msgs && msgs.length > 0 ? msgs[msgs.length - 1] : null; + return !!last && last.id === binding.messageId; + }); + if (!isOpen) return null; // Plain text and "no language" both render as the text variant — same as @@ -221,7 +232,11 @@ export function CodeCanvas() {
- + ); } diff --git a/src/components/CodeView.tsx b/src/components/CodeView.tsx index 62e841b..a79d3c9 100644 --- a/src/components/CodeView.tsx +++ b/src/components/CodeView.tsx @@ -17,9 +17,13 @@ const MAX_HIGHLIGHT_CHARS = 50_000; export function CodeView({ code, language, + isLive, }: { code: string; language: string | null; + /** True while the canvas mirrors a still-streaming block. Suppresses + * syntax highlighting (see `highlighted` below) until the block settles. */ + isLive?: boolean; }) { const lineCount = useMemo(() => { if (!code) return 1; @@ -30,6 +34,13 @@ export function CodeView({ const highlighted = useMemo(() => { if (!code) return ""; + // While the canvas mirrors a still-streaming block, skip highlighting + // entirely: re-running the highlighter — worst case `highlightAuto` over + // every grammar for an as-yet-unknown language — on each token flush is the + // canvas's dominant streaming cost. Plain escaped text is instant; the full + // highlight lands the moment streaming stops (isLive flips false and this + // recomputes). + if (isLive) return escapeHtml(code); if (code.length > MAX_HIGHLIGHT_CHARS) return escapeHtml(code); if (language) { const known = hljs.getLanguage(language); @@ -46,7 +57,7 @@ export function CodeView({ } catch { return escapeHtml(code); } - }, [code, language]); + }, [code, language, isLive]); return (
diff --git a/src/components/Message.tsx b/src/components/Message.tsx index 90fafbd..4cde8b9 100644 --- a/src/components/Message.tsx +++ b/src/components/Message.tsx @@ -471,10 +471,23 @@ function MessageItemImpl({ message, isStreaming, metrics, canRegenerate }: Messa } const isUser = message.role === "user"; - const persistedMetrics = parseMetrics(message.metrics_json); + // Memoise the JSON parses on their source strings. chatStore only swaps these + // strings when their dirty flags fire, so during a content-only streaming + // flush they stay reference-equal and the parse is skipped — otherwise the + // streaming bubble re-ran all three JSON.parse calls every animation frame. + const persistedMetrics = useMemo( + () => parseMetrics(message.metrics_json), + [message.metrics_json], + ); const showMetrics = metrics ?? persistedMetrics; - const toolCalls = !isUser ? parseToolCalls(message.tool_calls_json) : []; - const attachments = parseAttachments(message.attachments_json); + const toolCalls = useMemo( + () => (!isUser ? parseToolCalls(message.tool_calls_json) : []), + [isUser, message.tool_calls_json], + ); + const attachments = useMemo( + () => parseAttachments(message.attachments_json), + [message.attachments_json], + ); // Images can come from a user upload or, on an assistant turn, from an // MCP tool result (mapped to an image attachment in mcp/client.rs). const images = attachments.filter((a) => a.kind === "image"); diff --git a/src/components/ParameterPanel.tsx b/src/components/ParameterPanel.tsx index 0662201..956cd9c 100644 --- a/src/components/ParameterPanel.tsx +++ b/src/components/ParameterPanel.tsx @@ -687,6 +687,14 @@ function SliderRow({ }) { const usingStops = stops && stops.length > 0; + // Local drag position so dragging updates the thumb + readout live WITHOUT + // persisting on every pointermove. `onChange` (which clones the sessions + // array and writes to SQLite over IPC) now fires only on `onValueCommit` — + // pointer-up or a key press — so a one-second drag is a single write, not + // the 60-120 it used to spray. `null` = not dragging; fall back to the + // prop-derived position. + const [drag, setDrag] = useState(null); + // Find the stop index closest to the current value so a pre-existing // legacy value (e.g. 6144 from before the stops migration) lands sensibly. const stopIdx = usingStops @@ -703,7 +711,10 @@ function SliderRow({ return best; })() : 0; - const displayValue = usingStops ? stops![stopIdx] : value; + // Live slider position: the in-flight drag value while dragging, else derived + // from props (`stopIdx` for stops, the raw `value` for continuous). + const pos = drag ?? (usingStops ? stopIdx : value); + const displayValue = usingStops ? stops![pos] : pos; const displayText = format ? format(displayValue) : displayValue.toFixed(precision); @@ -720,19 +731,27 @@ function SliderRow({
{usingStops ? ( onChange(stops![v[0]])} + onValueChange={(v) => setDrag(v[0])} + onValueCommit={(v) => { + onChange(stops![v[0]]); + setDrag(null); + }} /> ) : ( onChange(v[0])} + onValueChange={(v) => setDrag(v[0])} + onValueCommit={(v) => { + onChange(v[0]); + setDrag(null); + }} /> )} {hint && ( diff --git a/src/lib/memory.ts b/src/lib/memory.ts index 34b1106..b85fc35 100644 --- a/src/lib/memory.ts +++ b/src/lib/memory.ts @@ -139,6 +139,24 @@ export async function extractMemories(args: { } } +// Handle to the in-flight extractor stream's `stop()` (which routes through +// chat_cancel on the Rust side), or null when no extraction is running. Lets a +// new user send abort a still-running extractor so it stops competing for the +// model's generation slot. See `cancelMemoryExtraction`. +let memoryStopFn: (() => Promise) | null = null; + +/** Abort any in-flight memory extraction. Called when the user sends a new + * message so the extractor — a second full generation against the same model + * — doesn't keep hogging the generation slot and inflating the new turn's + * time-to-first-token. No-op when nothing is running. Best-effort: the Rust + * stream is told to cancel and the extractor promise settles empty (its + * partial output is never parsed). */ +export function cancelMemoryExtraction(): void { + const stop = memoryStopFn; + memoryStopFn = null; + if (stop) void stop().catch(() => {}); +} + /** * Issue a single non-streaming chat call against the user-selected model * and resolve to the concatenated assistant text. We re-use @@ -148,7 +166,8 @@ export async function extractMemories(args: { * * Concurrent with the user's next chat: yes, but Ollama serialises * generation per-model and the extractor finishes quickly given the small - * prompt + tight max_tokens. + * prompt + tight max_tokens. A new user send aborts it via + * `cancelMemoryExtraction` so it never delays the visible turn. */ async function runOneShotStream(args: { provider: ProviderId; @@ -178,6 +197,7 @@ async function runOneShotStream(args: { /* already unlistened — harmless */ } unlistenFn = null; + memoryStopFn = null; }; // Hard ceiling on extractor wall-clock. If a model goes off the rails @@ -221,6 +241,13 @@ async function runOneShotStream(args: { window.clearTimeout(timeoutId); cleanup(); reject(new Error(ev.message)); + } else if (ev.kind === "cancelled") { + // Aborted by a new user send (cancelMemoryExtraction). Resolve EMPTY + // — never parse the partial output — so the caller saves no memories + // and logs no failure. + window.clearTimeout(timeoutId); + cleanup(); + resolve(""); } }, ) @@ -238,6 +265,7 @@ async function runOneShotStream(args: { return; } unlistenFn = handle.unlisten; + memoryStopFn = handle.stop; }) .catch((e) => { window.clearTimeout(timeoutId); diff --git a/src/lib/spaceContextCache.ts b/src/lib/spaceContextCache.ts new file mode 100644 index 0000000..3d12c04 --- /dev/null +++ b/src/lib/spaceContextCache.ts @@ -0,0 +1,38 @@ +import type { SpaceContext } from "@/types"; + +/** + * Per-space cache of the assembled {@link SpaceContext} (instructions + + * reference files + memories). + * + * Without it, every chat send in a Space re-ships all reference-file data + * across IPC twice — Rust→JS in `getSpaceContext`, then JS→Rust embedded in the + * outgoing `system_prompt` — which is squarely on the time-to-first-token path + * and scales with how much reference material the Space holds. + * + * Correctness rests on EXPLICIT invalidation: every `spaceStore` mutation that + * can change a space's instructions, files, or memories calls + * {@link invalidateSpaceContext}. Memory extraction routes through + * `spaceStore.addMemory`, so it's covered too. (A cache keyed on + * `space.updated_at` would NOT be safe — `add_space_file` / `add_space_memory` + * deliberately don't bump that column.) The cache is module-level, so it's also + * naturally dropped on app restart. + */ +const cache = new Map(); + +export function getCachedSpaceContext(spaceId: string): SpaceContext | undefined { + return cache.get(spaceId); +} + +export function setCachedSpaceContext(spaceId: string, ctx: SpaceContext): void { + cache.set(spaceId, ctx); +} + +export function invalidateSpaceContext(spaceId: string): void { + cache.delete(spaceId); +} + +/** Drop the whole cache — used on (re)hydration, e.g. after a data import + * rewrites the spaces tables out from under us. */ +export function invalidateAllSpaceContext(): void { + cache.clear(); +} diff --git a/src/lib/tauri.ts b/src/lib/tauri.ts index 60054a4..21dcc1f 100644 --- a/src/lib/tauri.ts +++ b/src/lib/tauri.ts @@ -416,9 +416,23 @@ export function ollamaUnloadModel(baseUrl: string, model: string): Promise return invoke("ollama_unload_model", { baseUrl, model }); } -export function ollamaPreloadModel(baseUrl: string, model: string): Promise { +export function ollamaPreloadModel( + baseUrl: string, + model: string, + numCtx?: number, + lowVram?: boolean, + numGpu?: number, +): Promise { if (!isTauri) return notInTauri(undefined); - return invoke("ollama_preload_model", { baseUrl, model }); + // Pass the runner-affecting params the first real request will send so the + // warmed model isn't immediately reloaded. Undefined → null → Rust None. + return invoke("ollama_preload_model", { + baseUrl, + model, + numCtx: numCtx ?? null, + lowVram: lowVram ?? null, + numGpu: numGpu ?? null, + }); } export function openaiListModels(baseUrl: string): Promise { diff --git a/src/stores/chatStore.ts b/src/stores/chatStore.ts index b59e595..54c97db 100644 --- a/src/stores/chatStore.ts +++ b/src/stores/chatStore.ts @@ -30,7 +30,7 @@ import { imagesFromAttachments, inlineTextAttachments, } from "@/lib/files"; -import { extractMemories } from "@/lib/memory"; +import { extractMemories, cancelMemoryExtraction } from "@/lib/memory"; import { buildCompactedMarkdown } from "@/lib/export"; import { extractSummary, @@ -42,6 +42,10 @@ import { formatProviderError } from "@/lib/providerErrors"; import { getPersona } from "@/lib/personas"; import { getTone } from "@/lib/tones"; import { applyTemporalAwareness } from "@/lib/temporal"; +import { + getCachedSpaceContext, + setCachedSpaceContext, +} from "@/lib/spaceContextCache"; import { buildFetchToolRecords, extractUrls, @@ -633,7 +637,15 @@ async function buildTaskRequest( const spaceImages: string[] = []; if (session.space_id) { try { - const ctx = await getSpaceContext(session.space_id); + // Serve the space context from the per-space cache when warm, so repeat + // sends in a Space don't re-ship every reference file across IPC. The + // cache is invalidated by every spaceStore mutation that can change a + // space's instructions / files / memories (see spaceContextCache.ts). + let ctx = getCachedSpaceContext(session.space_id); + if (!ctx) { + ctx = await getSpaceContext(session.space_id); + setCachedSpaceContext(session.space_id, ctx); + } const spaceInstructions = ctx.space.instructions.trim(); let filesBlock = ""; const textFiles = ctx.files.filter((f) => f.kind === "text"); @@ -891,9 +903,21 @@ function finishRunning( // promoted so the user's next message starts streaming without waiting on // us; failures are logged inside the extractor and never bubble up. if (memorySnapshot) { - void extractMemories(memorySnapshot).catch((e) => { - logger.warn("memory extraction failed", e); - }); + // Skip extraction when the chat has more work pending — a queued message + // (still in `queue` here; `promoteQueueHead` only shifts it on the next + // microtask) or something already running. The extractor is a second full + // LLM generation against the same model, so firing it now would make it + // compete with the user's visible turn for the generation slot and inflate + // that turn's time-to-first-token. Extraction is best-effort, so dropping + // this turn's is acceptable. + const busy = get().queue.length > 0 || get().runningTask !== null; + if (busy) { + logger.debug("memory extraction skipped — chat busy"); + } else { + void extractMemories(memorySnapshot).catch((e) => { + logger.warn("memory extraction failed", e); + }); + } } } @@ -1723,6 +1747,9 @@ export const useChatStore = create((set, get) => ({ }, sendUserMessage: async (rawContent, attachments) => { + // Abort any in-flight memory extraction so it stops competing with this + // turn for the model's generation slot (best-effort; no-op when idle). + cancelMemoryExtraction(); // If viewing a space, exit the space view and create a session in that space const spaceStore = useSpaceStore.getState(); const viewingSpaceId = spaceStore.viewingSpaceId; diff --git a/src/stores/privateChatStore.ts b/src/stores/privateChatStore.ts index b4a4e9f..520c362 100644 --- a/src/stores/privateChatStore.ts +++ b/src/stores/privateChatStore.ts @@ -115,6 +115,67 @@ function composeSystemPrompt(args: { return out.length > 0 ? out : null; } +// rAF-batched flush for the streaming assistant message, mirroring chatStore: +// the provider emits 100+ token events/sec and a per-event `set()` + full +// `messages.map()` re-renders the whole overlay each time. We accumulate deltas +// into this module-level buffer and write to the store at most once per frame. +// +// Invariants (privacy-load-bearing for the overlay): +// - Terminal paths (done / cancelled / error / user cancel) MUST sync-flush +// before teardown so the last tokens stay visible. +// - `pcReset` clears the pending frame AND the buffer — every teardown and +// `wipe()` calls it so a stale rAF can never write into a torn-down (or +// wiped) overlay, which would otherwise resurrect private content. +let pcPendingFrame: number | null = null; +let pcBuffer: + | { + id: string; + content: string; + thinking: string; + contentDirty: boolean; + thinkingDirty: boolean; + } + | null = null; + +function pcFlush() { + if (pcPendingFrame !== null) { + cancelAnimationFrame(pcPendingFrame); + pcPendingFrame = null; + } + const buf = pcBuffer; + if (!buf || (!buf.contentDirty && !buf.thinkingDirty)) return; + const { id, content, thinking, contentDirty, thinkingDirty } = buf; + buf.contentDirty = false; + buf.thinkingDirty = false; + usePrivateChatStore.setState((s) => ({ + messages: s.messages.map((m) => + m.id === id + ? { + ...m, + ...(contentDirty ? { content } : {}), + ...(thinkingDirty ? { thinking } : {}), + } + : m, + ), + })); +} + +function pcSchedule() { + if (pcPendingFrame !== null) return; + pcPendingFrame = requestAnimationFrame(() => { + pcPendingFrame = null; + pcFlush(); + }); +} + +function pcReset() { + if (pcPendingFrame !== null) { + cancelAnimationFrame(pcPendingFrame); + pcPendingFrame = null; + } + pcBuffer = null; +} + export const usePrivateChatStore = create((set, get) => ({ open: false, messages: [], @@ -190,6 +251,17 @@ export const usePrivateChatStore = create((set, get) => ({ const streamId = makeRequestId(); const assistantId = assistantMsg.id; + // Start a fresh rAF-batched buffer for this assistant message (drop any + // stale one from a prior stream first). + pcReset(); + pcBuffer = { + id: assistantId, + content: "", + thinking: "", + contentDirty: false, + thinkingDirty: false, + }; + // Apply the global Low-VRAM pin (Settings → Features) when the user // hasn't set anything in the per-chat panel. Per-chat override wins — // toggling Low VRAM off in the Private Chat sidebar stores `undefined`, @@ -215,6 +287,9 @@ export const usePrivateChatStore = create((set, get) => ({ }; const teardown = () => { + // Drop any pending flush + buffer so a stale rAF can't write into the + // torn-down overlay. Callers flush first when the final frame matters. + pcReset(); const handle = get().activeStream; if (handle) { try { @@ -249,12 +324,20 @@ export const usePrivateChatStore = create((set, get) => ({ }, (ev) => { if (ev.kind === "token") { - patchAssistant((m) => ({ content: (m.content ?? "") + ev.delta })); + if (pcBuffer) { + pcBuffer.content += ev.delta; + pcBuffer.contentDirty = true; + pcSchedule(); + } } else if (ev.kind === "thinking") { - patchAssistant((m) => ({ - thinking: (m.thinking ?? "") + ev.delta, - })); + if (pcBuffer) { + pcBuffer.thinking += ev.delta; + pcBuffer.thinkingDirty = true; + pcSchedule(); + } } else if (ev.kind === "metrics") { + // Metrics arrive once near the end — apply immediately. They don't + // touch content, so they don't race the content buffer. patchAssistant(() => ({ metrics: { tokens: ev.tokens, @@ -263,11 +346,18 @@ export const usePrivateChatStore = create((set, get) => ({ }, })); } else if (ev.kind === "error") { - patchAssistant((m) => ({ - content: (m.content ?? "") + `\n\n_⚠ ${ev.message}_`, - })); + // Fold the notice into the buffered content and flush once so the + // partial reply + error land together before teardown. + if (pcBuffer) { + pcBuffer.content += `\n\n_⚠ ${ev.message}_`; + pcBuffer.contentDirty = true; + } + pcFlush(); teardown(); } else if (ev.kind === "cancelled" || ev.kind === "done") { + // Sync-flush the last buffered tokens before tearing down so the + // final frame of the reply doesn't vanish. + pcFlush(); teardown(); } }, @@ -284,6 +374,11 @@ export const usePrivateChatStore = create((set, get) => ({ cancel: async () => { const stream = get().activeStream; if (!stream) return; + // Land any buffered tokens so the partial reply stays visible, then drop + // the buffer — this path unlistens, so the stream's own Cancelled event + // never re-enters the handler to flush for us. + pcFlush(); + pcReset(); try { stream.unlisten(); } catch { @@ -302,6 +397,9 @@ export const usePrivateChatStore = create((set, get) => ({ }, wipe: () => { + // Drop any pending streaming flush + buffered tokens FIRST so a queued rAF + // can't write private content back into the overlay we're about to clear. + pcReset(); const stream = get().activeStream; if (stream) { try { diff --git a/src/stores/spaceStore.ts b/src/stores/spaceStore.ts index 6c3b400..29d60a8 100644 --- a/src/stores/spaceStore.ts +++ b/src/stores/spaceStore.ts @@ -15,6 +15,10 @@ import { updateSpaceMemory, } from "@/lib/tauri"; import { SPACE_BYTES_CAP } from "@/lib/files"; +import { + invalidateSpaceContext, + invalidateAllSpaceContext, +} from "@/lib/spaceContextCache"; import type { Space, SpaceFile, SpaceMemory } from "@/types"; interface SpaceState { @@ -92,6 +96,9 @@ export const useSpaceStore = create((set, get) => ({ try { const spaces = await listSpaces(); set({ spaces }); + // A fresh hydrate (startup, or after a data import rewrote the spaces + // tables) means any previously-cached context may be stale. + invalidateAllSpaceContext(); } catch (e) { logger.error("space hydrate failed", e); } @@ -138,6 +145,7 @@ export const useSpaceStore = create((set, get) => ({ fields.memory_enabled !== undefined ? fields.memory_enabled : undefined, }; await updateSpace({ id, ...next }); + invalidateSpaceContext(id); set((s) => ({ spaces: s.spaces.map((sp) => sp.id === id @@ -168,6 +176,7 @@ export const useSpaceStore = create((set, get) => ({ deleteSpace: async (id) => { try { await deleteSpace(id); + invalidateSpaceContext(id); } catch (e) { useToastStore.getState().push({ kind: "error", @@ -218,6 +227,7 @@ export const useSpaceStore = create((set, get) => ({ size, position: existing.length, }); + invalidateSpaceContext(spaceId); set((s) => ({ spaceFiles: { ...s.spaceFiles, @@ -230,6 +240,7 @@ export const useSpaceStore = create((set, get) => ({ removeFile: async (fileId, spaceId) => { try { await removeSpaceFile(fileId); + invalidateSpaceContext(spaceId); } catch (e) { useToastStore.getState().push({ kind: "error", @@ -254,6 +265,7 @@ export const useSpaceStore = create((set, get) => ({ addMemory: async (args) => { const memory = await addSpaceMemory(args); + invalidateSpaceContext(args.space_id); set((s) => ({ spaceMemories: { ...s.spaceMemories, @@ -266,6 +278,7 @@ export const useSpaceStore = create((set, get) => ({ updateMemory: async (id, spaceId, content) => { const trimmed = content.trim(); await updateSpaceMemory({ id, space_id: spaceId, content: trimmed }); + invalidateSpaceContext(spaceId); const now = Date.now(); set((s) => ({ spaceMemories: { @@ -283,6 +296,7 @@ export const useSpaceStore = create((set, get) => ({ // panel. The SpaceView UI caller attaches its own toast on the rejection. removeMemory: async (id, spaceId) => { await removeSpaceMemory({ id, space_id: spaceId }); + invalidateSpaceContext(spaceId); set((s) => ({ spaceMemories: { ...s.spaceMemories, From 8c1c52bf26ba83b41a5786c8d7cf1efe0f96a3c8 Mon Sep 17 00:00:00 2001 From: ZTCS Date: Fri, 12 Jun 2026 16:14:03 +0200 Subject: [PATCH 3/9] land LOW optimizations (lazy read pool, pooled cold reads, single-tx message append, eval_duration tok/s, localhost SSRF short-circuit, concurrent settings/model hydration, deferred context-usage + PDF dims) and document Ollama server-side levers --- docs/TROUBLESHOOTING.md | 36 ++++++++ src-tauri/src/db.rs | 139 ++++++++++++++++------------- src-tauri/src/providers/mod.rs | 18 ++++ src-tauri/src/providers/ollama.rs | 52 +++++++++-- src/components/ContextUsageBar.tsx | 22 +++-- src/components/PdfPreview.tsx | 40 ++++----- src/stores/modelsStore.ts | 54 ++++++----- src/stores/settingsStore.ts | 11 ++- 8 files changed, 252 insertions(+), 120 deletions(-) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index c34868f..0f94f61 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -174,6 +174,42 @@ streaming starts; subsequent replies are instant. launch, turn on **Settings → General → Default model preload**. The first chat will start fast, at the cost of pinning VRAM as soon as Loach opens. +### Making Ollama generate faster + +**Problem.** Generation is slower than you'd expect, or you want to squeeze +more tokens/sec out of your hardware. + +**Solution.** These are environment variables on the **Ollama server** (not +Loach settings) — set them where `ollama serve` runs, then restart it. As of +Ollama 0.5+: + +- **`OLLAMA_FLASH_ATTENTION=1`** — enables flash attention, which lowers memory + bandwidth and usually speeds up generation, especially at longer contexts. + Newer builds enable it by default for some models on the new engine; setting + it explicitly is harmless. +- **`OLLAMA_KV_CACHE_TYPE=q8_0`** — quantizes the KV cache to 8-bit, roughly + halving its memory so you can run a bigger context (or a bigger model) in the + same VRAM. Only takes effect **with flash attention enabled**. `q4_0` is + smaller but lossier; `f16` (the default) is highest quality. +- **`OLLAMA_KEEP_ALIVE=30m`** (or `-1` for "until unloaded") — how long Ollama + keeps a model resident after a request, so a reply after a pause skips the + cold reload. Loach also exposes this under **Settings → Providers → Keep + model loaded**, which is sent with every request and overrides the env + default. +- **`OLLAMA_NUM_PARALLEL=1`** — caps concurrent requests per model. The default + splits VRAM across parallel slots; pinning it to 1 gives a single chat the + whole budget (and the largest usable context). +- **`OLLAMA_MAX_LOADED_MODELS=1`** — keeps only one model resident at a time. + Useful on a single-GPU machine where a second model would evict the first or + spill to system RAM. + +Setting environment variables for Ollama: on Windows, use **System Properties → +Environment Variables** (or `setx OLLAMA_FLASH_ATTENTION 1`) and restart the +Ollama tray app. On Linux with systemd, `systemctl edit ollama` (add an +`[Service]` block of `Environment="OLLAMA_…=…"` lines) then +`systemctl restart ollama`. On macOS, `launchctl setenv OLLAMA_FLASH_ATTENTION 1` +and relaunch the Ollama app. + --- ## 4. Attachments diff --git a/src-tauri/src/db.rs b/src-tauri/src/db.rs index a0a346b..1155523 100644 --- a/src-tauri/src/db.rs +++ b/src-tauri/src/db.rs @@ -295,6 +295,16 @@ impl Database { }); let read_pool = Pool::builder() .max_size(4) + // Don't block startup opening all 4 reader connections eagerly: + // r2d2's `min_idle` defaults to `max_size`, so `build()` would open + // and pragma-init every one (each runs the mmap_size pragma) on the + // setup thread before first paint. Open them lazily on first SELECT + // — which only happens post-unlock anyway. Trade-off: a reader that + // fails to open then surfaces as a per-query "acquire read + // connection" error rather than the friendly boot dialog, but the + // writer open of the same file already catches the fatal cases at + // startup. + .min_idle(Some(0)) .build(manager) .context("build read pool")?; @@ -806,16 +816,6 @@ impl Database { Ok(()) } - pub fn touch_session(&self, id: &str) -> Result<()> { - let now = Utc::now().timestamp_millis(); - let conn = self.conn.lock(); - conn.execute( - "UPDATE sessions SET updated_at = ?1 WHERE id = ?2", - params![now, id], - )?; - Ok(()) - } - pub fn pin_session(&self, id: &str, pinned: bool) -> Result<()> { let conn = self.conn.lock(); let now = Utc::now().timestamp_millis(); @@ -941,14 +941,25 @@ impl Database { let id = Uuid::new_v4().to_string(); let now = Utc::now().timestamp_millis(); { - let conn = self.conn.lock(); - conn.execute( + // Insert the message and bump the session's sort timestamp in ONE + // transaction: a single lock acquisition + a single WAL commit + // rather than two (the old shape dropped the lock, then + // `touch_session` reacquired it for a separate implicit commit). + // Also tightens crash atomicity — a message can't land with its + // session's updated_at left un-bumped. + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + tx.execute( "INSERT INTO messages (id, session_id, role, content, thinking, attachments_json, metrics_json, tool_calls_json, compacted_at, created_at, import_group, import_hidden) VALUES (?1, ?2, ?3, ?4, NULL, ?5, NULL, NULL, NULL, ?6, NULL, 0)", params![id, session_id, role, content, attachments_json, now], )?; + tx.execute( + "UPDATE sessions SET updated_at = ?1 WHERE id = ?2", + params![now, session_id], + )?; + tx.commit()?; } - self.touch_session(session_id)?; Ok(Message { id, session_id: session_id.to_string(), @@ -1013,9 +1024,14 @@ impl Database { }); } } + // Bump the session's sort timestamp in the same transaction so the + // import is one commit, not two (matches append_message). + tx.execute( + "UPDATE sessions SET updated_at = ?1 WHERE id = ?2", + params![base, session_id], + )?; tx.commit()?; } - self.touch_session(session_id)?; Ok(out) } @@ -1161,12 +1177,13 @@ impl Database { } pub fn all_settings(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare("SELECT key, value FROM settings")?; - let rows = stmt - .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? - .collect::, _>>()?; - Ok(rows) + self.with_read(|conn| { + let mut stmt = conn.prepare("SELECT key, value FROM settings")?; + let rows = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? + .collect::, _>>()?; + Ok(rows) + }) } // ------------ spaces ------------ @@ -1475,27 +1492,28 @@ impl Database { // ------------ snippets ------------ pub fn list_snippets(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "SELECT id, title, prompt, attachments_json, provider, model, - created_at, updated_at - FROM snippets ORDER BY updated_at DESC", - )?; - let rows = stmt - .query_map([], |r| { - Ok(Snippet { - id: r.get(0)?, - title: r.get(1)?, - prompt: r.get(2)?, - attachments_json: r.get(3)?, - provider: r.get(4)?, - model: r.get(5)?, - created_at: r.get(6)?, - updated_at: r.get(7)?, - }) - })? - .collect::, _>>()?; - Ok(rows) + self.with_read(|conn| { + let mut stmt = conn.prepare( + "SELECT id, title, prompt, attachments_json, provider, model, + created_at, updated_at + FROM snippets ORDER BY updated_at DESC", + )?; + let rows = stmt + .query_map([], |r| { + Ok(Snippet { + id: r.get(0)?, + title: r.get(1)?, + prompt: r.get(2)?, + attachments_json: r.get(3)?, + provider: r.get(4)?, + model: r.get(5)?, + created_at: r.get(6)?, + updated_at: r.get(7)?, + }) + })? + .collect::, _>>()?; + Ok(rows) + }) } pub fn create_snippet( @@ -1706,25 +1724,26 @@ impl Database { // legacy stdio-flavoured columns stay NULL. pub fn list_mcp_servers(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "SELECT id, name, url, headers_json, enabled, created_at, updated_at - FROM mcp_servers ORDER BY name ASC", - )?; - let rows = stmt - .query_map([], |r| { - Ok(McpServer { - id: r.get(0)?, - name: r.get(1)?, - url: r.get::<_, Option>(2)?.unwrap_or_default(), - headers_json: r.get(3)?, - enabled: r.get::<_, i64>(4)? != 0, - created_at: r.get(5)?, - updated_at: r.get(6)?, - }) - })? - .collect::, _>>()?; - Ok(rows) + self.with_read(|conn| { + let mut stmt = conn.prepare( + "SELECT id, name, url, headers_json, enabled, created_at, updated_at + FROM mcp_servers ORDER BY name ASC", + )?; + let rows = stmt + .query_map([], |r| { + Ok(McpServer { + id: r.get(0)?, + name: r.get(1)?, + url: r.get::<_, Option>(2)?.unwrap_or_default(), + headers_json: r.get(3)?, + enabled: r.get::<_, i64>(4)? != 0, + created_at: r.get(5)?, + updated_at: r.get(6)?, + }) + })? + .collect::, _>>()?; + Ok(rows) + }) } /// Upsert: create a new row if `id` is empty, otherwise update the diff --git a/src-tauri/src/providers/mod.rs b/src-tauri/src/providers/mod.rs index 48918d4..1cd4d2f 100644 --- a/src-tauri/src/providers/mod.rs +++ b/src-tauri/src/providers/mod.rs @@ -171,6 +171,15 @@ pub(super) async fn refuse_link_local_host(base_url: &str) -> Result<(), String> if let Ok(ip) = host_for_ip.parse::() { return check(ip); } + // `localhost` is loopback by definition, and the guard explicitly allows + // loopback — so skip the OS resolution reqwest is about to repeat anyway. + // This guard runs on the TTFT path of every send (and every admin call). + // Only the bare literal is short-circuited; every other hostname still flows + // through the resolver + link-local screen below, so there's no DNS-rebind + // window for arbitrary names. + if host.eq_ignore_ascii_case("localhost") { + return Ok(()); + } // Hostname: resolve through the OS. If resolution fails, leave the // request alone — the subsequent `reqwest::send` will produce a more // accurate error than we could synthesize here. @@ -218,6 +227,15 @@ mod tests { assert!(refuse_link_local_host("http://8.8.8.8/").await.is_ok()); } + #[tokio::test] + async fn refuse_link_local_short_circuits_localhost() { + // `localhost` is loopback by definition — the guard returns Ok without + // an OS resolution (which reqwest repeats anyway), case-insensitively. + assert!(refuse_link_local_host("http://localhost:11434").await.is_ok()); + assert!(refuse_link_local_host("http://localhost").await.is_ok()); + assert!(refuse_link_local_host("https://LocalHost:1234").await.is_ok()); + } + #[test] fn bound_messages_payload_is_noop_under_cap() { let mut msgs = vec![ diff --git a/src-tauri/src/providers/ollama.rs b/src-tauri/src/providers/ollama.rs index c71d553..8d308c3 100644 --- a/src-tauri/src/providers/ollama.rs +++ b/src-tauri/src/providers/ollama.rs @@ -508,6 +508,13 @@ struct OllamaChunk { /// tokens — which happens routinely on faster local models. #[serde(default)] eval_count: Option, + /// Pure decode time in nanoseconds from the final `done: true` chunk + /// (`eval_duration`). Paired with `eval_count` it gives the true generation + /// rate — unlike wall-clock, which also folds in model load, prompt + /// evaluation, and tool round-trips. `None` on intermediate chunks and on + /// builds that don't report it. + #[serde(default)] + eval_duration: Option, } #[derive(Debug, Deserialize)] @@ -617,6 +624,9 @@ pub async fn chat_stream( // (`eval_count`). Preferred over the per-chunk approximation — // faster local models routinely batch several tokens per chunk. let mut reported_tokens: Option = None; + // Accumulated decode time (eval_duration, ns) across turns, paired with + // reported_tokens to compute an accurate generation rate at the end. + let mut reported_eval_ns: Option = None; for turn in 0..MAX_TOOL_TURNS { // Keep the running history under the soft cap before each turn. @@ -649,6 +659,7 @@ pub async fn chat_stream( &mut body, &mut total_tokens, &mut reported_tokens, + &mut reported_eval_ns, &mut think_already_drop, &req.stream_id, ®istry, @@ -663,7 +674,7 @@ pub async fn chat_stream( match outcome { TurnOutcome::Done => { let tokens_for_metrics = reported_tokens.unwrap_or(total_tokens); - emit_metrics(&app, &channel, tokens_for_metrics, start); + emit_metrics(&app, &channel, tokens_for_metrics, reported_eval_ns, start); let _ = app.emit(&channel, StreamEvent::Done); registry.finish(&req.stream_id); return Ok(()); @@ -816,6 +827,7 @@ async fn run_one_turn( body: &mut Value, total_tokens: &mut u32, reported_tokens: &mut Option, + reported_eval_ns: &mut Option, think_already_drop: &mut bool, stream_id: &str, registry: &StreamRegistry, @@ -899,6 +911,7 @@ async fn run_one_turn( // and a forked build's intermediate emissions don't inflate the // count either. let mut turn_eval: Option = None; + let mut turn_eval_duration: Option = None; loop { select! { @@ -939,6 +952,9 @@ async fn run_one_turn( if let Some(n) = parsed.eval_count { turn_eval = Some(n); } + if let Some(d) = parsed.eval_duration { + turn_eval_duration = Some(d); + } if let Some(msg) = parsed.message { if let Some(think) = msg.thinking { if !think.is_empty() { @@ -988,6 +1004,10 @@ async fn run_one_turn( *reported_tokens = Some(reported_tokens.unwrap_or(0).saturating_add(n)); } + if let Some(d) = turn_eval_duration { + *reported_eval_ns = + Some(reported_eval_ns.unwrap_or(0).saturating_add(d)); + } return Ok(Some(decide_outcome(pending_tool_calls))); } } @@ -1002,11 +1022,15 @@ async fn run_one_turn( None => { // EOF without an explicit `done: true` — treat as a // natural terminator and let the caller emit Done. - // Still fold any partial turn_eval we got. + // Still fold any partial turn_eval / eval_duration. if let Some(n) = turn_eval { *reported_tokens = Some(reported_tokens.unwrap_or(0).saturating_add(n)); } + if let Some(d) = turn_eval_duration { + *reported_eval_ns = + Some(reported_eval_ns.unwrap_or(0).saturating_add(d)); + } return Ok(Some(decide_outcome(pending_tool_calls))); } } @@ -1023,12 +1047,26 @@ fn decide_outcome(calls: Vec) -> TurnOutcome { } } -fn emit_metrics(app: &AppHandle, channel: &str, tokens: u32, start: Instant) { +fn emit_metrics( + app: &AppHandle, + channel: &str, + tokens: u32, + eval_ns: Option, + start: Instant, +) { let elapsed = start.elapsed().as_millis() as u64; - let tps = if elapsed > 0 { - (tokens as f64) * 1000.0 / (elapsed as f64) - } else { - 0.0 + // Prefer Ollama's own `eval_duration` (pure decode time) so the rate + // reflects generation speed, not the wall clock — which also folds in model + // load, prompt evaluation, and any tool round-trips, understating tok/s + // (badly so after a cold load). Fall back to wall-clock when the server + // didn't report it (older builds) or nothing was generated. `elapsed_ms` + // stays the full turn wall-clock — still useful, and unchanged on the wire. + let tps = match eval_ns { + Some(ns) if ns > 0 && tokens > 0 => { + (tokens as f64) * 1_000_000_000.0 / (ns as f64) + } + _ if elapsed > 0 => (tokens as f64) * 1000.0 / (elapsed as f64), + _ => 0.0, }; let _ = app.emit( channel, diff --git a/src/components/ContextUsageBar.tsx b/src/components/ContextUsageBar.tsx index 8793bfd..55555e3 100644 --- a/src/components/ContextUsageBar.tsx +++ b/src/components/ContextUsageBar.tsx @@ -1,16 +1,20 @@ -import { useEffect, useMemo, useRef, useState } from "react"; +import { useDeferredValue, useEffect, useMemo, useRef, useState } from "react"; import { Gauge, Loader2, Maximize2, Sparkles, X } from "lucide-react"; import { useChatStore } from "@/stores/chatStore"; import { useSettingsStore } from "@/stores/settingsStore"; import { useModelsStore } from "@/stores/modelsStore"; import { useSpaceStore } from "@/stores/spaceStore"; -import { DEFAULT_PARAMS, type GenerationParams } from "@/types"; +import { DEFAULT_PARAMS, type GenerationParams, type Message } from "@/types"; import { computeContextUsage, formatTokens, } from "@/lib/contextUsage"; import { cn } from "@/lib/utils"; +// Stable empty array so the messages selector's no-messages branch keeps a +// constant reference (a fresh `[]` would re-fire the selector every render). +const EMPTY_MESSAGES: Message[] = []; + /** * Layer the same set of GenerationParams the chat send path uses, just * enough to find the effective `num_ctx` for the active session. Mirrors @@ -53,7 +57,9 @@ export function ContextUsageBar() { : undefined, ); const messages = useChatStore((s) => - s.activeSessionId ? s.messages[s.activeSessionId] ?? [] : [], + s.activeSessionId + ? s.messages[s.activeSessionId] ?? EMPTY_MESSAGES + : EMPTY_MESSAGES, ); const globalSystemPrompt = useSettingsStore((s) => s.global_system_prompt); const modelDefaults = useModelsStore((s) => @@ -115,9 +121,15 @@ export function ContextUsageBar() { ? session.system_prompt : globalSystemPrompt || ""; + // Defer the recompute: `messages` changes identity on every streaming flush + // and computeContextUsage rescans the whole transcript. useDeferredValue lets + // React skip intermediate values during a burst and recompute when idle — + // and it always catches up to the final value once streaming stops, so the + // bar can't end stale (the failure mode a leading-edge timer throttle has). + const deferredMessages = useDeferredValue(messages); const usage = useMemo( - () => computeContextUsage(messages, effectiveSystemPrompt, params), - [messages, effectiveSystemPrompt, params], + () => computeContextUsage(deferredMessages, effectiveSystemPrompt, params), + [deferredMessages, effectiveSystemPrompt, params], ); // Don't show the bar before the user has any conversation to measure — diff --git a/src/components/PdfPreview.tsx b/src/components/PdfPreview.tsx index 66a7efc..96f7f43 100644 --- a/src/components/PdfPreview.tsx +++ b/src/components/PdfPreview.tsx @@ -166,11 +166,14 @@ function PdfPreviewBody({ open, onOpenChange, attachment }: PdfPreviewProps) { } /** - * One PDF page, lazy-rendered. We start as a placeholder sized to the page's - * natural aspect ratio (cheap to compute — `getPage()` is fast, only the - * raster step is expensive) and trigger the actual `page.render()` once the - * placeholder enters the viewport. This keeps a 500-page legal PDF from - * pinning the renderer for seconds on open. + * One PDF page, lazy-rendered. The placeholder starts at a default + * letter-paper aspect and resolves its TRUE aspect — together with the actual + * `page.render()` raster — only once it nears the viewport (via the + * IntersectionObserver below). Resolving dimensions lazily, instead of eagerly + * calling `getPage()` for every page on open, keeps a 500-page legal PDF from + * issuing hundreds of page parses (and pinning the worker) the instant it + * opens. Mixed-size PDFs show the default aspect until a page is scrolled near, + * then settle to their real proportions. */ function PdfPage({ doc, pageNumber }: { doc: PDFDocumentProxy; pageNumber: number }) { const containerRef = useRef(null); @@ -179,26 +182,6 @@ function PdfPage({ doc, pageNumber }: { doc: PDFDocumentProxy; pageNumber: numbe const [visible, setVisible] = useState(false); const [rendered, setRendered] = useState(false); - // Compute the page's natural pixel dimensions immediately so the placeholder - // takes the right amount of space — otherwise every page would be a zero- - // height div and IntersectionObserver would fire for all of them at once. - useEffect(() => { - let cancelled = false; - (async () => { - const page = await doc.getPage(pageNumber); - // 1.5× scale is a good readability default — sharp on standard - // monitors without making A4 pages overflow on common laptop widths. - const viewport = page.getViewport({ scale: 1.5 }); - if (!cancelled) { - setDims({ width: viewport.width, height: viewport.height }); - } - page.cleanup(); - })(); - return () => { - cancelled = true; - }; - }, [doc, pageNumber]); - // IntersectionObserver flips `visible` once the placeholder enters (or // gets near) the viewport. `rootMargin` extends the trigger zone so pages // start rendering slightly before they're scrolled into view — masks the @@ -240,6 +223,13 @@ function PdfPage({ doc, pageNumber }: { doc: PDFDocumentProxy; pageNumber: numbe // 100-page PDF). const dpr = Math.min(window.devicePixelRatio || 1, 2); const viewport = page.getViewport({ scale: 2 * dpr }); + // Resolve this page's true aspect now, lazily (deferred from open). The + // ratio is scale-invariant, so the raster viewport doubles as the + // dimensions source; the placeholder used the letter-paper default until + // this point. + if (!cancelled) { + setDims({ width: viewport.width, height: viewport.height }); + } const canvas = canvasRef.current; if (!canvas || cancelled) { page.cleanup(); diff --git a/src/stores/modelsStore.ts b/src/stores/modelsStore.ts index ce4b650..048f92b 100644 --- a/src/stores/modelsStore.ts +++ b/src/stores/modelsStore.ts @@ -144,33 +144,45 @@ export const useModelsStore = create((set, get) => ({ refresh: async () => { set({ loading: true, error: null }); const s = useSettingsStore.getState(); - const out: ModelInfo[] = []; - let err: string | null = null; - - try { - const ollama = await ollamaListModels(s.ollama_base_url); - out.push(...ollama); - } catch (e) { - // Ollama being unreachable is a common "not running" state rather - // than a hard failure — surface it as a soft error so the panel can - // show a "start ollama serve" hint without hiding the OpenAI list. - err = e instanceof Error ? e.message : String(e); - } // Try the OpenAI-compatible listing if either a key is stored or the base // URL has been pointed away from the public default — local/proxy servers // (llama-server, LM Studio, vLLM, LiteLLM) don't require auth, so gating // purely on `openai_key_set` would hide their models from the picker. const baseChanged = s.openai_base_url !== DEFAULT_SETTINGS.openai_base_url; - if (s.openai_key_set || baseChanged) { - try { - const openai = await openaiListModels(s.openai_base_url); - out.push(...openai); - } catch (e) { - // Don't clobber a more meaningful Ollama error with a minor OpenAI - // listing hiccup; only set if nothing failed yet. - if (!err) err = e instanceof Error ? e.message : String(e); - } + const wantOpenai = s.openai_key_set || baseChanged; + + // Fire both listings concurrently — a slow or unreachable Ollama (connect + // timeout) used to block the cloud list behind its sequential await. + // `allSettled` so one failing never rejects the other. + const [ollamaRes, openaiRes] = await Promise.allSettled([ + ollamaListModels(s.ollama_base_url), + wantOpenai ? openaiListModels(s.openai_base_url) : Promise.resolve([]), + ]); + + // Assemble in fixed order (Ollama first) and preserve the error priority: + // a real Ollama failure wins, OpenAI only fills in if nothing failed yet. + const out: ModelInfo[] = []; + let err: string | null = null; + if (ollamaRes.status === "fulfilled") { + out.push(...ollamaRes.value); + } else { + // Ollama unreachable is a common "not running" state — surface it as a + // soft error (the panel shows a "start ollama serve" hint) without + // hiding the OpenAI list. + err = + ollamaRes.reason instanceof Error + ? ollamaRes.reason.message + : String(ollamaRes.reason); + } + if (openaiRes.status === "fulfilled") { + out.push(...openaiRes.value); + } else if (!err) { + // Don't clobber a more meaningful Ollama error with a minor OpenAI hiccup. + err = + openaiRes.reason instanceof Error + ? openaiRes.reason.message + : String(openaiRes.reason); } set({ models: out, loading: false, error: err }); diff --git a/src/stores/settingsStore.ts b/src/stores/settingsStore.ts index 56afda8..57332fd 100644 --- a/src/stores/settingsStore.ts +++ b/src/stores/settingsStore.ts @@ -32,7 +32,15 @@ export const useSettingsStore = create((set, get) => ({ hydrate: async () => { try { - const rows = await getSettings(); + // Fire the settings read and the OpenAI-key keyring probe concurrently: + // they're independent, and the probe (a spawn_blocking keyring read on + // the Rust side) otherwise serialized a whole round-trip after the SQLite + // read. getSettings rejecting still hits the outer catch; the key probe + // keeps its own fallback so a keyring failure never blocks hydration. + const [rows, hasKey] = await Promise.all([ + getSettings(), + getOpenAIKeyStatus().catch(() => false), + ]); const merged: Settings = { ...DEFAULT_SETTINGS }; // Settings live in a string-keyed KV table; coerce each value back into // the type implied by DEFAULT_SETTINGS so booleans don't arrive as the @@ -47,7 +55,6 @@ export const useSettingsStore = create((set, get) => ({ (merged as unknown as Record)[k as string] = v; } }); - const hasKey = await getOpenAIKeyStatus().catch(() => false); set({ ...merged, openai_key_set: hasKey, hydrated: true }); applyTheme(merged.theme); applyFontSize(merged.font_size); From 3157b295abf3f932b10e2485f0133b3bc6fb20a8 Mon Sep 17 00:00:00 2001 From: ZTCS Date: Fri, 12 Jun 2026 16:23:38 +0200 Subject: [PATCH 4/9] =?UTF-8?q?restore=20Aurora=20gradient=20by=20pre-pain?= =?UTF-8?q?ting=20body,=20not=20html=20(html=20background=20disabled=20bod?= =?UTF-8?q?y=E2=86=92canvas=20propagation,=20hiding=20the=20-z-10=20mesh)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.html | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/index.html b/index.html index f89ada5..e8a9393 100644 --- a/index.html +++ b/index.html @@ -9,8 +9,15 @@ globals.css; this inline + — same as before, just without the preceding white frame. + + MUST target `body`, NOT `html`: the Aurora gradient is a `-z-10` fixed + div (`.app-mesh`). The body→canvas background propagation that lets a + negative-z-index element show through only happens while `` has + NO background of its own. Giving `` a background disables that + propagation, so the opaque `body` background then paints OVER the mesh + and Aurora collapses to a flat fill (looks like the Solid theme). --> +
From c9617dfb8e0c3e2a5d9d3ca683aa37cc8052c5be Mon Sep 17 00:00:00 2001 From: ZTCS Date: Fri, 12 Jun 2026 16:58:30 +0200 Subject: [PATCH 5/9] enable in-app OTA updates for Linux .deb/.rpm installs --- src-tauri/src/commands.rs | 17 ++++++++++++----- src/components/UpdatesPanel.tsx | 2 +- src/lib/tauri.ts | 11 ++++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index db748f2..36ac987 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -2086,8 +2086,11 @@ pub fn open_in_vscode(code: String, filename: String) -> Result<(), String> { // ---------- updater support ---------- // // The Tauri updater can replace a Windows NSIS install or a Linux AppImage in -// place, but it cannot upgrade a `.deb`/`.rpm` install — that has to go -// through the system package manager. On macOS the updater patches the `.app` +// place. Since updater plugin 2.10 it can also upgrade `.deb`/`.rpm` installs: +// it downloads the signed package advertised by the format-specific +// `latest.json` key (`linux-x86_64-deb` / `-rpm`) and elevates via pkexec +// (falling back to zenity/kdialog + sudo) to run `dpkg -i` / `rpm -U`, so the +// package database stays consistent. On macOS the updater patches the `.app` // bundle in place; that works even though we don't sign with Apple, because // the updater's integrity check uses our own Ed25519 signature (separate from // Apple notarization). We expose this so the UI can hide the "Check for @@ -2102,10 +2105,14 @@ pub fn updater_supported() -> bool { #[cfg(target_os = "linux")] { // AppImage runtimes set $APPIMAGE to the absolute path of the running - // bundle; nothing else does. Absence of the var means we're running - // from a `.deb`, a dev build, or `cargo run` — none of which the - // updater plugin can patch. + // bundle; nothing else does. For `.deb`/`.rpm` we read the bundle-type + // marker the bundler patches into the binary at build time — the same + // marker the updater plugin keys its install path off, so this gate + // can't disagree with what the plugin would actually do. Dev builds + // and `cargo run` have no marker and report unsupported, as before. + use tauri::utils::{config::BundleType, platform::bundle_type}; std::env::var("APPIMAGE").is_ok() + || matches!(bundle_type(), Some(BundleType::Deb | BundleType::Rpm)) } #[cfg(target_os = "macos")] { diff --git a/src/components/UpdatesPanel.tsx b/src/components/UpdatesPanel.tsx index 6e6bfd9..2210bde 100644 --- a/src/components/UpdatesPanel.tsx +++ b/src/components/UpdatesPanel.tsx @@ -46,7 +46,7 @@ export function UpdatesPanel() { void isUpdaterSupported().then(setSupported); }, []); - // Unsupported install (Linux .deb, dev build, etc.) — show a friendly + // Unsupported install (dev build, plain binary, etc.) — show a friendly // fallback that points users at the GitHub release page instead of // letting them click into a guaranteed failure. if (supported === false) { diff --git a/src/lib/tauri.ts b/src/lib/tauri.ts index 21dcc1f..a9664a8 100644 --- a/src/lib/tauri.ts +++ b/src/lib/tauri.ts @@ -1025,11 +1025,12 @@ export function factoryReset(auth?: DestructiveAuth): Promise { // ------------ updater ------------ /** Whether the running binary's bundle format supports in-app updates. - * Returns true on Windows (NSIS), and on Linux only when running inside - * an AppImage (`.deb` / `.rpm` installs are stuck with whatever the - * system package manager last installed). Wrapped here so callers don't - * have to construct a raw `invoke("updater_supported")` and stay - * consistent with the rest of the IPC layer. */ + * Returns true on Windows (NSIS), on macOS, and on Linux for AppImage, + * `.deb`, and `.rpm` installs (deb/rpm updates install through pkexec + + * the system package manager, so they prompt for the user's password; + * dev builds report false). Wrapped here so callers don't have to + * construct a raw `invoke("updater_supported")` and stay consistent + * with the rest of the IPC layer. */ export function updaterSupported(): Promise { if (!isTauri) return notInTauri(false); return invoke("updater_supported"); From 4411c07c3ab4ba9eb59ef3371f7fb6981d1d2ba4 Mon Sep 17 00:00:00 2001 From: ZTCS Date: Sat, 13 Jun 2026 04:00:38 +0200 Subject: [PATCH 6/9] add ollama_keep_alive to the settings write allowlist, with a lockstep test --- src-tauri/src/commands.rs | 1 + tests/settingsAllowlist.test.ts | 57 +++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 tests/settingsAllowlist.test.ts diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 36ac987..b3f860f 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -444,6 +444,7 @@ const WRITABLE_SETTING_KEYS: &[&str] = &[ "temporal_awareness", "web_fetch_enabled", "low_vram_global", + "ollama_keep_alive", "thinking_default", "default_tone_id", "onboarding_completed", diff --git a/tests/settingsAllowlist.test.ts b/tests/settingsAllowlist.test.ts new file mode 100644 index 0000000..8c85412 --- /dev/null +++ b/tests/settingsAllowlist.test.ts @@ -0,0 +1,57 @@ +// Settings-write invariant: the renderer persists every key of +// `DEFAULT_SETTINGS` (src/types.ts) through the `set_setting` IPC command, +// which rejects keys missing from `WRITABLE_SETTING_KEYS` in +// src-tauri/src/commands.rs (built-in tool toggles come from the tool +// registry instead and all end in `_tool_enabled`). The two files are +// maintained by hand in different languages, so neither `tsc` nor `cargo +// check` can see them drift — a key added on the TS side but not the Rust +// side only fails at runtime, as a "not on the writable allowlist" toast +// when the user touches that setting. This test moves the failure to +// `npm test` time. (Regression: `ollama_keep_alive` shipped in the UI +// without an allowlist entry, making the new control un-saveable.) + +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import { DEFAULT_SETTINGS } from "../src/types"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const commandsRs = readFileSync( + path.join(root, "src-tauri", "src", "commands.rs"), + "utf8", +); + +const block = /const WRITABLE_SETTING_KEYS:[^=]*=\s*&\[([\s\S]*?)\];/.exec( + commandsRs, +)?.[1]; +const allowlist = [...(block ?? "").matchAll(/"([^"]+)"/g)].map((m) => m[1]); + +describe("settings allowlist agreement (types.ts vs commands.rs)", () => { + it("finds the WRITABLE_SETTING_KEYS const in commands.rs", () => { + // Parse guard so a refactor of commands.rs fails loudly here instead of + // letting the assertions below pass vacuously against an empty list. + expect(allowlist).toContain("theme"); + }); + + it("every DEFAULT_SETTINGS key is writable via set_setting", () => { + const unwritable = Object.keys(DEFAULT_SETTINGS).filter( + (k) => !k.endsWith("_tool_enabled") && !allowlist.includes(k), + ); + expect( + unwritable, + "add these to WRITABLE_SETTING_KEYS in src-tauri/src/commands.rs — " + + "the Settings UI cannot save them otherwise", + ).toEqual([]); + }); + + it("the allowlist holds no stale keys absent from DEFAULT_SETTINGS", () => { + const keys = Object.keys(DEFAULT_SETTINGS); + const stale = allowlist.filter((k) => !keys.includes(k)); + expect( + stale, + "remove these from WRITABLE_SETTING_KEYS (or add them to DEFAULT_SETTINGS) — " + + "the comment on the const promises the two stay in lockstep", + ).toEqual([]); + }); +}); From a725d64519cff95d2025e164affe3ad8fb18a109 Mon Sep 17 00:00:00 2001 From: ZTCS Date: Sat, 13 Jun 2026 04:11:21 +0200 Subject: [PATCH 7/9] launch VS Code via LaunchServices so "Open in VS Code" works under Finder's minimal PATH --- src-tauri/src/commands.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index b3f860f..ffd2ae5 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -2070,7 +2070,26 @@ pub fn open_in_vscode(code: String, filename: String) -> Result<(), String> { .spawn() .map_err(|e| format!("Couldn't launch VS Code: {e}"))?; } - #[cfg(not(windows))] + #[cfg(target_os = "macos")] + { + // GUI apps launched from Finder/Dock inherit a minimal PATH + // (/usr/bin:/bin:/usr/sbin:/sbin) that excludes /usr/local/bin (Intel) + // and /opt/homebrew/bin (Apple Silicon) where VS Code's `code` shim + // lives, so spawning `code` directly fails even when it works from a + // terminal. `open` is always at /usr/bin/open and resolves the app + // through LaunchServices, which doesn't depend on PATH. `.status()` + // returns as soon as `open` hands off (fast) and is non-zero when the + // app can't be found, so we can still surface the `not_found` hint. + let status = Command::new("open") + .args(["-a", "Visual Studio Code"]) + .arg(&path) + .status() + .map_err(|e| format!("Couldn't launch VS Code: {e}"))?; + if !status.success() { + return Err(not_found); + } + } + #[cfg(all(unix, not(target_os = "macos")))] { Command::new("code").arg(&path).spawn().map_err(|e| { if e.kind() == std::io::ErrorKind::NotFound { From 1e713f65bbedd4b54560881618c676bd2d97d893 Mon Sep 17 00:00:00 2001 From: ZTCS Date: Sat, 13 Jun 2026 04:38:31 +0200 Subject: [PATCH 8/9] run on pushes to main and PRs only, not every dev-branch push --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ff28be..712e18a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,9 @@ name: CI # gate in tests/versions.test.ts), and the Rust unit tests. Releases are # handled by release.yml; this workflow never publishes anything. # -# Triggers on every push so dev branches get feedback without a PR, and on -# PRs to main so the merge gate sees a check. NOTE: this workflow does NOT +# Triggers on pushes to main so each merge commit gets its own verdict, and +# on PRs to main so the merge gate sees a check before merge. Dev branches +# get CI once they open a PR. NOTE: this workflow does NOT # gate release.yml — both fire on the same push-to-main event and run in # parallel, and release.yml drafts its release without waiting for CI (the # draft still needs a manual publish, which is the human backstop). The @@ -16,6 +17,7 @@ name: CI on: push: + branches: [main] pull_request: branches: [main] workflow_dispatch: From 922568be1d72c68d6beda0696cc24b6120d37589 Mon Sep 17 00:00:00 2001 From: ZTCS Date: Wed, 17 Jun 2026 17:14:15 +0200 Subject: [PATCH 9/9] Update to v1.2.4 --- package.json | 2 +- releases/1.2.4.md | 8 ++++++++ src-tauri/Cargo.lock | 2 +- src-tauri/Cargo.toml | 2 +- src-tauri/tauri.conf.json | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 releases/1.2.4.md diff --git a/package.json b/package.json index 2564cdc..ba58960 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "loach", "private": true, - "version": "1.2.3", + "version": "1.2.4", "type": "module", "description": "Native local LLM chat client for Windows, Linux and macOS (Ollama + OpenAI-compatible).", "license": "MIT", diff --git a/releases/1.2.4.md b/releases/1.2.4.md new file mode 100644 index 0000000..1ffab71 --- /dev/null +++ b/releases/1.2.4.md @@ -0,0 +1,8 @@ +**What's new** +- **Faster across the board.** Loach cold-starts quicker, replies begin sooner, and long answers stream more smoothly even in big sessions with large transcripts. Under the hood: lazier startup and hydration, cached model and MCP catalogs, windowed transcript rendering, chunked markdown while streaming, a pooled database reader, and a leaner release build. +- **In-app updates for Linux `.deb` and `.rpm`.** Debian and RPM installs now update from within Loach the way AppImage already did — Loach downloads the signed package, elevates to install it, and keeps your package database consistent. (Updating *to* this release still needs one manual download; after that, updates are in-app.) + +**Fixes** +- **Ollama keep-alive now saves.** Changing how long models stay resident in memory now persists instead of silently reverting to the default. +- **"Open in VS Code" on macOS.** The Code Canvas's *Open in VS Code* button now works when Loach was launched from Finder. +- More accurate tokens/sec readout, plus assorted stability hardening under the hood. diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index a87293d..0cbc17e 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -2420,7 +2420,7 @@ checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "loach" -version = "1.2.3" +version = "1.2.4" dependencies = [ "anyhow", "argon2", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 72604eb..c95813f 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "loach" -version = "1.2.3" +version = "1.2.4" description = "Native local LLM chat client (Ollama + OpenAI-compatible)." authors = ["Loach"] license = "MIT" diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 9e41a5a..e2ab040 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Loach", - "version": "1.2.3", + "version": "1.2.4", "identifier": "dev.loach.app", "build": { "beforeDevCommand": "npm run dev",