docs: document backend architecture

crates/abyss-backend/migrations/20260824000000_initial_schema/down.sql

@@ -1,3 +1,5 @@
+-- Drop in reverse dependency order so every foreign key remains valid while
+-- the consolidated standalone schema is removed.
 DROP TABLE IF EXISTS search_outbox;
 DROP TABLE IF EXISTS agent_diagnostic_capture_events;
 DROP TABLE IF EXISTS agent_diagnostic_captures;

crates/abyss-backend/migrations/20260824000000_initial_schema/up.sql

@@ -1,3 +1,6 @@
+-- The standalone backend currently authenticates one deployment-wide owner.
+-- Keeping the owner as a real row preserves explicit foreign-key ownership and
+-- leaves the event schema ready for a future identity model without nullable IDs.
 CREATE TABLE app_users (
     id uuid PRIMARY KEY,
     email text NOT NULL,
@@ -12,6 +15,8 @@ VALUES (
     'Abyss Owner'
 );
 
+-- A device is a user-visible context, not a hardware identity. Host name and
+-- platform form the stable bucket; observed OS version and time bounds evolve.
 CREATE TABLE devices (
     id uuid PRIMARY KEY,
     user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE,
@@ -28,6 +33,8 @@ CREATE TABLE devices (
 CREATE INDEX devices_user_seen_idx
     ON devices (user_id, last_seen_at DESC);
 
+-- Sessions use the Agent-native session ID plus canonical Agent name. The
+-- device reference follows the most recently observed device for that session.
 CREATE TABLE agent_sessions (
     id uuid PRIMARY KEY,
     user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE,
@@ -46,6 +53,8 @@ CREATE TABLE agent_sessions (
 CREATE INDEX agent_sessions_user_time_idx
     ON agent_sessions (user_id, started_at DESC);
 
+-- Turn indexes are backend-normalized during ingest when stable provider or
+-- Agent metadata reveals that a collector restarted its local counter.
 CREATE TABLE agent_turns (
     id uuid PRIMARY KEY,
     user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE,
@@ -61,6 +70,8 @@ CREATE TABLE agent_turns (
 CREATE INDEX agent_turns_user_session_idx
     ON agent_turns (user_id, session_pk, turn_index);
 
+-- Usage events are immutable source records. event_id provides global ingest
+-- idempotency, while denormalized labels keep filters and aggregation direct.
 CREATE TABLE llm_usage_events (
     id uuid PRIMARY KEY,
     user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE,
@@ -100,6 +111,8 @@ CREATE INDEX llm_usage_events_session_turn_idx
 CREATE INDEX llm_usage_events_agent_model_time_idx
     ON llm_usage_events (agent_name, llm_provider, llm_model, observed_at DESC);
 
+-- Attachment content is optional so collectors can report image usage metadata
+-- without uploading bytes. Hash, size, type, and position remain queryable.
 CREATE TABLE llm_usage_event_attachments (
     id uuid PRIMARY KEY,
     user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE,
@@ -119,6 +132,8 @@ CREATE TABLE llm_usage_event_attachments (
 CREATE INDEX llm_usage_event_attachments_user_event_idx
     ON llm_usage_event_attachments (user_id, event_pk, position);
 
+-- Diagnostic payloads are deliberately opaque; the join table records the
+-- usage events that establish their owner, session, and device context.
 CREATE TABLE agent_diagnostic_captures (
     id uuid PRIMARY KEY,
     user_id uuid NOT NULL REFERENCES app_users(id) ON DELETE CASCADE,
@@ -148,6 +163,9 @@ CREATE TABLE agent_diagnostic_capture_events (
 CREATE INDEX agent_diagnostic_capture_events_event_idx
     ON agent_diagnostic_capture_events (event_pk);
 
+-- Search is a rebuildable projection. Outbox rows intentionally do not foreign
+-- key event_pk so a delete operation can survive removal of its source row.
+-- Expiring claims and terminal dead letters support concurrent crash recovery.
 CREATE TABLE search_outbox (
     id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
     event_pk uuid NOT NULL,

crates/abyss-backend/src/api.rs

@@ -1,4 +1,9 @@
-//! HTTP routes for standalone Agent event ingestion and queries.
+//! HTTP boundary for standalone Agent event ingestion and queries.
+//!
+//! The health endpoints are public, while every event, attachment, summary,
+//! timeline, and search endpoint authenticates the deployment bearer token.
+//! Diesel is synchronous, so all database access is moved onto Tokio's blocking
+//! pool through [`run_db`] rather than running on asynchronous executor threads.
 
 use axum::{
     Json, Router,
@@ -31,16 +36,28 @@ use crate::{
 const MAX_INGEST_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024;
 
 #[derive(Clone)]
+/// Cloneable dependencies and request limits shared by all Axum handlers.
 pub struct AppState {
+    /// Deployment label exposed by the informational root endpoint.
     pub environment: String,
+    /// Maximum events and diagnostic captures accepted per ingest request.
     pub max_ingest_batch_size: usize,
+    /// Default upper bound for summary aggregation rows.
     pub summary_scan_limit: i64,
+    /// Default raw-event page size.
     pub default_page_size: i64,
+    /// Deployment-wide bearer-token validator.
     pub identity: IdentityAuthenticator,
+    /// Optional full-text search service.
     pub search: Option<SearchService>,
+    /// PostgreSQL connection pool used by request handlers.
     pub pool: DbPool,
 }
 
+/// Builds the complete HTTP router for one backend process.
+///
+/// The ingest body limit is intentionally route-local so future endpoints do
+/// not inherit a large request allowance by accident.
 pub fn router(state: AppState) -> Router {
     Router::new()
         .route("/", get(root))
@@ -82,6 +99,8 @@ async fn health() -> Json<ServiceStatus> {
 }
 
 async fn ready(State(state): State<AppState>) -> Result<Json<ServiceStatus>, AppError> {
+    // Readiness checks the source of truth only. Search is an optional derived
+    // service and its temporary failure must not remove ingestion capacity.
     run_db(state, db::check_ready).await?;
     Ok(Json(ServiceStatus {
         service: "abyss-backend",
@@ -133,6 +152,8 @@ async fn session_search(
         .ok_or_else(|| AppError::unavailable("session search is not configured".to_owned()))?;
     let execution = search.search(user_id, query).await?;
     let session_ids = execution.session_ids();
+    // Elasticsearch contains only a bounded search projection. Authoritative
+    // session/device details are reloaded from PostgreSQL under the owner scope.
     let details = run_db(state, move |connection| {
         SearchOutboxRepository::session_details(connection, user_id, &session_ids)
     })
@@ -189,6 +210,8 @@ async fn image_attachment(
         header::CACHE_CONTROL,
         HeaderValue::from_static("private, no-store"),
     );
+    // Attachments may contain sensitive conversation context. Disallow MIME
+    // sniffing, shared caching, and cross-origin embedding even for valid images.
     response_headers.insert(
         header::X_CONTENT_TYPE_OPTIONS,
         HeaderValue::from_static("nosniff"),
@@ -219,6 +242,8 @@ where
     T: Send + 'static,
     F: FnOnce(&mut PgConnection) -> Result<T, AppError> + Send + 'static,
 {
+    // Diesel and r2d2 are blocking APIs. Acquiring the pool connection inside
+    // spawn_blocking also keeps pool contention off Tokio worker threads.
     task::spawn_blocking(move || {
         let mut connection = state.pool.get()?;
         task_fn(&mut connection)

crates/abyss-backend/src/config.rs

@@ -1,4 +1,9 @@
 //! Environment-backed standalone backend configuration.
+//!
+//! Every process setting is read once during startup. Empty or whitespace-only
+//! values are treated as absent, positive numeric settings are rejected at the
+//! boundary, and Elasticsearch is enabled only when its URL is present. This
+//! leaves the rest of the service with a fully validated, immutable snapshot.
 
 use std::{env, net::SocketAddr, num::ParseIntError};
 
@@ -15,22 +20,36 @@ const DEFAULT_SEARCH_REQUEST_TIMEOUT_SECONDS: u64 = 10;
 const DEFAULT_SEARCH_POLL_INTERVAL_MILLISECONDS: u64 = 500;
 const DEFAULT_SEARCH_BATCH_SIZE: i64 = 100;
 
+/// Complete runtime configuration shared by process startup and HTTP state.
 pub struct Config {
+    /// Socket address on which the HTTP server listens.
     pub addr: SocketAddr,
+    /// Deployment label returned by the root endpoint and used by safety checks.
     pub environment: String,
+    /// Explicit escape hatch for running black-box instances in containers.
     pub blackbox_allow_non_loopback: bool,
+    /// Default tracing filter used when `RUST_LOG` is not set.
     pub log_level: String,
+    /// PostgreSQL connection string.
     pub database_url: String,
+    /// Maximum number of PostgreSQL connections held by the r2d2 pool.
     pub database_pool_size: u32,
+    /// Whether embedded Diesel migrations run before the listener starts.
     pub run_migrations: bool,
+    /// Maximum events and diagnostic captures accepted in one ingest request.
     pub max_ingest_batch_size: usize,
+    /// Default and maximum row limit used by summary aggregation.
     pub summary_scan_limit: i64,
+    /// Default number of events returned by paginated raw-event queries.
     pub default_page_size: i64,
+    /// Deployment-wide bearer-token authentication configuration.
     pub identity: IdentityConfig,
+    /// Optional Elasticsearch projection configuration.
     pub search: Option<SearchConfig>,
 }
 
 impl Config {
+    /// Reads and validates all `ABYSS_BACKEND_*` environment variables.
     pub fn from_env() -> Result<Self, AppError> {
         let addr = read_env("ABYSS_BACKEND_ADDR", DEFAULT_ADDR)
             .parse::<SocketAddr>()
@@ -73,12 +92,19 @@ impl Config {
 }
 
 #[derive(Clone)]
+/// Settings required by the Elasticsearch client and projection worker.
 pub struct SearchConfig {
+    /// Base Elasticsearch URL, without an index or API path suffix.
     pub endpoint: String,
+    /// Optional HTTP Basic Authentication username.
     pub username: Option<String>,
+    /// Optional HTTP Basic Authentication password.
     pub password: Option<String>,
+    /// Per-request Elasticsearch timeout in seconds.
     pub request_timeout_seconds: u64,
+    /// Idle polling delay for the search outbox worker.
     pub poll_interval_milliseconds: u64,
+    /// Maximum outbox rows claimed or backfilled per worker iteration.
     pub batch_size: i64,
 }
 
@@ -88,6 +114,8 @@ impl SearchConfig {
         let username = env_value("ABYSS_BACKEND_ELASTICSEARCH_USERNAME");
         let password = env_value("ABYSS_BACKEND_ELASTICSEARCH_PASSWORD");
 
+        // Credentials without an endpoint almost always indicate a misspelled
+        // or missing secret, so fail startup instead of silently disabling search.
         let Some(endpoint) = endpoint else {
             if username.is_some() || password.is_some() {
                 return Err(AppError::config(
@@ -98,6 +126,7 @@ impl SearchConfig {
             return Ok(None);
         };
 
+        // Basic Authentication is only meaningful as a complete pair.
         if username.is_some() != password.is_some() {
             return Err(AppError::config(
                 "ABYSS_BACKEND_ELASTICSEARCH_USERNAME and ABYSS_BACKEND_ELASTICSEARCH_PASSWORD must be configured together"
@@ -134,6 +163,8 @@ fn read_required_env(key: &str) -> Result<String, AppError> {
 }
 
 fn env_value(key: &str) -> Option<String> {
+    // Normalizing empty secrets to None makes mounted-but-empty Kubernetes
+    // secrets behave the same as missing environment variables.
     env::var(key)
         .ok()
         .map(|value| value.trim().to_owned())

crates/abyss-backend/src/db/mod.rs

@@ -1,17 +1,25 @@
-//! Database connection pool, migrations, and persistence modules.
+//! PostgreSQL connection pool, embedded migrations, and persistence modules.
+//!
+//! A synchronous Diesel pool is shared by HTTP handlers and the search worker.
+//! Callers running on Tokio are responsible for entering this module through a
+//! blocking task so neither connection acquisition nor SQL blocks the executor.
 
+/// Diesel models used for reads and inserts.
 pub mod models;
+/// Diesel's compile-time representation of the PostgreSQL schema.
 pub mod schema;
 
 use diesel::{PgConnection, RunQueryDsl, r2d2::ConnectionManager, sql_query};
 use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations};
 
 use crate::{config::Config, error::AppError};
 
+/// Cloneable pool of synchronous PostgreSQL connections.
 pub type DbPool = r2d2::Pool<ConnectionManager<PgConnection>>;
 
 const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
 
+/// Creates the PostgreSQL pool and verifies that its initial connection opens.
 pub fn create_pool(config: &Config) -> Result<DbPool, AppError> {
     let manager = ConnectionManager::<PgConnection>::new(config.database_url.clone());
     r2d2::Pool::builder()
@@ -20,12 +28,14 @@ pub fn create_pool(config: &Config) -> Result<DbPool, AppError> {
         .map_err(AppError::from)
 }
 
+/// Applies all embedded migrations that are not recorded by Diesel yet.
 pub fn run_migrations(pool: &DbPool) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let mut connection = pool.get()?;
     connection.run_pending_migrations(MIGRATIONS)?;
     Ok(())
 }
 
+/// Executes the minimal PostgreSQL query used by the readiness endpoint.
 pub fn check_ready(connection: &mut PgConnection) -> Result<(), AppError> {
     sql_query("SELECT 1").execute(connection)?;
     Ok(())