From 5264063f8d146c73306384f3397fd3f36b8d0d02 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 12:59:37 +0100 Subject: [PATCH 01/43] feat: add audit log for tracking user actions (#94) - TimescaleDB hypertable with compression and 1y retention - buffer + flush service for non-blocking writes - org owner/admin access via settings page - expandable table rows with full event details - CSV export with filters - audit calls on auth, orgs, projects, api keys, admin ops --- CHANGELOG.md | 17 + packages/backend/migrations/025_audit_log.sql | 48 ++ packages/backend/src/database/types.ts | 31 + packages/backend/src/modules/admin/routes.ts | 67 +++ .../backend/src/modules/api-keys/routes.ts | 27 + .../backend/src/modules/audit-log/index.ts | 3 + .../backend/src/modules/audit-log/routes.ts | 211 +++++++ .../backend/src/modules/audit-log/service.ts | 161 ++++++ .../src/modules/organizations/routes.ts | 76 +++ .../backend/src/modules/projects/routes.ts | 46 ++ packages/backend/src/modules/users/routes.ts | 42 ++ packages/backend/src/server.ts | 4 + .../tests/modules/audit-log/routes.test.ts | 477 ++++++++++++++++ .../tests/modules/audit-log/service.test.ts | 527 +++++++++++++++++ packages/backend/src/tests/setup.ts | 1 + packages/frontend/src/lib/api/audit-log.ts | 102 ++++ .../routes/dashboard/settings/+page.svelte | 24 + .../dashboard/settings/audit-log/+page.svelte | 537 ++++++++++++++++++ 18 files changed, 2401 insertions(+) create mode 100644 packages/backend/migrations/025_audit_log.sql create mode 100644 packages/backend/src/modules/audit-log/index.ts create mode 100644 packages/backend/src/modules/audit-log/routes.ts create mode 100644 packages/backend/src/modules/audit-log/service.ts create mode 100644 packages/backend/src/tests/modules/audit-log/routes.test.ts create mode 100644 packages/backend/src/tests/modules/audit-log/service.test.ts create mode 100644 packages/frontend/src/lib/api/audit-log.ts create mode 100644 packages/frontend/src/routes/dashboard/settings/audit-log/+page.svelte diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e833fca..cd68877e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to LogTide will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.7.0] - Unreleased + +### Added + +- **Audit Log**: comprehensive audit trail tracking all user actions across the platform for compliance and security (SOC 2, ISO 27001, HIPAA) + - Tracks 4 event categories: log access, config changes, user management, data modifications + - Logged actions: login, logout, register, create/update/delete organizations, create/update/delete projects, create/revoke API keys, member role changes, member removal, leave organization, admin operations + - TimescaleDB hypertable with 7-day chunks, automatic compression (30 days), and retention policy (365 days) + - High-performance in-memory buffer with periodic flush (50 entries or 1s interval) for non-blocking writes + - Accessible to organization owners and admins via Organization Settings + - Expandable table rows showing full event details: metadata, resource IDs, user agent, IP address + - Category and action filters + - CSV export with current filters applied (up to 10k rows) + - Export actions are themselves audit-logged (meta-meta logging) + +--- + ## [0.6.3] - 2026-02-22 ### Fixed diff --git a/packages/backend/migrations/025_audit_log.sql b/packages/backend/migrations/025_audit_log.sql new file mode 100644 index 00000000..6b4e8d88 --- /dev/null +++ b/packages/backend/migrations/025_audit_log.sql @@ -0,0 +1,48 @@ +-- Migration 025: Audit log table +-- Append-only table for compliance audit trail +-- TimescaleDB hypertable for automatic compression and retention + +CREATE TABLE IF NOT EXISTS audit_log ( + time TIMESTAMPTZ NOT NULL DEFAULT NOW(), + id UUID NOT NULL DEFAULT gen_random_uuid(), + PRIMARY KEY (time, id), + + organization_id UUID, + user_id UUID, + user_email TEXT, + action TEXT NOT NULL, + category TEXT NOT NULL, + resource_type TEXT, + resource_id TEXT, + ip_address TEXT, + user_agent TEXT, + metadata JSONB, + + CONSTRAINT audit_log_category_check CHECK ( + category IN ('log_access', 'config_change', 'user_management', 'data_modification') + ) +); + +SELECT create_hypertable('audit_log', 'time', + chunk_time_interval => INTERVAL '7 days', + if_not_exists => TRUE +); + +ALTER TABLE audit_log SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'organization_id', + timescaledb.compress_orderby = 'time DESC' +); + +SELECT add_compression_policy('audit_log', INTERVAL '30 days', if_not_exists => TRUE); +SELECT add_retention_policy('audit_log', INTERVAL '365 days', if_not_exists => TRUE); + +CREATE INDEX IF NOT EXISTS idx_audit_log_org_time + ON audit_log (organization_id, time DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_log_org_category + ON audit_log (organization_id, category, time DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_log_org_user + ON audit_log (organization_id, user_id, time DESC) + WHERE user_id IS NOT NULL; diff --git a/packages/backend/src/database/types.ts b/packages/backend/src/database/types.ts index 7f567a4f..02c8bd3e 100644 --- a/packages/backend/src/database/types.ts +++ b/packages/backend/src/database/types.ts @@ -726,6 +726,35 @@ export interface OrganizationPiiSaltsTable { created_at: Generated; } +// ============================================================================ +// AUDIT LOG TABLE +// ============================================================================ + +export type AuditCategory = + | 'log_access' + | 'config_change' + | 'user_management' + | 'data_modification'; + +export interface AuditLogTable { + time: Generated; + id: Generated; + organization_id: string | null; + user_id: string | null; + user_email: string | null; + action: string; + category: AuditCategory; + resource_type: string | null; + resource_id: string | null; + ip_address: string | null; + user_agent: string | null; + metadata: ColumnType< + Record | null, + Record | null, + Record | null + >; +} + export interface Database { logs: LogsTable; users: UsersTable; @@ -781,4 +810,6 @@ export interface Database { // PII masking pii_masking_rules: PiiMaskingRulesTable; organization_pii_salts: OrganizationPiiSaltsTable; + // Audit log + audit_log: AuditLogTable; } diff --git a/packages/backend/src/modules/admin/routes.ts b/packages/backend/src/modules/admin/routes.ts index 32139602..45f78936 100644 --- a/packages/backend/src/modules/admin/routes.ts +++ b/packages/backend/src/modules/admin/routes.ts @@ -2,6 +2,7 @@ import type { FastifyInstance } from 'fastify'; import { adminService } from './service.js'; import { authenticate } from '../auth/middleware.js'; import { requireAdmin } from './middleware.js'; +import { auditLogService } from '../audit-log/index.js'; export async function adminRoutes(fastify: FastifyInstance) { // All routes require session authentication + admin role @@ -243,6 +244,19 @@ export async function adminRoutes(fastify: FastifyInstance) { const user = await adminService.updateUserStatus(id, disabled); + auditLogService.log({ + organizationId: null, + userId: (request as any).user?.id, + userEmail: (request as any).user?.email, + action: disabled ? 'disable_user' : 'enable_user', + category: 'user_management', + resourceType: 'user', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { targetEmail: user.email }, + }); + return reply.send({ message: `User ${disabled ? 'disabled' : 'enabled'} successfully`, user, @@ -285,6 +299,19 @@ export async function adminRoutes(fastify: FastifyInstance) { const user = await adminService.updateUserRole(id, is_admin); + auditLogService.log({ + organizationId: null, + userId: (request as any).user?.id, + userEmail: (request as any).user?.email, + action: 'update_user_role', + category: 'user_management', + resourceType: 'user', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { is_admin, targetEmail: user.email }, + }); + return reply.send({ message: `User ${is_admin ? 'promoted to admin' : 'demoted from admin'} successfully`, user, @@ -319,6 +346,19 @@ export async function adminRoutes(fastify: FastifyInstance) { const user = await adminService.resetUserPassword(id, newPassword); + auditLogService.log({ + organizationId: null, + userId: (request as any).user?.id, + userEmail: (request as any).user?.email, + action: 'reset_user_password', + category: 'user_management', + resourceType: 'user', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { targetEmail: user.email }, + }); + return reply.send({ message: 'Password reset successfully', user, @@ -414,6 +454,19 @@ export async function adminRoutes(fastify: FastifyInstance) { try { const { id } = request.params as { id: string }; const result = await adminService.deleteOrganization(id); + + auditLogService.log({ + organizationId: id, + userId: (request as any).user?.id, + userEmail: (request as any).user?.email, + action: 'admin_delete_organization', + category: 'data_modification', + resourceType: 'organization', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.send(result); } catch (error: any) { console.error('Error deleting organization:', error); @@ -505,7 +558,21 @@ export async function adminRoutes(fastify: FastifyInstance) { async (request, reply) => { try { const { id } = request.params as { id: string }; + const project = await adminService.getProjectDetails(id); const result = await adminService.deleteProject(id); + + auditLogService.log({ + organizationId: project.organization_id, + userId: (request as any).user?.id, + userEmail: (request as any).user?.email, + action: 'admin_delete_project', + category: 'data_modification', + resourceType: 'project', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.send(result); } catch (error: any) { console.error('Error deleting project:', error); diff --git a/packages/backend/src/modules/api-keys/routes.ts b/packages/backend/src/modules/api-keys/routes.ts index 59efa6fb..a20f031d 100644 --- a/packages/backend/src/modules/api-keys/routes.ts +++ b/packages/backend/src/modules/api-keys/routes.ts @@ -4,6 +4,7 @@ import { API_KEY_TYPES } from '@logtide/shared'; import { apiKeysService } from './service.js'; import { authenticate } from '../auth/middleware.js'; import { projectsService } from '../projects/service.js'; +import { auditLogService } from '../audit-log/index.js'; const createApiKeySchema = z.object({ name: z.string().min(1, 'Name is required').max(100, 'Name too long'), @@ -75,6 +76,19 @@ export async function apiKeysRoutes(fastify: FastifyInstance) { allowedOrigins: body.allowedOrigins ?? null, }); + auditLogService.log({ + organizationId: project.organizationId, + userId: request.user.id, + userEmail: request.user.email, + action: 'create_api_key', + category: 'config_change', + resourceType: 'api_key', + resourceId: result.id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { name: body.name, type: body.type, projectId }, + }); + return reply.status(201).send({ id: result.id, apiKey: result.apiKey, @@ -114,6 +128,19 @@ export async function apiKeysRoutes(fastify: FastifyInstance) { }); } + auditLogService.log({ + organizationId: project.organizationId, + userId: request.user.id, + userEmail: request.user.email, + action: 'revoke_api_key', + category: 'config_change', + resourceType: 'api_key', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { projectId }, + }); + return reply.status(204).send(); } catch (error) { if (error instanceof z.ZodError) { diff --git a/packages/backend/src/modules/audit-log/index.ts b/packages/backend/src/modules/audit-log/index.ts new file mode 100644 index 00000000..0d6a74f9 --- /dev/null +++ b/packages/backend/src/modules/audit-log/index.ts @@ -0,0 +1,3 @@ +export { auditLogService } from './service.js'; +export type { AuditLogEntry, AuditLogQueryParams, AuditLogResult } from './service.js'; +export { auditLogRoutes } from './routes.js'; diff --git a/packages/backend/src/modules/audit-log/routes.ts b/packages/backend/src/modules/audit-log/routes.ts new file mode 100644 index 00000000..4593f58b --- /dev/null +++ b/packages/backend/src/modules/audit-log/routes.ts @@ -0,0 +1,211 @@ +import type { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { auditLogService } from './service.js'; +import { authenticate } from '../auth/middleware.js'; +import { OrganizationsService } from '../organizations/service.js'; +import type { AuditCategory } from '../../database/types.js'; + +const organizationsService = new OrganizationsService(); + +const AUDIT_CATEGORIES = ['log_access', 'config_change', 'user_management', 'data_modification'] as const; + +const querySchema = z.object({ + organizationId: z.string().uuid(), + category: z.enum(AUDIT_CATEGORIES).optional(), + action: z.string().optional(), + resourceType: z.string().optional(), + userId: z.string().uuid().optional(), + from: z.string().datetime().optional(), + to: z.string().datetime().optional(), + limit: z.coerce.number().min(1).max(200).optional().default(50), + offset: z.coerce.number().min(0).optional().default(0), +}); + +const exportSchema = z.object({ + organizationId: z.string().uuid(), + category: z.enum(AUDIT_CATEGORIES).optional(), + action: z.string().optional(), + from: z.string().datetime().optional(), + to: z.string().datetime().optional(), +}); + +export async function auditLogRoutes(fastify: FastifyInstance) { + fastify.addHook('onRequest', authenticate); + + // GET /api/v1/audit-log + fastify.get( + '/', + { + config: { + rateLimit: { max: 60, timeWindow: '1 minute' }, + }, + }, + async (request: any, reply) => { + try { + const params = querySchema.parse(request.query); + + const isAdmin = await organizationsService.isOwnerOrAdmin( + params.organizationId, + request.user.id + ); + if (!isAdmin) { + return reply.status(403).send({ + error: 'Only organization owners and admins can view audit logs', + }); + } + + const result = await auditLogService.query({ + organizationId: params.organizationId, + category: params.category as AuditCategory | undefined, + action: params.action, + resourceType: params.resourceType, + userId: params.userId, + from: params.from ? new Date(params.from) : undefined, + to: params.to ? new Date(params.to) : undefined, + limit: params.limit, + offset: params.offset, + }); + + return reply.send(result); + } catch (error) { + if (error instanceof z.ZodError) { + return reply.status(400).send({ + error: 'Validation error', + details: error.errors, + }); + } + console.error('[AuditLog] Error querying audit logs:', error); + return reply.status(500).send({ + error: 'Failed to retrieve audit logs', + }); + } + } + ); + + // GET /api/v1/audit-log/actions - Get distinct action names for filter dropdown + fastify.get( + '/actions', + { + config: { + rateLimit: { max: 30, timeWindow: '1 minute' }, + }, + }, + async (request: any, reply) => { + try { + const { organizationId } = z + .object({ organizationId: z.string().uuid() }) + .parse(request.query); + + const isAdmin = await organizationsService.isOwnerOrAdmin( + organizationId, + request.user.id + ); + if (!isAdmin) { + return reply.status(403).send({ + error: 'Only organization owners and admins can view audit logs', + }); + } + + const actions = await auditLogService.getDistinctActions(organizationId); + return reply.send({ actions }); + } catch (error) { + if (error instanceof z.ZodError) { + return reply.status(400).send({ + error: 'Validation error', + details: error.errors, + }); + } + throw error; + } + } + ); + + // GET /api/v1/audit-log/export - Export audit log as CSV + fastify.get( + '/export', + { + config: { + rateLimit: { max: 5, timeWindow: '1 minute' }, + }, + }, + async (request: any, reply) => { + try { + const params = exportSchema.parse(request.query); + + const isAdmin = await organizationsService.isOwnerOrAdmin( + params.organizationId, + request.user.id + ); + if (!isAdmin) { + return reply.status(403).send({ + error: 'Only organization owners and admins can export audit logs', + }); + } + + const result = await auditLogService.query({ + organizationId: params.organizationId, + category: params.category as AuditCategory | undefined, + action: params.action, + from: params.from ? new Date(params.from) : undefined, + to: params.to ? new Date(params.to) : undefined, + limit: 10000, + offset: 0, + }); + + const csvHeader = 'Time,User,Category,Action,Resource Type,Resource ID,IP Address,User Agent,Details'; + const csvRows = result.entries.map((e: any) => { + const escape = (v: string | null) => { + if (v == null) return ''; + const s = String(v).replace(/"/g, '""'); + return s.includes(',') || s.includes('"') || s.includes('\n') ? `"${s}"` : s; + }; + const meta = e.metadata ? JSON.stringify(e.metadata) : ''; + return [ + escape(e.time?.toISOString?.() ?? String(e.time)), + escape(e.user_email), + escape(e.category), + escape(e.action), + escape(e.resource_type), + escape(e.resource_id), + escape(e.ip_address), + escape(e.user_agent), + escape(meta), + ].join(','); + }); + + const csv = [csvHeader, ...csvRows].join('\n'); + + auditLogService.log({ + organizationId: params.organizationId, + userId: request.user.id, + userEmail: request.user.email, + action: 'export_audit_log', + category: 'log_access', + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { + format: 'csv', + rowCount: result.entries.length, + filters: { category: params.category, action: params.action, from: params.from, to: params.to }, + }, + }); + + return reply + .header('Content-Type', 'text/csv') + .header('Content-Disposition', `attachment; filename="audit-log-${new Date().toISOString().slice(0, 10)}.csv"`) + .send(csv); + } catch (error) { + if (error instanceof z.ZodError) { + return reply.status(400).send({ + error: 'Validation error', + details: error.errors, + }); + } + console.error('[AuditLog] Error exporting audit logs:', error); + return reply.status(500).send({ + error: 'Failed to export audit logs', + }); + } + } + ); +} diff --git a/packages/backend/src/modules/audit-log/service.ts b/packages/backend/src/modules/audit-log/service.ts new file mode 100644 index 00000000..b0c40277 --- /dev/null +++ b/packages/backend/src/modules/audit-log/service.ts @@ -0,0 +1,161 @@ +import { db } from '../../database/index.js'; +import type { AuditCategory } from '../../database/types.js'; + +export interface AuditLogEntry { + organizationId: string | null; + userId?: string | null; + userEmail?: string | null; + action: string; + category: AuditCategory; + resourceType?: string | null; + resourceId?: string | null; + ipAddress?: string | null; + userAgent?: string | null; + metadata?: Record | null; +} + +export interface AuditLogQueryParams { + organizationId: string; + category?: AuditCategory; + action?: string; + resourceType?: string; + userId?: string; + from?: Date; + to?: Date; + limit?: number; + offset?: number; +} + +export interface AuditLogRow { + id: string; + time: Date; + organization_id: string | null; + user_id: string | null; + user_email: string | null; + action: string; + category: string; + resource_type: string | null; + resource_id: string | null; + ip_address: string | null; + user_agent: string | null; + metadata: Record | null; +} + +export interface AuditLogResult { + entries: AuditLogRow[]; + total: number; +} + +const BUFFER_MAX = 50; +const FLUSH_INTERVAL_MS = 1000; + +export class AuditLogService { + private buffer: AuditLogEntry[] = []; + private flushTimer: ReturnType | null = null; + private flushing = false; + + start(): void { + this.flushTimer = setInterval(() => this.flush(), FLUSH_INTERVAL_MS); + } + + log(entry: AuditLogEntry): void { + this.buffer.push(entry); + if (this.buffer.length >= BUFFER_MAX) { + void this.flush(); + } + } + + private async flush(): Promise { + if (this.flushing || this.buffer.length === 0) return; + this.flushing = true; + + const toInsert = this.buffer.splice(0, this.buffer.length); + try { + await db + .insertInto('audit_log') + .values( + toInsert.map((e) => ({ + organization_id: e.organizationId, + user_id: e.userId ?? null, + user_email: e.userEmail ?? null, + action: e.action, + category: e.category, + resource_type: e.resourceType ?? null, + resource_id: e.resourceId ?? null, + ip_address: e.ipAddress ?? null, + user_agent: e.userAgent ?? null, + metadata: e.metadata ?? null, + })) + ) + .execute(); + } catch (err) { + console.error('[AuditLog] flush error:', err); + this.buffer.unshift(...toInsert); + } finally { + this.flushing = false; + } + } + + async query(params: AuditLogQueryParams): Promise { + const limit = Math.min(params.limit ?? 50, 200); + const offset = params.offset ?? 0; + + let baseQuery = db + .selectFrom('audit_log') + .where('organization_id', '=', params.organizationId); + + if (params.category) { + baseQuery = baseQuery.where('category', '=', params.category); + } + if (params.action) { + baseQuery = baseQuery.where('action', '=', params.action); + } + if (params.resourceType) { + baseQuery = baseQuery.where('resource_type', '=', params.resourceType); + } + if (params.userId) { + baseQuery = baseQuery.where('user_id', '=', params.userId); + } + if (params.from) { + baseQuery = baseQuery.where('time', '>=', params.from); + } + if (params.to) { + baseQuery = baseQuery.where('time', '<=', params.to); + } + + const [entries, countResult] = await Promise.all([ + baseQuery + .selectAll() + .orderBy('time', 'desc') + .limit(limit) + .offset(offset) + .execute(), + baseQuery + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(), + ]); + + return { + entries: entries as AuditLogRow[], + total: Number(countResult.count), + }; + } + + async getDistinctActions(organizationId: string): Promise { + const results = await db + .selectFrom('audit_log') + .select('action') + .distinct() + .where('organization_id', '=', organizationId) + .orderBy('action') + .execute(); + return results.map((r) => r.action); + } + + async shutdown(): Promise { + if (this.flushTimer) clearInterval(this.flushTimer); + await this.flush(); + } +} + +export const auditLogService = new AuditLogService(); diff --git a/packages/backend/src/modules/organizations/routes.ts b/packages/backend/src/modules/organizations/routes.ts index 71267083..3b0a863a 100644 --- a/packages/backend/src/modules/organizations/routes.ts +++ b/packages/backend/src/modules/organizations/routes.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import { OrganizationsService } from './service.js'; import { authenticate } from '../auth/middleware.js'; import type { OrgRole } from '@logtide/shared'; +import { auditLogService } from '../audit-log/index.js'; const organizationsService = new OrganizationsService(); @@ -129,6 +130,19 @@ export async function organizationsRoutes(fastify: FastifyInstance) { await organizationsService.updateMemberRole(id, memberId, role as OrgRole, request.user.id); + auditLogService.log({ + organizationId: id, + userId: request.user.id, + userEmail: request.user.email, + action: 'update_member_role', + category: 'user_management', + resourceType: 'organization_member', + resourceId: memberId, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { role }, + }); + return reply.send({ success: true }); } catch (error) { if (error instanceof z.ZodError) { @@ -172,6 +186,18 @@ export async function organizationsRoutes(fastify: FastifyInstance) { await organizationsService.removeMember(id, memberId, request.user.id); + auditLogService.log({ + organizationId: id, + userId: request.user.id, + userEmail: request.user.email, + action: 'remove_member', + category: 'user_management', + resourceType: 'organization_member', + resourceId: memberId, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.status(204).send(); } catch (error) { if (error instanceof z.ZodError) { @@ -224,6 +250,18 @@ export async function organizationsRoutes(fastify: FastifyInstance) { await organizationsService.leaveOrganization(id, request.user.id); + auditLogService.log({ + organizationId: id, + userId: request.user.id, + userEmail: request.user.email, + action: 'leave_organization', + category: 'user_management', + resourceType: 'organization', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.status(204).send(); } catch (error) { if (error instanceof z.ZodError) { @@ -260,6 +298,19 @@ export async function organizationsRoutes(fastify: FastifyInstance) { description: body.description, }); + auditLogService.log({ + organizationId: organization.id, + userId: request.user.id, + userEmail: request.user.email, + action: 'create_organization', + category: 'config_change', + resourceType: 'organization', + resourceId: organization.id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { name: organization.name }, + }); + return reply.status(201).send({ organization }); } catch (error) { if (error instanceof z.ZodError) { @@ -289,6 +340,19 @@ export async function organizationsRoutes(fastify: FastifyInstance) { const organization = await organizationsService.updateOrganization(id, request.user.id, body); + auditLogService.log({ + organizationId: id, + userId: request.user.id, + userEmail: request.user.email, + action: 'update_organization', + category: 'config_change', + resourceType: 'organization', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: body, + }); + return reply.send({ organization }); } catch (error) { if (error instanceof z.ZodError) { @@ -333,6 +397,18 @@ export async function organizationsRoutes(fastify: FastifyInstance) { await organizationsService.deleteOrganization(id, request.user.id); + auditLogService.log({ + organizationId: id, + userId: request.user.id, + userEmail: request.user.email, + action: 'delete_organization', + category: 'data_modification', + resourceType: 'organization', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.status(204).send(); } catch (error) { if (error instanceof z.ZodError) { diff --git a/packages/backend/src/modules/projects/routes.ts b/packages/backend/src/modules/projects/routes.ts index c0249962..44a2210f 100644 --- a/packages/backend/src/modules/projects/routes.ts +++ b/packages/backend/src/modules/projects/routes.ts @@ -2,6 +2,7 @@ import type { FastifyInstance } from 'fastify'; import { z } from 'zod'; import { projectsService } from './service.js'; import { authenticate } from '../auth/middleware.js'; +import { auditLogService } from '../audit-log/index.js'; const createProjectSchema = z.object({ organizationId: z.string().uuid('Invalid organization ID'), @@ -82,6 +83,19 @@ export async function projectsRoutes(fastify: FastifyInstance) { description: body.description, }); + auditLogService.log({ + organizationId: body.organizationId, + userId: request.user.id, + userEmail: request.user.email, + action: 'create_project', + category: 'config_change', + resourceType: 'project', + resourceId: project.id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: { name: project.name }, + }); + return reply.status(201).send({ project }); } catch (error) { if (error instanceof z.ZodError) { @@ -122,6 +136,19 @@ export async function projectsRoutes(fastify: FastifyInstance) { }); } + auditLogService.log({ + organizationId: project.organizationId, + userId: request.user.id, + userEmail: request.user.email, + action: 'update_project', + category: 'config_change', + resourceType: 'project', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + metadata: body, + }); + return reply.send({ project }); } catch (error) { if (error instanceof z.ZodError) { @@ -155,6 +182,13 @@ export async function projectsRoutes(fastify: FastifyInstance) { try { const { id } = projectIdSchema.parse(request.params); + const project = await projectsService.getProjectById(id, request.user.id); + if (!project) { + return reply.status(404).send({ + error: 'Project not found', + }); + } + const deleted = await projectsService.deleteProject(id, request.user.id); if (!deleted) { @@ -163,6 +197,18 @@ export async function projectsRoutes(fastify: FastifyInstance) { }); } + auditLogService.log({ + organizationId: project.organizationId, + userId: request.user.id, + userEmail: request.user.email, + action: 'delete_project', + category: 'data_modification', + resourceType: 'project', + resourceId: id, + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.status(204).send(); } catch (error) { if (error instanceof z.ZodError) { diff --git a/packages/backend/src/modules/users/routes.ts b/packages/backend/src/modules/users/routes.ts index 1c238c0b..ac83769d 100644 --- a/packages/backend/src/modules/users/routes.ts +++ b/packages/backend/src/modules/users/routes.ts @@ -4,6 +4,7 @@ import { usersService } from './service.js'; import { config } from '../../config/index.js'; import { settingsService } from '../settings/service.js'; import { bootstrapService } from '../bootstrap/service.js'; +import { auditLogService } from '../audit-log/index.js'; const registerSchema = z.object({ email: z.string().email(), @@ -57,6 +58,16 @@ export async function usersRoutes(fastify: FastifyInstance) { password: body.password, }); + auditLogService.log({ + organizationId: null, + userId: user.id, + userEmail: user.email, + action: 'register', + category: 'user_management', + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.status(201).send({ user: { id: user.id, @@ -111,6 +122,16 @@ export async function usersRoutes(fastify: FastifyInstance) { }); } + auditLogService.log({ + organizationId: null, + userId: user.id, + userEmail: user.email, + action: 'login', + category: 'user_management', + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.send({ user: { id: user.id, @@ -154,8 +175,19 @@ export async function usersRoutes(fastify: FastifyInstance) { }); } + const user = await usersService.validateSession(token); await usersService.logout(token); + auditLogService.log({ + organizationId: null, + userId: user?.id ?? null, + userEmail: user?.email ?? null, + action: 'logout', + category: 'user_management', + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + return reply.send({ message: 'Logged out successfully', }); @@ -307,6 +339,16 @@ export async function usersRoutes(fastify: FastifyInstance) { await usersService.deleteUser(currentUser.id, body.password); + auditLogService.log({ + organizationId: null, + userId: currentUser.id, + userEmail: currentUser.email, + action: 'delete_account', + category: 'user_management', + ipAddress: request.ip, + userAgent: request.headers['user-agent'], + }); + // Logout (delete session) await usersService.logout(token); diff --git a/packages/backend/src/server.ts b/packages/backend/src/server.ts index 57f25313..3b29096a 100644 --- a/packages/backend/src/server.ts +++ b/packages/backend/src/server.ts @@ -30,6 +30,7 @@ import { settingsRoutes, publicSettingsRoutes, settingsService } from './modules import { retentionRoutes } from './modules/retention/index.js'; import { correlationRoutes, patternRoutes } from './modules/correlation/index.js'; import { piiMaskingRoutes } from './modules/pii-masking/index.js'; +import { auditLogRoutes, auditLogService } from './modules/audit-log/index.js'; import { bootstrapService } from './modules/bootstrap/index.js'; import { notificationChannelsRoutes } from './modules/notification-channels/index.js'; import internalLoggingPlugin from './plugins/internal-logging-plugin.js'; @@ -175,6 +176,7 @@ export async function build(opts = {}) { await fastify.register(dashboardRoutes); await fastify.register(adminRoutes, { prefix: '/api/v1/admin' }); await fastify.register(settingsRoutes, { prefix: '/api/v1/admin/settings' }); + await fastify.register(auditLogRoutes, { prefix: '/api/v1/audit-log' }); await fastify.register(retentionRoutes, { prefix: '/api/v1/admin' }); await fastify.register(authPlugin); @@ -197,6 +199,7 @@ async function start() { await bootstrapService.runInitialBootstrap(); await initializeInternalLogging(); + auditLogService.start(); await enrichmentService.initialize(); await notificationManager.initialize(config.DATABASE_URL); @@ -210,6 +213,7 @@ async function start() { const shutdown = async () => { console.log('[Server] Shutting down gracefully...'); + await auditLogService.shutdown(); await notificationManager.shutdown(); await shutdownInternalLogging(); await app.close(); diff --git a/packages/backend/src/tests/modules/audit-log/routes.test.ts b/packages/backend/src/tests/modules/audit-log/routes.test.ts new file mode 100644 index 00000000..22e60812 --- /dev/null +++ b/packages/backend/src/tests/modules/audit-log/routes.test.ts @@ -0,0 +1,477 @@ +import { describe, it, expect, beforeEach, afterAll, beforeAll } from 'vitest'; +import Fastify, { FastifyInstance } from 'fastify'; +import { db } from '../../../database/index.js'; +import { auditLogRoutes } from '../../../modules/audit-log/routes.js'; +import { createTestUser, createTestOrganization } from '../../helpers/factories.js'; +import crypto from 'crypto'; + +async function createTestSession(userId: string) { + const token = crypto.randomBytes(32).toString('hex'); + const expiresAt = new Date(Date.now() + 24 * 60 * 60 * 1000); + + await db + .insertInto('sessions') + .values({ + user_id: userId, + token, + expires_at: expiresAt, + }) + .execute(); + + return { token, expiresAt }; +} + +async function createAdminUser() { + const user = await createTestUser({ email: `admin-${Date.now()}@test.com`, name: 'Admin User' }); + await db + .updateTable('users') + .set({ is_admin: true }) + .where('id', '=', user.id) + .execute(); + return { ...user, is_admin: true }; +} + +async function insertAuditEntry(overrides: { + organization_id: string; + user_id?: string | null; + user_email?: string | null; + action?: string; + category?: string; + resource_type?: string | null; + resource_id?: string | null; + ip_address?: string | null; + user_agent?: string | null; + metadata?: Record | null; +}) { + return db + .insertInto('audit_log') + .values({ + organization_id: overrides.organization_id, + user_id: overrides.user_id ?? null, + user_email: overrides.user_email ?? null, + action: overrides.action ?? 'test_action', + category: (overrides.category ?? 'config_change') as any, + resource_type: overrides.resource_type ?? null, + resource_id: overrides.resource_id ?? null, + ip_address: overrides.ip_address ?? '127.0.0.1', + user_agent: overrides.user_agent ?? 'test-agent', + metadata: overrides.metadata ?? null, + }) + .returningAll() + .executeTakeFirstOrThrow(); +} + +describe('Audit Log Routes', () => { + let app: FastifyInstance; + let adminToken: string; + let userToken: string; + let adminUser: any; + let regularUser: any; + let testOrg: any; + + beforeAll(async () => { + app = Fastify(); + await app.register(auditLogRoutes, { prefix: '/api/v1/admin/audit-log' }); + await app.ready(); + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + await db.deleteFrom('audit_log').execute(); + await db.deleteFrom('sessions').execute(); + await db.deleteFrom('organization_members').execute(); + await db.deleteFrom('projects').execute(); + await db.deleteFrom('organizations').execute(); + await db.deleteFrom('users').execute(); + + adminUser = await createAdminUser(); + const adminSession = await createTestSession(adminUser.id); + adminToken = adminSession.token; + + regularUser = await createTestUser({ email: 'regular@test.com' }); + const userSession = await createTestSession(regularUser.id); + userToken = userSession.token; + + testOrg = await createTestOrganization({ ownerId: adminUser.id }); + }); + + describe('Authentication & Authorization', () => { + it('should return 401 without auth token', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}`, + }); + + expect(response.statusCode).toBe(401); + }); + + it('should return 403 for non-admin users', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}`, + headers: { + Authorization: `Bearer ${userToken}`, + }, + }); + + expect(response.statusCode).toBe(403); + }); + }); + + describe('GET /api/v1/admin/audit-log', () => { + it('should return audit log entries', async () => { + await insertAuditEntry({ + organization_id: testOrg.id, + user_id: adminUser.id, + user_email: adminUser.email, + action: 'create_project', + category: 'config_change', + resource_type: 'project', + resource_id: 'proj-123', + }); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body).toHaveProperty('entries'); + expect(body).toHaveProperty('total'); + expect(body.entries).toHaveLength(1); + expect(body.total).toBe(1); + expect(body.entries[0].action).toBe('create_project'); + }); + + it('should return empty result when no entries exist', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.entries).toHaveLength(0); + expect(body.total).toBe(0); + }); + + it('should return 400 when organizationId is missing', async () => { + const response = await app.inject({ + method: 'GET', + url: '/api/v1/admin/audit-log', + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(400); + const body = JSON.parse(response.payload); + expect(body.error).toBe('Validation error'); + }); + + it('should return 400 for invalid organizationId', async () => { + const response = await app.inject({ + method: 'GET', + url: '/api/v1/admin/audit-log?organizationId=not-a-uuid', + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(400); + }); + + it('should filter by category', async () => { + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'change_1', + category: 'config_change', + }); + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'user_1', + category: 'user_management', + }); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&category=config_change`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.entries).toHaveLength(1); + expect(body.entries[0].action).toBe('change_1'); + }); + + it('should return 400 for invalid category', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&category=invalid_category`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(400); + }); + + it('should filter by action', async () => { + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'create_project', + }); + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'delete_project', + }); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&action=create_project`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.entries).toHaveLength(1); + expect(body.entries[0].action).toBe('create_project'); + }); + + it('should filter by resourceType', async () => { + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'a1', + resource_type: 'project', + }); + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'a2', + resource_type: 'user', + }); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&resourceType=project`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.entries).toHaveLength(1); + expect(body.entries[0].resource_type).toBe('project'); + }); + + it('should filter by userId', async () => { + await insertAuditEntry({ + organization_id: testOrg.id, + user_id: adminUser.id, + action: 'admin_action', + }); + await insertAuditEntry({ + organization_id: testOrg.id, + user_id: regularUser.id, + action: 'user_action', + }); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&userId=${adminUser.id}`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.entries).toHaveLength(1); + expect(body.entries[0].action).toBe('admin_action'); + }); + + it('should filter by from/to date range', async () => { + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'recent_action', + }); + + const from = new Date(Date.now() - 60000).toISOString(); + const to = new Date(Date.now() + 60000).toISOString(); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&from=${from}&to=${to}`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.entries).toHaveLength(1); + }); + + it('should handle pagination with limit and offset', async () => { + for (let i = 0; i < 5; i++) { + await insertAuditEntry({ + organization_id: testOrg.id, + action: `action_${i}`, + }); + } + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&limit=2&offset=0`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.entries).toHaveLength(2); + expect(body.total).toBe(5); + }); + + it('should return 400 for limit below 1', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&limit=0`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(400); + }); + + it('should return 400 for limit above 200', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&limit=201`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(400); + }); + + it('should return 400 for negative offset', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log?organizationId=${testOrg.id}&offset=-1`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(400); + }); + }); + + describe('GET /api/v1/admin/audit-log/actions', () => { + it('should return distinct actions', async () => { + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'create_project', + }); + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'delete_project', + }); + await insertAuditEntry({ + organization_id: testOrg.id, + action: 'create_project', // duplicate + }); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log/actions?organizationId=${testOrg.id}`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body).toHaveProperty('actions'); + expect(body.actions).toEqual(['create_project', 'delete_project']); + }); + + it('should return empty actions array when no entries exist', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log/actions?organizationId=${testOrg.id}`, + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.actions).toEqual([]); + }); + + it('should return 401 without auth token', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log/actions?organizationId=${testOrg.id}`, + }); + + expect(response.statusCode).toBe(401); + }); + + it('should return 403 for non-admin users', async () => { + const response = await app.inject({ + method: 'GET', + url: `/api/v1/admin/audit-log/actions?organizationId=${testOrg.id}`, + headers: { + Authorization: `Bearer ${userToken}`, + }, + }); + + expect(response.statusCode).toBe(403); + }); + + it('should return 400 when organizationId is missing', async () => { + const response = await app.inject({ + method: 'GET', + url: '/api/v1/admin/audit-log/actions', + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(400); + }); + + it('should return 400 for invalid organizationId', async () => { + const response = await app.inject({ + method: 'GET', + url: '/api/v1/admin/audit-log/actions?organizationId=not-a-uuid', + headers: { + Authorization: `Bearer ${adminToken}`, + }, + }); + + expect(response.statusCode).toBe(400); + }); + }); +}); diff --git a/packages/backend/src/tests/modules/audit-log/service.test.ts b/packages/backend/src/tests/modules/audit-log/service.test.ts new file mode 100644 index 00000000..0695f3cd --- /dev/null +++ b/packages/backend/src/tests/modules/audit-log/service.test.ts @@ -0,0 +1,527 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { db } from '../../../database/index.js'; +import { AuditLogService } from '../../../modules/audit-log/service.js'; +import type { AuditLogEntry } from '../../../modules/audit-log/service.js'; +import { createTestUser, createTestOrganization } from '../../helpers/factories.js'; + +function makeEntry(overrides: Partial = {}): AuditLogEntry { + return { + organizationId: overrides.organizationId ?? null, + userId: overrides.userId ?? null, + userEmail: overrides.userEmail ?? null, + action: overrides.action ?? 'test_action', + category: overrides.category ?? 'config_change', + resourceType: overrides.resourceType ?? null, + resourceId: overrides.resourceId ?? null, + ipAddress: overrides.ipAddress ?? null, + userAgent: overrides.userAgent ?? null, + metadata: overrides.metadata ?? null, + }; +} + +describe('AuditLogService', () => { + let service: AuditLogService; + let orgId: string; + let userId: string; + let userEmail: string; + + beforeEach(async () => { + await db.deleteFrom('audit_log').execute(); + service = new AuditLogService(); + + const user = await createTestUser(); + const org = await createTestOrganization({ ownerId: user.id }); + orgId = org.id; + userId = user.id; + userEmail = user.email; + }); + + afterEach(async () => { + await service.shutdown(); + }); + + // Helper to insert entries directly into DB for query tests + async function insertEntry(overrides: Partial<{ + organization_id: string | null; + user_id: string | null; + user_email: string | null; + action: string; + category: string; + resource_type: string | null; + resource_id: string | null; + ip_address: string | null; + user_agent: string | null; + metadata: Record | null; + time: Date; + }> = {}) { + return db + .insertInto('audit_log') + .values({ + organization_id: overrides.organization_id ?? orgId, + user_id: overrides.user_id ?? userId, + user_email: overrides.user_email ?? userEmail, + action: overrides.action ?? 'test_action', + category: (overrides.category ?? 'config_change') as any, + resource_type: overrides.resource_type ?? null, + resource_id: overrides.resource_id ?? null, + ip_address: overrides.ip_address ?? '127.0.0.1', + user_agent: overrides.user_agent ?? 'test-agent', + metadata: overrides.metadata ?? null, + }) + .returningAll() + .executeTakeFirstOrThrow(); + } + + describe('log() and flush()', () => { + it('should buffer entries and flush them to DB on shutdown', async () => { + service.log(makeEntry({ + organizationId: orgId, + userId, + userEmail, + action: 'create_project', + category: 'config_change', + })); + + // Before shutdown, nothing in DB + const before = await db + .selectFrom('audit_log') + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + expect(Number(before.count)).toBe(0); + + // Shutdown flushes the buffer + await service.shutdown(); + + const after = await db + .selectFrom('audit_log') + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + expect(Number(after.count)).toBe(1); + }); + + it('should flush multiple entries at once', async () => { + for (let i = 0; i < 5; i++) { + service.log(makeEntry({ + organizationId: orgId, + action: `action_${i}`, + category: 'config_change', + })); + } + + await service.shutdown(); + + const result = await db + .selectFrom('audit_log') + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + expect(Number(result.count)).toBe(5); + }); + + it('should map camelCase fields to snake_case columns', async () => { + service.log(makeEntry({ + organizationId: orgId, + userId, + userEmail, + action: 'login', + category: 'user_management', + resourceType: 'session', + resourceId: 'sess-123', + ipAddress: '192.168.1.1', + userAgent: 'Mozilla/5.0', + metadata: { browser: 'Chrome' }, + })); + + await service.shutdown(); + + const row = await db + .selectFrom('audit_log') + .selectAll() + .executeTakeFirstOrThrow(); + + expect(row.organization_id).toBe(orgId); + expect(row.user_id).toBe(userId); + expect(row.user_email).toBe(userEmail); + expect(row.action).toBe('login'); + expect(row.category).toBe('user_management'); + expect(row.resource_type).toBe('session'); + expect(row.resource_id).toBe('sess-123'); + expect(row.ip_address).toBe('192.168.1.1'); + expect(row.user_agent).toBe('Mozilla/5.0'); + expect(row.metadata).toEqual({ browser: 'Chrome' }); + }); + + it('should handle null/undefined optional fields', async () => { + service.log(makeEntry({ + organizationId: orgId, + action: 'test', + category: 'log_access', + })); + + await service.shutdown(); + + const row = await db + .selectFrom('audit_log') + .selectAll() + .executeTakeFirstOrThrow(); + + expect(row.user_id).toBeNull(); + expect(row.user_email).toBeNull(); + expect(row.resource_type).toBeNull(); + expect(row.resource_id).toBeNull(); + expect(row.ip_address).toBeNull(); + expect(row.user_agent).toBeNull(); + expect(row.metadata).toBeNull(); + }); + + it('should auto-flush when buffer reaches BUFFER_MAX (50)', async () => { + for (let i = 0; i < 50; i++) { + service.log(makeEntry({ + organizationId: orgId, + action: `bulk_action_${i}`, + category: 'config_change', + })); + } + + // Give the async flush a moment to complete + await new Promise((r) => setTimeout(r, 200)); + + const result = await db + .selectFrom('audit_log') + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + expect(Number(result.count)).toBe(50); + }); + + it('should not flush when buffer is empty', async () => { + // shutdown on empty buffer should not error + await service.shutdown(); + + const result = await db + .selectFrom('audit_log') + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + expect(Number(result.count)).toBe(0); + }); + + it('should re-queue entries on flush error', async () => { + const insertSpy = vi.spyOn(db, 'insertInto'); + + // First call throws, second call succeeds + insertSpy.mockImplementationOnce(() => { + throw new Error('DB connection error'); + }); + + service.log(makeEntry({ + organizationId: orgId, + action: 'will_fail', + category: 'config_change', + })); + + // Trigger flush via shutdown - first attempt fails, entries re-queued + await service.shutdown(); + + insertSpy.mockRestore(); + + // Create a new service to flush the re-queued entries + // Since the entries were re-queued in the same service instance's buffer, + // we need to flush again + // Actually, shutdown calls flush which failed and re-queued, then + // the entries are still in the buffer. Let's create a new service + // and verify the original service's buffer state. + // The entries should be back in the buffer after the error. + // Let's try flushing again by calling shutdown on a fresh service + // that we populated manually. + + // Since the flush failed and re-queued, we can't easily test + // the buffer state from outside. Let's verify by checking DB is empty. + const result = await db + .selectFrom('audit_log') + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + // The entries were re-queued but shutdown only calls flush once, + // so they're still in the buffer (DB should be empty) + expect(Number(result.count)).toBe(0); + }); + }); + + describe('start() and shutdown()', () => { + it('should start the flush timer and stop it on shutdown', async () => { + service.start(); + + service.log(makeEntry({ + organizationId: orgId, + action: 'timed_flush', + category: 'config_change', + })); + + // Wait for the flush interval (1000ms + buffer) + await new Promise((r) => setTimeout(r, 1500)); + + const result = await db + .selectFrom('audit_log') + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + expect(Number(result.count)).toBe(1); + + await service.shutdown(); + }); + + it('should flush remaining entries on shutdown', async () => { + service.start(); + + service.log(makeEntry({ + organizationId: orgId, + action: 'shutdown_flush', + category: 'data_modification', + })); + + // Immediately shutdown (don't wait for timer) + await service.shutdown(); + + const result = await db + .selectFrom('audit_log') + .select(db.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + expect(Number(result.count)).toBe(1); + }); + }); + + describe('query()', () => { + it('should return entries for an organization', async () => { + await insertEntry({ action: 'action_1' }); + await insertEntry({ action: 'action_2' }); + + const result = await service.query({ organizationId: orgId }); + + expect(result.entries).toHaveLength(2); + expect(result.total).toBe(2); + }); + + it('should not return entries from other organizations', async () => { + const otherUser = await createTestUser({ email: `other-${Date.now()}@test.com` }); + const otherOrg = await createTestOrganization({ ownerId: otherUser.id }); + + await insertEntry({ action: 'my_action' }); + await insertEntry({ organization_id: otherOrg.id, action: 'other_action' }); + + const result = await service.query({ organizationId: orgId }); + expect(result.entries).toHaveLength(1); + expect(result.entries[0].action).toBe('my_action'); + }); + + it('should filter by category', async () => { + await insertEntry({ category: 'config_change', action: 'change_1' }); + await insertEntry({ category: 'user_management', action: 'user_1' }); + await insertEntry({ category: 'log_access', action: 'access_1' }); + + const result = await service.query({ + organizationId: orgId, + category: 'config_change', + }); + + expect(result.entries).toHaveLength(1); + expect(result.entries[0].action).toBe('change_1'); + }); + + it('should filter by action', async () => { + await insertEntry({ action: 'create_project' }); + await insertEntry({ action: 'delete_project' }); + await insertEntry({ action: 'create_project' }); + + const result = await service.query({ + organizationId: orgId, + action: 'create_project', + }); + + expect(result.entries).toHaveLength(2); + }); + + it('should filter by resourceType', async () => { + await insertEntry({ resource_type: 'project', action: 'a1' }); + await insertEntry({ resource_type: 'user', action: 'a2' }); + await insertEntry({ resource_type: 'project', action: 'a3' }); + + const result = await service.query({ + organizationId: orgId, + resourceType: 'project', + }); + + expect(result.entries).toHaveLength(2); + }); + + it('should filter by userId', async () => { + const otherUser = await createTestUser({ email: `filter-user-${Date.now()}@test.com` }); + + await insertEntry({ user_id: userId, action: 'a1' }); + await insertEntry({ user_id: otherUser.id, action: 'a2' }); + + const result = await service.query({ + organizationId: orgId, + userId, + }); + + expect(result.entries).toHaveLength(1); + expect(result.entries[0].action).toBe('a1'); + }); + + it('should filter by from date', async () => { + const oldDate = new Date('2024-01-01'); + const recentDate = new Date(); + + // Insert using raw SQL for time control + await db.insertInto('audit_log').values({ + organization_id: orgId, + action: 'old_action', + category: 'config_change' as any, + user_id: null, + user_email: null, + resource_type: null, + resource_id: null, + ip_address: null, + user_agent: null, + metadata: null, + }).execute(); + + const result = await service.query({ + organizationId: orgId, + from: new Date(Date.now() - 60000), // last minute + }); + + expect(result.entries.length).toBeGreaterThanOrEqual(1); + }); + + it('should filter by to date', async () => { + await insertEntry({ action: 'recent_action' }); + + const result = await service.query({ + organizationId: orgId, + to: new Date(Date.now() + 60000), // future + }); + + expect(result.entries).toHaveLength(1); + + const resultPast = await service.query({ + organizationId: orgId, + to: new Date('2020-01-01'), + }); + + expect(resultPast.entries).toHaveLength(0); + }); + + it('should apply default limit of 50', async () => { + const result = await service.query({ organizationId: orgId }); + // Just verify it doesn't error - with 0 entries it returns empty + expect(result.entries).toHaveLength(0); + expect(result.total).toBe(0); + }); + + it('should cap limit at 200', async () => { + // Insert 3 entries + await insertEntry({ action: 'a1' }); + await insertEntry({ action: 'a2' }); + await insertEntry({ action: 'a3' }); + + const result = await service.query({ + organizationId: orgId, + limit: 999, // above 200 cap + }); + + // Should still return all 3 (cap is 200 but we only have 3) + expect(result.entries).toHaveLength(3); + }); + + it('should handle offset for pagination', async () => { + for (let i = 0; i < 5; i++) { + await insertEntry({ action: `action_${i}` }); + } + + const page1 = await service.query({ + organizationId: orgId, + limit: 2, + offset: 0, + }); + + const page2 = await service.query({ + organizationId: orgId, + limit: 2, + offset: 2, + }); + + expect(page1.entries).toHaveLength(2); + expect(page2.entries).toHaveLength(2); + expect(page1.total).toBe(5); + expect(page2.total).toBe(5); + + // Entries should be different + expect(page1.entries[0].action).not.toBe(page2.entries[0].action); + }); + + it('should order entries by time descending', async () => { + await insertEntry({ action: 'first' }); + // Small delay to ensure different timestamps + await new Promise((r) => setTimeout(r, 10)); + await insertEntry({ action: 'second' }); + + const result = await service.query({ organizationId: orgId }); + + expect(result.entries[0].action).toBe('second'); + expect(result.entries[1].action).toBe('first'); + }); + + it('should combine multiple filters', async () => { + await insertEntry({ + category: 'config_change', + action: 'create_project', + resource_type: 'project', + }); + await insertEntry({ + category: 'config_change', + action: 'create_project', + resource_type: 'api_key', + }); + await insertEntry({ + category: 'user_management', + action: 'create_project', + resource_type: 'project', + }); + + const result = await service.query({ + organizationId: orgId, + category: 'config_change', + resourceType: 'project', + }); + + expect(result.entries).toHaveLength(1); + expect(result.entries[0].action).toBe('create_project'); + }); + }); + + describe('getDistinctActions()', () => { + it('should return sorted distinct actions', async () => { + await insertEntry({ action: 'delete_project' }); + await insertEntry({ action: 'create_project' }); + await insertEntry({ action: 'create_project' }); // duplicate + await insertEntry({ action: 'login' }); + + const actions = await service.getDistinctActions(orgId); + + expect(actions).toEqual(['create_project', 'delete_project', 'login']); + }); + + it('should return empty array for org with no entries', async () => { + const actions = await service.getDistinctActions(orgId); + expect(actions).toEqual([]); + }); + + it('should not return actions from other organizations', async () => { + const otherUser = await createTestUser({ email: `distinct-${Date.now()}@test.com` }); + const otherOrg = await createTestOrganization({ ownerId: otherUser.id }); + + await insertEntry({ action: 'my_action' }); + await insertEntry({ organization_id: otherOrg.id, action: 'other_action' }); + + const actions = await service.getDistinctActions(orgId); + expect(actions).toEqual(['my_action']); + }); + }); +}); diff --git a/packages/backend/src/tests/setup.ts b/packages/backend/src/tests/setup.ts index e78ae793..ff6ee154 100644 --- a/packages/backend/src/tests/setup.ts +++ b/packages/backend/src/tests/setup.ts @@ -55,6 +55,7 @@ beforeEach(async () => { await db.deleteFrom('organization_members').execute(); await db.deleteFrom('projects').execute(); await db.deleteFrom('organizations').execute(); + await db.deleteFrom('audit_log').execute(); await db.deleteFrom('sessions').execute(); await db.deleteFrom('users').execute(); }); diff --git a/packages/frontend/src/lib/api/audit-log.ts b/packages/frontend/src/lib/api/audit-log.ts new file mode 100644 index 00000000..26400a53 --- /dev/null +++ b/packages/frontend/src/lib/api/audit-log.ts @@ -0,0 +1,102 @@ +import { getApiBaseUrl } from '$lib/config'; +import { getAuthToken } from '$lib/utils/auth'; + +export type AuditCategory = 'log_access' | 'config_change' | 'user_management' | 'data_modification'; + +export interface AuditLogEntry { + id: string; + time: string; + organization_id: string | null; + user_id: string | null; + user_email: string | null; + action: string; + category: AuditCategory; + resource_type: string | null; + resource_id: string | null; + ip_address: string | null; + user_agent: string | null; + metadata: Record | null; +} + +export interface AuditLogFilters { + organizationId: string; + category?: AuditCategory; + action?: string; + resourceType?: string; + userId?: string; + from?: string; + to?: string; + limit?: number; + offset?: number; +} + +export interface AuditLogResponse { + entries: AuditLogEntry[]; + total: number; +} + +async function request(endpoint: string): Promise { + const token = getAuthToken(); + const response = await fetch(`${getApiBaseUrl()}${endpoint}`, { + headers: { + 'Content-Type': 'application/json', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + }); + + if (!response.ok) { + const err = await response.json().catch(() => ({})); + throw new Error((err as any).error ?? `HTTP ${response.status}`); + } + + return response; +} + +export async function getAuditLog(filters: AuditLogFilters): Promise { + const params = new URLSearchParams({ organizationId: filters.organizationId }); + + if (filters.category) params.set('category', filters.category); + if (filters.action) params.set('action', filters.action); + if (filters.resourceType) params.set('resourceType', filters.resourceType); + if (filters.userId) params.set('userId', filters.userId); + if (filters.from) params.set('from', filters.from); + if (filters.to) params.set('to', filters.to); + if (filters.limit != null) params.set('limit', String(filters.limit)); + if (filters.offset != null) params.set('offset', String(filters.offset)); + + const response = await request(`/audit-log?${params}`); + return response.json(); +} + +export async function getAuditLogActions(organizationId: string): Promise { + const response = await request(`/audit-log/actions?organizationId=${organizationId}`); + const data = await response.json(); + return data.actions; +} + +export interface AuditLogExportFilters { + organizationId: string; + category?: AuditCategory; + action?: string; + from?: string; + to?: string; +} + +export async function exportAuditLogCsv(filters: AuditLogExportFilters): Promise { + const params = new URLSearchParams({ organizationId: filters.organizationId }); + if (filters.category) params.set('category', filters.category); + if (filters.action) params.set('action', filters.action); + if (filters.from) params.set('from', filters.from); + if (filters.to) params.set('to', filters.to); + + const response = await request(`/audit-log/export?${params}`); + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `audit-log-${new Date().toISOString().slice(0, 10)}.csv`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); +} diff --git a/packages/frontend/src/routes/dashboard/settings/+page.svelte b/packages/frontend/src/routes/dashboard/settings/+page.svelte index 4d8f6574..85faee92 100644 --- a/packages/frontend/src/routes/dashboard/settings/+page.svelte +++ b/packages/frontend/src/routes/dashboard/settings/+page.svelte @@ -33,6 +33,7 @@ import ChevronRight from '@lucide/svelte/icons/chevron-right'; import BellRing from '@lucide/svelte/icons/bell-ring'; import ShieldAlert from '@lucide/svelte/icons/shield-alert'; + import ClipboardList from '@lucide/svelte/icons/clipboard-list'; import { Table, TableBody, @@ -512,6 +513,29 @@ + {#if canManage} + goto('/dashboard/settings/audit-log')}> + +
+
+ +
+ Audit Log + Track all actions performed in your organization +
+
+ +
+ + +

+ View a complete audit trail of user actions including configuration changes, + member management, and data modifications for compliance and security. +

+ + + {/if} +
diff --git a/packages/frontend/src/routes/dashboard/settings/audit-log/+page.svelte b/packages/frontend/src/routes/dashboard/settings/audit-log/+page.svelte new file mode 100644 index 00000000..b9651779 --- /dev/null +++ b/packages/frontend/src/routes/dashboard/settings/audit-log/+page.svelte @@ -0,0 +1,537 @@ + + + + Audit Log - LogTide + + +
+
+ +
+
+

Audit Log

+
+ +

+ Track all actions performed in {currentOrg?.name || 'your organization'} +

+
+
+
+ + +
+
+
+ + {#if !canManage} + + + +

+ Only organization owners and admins can view the audit log. +

+ + + {:else} + + + + Filters + + +
+
+ + +
+ +
+ + +
+ + {#if categoryFilter || actionFilter} + + {/if} +
+ + + + + + + + {total} event{total !== 1 ? 's' : ''} + + + + {#if loading} +
+ +
+ {:else if error} +

{error}

+ {:else if entries.length === 0} +
+ +

No audit events found.

+
+ {:else} +
+ + + + + Time + User + Category + Action + Resource + IP Address + + + + {#each entries as entry (entry.id)} + {@const catInfo = getCategoryInfo(entry.category)} + {@const isExpanded = expandedId === entry.id} + + toggleExpand(entry.id)} + > + + + + + {formatDate(entry.time)} + + + {entry.user_email ?? '\u2014'} + + + + {catInfo.label} + + + + {formatAction(entry.action)} + + + {#if entry.resource_type} + {entry.resource_type} + {#if entry.resource_id} + {entry.resource_id.length > 12 + ? entry.resource_id.slice(0, 8) + '\u2026' + : entry.resource_id} + {/if} + {:else} + \u2014 + {/if} + + + {entry.ip_address ?? '\u2014'} + + + {#if isExpanded} + + +
+

{describeEntry(entry)}

+
+
+ Time +

{new Date(entry.time).toLocaleString(undefined, { dateStyle: 'full', timeStyle: 'long' })}

+
+
+ User +

{entry.user_email ?? '\u2014'}

+ {#if entry.user_id} +

{entry.user_id}

+ {/if} +
+ {#if entry.resource_type} +
+ Resource Type +

{entry.resource_type}

+
+ {/if} + {#if entry.resource_id} +
+ Resource ID +

{entry.resource_id}

+
+ {/if} +
+ IP Address +

{entry.ip_address ?? '\u2014'}

+
+ {#if entry.user_agent} +
+ User Agent +

{entry.user_agent}

+
+ {/if} + {#if entry.metadata && Object.keys(entry.metadata).length > 0} +
+ Details +
+
+ {#each Object.entries(entry.metadata) as [key, value]} +
+
{key}:
+
+ {typeof value === 'object' ? JSON.stringify(value) : String(value)} +
+
+ {/each} +
+
+
+ {/if} +
+
+ + + {/if} + {/each} + +
+
+ {/if} + + + + + {#if totalPages > 1} +
+

+ Page {currentPage} of {totalPages} ({total} total) +

+
+ + +
+
+ {/if} + {/if} +
From b5ac10ea00714476dae29c48d632e80547f46568 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 15:18:37 +0100 Subject: [PATCH 02/43] add service dependency graph page (#40) - new /dashboard/service-map page with force-directed graph - enriched endpoint GET /api/v1/traces/service-map combining span deps + log co-occurrence + health stats - health color-coding on nodes (green/amber/red by error rate) - side panel on click with latency, error rate, p95, upstream/downstream edges - dashed edges for log correlation, solid for span-based deps - png export, time range filtering, project picker --- packages/backend/src/modules/traces/routes.ts | 47 ++ .../backend/src/modules/traces/service.ts | 222 +++++++++ .../src/tests/modules/traces/routes.test.ts | 268 +++++++++++ .../src/tests/modules/traces/service.test.ts | 423 +++++++++++++++- packages/frontend/src/lib/api/traces.ts | 43 ++ .../src/lib/components/AppLayout.svelte | 6 + .../src/lib/components/ServiceMap.svelte | 123 +++-- .../routes/dashboard/service-map/+page.svelte | 452 ++++++++++++++++++ 8 files changed, 1549 insertions(+), 35 deletions(-) create mode 100644 packages/frontend/src/routes/dashboard/service-map/+page.svelte diff --git a/packages/backend/src/modules/traces/routes.ts b/packages/backend/src/modules/traces/routes.ts index d8699885..8f3e9241 100644 --- a/packages/backend/src/modules/traces/routes.ts +++ b/packages/backend/src/modules/traces/routes.ts @@ -255,6 +255,53 @@ const tracesRoutes: FastifyPluginAsync = async (fastify) => { }, }); + fastify.get('/api/v1/traces/service-map', { + schema: { + querystring: { + type: 'object', + properties: { + projectId: { type: 'string' }, + from: { type: 'string', format: 'date-time' }, + to: { type: 'string', format: 'date-time' }, + }, + }, + }, + handler: async (request: any, reply) => { + if (!await requireFullAccess(request, reply)) return; + + const { projectId: queryProjectId, from, to } = request.query as { + projectId?: string; + from?: string; + to?: string; + }; + + const projectId = queryProjectId || request.projectId; + + if (!projectId) { + return reply.code(400).send({ + error: 'Project context missing - provide projectId query parameter', + }); + } + + if (request.user?.id) { + const hasAccess = await verifyProjectAccess(projectId, request.user.id); + if (!hasAccess) { + return reply.code(403).send({ + error: 'Access denied - you do not have access to this project', + }); + } + } + + const data = await tracesService.getEnrichedServiceDependencies( + projectId, + from ? new Date(from) : undefined, + to ? new Date(to) : undefined + ); + + return data; + }, + }); + fastify.get('/api/v1/traces/stats', { schema: { querystring: { diff --git a/packages/backend/src/modules/traces/service.ts b/packages/backend/src/modules/traces/service.ts index c3394c85..f09fa9d2 100644 --- a/packages/backend/src/modules/traces/service.ts +++ b/packages/backend/src/modules/traces/service.ts @@ -1,4 +1,5 @@ import { db } from '../../database/index.js'; +import { pool } from '../../database/connection.js'; import { reservoir } from '../../database/reservoir.js'; import type { TransformedSpan, AggregatedTrace } from '../otlp/trace-transformer.js'; import type { @@ -51,6 +52,44 @@ export interface SpanRecord { resource_attributes: Record | null; } +// Service map enriched types +export interface ServiceHealthStats { + service_name: string; + total_calls: number; + total_errors: number; + error_rate: number; + avg_latency_ms: number; + p95_latency_ms: number | null; +} + +export interface EnrichedServiceDependencyNode { + id: string; + name: string; + callCount: number; + errorRate: number; + avgLatencyMs: number; + p95LatencyMs: number | null; + totalCalls: number; +} + +export interface EnrichedServiceDependencyEdge { + source: string; + target: string; + callCount: number; + type: 'span' | 'log_correlation'; +} + +export interface EnrichedServiceDependencies { + nodes: EnrichedServiceDependencyNode[]; + edges: EnrichedServiceDependencyEdge[]; +} + +interface LogCoOccurrenceRow { + source_service: string; + target_service: string; + co_occurrence_count: number; +} + export class TracesService { async ingestSpans( spans: TransformedSpan[], @@ -164,6 +203,189 @@ export class TracesService { return reservoir.getServiceDependencies(projectId, from, to); } + async getEnrichedServiceDependencies( + projectId: string, + from?: Date, + to?: Date, + ): Promise { + const effectiveFrom = from || new Date(Date.now() - 24 * 60 * 60 * 1000); + const effectiveTo = to || new Date(); + const rangeHours = (effectiveTo.getTime() - effectiveFrom.getTime()) / (1000 * 60 * 60); + + // Only include log co-occurrence for ranges <= 7 days (performance guard) + const includeLogCorrelation = rangeHours <= 168; + + const results = await Promise.allSettled([ + reservoir.getServiceDependencies(projectId, effectiveFrom, effectiveTo), + this.getServiceHealthStats(projectId, effectiveFrom, effectiveTo, rangeHours), + includeLogCorrelation + ? this.getLogCoOccurrenceEdges(projectId, effectiveFrom, effectiveTo) + : Promise.resolve([]), + ]); + + const spanDeps = results[0].status === 'fulfilled' ? results[0].value : { nodes: [], edges: [] }; + const healthStats = results[1].status === 'fulfilled' ? results[1].value : []; + const logCoOccurrence = results[2].status === 'fulfilled' ? results[2].value : []; + + // Build health map for quick lookup + const healthMap = new Map( + healthStats.map((s) => [s.service_name, s]), + ); + + // Merge nodes: start from span-based, add log-only services + const nodeMap = new Map(); + + for (const node of spanDeps.nodes) { + const health = healthMap.get(node.name); + nodeMap.set(node.name, { + id: node.name, + name: node.name, + callCount: node.callCount, + errorRate: health?.error_rate ?? 0, + avgLatencyMs: health?.avg_latency_ms ?? 0, + p95LatencyMs: health?.p95_latency_ms ?? null, + totalCalls: health?.total_calls ?? node.callCount, + }); + } + + // Add services that appear only in log co-occurrence (no spans) + for (const edge of logCoOccurrence) { + for (const svcName of [edge.source_service, edge.target_service]) { + if (!nodeMap.has(svcName)) { + const health = healthMap.get(svcName); + nodeMap.set(svcName, { + id: svcName, + name: svcName, + callCount: 0, + errorRate: health?.error_rate ?? 0, + avgLatencyMs: health?.avg_latency_ms ?? 0, + p95LatencyMs: health?.p95_latency_ms ?? null, + totalCalls: health?.total_calls ?? 0, + }); + } + } + } + + // Merge edges: span edges take priority, log edges fill gaps + const edgeKey = (s: string, t: string) => `${s}-->${t}`; + const edgeMap = new Map(); + + for (const edge of spanDeps.edges) { + edgeMap.set(edgeKey(edge.source, edge.target), { + source: edge.source, + target: edge.target, + callCount: edge.callCount, + type: 'span', + }); + } + + for (const edge of logCoOccurrence) { + const fwdKey = edgeKey(edge.source_service, edge.target_service); + const revKey = edgeKey(edge.target_service, edge.source_service); + if (!edgeMap.has(fwdKey) && !edgeMap.has(revKey)) { + edgeMap.set(fwdKey, { + source: edge.source_service, + target: edge.target_service, + callCount: edge.co_occurrence_count, + type: 'log_correlation', + }); + } + } + + return { + nodes: Array.from(nodeMap.values()), + edges: Array.from(edgeMap.values()), + }; + } + + private async getServiceHealthStats( + projectId: string, + from: Date, + to: Date, + rangeHours: number, + ): Promise { + if (reservoir.getEngineType() !== 'timescale') { + return []; + } + + const { sql } = await import('kysely'); + const table = rangeHours <= 48 ? 'spans_hourly_stats' as const : 'spans_daily_stats' as const; + + const result = await db + .selectFrom(table) + .select([ + 'service_name', + ]) + .select([ + db.fn.sum('span_count').as('total_calls'), + db.fn.sum('error_count').as('total_errors'), + // Weighted average: SUM(avg * count) / SUM(count) + sql`CASE WHEN SUM(span_count) > 0 + THEN SUM(COALESCE(duration_avg_ms, 0) * span_count) / SUM(span_count) + ELSE 0 END`.as('avg_latency_ms'), + db.fn.max('duration_p95_ms').as('p95_latency_ms'), + ]) + .where('project_id', '=', projectId) + .where('bucket', '>=', from) + .where('bucket', '<=', to) + .groupBy('service_name') + .execute(); + + return result.map((r) => ({ + service_name: r.service_name, + total_calls: Number(r.total_calls ?? 0), + total_errors: Number(r.total_errors ?? 0), + error_rate: Number(r.total_calls) > 0 + ? Number(r.total_errors) / Number(r.total_calls) + : 0, + avg_latency_ms: Number(r.avg_latency_ms ?? 0), + p95_latency_ms: r.p95_latency_ms != null ? Number(r.p95_latency_ms) : null, + })); + } + + private async getLogCoOccurrenceEdges( + projectId: string, + from: Date, + to: Date, + ): Promise { + if (reservoir.getEngineType() !== 'timescale') { + return []; + } + + const result = await pool.query<{ + source_service: string; + target_service: string; + co_occurrence_count: string; + }>( + `SELECT + a.service AS source_service, + b.service AS target_service, + COUNT(*)::int AS co_occurrence_count + FROM logs a + JOIN logs b + ON a.trace_id = b.trace_id + AND a.project_id = b.project_id + AND a.service < b.service + WHERE a.project_id = $1 + AND a.trace_id IS NOT NULL + AND a.time >= $2 + AND a.time <= $3 + AND b.time >= $2 + AND b.time <= $3 + GROUP BY a.service, b.service + HAVING COUNT(*) >= 2 + ORDER BY co_occurrence_count DESC + LIMIT 500`, + [projectId, from, to], + ); + + return result.rows.map((r) => ({ + source_service: r.source_service, + target_service: r.target_service, + co_occurrence_count: Number(r.co_occurrence_count), + })); + } + async getStats(projectId: string, from?: Date, to?: Date) { // Stats require aggregation (count, sum, avg, max) - use Kysely for timescale if (reservoir.getEngineType() === 'timescale') { diff --git a/packages/backend/src/tests/modules/traces/routes.test.ts b/packages/backend/src/tests/modules/traces/routes.test.ts index ddb433f3..155c5951 100644 --- a/packages/backend/src/tests/modules/traces/routes.test.ts +++ b/packages/backend/src/tests/modules/traces/routes.test.ts @@ -2,8 +2,19 @@ import { describe, it, expect, beforeEach, afterAll } from 'vitest'; import request from 'supertest'; import { build } from '../../../server.js'; import { createTestContext, createTestTrace, createTestSpan, createTestApiKey } from '../../helpers/index.js'; +import { db } from '../../../database/index.js'; import crypto from 'crypto'; +async function createSession(userId: string) { + const token = crypto.randomBytes(32).toString('hex'); + const expiresAt = new Date(Date.now() + 24 * 60 * 60 * 1000); + await db + .insertInto('sessions') + .values({ user_id: userId, token, expires_at: expiresAt }) + .execute(); + return { token }; +} + describe('Traces Routes', () => { let app: any; let context: Awaited>; @@ -445,4 +456,261 @@ describe('Traces Routes', () => { expect(response.body.total_traces).toBe(0); }); }); + + // ========================================================================== + // GET /api/v1/traces/service-map + // ========================================================================== + describe('GET /api/v1/traces/service-map', () => { + it('should return 401 without auth', async () => { + const response = await request(app.server) + .get('/api/v1/traces/service-map') + .expect(401); + + expect(response.body.error).toBe('Unauthorized'); + }); + + it('should return empty graph when no data', async () => { + const response = await request(app.server) + .get('/api/v1/traces/service-map') + .set('x-api-key', apiKey) + .expect(200); + + expect(response.body.nodes).toEqual([]); + expect(response.body.edges).toEqual([]); + }); + + it('should return enriched service map with span dependencies', async () => { + const traceId = crypto.randomBytes(16).toString('hex'); + const now = new Date(); + + const parentSpan = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId, + spanId: 'route-parent', + serviceName: 'web-app', + startTime: now, + }); + + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId, + parentSpanId: parentSpan.span_id, + serviceName: 'api-server', + startTime: new Date(now.getTime() + 10), + }); + + const response = await request(app.server) + .get('/api/v1/traces/service-map') + .query({ + from: new Date(now.getTime() - 5000).toISOString(), + to: new Date(now.getTime() + 5000).toISOString(), + }) + .set('x-api-key', apiKey) + .expect(200); + + expect(response.body.nodes).toHaveLength(2); + expect(response.body.edges).toHaveLength(1); + + // Verify enriched node structure + const webNode = response.body.nodes.find((n: any) => n.name === 'web-app'); + expect(webNode).toBeDefined(); + expect(webNode.id).toBe('web-app'); + expect(typeof webNode.callCount).toBe('number'); + expect(typeof webNode.errorRate).toBe('number'); + expect(typeof webNode.avgLatencyMs).toBe('number'); + expect(typeof webNode.totalCalls).toBe('number'); + + // Verify enriched edge structure + expect(response.body.edges[0]).toMatchObject({ + source: 'web-app', + target: 'api-server', + callCount: 1, + type: 'span', + }); + }); + + it('should filter by time range', async () => { + const now = new Date(); + const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000); + + // Old data + const oldTraceId = crypto.randomBytes(16).toString('hex'); + const oldParent = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: oldTraceId, + spanId: 'route-old-p', + serviceName: 'old-a', + startTime: yesterday, + }); + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: oldTraceId, + parentSpanId: oldParent.span_id, + serviceName: 'old-b', + startTime: yesterday, + }); + + // Query recent window only + const response = await request(app.server) + .get('/api/v1/traces/service-map') + .query({ + from: new Date(now.getTime() - 1000).toISOString(), + to: new Date(now.getTime() + 1000).toISOString(), + }) + .set('x-api-key', apiKey) + .expect(200); + + expect(response.body.nodes).toEqual([]); + expect(response.body.edges).toEqual([]); + }); + + it('should return service map without time range (defaults)', async () => { + const response = await request(app.server) + .get('/api/v1/traces/service-map') + .set('x-api-key', apiKey) + .expect(200); + + expect(response.body).toHaveProperty('nodes'); + expect(response.body).toHaveProperty('edges'); + expect(Array.isArray(response.body.nodes)).toBe(true); + expect(Array.isArray(response.body.edges)).toBe(true); + }); + + it('should work with session auth', async () => { + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces/service-map') + .query({ projectId: context.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(200); + + expect(response.body.nodes).toEqual([]); + expect(response.body.edges).toEqual([]); + }); + + it('should return 403 for unauthorized project via session', async () => { + const otherContext = await createTestContext(); + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces/service-map') + .query({ projectId: otherContext.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(403); + + expect(response.body.error).toContain('Access denied'); + }); + }); + + // ========================================================================== + // Session auth coverage for existing routes + // ========================================================================== + describe('Session auth - project access control', () => { + it('GET /api/v1/traces should work with session auth', async () => { + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces') + .query({ projectId: context.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(200); + + expect(response.body.traces).toEqual([]); + }); + + it('GET /api/v1/traces should return 403 for unauthorized project', async () => { + const otherContext = await createTestContext(); + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces') + .query({ projectId: otherContext.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(403); + + expect(response.body.error).toContain('Access denied'); + }); + + it('GET /api/v1/traces/:traceId should return 403 for unauthorized project', async () => { + const otherContext = await createTestContext(); + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces/some-trace') + .query({ projectId: otherContext.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(403); + + expect(response.body.error).toContain('Access denied'); + }); + + it('GET /api/v1/traces/:traceId/spans should return 403 for unauthorized project', async () => { + const otherContext = await createTestContext(); + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces/some-trace/spans') + .query({ projectId: otherContext.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(403); + + expect(response.body.error).toContain('Access denied'); + }); + + it('GET /api/v1/traces/services should return 403 for unauthorized project', async () => { + const otherContext = await createTestContext(); + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces/services') + .query({ projectId: otherContext.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(403); + + expect(response.body.error).toContain('Access denied'); + }); + + it('GET /api/v1/traces/dependencies should return 403 for unauthorized project', async () => { + const otherContext = await createTestContext(); + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces/dependencies') + .query({ projectId: otherContext.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(403); + + expect(response.body.error).toContain('Access denied'); + }); + + it('GET /api/v1/traces/stats should return 403 for unauthorized project', async () => { + const otherContext = await createTestContext(); + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces/stats') + .query({ projectId: otherContext.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(403); + + expect(response.body.error).toContain('Access denied'); + }); + + it('GET /api/v1/traces/stats should work with session auth', async () => { + const session = await createSession(context.user.id); + + const response = await request(app.server) + .get('/api/v1/traces/stats') + .query({ projectId: context.project.id }) + .set('Authorization', `Bearer ${session.token}`) + .expect(200); + + expect(response.body.total_traces).toBe(0); + }); + }); }); diff --git a/packages/backend/src/tests/modules/traces/service.test.ts b/packages/backend/src/tests/modules/traces/service.test.ts index c0bd7cec..efbfdb0f 100644 --- a/packages/backend/src/tests/modules/traces/service.test.ts +++ b/packages/backend/src/tests/modules/traces/service.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { TracesService } from '../../../modules/traces/service.js'; -import { createTestContext, createTestTrace, createTestSpan } from '../../helpers/index.js'; +import { createTestContext, createTestTrace, createTestSpan, createTestLog } from '../../helpers/index.js'; import { db } from '../../../database/index.js'; import type { TransformedSpan, AggregatedTrace } from '../../../modules/otlp/trace-transformer.js'; import crypto from 'crypto'; @@ -800,4 +800,425 @@ describe('TracesService', () => { expect(result.total_traces).toBe(1); }); }); + + // ========================================================================== + // getEnrichedServiceDependencies + // ========================================================================== + describe('getEnrichedServiceDependencies', () => { + it('should return empty graph when no data exists', async () => { + const result = await service.getEnrichedServiceDependencies(context.project.id); + + expect(result.nodes).toEqual([]); + expect(result.edges).toEqual([]); + }); + + it('should return enriched nodes from span dependencies', async () => { + const traceId = crypto.randomBytes(16).toString('hex'); + const now = new Date(); + + const parentSpan = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId, + spanId: 'enriched-parent', + serviceName: 'api-gateway', + startTime: now, + }); + + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId, + parentSpanId: parentSpan.span_id, + serviceName: 'user-service', + startTime: new Date(now.getTime() + 10), + }); + + const result = await service.getEnrichedServiceDependencies( + context.project.id, + new Date(now.getTime() - 5000), + new Date(now.getTime() + 5000), + ); + + expect(result.nodes).toHaveLength(2); + expect(result.edges).toHaveLength(1); + + // Verify enriched node structure + const gatewayNode = result.nodes.find((n) => n.name === 'api-gateway'); + expect(gatewayNode).toBeDefined(); + expect(gatewayNode?.id).toBe('api-gateway'); + expect(gatewayNode?.callCount).toBeGreaterThanOrEqual(0); + expect(typeof gatewayNode?.errorRate).toBe('number'); + expect(typeof gatewayNode?.avgLatencyMs).toBe('number'); + expect(typeof gatewayNode?.totalCalls).toBe('number'); + + // Verify edge has type 'span' + expect(result.edges[0].type).toBe('span'); + expect(result.edges[0].source).toBe('api-gateway'); + expect(result.edges[0].target).toBe('user-service'); + expect(result.edges[0].callCount).toBe(1); + }); + + it('should include log co-occurrence edges when within 7-day range', async () => { + const traceId = crypto.randomBytes(16).toString('hex'); + const now = new Date(); + + // Create 2+ log pairs with the same trace_id across different services + // (HAVING COUNT(*) >= 2 requires at least 2 co-occurrences) + for (let i = 0; i < 2; i++) { + await createTestLog({ + projectId: context.project.id, + service: 'log-service-a', + trace_id: traceId, + time: new Date(now.getTime() + i), + }); + await createTestLog({ + projectId: context.project.id, + service: 'log-service-b', + trace_id: traceId, + time: new Date(now.getTime() + i + 1), + }); + } + + const result = await service.getEnrichedServiceDependencies( + context.project.id, + new Date(now.getTime() - 5000), + new Date(now.getTime() + 5000), + ); + + // Should have nodes for log-only services + const logServiceA = result.nodes.find((n) => n.name === 'log-service-a'); + const logServiceB = result.nodes.find((n) => n.name === 'log-service-b'); + expect(logServiceA).toBeDefined(); + expect(logServiceB).toBeDefined(); + + // Log-only services should have callCount 0 + expect(logServiceA?.callCount).toBe(0); + + // Should have a log_correlation edge + const logEdge = result.edges.find((e) => e.type === 'log_correlation'); + expect(logEdge).toBeDefined(); + expect(logEdge?.callCount).toBeGreaterThanOrEqual(2); + }); + + it('should prioritize span edges over log co-occurrence edges', async () => { + const traceId = crypto.randomBytes(16).toString('hex'); + const now = new Date(); + + // Create span-based dependency: gateway → backend + const parentSpan = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId, + spanId: 'prio-parent', + serviceName: 'gateway', + startTime: now, + }); + + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId, + parentSpanId: parentSpan.span_id, + serviceName: 'backend', + startTime: new Date(now.getTime() + 10), + }); + + // Also create log co-occurrence for same service pair + for (let i = 0; i < 3; i++) { + await createTestLog({ + projectId: context.project.id, + service: 'backend', + trace_id: traceId, + time: new Date(now.getTime() + i), + }); + await createTestLog({ + projectId: context.project.id, + service: 'gateway', + trace_id: traceId, + time: new Date(now.getTime() + i + 1), + }); + } + + const result = await service.getEnrichedServiceDependencies( + context.project.id, + new Date(now.getTime() - 5000), + new Date(now.getTime() + 5000), + ); + + // Should only have span edge, not duplicate log_correlation edge + const edgesBetween = result.edges.filter( + (e) => + (e.source === 'gateway' && e.target === 'backend') || + (e.source === 'backend' && e.target === 'gateway'), + ); + expect(edgesBetween).toHaveLength(1); + expect(edgesBetween[0].type).toBe('span'); + }); + + it('should skip log correlation for ranges > 7 days', async () => { + const traceId = crypto.randomBytes(16).toString('hex'); + const now = new Date(); + + // Create log co-occurrence data + for (let i = 0; i < 3; i++) { + await createTestLog({ + projectId: context.project.id, + service: 'svc-x', + trace_id: traceId, + time: now, + }); + await createTestLog({ + projectId: context.project.id, + service: 'svc-y', + trace_id: traceId, + time: now, + }); + } + + // Query with > 7 day range (8 days) + const eightDaysAgo = new Date(now.getTime() - 8 * 24 * 60 * 60 * 1000); + const result = await service.getEnrichedServiceDependencies( + context.project.id, + eightDaysAgo, + new Date(now.getTime() + 5000), + ); + + // Should not have log_correlation edges + const logEdges = result.edges.filter((e) => e.type === 'log_correlation'); + expect(logEdges).toHaveLength(0); + }); + + it('should use default time range when not provided', async () => { + const result = await service.getEnrichedServiceDependencies(context.project.id); + + // Should not throw, default is last 24h + expect(result).toBeDefined(); + expect(result.nodes).toBeDefined(); + expect(result.edges).toBeDefined(); + }); + + it('should set default values when health stats are empty', async () => { + const traceId = crypto.randomBytes(16).toString('hex'); + const now = new Date(); + + const parentSpan = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId, + spanId: 'health-parent', + serviceName: 'svc-no-health', + startTime: now, + }); + + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId, + parentSpanId: parentSpan.span_id, + serviceName: 'svc-no-health-child', + startTime: new Date(now.getTime() + 10), + }); + + const result = await service.getEnrichedServiceDependencies( + context.project.id, + new Date(now.getTime() - 5000), + new Date(now.getTime() + 5000), + ); + + // Health stats won't be populated (continuous aggregates not refreshed in tests) + // So defaults should be applied + for (const node of result.nodes) { + expect(node.errorRate).toBe(0); + expect(node.avgLatencyMs).toBe(0); + expect(node.p95LatencyMs).toBeNull(); + } + }); + + it('should handle multiple independent trace dependencies', async () => { + const now = new Date(); + + // Trace 1: A → B + const trace1 = crypto.randomBytes(16).toString('hex'); + const parent1 = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: trace1, + spanId: 'multi-parent-1', + serviceName: 'service-a', + startTime: now, + }); + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: trace1, + parentSpanId: parent1.span_id, + serviceName: 'service-b', + startTime: new Date(now.getTime() + 10), + }); + + // Trace 2: B → C + const trace2 = crypto.randomBytes(16).toString('hex'); + const parent2 = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: trace2, + spanId: 'multi-parent-2', + serviceName: 'service-b', + startTime: now, + }); + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: trace2, + parentSpanId: parent2.span_id, + serviceName: 'service-c', + startTime: new Date(now.getTime() + 10), + }); + + const result = await service.getEnrichedServiceDependencies( + context.project.id, + new Date(now.getTime() - 5000), + new Date(now.getTime() + 5000), + ); + + expect(result.nodes).toHaveLength(3); + expect(result.edges).toHaveLength(2); + + const edgeAB = result.edges.find((e) => e.source === 'service-a' && e.target === 'service-b'); + const edgeBC = result.edges.find((e) => e.source === 'service-b' && e.target === 'service-c'); + expect(edgeAB).toBeDefined(); + expect(edgeBC).toBeDefined(); + }); + + it('should add log-only services as nodes with callCount 0', async () => { + const traceId = crypto.randomBytes(16).toString('hex'); + const now = new Date(); + + // Only log co-occurrence, no spans + for (let i = 0; i < 3; i++) { + await createTestLog({ + projectId: context.project.id, + service: 'log-only-a', + trace_id: traceId, + time: new Date(now.getTime() + i), + }); + await createTestLog({ + projectId: context.project.id, + service: 'log-only-b', + trace_id: traceId, + time: new Date(now.getTime() + i + 1), + }); + } + + const result = await service.getEnrichedServiceDependencies( + context.project.id, + new Date(now.getTime() - 5000), + new Date(now.getTime() + 5000), + ); + + const nodeA = result.nodes.find((n) => n.name === 'log-only-a'); + const nodeB = result.nodes.find((n) => n.name === 'log-only-b'); + + expect(nodeA).toBeDefined(); + expect(nodeA?.callCount).toBe(0); + expect(nodeA?.totalCalls).toBe(0); + + expect(nodeB).toBeDefined(); + expect(nodeB?.callCount).toBe(0); + expect(nodeB?.totalCalls).toBe(0); + }); + + it('should not add log edges below threshold (< 2 co-occurrences)', async () => { + const traceId = crypto.randomBytes(16).toString('hex'); + const now = new Date(); + + // Only 1 co-occurrence (below HAVING COUNT(*) >= 2 threshold) + await createTestLog({ + projectId: context.project.id, + service: 'below-thresh-a', + trace_id: traceId, + time: now, + }); + await createTestLog({ + projectId: context.project.id, + service: 'below-thresh-b', + trace_id: traceId, + time: new Date(now.getTime() + 1), + }); + + const result = await service.getEnrichedServiceDependencies( + context.project.id, + new Date(now.getTime() - 5000), + new Date(now.getTime() + 5000), + ); + + // Should not find edges for this pair + const edges = result.edges.filter( + (e) => + (e.source === 'below-thresh-a' || e.target === 'below-thresh-a') && + (e.source === 'below-thresh-b' || e.target === 'below-thresh-b'), + ); + expect(edges).toHaveLength(0); + }); + + it('should filter span dependencies by time range', async () => { + const now = new Date(); + const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000); + + // Old span dependency + const oldTraceId = crypto.randomBytes(16).toString('hex'); + const oldParent = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: oldTraceId, + spanId: 'old-enr-parent', + serviceName: 'old-svc-a', + startTime: yesterday, + }); + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: oldTraceId, + parentSpanId: oldParent.span_id, + serviceName: 'old-svc-b', + startTime: yesterday, + }); + + // Recent span dependency + const newTraceId = crypto.randomBytes(16).toString('hex'); + const newParent = await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: newTraceId, + spanId: 'new-enr-parent', + serviceName: 'new-svc-a', + startTime: now, + }); + await createTestSpan({ + projectId: context.project.id, + organizationId: context.organization.id, + traceId: newTraceId, + parentSpanId: newParent.span_id, + serviceName: 'new-svc-b', + startTime: now, + }); + + const result = await service.getEnrichedServiceDependencies( + context.project.id, + new Date(now.getTime() - 1000), + new Date(now.getTime() + 1000), + ); + + // Should only include recent dependency + const oldEdge = result.edges.find( + (e) => e.source === 'old-svc-a' && e.target === 'old-svc-b', + ); + const newEdge = result.edges.find( + (e) => e.source === 'new-svc-a' && e.target === 'new-svc-b', + ); + expect(oldEdge).toBeUndefined(); + expect(newEdge).toBeDefined(); + }); + }); }); diff --git a/packages/frontend/src/lib/api/traces.ts b/packages/frontend/src/lib/api/traces.ts index d0a77a4d..f4c9d72d 100644 --- a/packages/frontend/src/lib/api/traces.ts +++ b/packages/frontend/src/lib/api/traces.ts @@ -72,6 +72,29 @@ export interface ServiceDependencies { edges: ServiceDependencyEdge[]; } +// Enriched types for the service map page +export interface EnrichedServiceDependencyNode { + id: string; + name: string; + callCount: number; + errorRate: number; + avgLatencyMs: number; + p95LatencyMs: number | null; + totalCalls: number; +} + +export interface EnrichedServiceDependencyEdge { + source: string; + target: string; + callCount: number; + type: 'span' | 'log_correlation'; +} + +export interface EnrichedServiceDependencies { + nodes: EnrichedServiceDependencyNode[]; + edges: EnrichedServiceDependencyEdge[]; +} + export class TracesAPI { constructor(private getToken: () => string | null) {} @@ -191,6 +214,26 @@ export class TracesAPI { return response.json(); } + async getServiceMap(projectId: string, from?: string, to?: string): Promise { + const params = new URLSearchParams(); + params.append('projectId', projectId); + if (from) params.append('from', from); + if (to) params.append('to', to); + + const url = `${getApiBaseUrl()}/traces/service-map?${params.toString()}`; + + const response = await fetch(url, { + method: 'GET', + headers: this.getHeaders(), + }); + + if (!response.ok) { + throw new Error(`Failed to fetch service map: ${response.statusText}`); + } + + return response.json(); + } + async getDependencies(projectId: string, from?: string, to?: string): Promise { const params = new URLSearchParams(); params.append('projectId', projectId); diff --git a/packages/frontend/src/lib/components/AppLayout.svelte b/packages/frontend/src/lib/components/AppLayout.svelte index 1f81c780..fa440bf9 100644 --- a/packages/frontend/src/lib/components/AppLayout.svelte +++ b/packages/frontend/src/lib/components/AppLayout.svelte @@ -38,6 +38,7 @@ import LogOut from "@lucide/svelte/icons/log-out"; import Menu from "@lucide/svelte/icons/menu"; import Shield from "@lucide/svelte/icons/shield"; + import Network from "@lucide/svelte/icons/network"; import Book from "@lucide/svelte/icons/book"; import Github from "@lucide/svelte/icons/github"; import X from "@lucide/svelte/icons/x"; @@ -238,6 +239,11 @@ icon: GitBranch, badge: { id: 'traces-feature', type: 'new', showUntil: '2025-03-01' } }, + { + label: "Service Map", + href: "/dashboard/service-map", + icon: Network, + }, { label: "Alerts", href: "/dashboard/alerts", icon: AlertTriangle }, { label: "Errors", diff --git a/packages/frontend/src/lib/components/ServiceMap.svelte b/packages/frontend/src/lib/components/ServiceMap.svelte index bb924577..79c894b4 100644 --- a/packages/frontend/src/lib/components/ServiceMap.svelte +++ b/packages/frontend/src/lib/components/ServiceMap.svelte @@ -1,22 +1,23 @@ + + + Service Map - LogTide + + +
+ +
+
+ +

Service Map

+
+

+ Visualize service dependencies and correlations across your infrastructure +

+
+ + + + + Filters + + +
+
+ + { + if (v) { + selectedProject = v; + selectedNode = null; + loadMap(); + } + }} + > + + {projects.find((p) => p.id === selectedProject)?.name || + "Select project"} + + + {#each projects as project} + {project.name} + {/each} + + +
+ +
+ + { + selectedNode = null; + loadMap(); + }} + /> +
+
+ + + + +
+
+
+ + Healthy (<1%) +
+
+ + Degraded (1-10%) +
+
+ + Unhealthy (>10%) +
+
+ + Log correlation +
+
+ +
+ + +
+ +
+ + + {#if isLoading} +
+ +
+ {:else if loadError} +
+

{loadError}

+
+ {:else if mapData && mapData.nodes.length > 0} + + {:else if mapData} +
+
+ +

No service dependencies found

+

+ Send traces with parent-child spans or logs with trace_id to + see service relationships +

+
+
+ {:else} +
+

Select a project to view the service map

+
+ {/if} + + +
+ + + {#if selectedNode} + {@const health = getHealthLabel(selectedNode.errorRate)} +
+ + +
+ {selectedNode.name} + + {health.label} + +
+ + + + +
+
+

Error Rate

+

+ {(selectedNode.errorRate * 100).toFixed(1)}% +

+
+
+

Avg Latency

+

+ {formatLatency(selectedNode.avgLatencyMs)} +

+
+
+

P95 Latency

+

+ {selectedNode.p95LatencyMs != null + ? formatLatency(selectedNode.p95LatencyMs) + : "N/A"} +

+
+
+

Total Calls

+

+ {selectedNode.totalCalls.toLocaleString()} +

+
+
+ + + {#if downstreamEdges.length > 0} +
+

+ + Calls to +

+
+ {#each downstreamEdges as edge} +
+ {edge.target} +
+ + {edge.callCount} + + {#if edge.type === "log_correlation"} + log + {/if} +
+
+ {/each} +
+
+ {/if} + + + {#if upstreamEdges.length > 0} +
+

+ + Called by +

+
+ {#each upstreamEdges as edge} +
+ {edge.source} +
+ + {edge.callCount} + + {#if edge.type === "log_correlation"} + log + {/if} +
+
+ {/each} +
+
+ {/if} + + + + +
+ {/if} +
+
From a2721b47a963ecce2d11873a8999254345dd8475 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 15:21:15 +0100 Subject: [PATCH 03/43] feat: updated changelog for added service dependency graph and correlation analysis to dashboard --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd68877e..12554278 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Service Dependency Graph & Correlation Analysis** (#40): dedicated `/dashboard/service-map` page visualizing microservice interactions + - Force-directed graph (ECharts) built from span parent-child relationships + log co-occurrence analysis + - Enriched backend endpoint `GET /api/v1/traces/service-map` runs 3 parallel queries: span deps (reservoir), per-service health stats (continuous aggregates), log co-occurrence (trace_id self-join) + - Health color-coding on nodes: green (<1% errors), amber (1-10%), red (>10%) + - Click-to-inspect side panel showing error rate, avg/p95 latency, total calls, upstream/downstream edges + - Dashed edges for log correlation, solid for span-based dependencies + - PNG export, time range filtering, project picker + - New "Service Map" nav item in sidebar + - **Audit Log**: comprehensive audit trail tracking all user actions across the platform for compliance and security (SOC 2, ISO 27001, HIPAA) - Tracks 4 event categories: log access, config changes, user management, data modifications - Logged actions: login, logout, register, create/update/delete organizations, create/update/delete projects, create/revoke API keys, member role changes, member removal, leave organization, admin operations From d77a822cff5dbf385ad82c28e2f18bb743283d10 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 16:35:21 +0100 Subject: [PATCH 04/43] feat: implement OTLP metrics ingestion with API routes and database schema --- .../backend/migrations/026_add_metrics.sql | 93 ++ packages/backend/src/database/types.ts | 35 + packages/backend/src/modules/metrics/index.ts | 2 + .../backend/src/modules/metrics/routes.ts | 378 +++++++ .../backend/src/modules/metrics/service.ts | 89 ++ packages/backend/src/modules/otlp/index.ts | 1 + .../backend/src/modules/otlp/metric-routes.ts | 216 ++++ .../src/modules/otlp/metric-transformer.ts | 955 ++++++++++++++++++ packages/backend/src/server.ts | 5 +- packages/frontend/src/lib/api/metrics.ts | 151 +++ .../src/lib/components/AppLayout.svelte | 7 + packages/frontend/src/lib/stores/metrics.ts | 157 +++ .../src/routes/dashboard/metrics/+page.svelte | 907 +++++++++++++++++ .../src/routes/dashboard/metrics/+page.ts | 1 + packages/reservoir/src/client.ts | 50 + packages/reservoir/src/core/storage-engine.ts | 36 + packages/reservoir/src/core/types.ts | 169 ++++ .../engines/clickhouse/clickhouse-engine.ts | 601 +++++++++++ .../src/engines/timescale/timescale-engine.ts | 598 +++++++++++ packages/reservoir/src/index.ts | 17 + 20 files changed, 4467 insertions(+), 1 deletion(-) create mode 100644 packages/backend/migrations/026_add_metrics.sql create mode 100644 packages/backend/src/modules/metrics/index.ts create mode 100644 packages/backend/src/modules/metrics/routes.ts create mode 100644 packages/backend/src/modules/metrics/service.ts create mode 100644 packages/backend/src/modules/otlp/metric-routes.ts create mode 100644 packages/backend/src/modules/otlp/metric-transformer.ts create mode 100644 packages/frontend/src/lib/api/metrics.ts create mode 100644 packages/frontend/src/lib/stores/metrics.ts create mode 100644 packages/frontend/src/routes/dashboard/metrics/+page.svelte create mode 100644 packages/frontend/src/routes/dashboard/metrics/+page.ts diff --git a/packages/backend/migrations/026_add_metrics.sql b/packages/backend/migrations/026_add_metrics.sql new file mode 100644 index 00000000..bfffbadc --- /dev/null +++ b/packages/backend/migrations/026_add_metrics.sql @@ -0,0 +1,93 @@ +-- ============================================================================ +-- Migration 026: OTLP Metrics Ingestion +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS metrics ( + time TIMESTAMPTZ NOT NULL, + id UUID NOT NULL DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + metric_name TEXT NOT NULL, + metric_type TEXT NOT NULL, + value DOUBLE PRECISION NOT NULL DEFAULT 0, + is_monotonic BOOLEAN, + service_name TEXT NOT NULL DEFAULT 'unknown', + attributes JSONB, + resource_attributes JSONB, + histogram_data JSONB, + has_exemplars BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY (time, id) +); + +SELECT create_hypertable('metrics', 'time', if_not_exists => TRUE); + +CREATE INDEX IF NOT EXISTS idx_metrics_name_time + ON metrics (metric_name, time DESC); + +CREATE INDEX IF NOT EXISTS idx_metrics_project_name_time + ON metrics (project_id, metric_name, time DESC); + +CREATE INDEX IF NOT EXISTS idx_metrics_service_time + ON metrics (service_name, time DESC); + +CREATE INDEX IF NOT EXISTS idx_metrics_type + ON metrics (metric_type, time DESC); + +CREATE INDEX IF NOT EXISTS idx_metrics_attributes + ON metrics USING GIN (attributes jsonb_path_ops); + +CREATE INDEX IF NOT EXISTS idx_metrics_org_time + ON metrics (organization_id, time DESC); + +-- ============================================================================ +-- METRIC EXEMPLARS TABLE +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS metric_exemplars ( + time TIMESTAMPTZ NOT NULL, + id UUID NOT NULL DEFAULT gen_random_uuid(), + metric_id UUID NOT NULL, + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + exemplar_value DOUBLE PRECISION NOT NULL, + exemplar_time TIMESTAMPTZ, + trace_id TEXT, + span_id TEXT, + attributes JSONB, + PRIMARY KEY (time, id) +); + +SELECT create_hypertable('metric_exemplars', 'time', if_not_exists => TRUE); + +CREATE INDEX IF NOT EXISTS idx_exemplars_metric_id + ON metric_exemplars (metric_id, time DESC); + +CREATE INDEX IF NOT EXISTS idx_exemplars_trace_id + ON metric_exemplars (trace_id, time DESC) WHERE trace_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_exemplars_project_time + ON metric_exemplars (project_id, time DESC); + +-- ============================================================================ +-- COMPRESSION POLICIES +-- ============================================================================ + +ALTER TABLE metrics SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'project_id, metric_name', + timescaledb.compress_orderby = 'time DESC' +); +SELECT add_compression_policy('metrics', INTERVAL '7 days', if_not_exists => TRUE); + +ALTER TABLE metric_exemplars SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'project_id', + timescaledb.compress_orderby = 'time DESC' +); +SELECT add_compression_policy('metric_exemplars', INTERVAL '7 days', if_not_exists => TRUE); + +-- ============================================================================ +-- RETENTION POLICIES (default 90 days, org-configurable like logs) +-- ============================================================================ + +SELECT add_retention_policy('metrics', INTERVAL '90 days', if_not_exists => TRUE); +SELECT add_retention_policy('metric_exemplars', INTERVAL '90 days', if_not_exists => TRUE); diff --git a/packages/backend/src/database/types.ts b/packages/backend/src/database/types.ts index 02c8bd3e..6f710796 100644 --- a/packages/backend/src/database/types.ts +++ b/packages/backend/src/database/types.ts @@ -755,6 +755,38 @@ export interface AuditLogTable { >; } +// ============================================================================ +// METRICS TABLES (OTLP Metrics Ingestion) +// ============================================================================ + +export interface MetricsTable { + time: Timestamp; + id: Generated; + organization_id: string; + project_id: string; + metric_name: string; + metric_type: string; + value: number; + is_monotonic: boolean | null; + service_name: string; + attributes: ColumnType | null, Record | null, Record | null>; + resource_attributes: ColumnType | null, Record | null, Record | null>; + histogram_data: ColumnType | null, Record | null, Record | null>; + has_exemplars: boolean; +} + +export interface MetricExemplarsTable { + time: Timestamp; + id: Generated; + metric_id: string; + project_id: string; + exemplar_value: number; + exemplar_time: Timestamp | null; + trace_id: string | null; + span_id: string | null; + attributes: ColumnType | null, Record | null, Record | null>; +} + export interface Database { logs: LogsTable; users: UsersTable; @@ -812,4 +844,7 @@ export interface Database { organization_pii_salts: OrganizationPiiSaltsTable; // Audit log audit_log: AuditLogTable; + // Metrics (OTLP) + metrics: MetricsTable; + metric_exemplars: MetricExemplarsTable; } diff --git a/packages/backend/src/modules/metrics/index.ts b/packages/backend/src/modules/metrics/index.ts new file mode 100644 index 00000000..17efea2a --- /dev/null +++ b/packages/backend/src/modules/metrics/index.ts @@ -0,0 +1,2 @@ +export { metricsService } from './service.js'; +export { default as metricsRoutes } from './routes.js'; diff --git a/packages/backend/src/modules/metrics/routes.ts b/packages/backend/src/modules/metrics/routes.ts new file mode 100644 index 00000000..fd4d9434 --- /dev/null +++ b/packages/backend/src/modules/metrics/routes.ts @@ -0,0 +1,378 @@ +import type { FastifyPluginAsync } from 'fastify'; +import { metricsService } from './service.js'; +import { db } from '../../database/index.js'; +import { requireFullAccess } from '../auth/guards.js'; +import type { AggregationInterval, MetricAggregationFn } from '@logtide/reservoir'; + +async function verifyProjectAccess(projectId: string, userId: string): Promise { + const result = await db + .selectFrom('projects') + .innerJoin('organization_members', 'projects.organization_id', 'organization_members.organization_id') + .select(['projects.id']) + .where('projects.id', '=', projectId) + .where('organization_members.user_id', '=', userId) + .executeTakeFirst(); + + return !!result; +} + +/** + * Resolve the effective projectId for the request. + * API key auth: always use request.projectId (scoped to one project). + * Session auth: use queryProjectId if provided, with access verification. + */ +function resolveProjectId(request: any, queryProjectId?: string): string | undefined { + if (request.projectId) { + // API key auth: always scoped to the key's project + return request.projectId; + } + return queryProjectId; +} + +function parseAttributes(query: Record): Record | undefined { + const attrs: Record = {}; + let found = false; + + for (const key of Object.keys(query)) { + const match = key.match(/^attributes\[(.+)]$/); + if (match && typeof query[key] === 'string') { + attrs[match[1]] = query[key] as string; + found = true; + } + } + + return found ? attrs : undefined; +} + +const metricsRoutes: FastifyPluginAsync = async (fastify) => { + // GET /api/v1/metrics/names + fastify.get('/names', { + schema: { + querystring: { + type: 'object', + properties: { + projectId: { type: 'string' }, + from: { type: 'string', format: 'date-time' }, + to: { type: 'string', format: 'date-time' }, + }, + }, + }, + handler: async (request: any, reply) => { + if (!await requireFullAccess(request, reply)) return; + + const { projectId: queryProjectId, from, to } = request.query as { + projectId?: string; + from?: string; + to?: string; + }; + + const projectId = resolveProjectId(request, queryProjectId); + + if (!projectId) { + return reply.code(400).send({ + error: 'Project context missing - provide projectId query parameter', + }); + } + + if (request.user?.id) { + const hasAccess = await verifyProjectAccess(projectId, request.user.id); + if (!hasAccess) { + return reply.code(403).send({ + error: 'Access denied - you do not have access to this project', + }); + } + } + + return metricsService.listMetricNames( + projectId, + from ? new Date(from) : undefined, + to ? new Date(to) : undefined, + ); + }, + }); + + // GET /api/v1/metrics/labels/keys + fastify.get('/labels/keys', { + schema: { + querystring: { + type: 'object', + properties: { + projectId: { type: 'string' }, + metricName: { type: 'string' }, + from: { type: 'string', format: 'date-time' }, + to: { type: 'string', format: 'date-time' }, + }, + }, + }, + handler: async (request: any, reply) => { + if (!await requireFullAccess(request, reply)) return; + + const { projectId: queryProjectId, metricName, from, to } = request.query as { + projectId?: string; + metricName?: string; + from?: string; + to?: string; + }; + + const projectId = resolveProjectId(request, queryProjectId); + + if (!projectId) { + return reply.code(400).send({ + error: 'Project context missing - provide projectId query parameter', + }); + } + + if (!metricName) { + return reply.code(400).send({ + error: 'metricName query parameter is required', + }); + } + + if (request.user?.id) { + const hasAccess = await verifyProjectAccess(projectId, request.user.id); + if (!hasAccess) { + return reply.code(403).send({ + error: 'Access denied - you do not have access to this project', + }); + } + } + + return metricsService.getLabelKeys( + projectId, + metricName, + from ? new Date(from) : undefined, + to ? new Date(to) : undefined, + ); + }, + }); + + // GET /api/v1/metrics/labels/values + fastify.get('/labels/values', { + schema: { + querystring: { + type: 'object', + properties: { + projectId: { type: 'string' }, + metricName: { type: 'string' }, + labelKey: { type: 'string' }, + from: { type: 'string', format: 'date-time' }, + to: { type: 'string', format: 'date-time' }, + }, + }, + }, + handler: async (request: any, reply) => { + if (!await requireFullAccess(request, reply)) return; + + const { projectId: queryProjectId, metricName, labelKey, from, to } = request.query as { + projectId?: string; + metricName?: string; + labelKey?: string; + from?: string; + to?: string; + }; + + const projectId = resolveProjectId(request, queryProjectId); + + if (!projectId) { + return reply.code(400).send({ + error: 'Project context missing - provide projectId query parameter', + }); + } + + if (!metricName) { + return reply.code(400).send({ + error: 'metricName query parameter is required', + }); + } + + if (!labelKey) { + return reply.code(400).send({ + error: 'labelKey query parameter is required', + }); + } + + if (request.user?.id) { + const hasAccess = await verifyProjectAccess(projectId, request.user.id); + if (!hasAccess) { + return reply.code(403).send({ + error: 'Access denied - you do not have access to this project', + }); + } + } + + return metricsService.getLabelValues( + projectId, + metricName, + labelKey, + from ? new Date(from) : undefined, + to ? new Date(to) : undefined, + ); + }, + }); + + // GET /api/v1/metrics/data + fastify.get('/data', { + schema: { + querystring: { + type: 'object', + properties: { + projectId: { type: 'string' }, + metricName: { type: 'string' }, + from: { type: 'string', format: 'date-time' }, + to: { type: 'string', format: 'date-time' }, + limit: { type: 'number', minimum: 1, maximum: 10000, default: 1000 }, + offset: { type: 'number', minimum: 0, default: 0 }, + includeExemplars: { type: 'boolean', default: false }, + }, + }, + }, + handler: async (request: any, reply) => { + if (!await requireFullAccess(request, reply)) return; + + const { + projectId: queryProjectId, + metricName, + from, + to, + limit, + offset, + includeExemplars, + } = request.query as { + projectId?: string; + metricName?: string; + from?: string; + to?: string; + limit?: number; + offset?: number; + includeExemplars?: boolean; + }; + + const projectId = resolveProjectId(request, queryProjectId); + + if (!projectId) { + return reply.code(400).send({ + error: 'Project context missing - provide projectId query parameter', + }); + } + + if (!from || !to) { + return reply.code(400).send({ + error: 'from and to query parameters are required', + }); + } + + if (request.user?.id) { + const hasAccess = await verifyProjectAccess(projectId, request.user.id); + if (!hasAccess) { + return reply.code(403).send({ + error: 'Access denied - you do not have access to this project', + }); + } + } + + const attributes = parseAttributes(request.query); + + return metricsService.queryMetrics({ + projectId, + metricName, + from: new Date(from), + to: new Date(to), + attributes, + limit: limit || 1000, + offset: offset || 0, + includeExemplars: includeExemplars || false, + }); + }, + }); + + // GET /api/v1/metrics/aggregate + fastify.get('/aggregate', { + schema: { + querystring: { + type: 'object', + properties: { + projectId: { type: 'string' }, + metricName: { type: 'string' }, + from: { type: 'string', format: 'date-time' }, + to: { type: 'string', format: 'date-time' }, + interval: { type: 'string', enum: ['1m', '5m', '15m', '1h', '6h', '1d', '1w'], default: '1h' }, + aggregation: { type: 'string', enum: ['avg', 'sum', 'min', 'max', 'count', 'last'], default: 'avg' }, + groupBy: { + oneOf: [ + { type: 'string' }, + { type: 'array', items: { type: 'string' } }, + ], + }, + }, + }, + }, + handler: async (request: any, reply) => { + if (!await requireFullAccess(request, reply)) return; + + const { + projectId: queryProjectId, + metricName, + from, + to, + interval, + aggregation, + groupBy, + } = request.query as { + projectId?: string; + metricName?: string; + from?: string; + to?: string; + interval?: string; + aggregation?: string; + groupBy?: string | string[]; + }; + + const projectId = resolveProjectId(request, queryProjectId); + + if (!projectId) { + return reply.code(400).send({ + error: 'Project context missing - provide projectId query parameter', + }); + } + + if (!metricName) { + return reply.code(400).send({ + error: 'metricName query parameter is required', + }); + } + + if (!from || !to) { + return reply.code(400).send({ + error: 'from and to query parameters are required', + }); + } + + if (request.user?.id) { + const hasAccess = await verifyProjectAccess(projectId, request.user.id); + if (!hasAccess) { + return reply.code(403).send({ + error: 'Access denied - you do not have access to this project', + }); + } + } + + const attributes = parseAttributes(request.query); + const groupByArr = groupBy + ? Array.isArray(groupBy) ? groupBy : [groupBy] + : undefined; + + return metricsService.aggregateMetrics({ + projectId, + metricName, + from: new Date(from), + to: new Date(to), + interval: (interval || '1h') as AggregationInterval, + aggregation: (aggregation || 'avg') as MetricAggregationFn, + groupBy: groupByArr, + attributes, + }); + }, + }); +}; + +export default metricsRoutes; diff --git a/packages/backend/src/modules/metrics/service.ts b/packages/backend/src/modules/metrics/service.ts new file mode 100644 index 00000000..1c727b44 --- /dev/null +++ b/packages/backend/src/modules/metrics/service.ts @@ -0,0 +1,89 @@ +import { reservoir } from '../../database/reservoir.js'; +import type { + MetricRecord, + AggregationInterval, + MetricAggregationFn, +} from '@logtide/reservoir'; + +export class MetricsService { + async ingestMetrics( + records: MetricRecord[], + projectId: string, + organizationId: string, + ): Promise { + if (records.length === 0) return 0; + + const enriched = records.map((r) => ({ + ...r, + projectId, + organizationId, + })); + + const result = await reservoir.ingestMetrics(enriched); + return result.ingested; + } + + async listMetricNames(projectId: string | string[], from?: Date, to?: Date) { + return reservoir.getMetricNames({ projectId, from, to }); + } + + async getLabelKeys(projectId: string | string[], metricName: string, from?: Date, to?: Date) { + return reservoir.getMetricLabelKeys({ projectId, metricName, from, to }); + } + + async getLabelValues( + projectId: string | string[], + metricName: string, + labelKey: string, + from?: Date, + to?: Date, + ) { + return reservoir.getMetricLabelValues({ projectId, metricName, from, to }, labelKey); + } + + async queryMetrics(params: { + projectId: string | string[]; + metricName?: string | string[]; + from: Date; + to: Date; + attributes?: Record; + limit?: number; + offset?: number; + includeExemplars?: boolean; + }) { + return reservoir.queryMetrics({ + projectId: params.projectId, + metricName: params.metricName, + from: params.from, + to: params.to, + attributes: params.attributes, + limit: params.limit, + offset: params.offset, + includeExemplars: params.includeExemplars, + }); + } + + async aggregateMetrics(params: { + projectId: string | string[]; + metricName: string; + from: Date; + to: Date; + interval: AggregationInterval; + aggregation: MetricAggregationFn; + groupBy?: string[]; + attributes?: Record; + }) { + return reservoir.aggregateMetrics({ + projectId: params.projectId, + metricName: params.metricName, + from: params.from, + to: params.to, + interval: params.interval, + aggregation: params.aggregation, + groupBy: params.groupBy, + attributes: params.attributes, + }); + } +} + +export const metricsService = new MetricsService(); diff --git a/packages/backend/src/modules/otlp/index.ts b/packages/backend/src/modules/otlp/index.ts index 97541c41..0c01120a 100644 --- a/packages/backend/src/modules/otlp/index.ts +++ b/packages/backend/src/modules/otlp/index.ts @@ -1,5 +1,6 @@ export { default as otlpRoutes } from './routes.js'; export { default as otlpTraceRoutes } from './trace-routes.js'; +export { default as otlpMetricRoutes } from './metric-routes.js'; export * from './parser.js'; export * from './transformer.js'; export * from './severity-mapper.js'; diff --git a/packages/backend/src/modules/otlp/metric-routes.ts b/packages/backend/src/modules/otlp/metric-routes.ts new file mode 100644 index 00000000..a7092db7 --- /dev/null +++ b/packages/backend/src/modules/otlp/metric-routes.ts @@ -0,0 +1,216 @@ +/** + * OTLP Metric Routes + * + * OpenTelemetry Protocol HTTP endpoints for metric ingestion. + * + * Endpoint: POST /v1/otlp/metrics + * Content-Types: application/json, application/x-protobuf + * Content-Encoding: gzip (supported) + * + * @see https://opentelemetry.io/docs/specs/otlp/ + */ + +import type { FastifyPluginAsync, FastifyRequest } from 'fastify'; +import { parseOtlpMetricsJson, parseOtlpMetricsProtobuf, transformOtlpToMetrics } from './metric-transformer.js'; +import { detectContentType, isGzipCompressed, decompressGzip } from './parser.js'; +import { metricsService } from '../metrics/service.js'; +import { config } from '../../config/index.js'; +import { db } from '../../database/index.js'; + +const collectStreamToBuffer = (stream: NodeJS.ReadableStream): Promise => + new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk: Buffer) => chunks.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(chunks))); + stream.on('error', reject); + }); + +const otlpMetricRoutes: FastifyPluginAsync = async (fastify) => { + // Remove default JSON parser to add our own with gzip support + fastify.removeContentTypeParser('application/json'); + + // Custom JSON parser with gzip decompression support + fastify.addContentTypeParser( + 'application/json', + async (request: FastifyRequest) => { + const contentEncoding = request.headers['content-encoding'] as string | undefined; + let buffer = await collectStreamToBuffer(request.raw); + + const needsDecompression = contentEncoding?.toLowerCase() === 'gzip' || isGzipCompressed(buffer); + if (needsDecompression) { + try { + buffer = await decompressGzip(buffer); + } catch (error) { + const errMsg = error instanceof Error ? error.message : 'Unknown error'; + const decompressError = new Error(`Failed to decompress gzip JSON data: ${errMsg}`) as Error & { statusCode: number }; + decompressError.statusCode = 400; + throw decompressError; + } + } + + try { + return JSON.parse(buffer.toString('utf-8')); + } catch (error) { + const errMsg = error instanceof Error ? error.message : 'Invalid JSON'; + const parseError = new Error(`Invalid JSON: ${errMsg}`) as Error & { statusCode: number }; + parseError.statusCode = 400; + throw parseError; + } + } + ); + + fastify.addContentTypeParser( + 'application/x-protobuf', + async (request: FastifyRequest) => collectStreamToBuffer(request.raw), + ); + + fastify.addContentTypeParser( + 'application/protobuf', + async (request: FastifyRequest) => collectStreamToBuffer(request.raw), + ); + + /** + * POST /v1/otlp/metrics + * + * Ingest metrics via OpenTelemetry Protocol. + * Accepts both JSON and Protobuf content types. + */ + fastify.post('/v1/otlp/metrics', { + bodyLimit: 50 * 1024 * 1024, + config: { + rateLimit: { + max: config.RATE_LIMIT_MAX, + timeWindow: config.RATE_LIMIT_WINDOW, + }, + }, + schema: { + response: { + 200: { + type: 'object', + properties: { + partialSuccess: { + type: 'object', + properties: { + rejectedDataPoints: { type: 'number' }, + errorMessage: { type: 'string' }, + }, + }, + }, + }, + 400: { + type: 'object', + properties: { + partialSuccess: { + type: 'object', + properties: { + rejectedDataPoints: { type: 'number' }, + errorMessage: { type: 'string' }, + }, + }, + }, + }, + 401: { + type: 'object', + properties: { + error: { type: 'string' }, + }, + }, + }, + }, + handler: async (request: any, reply) => { + const projectId = request.projectId; + + if (!projectId) { + return reply.code(401).send({ + partialSuccess: { + rejectedDataPoints: -1, + errorMessage: 'Unauthorized: Missing or invalid API key', + }, + }); + } + + const project = await db + .selectFrom('projects') + .select(['organization_id']) + .where('id', '=', projectId) + .executeTakeFirst(); + + if (!project) { + return reply.code(401).send({ + partialSuccess: { + rejectedDataPoints: -1, + errorMessage: 'Unauthorized: Project not found', + }, + }); + } + + const contentType = request.headers['content-type'] as string | undefined; + const contentEncoding = request.headers['content-encoding'] as string | undefined; + const detectedType = detectContentType(contentType); + + try { + let otlpRequest; + if (detectedType === 'protobuf') { + let body = request.body; + if (Buffer.isBuffer(body)) { + const needsDecompression = contentEncoding?.toLowerCase() === 'gzip' || isGzipCompressed(body); + if (needsDecompression) { + try { + body = await decompressGzip(body); + } catch (decompressError) { + const errMsg = decompressError instanceof Error ? decompressError.message : 'Unknown error'; + throw new Error(`Failed to decompress gzip data: ${errMsg}`); + } + } + otlpRequest = await parseOtlpMetricsProtobuf(body); + } else { + throw new Error('Protobuf content-type requires Buffer body'); + } + } else { + otlpRequest = parseOtlpMetricsJson(request.body); + } + + const records = transformOtlpToMetrics(otlpRequest); + + if (records.length === 0) { + return { + partialSuccess: { + rejectedDataPoints: 0, + errorMessage: '', + }, + }; + } + + await metricsService.ingestMetrics(records, projectId, project.organization_id); + + console.log(`[OTLP Metrics] Ingested ${records.length} data points for project ${projectId}`); + + return { + partialSuccess: { + rejectedDataPoints: 0, + errorMessage: '', + }, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error('[OTLP Metrics] Ingestion error:', errorMessage); + + return reply.code(400).send({ + partialSuccess: { + rejectedDataPoints: -1, + errorMessage, + }, + }); + } + }, + }); + + /** + * Health check endpoint for OTLP metrics + */ + fastify.get('/v1/otlp/metrics', async () => { + return { status: 'ok' }; + }); +}; + +export default otlpMetricRoutes; diff --git a/packages/backend/src/modules/otlp/metric-transformer.ts b/packages/backend/src/modules/otlp/metric-transformer.ts new file mode 100644 index 00000000..c692bb44 --- /dev/null +++ b/packages/backend/src/modules/otlp/metric-transformer.ts @@ -0,0 +1,955 @@ +/** + * OTLP Metric Transformer + * + * Transforms OpenTelemetry Metric messages to LogTide MetricRecord format. + * Supports gauge, sum, histogram, exponential histogram, and summary metric types. + * + * @see https://opentelemetry.io/docs/specs/otel/metrics/data-model/ + */ + +import type { MetricRecord, HistogramData, MetricExemplar } from '@logtide/reservoir'; +import { attributesToRecord, sanitizeForPostgres, extractServiceName, nanosToIso, type OtlpKeyValue } from './transformer.js'; +import { isGzipCompressed, decompressGzip } from './parser.js'; +import { createRequire } from 'module'; + +// Import the generated protobuf definitions from @opentelemetry/otlp-transformer +const require = createRequire(import.meta.url); +const $root = require('@opentelemetry/otlp-transformer/build/esm/generated/root.js'); + +// Get the ExportMetricsServiceRequest message type for decoding protobuf messages +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const ExportMetricsServiceRequest: any = $root.opentelemetry?.proto?.collector?.metrics?.v1?.ExportMetricsServiceRequest; + +// ============================================================================ +// OTLP Metric Type Definitions (based on OpenTelemetry proto) +// ============================================================================ + +/** + * OTLP exemplar attached to a data point for trace correlation + */ +export interface OtlpExemplar { + filteredAttributes?: OtlpKeyValue[]; + timeUnixNano?: string | bigint; + asDouble?: number; + asInt?: string | number; + spanId?: string; + traceId?: string; +} + +/** + * OTLP NumberDataPoint - used by gauge and sum metrics + */ +export interface OtlpNumberDataPoint { + attributes?: OtlpKeyValue[]; + startTimeUnixNano?: string | bigint; + timeUnixNano?: string | bigint; + asDouble?: number; + asInt?: string | number; + exemplars?: OtlpExemplar[]; + flags?: number; +} + +/** + * OTLP HistogramDataPoint - explicit bucket histogram + */ +export interface OtlpHistogramDataPoint { + attributes?: OtlpKeyValue[]; + startTimeUnixNano?: string | bigint; + timeUnixNano?: string | bigint; + count?: string | number; + sum?: number; + bucketCounts?: (string | number)[]; + explicitBounds?: number[]; + exemplars?: OtlpExemplar[]; + flags?: number; + min?: number; + max?: number; +} + +/** + * OTLP ExponentialHistogramDataPoint - base-2 exponential bucket histogram + */ +export interface OtlpExponentialHistogramDataPoint { + attributes?: OtlpKeyValue[]; + startTimeUnixNano?: string | bigint; + timeUnixNano?: string | bigint; + count?: string | number; + sum?: number; + scale?: number; + zeroCount?: string | number; + positive?: { + offset?: number; + bucketCounts?: (string | number)[]; + }; + negative?: { + offset?: number; + bucketCounts?: (string | number)[]; + }; + flags?: number; + exemplars?: OtlpExemplar[]; + min?: number; + max?: number; + zeroThreshold?: number; +} + +/** + * OTLP SummaryDataPoint - pre-computed quantile summary + */ +export interface OtlpSummaryDataPoint { + attributes?: OtlpKeyValue[]; + startTimeUnixNano?: string | bigint; + timeUnixNano?: string | bigint; + count?: string | number; + sum?: number; + quantileValues?: Array<{ + quantile?: number; + value?: number; + }>; + flags?: number; +} + +/** + * OTLP Gauge metric - instantaneous measurement + */ +export interface OtlpGauge { + dataPoints?: OtlpNumberDataPoint[]; +} + +/** + * OTLP Sum metric - cumulative or delta counter + */ +export interface OtlpSum { + dataPoints?: OtlpNumberDataPoint[]; + aggregationTemporality?: number; + isMonotonic?: boolean; +} + +/** + * OTLP Histogram metric - explicit bucket histogram + */ +export interface OtlpHistogram { + dataPoints?: OtlpHistogramDataPoint[]; + aggregationTemporality?: number; +} + +/** + * OTLP ExponentialHistogram metric - base-2 exponential bucket histogram + */ +export interface OtlpExponentialHistogram { + dataPoints?: OtlpExponentialHistogramDataPoint[]; + aggregationTemporality?: number; +} + +/** + * OTLP Summary metric - pre-computed quantile summary + */ +export interface OtlpSummary { + dataPoints?: OtlpSummaryDataPoint[]; +} + +/** + * OTLP Metric - a single named metric with one of gauge/sum/histogram/expHistogram/summary + */ +export interface OtlpMetric { + name?: string; + description?: string; + unit?: string; + gauge?: OtlpGauge; + sum?: OtlpSum; + histogram?: OtlpHistogram; + exponentialHistogram?: OtlpExponentialHistogram; + summary?: OtlpSummary; +} + +/** + * OTLP InstrumentationScope + */ +export interface OtlpMetricInstrumentationScope { + name?: string; + version?: string; + attributes?: OtlpKeyValue[]; +} + +/** + * OTLP ScopeMetrics - metrics from a single instrumentation scope + */ +export interface OtlpScopeMetrics { + scope?: OtlpMetricInstrumentationScope; + metrics?: OtlpMetric[]; + schemaUrl?: string; +} + +/** + * OTLP Resource (same structure as logs/traces) + */ +export interface OtlpMetricResource { + attributes?: OtlpKeyValue[]; + droppedAttributesCount?: number; +} + +/** + * OTLP ResourceMetrics - metrics from a single resource + */ +export interface OtlpResourceMetrics { + resource?: OtlpMetricResource; + scopeMetrics?: OtlpScopeMetrics[]; + schemaUrl?: string; +} + +/** + * OTLP ExportMetricsServiceRequest - top-level request message + */ +export interface OtlpExportMetricsRequest { + resourceMetrics?: OtlpResourceMetrics[]; +} + +// ============================================================================ +// Transformation Functions +// ============================================================================ + +/** + * Transform OTLP ExportMetricsServiceRequest to LogTide MetricRecord[]. + * + * Iterates through resourceMetrics -> scopeMetrics -> metrics, + * extracting the service name from resource attributes and dispatching + * each metric to its type-specific handler. + * + * Note: organizationId and projectId are left as empty strings here; + * they are filled in by the route handler. + * + * @param request - Parsed OTLP export metrics request + * @returns Array of MetricRecord ready for ingestion + */ +export function transformOtlpToMetrics( + request: OtlpExportMetricsRequest +): MetricRecord[] { + const records: MetricRecord[] = []; + + for (const resourceMetric of request.resourceMetrics ?? []) { + const serviceName = extractServiceName(resourceMetric.resource?.attributes); + const resourceAttributes = attributesToRecord(resourceMetric.resource?.attributes); + + for (const scopeMetric of resourceMetric.scopeMetrics ?? []) { + for (const metric of scopeMetric.metrics ?? []) { + const metricName = sanitizeForPostgres(metric.name || 'unknown'); + + if (metric.gauge) { + records.push( + ...transformGaugeDataPoints(metric.gauge, metricName, serviceName, resourceAttributes) + ); + } else if (metric.sum) { + records.push( + ...transformSumDataPoints(metric.sum, metricName, serviceName, resourceAttributes) + ); + } else if (metric.histogram) { + records.push( + ...transformHistogramDataPoints(metric.histogram, metricName, serviceName, resourceAttributes) + ); + } else if (metric.exponentialHistogram) { + records.push( + ...transformExpHistogramDataPoints(metric.exponentialHistogram, metricName, serviceName, resourceAttributes) + ); + } else if (metric.summary) { + records.push( + ...transformSummaryDataPoints(metric.summary, metricName, serviceName, resourceAttributes) + ); + } + } + } + } + + return records; +} + +// ============================================================================ +// Type-specific handlers +// ============================================================================ + +/** + * Transform gauge data points to MetricRecord[]. + */ +function transformGaugeDataPoints( + gauge: OtlpGauge, + metricName: string, + serviceName: string, + resourceAttributes: Record +): MetricRecord[] { + return (gauge.dataPoints ?? []).map((dp) => ({ + time: nanosToDate(dp.timeUnixNano), + organizationId: '', + projectId: '', + metricName, + metricType: 'gauge' as const, + value: extractScalarValue(dp), + serviceName, + attributes: attributesToRecord(dp.attributes), + resourceAttributes, + exemplars: extractExemplars(dp.exemplars), + })); +} + +/** + * Transform sum data points to MetricRecord[]. + */ +function transformSumDataPoints( + sum: OtlpSum, + metricName: string, + serviceName: string, + resourceAttributes: Record +): MetricRecord[] { + return (sum.dataPoints ?? []).map((dp) => ({ + time: nanosToDate(dp.timeUnixNano), + organizationId: '', + projectId: '', + metricName, + metricType: 'sum' as const, + value: extractScalarValue(dp), + isMonotonic: sum.isMonotonic, + serviceName, + attributes: attributesToRecord(dp.attributes), + resourceAttributes, + exemplars: extractExemplars(dp.exemplars), + })); +} + +/** + * Transform histogram data points to MetricRecord[]. + */ +function transformHistogramDataPoints( + histogram: OtlpHistogram, + metricName: string, + serviceName: string, + resourceAttributes: Record +): MetricRecord[] { + return (histogram.dataPoints ?? []).map((dp) => { + const histogramData: HistogramData = { + sum: dp.sum, + count: toNumber(dp.count), + min: dp.min, + max: dp.max, + bucket_counts: dp.bucketCounts?.map(toNumber), + explicit_bounds: dp.explicitBounds, + }; + + // Use sum as the representative value, fallback to 0 + const value = dp.sum ?? 0; + + return { + time: nanosToDate(dp.timeUnixNano), + organizationId: '', + projectId: '', + metricName, + metricType: 'histogram' as const, + value, + serviceName, + attributes: attributesToRecord(dp.attributes), + resourceAttributes, + histogramData, + exemplars: extractExemplars(dp.exemplars), + }; + }); +} + +/** + * Transform exponential histogram data points to MetricRecord[]. + */ +function transformExpHistogramDataPoints( + expHistogram: OtlpExponentialHistogram, + metricName: string, + serviceName: string, + resourceAttributes: Record +): MetricRecord[] { + return (expHistogram.dataPoints ?? []).map((dp) => { + const histogramData: HistogramData = { + sum: dp.sum, + count: toNumber(dp.count), + min: dp.min, + max: dp.max, + scale: dp.scale, + zero_count: toNumber(dp.zeroCount), + positive: dp.positive ? { + offset: dp.positive.offset ?? 0, + bucket_counts: dp.positive.bucketCounts?.map(toNumber) ?? [], + } : undefined, + negative: dp.negative ? { + offset: dp.negative.offset ?? 0, + bucket_counts: dp.negative.bucketCounts?.map(toNumber) ?? [], + } : undefined, + }; + + const value = dp.sum ?? 0; + + return { + time: nanosToDate(dp.timeUnixNano), + organizationId: '', + projectId: '', + metricName, + metricType: 'exp_histogram' as const, + value, + serviceName, + attributes: attributesToRecord(dp.attributes), + resourceAttributes, + histogramData, + exemplars: extractExemplars(dp.exemplars), + }; + }); +} + +/** + * Transform summary data points to MetricRecord[]. + */ +function transformSummaryDataPoints( + summary: OtlpSummary, + metricName: string, + serviceName: string, + resourceAttributes: Record +): MetricRecord[] { + return (summary.dataPoints ?? []).map((dp) => { + const histogramData: HistogramData = { + sum: dp.sum, + count: toNumber(dp.count), + quantile_values: dp.quantileValues?.map((qv) => ({ + quantile: qv.quantile ?? 0, + value: qv.value ?? 0, + })), + }; + + const value = dp.sum ?? 0; + + return { + time: nanosToDate(dp.timeUnixNano), + organizationId: '', + projectId: '', + metricName, + metricType: 'summary' as const, + value, + serviceName, + attributes: attributesToRecord(dp.attributes), + resourceAttributes, + histogramData, + exemplars: undefined, + }; + }); +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/** + * Extract the numeric value from a NumberDataPoint. + * Prefers asDouble, falls back to asInt, then 0. + */ +function extractScalarValue(dp: OtlpNumberDataPoint): number { + if (dp.asDouble !== undefined) { + return dp.asDouble; + } + if (dp.asInt !== undefined) { + return toNumber(dp.asInt); + } + return 0; +} + +/** + * Convert a value that may be a string (int64 from JSON/protobuf) to a number. + */ +function toNumber(v: string | number | undefined): number { + if (v === undefined || v === null) return 0; + if (typeof v === 'number') return v; + const parsed = Number(v); + return Number.isNaN(parsed) ? 0 : parsed; +} + +/** + * Convert nanoseconds timestamp to Date object. + */ +function nanosToDate(nanos?: string | bigint): Date { + const iso = nanosToIso(nanos); + return new Date(iso); +} + +/** + * Extract exemplars from OTLP data points into MetricExemplar[]. + * Returns undefined if no exemplars are present. + */ +function extractExemplars(exemplars?: OtlpExemplar[]): MetricExemplar[] | undefined { + if (!exemplars || exemplars.length === 0) { + return undefined; + } + + return exemplars.map((ex) => { + const value = ex.asDouble !== undefined + ? ex.asDouble + : toNumber(ex.asInt); + + return { + exemplarValue: value, + exemplarTime: ex.timeUnixNano ? nanosToDate(ex.timeUnixNano) : undefined, + traceId: normalizeHexId(ex.traceId), + spanId: normalizeHexId(ex.spanId), + attributes: ex.filteredAttributes + ? attributesToRecord(ex.filteredAttributes) + : undefined, + }; + }); +} + +/** + * Normalize a hex-encoded ID (traceId/spanId). + * Returns undefined for empty/all-zero IDs. + * Handles base64 encoded values from protobuf. + */ +function normalizeHexId(id?: string): string | undefined { + if (!id) return undefined; + + // Check if all zeros (invalid per OTLP spec) + if (/^0+$/.test(id)) return undefined; + + // Check if it's base64 encoded (from protobuf toObject with bytes: String) + if (id.length > 0 && !/^[0-9a-fA-F]+$/.test(id)) { + try { + const buffer = Buffer.from(id, 'base64'); + const hex = buffer.toString('hex'); + if (/^0+$/.test(hex)) return undefined; + return hex; + } catch { + return id; + } + } + + return id; +} + +// ============================================================================ +// JSON Parser +// ============================================================================ + +/** + * Parse OTLP JSON metrics request body. + * Handles both camelCase and snake_case field names since some OTLP + * exporters use snake_case instead of the canonical camelCase. + * + * @param body - Raw request body (string or object) + * @returns Parsed OTLP metrics request + * @throws Error if parsing fails + */ +export function parseOtlpMetricsJson(body: unknown): OtlpExportMetricsRequest { + if (!body) { + return { resourceMetrics: [] }; + } + + if (typeof body === 'object') { + return normalizeMetricsRequest(body as Record); + } + + if (typeof body === 'string') { + try { + const parsed = JSON.parse(body); + return normalizeMetricsRequest(parsed); + } catch (error) { + throw new Error(`Invalid OTLP Metrics JSON: ${(error as Error).message}`); + } + } + + throw new Error('Invalid OTLP metrics request body type'); +} + +/** + * Normalize metrics request handling both camelCase and snake_case. + */ +function normalizeMetricsRequest(data: Record): OtlpExportMetricsRequest { + const resourceMetrics = (data.resourceMetrics ?? data.resource_metrics) as unknown[]; + + if (!Array.isArray(resourceMetrics)) { + return { resourceMetrics: [] }; + } + + return { + resourceMetrics: resourceMetrics.map(normalizeResourceMetrics), + }; +} + +function normalizeResourceMetrics(rm: unknown): OtlpResourceMetrics { + if (!rm || typeof rm !== 'object') return {}; + + const data = rm as Record; + + return { + resource: data.resource as OtlpMetricResource | undefined, + scopeMetrics: normalizeScopeMetrics(data.scopeMetrics ?? data.scope_metrics), + schemaUrl: (data.schemaUrl ?? data.schema_url) as string | undefined, + }; +} + +function normalizeScopeMetrics(sm: unknown): OtlpScopeMetrics[] | undefined { + if (!Array.isArray(sm)) return undefined; + + return sm.map((s) => { + if (!s || typeof s !== 'object') return {}; + const data = s as Record; + + return { + scope: data.scope as OtlpMetricInstrumentationScope | undefined, + metrics: normalizeMetrics(data.metrics), + schemaUrl: (data.schemaUrl ?? data.schema_url) as string | undefined, + }; + }); +} + +function normalizeMetrics(metrics: unknown): OtlpMetric[] | undefined { + if (!Array.isArray(metrics)) return undefined; + + return metrics.map((m) => { + if (!m || typeof m !== 'object') return {}; + const data = m as Record; + + return { + name: data.name as string | undefined, + description: data.description as string | undefined, + unit: data.unit as string | undefined, + gauge: data.gauge ? normalizeGauge(data.gauge) : undefined, + sum: data.sum ? normalizeSum(data.sum) : undefined, + histogram: data.histogram ? normalizeHistogram(data.histogram) : undefined, + exponentialHistogram: normalizeExpHistogramField( + data.exponentialHistogram ?? data.exponential_histogram + ), + summary: data.summary ? normalizeSummary(data.summary) : undefined, + }; + }); +} + +function normalizeGauge(gauge: unknown): OtlpGauge | undefined { + if (!gauge || typeof gauge !== 'object') return undefined; + const data = gauge as Record; + + return { + dataPoints: normalizeNumberDataPoints(data.dataPoints ?? data.data_points), + }; +} + +function normalizeSum(sum: unknown): OtlpSum | undefined { + if (!sum || typeof sum !== 'object') return undefined; + const data = sum as Record; + + return { + dataPoints: normalizeNumberDataPoints(data.dataPoints ?? data.data_points), + aggregationTemporality: (data.aggregationTemporality ?? data.aggregation_temporality) as number | undefined, + isMonotonic: (data.isMonotonic ?? data.is_monotonic) as boolean | undefined, + }; +} + +function normalizeHistogram(histogram: unknown): OtlpHistogram | undefined { + if (!histogram || typeof histogram !== 'object') return undefined; + const data = histogram as Record; + + return { + dataPoints: normalizeHistogramDataPoints(data.dataPoints ?? data.data_points), + aggregationTemporality: (data.aggregationTemporality ?? data.aggregation_temporality) as number | undefined, + }; +} + +function normalizeExpHistogramField(expHist: unknown): OtlpExponentialHistogram | undefined { + if (!expHist || typeof expHist !== 'object') return undefined; + const data = expHist as Record; + + return { + dataPoints: normalizeExpHistogramDataPoints(data.dataPoints ?? data.data_points), + aggregationTemporality: (data.aggregationTemporality ?? data.aggregation_temporality) as number | undefined, + }; +} + +function normalizeSummary(summary: unknown): OtlpSummary | undefined { + if (!summary || typeof summary !== 'object') return undefined; + const data = summary as Record; + + return { + dataPoints: normalizeSummaryDataPoints(data.dataPoints ?? data.data_points), + }; +} + +// ============================================================================ +// Data point normalization (snake_case -> camelCase) +// ============================================================================ + +function normalizeNumberDataPoints(dps: unknown): OtlpNumberDataPoint[] | undefined { + if (!Array.isArray(dps)) return undefined; + + return dps.map((dp) => { + if (!dp || typeof dp !== 'object') return {}; + const data = dp as Record; + + return { + attributes: data.attributes as OtlpKeyValue[] | undefined, + startTimeUnixNano: (data.startTimeUnixNano ?? data.start_time_unix_nano) as string | bigint | undefined, + timeUnixNano: (data.timeUnixNano ?? data.time_unix_nano) as string | bigint | undefined, + asDouble: (data.asDouble ?? data.as_double) as number | undefined, + asInt: (data.asInt ?? data.as_int) as string | number | undefined, + exemplars: normalizeExemplars(data.exemplars), + flags: data.flags as number | undefined, + }; + }); +} + +function normalizeHistogramDataPoints(dps: unknown): OtlpHistogramDataPoint[] | undefined { + if (!Array.isArray(dps)) return undefined; + + return dps.map((dp) => { + if (!dp || typeof dp !== 'object') return {}; + const data = dp as Record; + + return { + attributes: data.attributes as OtlpKeyValue[] | undefined, + startTimeUnixNano: (data.startTimeUnixNano ?? data.start_time_unix_nano) as string | bigint | undefined, + timeUnixNano: (data.timeUnixNano ?? data.time_unix_nano) as string | bigint | undefined, + count: data.count as string | number | undefined, + sum: data.sum as number | undefined, + bucketCounts: (data.bucketCounts ?? data.bucket_counts) as (string | number)[] | undefined, + explicitBounds: (data.explicitBounds ?? data.explicit_bounds) as number[] | undefined, + exemplars: normalizeExemplars(data.exemplars), + flags: data.flags as number | undefined, + min: data.min as number | undefined, + max: data.max as number | undefined, + }; + }); +} + +function normalizeExpHistogramDataPoints(dps: unknown): OtlpExponentialHistogramDataPoint[] | undefined { + if (!Array.isArray(dps)) return undefined; + + return dps.map((dp) => { + if (!dp || typeof dp !== 'object') return {}; + const data = dp as Record; + + const positive = data.positive as Record | undefined; + const negative = data.negative as Record | undefined; + + return { + attributes: data.attributes as OtlpKeyValue[] | undefined, + startTimeUnixNano: (data.startTimeUnixNano ?? data.start_time_unix_nano) as string | bigint | undefined, + timeUnixNano: (data.timeUnixNano ?? data.time_unix_nano) as string | bigint | undefined, + count: data.count as string | number | undefined, + sum: data.sum as number | undefined, + scale: data.scale as number | undefined, + zeroCount: (data.zeroCount ?? data.zero_count) as string | number | undefined, + positive: positive ? { + offset: positive.offset as number | undefined, + bucketCounts: (positive.bucketCounts ?? positive.bucket_counts) as (string | number)[] | undefined, + } : undefined, + negative: negative ? { + offset: negative.offset as number | undefined, + bucketCounts: (negative.bucketCounts ?? negative.bucket_counts) as (string | number)[] | undefined, + } : undefined, + flags: data.flags as number | undefined, + exemplars: normalizeExemplars(data.exemplars), + min: data.min as number | undefined, + max: data.max as number | undefined, + zeroThreshold: (data.zeroThreshold ?? data.zero_threshold) as number | undefined, + }; + }); +} + +function normalizeSummaryDataPoints(dps: unknown): OtlpSummaryDataPoint[] | undefined { + if (!Array.isArray(dps)) return undefined; + + return dps.map((dp) => { + if (!dp || typeof dp !== 'object') return {}; + const data = dp as Record; + + const rawQuantiles = (data.quantileValues ?? data.quantile_values) as unknown[] | undefined; + + return { + attributes: data.attributes as OtlpKeyValue[] | undefined, + startTimeUnixNano: (data.startTimeUnixNano ?? data.start_time_unix_nano) as string | bigint | undefined, + timeUnixNano: (data.timeUnixNano ?? data.time_unix_nano) as string | bigint | undefined, + count: data.count as string | number | undefined, + sum: data.sum as number | undefined, + quantileValues: rawQuantiles?.map((qv) => { + if (!qv || typeof qv !== 'object') return { quantile: 0, value: 0 }; + const q = qv as Record; + return { + quantile: q.quantile as number | undefined, + value: q.value as number | undefined, + }; + }), + flags: data.flags as number | undefined, + }; + }); +} + +function normalizeExemplars(exemplars: unknown): OtlpExemplar[] | undefined { + if (!Array.isArray(exemplars)) return undefined; + + return exemplars.map((ex) => { + if (!ex || typeof ex !== 'object') return {}; + const data = ex as Record; + + return { + filteredAttributes: (data.filteredAttributes ?? data.filtered_attributes) as OtlpKeyValue[] | undefined, + timeUnixNano: (data.timeUnixNano ?? data.time_unix_nano) as string | bigint | undefined, + asDouble: (data.asDouble ?? data.as_double) as number | undefined, + asInt: (data.asInt ?? data.as_int) as string | number | undefined, + spanId: (data.spanId ?? data.span_id) as string | undefined, + traceId: (data.traceId ?? data.trace_id) as string | undefined, + }; + }); +} + +// ============================================================================ +// Protobuf Parser +// ============================================================================ + +/** + * Parse OTLP Protobuf metrics request body. + * + * Uses the OpenTelemetry proto definitions from @opentelemetry/otlp-transformer + * to properly decode binary protobuf messages. + * + * Automatically detects and decompresses gzip-compressed data by checking + * for gzip magic bytes (0x1f 0x8b), regardless of Content-Encoding header. + * + * @param buffer - Raw protobuf buffer (may be gzip compressed) + * @returns Parsed OTLP metrics request + * @throws Error if parsing fails + */ +export async function parseOtlpMetricsProtobuf(buffer: Buffer): Promise { + // Auto-detect gzip compression by magic bytes (0x1f 0x8b) + if (isGzipCompressed(buffer)) { + console.log('[OTLP Metrics] Auto-detected gzip compression by magic bytes, decompressing...'); + try { + buffer = await decompressGzip(buffer); + console.log(`[OTLP Metrics] Decompressed protobuf data to ${buffer.length} bytes`); + } catch (error) { + const errMsg = error instanceof Error ? error.message : 'Unknown error'; + console.error('[OTLP Metrics] Gzip decompression failed:', errMsg); + throw new Error(`Failed to decompress gzip data: ${errMsg}`); + } + } + + // First, try to parse as JSON (some clients send JSON with protobuf content-type) + try { + const jsonString = buffer.toString('utf-8'); + if (jsonString.startsWith('{') || jsonString.startsWith('[')) { + console.log('[OTLP Metrics] Protobuf content-type but JSON payload detected, parsing as JSON'); + return parseOtlpMetricsJson(jsonString); + } + } catch { + // Not JSON, continue to protobuf parsing + } + + // Verify ExportMetricsServiceRequest is available + if (!ExportMetricsServiceRequest) { + throw new Error( + 'OTLP protobuf support not available. The OpenTelemetry proto definitions could not be loaded. ' + + 'Please use application/json content-type.' + ); + } + + // Decode the protobuf message using OpenTelemetry proto definitions + try { + const decoded = ExportMetricsServiceRequest.decode(buffer); + + // Convert to plain JavaScript object for processing + const message = ExportMetricsServiceRequest.toObject(decoded, { + longs: String, // Convert Long to string for JSON compatibility + bytes: String, // Convert bytes to base64 string + defaults: false, // Don't include default values + arrays: true, // Always return arrays even if empty + objects: true, // Always return nested objects + }); + + console.log('[OTLP Metrics] Successfully decoded protobuf message with', + message.resourceMetrics?.length || 0, 'resourceMetrics'); + + // Normalize the decoded message to match our OtlpExportMetricsRequest interface + return normalizeDecodedMetricsProtobuf(message); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error('[OTLP Metrics] Failed to decode protobuf:', errorMessage); + throw new Error(`Failed to decode OTLP metrics protobuf: ${errorMessage}`); + } +} + +/** + * Normalize decoded protobuf message to OtlpExportMetricsRequest format. + */ +function normalizeDecodedMetricsProtobuf(message: Record): OtlpExportMetricsRequest { + const resourceMetrics = message.resourceMetrics as unknown[] | undefined; + + if (!Array.isArray(resourceMetrics)) { + return { resourceMetrics: [] }; + } + + return { + resourceMetrics: resourceMetrics.map(normalizeResourceMetricsFromProtobuf), + }; +} + +/** + * Normalize ResourceMetrics from protobuf format. + */ +function normalizeResourceMetricsFromProtobuf(rm: unknown): OtlpResourceMetrics { + if (!rm || typeof rm !== 'object') return {}; + + const data = rm as Record; + + return { + resource: data.resource as OtlpMetricResource | undefined, + scopeMetrics: normalizeScopeMetricsFromProtobuf(data.scopeMetrics), + schemaUrl: data.schemaUrl as string | undefined, + }; +} + +/** + * Normalize ScopeMetrics from protobuf format. + */ +function normalizeScopeMetricsFromProtobuf(sm: unknown): OtlpScopeMetrics[] | undefined { + if (!Array.isArray(sm)) return undefined; + + return sm.map((s) => { + if (!s || typeof s !== 'object') return {}; + const data = s as Record; + + return { + scope: data.scope as OtlpMetricInstrumentationScope | undefined, + metrics: normalizeMetricsFromProtobuf(data.metrics), + schemaUrl: data.schemaUrl as string | undefined, + }; + }); +} + +/** + * Normalize individual metrics from protobuf format. + */ +function normalizeMetricsFromProtobuf(metrics: unknown): OtlpMetric[] | undefined { + if (!Array.isArray(metrics)) return undefined; + + return metrics.map((m) => { + if (!m || typeof m !== 'object') return {}; + const data = m as Record; + + return { + name: data.name as string | undefined, + description: data.description as string | undefined, + unit: data.unit as string | undefined, + gauge: data.gauge as OtlpGauge | undefined, + sum: data.sum ? normalizeProtobufSum(data.sum) : undefined, + histogram: data.histogram as OtlpHistogram | undefined, + exponentialHistogram: data.exponentialHistogram as OtlpExponentialHistogram | undefined, + summary: data.summary as OtlpSummary | undefined, + }; + }); +} + +/** + * Normalize sum from protobuf to ensure isMonotonic is properly read. + */ +function normalizeProtobufSum(sum: unknown): OtlpSum | undefined { + if (!sum || typeof sum !== 'object') return undefined; + const data = sum as Record; + + return { + dataPoints: data.dataPoints as OtlpNumberDataPoint[] | undefined, + aggregationTemporality: data.aggregationTemporality as number | undefined, + isMonotonic: data.isMonotonic as boolean | undefined, + }; +} diff --git a/packages/backend/src/server.ts b/packages/backend/src/server.ts index 3b29096a..17d7b35b 100644 --- a/packages/backend/src/server.ts +++ b/packages/backend/src/server.ts @@ -22,8 +22,9 @@ import { siemRoutes } from './modules/siem/routes.js'; import { registerSiemSseRoutes } from './modules/siem/sse-events.js'; import { adminRoutes } from './modules/admin/index.js'; import { publicAuthRoutes, authenticatedAuthRoutes, adminAuthRoutes } from './modules/auth/external-routes.js'; -import { otlpRoutes, otlpTraceRoutes } from './modules/otlp/index.js'; +import { otlpRoutes, otlpTraceRoutes, otlpMetricRoutes } from './modules/otlp/index.js'; import { tracesRoutes } from './modules/traces/index.js'; +import { metricsRoutes } from './modules/metrics/index.js'; import { onboardingRoutes } from './modules/onboarding/index.js'; import { exceptionsRoutes } from './modules/exceptions/index.js'; import { settingsRoutes, publicSettingsRoutes, settingsService } from './modules/settings/index.js'; @@ -187,7 +188,9 @@ export async function build(opts = {}) { await fastify.register(piiMaskingRoutes, { prefix: '/api' }); await fastify.register(otlpRoutes); await fastify.register(otlpTraceRoutes); + await fastify.register(otlpMetricRoutes); await fastify.register(tracesRoutes); + await fastify.register(metricsRoutes, { prefix: '/api/v1/metrics' }); await fastify.register(websocketPlugin); await fastify.register(websocketRoutes); diff --git a/packages/frontend/src/lib/api/metrics.ts b/packages/frontend/src/lib/api/metrics.ts new file mode 100644 index 00000000..f9f19450 --- /dev/null +++ b/packages/frontend/src/lib/api/metrics.ts @@ -0,0 +1,151 @@ +import { getApiBaseUrl } from '$lib/config'; +import { getAuthToken } from '$lib/utils/auth'; + +export type MetricType = 'gauge' | 'sum' | 'histogram' | 'exp_histogram' | 'summary'; +export type MetricAggregationFn = 'avg' | 'sum' | 'min' | 'max' | 'count' | 'last'; + +export interface MetricName { + name: string; + type: MetricType; +} + +export interface MetricTimeBucket { + bucket: string; + value: number; + labels?: Record; +} + +export interface MetricAggregateResult { + metricName: string; + metricType: MetricType; + timeseries: MetricTimeBucket[]; +} + +export interface MetricDataPoint { + id: string; + time: string; + metricName: string; + metricType: MetricType; + value: number; + serviceName: string; + attributes: Record | null; + resourceAttributes: Record | null; + histogramData: Record | null; + hasExemplars: boolean; + exemplars?: Array<{ + exemplarValue: number; + exemplarTime?: string; + traceId?: string; + spanId?: string; + attributes?: Record; + }>; +} + +export interface MetricDataResponse { + metrics: MetricDataPoint[]; + total: number; + hasMore: boolean; + limit: number; + offset: number; +} + +export class MetricsAPI { + constructor(private getToken: () => string | null) {} + + private getHeaders(): HeadersInit { + const token = this.getToken(); + const headers: HeadersInit = { 'Content-Type': 'application/json' }; + if (token) headers['Authorization'] = `Bearer ${token}`; + return headers; + } + + async getMetricNames(projectId: string, from?: string, to?: string): Promise { + const params = new URLSearchParams({ projectId }); + if (from) params.append('from', from); + if (to) params.append('to', to); + const res = await fetch(`${getApiBaseUrl()}/metrics/names?${params}`, { headers: this.getHeaders() }); + if (!res.ok) throw new Error(`Failed to fetch metric names: ${res.statusText}`); + const data = await res.json(); + return data.names; + } + + async getLabelKeys(projectId: string, metricName: string, from?: string, to?: string): Promise { + const params = new URLSearchParams({ projectId, metricName }); + if (from) params.append('from', from); + if (to) params.append('to', to); + const res = await fetch(`${getApiBaseUrl()}/metrics/labels/keys?${params}`, { headers: this.getHeaders() }); + if (!res.ok) throw new Error(`Failed to fetch label keys: ${res.statusText}`); + const data = await res.json(); + return data.keys ?? []; + } + + async getLabelValues(projectId: string, metricName: string, labelKey: string, from?: string, to?: string): Promise { + const params = new URLSearchParams({ projectId, metricName, labelKey }); + if (from) params.append('from', from); + if (to) params.append('to', to); + const res = await fetch(`${getApiBaseUrl()}/metrics/labels/values?${params}`, { headers: this.getHeaders() }); + if (!res.ok) throw new Error(`Failed to fetch label values: ${res.statusText}`); + const data = await res.json(); + return data.values ?? []; + } + + async getMetricData(params: { + projectId: string; + metricName: string; + from: string; + to: string; + attributes?: Record; + limit?: number; + offset?: number; + includeExemplars?: boolean; + }): Promise { + const searchParams = new URLSearchParams({ + projectId: params.projectId, + metricName: params.metricName, + from: params.from, + to: params.to, + }); + if (params.limit) searchParams.append('limit', String(params.limit)); + if (params.offset) searchParams.append('offset', String(params.offset)); + if (params.includeExemplars) searchParams.append('includeExemplars', 'true'); + if (params.attributes) { + for (const [k, v] of Object.entries(params.attributes)) { + searchParams.append(`attributes[${k}]`, v); + } + } + const res = await fetch(`${getApiBaseUrl()}/metrics/data?${searchParams}`, { headers: this.getHeaders() }); + if (!res.ok) throw new Error(`Failed to fetch metric data: ${res.statusText}`); + return res.json(); + } + + async aggregateMetrics(params: { + projectId: string; + metricName: string; + from: string; + to: string; + interval?: string; + aggregation?: MetricAggregationFn; + groupBy?: string[]; + attributes?: Record; + }): Promise { + const searchParams = new URLSearchParams({ + projectId: params.projectId, + metricName: params.metricName, + from: params.from, + to: params.to, + interval: params.interval ?? '1h', + aggregation: params.aggregation ?? 'avg', + }); + if (params.groupBy) params.groupBy.forEach(g => searchParams.append('groupBy', g)); + if (params.attributes) { + for (const [k, v] of Object.entries(params.attributes)) { + searchParams.append(`attributes[${k}]`, v); + } + } + const res = await fetch(`${getApiBaseUrl()}/metrics/aggregate?${searchParams}`, { headers: this.getHeaders() }); + if (!res.ok) throw new Error(`Failed to aggregate metrics: ${res.statusText}`); + return res.json(); + } +} + +export const metricsAPI = new MetricsAPI(getAuthToken); diff --git a/packages/frontend/src/lib/components/AppLayout.svelte b/packages/frontend/src/lib/components/AppLayout.svelte index fa440bf9..5069d965 100644 --- a/packages/frontend/src/lib/components/AppLayout.svelte +++ b/packages/frontend/src/lib/components/AppLayout.svelte @@ -47,6 +47,7 @@ import LayoutGrid from "@lucide/svelte/icons/layout-grid"; import Check from "@lucide/svelte/icons/check"; import SearchIcon from "@lucide/svelte/icons/search"; + import BarChart3 from "@lucide/svelte/icons/bar-chart-3"; import { formatTimeAgo } from "$lib/utils/datetime"; import Footer from "$lib/components/Footer.svelte"; import OnboardingChecklist from "$lib/components/OnboardingChecklist.svelte"; @@ -239,6 +240,12 @@ icon: GitBranch, badge: { id: 'traces-feature', type: 'new', showUntil: '2025-03-01' } }, + { + label: "Metrics", + href: "/dashboard/metrics", + icon: BarChart3, + badge: { id: 'metrics-feature', type: 'new', showUntil: '2026-09-01' } + }, { label: "Service Map", href: "/dashboard/service-map", diff --git a/packages/frontend/src/lib/stores/metrics.ts b/packages/frontend/src/lib/stores/metrics.ts new file mode 100644 index 00000000..ab2940ee --- /dev/null +++ b/packages/frontend/src/lib/stores/metrics.ts @@ -0,0 +1,157 @@ +import { writable, derived } from 'svelte/store'; +import { metricsAPI, type MetricName, type MetricAggregateResult, type MetricDataResponse, type MetricAggregationFn } from '$lib/api/metrics'; + +interface MetricsState { + metricNames: MetricName[]; + metricNamesLoading: boolean; + metricNamesError: string | null; + selectedMetric: string | null; + selectedInterval: string; + selectedAggregation: MetricAggregationFn; + selectedGroupBy: string[]; + activeLabels: Record; + timeseries: MetricAggregateResult | null; + timeseriesLoading: boolean; + timeseriesError: string | null; + labelKeys: string[]; + labelValues: Record; + dataPoints: MetricDataResponse | null; + dataPointsLoading: boolean; +} + +const initialState: MetricsState = { + metricNames: [], + metricNamesLoading: false, + metricNamesError: null, + selectedMetric: null, + selectedInterval: '1h', + selectedAggregation: 'avg', + selectedGroupBy: [], + activeLabels: {}, + timeseries: null, + timeseriesLoading: false, + timeseriesError: null, + labelKeys: [], + labelValues: {}, + dataPoints: null, + dataPointsLoading: false, +}; + +function createMetricsStore() { + const { subscribe, set, update } = writable(initialState); + + return { + subscribe, + + async loadMetricNames(projectId: string, from?: string, to?: string) { + update(s => ({ ...s, metricNamesLoading: true, metricNamesError: null })); + try { + const names = await metricsAPI.getMetricNames(projectId, from, to); + update(s => ({ ...s, metricNames: names, metricNamesLoading: false })); + } catch (error) { + update(s => ({ ...s, metricNamesError: (error as Error).message, metricNamesLoading: false })); + } + }, + + async loadTimeseries(projectId: string, metricName: string, from: string, to: string) { + update(s => ({ ...s, timeseriesLoading: true, timeseriesError: null })); + try { + let currentState: MetricsState; + const unsub = subscribe(s => currentState = s); + unsub(); + + const result = await metricsAPI.aggregateMetrics({ + projectId, + metricName, + from, + to, + interval: currentState!.selectedInterval, + aggregation: currentState!.selectedAggregation, + groupBy: currentState!.selectedGroupBy.length > 0 ? currentState!.selectedGroupBy : undefined, + attributes: Object.keys(currentState!.activeLabels).length > 0 ? currentState!.activeLabels : undefined, + }); + update(s => ({ ...s, timeseries: result, timeseriesLoading: false })); + } catch (error) { + update(s => ({ ...s, timeseriesError: (error as Error).message, timeseriesLoading: false })); + } + }, + + async loadLabelKeys(projectId: string, metricName: string, from?: string, to?: string) { + try { + const keys = await metricsAPI.getLabelKeys(projectId, metricName, from, to); + update(s => ({ ...s, labelKeys: keys })); + } catch { + update(s => ({ ...s, labelKeys: [] })); + } + }, + + async loadLabelValues(projectId: string, metricName: string, labelKey: string, from?: string, to?: string) { + try { + const values = await metricsAPI.getLabelValues(projectId, metricName, labelKey, from, to); + update(s => ({ ...s, labelValues: { ...s.labelValues, [labelKey]: values } })); + } catch { + // ignore + } + }, + + async loadDataPoints(projectId: string, metricName: string, from: string, to: string, includeExemplars = false) { + update(s => ({ ...s, dataPointsLoading: true })); + try { + let currentState: MetricsState; + const unsub = subscribe(s => currentState = s); + unsub(); + + const result = await metricsAPI.getMetricData({ + projectId, + metricName, + from, + to, + includeExemplars, + attributes: Object.keys(currentState!.activeLabels).length > 0 ? currentState!.activeLabels : undefined, + limit: 100, + }); + update(s => ({ ...s, dataPoints: result, dataPointsLoading: false })); + } catch { + update(s => ({ ...s, dataPointsLoading: false })); + } + }, + + selectMetric(name: string | null) { + update(s => ({ ...s, selectedMetric: name, timeseries: null, dataPoints: null, labelKeys: [], labelValues: {}, activeLabels: {} })); + }, + + setInterval(interval: string) { + update(s => ({ ...s, selectedInterval: interval })); + }, + + setAggregation(agg: MetricAggregationFn) { + update(s => ({ ...s, selectedAggregation: agg })); + }, + + setGroupBy(keys: string[]) { + update(s => ({ ...s, selectedGroupBy: keys })); + }, + + setLabel(key: string, value: string) { + update(s => ({ ...s, activeLabels: { ...s.activeLabels, [key]: value } })); + }, + + removeLabel(key: string) { + update(s => { + const labels = { ...s.activeLabels }; + delete labels[key]; + return { ...s, activeLabels: labels }; + }); + }, + + reset() { + set(initialState); + }, + }; +} + +export const metricsStore = createMetricsStore(); +export const metricNames = derived({ subscribe: metricsStore.subscribe }, $s => $s.metricNames); +export const selectedMetric = derived({ subscribe: metricsStore.subscribe }, $s => $s.selectedMetric); +export const timeseries = derived({ subscribe: metricsStore.subscribe }, $s => $s.timeseries); +export const timeseriesLoading = derived({ subscribe: metricsStore.subscribe }, $s => $s.timeseriesLoading); diff --git a/packages/frontend/src/routes/dashboard/metrics/+page.svelte b/packages/frontend/src/routes/dashboard/metrics/+page.svelte new file mode 100644 index 00000000..f729009b --- /dev/null +++ b/packages/frontend/src/routes/dashboard/metrics/+page.svelte @@ -0,0 +1,907 @@ + + + + Metrics Explorer - LogTide + + +
+ +
+
+ +

Metrics Explorer

+
+

+ Explore and visualize OTLP metrics from your applications +

+
+ + + + + Filters + + +
+ +
+ + { + if (v) handleProjectChange(v); + }} + > + + {projects.find((p) => p.id === selectedProject)?.name || + "Select project"} + + + {#each projects as project} + {project.name} + {/each} + + +
+ + +
+ + { + if (v) handleMetricSelect(v); + }} + > + + {#if storeState.metricNamesLoading} + Loading... + {:else} + {storeState.selectedMetric || "Select metric"} + {/if} + + + {#each storeState.metricNames as metric} + + + {metric.name} + {metric.type} + + + {/each} + {#if storeState.metricNames.length === 0 && !storeState.metricNamesLoading} +
+ No metrics found +
+ {/if} + + +
+
+ + +
+ +
+ + + + + {#if storeState.selectedMetric} + + +
+ + Chart Controls +
+ + +
+ +
+ + { + if (v) handleIntervalChange(v); + }} + > + + {intervals.find((i) => i.value === storeState.selectedInterval) + ?.label || storeState.selectedInterval} + + + {#each intervals as interval} + {interval.label} + {/each} + + +
+ + +
+ + { + if (v) handleAggregationChange(v); + }} + > + + {aggregations.find( + (a) => a.value === storeState.selectedAggregation + )?.label || storeState.selectedAggregation} + + + {#each aggregations as agg} + {agg.label} + {/each} + + +
+ + +
+ + { + if (v) handleLabelKeyChange(v); + }} + > + + {selectedLabelKey || "Select label key"} + + + {#each storeState.labelKeys as key} + {key} + {/each} + {#if storeState.labelKeys.length === 0} +
+ No labels available +
+ {/if} + + +
+ + +
+ +
+ { + if (v) selectedLabelValue = v; + }} + > + + {selectedLabelValue || "Select value"} + + + {#each storeState.labelValues[selectedLabelKey ?? ""] ?? [] as val} + {val} + {/each} + {#if !selectedLabelKey} +
+ Select a label key first +
+ {/if} + + + +
+
+
+ + + {#if Object.keys(storeState.activeLabels).length > 0} +
+ + {#each Object.entries(storeState.activeLabels) as [key, value]} + + {key}={value} + + + {/each} +
+ {/if} + + + {/if} + + + + + + {#if storeState.selectedMetric} + {storeState.selectedMetric} + + {storeState.selectedAggregation} / {storeState.selectedInterval} + + {:else} + Time Series + {/if} + + + + {#if storeState.timeseriesLoading} +
+
+
+ {:else if storeState.timeseriesError} +
+

Error: {storeState.timeseriesError}

+
+ {:else if !storeState.selectedMetric} +
+ +

Select a metric to visualize

+
+ {:else} +
+ {/if} + + + + + {#if storeState.selectedMetric} + + +
+ + {#if storeState.dataPoints} + {storeState.dataPoints.total} data + {storeState.dataPoints.total === 1 ? "point" : "points"} + {:else} + Data Points + {/if} + +
+ + + {#if storeState.dataPointsLoading} +
+
+
+ {:else if storeState.dataPoints && storeState.dataPoints.metrics.length > 0} +
+ + + + Time + Value + Type + Service + Attributes + Exemplar + + + + {#each storeState.dataPoints.metrics as point} + + + {formatDateTime(point.time)} + + + {typeof point.value === "number" + ? point.value.toFixed(4) + : point.value} + + + {point.metricType} + + + {point.serviceName} + + + {truncateJson(point.attributes)} + + + {#if point.hasExemplars && point.exemplars?.length} + {#each point.exemplars.filter((e) => e.traceId) as exemplar} + + {/each} + {#if !point.exemplars.some((e) => e.traceId)} + - + {/if} + {:else} + - + {/if} + + + {/each} + +
+
+ + {#if storeState.dataPoints.hasMore} +
+

+ Showing {storeState.dataPoints.metrics.length} of {storeState + .dataPoints.total} data points +

+
+ {/if} + {:else} +
+ +

No data points found for the selected metric and time range

+
+ {/if} + + + {/if} +
diff --git a/packages/frontend/src/routes/dashboard/metrics/+page.ts b/packages/frontend/src/routes/dashboard/metrics/+page.ts new file mode 100644 index 00000000..a3d15781 --- /dev/null +++ b/packages/frontend/src/routes/dashboard/metrics/+page.ts @@ -0,0 +1 @@ +export const ssr = false; diff --git a/packages/reservoir/src/client.ts b/packages/reservoir/src/client.ts index 835e5e5a..41bf799e 100644 --- a/packages/reservoir/src/client.ts +++ b/packages/reservoir/src/client.ts @@ -30,6 +30,17 @@ import type { IngestSpansResult, ServiceDependencyResult, DeleteSpansByTimeRangeParams, + MetricRecord, + MetricQueryParams, + MetricQueryResult, + MetricAggregateParams, + MetricAggregateResult, + MetricNamesParams, + MetricNamesResult, + MetricLabelParams, + MetricLabelResult, + IngestMetricsResult, + DeleteMetricsByTimeRangeParams, } from './core/types.js'; import type { StorageEngine } from './core/storage-engine.js'; import { StorageEngineFactory } from './factory.js'; @@ -192,6 +203,45 @@ export class Reservoir { return this.engine.deleteSpansByTimeRange(params); } + // ========================================================================= + // Metric Operations + // ========================================================================= + + async ingestMetrics(metrics: MetricRecord[]): Promise { + this.ensureInitialized(); + return this.engine.ingestMetrics(metrics); + } + + async queryMetrics(params: MetricQueryParams): Promise { + this.ensureInitialized(); + return this.engine.queryMetrics(params); + } + + async aggregateMetrics(params: MetricAggregateParams): Promise { + this.ensureInitialized(); + return this.engine.aggregateMetrics(params); + } + + async getMetricNames(params: MetricNamesParams): Promise { + this.ensureInitialized(); + return this.engine.getMetricNames(params); + } + + async getMetricLabelKeys(params: MetricLabelParams): Promise { + this.ensureInitialized(); + return this.engine.getMetricLabelKeys(params); + } + + async getMetricLabelValues(params: MetricLabelParams, labelKey: string): Promise { + this.ensureInitialized(); + return this.engine.getMetricLabelValues(params, labelKey); + } + + async deleteMetricsByTimeRange(params: DeleteMetricsByTimeRangeParams): Promise { + this.ensureInitialized(); + return this.engine.deleteMetricsByTimeRange(params); + } + getEngineType(): EngineType { return this.engine.getCapabilities().engine; } diff --git a/packages/reservoir/src/core/storage-engine.ts b/packages/reservoir/src/core/storage-engine.ts index a6ba9298..afdcc109 100644 --- a/packages/reservoir/src/core/storage-engine.ts +++ b/packages/reservoir/src/core/storage-engine.ts @@ -30,6 +30,17 @@ import type { IngestSpansResult, ServiceDependencyResult, DeleteSpansByTimeRangeParams, + MetricRecord, + MetricQueryParams, + MetricQueryResult, + MetricAggregateParams, + MetricAggregateResult, + MetricNamesParams, + MetricNamesResult, + MetricLabelParams, + MetricLabelResult, + IngestMetricsResult, + DeleteMetricsByTimeRangeParams, } from './types.js'; /** @@ -127,4 +138,29 @@ export abstract class StorageEngine { /** Delete spans by time range */ abstract deleteSpansByTimeRange(params: DeleteSpansByTimeRangeParams): Promise; + + // ========================================================================= + // Metric Operations + // ========================================================================= + + /** Ingest a batch of metric data points */ + abstract ingestMetrics(metrics: MetricRecord[]): Promise; + + /** Query raw metric data points */ + abstract queryMetrics(params: MetricQueryParams): Promise; + + /** Aggregate metrics into time buckets */ + abstract aggregateMetrics(params: MetricAggregateParams): Promise; + + /** List distinct metric names for a project */ + abstract getMetricNames(params: MetricNamesParams): Promise; + + /** Get distinct label keys for a specific metric */ + abstract getMetricLabelKeys(params: MetricLabelParams): Promise; + + /** Get distinct label values for a specific metric + label key */ + abstract getMetricLabelValues(params: MetricLabelParams, labelKey: string): Promise; + + /** Delete metrics by time range */ + abstract deleteMetricsByTimeRange(params: DeleteMetricsByTimeRangeParams): Promise; } diff --git a/packages/reservoir/src/core/types.ts b/packages/reservoir/src/core/types.ts index 685ce448..18224f4b 100644 --- a/packages/reservoir/src/core/types.ts +++ b/packages/reservoir/src/core/types.ts @@ -402,3 +402,172 @@ export interface DeleteSpansByTimeRangeParams { to: Date; serviceName?: string | string[]; } + +// ============================================================================ +// Metric Types +// ============================================================================ + +/** OTLP metric types */ +export type MetricType = 'gauge' | 'sum' | 'histogram' | 'exp_histogram' | 'summary'; + +/** Histogram/summary bucket data stored in JSONB */ +export interface HistogramData { + sum?: number; + count?: number; + min?: number; + max?: number; + /** Histogram: bucket counts per explicit bound */ + bucket_counts?: number[]; + explicit_bounds?: number[]; + /** ExponentialHistogram fields */ + scale?: number; + zero_count?: number; + positive?: { offset: number; bucket_counts: number[] }; + negative?: { offset: number; bucket_counts: number[] }; + /** Summary: quantile values */ + quantile_values?: Array<{ quantile: number; value: number }>; +} + +/** A single metric exemplar with trace correlation */ +export interface MetricExemplar { + exemplarValue: number; + exemplarTime?: Date; + traceId?: string; + spanId?: string; + attributes?: Record; +} + +/** A metric data point record for ingestion */ +export interface MetricRecord { + time: Date; + organizationId: string; + projectId: string; + metricName: string; + metricType: MetricType; + value: number; + isMonotonic?: boolean; + serviceName: string; + attributes?: Record; + resourceAttributes?: Record; + histogramData?: HistogramData; + exemplars?: MetricExemplar[]; +} + +/** A stored metric record (includes DB-generated id) */ +export interface StoredMetricRecord extends MetricRecord { + id: string; + hasExemplars: boolean; +} + +/** Parameters for querying raw metric data points */ +export interface MetricQueryParams { + organizationId?: string | string[]; + projectId: string | string[]; + metricName?: string | string[]; + metricType?: MetricType | MetricType[]; + serviceName?: string | string[]; + from: Date; + to: Date; + fromExclusive?: boolean; + toExclusive?: boolean; + /** Filter by label key-value pairs */ + attributes?: Record; + limit?: number; + offset?: number; + sortOrder?: 'asc' | 'desc'; + includeExemplars?: boolean; +} + +/** Result of a raw metric query */ +export interface MetricQueryResult { + metrics: StoredMetricRecord[]; + total: number; + hasMore: boolean; + limit: number; + offset: number; + executionTimeMs?: number; +} + +/** Aggregation function for metric time-series */ +export type MetricAggregationFn = 'avg' | 'sum' | 'min' | 'max' | 'count' | 'last'; + +/** Parameters for time-series aggregation of metrics */ +export interface MetricAggregateParams { + organizationId?: string | string[]; + projectId: string | string[]; + metricName: string; + metricType?: MetricType; + serviceName?: string | string[]; + from: Date; + to: Date; + interval: AggregationInterval; + aggregation: MetricAggregationFn; + /** Group results by these label keys */ + groupBy?: string[]; + attributes?: Record; +} + +/** A single time bucket in a metric aggregation result */ +export interface MetricTimeBucket { + bucket: Date; + value: number; + labels?: Record; +} + +/** Result of a metric aggregation query */ +export interface MetricAggregateResult { + metricName: string; + metricType: MetricType; + timeseries: MetricTimeBucket[]; + executionTimeMs?: number; +} + +/** Parameters for listing distinct metric names */ +export interface MetricNamesParams { + organizationId?: string | string[]; + projectId: string | string[]; + metricType?: MetricType | MetricType[]; + from?: Date; + to?: Date; + limit?: number; +} + +/** Result of metric name listing */ +export interface MetricNamesResult { + names: Array<{ name: string; type: MetricType }>; + executionTimeMs?: number; +} + +/** Parameters for label key/value discovery */ +export interface MetricLabelParams { + organizationId?: string | string[]; + projectId: string | string[]; + metricName: string; + from?: Date; + to?: Date; + limit?: number; +} + +/** Result of metric label query */ +export interface MetricLabelResult { + keys?: string[]; + values?: string[]; + executionTimeMs?: number; +} + +/** Result of batch metric ingestion */ +export interface IngestMetricsResult { + ingested: number; + failed: number; + durationMs: number; + errors?: Array<{ index: number; error: string }>; +} + +/** Parameters for deleting metrics by time range */ +export interface DeleteMetricsByTimeRangeParams { + projectId: string | string[]; + from: Date; + to: Date; + metricName?: string | string[]; + serviceName?: string | string[]; +} diff --git a/packages/reservoir/src/engines/clickhouse/clickhouse-engine.ts b/packages/reservoir/src/engines/clickhouse/clickhouse-engine.ts index cd478ed1..715968e5 100644 --- a/packages/reservoir/src/engines/clickhouse/clickhouse-engine.ts +++ b/packages/reservoir/src/engines/clickhouse/clickhouse-engine.ts @@ -38,6 +38,21 @@ import type { DeleteSpansByTimeRangeParams, SpanKind, SpanStatusCode, + AggregationInterval, + MetricRecord, + StoredMetricRecord, + MetricType, + MetricQueryParams, + MetricQueryResult, + MetricAggregateParams, + MetricAggregateResult, + MetricNamesParams, + MetricNamesResult, + MetricLabelParams, + MetricLabelResult, + IngestMetricsResult, + DeleteMetricsByTimeRangeParams, + MetricExemplar, } from '../../core/types.js'; import { ClickHouseQueryTranslator } from './query-translator.js'; @@ -230,6 +245,52 @@ export class ClickHouseEngine extends StorageEngine { SETTINGS index_granularity = 8192 `, }); + + // Metrics table + await client.command({ + query: ` + CREATE TABLE IF NOT EXISTS metrics ( + time DateTime64(3) NOT NULL, + id UUID DEFAULT generateUUIDv4(), + organization_id Nullable(String) DEFAULT NULL, + project_id String NOT NULL, + metric_name LowCardinality(String) NOT NULL, + metric_type LowCardinality(String) NOT NULL, + value Float64 NOT NULL DEFAULT 0, + is_monotonic Nullable(UInt8) DEFAULT NULL, + service_name LowCardinality(String) NOT NULL DEFAULT 'unknown', + attributes String DEFAULT '{}', + resource_attributes String DEFAULT '{}', + histogram_data Nullable(String) DEFAULT NULL, + has_exemplars UInt8 NOT NULL DEFAULT 0 + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(time) + ORDER BY (project_id, metric_name, time) + SETTINGS index_granularity = 8192 + `, + }); + + // Metric exemplars table + await client.command({ + query: ` + CREATE TABLE IF NOT EXISTS metric_exemplars ( + time DateTime64(3) NOT NULL, + id UUID DEFAULT generateUUIDv4(), + metric_id String NOT NULL, + project_id String NOT NULL, + exemplar_value Float64 NOT NULL, + exemplar_time Nullable(DateTime64(3)) DEFAULT NULL, + trace_id Nullable(String) DEFAULT NULL, + span_id Nullable(String) DEFAULT NULL, + attributes String DEFAULT '{}' + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(time) + ORDER BY (project_id, metric_id, time) + SETTINGS index_granularity = 8192 + `, + }); } async migrate(_version: string): Promise { @@ -854,6 +915,519 @@ export class ClickHouseEngine extends StorageEngine { return { deleted: 0, executionTimeMs: Date.now() - start }; } + + // ========================================================================= + // Metric Operations + // ========================================================================= + + async ingestMetrics(metrics: MetricRecord[]): Promise { + if (metrics.length === 0) return { ingested: 0, failed: 0, durationMs: 0 }; + + const start = Date.now(); + const client = this.getClient(); + + try { + const metricRows: Record[] = []; + const exemplarRows: Record[] = []; + + for (const metric of metrics) { + const metricId = randomUUID(); + const hasExemplars = (metric.exemplars?.length ?? 0) > 0; + + metricRows.push({ + time: metric.time.getTime(), + id: metricId, + organization_id: metric.organizationId ?? null, + project_id: metric.projectId, + metric_name: metric.metricName, + metric_type: metric.metricType, + value: metric.value, + is_monotonic: metric.isMonotonic != null ? (metric.isMonotonic ? 1 : 0) : null, + service_name: metric.serviceName || 'unknown', + attributes: metric.attributes ? JSON.stringify(metric.attributes) : '{}', + resource_attributes: metric.resourceAttributes ? JSON.stringify(metric.resourceAttributes) : '{}', + histogram_data: metric.histogramData ? JSON.stringify(metric.histogramData) : null, + has_exemplars: hasExemplars ? 1 : 0, + }); + + if (hasExemplars && metric.exemplars) { + for (const ex of metric.exemplars) { + exemplarRows.push({ + time: metric.time.getTime(), + metric_id: metricId, + project_id: metric.projectId, + exemplar_value: ex.exemplarValue, + exemplar_time: ex.exemplarTime ? ex.exemplarTime.getTime() : null, + trace_id: ex.traceId ?? null, + span_id: ex.spanId ?? null, + attributes: ex.attributes ? JSON.stringify(ex.attributes) : '{}', + }); + } + } + } + + await client.insert({ table: 'metrics', values: metricRows, format: 'JSONEachRow' }); + + if (exemplarRows.length > 0) { + await client.insert({ table: 'metric_exemplars', values: exemplarRows, format: 'JSONEachRow' }); + } + + return { ingested: metrics.length, failed: 0, durationMs: Date.now() - start }; + } catch (err) { + return { + ingested: 0, + failed: metrics.length, + durationMs: Date.now() - start, + errors: [{ index: 0, error: err instanceof Error ? err.message : String(err) }], + }; + } + } + + async queryMetrics(params: MetricQueryParams): Promise { + const start = Date.now(); + const client = this.getClient(); + const limit = params.limit ?? 50; + const offset = params.offset ?? 0; + + const conditions: string[] = []; + const queryParams: Record = {}; + + // Project ID (required) + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id IN {p_pids:Array(String)}`); + queryParams.p_pids = pids; + + // Time range + conditions.push(`time ${params.fromExclusive ? '>' : '>='} {p_from:DateTime64(3)}`); + queryParams.p_from = Math.floor(params.from.getTime() / 1000); + conditions.push(`time ${params.toExclusive ? '<' : '<='} {p_to:DateTime64(3)}`); + queryParams.p_to = Math.floor(params.to.getTime() / 1000); + + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id IN {p_oids:Array(String)}`); + queryParams.p_oids = oids; + } + if (params.metricName) { + const names = Array.isArray(params.metricName) ? params.metricName : [params.metricName]; + conditions.push(`metric_name IN {p_names:Array(String)}`); + queryParams.p_names = names; + } + if (params.metricType) { + const types = Array.isArray(params.metricType) ? params.metricType : [params.metricType]; + conditions.push(`metric_type IN {p_types:Array(String)}`); + queryParams.p_types = types; + } + if (params.serviceName) { + const svc = Array.isArray(params.serviceName) ? params.serviceName : [params.serviceName]; + conditions.push(`service_name IN {p_svc:Array(String)}`); + queryParams.p_svc = svc; + } + + // Attribute label filtering + if (params.attributes) { + let attrIdx = 0; + for (const [key, val] of Object.entries(params.attributes)) { + const keyParam = `p_attr_key_${attrIdx}`; + const valParam = `p_attr_val_${attrIdx}`; + conditions.push(`JSONExtractString(attributes, {${keyParam}:String}) = {${valParam}:String}`); + queryParams[keyParam] = key; + queryParams[valParam] = val; + attrIdx++; + } + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + const sortOrder = params.sortOrder ?? 'desc'; + + // Count total + const countResult = await client.query({ + query: `SELECT count() AS count FROM metrics ${where}`, + query_params: queryParams, + format: 'JSONEachRow', + }); + const total = Number((await countResult.json<{ count: string }>())[0]?.count ?? 0); + + // Fetch rows + const resultSet = await client.query({ + query: `SELECT * FROM metrics ${where} ORDER BY time ${sortOrder} LIMIT ${limit} OFFSET ${offset}`, + query_params: queryParams, + format: 'JSONEachRow', + }); + const rows = await resultSet.json>(); + + let metricsResult = rows.map(mapClickHouseRowToMetricRecord); + + // Fetch exemplars if requested + if (params.includeExemplars) { + const metricIds = metricsResult.filter(m => m.hasExemplars).map(m => m.id); + if (metricIds.length > 0) { + const exemplarResult = await client.query({ + query: `SELECT * FROM metric_exemplars WHERE metric_id IN {p_mids:Array(String)} ORDER BY time ASC`, + query_params: { p_mids: metricIds }, + format: 'JSONEachRow', + }); + const exemplarRows = await exemplarResult.json>(); + + const exemplarsByMetricId = new Map(); + for (const row of exemplarRows) { + const metricId = String(row.metric_id); + if (!exemplarsByMetricId.has(metricId)) { + exemplarsByMetricId.set(metricId, []); + } + exemplarsByMetricId.get(metricId)!.push({ + exemplarValue: Number(row.exemplar_value), + exemplarTime: row.exemplar_time ? parseClickHouseTime(row.exemplar_time) : undefined, + traceId: row.trace_id ? String(row.trace_id) : undefined, + spanId: row.span_id ? String(row.span_id) : undefined, + attributes: parseJsonField(row.attributes) as Record | undefined, + }); + } + + metricsResult = metricsResult.map(m => ({ + ...m, + exemplars: exemplarsByMetricId.get(m.id) ?? m.exemplars, + })); + } + } + + return { + metrics: metricsResult, + total, + hasMore: offset + rows.length < total, + limit, + offset, + executionTimeMs: Date.now() - start, + }; + } + + async aggregateMetrics(params: MetricAggregateParams): Promise { + const start = Date.now(); + const client = this.getClient(); + + const intervalMap: Record = { + '1m': '1 MINUTE', + '5m': '5 MINUTE', + '15m': '15 MINUTE', + '1h': '1 HOUR', + '6h': '6 HOUR', + '1d': '1 DAY', + '1w': '1 WEEK', + }; + const interval = intervalMap[params.interval]; + + const aggFnMap: Record = { + avg: 'avg(value)', + sum: 'sum(value)', + min: 'min(value)', + max: 'max(value)', + count: 'count()', + last: 'argMax(value, time)', + }; + const aggExpr = aggFnMap[params.aggregation] ?? 'avg(value)'; + + const conditions: string[] = []; + const queryParams: Record = {}; + + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id IN {p_pids:Array(String)}`); + queryParams.p_pids = pids; + + conditions.push(`time >= {p_from:DateTime64(3)}`); + queryParams.p_from = Math.floor(params.from.getTime() / 1000); + conditions.push(`time <= {p_to:DateTime64(3)}`); + queryParams.p_to = Math.floor(params.to.getTime() / 1000); + + conditions.push(`metric_name = {p_name:String}`); + queryParams.p_name = params.metricName; + + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id IN {p_oids:Array(String)}`); + queryParams.p_oids = oids; + } + if (params.metricType) { + conditions.push(`metric_type = {p_type:String}`); + queryParams.p_type = params.metricType; + } + if (params.serviceName) { + const svc = Array.isArray(params.serviceName) ? params.serviceName : [params.serviceName]; + conditions.push(`service_name IN {p_svc:Array(String)}`); + queryParams.p_svc = svc; + } + + // Attribute label filtering + if (params.attributes) { + let attrIdx = 0; + for (const [key, val] of Object.entries(params.attributes)) { + const keyParam = `p_attr_key_${attrIdx}`; + const valParam = `p_attr_val_${attrIdx}`; + conditions.push(`JSONExtractString(attributes, {${keyParam}:String}) = {${valParam}:String}`); + queryParams[keyParam] = key; + queryParams[valParam] = val; + attrIdx++; + } + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + + // Build GROUP BY columns for groupBy label keys + const groupByColumns = ['bucket']; + const selectExtra: string[] = []; + if (params.groupBy && params.groupBy.length > 0) { + for (let i = 0; i < params.groupBy.length; i++) { + const labelKey = params.groupBy[i]; + const alias = `label_${i}`; + const keyParam = `p_gb_key_${i}`; + selectExtra.push(`JSONExtractString(attributes, {${keyParam}:String}) AS ${alias}`); + queryParams[keyParam] = labelKey; + groupByColumns.push(alias); + } + } + + const selectCols = [ + `toStartOfInterval(time, INTERVAL ${interval}) AS bucket`, + `${aggExpr} AS agg_value`, + ...selectExtra, + ].join(', '); + + const query = `SELECT ${selectCols} FROM metrics ${where} GROUP BY ${groupByColumns.join(', ')} ORDER BY bucket ASC`; + + const resultSet = await client.query({ + query, + query_params: queryParams, + format: 'JSONEachRow', + }); + const rows = await resultSet.json>(); + + const timeseries = rows.map(row => { + const bucket: { bucket: Date; value: number; labels?: Record } = { + bucket: parseClickHouseTime(row.bucket), + value: Number(row.agg_value), + }; + + if (params.groupBy && params.groupBy.length > 0) { + const labels: Record = {}; + for (let i = 0; i < params.groupBy.length; i++) { + labels[params.groupBy[i]] = String(row[`label_${i}`] ?? ''); + } + bucket.labels = labels; + } + + return bucket; + }); + + // Determine metricType: use param or query DB + let metricType: MetricType = params.metricType ?? 'gauge'; + if (!params.metricType) { + const typeResult = await client.query({ + query: `SELECT metric_type FROM metrics WHERE metric_name = {p_name:String} AND project_id IN {p_pids:Array(String)} LIMIT 1`, + query_params: { p_name: params.metricName, p_pids: pids }, + format: 'JSONEachRow', + }); + const typeRows = await typeResult.json<{ metric_type: string }>(); + if (typeRows.length > 0) { + metricType = typeRows[0].metric_type as MetricType; + } + } + + return { + metricName: params.metricName, + metricType, + timeseries, + executionTimeMs: Date.now() - start, + }; + } + + async getMetricNames(params: MetricNamesParams): Promise { + const start = Date.now(); + const client = this.getClient(); + const limit = params.limit ?? 1000; + + const conditions: string[] = []; + const queryParams: Record = {}; + + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id IN {p_pids:Array(String)}`); + queryParams.p_pids = pids; + + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id IN {p_oids:Array(String)}`); + queryParams.p_oids = oids; + } + if (params.metricType) { + const types = Array.isArray(params.metricType) ? params.metricType : [params.metricType]; + conditions.push(`metric_type IN {p_types:Array(String)}`); + queryParams.p_types = types; + } + if (params.from) { + conditions.push(`time >= {p_from:DateTime64(3)}`); + queryParams.p_from = Math.floor(params.from.getTime() / 1000); + } + if (params.to) { + conditions.push(`time <= {p_to:DateTime64(3)}`); + queryParams.p_to = Math.floor(params.to.getTime() / 1000); + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + + const resultSet = await client.query({ + query: `SELECT metric_name, metric_type FROM metrics ${where} GROUP BY metric_name, metric_type ORDER BY metric_name ASC LIMIT ${limit}`, + query_params: queryParams, + format: 'JSONEachRow', + }); + const rows = await resultSet.json<{ metric_name: string; metric_type: string }>(); + + return { + names: rows.map(row => ({ + name: row.metric_name, + type: row.metric_type as MetricType, + })), + executionTimeMs: Date.now() - start, + }; + } + + async getMetricLabelKeys(params: MetricLabelParams): Promise { + const start = Date.now(); + const client = this.getClient(); + const limit = params.limit ?? 100; + + const conditions: string[] = []; + const queryParams: Record = {}; + + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id IN {p_pids:Array(String)}`); + queryParams.p_pids = pids; + + conditions.push(`metric_name = {p_name:String}`); + queryParams.p_name = params.metricName; + + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id IN {p_oids:Array(String)}`); + queryParams.p_oids = oids; + } + if (params.from) { + conditions.push(`time >= {p_from:DateTime64(3)}`); + queryParams.p_from = Math.floor(params.from.getTime() / 1000); + } + if (params.to) { + conditions.push(`time <= {p_to:DateTime64(3)}`); + queryParams.p_to = Math.floor(params.to.getTime() / 1000); + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + + // ClickHouse has no native JSONB keys function, so we sample rows + // and extract keys client-side using JSONExtractKeys + const resultSet = await client.query({ + query: `SELECT DISTINCT arrayJoin(JSONExtractKeys(attributes)) AS key FROM metrics ${where} LIMIT ${limit}`, + query_params: queryParams, + format: 'JSONEachRow', + }); + const rows = await resultSet.json<{ key: string }>(); + + return { + keys: rows.map(r => r.key), + executionTimeMs: Date.now() - start, + }; + } + + async getMetricLabelValues(params: MetricLabelParams, labelKey: string): Promise { + const start = Date.now(); + const client = this.getClient(); + const limit = params.limit ?? 100; + + const conditions: string[] = []; + const queryParams: Record = {}; + + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id IN {p_pids:Array(String)}`); + queryParams.p_pids = pids; + + conditions.push(`metric_name = {p_name:String}`); + queryParams.p_name = params.metricName; + + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id IN {p_oids:Array(String)}`); + queryParams.p_oids = oids; + } + if (params.from) { + conditions.push(`time >= {p_from:DateTime64(3)}`); + queryParams.p_from = Math.floor(params.from.getTime() / 1000); + } + if (params.to) { + conditions.push(`time <= {p_to:DateTime64(3)}`); + queryParams.p_to = Math.floor(params.to.getTime() / 1000); + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + + const resultSet = await client.query({ + query: `SELECT DISTINCT JSONExtractString(attributes, {p_label_key:String}) AS val FROM metrics ${where} HAVING val != '' LIMIT ${limit}`, + query_params: { ...queryParams, p_label_key: labelKey }, + format: 'JSONEachRow', + }); + const rows = await resultSet.json<{ val: string }>(); + + return { + values: rows.map(r => r.val), + executionTimeMs: Date.now() - start, + }; + } + + async deleteMetricsByTimeRange(params: DeleteMetricsByTimeRangeParams): Promise { + const start = Date.now(); + const client = this.getClient(); + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + + const conditions = [ + `project_id IN {p_pids:Array(String)}`, + `time >= {p_from:DateTime64(3)}`, + `time <= {p_to:DateTime64(3)}`, + ]; + const queryParams: Record = { + p_pids: pids, + p_from: Math.floor(params.from.getTime() / 1000), + p_to: Math.floor(params.to.getTime() / 1000), + }; + + if (params.metricName) { + const names = Array.isArray(params.metricName) ? params.metricName : [params.metricName]; + conditions.push(`metric_name IN {p_names:Array(String)}`); + queryParams.p_names = names; + } + if (params.serviceName) { + const svc = Array.isArray(params.serviceName) ? params.serviceName : [params.serviceName]; + conditions.push(`service_name IN {p_svc:Array(String)}`); + queryParams.p_svc = svc; + } + + // Delete from metrics table (async mutation) + await client.command({ + query: `ALTER TABLE metrics DELETE WHERE ${conditions.join(' AND ')}`, + query_params: queryParams, + }); + + // Also delete exemplars for the same time range / project + const exemplarConditions = [ + `project_id IN {p_pids:Array(String)}`, + `time >= {p_from:DateTime64(3)}`, + `time <= {p_to:DateTime64(3)}`, + ]; + await client.command({ + query: `ALTER TABLE metric_exemplars DELETE WHERE ${exemplarConditions.join(' AND ')}`, + query_params: { + p_pids: pids, + p_from: Math.floor(params.from.getTime() / 1000), + p_to: Math.floor(params.to.getTime() / 1000), + }, + }); + + return { deleted: 0, executionTimeMs: Date.now() - start }; + } } function parseClickHouseTime(value: unknown): Date { @@ -951,3 +1525,30 @@ function mapClickHouseRowToTraceRecord(row: Record): TraceRecor error: !!Number(row.error), }; } + +function mapClickHouseRowToMetricRecord(row: Record): StoredMetricRecord { + let histogramData: MetricRecord['histogramData']; + if (row.histogram_data && typeof row.histogram_data === 'string' && row.histogram_data !== 'null') { + try { + histogramData = JSON.parse(row.histogram_data as string); + } catch { + histogramData = undefined; + } + } + + return { + id: String(row.id), + time: parseClickHouseTime(row.time), + organizationId: row.organization_id ? String(row.organization_id) : '', + projectId: String(row.project_id), + metricName: String(row.metric_name), + metricType: String(row.metric_type) as MetricType, + value: Number(row.value), + isMonotonic: row.is_monotonic != null ? !!Number(row.is_monotonic) : undefined, + serviceName: String(row.service_name), + attributes: parseJsonField(row.attributes) as Record | undefined, + resourceAttributes: parseJsonField(row.resource_attributes) as Record | undefined, + histogramData, + hasExemplars: !!Number(row.has_exemplars), + }; +} diff --git a/packages/reservoir/src/engines/timescale/timescale-engine.ts b/packages/reservoir/src/engines/timescale/timescale-engine.ts index db0646f5..35357569 100644 --- a/packages/reservoir/src/engines/timescale/timescale-engine.ts +++ b/packages/reservoir/src/engines/timescale/timescale-engine.ts @@ -8,6 +8,7 @@ import type { QueryResult, AggregateParams, AggregateResult, + AggregationInterval, IngestResult, IngestReturningResult, HealthStatus, @@ -37,6 +38,21 @@ import type { DeleteSpansByTimeRangeParams, SpanKind, SpanStatusCode, + MetricRecord, + StoredMetricRecord, + MetricQueryParams, + MetricQueryResult, + MetricAggregateParams, + MetricAggregateResult, + MetricNamesParams, + MetricNamesResult, + MetricLabelParams, + MetricLabelResult, + IngestMetricsResult, + DeleteMetricsByTimeRangeParams, + MetricType, + MetricTimeBucket, + MetricExemplar, } from '../../core/types.js'; import { TimescaleQueryTranslator } from './query-translator.js'; @@ -46,6 +62,16 @@ function sanitizeNull(value: string): string { return value.includes('\0') ? value.replace(/\0/g, '') : value; } +const METRIC_INTERVAL_MAP: Record = { + '1m': '1 minute', + '5m': '5 minutes', + '15m': '15 minutes', + '1h': '1 hour', + '6h': '6 hours', + '1d': '1 day', + '1w': '1 week', +}; + export interface TimescaleEngineOptions { /** Use an existing pg.Pool instead of creating a new one */ pool?: pg.Pool; @@ -802,6 +828,560 @@ export class TimescaleEngine extends StorageEngine { executionTimeMs: Date.now() - start, }; } + + // ========================================================================= + // Metric Operations + // ========================================================================= + + async ingestMetrics(metrics: MetricRecord[]): Promise { + if (metrics.length === 0) { + return { ingested: 0, failed: 0, durationMs: 0 }; + } + + const start = Date.now(); + const pool = this.getPool(); + const s = this.schema; + + const times: Date[] = []; + const orgIds: string[] = []; + const projectIds: string[] = []; + const metricNames: string[] = []; + const metricTypes: string[] = []; + const values: number[] = []; + const isMonotonics: (boolean | null)[] = []; + const serviceNames: string[] = []; + const attributesJsons: (string | null)[] = []; + const resourceAttrsJsons: (string | null)[] = []; + const histogramDataJsons: (string | null)[] = []; + + for (const m of metrics) { + times.push(m.time); + orgIds.push(sanitizeNull(m.organizationId)); + projectIds.push(sanitizeNull(m.projectId)); + metricNames.push(sanitizeNull(m.metricName)); + metricTypes.push(m.metricType); + values.push(m.value); + isMonotonics.push(m.isMonotonic ?? null); + serviceNames.push(sanitizeNull(m.serviceName)); + attributesJsons.push(m.attributes ? JSON.stringify(m.attributes) : null); + resourceAttrsJsons.push(m.resourceAttributes ? JSON.stringify(m.resourceAttributes) : null); + histogramDataJsons.push(m.histogramData ? JSON.stringify(m.histogramData) : null); + } + + // Compute has_exemplars flags + const hasExemplarsFlags: boolean[] = metrics.map(m => (m.exemplars?.length ?? 0) > 0); + + try { + const insertResult = await pool.query( + `INSERT INTO ${s}.metrics ( + time, organization_id, project_id, metric_name, metric_type, + value, is_monotonic, service_name, attributes, resource_attributes, histogram_data, has_exemplars + ) + SELECT * FROM UNNEST( + $1::timestamptz[], $2::uuid[], $3::uuid[], $4::text[], $5::text[], + $6::double precision[], $7::boolean[], $8::text[], $9::jsonb[], $10::jsonb[], $11::jsonb[], $12::boolean[] + ) + RETURNING id, time`, + [times, orgIds, projectIds, metricNames, metricTypes, + values, isMonotonics, serviceNames, attributesJsons, resourceAttrsJsons, histogramDataJsons, hasExemplarsFlags], + ); + + // Insert exemplars if any metrics have them + const exemplarTimes: Date[] = []; + const exemplarMetricIds: string[] = []; + const exemplarProjectIds: string[] = []; + const exemplarValues: number[] = []; + const exemplarTimesReal: (Date | null)[] = []; + const exemplarTraceIds: (string | null)[] = []; + const exemplarSpanIds: (string | null)[] = []; + const exemplarAttrsJsons: (string | null)[] = []; + + for (let i = 0; i < metrics.length; i++) { + const m = metrics[i]; + if (m.exemplars && m.exemplars.length > 0) { + const row = insertResult.rows[i]; + const metricId = row.id as string; + const metricTime = row.time as Date; + + for (const ex of m.exemplars) { + exemplarTimes.push(metricTime); + exemplarMetricIds.push(metricId); + exemplarProjectIds.push(sanitizeNull(m.projectId)); + exemplarValues.push(ex.exemplarValue); + exemplarTimesReal.push(ex.exemplarTime ?? null); + exemplarTraceIds.push(ex.traceId ?? null); + exemplarSpanIds.push(ex.spanId ?? null); + exemplarAttrsJsons.push(ex.attributes ? JSON.stringify(ex.attributes) : null); + } + } + } + + if (exemplarTimes.length > 0) { + await pool.query( + `INSERT INTO ${s}.metric_exemplars ( + time, metric_id, project_id, + exemplar_value, exemplar_time, trace_id, span_id, attributes + ) + SELECT * FROM UNNEST( + $1::timestamptz[], $2::uuid[], $3::uuid[], + $4::double precision[], $5::timestamptz[], $6::text[], $7::text[], $8::jsonb[] + )`, + [exemplarTimes, exemplarMetricIds, exemplarProjectIds, + exemplarValues, exemplarTimesReal, exemplarTraceIds, exemplarSpanIds, exemplarAttrsJsons], + ); + } + + return { ingested: metrics.length, failed: 0, durationMs: Date.now() - start }; + } catch (err) { + return { + ingested: 0, + failed: metrics.length, + durationMs: Date.now() - start, + errors: [{ index: 0, error: err instanceof Error ? err.message : String(err) }], + }; + } + } + + async queryMetrics(params: MetricQueryParams): Promise { + const start = Date.now(); + const pool = this.getPool(); + const s = this.schema; + const limit = params.limit ?? 50; + const offset = params.offset ?? 0; + + const conditions: string[] = []; + const values: unknown[] = []; + let idx = 1; + + // Time range + conditions.push(`m.time ${params.fromExclusive ? '>' : '>='} $${idx++}`); + values.push(params.from); + conditions.push(`m.time ${params.toExclusive ? '<' : '<='} $${idx++}`); + values.push(params.to); + + // Project filter + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`m.project_id = ANY($${idx++})`); + values.push(pids); + + // Optional filters + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`m.organization_id = ANY($${idx++})`); + values.push(oids); + } + if (params.metricName) { + const names = Array.isArray(params.metricName) ? params.metricName : [params.metricName]; + conditions.push(`m.metric_name = ANY($${idx++})`); + values.push(names); + } + if (params.metricType) { + const types = Array.isArray(params.metricType) ? params.metricType : [params.metricType]; + conditions.push(`m.metric_type = ANY($${idx++})`); + values.push(types); + } + if (params.serviceName) { + const svcs = Array.isArray(params.serviceName) ? params.serviceName : [params.serviceName]; + conditions.push(`m.service_name = ANY($${idx++})`); + values.push(svcs); + } + if (params.attributes) { + conditions.push(`m.attributes @> $${idx++}::jsonb`); + values.push(JSON.stringify(params.attributes)); + } + + const where = `WHERE ${conditions.join(' AND ')}`; + const sortOrder = params.sortOrder ?? 'desc'; + + // Count total + const countResult = await pool.query( + `SELECT COUNT(*)::int AS count FROM ${s}.metrics m ${where}`, + values, + ); + const total = countResult.rows[0]?.count ?? 0; + + // Fetch rows + const dataResult = await pool.query( + `SELECT m.* FROM ${s}.metrics m ${where} + ORDER BY m.time ${sortOrder} + LIMIT $${idx++} OFFSET $${idx++}`, + [...values, limit, offset], + ); + + let metricsResult = dataResult.rows.map(mapRowToStoredMetricRecord); + + // Optionally load exemplars + if (params.includeExemplars && metricsResult.length > 0) { + const metricIds = metricsResult.map((m) => m.id); + const exResult = await pool.query( + `SELECT * FROM ${s}.metric_exemplars WHERE metric_id = ANY($1::uuid[])`, + [metricIds], + ); + + const exemplarsByMetricId = new Map(); + for (const row of exResult.rows) { + const mid = row.metric_id as string; + if (!exemplarsByMetricId.has(mid)) { + exemplarsByMetricId.set(mid, []); + } + exemplarsByMetricId.get(mid)!.push({ + exemplarValue: Number(row.exemplar_value), + exemplarTime: row.exemplar_time ? (row.exemplar_time as Date) : undefined, + traceId: row.trace_id as string | undefined, + spanId: row.span_id as string | undefined, + attributes: row.attributes as Record | undefined, + }); + } + + metricsResult = metricsResult.map((m) => ({ + ...m, + exemplars: exemplarsByMetricId.get(m.id) ?? undefined, + hasExemplars: exemplarsByMetricId.has(m.id), + })); + } + + return { + metrics: metricsResult, + total, + hasMore: offset + metricsResult.length < total, + limit, + offset, + executionTimeMs: Date.now() - start, + }; + } + + async aggregateMetrics(params: MetricAggregateParams): Promise { + const start = Date.now(); + const pool = this.getPool(); + const s = this.schema; + + const intervalSql = METRIC_INTERVAL_MAP[params.interval]; + const conditions: string[] = []; + const values: unknown[] = [intervalSql]; + let idx = 2; + + // Time range + conditions.push(`time >= $${idx++}`); + values.push(params.from); + conditions.push(`time <= $${idx++}`); + values.push(params.to); + + // Project filter + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id = ANY($${idx++})`); + values.push(pids); + + // Metric name + conditions.push(`metric_name = $${idx++}`); + values.push(params.metricName); + + // Optional filters + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id = ANY($${idx++})`); + values.push(oids); + } + if (params.metricType) { + conditions.push(`metric_type = $${idx++}`); + values.push(params.metricType); + } + if (params.serviceName) { + const svcs = Array.isArray(params.serviceName) ? params.serviceName : [params.serviceName]; + conditions.push(`service_name = ANY($${idx++})`); + values.push(svcs); + } + if (params.attributes) { + conditions.push(`attributes @> $${idx++}::jsonb`); + values.push(JSON.stringify(params.attributes)); + } + + const where = `WHERE ${conditions.join(' AND ')}`; + + // Build aggregation expression + let aggExpr: string; + switch (params.aggregation) { + case 'avg': + aggExpr = 'AVG(value)'; + break; + case 'sum': + aggExpr = 'SUM(value)'; + break; + case 'min': + aggExpr = 'MIN(value)'; + break; + case 'max': + aggExpr = 'MAX(value)'; + break; + case 'count': + aggExpr = 'COUNT(*)'; + break; + case 'last': + aggExpr = '(array_agg(value ORDER BY time DESC))[1]'; + break; + default: + aggExpr = 'AVG(value)'; + } + + // Build groupBy columns (parameterized to prevent SQL injection) + const groupByColumns: string[] = []; + const selectExtra: string[] = []; + if (params.groupBy && params.groupBy.length > 0) { + for (const key of params.groupBy) { + const alias = `label_${groupByColumns.length}`; + selectExtra.push(`attributes->>$${idx++} AS ${alias}`); + values.push(key); + groupByColumns.push(alias); + } + } + + const selectCols = [ + `time_bucket($1, time) AS bucket`, + `${aggExpr} AS agg_value`, + ...selectExtra, + ].join(', '); + + const groupByCols = ['bucket', ...groupByColumns].join(', '); + + const result = await pool.query( + `SELECT ${selectCols} + FROM ${s}.metrics + ${where} + GROUP BY ${groupByCols} + ORDER BY bucket ASC`, + values, + ); + + const timeseries: MetricTimeBucket[] = result.rows.map((row: Record) => { + const bucket: MetricTimeBucket = { + bucket: row.bucket as Date, + value: Number(row.agg_value), + }; + if (params.groupBy && params.groupBy.length > 0) { + const labels: Record = {}; + for (let i = 0; i < params.groupBy.length; i++) { + labels[params.groupBy[i]] = (row[`label_${i}`] as string) ?? ''; + } + bucket.labels = labels; + } + return bucket; + }); + + return { + metricName: params.metricName, + metricType: params.metricType ?? 'gauge', + timeseries, + executionTimeMs: Date.now() - start, + }; + } + + async getMetricNames(params: MetricNamesParams): Promise { + const start = Date.now(); + const pool = this.getPool(); + const s = this.schema; + + const conditions: string[] = []; + const values: unknown[] = []; + let idx = 1; + + // Project filter + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id = ANY($${idx++})`); + values.push(pids); + + // Optional filters + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id = ANY($${idx++})`); + values.push(oids); + } + if (params.metricType) { + const types = Array.isArray(params.metricType) ? params.metricType : [params.metricType]; + conditions.push(`metric_type = ANY($${idx++})`); + values.push(types); + } + if (params.from) { + conditions.push(`time >= $${idx++}`); + values.push(params.from); + } + if (params.to) { + conditions.push(`time <= $${idx++}`); + values.push(params.to); + } + + const where = `WHERE ${conditions.join(' AND ')}`; + const limitClause = params.limit ? `LIMIT $${idx++}` : ''; + const limitValues = params.limit ? [params.limit] : []; + + const result = await pool.query( + `SELECT DISTINCT metric_name, metric_type + FROM ${s}.metrics + ${where} + ORDER BY metric_name ASC + ${limitClause}`, + [...values, ...limitValues], + ); + + return { + names: result.rows.map((row: Record) => ({ + name: row.metric_name as string, + type: row.metric_type as MetricType, + })), + executionTimeMs: Date.now() - start, + }; + } + + async getMetricLabelKeys(params: MetricLabelParams): Promise { + const start = Date.now(); + const pool = this.getPool(); + const s = this.schema; + + const conditions: string[] = []; + const values: unknown[] = []; + let idx = 1; + + // Project filter + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id = ANY($${idx++})`); + values.push(pids); + + // Metric name + conditions.push(`metric_name = $${idx++}`); + values.push(params.metricName); + + // Optional filters + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id = ANY($${idx++})`); + values.push(oids); + } + if (params.from) { + conditions.push(`time >= $${idx++}`); + values.push(params.from); + } + if (params.to) { + conditions.push(`time <= $${idx++}`); + values.push(params.to); + } + + const where = `WHERE ${conditions.join(' AND ')}`; + const limitClause = params.limit ? `LIMIT $${idx++}` : ''; + const limitValues = params.limit ? [params.limit] : []; + + const result = await pool.query( + `SELECT DISTINCT jsonb_object_keys(attributes) AS key + FROM ${s}.metrics + ${where} AND attributes IS NOT NULL + ORDER BY key ASC + ${limitClause}`, + [...values, ...limitValues], + ); + + return { + keys: result.rows.map((row: Record) => row.key as string), + executionTimeMs: Date.now() - start, + }; + } + + async getMetricLabelValues(params: MetricLabelParams, labelKey: string): Promise { + const start = Date.now(); + const pool = this.getPool(); + const s = this.schema; + + const conditions: string[] = []; + const values: unknown[] = []; + let idx = 1; + + // Project filter + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + conditions.push(`project_id = ANY($${idx++})`); + values.push(pids); + + // Metric name + conditions.push(`metric_name = $${idx++}`); + values.push(params.metricName); + + // Must have the key + conditions.push(`attributes ? $${idx++}`); + values.push(labelKey); + + // Optional filters + if (params.organizationId) { + const oids = Array.isArray(params.organizationId) ? params.organizationId : [params.organizationId]; + conditions.push(`organization_id = ANY($${idx++})`); + values.push(oids); + } + if (params.from) { + conditions.push(`time >= $${idx++}`); + values.push(params.from); + } + if (params.to) { + conditions.push(`time <= $${idx++}`); + values.push(params.to); + } + + const where = `WHERE ${conditions.join(' AND ')}`; + const limitClause = params.limit ? `LIMIT $${idx++}` : ''; + const limitValues = params.limit ? [params.limit] : []; + + const result = await pool.query( + `SELECT DISTINCT attributes->>$${idx++} AS value + FROM ${s}.metrics + ${where} + ORDER BY value ASC + ${limitClause}`, + [...values, ...limitValues, labelKey], + ); + + return { + values: result.rows + .map((row: Record) => row.value as string) + .filter((v) => v != null), + executionTimeMs: Date.now() - start, + }; + } + + async deleteMetricsByTimeRange(params: DeleteMetricsByTimeRangeParams): Promise { + const start = Date.now(); + const pool = this.getPool(); + const s = this.schema; + const pids = Array.isArray(params.projectId) ? params.projectId : [params.projectId]; + + const conditions = ['project_id = ANY($1)', 'time >= $2', 'time <= $3']; + const values: unknown[] = [pids, params.from, params.to]; + let idx = 4; + + if (params.metricName) { + const names = Array.isArray(params.metricName) ? params.metricName : [params.metricName]; + conditions.push(`metric_name = ANY($${idx++})`); + values.push(names); + } + if (params.serviceName) { + const svcs = Array.isArray(params.serviceName) ? params.serviceName : [params.serviceName]; + conditions.push(`service_name = ANY($${idx++})`); + values.push(svcs); + } + + const where = conditions.join(' AND '); + + // Delete exemplars first (they reference metrics) + await pool.query( + `DELETE FROM ${s}.metric_exemplars WHERE metric_id IN ( + SELECT id FROM ${s}.metrics WHERE ${where} + )`, + values, + ); + + // Delete metrics + const result = await pool.query( + `DELETE FROM ${s}.metrics WHERE ${where}`, + values, + ); + + return { + deleted: Number(result.rowCount ?? 0), + executionTimeMs: Date.now() - start, + }; + } } function mapRowToLogRecord(row: Record): LogRecord { @@ -864,3 +1444,21 @@ function mapRowToTraceRecord(row: Record): TraceRecord { error: row.error as boolean, }; } + +function mapRowToStoredMetricRecord(row: Record): StoredMetricRecord { + return { + id: row.id as string, + time: row.time as Date, + organizationId: row.organization_id as string, + projectId: row.project_id as string, + metricName: row.metric_name as string, + metricType: row.metric_type as MetricType, + value: Number(row.value), + isMonotonic: row.is_monotonic as boolean | undefined, + serviceName: row.service_name as string, + attributes: row.attributes as Record | undefined, + resourceAttributes: row.resource_attributes as Record | undefined, + histogramData: row.histogram_data as StoredMetricRecord['histogramData'], + hasExemplars: Boolean(row.has_exemplars), + }; +} diff --git a/packages/reservoir/src/index.ts b/packages/reservoir/src/index.ts index a7d37423..c8dd83ee 100644 --- a/packages/reservoir/src/index.ts +++ b/packages/reservoir/src/index.ts @@ -42,6 +42,23 @@ export type { ServiceDependency, ServiceDependencyResult, DeleteSpansByTimeRangeParams, + MetricType, + HistogramData, + MetricExemplar, + MetricRecord, + StoredMetricRecord, + MetricQueryParams, + MetricQueryResult, + MetricAggregationFn, + MetricAggregateParams, + MetricTimeBucket, + MetricAggregateResult, + MetricNamesParams, + MetricNamesResult, + MetricLabelParams, + MetricLabelResult, + IngestMetricsResult, + DeleteMetricsByTimeRangeParams, } from './core/types.js'; // Core abstractions From 6d58633f5abfa55d9ac2d4d5100753f2c9783222 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 19:02:23 +0100 Subject: [PATCH 05/43] add tests for OTLP metrics ingestion (188 tests) covers metric-transformer, metric-routes, metrics routes/service, timescale and clickhouse engine metric methods --- .../src/tests/modules/metrics/routes.test.ts | 585 +++++++ .../src/tests/modules/metrics/service.test.ts | 351 ++++ .../tests/modules/otlp/metric-routes.test.ts | 583 +++++++ .../modules/otlp/metric-transformer.test.ts | 1450 +++++++++++++++++ .../clickhouse-engine-metrics.test.ts | 782 +++++++++ .../timescale-engine-metrics.test.ts | 860 ++++++++++ 6 files changed, 4611 insertions(+) create mode 100644 packages/backend/src/tests/modules/metrics/routes.test.ts create mode 100644 packages/backend/src/tests/modules/metrics/service.test.ts create mode 100644 packages/backend/src/tests/modules/otlp/metric-routes.test.ts create mode 100644 packages/backend/src/tests/modules/otlp/metric-transformer.test.ts create mode 100644 packages/reservoir/src/engines/clickhouse/clickhouse-engine-metrics.test.ts create mode 100644 packages/reservoir/src/engines/timescale/timescale-engine-metrics.test.ts diff --git a/packages/backend/src/tests/modules/metrics/routes.test.ts b/packages/backend/src/tests/modules/metrics/routes.test.ts new file mode 100644 index 00000000..9d1e7a20 --- /dev/null +++ b/packages/backend/src/tests/modules/metrics/routes.test.ts @@ -0,0 +1,585 @@ +import { describe, it, expect, beforeEach, afterAll } from 'vitest'; +import request from 'supertest'; +import { build } from '../../../server.js'; +import { createTestContext, createTestApiKey } from '../../helpers/index.js'; +import { db } from '../../../database/index.js'; +import crypto from 'crypto'; + +describe('Metrics Routes', () => { + let app: any; + let ctx: Awaited>; + let apiKey: string; + let projectId: string; + let sessionToken: string; + + beforeEach(async () => { + if (!app) { + app = await build(); + await app.ready(); + } + + ctx = await createTestContext(); + apiKey = ctx.apiKey.plainKey; + projectId = ctx.project.id; + + // Create session + const session = await db + .insertInto('sessions') + .values({ + user_id: ctx.user.id, + token: `test-session-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`, + expires_at: new Date(Date.now() + 86400000), + }) + .returningAll() + .executeTakeFirstOrThrow(); + sessionToken = session.token; + + // Ingest some test metrics via OTLP endpoint + await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send({ + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'test-api' } }, + ], + }, + scopeMetrics: [ + { + metrics: [ + { + name: 'http.request.duration', + gauge: { + dataPoints: [ + { + timeUnixNano: String(Date.now() * 1000000), + asDouble: 150.5, + attributes: [ + { key: 'method', value: { stringValue: 'GET' } }, + { key: 'path', value: { stringValue: '/api/users' } }, + ], + }, + ], + }, + }, + { + name: 'http.request.count', + sum: { + dataPoints: [ + { + timeUnixNano: String(Date.now() * 1000000), + asInt: '42', + }, + ], + isMonotonic: true, + }, + }, + ], + }, + ], + }, + ], + }); + }); + + afterAll(async () => { + if (app) await app.close(); + }); + + // ========================================================================== + // GET /api/v1/metrics/names + // ========================================================================== + describe('GET /api/v1/metrics/names', () => { + it('should return metric names with API key auth', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/names') + .set('x-api-key', apiKey) + .query({ projectId }); + + expect(response.status).toBe(200); + expect(response.body).toHaveProperty('names'); + expect(Array.isArray(response.body.names)).toBe(true); + }); + + it('should return metric names with session auth', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/names') + .set('Authorization', `Bearer ${sessionToken}`) + .query({ projectId }); + + expect(response.status).toBe(200); + expect(response.body).toHaveProperty('names'); + expect(Array.isArray(response.body.names)).toBe(true); + }); + + it('should return 400 when projectId is missing (session auth)', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/names') + .set('Authorization', `Bearer ${sessionToken}`); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + }); + + it('should return 401 without auth', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/names') + .query({ projectId }); + + expect(response.status).toBe(401); + }); + + it('should accept optional from/to time range filters', async () => { + const from = new Date(Date.now() - 3600000).toISOString(); + const to = new Date(Date.now() + 3600000).toISOString(); + + const response = await request(app.server) + .get('/api/v1/metrics/names') + .set('x-api-key', apiKey) + .query({ projectId, from, to }); + + expect(response.status).toBe(200); + expect(response.body).toHaveProperty('names'); + expect(Array.isArray(response.body.names)).toBe(true); + }); + + it('should return empty array for project with no metrics', async () => { + // Create a fresh context with no metrics ingested + const freshCtx = await createTestContext(); + + const response = await request(app.server) + .get('/api/v1/metrics/names') + .set('x-api-key', freshCtx.apiKey.plainKey) + .query({ projectId: freshCtx.project.id }); + + expect(response.status).toBe(200); + expect(response.body).toHaveProperty('names'); + expect(Array.isArray(response.body.names)).toBe(true); + }); + }); + + // ========================================================================== + // GET /api/v1/metrics/labels/keys + // ========================================================================== + describe('GET /api/v1/metrics/labels/keys', () => { + it('should return label keys for a metric', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/keys') + .set('x-api-key', apiKey) + .query({ projectId, metricName: 'http.request.duration' }); + + expect(response.status).toBe(200); + expect(response.body).toHaveProperty('keys'); + expect(Array.isArray(response.body.keys)).toBe(true); + }); + + it('should return 400 when projectId is missing', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/keys') + .set('Authorization', `Bearer ${sessionToken}`) + .query({ metricName: 'http.request.duration' }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + }); + + it('should return 400 when metricName is missing', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/keys') + .set('x-api-key', apiKey) + .query({ projectId }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + expect(response.body.error).toContain('metricName'); + }); + + it('should return 401 without auth', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/keys') + .query({ projectId, metricName: 'http.request.duration' }); + + expect(response.status).toBe(401); + }); + }); + + // ========================================================================== + // GET /api/v1/metrics/labels/values + // ========================================================================== + describe('GET /api/v1/metrics/labels/values', () => { + it('should return label values for a metric and label key', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/values') + .set('x-api-key', apiKey) + .query({ + projectId, + metricName: 'http.request.duration', + labelKey: 'method', + }); + + expect(response.status).toBe(200); + expect(response.body).toHaveProperty('values'); + expect(Array.isArray(response.body.values)).toBe(true); + }); + + it('should return 400 when projectId is missing', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/values') + .set('Authorization', `Bearer ${sessionToken}`) + .query({ + metricName: 'http.request.duration', + labelKey: 'method', + }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + }); + + it('should return 400 when metricName is missing', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/values') + .set('x-api-key', apiKey) + .query({ projectId, labelKey: 'method' }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + expect(response.body.error).toContain('metricName'); + }); + + it('should return 400 when labelKey is missing', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/values') + .set('x-api-key', apiKey) + .query({ projectId, metricName: 'http.request.duration' }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + expect(response.body.error).toContain('labelKey'); + }); + + it('should return 401 without auth', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/labels/values') + .query({ + projectId, + metricName: 'http.request.duration', + labelKey: 'method', + }); + + expect(response.status).toBe(401); + }); + }); + + // ========================================================================== + // GET /api/v1/metrics/data + // ========================================================================== + describe('GET /api/v1/metrics/data', () => { + const timeRange = () => ({ + from: new Date(Date.now() - 3600000).toISOString(), + to: new Date(Date.now() + 3600000).toISOString(), + }); + + it('should return metric data points', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/data') + .set('x-api-key', apiKey) + .query({ projectId, from, to }); + + expect(response.status).toBe(200); + // Response should be an object or array depending on implementation + expect(response.body).toBeDefined(); + }); + + it('should return 400 when projectId is missing', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/data') + .set('Authorization', `Bearer ${sessionToken}`) + .query({ from, to }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + }); + + it('should return 400 when from or to is missing', async () => { + // Missing both from and to + const response = await request(app.server) + .get('/api/v1/metrics/data') + .set('x-api-key', apiKey) + .query({ projectId }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + expect(response.body.error).toContain('from'); + }); + + it('should accept pagination (limit, offset)', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/data') + .set('x-api-key', apiKey) + .query({ projectId, from, to, limit: 10, offset: 0 }); + + expect(response.status).toBe(200); + expect(response.body).toBeDefined(); + }); + + it('should accept includeExemplars parameter', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/data') + .set('x-api-key', apiKey) + .query({ projectId, from, to, includeExemplars: true }); + + expect(response.status).toBe(200); + expect(response.body).toBeDefined(); + }); + + it('should return 401 without auth', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/data') + .query({ projectId, from, to }); + + expect(response.status).toBe(401); + }); + + it('should accept metricName filter', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/data') + .set('x-api-key', apiKey) + .query({ + projectId, + from, + to, + metricName: 'http.request.duration', + }); + + expect(response.status).toBe(200); + expect(response.body).toBeDefined(); + }); + + it('should accept attributes filter', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/data') + .set('x-api-key', apiKey) + .query({ + projectId, + from, + to, + 'attributes[method]': 'GET', + }); + + expect(response.status).toBe(200); + expect(response.body).toBeDefined(); + }); + }); + + // ========================================================================== + // GET /api/v1/metrics/aggregate + // ========================================================================== + describe('GET /api/v1/metrics/aggregate', () => { + const timeRange = () => ({ + from: new Date(Date.now() - 3600000).toISOString(), + to: new Date(Date.now() + 3600000).toISOString(), + }); + + it('should return aggregated time series', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/aggregate') + .set('x-api-key', apiKey) + .query({ + projectId, + metricName: 'http.request.duration', + from, + to, + }); + + expect(response.status).toBe(200); + expect(response.body).toBeDefined(); + }); + + it('should return 400 when projectId is missing', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/aggregate') + .set('Authorization', `Bearer ${sessionToken}`) + .query({ + metricName: 'http.request.duration', + from, + to, + }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + }); + + it('should return 400 when metricName is missing', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/aggregate') + .set('x-api-key', apiKey) + .query({ projectId, from, to }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + expect(response.body.error).toContain('metricName'); + }); + + it('should return 400 when from or to is missing', async () => { + const response = await request(app.server) + .get('/api/v1/metrics/aggregate') + .set('x-api-key', apiKey) + .query({ + projectId, + metricName: 'http.request.duration', + }); + + expect(response.status).toBe(400); + expect(response.body).toHaveProperty('error'); + expect(response.body.error).toContain('from'); + }); + + it('should accept interval parameter', async () => { + const { from, to } = timeRange(); + const intervals = ['1m', '5m', '15m', '1h', '6h', '1d', '1w']; + + for (const interval of intervals) { + const response = await request(app.server) + .get('/api/v1/metrics/aggregate') + .set('x-api-key', apiKey) + .query({ + projectId, + metricName: 'http.request.duration', + from, + to, + interval, + }); + + expect(response.status).toBe(200); + } + }); + + it('should accept aggregation parameter', async () => { + const { from, to } = timeRange(); + const aggregations = ['avg', 'sum', 'min', 'max', 'count', 'last']; + + for (const aggregation of aggregations) { + const response = await request(app.server) + .get('/api/v1/metrics/aggregate') + .set('x-api-key', apiKey) + .query({ + projectId, + metricName: 'http.request.duration', + from, + to, + aggregation, + }); + + expect(response.status).toBe(200); + } + }); + + it('should accept groupBy parameter as array', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/aggregate') + .set('x-api-key', apiKey) + .query({ + projectId, + metricName: 'http.request.duration', + from, + to, + 'groupBy[]': 'method', + }); + + // groupBy may be accepted or rejected depending on schema handling + // Just verify it doesn't return a 500 + expect([200, 400]).toContain(response.status); + }); + + it('should return 401 without auth', async () => { + const { from, to } = timeRange(); + + const response = await request(app.server) + .get('/api/v1/metrics/aggregate') + .query({ + projectId, + metricName: 'http.request.duration', + from, + to, + }); + + expect(response.status).toBe(401); + }); + }); + + // ========================================================================== + // Access control + // ========================================================================== + describe('access control', () => { + it('should return 403 for write-only API key', async () => { + const writeKey = await createTestApiKey({ + projectId, + type: 'write', + }); + + const endpoints = [ + { url: '/api/v1/metrics/names', query: { projectId } }, + { + url: '/api/v1/metrics/labels/keys', + query: { projectId, metricName: 'http.request.duration' }, + }, + { + url: '/api/v1/metrics/labels/values', + query: { projectId, metricName: 'http.request.duration', labelKey: 'method' }, + }, + { + url: '/api/v1/metrics/data', + query: { + projectId, + from: new Date(Date.now() - 3600000).toISOString(), + to: new Date(Date.now() + 3600000).toISOString(), + }, + }, + { + url: '/api/v1/metrics/aggregate', + query: { + projectId, + metricName: 'http.request.duration', + from: new Date(Date.now() - 3600000).toISOString(), + to: new Date(Date.now() + 3600000).toISOString(), + }, + }, + ]; + + for (const { url, query } of endpoints) { + const response = await request(app.server) + .get(url) + .set('x-api-key', writeKey.plainKey) + .query(query); + + expect(response.status).toBe(403); + expect(response.body).toHaveProperty('error', 'Forbidden'); + } + }); + }); +}); diff --git a/packages/backend/src/tests/modules/metrics/service.test.ts b/packages/backend/src/tests/modules/metrics/service.test.ts new file mode 100644 index 00000000..86500846 --- /dev/null +++ b/packages/backend/src/tests/modules/metrics/service.test.ts @@ -0,0 +1,351 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockIngestMetrics = vi.fn(); +const mockGetMetricNames = vi.fn(); +const mockGetMetricLabelKeys = vi.fn(); +const mockGetMetricLabelValues = vi.fn(); +const mockQueryMetrics = vi.fn(); +const mockAggregateMetrics = vi.fn(); + +vi.mock('../../../database/reservoir.js', () => ({ + reservoir: { + ingestMetrics: (...args: unknown[]) => mockIngestMetrics(...args), + getMetricNames: (...args: unknown[]) => mockGetMetricNames(...args), + getMetricLabelKeys: (...args: unknown[]) => mockGetMetricLabelKeys(...args), + getMetricLabelValues: (...args: unknown[]) => mockGetMetricLabelValues(...args), + queryMetrics: (...args: unknown[]) => mockQueryMetrics(...args), + aggregateMetrics: (...args: unknown[]) => mockAggregateMetrics(...args), + }, +})); + +import { MetricsService } from '../../../modules/metrics/service.js'; + +describe('MetricsService', () => { + let service: MetricsService; + + beforeEach(() => { + service = new MetricsService(); + vi.clearAllMocks(); + }); + + describe('ingestMetrics', () => { + it('should return 0 for empty records array without calling reservoir', async () => { + const result = await service.ingestMetrics([], 'proj-1', 'org-1'); + + expect(result).toBe(0); + expect(mockIngestMetrics).not.toHaveBeenCalled(); + }); + + it('should enrich records with projectId and organizationId', async () => { + mockIngestMetrics.mockResolvedValueOnce({ ingested: 2 }); + + const records = [ + { + time: new Date('2025-01-01T00:00:00Z'), + metricName: 'http_requests_total', + metricType: 'sum' as const, + value: 42, + serviceName: 'api-gateway', + organizationId: '', + projectId: '', + }, + { + time: new Date('2025-01-01T00:01:00Z'), + metricName: 'cpu_usage', + metricType: 'gauge' as const, + value: 0.75, + serviceName: 'worker', + organizationId: '', + projectId: '', + }, + ]; + + await service.ingestMetrics(records, 'proj-1', 'org-1'); + + expect(mockIngestMetrics).toHaveBeenCalledOnce(); + const enriched = mockIngestMetrics.mock.calls[0][0]; + expect(enriched).toHaveLength(2); + expect(enriched[0]).toEqual(expect.objectContaining({ + projectId: 'proj-1', + organizationId: 'org-1', + metricName: 'http_requests_total', + value: 42, + })); + expect(enriched[1]).toEqual(expect.objectContaining({ + projectId: 'proj-1', + organizationId: 'org-1', + metricName: 'cpu_usage', + value: 0.75, + })); + }); + + it('should return ingested count from reservoir result', async () => { + mockIngestMetrics.mockResolvedValueOnce({ ingested: 5 }); + + const records = [ + { + time: new Date('2025-01-01T00:00:00Z'), + metricName: 'requests', + metricType: 'sum' as const, + value: 1, + serviceName: 'api', + organizationId: '', + projectId: '', + }, + ]; + + const result = await service.ingestMetrics(records, 'proj-1', 'org-1'); + + expect(result).toBe(5); + }); + }); + + describe('listMetricNames', () => { + it('should delegate to reservoir.getMetricNames with correct params', async () => { + mockGetMetricNames.mockResolvedValueOnce(['http_requests_total', 'cpu_usage']); + + const result = await service.listMetricNames('proj-1'); + + expect(mockGetMetricNames).toHaveBeenCalledWith({ + projectId: 'proj-1', + from: undefined, + to: undefined, + }); + expect(result).toEqual(['http_requests_total', 'cpu_usage']); + }); + + it('should pass optional from/to dates', async () => { + mockGetMetricNames.mockResolvedValueOnce(['cpu_usage']); + + const from = new Date('2025-01-01T00:00:00Z'); + const to = new Date('2025-01-02T00:00:00Z'); + + const result = await service.listMetricNames('proj-1', from, to); + + expect(mockGetMetricNames).toHaveBeenCalledWith({ + projectId: 'proj-1', + from, + to, + }); + expect(result).toEqual(['cpu_usage']); + }); + }); + + describe('getLabelKeys', () => { + it('should delegate to reservoir.getMetricLabelKeys with correct params', async () => { + mockGetMetricLabelKeys.mockResolvedValueOnce(['host', 'method', 'status']); + + const from = new Date('2025-01-01T00:00:00Z'); + const to = new Date('2025-01-02T00:00:00Z'); + + const result = await service.getLabelKeys('proj-1', 'http_requests_total', from, to); + + expect(mockGetMetricLabelKeys).toHaveBeenCalledWith({ + projectId: 'proj-1', + metricName: 'http_requests_total', + from, + to, + }); + expect(result).toEqual(['host', 'method', 'status']); + }); + }); + + describe('getLabelValues', () => { + it('should delegate to reservoir.getMetricLabelValues with correct params including labelKey', async () => { + mockGetMetricLabelValues.mockResolvedValueOnce(['GET', 'POST', 'PUT']); + + const from = new Date('2025-01-01T00:00:00Z'); + const to = new Date('2025-01-02T00:00:00Z'); + + const result = await service.getLabelValues( + 'proj-1', + 'http_requests_total', + 'method', + from, + to, + ); + + expect(mockGetMetricLabelValues).toHaveBeenCalledWith( + { + projectId: 'proj-1', + metricName: 'http_requests_total', + from, + to, + }, + 'method', + ); + expect(result).toEqual(['GET', 'POST', 'PUT']); + }); + }); + + describe('queryMetrics', () => { + it('should delegate to reservoir.queryMetrics with all params', async () => { + const mockResult = { + metrics: [ + { id: 'm-1', metricName: 'cpu_usage', value: 0.8, time: new Date() }, + ], + total: 1, + hasMore: false, + limit: 100, + offset: 0, + }; + mockQueryMetrics.mockResolvedValueOnce(mockResult); + + const from = new Date('2025-01-01T00:00:00Z'); + const to = new Date('2025-01-02T00:00:00Z'); + + const result = await service.queryMetrics({ + projectId: 'proj-1', + metricName: 'cpu_usage', + from, + to, + limit: 100, + offset: 0, + }); + + expect(mockQueryMetrics).toHaveBeenCalledWith({ + projectId: 'proj-1', + metricName: 'cpu_usage', + from, + to, + attributes: undefined, + limit: 100, + offset: 0, + includeExemplars: undefined, + }); + expect(result).toBe(mockResult); + }); + + it('should pass through optional attributes and includeExemplars', async () => { + const mockResult = { + metrics: [], + total: 0, + hasMore: false, + limit: 50, + offset: 0, + }; + mockQueryMetrics.mockResolvedValueOnce(mockResult); + + const from = new Date('2025-01-01T00:00:00Z'); + const to = new Date('2025-01-02T00:00:00Z'); + + const result = await service.queryMetrics({ + projectId: ['proj-1', 'proj-2'], + metricName: ['cpu_usage', 'memory_usage'], + from, + to, + attributes: { host: 'server-1', region: 'eu-west' }, + limit: 50, + offset: 10, + includeExemplars: true, + }); + + expect(mockQueryMetrics).toHaveBeenCalledWith({ + projectId: ['proj-1', 'proj-2'], + metricName: ['cpu_usage', 'memory_usage'], + from, + to, + attributes: { host: 'server-1', region: 'eu-west' }, + limit: 50, + offset: 10, + includeExemplars: true, + }); + expect(result).toBe(mockResult); + }); + }); + + describe('aggregateMetrics', () => { + it('should delegate to reservoir.aggregateMetrics with all params', async () => { + const mockResult = { + timeseries: [ + { time: new Date('2025-01-01T00:00:00Z'), value: 42 }, + { time: new Date('2025-01-01T01:00:00Z'), value: 55 }, + ], + }; + mockAggregateMetrics.mockResolvedValueOnce(mockResult); + + const from = new Date('2025-01-01T00:00:00Z'); + const to = new Date('2025-01-02T00:00:00Z'); + + const result = await service.aggregateMetrics({ + projectId: 'proj-1', + metricName: 'http_requests_total', + from, + to, + interval: '1h', + aggregation: 'sum', + }); + + expect(mockAggregateMetrics).toHaveBeenCalledWith({ + projectId: 'proj-1', + metricName: 'http_requests_total', + from, + to, + interval: '1h', + aggregation: 'sum', + groupBy: undefined, + attributes: undefined, + }); + expect(result).toBe(mockResult); + }); + + it('should pass through optional groupBy and attributes', async () => { + const mockResult = { + timeseries: [ + { time: new Date('2025-01-01T00:00:00Z'), value: 10, group: { method: 'GET' } }, + ], + }; + mockAggregateMetrics.mockResolvedValueOnce(mockResult); + + const from = new Date('2025-01-01T00:00:00Z'); + const to = new Date('2025-01-02T00:00:00Z'); + + const result = await service.aggregateMetrics({ + projectId: ['proj-1', 'proj-2'], + metricName: 'http_requests_total', + from, + to, + interval: '5m', + aggregation: 'avg', + groupBy: ['method', 'status'], + attributes: { host: 'server-1' }, + }); + + expect(mockAggregateMetrics).toHaveBeenCalledWith({ + projectId: ['proj-1', 'proj-2'], + metricName: 'http_requests_total', + from, + to, + interval: '5m', + aggregation: 'avg', + groupBy: ['method', 'status'], + attributes: { host: 'server-1' }, + }); + expect(result).toBe(mockResult); + }); + + it('should pass interval and aggregation correctly for all supported values', async () => { + mockAggregateMetrics.mockResolvedValueOnce({ timeseries: [] }); + + const from = new Date('2025-01-01T00:00:00Z'); + const to = new Date('2025-01-08T00:00:00Z'); + + await service.aggregateMetrics({ + projectId: 'proj-1', + metricName: 'memory_usage', + from, + to, + interval: '1d', + aggregation: 'max', + }); + + expect(mockAggregateMetrics).toHaveBeenCalledWith( + expect.objectContaining({ + interval: '1d', + aggregation: 'max', + metricName: 'memory_usage', + }), + ); + }); + }); +}); diff --git a/packages/backend/src/tests/modules/otlp/metric-routes.test.ts b/packages/backend/src/tests/modules/otlp/metric-routes.test.ts new file mode 100644 index 00000000..0d6fa10a --- /dev/null +++ b/packages/backend/src/tests/modules/otlp/metric-routes.test.ts @@ -0,0 +1,583 @@ +import { describe, it, expect, beforeEach, afterAll } from 'vitest'; +import request from 'supertest'; +import { gzipSync } from 'zlib'; +import { build } from '../../../server.js'; +import { createTestApiKey } from '../../helpers/index.js'; + +describe('OTLP Metrics API', () => { + let app: any; + let apiKey: string; + let projectId: string; + + beforeEach(async () => { + if (!app) { + app = await build(); + await app.ready(); + } + + const testKey = await createTestApiKey({ name: 'Test OTLP Metrics Key' }); + apiKey = testKey.plainKey; + projectId = testKey.project_id; + }); + + afterAll(async () => { + if (app) { + await app.close(); + } + }); + + // ========================================================================== + // POST /v1/otlp/metrics - OTLP Metrics Ingestion + // ========================================================================== + describe('POST /v1/otlp/metrics', () => { + it('should ingest a basic gauge metric via JSON', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'test-service' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'cpu.usage', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 0.75, + attributes: [{ key: 'host', value: { stringValue: 'server-1' } }], + }], + }, + }], + }], + }], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body).toHaveProperty('partialSuccess'); + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should ingest a sum metric (counter)', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'counter-service' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'http.requests.total', + sum: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asInt: '42', + attributes: [{ key: 'method', value: { stringValue: 'GET' } }], + }], + aggregationTemporality: 2, // CUMULATIVE + isMonotonic: true, + }, + }], + }], + }], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should ingest a histogram metric with bucketCounts and explicitBounds', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'histogram-service' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'http.request.duration', + histogram: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + count: '100', + sum: 5432.1, + min: 1.2, + max: 890.5, + bucketCounts: ['10', '25', '30', '20', '10', '5'], + explicitBounds: [10, 50, 100, 250, 500], + attributes: [{ key: 'endpoint', value: { stringValue: '/api/users' } }], + }], + aggregationTemporality: 2, + }, + }], + }], + }], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should ingest a summary metric with quantileValues', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'summary-service' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'rpc.server.duration', + summary: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + count: '200', + sum: 15000.0, + quantileValues: [ + { quantile: 0.5, value: 50.0 }, + { quantile: 0.9, value: 120.0 }, + { quantile: 0.99, value: 450.0 }, + ], + }], + }, + }], + }], + }], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should handle multiple resources with different service names', async () => { + const otlpRequest = { + resourceMetrics: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'frontend' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'page.load.time', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 2.5, + }], + }, + }], + }], + }, + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'backend' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'db.query.time', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 15.3, + }], + }, + }], + }], + }, + ], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should handle multiple metrics in a single request', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'multi-metric-svc' } }], + }, + scopeMetrics: [{ + metrics: [ + { + name: 'system.cpu.usage', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 0.65, + }], + }, + }, + { + name: 'system.memory.usage', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 0.82, + }], + }, + }, + { + name: 'http.server.requests', + sum: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asInt: '1500', + }], + aggregationTemporality: 2, + isMonotonic: true, + }, + }, + ], + }], + }], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should handle snake_case field names (Python SDK)', async () => { + const otlpRequest = { + resource_metrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'python-svc' } }], + }, + scope_metrics: [{ + metrics: [{ + name: 'http.duration', + gauge: { + data_points: [{ + time_unix_nano: String(Date.now() * 1000000), + as_double: 123.4, + }], + }, + }], + }], + }], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should handle empty request body (valid per OTLP spec)', async () => { + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send({ resourceMetrics: [] }) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should handle gzip-compressed JSON (Content-Encoding: gzip)', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'gzip-json-metrics' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'gzip.test.gauge', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 99.9, + }], + }, + }], + }], + }], + }; + + const jsonData = JSON.stringify(otlpRequest); + const gzippedData = gzipSync(Buffer.from(jsonData)); + + const response = await app.inject({ + method: 'POST', + url: '/v1/otlp/metrics', + headers: { + 'content-type': 'application/json', + 'content-encoding': 'gzip', + 'x-api-key': apiKey, + }, + payload: gzippedData, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.partialSuccess.rejectedDataPoints).toBe(0); + expect(body.partialSuccess.errorMessage).toBe(''); + }); + + it('should auto-detect gzip by magic bytes (no Content-Encoding header)', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'gzip-magic-metrics' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'magic.bytes.gauge', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 42.0, + }], + }, + }], + }], + }], + }; + + const jsonData = JSON.stringify(otlpRequest); + const gzippedData = gzipSync(Buffer.from(jsonData)); + + // Send gzip data WITHOUT Content-Encoding header + // The server should detect gzip by magic bytes (0x1f 0x8b) + const response = await app.inject({ + method: 'POST', + url: '/v1/otlp/metrics', + headers: { + 'content-type': 'application/x-protobuf', + // NOTE: No 'content-encoding' header! + 'x-api-key': apiKey, + }, + payload: gzippedData, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.partialSuccess.rejectedDataPoints).toBe(0); + expect(body.partialSuccess.errorMessage).toBe(''); + }); + + it('should handle JSON sent with protobuf content-type (fallback)', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'proto-fallback-svc' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'fallback.gauge', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 77.7, + }], + }, + }], + }], + }], + }; + + const jsonData = JSON.stringify(otlpRequest); + + const response = await app.inject({ + method: 'POST', + url: '/v1/otlp/metrics', + headers: { + 'content-type': 'application/x-protobuf', + 'x-api-key': apiKey, + }, + payload: Buffer.from(jsonData), + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.partialSuccess.rejectedDataPoints).toBe(0); + expect(body.partialSuccess.errorMessage).toBe(''); + }); + + it('should handle gzip-compressed protobuf content-type (JSON inside)', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'gzip-proto-json' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'gzip.proto.gauge', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 55.5, + }], + }, + }], + }], + }], + }; + + const jsonData = JSON.stringify(otlpRequest); + const gzippedData = gzipSync(Buffer.from(jsonData)); + + const response = await app.inject({ + method: 'POST', + url: '/v1/otlp/metrics', + headers: { + 'content-type': 'application/x-protobuf', + 'content-encoding': 'gzip', + 'x-api-key': apiKey, + }, + payload: gzippedData, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.partialSuccess.rejectedDataPoints).toBe(0); + expect(body.partialSuccess.errorMessage).toBe(''); + }); + + it('should reject request without API key', async () => { + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('Content-Type', 'application/json') + .send({ resourceMetrics: [] }) + .expect(401); + + expect(response.body).toHaveProperty('error', 'Unauthorized'); + }); + + it('should reject request with invalid API key', async () => { + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', 'invalid_key_12345') + .set('Content-Type', 'application/json') + .send({ resourceMetrics: [] }) + .expect(401); + + expect(response.body).toHaveProperty('error', 'Unauthorized'); + }); + + it('should handle malformed JSON', async () => { + await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send('invalid json{') + .expect(400); + }); + + it('should ingest gauge metric with exemplars', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'exemplar-test' } }], + }, + scopeMetrics: [{ + metrics: [{ + name: 'request.duration', + gauge: { + dataPoints: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 150.5, + exemplars: [{ + timeUnixNano: String(Date.now() * 1000000), + asDouble: 200.1, + traceId: 'abc123def456abc123def456abc123de', + spanId: '1234567890abcdef', + filteredAttributes: [{ key: 'http.method', value: { stringValue: 'GET' } }], + }], + }], + }, + }], + }], + }], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + + it('should return 200 with empty records (no data points in gauge)', async () => { + const otlpRequest = { + resourceMetrics: [{ + resource: { attributes: [] }, + scopeMetrics: [{ + metrics: [{ name: 'empty.metric', gauge: { dataPoints: [] } }], + }], + }], + }; + + const response = await request(app.server) + .post('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .set('Content-Type', 'application/json') + .send(otlpRequest) + .expect(200); + + expect(response.body.partialSuccess.rejectedDataPoints).toBe(0); + expect(response.body.partialSuccess.errorMessage).toBe(''); + }); + }); + + // ========================================================================== + // GET /v1/otlp/metrics - Health Check + // ========================================================================== + describe('GET /v1/otlp/metrics', () => { + it('should return ok status (health check)', async () => { + const response = await request(app.server) + .get('/v1/otlp/metrics') + .set('x-api-key', apiKey) + .expect(200); + + expect(response.body).toEqual({ status: 'ok' }); + }); + }); +}); diff --git a/packages/backend/src/tests/modules/otlp/metric-transformer.test.ts b/packages/backend/src/tests/modules/otlp/metric-transformer.test.ts new file mode 100644 index 00000000..d636a560 --- /dev/null +++ b/packages/backend/src/tests/modules/otlp/metric-transformer.test.ts @@ -0,0 +1,1450 @@ +import { describe, it, expect } from 'vitest'; +import { gzipSync } from 'zlib'; +import { + transformOtlpToMetrics, + parseOtlpMetricsJson, + parseOtlpMetricsProtobuf, + type OtlpExportMetricsRequest, +} from '../../../modules/otlp/metric-transformer.js'; + +// ============================================================================ +// Helper: build a minimal OTLP metrics request +// ============================================================================ + +function makeRequest( + overrides: Partial<{ + serviceName: string; + resourceAttrs: Array<{ key: string; value?: Record }>; + metrics: Array>; + scopeMetrics: unknown[]; + }> = {} +): OtlpExportMetricsRequest { + const resourceAttributes = overrides.resourceAttrs ?? (overrides.serviceName + ? [{ key: 'service.name', value: { stringValue: overrides.serviceName } }] + : []); + + return { + resourceMetrics: [ + { + resource: { attributes: resourceAttributes }, + scopeMetrics: overrides.scopeMetrics as OtlpExportMetricsRequest['resourceMetrics'] extends (infer U)[] ? U extends { scopeMetrics?: infer S } ? S : never : never ?? [ + { + metrics: overrides.metrics ?? [], + }, + ], + }, + ], + }; +} + +/** + * Shorthand to build a well-formed request with a single metric and avoid the + * type gymnastics of makeRequest for most tests. + */ +function singleMetricRequest( + metric: Record, + serviceName = 'my-service', + extraResourceAttrs: Array<{ key: string; value?: Record }> = [] +): OtlpExportMetricsRequest { + return { + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: serviceName } }, + ...extraResourceAttrs, + ], + }, + scopeMetrics: [ + { + metrics: [metric as never], + }, + ], + }, + ], + }; +} + +// A fixed timestamp in nanoseconds: 2024-01-15T09:50:00.000Z +const FIXED_NANOS = '1705312200000000000'; +const FIXED_DATE = new Date(1705312200000); + +// ============================================================================ +// transformOtlpToMetrics +// ============================================================================ + +describe('OTLP Metric Transformer', () => { + describe('transformOtlpToMetrics', () => { + it('should return empty array for empty request', () => { + const result = transformOtlpToMetrics({}); + expect(result).toEqual([]); + }); + + it('should return empty array for empty resourceMetrics', () => { + const result = transformOtlpToMetrics({ resourceMetrics: [] }); + expect(result).toEqual([]); + }); + + it('should transform a gauge metric', () => { + const request = singleMetricRequest({ + name: 'cpu.usage', + gauge: { + dataPoints: [ + { timeUnixNano: FIXED_NANOS, asDouble: 72.5 }, + ], + }, + }); + + const result = transformOtlpToMetrics(request); + + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + metricName: 'cpu.usage', + metricType: 'gauge', + value: 72.5, + serviceName: 'my-service', + organizationId: '', + projectId: '', + }); + expect(result[0].time).toEqual(FIXED_DATE); + }); + + it('should transform a sum metric with isMonotonic', () => { + const request = singleMetricRequest({ + name: 'http.requests', + sum: { + isMonotonic: true, + aggregationTemporality: 2, + dataPoints: [ + { timeUnixNano: FIXED_NANOS, asInt: '150' }, + ], + }, + }); + + const result = transformOtlpToMetrics(request); + + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + metricName: 'http.requests', + metricType: 'sum', + value: 150, + isMonotonic: true, + serviceName: 'my-service', + }); + }); + + it('should transform a histogram metric with bucketCounts and explicitBounds', () => { + const request = singleMetricRequest({ + name: 'http.request.duration', + histogram: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '100', + sum: 5432.1, + min: 1.2, + max: 987.6, + bucketCounts: ['10', '30', '40', '15', '5'], + explicitBounds: [10, 50, 100, 500], + }, + ], + }, + }); + + const result = transformOtlpToMetrics(request); + + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + metricName: 'http.request.duration', + metricType: 'histogram', + value: 5432.1, + serviceName: 'my-service', + }); + expect(result[0].histogramData).toEqual({ + sum: 5432.1, + count: 100, + min: 1.2, + max: 987.6, + bucket_counts: [10, 30, 40, 15, 5], + explicit_bounds: [10, 50, 100, 500], + }); + }); + + it('should transform an exponential histogram metric with scale, zeroCount, positive/negative', () => { + const request = singleMetricRequest({ + name: 'exp.hist.metric', + exponentialHistogram: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '200', + sum: 1000.0, + min: 0.5, + max: 100.0, + scale: 3, + zeroCount: '5', + positive: { offset: 1, bucketCounts: ['10', '20', '30'] }, + negative: { offset: 2, bucketCounts: ['5', '15'] }, + }, + ], + }, + }); + + const result = transformOtlpToMetrics(request); + + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + metricName: 'exp.hist.metric', + metricType: 'exp_histogram', + value: 1000.0, + }); + expect(result[0].histogramData).toEqual({ + sum: 1000.0, + count: 200, + min: 0.5, + max: 100.0, + scale: 3, + zero_count: 5, + positive: { offset: 1, bucket_counts: [10, 20, 30] }, + negative: { offset: 2, bucket_counts: [5, 15] }, + }); + }); + + it('should transform a summary metric with quantileValues', () => { + const request = singleMetricRequest({ + name: 'rpc.duration', + summary: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '500', + sum: 12345.0, + quantileValues: [ + { quantile: 0.5, value: 20.0 }, + { quantile: 0.99, value: 95.0 }, + ], + }, + ], + }, + }); + + const result = transformOtlpToMetrics(request); + + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + metricName: 'rpc.duration', + metricType: 'summary', + value: 12345.0, + }); + expect(result[0].histogramData).toEqual({ + sum: 12345.0, + count: 500, + quantile_values: [ + { quantile: 0.5, value: 20.0 }, + { quantile: 0.99, value: 95.0 }, + ], + }); + // summary always sets exemplars to undefined + expect(result[0].exemplars).toBeUndefined(); + }); + + it('should extract service name from resource attributes', () => { + const request = singleMetricRequest( + { + name: 'test.metric', + gauge: { dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 1 }] }, + }, + 'payment-service' + ); + + const result = transformOtlpToMetrics(request); + expect(result[0].serviceName).toBe('payment-service'); + }); + + it("should use 'unknown' when no service.name in resource", () => { + const request: OtlpExportMetricsRequest = { + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'host.name', value: { stringValue: 'server-01' } }, + ], + }, + scopeMetrics: [ + { + metrics: [ + { + name: 'test.metric', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 1 }], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = transformOtlpToMetrics(request); + expect(result[0].serviceName).toBe('unknown'); + }); + + it('should handle multiple resources with different services', () => { + const request: OtlpExportMetricsRequest = { + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'frontend' } }, + ], + }, + scopeMetrics: [ + { + metrics: [ + { + name: 'req.count', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 10 }], + }, + }, + ], + }, + ], + }, + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'backend' } }, + ], + }, + scopeMetrics: [ + { + metrics: [ + { + name: 'req.count', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 20 }], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = transformOtlpToMetrics(request); + + expect(result).toHaveLength(2); + expect(result[0].serviceName).toBe('frontend'); + expect(result[0].value).toBe(10); + expect(result[1].serviceName).toBe('backend'); + expect(result[1].value).toBe(20); + }); + + it('should handle multiple scopes within a resource', () => { + const request: OtlpExportMetricsRequest = { + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'svc' } }, + ], + }, + scopeMetrics: [ + { + scope: { name: 'scope-a' }, + metrics: [ + { + name: 'metric.a', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 1 }], + }, + }, + ], + }, + { + scope: { name: 'scope-b' }, + metrics: [ + { + name: 'metric.b', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 2 }], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = transformOtlpToMetrics(request); + + expect(result).toHaveLength(2); + expect(result[0].metricName).toBe('metric.a'); + expect(result[1].metricName).toBe('metric.b'); + }); + + it('should handle multiple metrics within a scope', () => { + const request = singleMetricRequest({ + name: 'will-be-overridden', + gauge: { dataPoints: [] }, + }); + // Replace with two metrics in the same scope + request.resourceMetrics![0].scopeMetrics![0].metrics = [ + { + name: 'metric.one', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 100 }], + }, + }, + { + name: 'metric.two', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 200 }], + }, + }, + ]; + + const result = transformOtlpToMetrics(request); + + expect(result).toHaveLength(2); + expect(result[0].metricName).toBe('metric.one'); + expect(result[1].metricName).toBe('metric.two'); + }); + + it("should use 'unknown' for metric name when name is missing", () => { + const request = singleMetricRequest({ + // name intentionally omitted + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 42 }], + }, + }); + + const result = transformOtlpToMetrics(request); + expect(result[0].metricName).toBe('unknown'); + }); + + it('should include resource attributes in each record', () => { + const request = singleMetricRequest( + { + name: 'test', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 1 }], + }, + }, + 'svc', + [{ key: 'deployment.environment', value: { stringValue: 'production' } }] + ); + + const result = transformOtlpToMetrics(request); + + expect(result[0].resourceAttributes).toMatchObject({ + 'service.name': 'svc', + 'deployment.environment': 'production', + }); + }); + + it('should include data point attributes', () => { + const request = singleMetricRequest({ + name: 'http.duration', + gauge: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + asDouble: 55, + attributes: [ + { key: 'http.method', value: { stringValue: 'GET' } }, + { key: 'http.status_code', value: { intValue: 200 } }, + ], + }, + ], + }, + }); + + const result = transformOtlpToMetrics(request); + + expect(result[0].attributes).toEqual({ + 'http.method': 'GET', + 'http.status_code': 200, + }); + }); + + it('should set organizationId and projectId to empty strings', () => { + const request = singleMetricRequest({ + name: 'test', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 1 }], + }, + }); + + const result = transformOtlpToMetrics(request); + + expect(result[0].organizationId).toBe(''); + expect(result[0].projectId).toBe(''); + }); + }); + + // ========================================================================== + // Gauge data points + // ========================================================================== + + describe('gauge data points', () => { + it('should use asDouble for value', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 3.14 }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].value).toBe(3.14); + }); + + it('should use asInt when asDouble is undefined', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asInt: 42 }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].value).toBe(42); + }); + + it('should use 0 when both asDouble and asInt are undefined', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].value).toBe(0); + }); + + it('should handle string asInt (int64 from JSON)', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asInt: '9007199254740991' }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].value).toBe(9007199254740991); + }); + + it('should convert timeUnixNano to Date', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [{ timeUnixNano: '1705312200000000000', asDouble: 1 }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].time).toEqual(FIXED_DATE); + }); + }); + + // ========================================================================== + // Sum data points + // ========================================================================== + + describe('sum data points', () => { + it('should include isMonotonic field from sum', () => { + const request = singleMetricRequest({ + name: 's', + sum: { + isMonotonic: true, + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 100 }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].isMonotonic).toBe(true); + }); + + it('should handle sum without isMonotonic', () => { + const request = singleMetricRequest({ + name: 's', + sum: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 50 }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].isMonotonic).toBeUndefined(); + }); + }); + + // ========================================================================== + // Histogram data points + // ========================================================================== + + describe('histogram data points', () => { + it('should include histogramData with all fields', () => { + const request = singleMetricRequest({ + name: 'h', + histogram: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: 50, + sum: 2500.0, + min: 5.0, + max: 200.0, + bucketCounts: [5, 15, 20, 8, 2], + explicitBounds: [10, 50, 100, 500], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].histogramData).toEqual({ + sum: 2500.0, + count: 50, + min: 5.0, + max: 200.0, + bucket_counts: [5, 15, 20, 8, 2], + explicit_bounds: [10, 50, 100, 500], + }); + }); + + it('should use sum as value, fallback to 0', () => { + const withSum = singleMetricRequest({ + name: 'h', + histogram: { + dataPoints: [ + { timeUnixNano: FIXED_NANOS, sum: 123.4, count: 10 }, + ], + }, + }); + const withoutSum = singleMetricRequest({ + name: 'h', + histogram: { + dataPoints: [ + { timeUnixNano: FIXED_NANOS, count: 10 }, + ], + }, + }); + + expect(transformOtlpToMetrics(withSum)[0].value).toBe(123.4); + expect(transformOtlpToMetrics(withoutSum)[0].value).toBe(0); + }); + + it('should handle missing optional fields (min, max)', () => { + const request = singleMetricRequest({ + name: 'h', + histogram: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: 10, + sum: 100.0, + bucketCounts: [5, 5], + explicitBounds: [50], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].histogramData!.min).toBeUndefined(); + expect(result[0].histogramData!.max).toBeUndefined(); + }); + + it('should map bucketCounts through toNumber', () => { + const request = singleMetricRequest({ + name: 'h', + histogram: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '30', + sum: 600, + bucketCounts: ['10', '15', '5'], + explicitBounds: [100, 500], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].histogramData!.bucket_counts).toEqual([10, 15, 5]); + }); + }); + + // ========================================================================== + // Exponential histogram data points + // ========================================================================== + + describe('exponential histogram data points', () => { + it('should include scale, zeroCount, positive, negative in histogramData', () => { + const request = singleMetricRequest({ + name: 'eh', + exponentialHistogram: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '50', + sum: 1234.5, + scale: 5, + zeroCount: '3', + positive: { offset: 2, bucketCounts: ['10', '20'] }, + negative: { offset: 1, bucketCounts: ['5'] }, + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].histogramData).toMatchObject({ + scale: 5, + zero_count: 3, + positive: { offset: 2, bucket_counts: [10, 20] }, + negative: { offset: 1, bucket_counts: [5] }, + }); + }); + + it('should handle missing positive/negative', () => { + const request = singleMetricRequest({ + name: 'eh', + exponentialHistogram: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '10', + sum: 100.0, + scale: 2, + zeroCount: '1', + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].histogramData!.positive).toBeUndefined(); + expect(result[0].histogramData!.negative).toBeUndefined(); + }); + + it('should default positive/negative offset to 0', () => { + const request = singleMetricRequest({ + name: 'eh', + exponentialHistogram: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '10', + sum: 100.0, + scale: 2, + zeroCount: '0', + positive: { bucketCounts: ['5', '5'] }, + negative: { bucketCounts: ['3'] }, + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].histogramData!.positive!.offset).toBe(0); + expect(result[0].histogramData!.negative!.offset).toBe(0); + }); + }); + + // ========================================================================== + // Summary data points + // ========================================================================== + + describe('summary data points', () => { + it('should include quantileValues in histogramData', () => { + const request = singleMetricRequest({ + name: 'sm', + summary: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '100', + sum: 5000.0, + quantileValues: [ + { quantile: 0.5, value: 45.0 }, + { quantile: 0.9, value: 88.0 }, + { quantile: 0.99, value: 99.0 }, + ], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].histogramData!.quantile_values).toEqual([ + { quantile: 0.5, value: 45.0 }, + { quantile: 0.9, value: 88.0 }, + { quantile: 0.99, value: 99.0 }, + ]); + }); + + it('should default quantile/value to 0 when missing', () => { + const request = singleMetricRequest({ + name: 'sm', + summary: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '10', + sum: 100.0, + quantileValues: [ + { /* quantile and value both omitted */ }, + ], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].histogramData!.quantile_values).toEqual([ + { quantile: 0, value: 0 }, + ]); + }); + + it('should set exemplars to undefined for summary', () => { + const request = singleMetricRequest({ + name: 'sm', + summary: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + count: '1', + sum: 10.0, + quantileValues: [], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].exemplars).toBeUndefined(); + }); + }); + + // ========================================================================== + // Exemplars + // ========================================================================== + + describe('exemplars', () => { + it('should return undefined when no exemplars', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 1 }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].exemplars).toBeUndefined(); + }); + + it('should return undefined for empty exemplars array', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 1, exemplars: [] }], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].exemplars).toBeUndefined(); + }); + + it('should extract exemplar with all fields (value, time, traceId, spanId, attributes)', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + asDouble: 1, + exemplars: [ + { + asDouble: 99.9, + timeUnixNano: FIXED_NANOS, + traceId: 'abcdef0123456789abcdef0123456789', + spanId: '0123456789abcdef', + filteredAttributes: [ + { key: 'http.route', value: { stringValue: '/api/v1/users' } }, + ], + }, + ], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + + expect(result[0].exemplars).toHaveLength(1); + expect(result[0].exemplars![0]).toEqual({ + exemplarValue: 99.9, + exemplarTime: FIXED_DATE, + traceId: 'abcdef0123456789abcdef0123456789', + spanId: '0123456789abcdef', + attributes: { 'http.route': '/api/v1/users' }, + }); + }); + + it('should prefer asDouble over asInt for exemplar value', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + asDouble: 1, + exemplars: [ + { + asDouble: 77.7, + asInt: '100', + timeUnixNano: FIXED_NANOS, + }, + ], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].exemplars![0].exemplarValue).toBe(77.7); + }); + + it('should use toNumber on asInt when asDouble is undefined', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + asDouble: 1, + exemplars: [ + { + asInt: '42', + timeUnixNano: FIXED_NANOS, + }, + ], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].exemplars![0].exemplarValue).toBe(42); + }); + + it('should normalize traceId and spanId (hex passthrough)', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + asDouble: 1, + exemplars: [ + { + asDouble: 10, + traceId: 'aabbccdd11223344aabbccdd11223344', + spanId: 'aabbccdd11223344', + }, + ], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].exemplars![0].traceId).toBe('aabbccdd11223344aabbccdd11223344'); + expect(result[0].exemplars![0].spanId).toBe('aabbccdd11223344'); + }); + + it('should return undefined traceId for all-zeros', () => { + const request = singleMetricRequest({ + name: 'g', + gauge: { + dataPoints: [ + { + timeUnixNano: FIXED_NANOS, + asDouble: 1, + exemplars: [ + { + asDouble: 10, + traceId: '00000000000000000000000000000000', + spanId: '0000000000000000', + }, + ], + }, + ], + }, + }); + const result = transformOtlpToMetrics(request); + expect(result[0].exemplars![0].traceId).toBeUndefined(); + expect(result[0].exemplars![0].spanId).toBeUndefined(); + }); + }); + + // ========================================================================== + // parseOtlpMetricsJson + // ========================================================================== + + describe('parseOtlpMetricsJson', () => { + it('should return empty resourceMetrics for null/undefined body', () => { + expect(parseOtlpMetricsJson(null)).toEqual({ resourceMetrics: [] }); + expect(parseOtlpMetricsJson(undefined)).toEqual({ resourceMetrics: [] }); + }); + + it('should parse object body directly', () => { + const body = { + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'test' } }, + ], + }, + scopeMetrics: [ + { + metrics: [ + { + name: 'cpu', + gauge: { + dataPoints: [{ timeUnixNano: FIXED_NANOS, asDouble: 50 }], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + + expect(result.resourceMetrics).toHaveLength(1); + expect(result.resourceMetrics![0].scopeMetrics![0].metrics![0].name).toBe('cpu'); + }); + + it('should parse string body as JSON', () => { + const body = JSON.stringify({ + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { name: 'mem', gauge: { dataPoints: [{ asDouble: 80 }] } }, + ], + }, + ], + }, + ], + }); + + const result = parseOtlpMetricsJson(body); + expect(result.resourceMetrics).toHaveLength(1); + expect(result.resourceMetrics![0].scopeMetrics![0].metrics![0].name).toBe('mem'); + }); + + it('should throw on invalid JSON string', () => { + expect(() => parseOtlpMetricsJson('{not valid json')).toThrow( + 'Invalid OTLP Metrics JSON' + ); + }); + + it('should throw on non-string, non-object body type', () => { + expect(() => parseOtlpMetricsJson(12345 as unknown)).toThrow( + 'Invalid OTLP metrics request body type' + ); + }); + + it('should handle camelCase fields (resourceMetrics, scopeMetrics, dataPoints, timeUnixNano, asDouble, asInt)', () => { + const body = { + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'test', + gauge: { + dataPoints: [ + { timeUnixNano: FIXED_NANOS, asDouble: 3.14, asInt: '7' }, + ], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + const dp = result.resourceMetrics![0].scopeMetrics![0].metrics![0].gauge!.dataPoints![0]; + + expect(dp.timeUnixNano).toBe(FIXED_NANOS); + expect(dp.asDouble).toBe(3.14); + expect(dp.asInt).toBe('7'); + }); + + it('should handle snake_case fields (resource_metrics, scope_metrics, data_points, time_unix_nano, as_double, as_int)', () => { + const body = { + resource_metrics: [ + { + resource: { attributes: [] }, + scope_metrics: [ + { + metrics: [ + { + name: 'snake_test', + gauge: { + data_points: [ + { time_unix_nano: FIXED_NANOS, as_double: 2.71, as_int: '3' }, + ], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + + expect(result.resourceMetrics).toHaveLength(1); + const dp = result.resourceMetrics![0].scopeMetrics![0].metrics![0].gauge!.dataPoints![0]; + expect(dp.timeUnixNano).toBe(FIXED_NANOS); + expect(dp.asDouble).toBe(2.71); + expect(dp.asInt).toBe('3'); + }); + + it('should normalize gauge data_points to dataPoints', () => { + const body = { + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'gauge_test', + gauge: { + data_points: [ + { time_unix_nano: FIXED_NANOS, as_double: 10 }, + ], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + const gauge = result.resourceMetrics![0].scopeMetrics![0].metrics![0].gauge!; + + expect(gauge.dataPoints).toHaveLength(1); + expect(gauge.dataPoints![0].asDouble).toBe(10); + }); + + it('should normalize sum with aggregation_temporality and is_monotonic', () => { + const body = { + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'sum_test', + sum: { + data_points: [ + { time_unix_nano: FIXED_NANOS, as_double: 100 }, + ], + aggregation_temporality: 2, + is_monotonic: true, + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + const sum = result.resourceMetrics![0].scopeMetrics![0].metrics![0].sum!; + + expect(sum.aggregationTemporality).toBe(2); + expect(sum.isMonotonic).toBe(true); + expect(sum.dataPoints).toHaveLength(1); + }); + + it('should normalize histogram with bucket_counts and explicit_bounds', () => { + const body = { + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'hist_test', + histogram: { + data_points: [ + { + time_unix_nano: FIXED_NANOS, + count: '20', + sum: 500, + bucket_counts: ['5', '10', '5'], + explicit_bounds: [100, 500], + min: 2, + max: 450, + }, + ], + aggregation_temporality: 1, + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + const dp = result.resourceMetrics![0].scopeMetrics![0].metrics![0].histogram!.dataPoints![0]; + + expect(dp.bucketCounts).toEqual(['5', '10', '5']); + expect(dp.explicitBounds).toEqual([100, 500]); + expect(dp.min).toBe(2); + expect(dp.max).toBe(450); + }); + + it('should normalize exponential_histogram with zero_count', () => { + const body = { + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'exp_hist_test', + exponential_histogram: { + data_points: [ + { + time_unix_nano: FIXED_NANOS, + count: '10', + sum: 100, + scale: 3, + zero_count: '2', + positive: { offset: 1, bucket_counts: ['4', '6'] }, + negative: { offset: 0, bucket_counts: ['2'] }, + }, + ], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + const metric = result.resourceMetrics![0].scopeMetrics![0].metrics![0]; + + expect(metric.exponentialHistogram).toBeDefined(); + const dp = metric.exponentialHistogram!.dataPoints![0]; + expect(dp.zeroCount).toBe('2'); + expect(dp.scale).toBe(3); + expect(dp.positive).toEqual({ offset: 1, bucketCounts: ['4', '6'] }); + }); + + it('should normalize summary with quantile_values', () => { + const body = { + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'summary_test', + summary: { + data_points: [ + { + time_unix_nano: FIXED_NANOS, + count: '50', + sum: 2500, + quantile_values: [ + { quantile: 0.5, value: 45 }, + { quantile: 0.99, value: 99 }, + ], + }, + ], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + const dp = result.resourceMetrics![0].scopeMetrics![0].metrics![0].summary!.dataPoints![0]; + + expect(dp.quantileValues).toEqual([ + { quantile: 0.5, value: 45 }, + { quantile: 0.99, value: 99 }, + ]); + }); + + it('should normalize exemplar filtered_attributes, span_id, trace_id', () => { + const body = { + resourceMetrics: [ + { + resource: { attributes: [] }, + scopeMetrics: [ + { + metrics: [ + { + name: 'exemplar_test', + gauge: { + data_points: [ + { + time_unix_nano: FIXED_NANOS, + as_double: 1, + exemplars: [ + { + as_double: 5.5, + time_unix_nano: FIXED_NANOS, + trace_id: 'aabb', + span_id: 'ccdd', + filtered_attributes: [ + { key: 'env', value: { stringValue: 'prod' } }, + ], + }, + ], + }, + ], + }, + }, + ], + }, + ], + }, + ], + }; + + const result = parseOtlpMetricsJson(body); + const exemplar = result.resourceMetrics![0] + .scopeMetrics![0] + .metrics![0] + .gauge! + .dataPoints![0] + .exemplars![0]; + + expect(exemplar.traceId).toBe('aabb'); + expect(exemplar.spanId).toBe('ccdd'); + expect(exemplar.asDouble).toBe(5.5); + expect(exemplar.filteredAttributes).toEqual([ + { key: 'env', value: { stringValue: 'prod' } }, + ]); + }); + + it('should return empty resourceMetrics when field is not array', () => { + const body = { resourceMetrics: 'not-an-array' }; + const result = parseOtlpMetricsJson(body); + expect(result.resourceMetrics).toEqual([]); + }); + }); + + // ========================================================================== + // parseOtlpMetricsProtobuf + // ========================================================================== + + describe('parseOtlpMetricsProtobuf', () => { + it('should parse JSON payload sent as protobuf (JSON-in-protobuf fallback)', async () => { + const jsonPayload = { + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'proto-svc' } }, + ], + }, + scopeMetrics: [ + { + metrics: [ + { + name: 'fallback.metric', + gauge: { + dataPoints: [ + { timeUnixNano: FIXED_NANOS, asDouble: 42 }, + ], + }, + }, + ], + }, + ], + }, + ], + }; + + const buffer = Buffer.from(JSON.stringify(jsonPayload), 'utf-8'); + const result = await parseOtlpMetricsProtobuf(buffer); + + expect(result.resourceMetrics).toHaveLength(1); + expect(result.resourceMetrics![0].scopeMetrics![0].metrics![0].name).toBe('fallback.metric'); + }); + + it('should handle gzip-compressed JSON (auto-detect by magic bytes)', async () => { + const jsonPayload = { + resourceMetrics: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'gzip-svc' } }, + ], + }, + scopeMetrics: [ + { + metrics: [ + { + name: 'gzip.metric', + gauge: { + dataPoints: [ + { timeUnixNano: FIXED_NANOS, asDouble: 77.7 }, + ], + }, + }, + ], + }, + ], + }, + ], + }; + + const jsonBuffer = Buffer.from(JSON.stringify(jsonPayload), 'utf-8'); + const gzipBuffer = gzipSync(jsonBuffer); + + const result = await parseOtlpMetricsProtobuf(gzipBuffer); + + expect(result.resourceMetrics).toHaveLength(1); + expect(result.resourceMetrics![0].scopeMetrics![0].metrics![0].name).toBe('gzip.metric'); + + // Verify end-to-end: parse then transform + const records = transformOtlpToMetrics(result); + expect(records).toHaveLength(1); + expect(records[0].metricName).toBe('gzip.metric'); + expect(records[0].value).toBe(77.7); + expect(records[0].serviceName).toBe('gzip-svc'); + }); + }); +}); diff --git a/packages/reservoir/src/engines/clickhouse/clickhouse-engine-metrics.test.ts b/packages/reservoir/src/engines/clickhouse/clickhouse-engine-metrics.test.ts new file mode 100644 index 00000000..b636af4e --- /dev/null +++ b/packages/reservoir/src/engines/clickhouse/clickhouse-engine-metrics.test.ts @@ -0,0 +1,782 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { StorageConfig, MetricRecord, MetricExemplar } from '../../core/types.js'; + +// Create mock functions +const mockInsert = vi.fn(); +const mockQuery = vi.fn(); +const mockCommand = vi.fn(); +const mockClose = vi.fn(); +const mockPing = vi.fn(); + +// Mock the @clickhouse/client module +vi.mock('@clickhouse/client', () => ({ + createClient: vi.fn(() => ({ + insert: mockInsert, + query: mockQuery, + command: mockCommand, + close: mockClose, + ping: mockPing, + })), +})); + +import { ClickHouseEngine } from './clickhouse-engine.js'; + +const config: StorageConfig = { + host: 'localhost', + port: 8123, + database: 'logtide_test', + username: 'default', + password: '', +}; + +function makeMetric(overrides: Partial = {}): MetricRecord { + return { + time: new Date('2024-01-01T00:00:00Z'), + organizationId: 'org-1', + projectId: 'proj-1', + metricName: 'cpu.usage', + metricType: 'gauge', + value: 0.75, + serviceName: 'api', + attributes: { host: 'server-1' }, + resourceAttributes: { 'service.name': 'api' }, + ...overrides, + }; +} + +function makeExemplar(overrides: Partial = {}): MetricExemplar { + return { + exemplarValue: 0.95, + exemplarTime: new Date('2024-01-01T00:00:01Z'), + traceId: 'trace-ex-1', + spanId: 'span-ex-1', + attributes: { sampled: 'true' }, + ...overrides, + }; +} + +// Helper to create mock query result +function mockQueryResult(data: unknown[]) { + return { + json: vi.fn().mockResolvedValue(data), + }; +} + +describe('ClickHouseEngine metric operations (unit)', () => { + let engine: ClickHouseEngine; + + beforeEach(async () => { + vi.clearAllMocks(); + engine = new ClickHouseEngine(config); + await engine.connect(); + }); + + // =========================================================================== + // ingestMetrics + // =========================================================================== + + describe('ingestMetrics', () => { + it('should return empty result for empty array', async () => { + const result = await engine.ingestMetrics([]); + + expect(result).toEqual({ ingested: 0, failed: 0, durationMs: 0 }); + expect(mockInsert).not.toHaveBeenCalled(); + }); + + it('should insert metrics with client.insert', async () => { + mockInsert.mockResolvedValueOnce(undefined); + const metric = makeMetric(); + + const result = await engine.ingestMetrics([metric]); + + expect(result.ingested).toBe(1); + expect(result.failed).toBe(0); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + expect(mockInsert).toHaveBeenCalledWith( + expect.objectContaining({ + table: 'metrics', + format: 'JSONEachRow', + }), + ); + }); + + it('should convert time to epoch milliseconds', async () => { + mockInsert.mockResolvedValueOnce(undefined); + const metric = makeMetric({ time: new Date('2024-06-15T10:30:00Z') }); + + await engine.ingestMetrics([metric]); + + const insertCall = mockInsert.mock.calls[0][0]; + const row = insertCall.values[0]; + expect(row.time).toBe(new Date('2024-06-15T10:30:00Z').getTime()); + }); + + it('should convert is_monotonic to 1/0/null', async () => { + mockInsert.mockResolvedValueOnce(undefined); + + const metrics = [ + makeMetric({ isMonotonic: true }), + makeMetric({ isMonotonic: false }), + makeMetric({ isMonotonic: undefined }), + ]; + + await engine.ingestMetrics(metrics); + + const insertCall = mockInsert.mock.calls[0][0]; + expect(insertCall.values[0].is_monotonic).toBe(1); + expect(insertCall.values[1].is_monotonic).toBe(0); + expect(insertCall.values[2].is_monotonic).toBeNull(); + }); + + it('should JSON.stringify attributes and resource_attributes', async () => { + mockInsert.mockResolvedValueOnce(undefined); + const metric = makeMetric({ + attributes: { env: 'production', region: 'us-east' }, + resourceAttributes: { 'service.version': '2.0' }, + }); + + await engine.ingestMetrics([metric]); + + const row = mockInsert.mock.calls[0][0].values[0]; + expect(row.attributes).toBe(JSON.stringify({ env: 'production', region: 'us-east' })); + expect(row.resource_attributes).toBe(JSON.stringify({ 'service.version': '2.0' })); + }); + + it('should insert exemplars in a second insert when present', async () => { + mockInsert.mockResolvedValueOnce(undefined); // metrics insert + mockInsert.mockResolvedValueOnce(undefined); // exemplars insert + + const metric = makeMetric({ + exemplars: [makeExemplar()], + }); + + await engine.ingestMetrics([metric]); + + expect(mockInsert).toHaveBeenCalledTimes(2); + + // First call: metrics table + expect(mockInsert.mock.calls[0][0].table).toBe('metrics'); + expect(mockInsert.mock.calls[0][0].values[0].has_exemplars).toBe(1); + + // Second call: metric_exemplars table + expect(mockInsert.mock.calls[1][0].table).toBe('metric_exemplars'); + const exemplarRow = mockInsert.mock.calls[1][0].values[0]; + expect(exemplarRow.exemplar_value).toBe(0.95); + expect(exemplarRow.trace_id).toBe('trace-ex-1'); + expect(exemplarRow.span_id).toBe('span-ex-1'); + expect(exemplarRow.attributes).toBe(JSON.stringify({ sampled: 'true' })); + }); + + it('should not insert exemplars when none present', async () => { + mockInsert.mockResolvedValueOnce(undefined); + + const metric = makeMetric({ exemplars: undefined }); + + await engine.ingestMetrics([metric]); + + expect(mockInsert).toHaveBeenCalledTimes(1); + expect(mockInsert.mock.calls[0][0].table).toBe('metrics'); + expect(mockInsert.mock.calls[0][0].values[0].has_exemplars).toBe(0); + }); + + it('should handle insert errors gracefully', async () => { + mockInsert.mockRejectedValueOnce(new Error('Connection refused')); + + const metrics = [makeMetric(), makeMetric()]; + const result = await engine.ingestMetrics(metrics); + + expect(result.ingested).toBe(0); + expect(result.failed).toBe(2); + expect(result.errors).toBeDefined(); + expect(result.errors![0].error).toBe('Connection refused'); + }); + }); + + // =========================================================================== + // queryMetrics + // =========================================================================== + + describe('queryMetrics', () => { + it('should query with time range and project filter', async () => { + // count query + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '5' }])); + // data query + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + const countQuery = mockQuery.mock.calls[0][0].query as string; + expect(countQuery).toContain('SELECT count() AS count FROM metrics'); + expect(countQuery).toContain('project_id IN {p_pids:Array(String)}'); + expect(countQuery).toContain('time >='); + expect(countQuery).toContain('time <='); + + const queryParams = mockQuery.mock.calls[0][0].query_params; + expect(queryParams.p_pids).toEqual(['proj-1']); + expect(queryParams.p_from).toBe(Math.floor(new Date('2024-01-01T00:00:00Z').getTime() / 1000)); + }); + + it('should include optional metricName filter', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '0' }])); + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.queryMetrics({ + projectId: 'proj-1', + metricName: 'cpu.usage', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('metric_name IN {p_names:Array(String)}'); + expect(mockQuery.mock.calls[0][0].query_params.p_names).toEqual(['cpu.usage']); + }); + + it('should include optional metricType filter', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '0' }])); + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.queryMetrics({ + projectId: 'proj-1', + metricType: 'gauge', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('metric_type IN {p_types:Array(String)}'); + expect(mockQuery.mock.calls[0][0].query_params.p_types).toEqual(['gauge']); + }); + + it('should include optional serviceName filter', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '0' }])); + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.queryMetrics({ + projectId: 'proj-1', + serviceName: 'api', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('service_name IN {p_svc:Array(String)}'); + expect(mockQuery.mock.calls[0][0].query_params.p_svc).toEqual(['api']); + }); + + it('should include attribute filter using JSONExtractString', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '0' }])); + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + attributes: { host: 'server-1' }, + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('JSONExtractString(attributes, {p_attr_key_0:String}) = {p_attr_val_0:String}'); + const params = mockQuery.mock.calls[0][0].query_params; + expect(params.p_attr_key_0).toBe('host'); + expect(params.p_attr_val_0).toBe('server-1'); + }); + + it('should handle pagination (limit/offset)', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '10' }])); + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + limit: 20, + offset: 5, + }); + + const dataQuery = mockQuery.mock.calls[1][0].query as string; + expect(dataQuery).toContain('LIMIT 20'); + expect(dataQuery).toContain('OFFSET 5'); + }); + + it('should map ClickHouse rows to StoredMetricRecord', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '1' }])); + mockQuery.mockResolvedValueOnce( + mockQueryResult([ + { + id: 'metric-id-1', + time: '1704067200', + organization_id: 'org-1', + project_id: 'proj-1', + metric_name: 'cpu.usage', + metric_type: 'gauge', + value: 0.75, + is_monotonic: null, + service_name: 'api', + attributes: '{"host":"server-1"}', + resource_attributes: '{"service.name":"api"}', + histogram_data: null, + has_exemplars: 0, + }, + ]), + ); + + const result = await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + expect(result.metrics).toHaveLength(1); + const m = result.metrics[0]; + expect(m.id).toBe('metric-id-1'); + expect(m.metricName).toBe('cpu.usage'); + expect(m.metricType).toBe('gauge'); + expect(m.value).toBe(0.75); + expect(m.serviceName).toBe('api'); + expect(m.attributes).toEqual({ host: 'server-1' }); + expect(m.hasExemplars).toBe(false); + }); + + it('should load exemplars when includeExemplars is true', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '1' }])); + mockQuery.mockResolvedValueOnce( + mockQueryResult([ + { + id: 'metric-id-1', + time: '1704067200', + organization_id: 'org-1', + project_id: 'proj-1', + metric_name: 'cpu.usage', + metric_type: 'gauge', + value: 0.75, + is_monotonic: null, + service_name: 'api', + attributes: '{}', + resource_attributes: '{}', + histogram_data: null, + has_exemplars: 1, + }, + ]), + ); + // exemplar query + mockQuery.mockResolvedValueOnce( + mockQueryResult([ + { + metric_id: 'metric-id-1', + exemplar_value: 0.95, + exemplar_time: '1704067201', + trace_id: 'trace-ex-1', + span_id: 'span-ex-1', + attributes: '{"sampled":"true"}', + }, + ]), + ); + + const result = await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + includeExemplars: true, + }); + + expect(mockQuery).toHaveBeenCalledTimes(3); + const exemplarQuery = mockQuery.mock.calls[2][0].query as string; + expect(exemplarQuery).toContain('SELECT * FROM metric_exemplars'); + expect(exemplarQuery).toContain('metric_id IN {p_mids:Array(String)}'); + + expect(result.metrics[0].exemplars).toBeDefined(); + expect(result.metrics[0].exemplars![0].exemplarValue).toBe(0.95); + expect(result.metrics[0].exemplars![0].traceId).toBe('trace-ex-1'); + }); + + it('should not load exemplars when includeExemplars is false', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '1' }])); + mockQuery.mockResolvedValueOnce( + mockQueryResult([ + { + id: 'metric-id-1', + time: '1704067200', + organization_id: 'org-1', + project_id: 'proj-1', + metric_name: 'cpu.usage', + metric_type: 'gauge', + value: 0.75, + is_monotonic: null, + service_name: 'api', + attributes: '{}', + resource_attributes: '{}', + histogram_data: null, + has_exemplars: 1, + }, + ]), + ); + + await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + includeExemplars: false, + }); + + // Only count + data queries, no exemplar query + expect(mockQuery).toHaveBeenCalledTimes(2); + }); + + it('should calculate hasMore correctly', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([{ count: '100' }])); + mockQuery.mockResolvedValueOnce( + mockQueryResult( + Array.from({ length: 50 }, (_, i) => ({ + id: `m-${i}`, + time: '1704067200', + organization_id: 'org-1', + project_id: 'proj-1', + metric_name: 'cpu.usage', + metric_type: 'gauge', + value: i, + is_monotonic: null, + service_name: 'api', + attributes: '{}', + resource_attributes: '{}', + histogram_data: null, + has_exemplars: 0, + })), + ), + ); + + const result = await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + limit: 50, + offset: 0, + }); + + // total=100, offset(0) + rows.length(50) < total(100) => hasMore=true + expect(result.hasMore).toBe(true); + expect(result.total).toBe(100); + }); + }); + + // =========================================================================== + // aggregateMetrics + // =========================================================================== + + describe('aggregateMetrics', () => { + const baseAggParams = { + projectId: 'proj-1' as string | string[], + metricName: 'cpu.usage', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + interval: '1h' as const, + }; + + it('should use avg aggregation', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.aggregateMetrics({ + ...baseAggParams, + aggregation: 'avg', + metricType: 'gauge', + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('avg(value) AS agg_value'); + }); + + it('should use sum aggregation', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.aggregateMetrics({ + ...baseAggParams, + aggregation: 'sum', + metricType: 'gauge', + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('sum(value) AS agg_value'); + }); + + it('should use count aggregation (count())', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.aggregateMetrics({ + ...baseAggParams, + aggregation: 'count', + metricType: 'gauge', + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('count() AS agg_value'); + }); + + it('should use last aggregation (argMax)', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.aggregateMetrics({ + ...baseAggParams, + aggregation: 'last', + metricType: 'gauge', + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('argMax(value, time) AS agg_value'); + }); + + it('should use toStartOfInterval for time bucketing', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.aggregateMetrics({ + ...baseAggParams, + interval: '5m', + aggregation: 'avg', + metricType: 'gauge', + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('toStartOfInterval(time, INTERVAL 5 MINUTE) AS bucket'); + }); + + it('should include groupBy with JSONExtractString', async () => { + mockQuery.mockResolvedValueOnce( + mockQueryResult([ + { bucket: '1704067200', agg_value: 0.8, label_0: 'server-1' }, + ]), + ); + + const result = await engine.aggregateMetrics({ + ...baseAggParams, + aggregation: 'avg', + metricType: 'gauge', + groupBy: ['host'], + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('JSONExtractString(attributes, {p_gb_key_0:String}) AS label_0'); + expect(query).toContain('GROUP BY bucket, label_0'); + + const params = mockQuery.mock.calls[0][0].query_params; + expect(params.p_gb_key_0).toBe('host'); + + expect(result.timeseries[0].labels).toEqual({ host: 'server-1' }); + }); + + it('should query metric type when not provided', async () => { + // aggregation query + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + // type lookup query + mockQuery.mockResolvedValueOnce(mockQueryResult([{ metric_type: 'sum' }])); + + const result = await engine.aggregateMetrics({ + ...baseAggParams, + aggregation: 'avg', + // no metricType + }); + + expect(mockQuery).toHaveBeenCalledTimes(2); + const typeQuery = mockQuery.mock.calls[1][0].query as string; + expect(typeQuery).toContain('SELECT metric_type FROM metrics'); + expect(typeQuery).toContain('metric_name = {p_name:String}'); + expect(result.metricType).toBe('sum'); + }); + + it('should include attribute filtering', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.aggregateMetrics({ + ...baseAggParams, + aggregation: 'avg', + metricType: 'gauge', + attributes: { region: 'us-east' }, + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('JSONExtractString(attributes, {p_attr_key_0:String}) = {p_attr_val_0:String}'); + const params = mockQuery.mock.calls[0][0].query_params; + expect(params.p_attr_key_0).toBe('region'); + expect(params.p_attr_val_0).toBe('us-east'); + }); + }); + + // =========================================================================== + // getMetricNames + // =========================================================================== + + describe('getMetricNames', () => { + it('should return metric names grouped by name and type', async () => { + mockQuery.mockResolvedValueOnce( + mockQueryResult([ + { metric_name: 'cpu.usage', metric_type: 'gauge' }, + { metric_name: 'http.requests', metric_type: 'sum' }, + ]), + ); + + const result = await engine.getMetricNames({ projectId: 'proj-1' }); + + expect(result.names).toHaveLength(2); + expect(result.names[0]).toEqual({ name: 'cpu.usage', type: 'gauge' }); + expect(result.names[1]).toEqual({ name: 'http.requests', type: 'sum' }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('SELECT metric_name, metric_type FROM metrics'); + expect(query).toContain('GROUP BY metric_name, metric_type'); + expect(query).toContain('ORDER BY metric_name ASC'); + }); + + it('should include time range filters when provided', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.getMetricNames({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('time >= {p_from:DateTime64(3)}'); + expect(query).toContain('time <= {p_to:DateTime64(3)}'); + }); + }); + + // =========================================================================== + // getMetricLabelKeys + // =========================================================================== + + describe('getMetricLabelKeys', () => { + it('should return keys using arrayJoin(JSONExtractKeys(...))', async () => { + mockQuery.mockResolvedValueOnce( + mockQueryResult([ + { key: 'host' }, + { key: 'region' }, + ]), + ); + + const result = await engine.getMetricLabelKeys({ + projectId: 'proj-1', + metricName: 'cpu.usage', + }); + + expect(result.keys).toEqual(['host', 'region']); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('SELECT DISTINCT arrayJoin(JSONExtractKeys(attributes)) AS key'); + expect(query).toContain('FROM metrics'); + }); + + it('should include project and metric name filters', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.getMetricLabelKeys({ + projectId: 'proj-1', + metricName: 'cpu.usage', + }); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('project_id IN {p_pids:Array(String)}'); + expect(query).toContain('metric_name = {p_name:String}'); + const params = mockQuery.mock.calls[0][0].query_params; + expect(params.p_pids).toEqual(['proj-1']); + expect(params.p_name).toBe('cpu.usage'); + }); + }); + + // =========================================================================== + // getMetricLabelValues + // =========================================================================== + + describe('getMetricLabelValues', () => { + it('should return values using JSONExtractString', async () => { + mockQuery.mockResolvedValueOnce( + mockQueryResult([ + { val: 'server-1' }, + { val: 'server-2' }, + ]), + ); + + const result = await engine.getMetricLabelValues( + { projectId: 'proj-1', metricName: 'cpu.usage' }, + 'host', + ); + + expect(result.values).toEqual(['server-1', 'server-2']); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain('SELECT DISTINCT JSONExtractString(attributes, {p_label_key:String}) AS val'); + expect(query).toContain('FROM metrics'); + expect(mockQuery.mock.calls[0][0].query_params.p_label_key).toBe('host'); + }); + + it('should include HAVING val != \'\' in query', async () => { + mockQuery.mockResolvedValueOnce(mockQueryResult([])); + + await engine.getMetricLabelValues( + { projectId: 'proj-1', metricName: 'cpu.usage' }, + 'host', + ); + + const query = mockQuery.mock.calls[0][0].query as string; + expect(query).toContain("HAVING val != ''"); + }); + }); + + // =========================================================================== + // deleteMetricsByTimeRange + // =========================================================================== + + describe('deleteMetricsByTimeRange', () => { + it('should use ALTER TABLE DELETE for metrics', async () => { + mockCommand.mockResolvedValueOnce(undefined); // metrics delete + mockCommand.mockResolvedValueOnce(undefined); // exemplars delete + + await engine.deleteMetricsByTimeRange({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + const metricsCmd = mockCommand.mock.calls[0][0].query as string; + expect(metricsCmd).toContain('ALTER TABLE metrics DELETE WHERE'); + expect(metricsCmd).toContain('project_id IN {p_pids:Array(String)}'); + expect(metricsCmd).toContain('time >= {p_from:DateTime64(3)}'); + expect(metricsCmd).toContain('time <= {p_to:DateTime64(3)}'); + }); + + it('should also delete from metric_exemplars', async () => { + mockCommand.mockResolvedValueOnce(undefined); + mockCommand.mockResolvedValueOnce(undefined); + + await engine.deleteMetricsByTimeRange({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + expect(mockCommand).toHaveBeenCalledTimes(2); + + const exemplarCmd = mockCommand.mock.calls[1][0].query as string; + expect(exemplarCmd).toContain('ALTER TABLE metric_exemplars DELETE WHERE'); + expect(exemplarCmd).toContain('project_id IN {p_pids:Array(String)}'); + expect(exemplarCmd).toContain('time >= {p_from:DateTime64(3)}'); + expect(exemplarCmd).toContain('time <= {p_to:DateTime64(3)}'); + }); + + it('should return deleted: 0 (ClickHouse mutations are async)', async () => { + mockCommand.mockResolvedValueOnce(undefined); + mockCommand.mockResolvedValueOnce(undefined); + + const result = await engine.deleteMetricsByTimeRange({ + projectId: 'proj-1', + from: new Date('2024-01-01T00:00:00Z'), + to: new Date('2024-01-02T00:00:00Z'), + }); + + expect(result.deleted).toBe(0); + expect(result.executionTimeMs).toBeGreaterThanOrEqual(0); + }); + }); +}); diff --git a/packages/reservoir/src/engines/timescale/timescale-engine-metrics.test.ts b/packages/reservoir/src/engines/timescale/timescale-engine-metrics.test.ts new file mode 100644 index 00000000..1990b4e6 --- /dev/null +++ b/packages/reservoir/src/engines/timescale/timescale-engine-metrics.test.ts @@ -0,0 +1,860 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { StorageConfig, MetricRecord, MetricExemplar } from '../../core/types.js'; + +const mockQuery = vi.fn(); +const mockEnd = vi.fn(); + +vi.mock('pg', () => { + return { + default: { + Pool: vi.fn(() => ({ + query: mockQuery, + end: mockEnd, + })), + }, + }; +}); + +import { TimescaleEngine } from './timescale-engine.js'; + +const config: StorageConfig = { + host: 'localhost', + port: 5432, + database: 'logtide', + username: 'logtide', + password: 'secret', + schema: 'public', +}; + +function makeMetric(overrides: Partial = {}): MetricRecord { + return { + time: new Date('2024-01-01T00:00:00Z'), + organizationId: 'org-1', + projectId: 'proj-1', + metricName: 'cpu.usage', + metricType: 'gauge', + value: 0.75, + serviceName: 'api', + attributes: { host: 'server-1' }, + resourceAttributes: { 'service.name': 'api' }, + ...overrides, + }; +} + +describe('TimescaleEngine - Metrics', () => { + let engine: TimescaleEngine; + + beforeEach(() => { + vi.clearAllMocks(); + engine = new TimescaleEngine(config); + }); + + // ========================================================================= + // ingestMetrics + // ========================================================================= + + describe('ingestMetrics', () => { + it('should return empty result for empty array without calling query', async () => { + await engine.connect(); + const result = await engine.ingestMetrics([]); + expect(result.ingested).toBe(0); + expect(result.failed).toBe(0); + expect(result.durationMs).toBe(0); + expect(mockQuery).not.toHaveBeenCalled(); + }); + + it('should insert metrics using UNNEST batch insert', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ id: 'metric-1', time: new Date('2024-01-01T00:00:00Z') }], + }); + await engine.connect(); + + const result = await engine.ingestMetrics([makeMetric()]); + + expect(result.ingested).toBe(1); + expect(result.failed).toBe(0); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('INSERT INTO public.metrics'); + expect(sql).toContain('UNNEST'); + expect(sql).toContain('RETURNING id, time'); + }); + + it('should include all 12 column arrays in UNNEST parameters', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ id: 'metric-1', time: new Date('2024-01-01T00:00:00Z') }], + }); + await engine.connect(); + + await engine.ingestMetrics([makeMetric()]); + + const params = mockQuery.mock.calls[0][1] as unknown[]; + expect(params.length).toBe(12); + + // Verify individual arrays + // $1: times + expect((params[0] as Date[])[0]).toEqual(new Date('2024-01-01T00:00:00Z')); + // $2: orgIds + expect((params[1] as string[])[0]).toBe('org-1'); + // $3: projectIds + expect((params[2] as string[])[0]).toBe('proj-1'); + // $4: metricNames + expect((params[3] as string[])[0]).toBe('cpu.usage'); + // $5: metricTypes + expect((params[4] as string[])[0]).toBe('gauge'); + // $6: values + expect((params[5] as number[])[0]).toBe(0.75); + // $7: isMonotonics + expect((params[6] as (boolean | null)[])[0]).toBeNull(); + // $8: serviceNames + expect((params[7] as string[])[0]).toBe('api'); + // $9: attributes JSON + expect((params[8] as (string | null)[])[0]).toBe(JSON.stringify({ host: 'server-1' })); + // $10: resourceAttributes JSON + expect((params[9] as (string | null)[])[0]).toBe(JSON.stringify({ 'service.name': 'api' })); + // $11: histogramData JSON + expect((params[10] as (string | null)[])[0]).toBeNull(); + // $12: hasExemplars flags + expect((params[11] as boolean[])[0]).toBe(false); + }); + + it('should set has_exemplars flag to true when exemplars present', async () => { + const metric = makeMetric({ + exemplars: [{ exemplarValue: 1.5, traceId: 'trace-1', spanId: 'span-1' }], + }); + mockQuery.mockResolvedValueOnce({ + rows: [{ id: 'metric-1', time: new Date('2024-01-01T00:00:00Z') }], + }); + // Exemplar insert + mockQuery.mockResolvedValueOnce({ rows: [] }); + await engine.connect(); + + await engine.ingestMetrics([metric]); + + const params = mockQuery.mock.calls[0][1] as unknown[]; + // $12: hasExemplars flags + expect((params[11] as boolean[])[0]).toBe(true); + }); + + it('should insert exemplars in a second query when present', async () => { + const exemplar: MetricExemplar = { + exemplarValue: 1.5, + exemplarTime: new Date('2024-01-01T00:00:01Z'), + traceId: 'trace-1', + spanId: 'span-1', + attributes: { key: 'val' }, + }; + const metric = makeMetric({ exemplars: [exemplar] }); + + mockQuery.mockResolvedValueOnce({ + rows: [{ id: 'metric-1', time: new Date('2024-01-01T00:00:00Z') }], + }); + mockQuery.mockResolvedValueOnce({ rows: [] }); + await engine.connect(); + + await engine.ingestMetrics([metric]); + + expect(mockQuery).toHaveBeenCalledTimes(2); + + // Second call is exemplar insert + const exSql = mockQuery.mock.calls[1][0] as string; + expect(exSql).toContain('INSERT INTO public.metric_exemplars'); + expect(exSql).toContain('UNNEST'); + + const exParams = mockQuery.mock.calls[1][1] as unknown[]; + expect(exParams.length).toBe(8); + + // Verify exemplar data + // $1: times (from metric row) + expect((exParams[0] as Date[])[0]).toEqual(new Date('2024-01-01T00:00:00Z')); + // $2: metricIds + expect((exParams[1] as string[])[0]).toBe('metric-1'); + // $3: projectIds + expect((exParams[2] as string[])[0]).toBe('proj-1'); + // $4: exemplarValues + expect((exParams[3] as number[])[0]).toBe(1.5); + // $5: exemplarTimes + expect((exParams[4] as (Date | null)[])[0]).toEqual(new Date('2024-01-01T00:00:01Z')); + // $6: traceIds + expect((exParams[5] as (string | null)[])[0]).toBe('trace-1'); + // $7: spanIds + expect((exParams[6] as (string | null)[])[0]).toBe('span-1'); + // $8: attributes JSON + expect((exParams[7] as (string | null)[])[0]).toBe(JSON.stringify({ key: 'val' })); + }); + + it('should not insert exemplars when none present (only 1 query call)', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ id: 'metric-1', time: new Date('2024-01-01T00:00:00Z') }], + }); + await engine.connect(); + + await engine.ingestMetrics([makeMetric()]); + + expect(mockQuery).toHaveBeenCalledTimes(1); + }); + + it('should handle insert errors gracefully', async () => { + mockQuery.mockRejectedValueOnce(new Error('disk full')); + await engine.connect(); + + const result = await engine.ingestMetrics([makeMetric(), makeMetric()]); + + expect(result.ingested).toBe(0); + expect(result.failed).toBe(2); + expect(result.errors).toHaveLength(1); + expect(result.errors![0].error).toBe('disk full'); + }); + + it('should serialize attributes and histogram_data as JSON strings', async () => { + const histData = { sum: 10, count: 5, min: 1, max: 3 }; + const metric = makeMetric({ + attributes: { env: 'prod' }, + histogramData: histData, + }); + + mockQuery.mockResolvedValueOnce({ + rows: [{ id: 'metric-1', time: new Date('2024-01-01T00:00:00Z') }], + }); + await engine.connect(); + + await engine.ingestMetrics([metric]); + + const params = mockQuery.mock.calls[0][1] as unknown[]; + // $9: attributes JSON + expect((params[8] as (string | null)[])[0]).toBe(JSON.stringify({ env: 'prod' })); + // $11: histogramData JSON + expect((params[10] as (string | null)[])[0]).toBe(JSON.stringify(histData)); + }); + }); + + // ========================================================================= + // queryMetrics + // ========================================================================= + + describe('queryMetrics', () => { + const defaultRow = { + id: 'metric-1', + time: new Date('2024-01-01T00:00:00Z'), + organization_id: 'org-1', + project_id: 'proj-1', + metric_name: 'cpu.usage', + metric_type: 'gauge', + value: 0.75, + is_monotonic: null, + service_name: 'api', + attributes: { host: 'server-1' }, + resource_attributes: { 'service.name': 'api' }, + histogram_data: null, + has_exemplars: false, + }; + + it('should query with time range and project filter', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 1 }] }); + mockQuery.mockResolvedValueOnce({ rows: [defaultRow] }); + await engine.connect(); + + const result = await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }); + + expect(result.metrics).toHaveLength(1); + expect(result.total).toBe(1); + expect(result.executionTimeMs).toBeGreaterThanOrEqual(0); + + const countSql = mockQuery.mock.calls[0][0] as string; + expect(countSql).toContain('COUNT(*)'); + expect(countSql).toContain('public.metrics'); + expect(countSql).toContain('m.time >='); + expect(countSql).toContain('m.time <='); + expect(countSql).toContain('m.project_id = ANY'); + + const dataSql = mockQuery.mock.calls[1][0] as string; + expect(dataSql).toContain('FROM public.metrics m'); + expect(dataSql).toContain('LIMIT'); + expect(dataSql).toContain('OFFSET'); + }); + + it('should include optional metricName filter', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 0 }] }); + mockQuery.mockResolvedValueOnce({ rows: [] }); + await engine.connect(); + + await engine.queryMetrics({ + projectId: 'proj-1', + metricName: 'cpu.usage', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }); + + const countSql = mockQuery.mock.calls[0][0] as string; + expect(countSql).toContain('m.metric_name = ANY'); + }); + + it('should include optional metricType filter', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 0 }] }); + mockQuery.mockResolvedValueOnce({ rows: [] }); + await engine.connect(); + + await engine.queryMetrics({ + projectId: 'proj-1', + metricType: 'gauge', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }); + + const countSql = mockQuery.mock.calls[0][0] as string; + expect(countSql).toContain('m.metric_type = ANY'); + }); + + it('should include optional serviceName filter', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 0 }] }); + mockQuery.mockResolvedValueOnce({ rows: [] }); + await engine.connect(); + + await engine.queryMetrics({ + projectId: 'proj-1', + serviceName: 'api', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }); + + const countSql = mockQuery.mock.calls[0][0] as string; + expect(countSql).toContain('m.service_name = ANY'); + }); + + it('should include optional attributes jsonb filter', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 0 }] }); + mockQuery.mockResolvedValueOnce({ rows: [] }); + await engine.connect(); + + await engine.queryMetrics({ + projectId: 'proj-1', + attributes: { host: 'server-1' }, + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }); + + const countSql = mockQuery.mock.calls[0][0] as string; + expect(countSql).toContain('m.attributes @>'); + expect(countSql).toContain('::jsonb'); + + const countParams = mockQuery.mock.calls[0][1] as unknown[]; + expect(countParams).toContain(JSON.stringify({ host: 'server-1' })); + }); + + it('should handle pagination (limit/offset)', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 100 }] }); + mockQuery.mockResolvedValueOnce({ rows: [defaultRow] }); + await engine.connect(); + + const result = await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + limit: 20, + offset: 40, + }); + + expect(result.limit).toBe(20); + expect(result.offset).toBe(40); + + const dataParams = mockQuery.mock.calls[1][1] as unknown[]; + // Last two params should be limit and offset + expect(dataParams[dataParams.length - 2]).toBe(20); + expect(dataParams[dataParams.length - 1]).toBe(40); + }); + + it('should map DB rows to StoredMetricRecord', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 1 }] }); + mockQuery.mockResolvedValueOnce({ rows: [defaultRow] }); + await engine.connect(); + + const result = await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }); + + const metric = result.metrics[0]; + expect(metric.id).toBe('metric-1'); + expect(metric.time).toEqual(new Date('2024-01-01T00:00:00Z')); + expect(metric.organizationId).toBe('org-1'); + expect(metric.projectId).toBe('proj-1'); + expect(metric.metricName).toBe('cpu.usage'); + expect(metric.metricType).toBe('gauge'); + expect(metric.value).toBe(0.75); + expect(metric.serviceName).toBe('api'); + expect(metric.attributes).toEqual({ host: 'server-1' }); + expect(metric.resourceAttributes).toEqual({ 'service.name': 'api' }); + expect(metric.hasExemplars).toBe(false); + }); + + it('should load exemplars when includeExemplars is true (3 queries)', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 1 }] }); + mockQuery.mockResolvedValueOnce({ rows: [{ ...defaultRow, has_exemplars: true }] }); + mockQuery.mockResolvedValueOnce({ + rows: [{ + metric_id: 'metric-1', + exemplar_value: 1.5, + exemplar_time: new Date('2024-01-01T00:00:01Z'), + trace_id: 'trace-1', + span_id: 'span-1', + attributes: { key: 'val' }, + }], + }); + await engine.connect(); + + const result = await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + includeExemplars: true, + }); + + expect(mockQuery).toHaveBeenCalledTimes(3); + + const exSql = mockQuery.mock.calls[2][0] as string; + expect(exSql).toContain('metric_exemplars'); + expect(exSql).toContain('metric_id = ANY'); + + const metric = result.metrics[0]; + expect(metric.hasExemplars).toBe(true); + expect(metric.exemplars).toHaveLength(1); + expect(metric.exemplars![0].exemplarValue).toBe(1.5); + expect(metric.exemplars![0].traceId).toBe('trace-1'); + }); + + it('should not load exemplars when includeExemplars is false (2 queries)', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 1 }] }); + mockQuery.mockResolvedValueOnce({ rows: [defaultRow] }); + await engine.connect(); + + await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + includeExemplars: false, + }); + + expect(mockQuery).toHaveBeenCalledTimes(2); + }); + + it('should calculate hasMore correctly', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ count: 100 }] }); + mockQuery.mockResolvedValueOnce({ rows: [defaultRow] }); + await engine.connect(); + + const result = await engine.queryMetrics({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + limit: 50, + offset: 0, + }); + + // offset(0) + rows.length(1) < total(100) → hasMore = true + expect(result.hasMore).toBe(true); + }); + }); + + // ========================================================================= + // aggregateMetrics + // ========================================================================= + + describe('aggregateMetrics', () => { + const baseParams = { + projectId: 'proj-1' as string | string[], + metricName: 'cpu.usage', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + interval: '1h' as const, + }; + + it('should query with time_bucket and AVG aggregation', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 0.5 }, + { bucket: new Date('2024-01-01T01:00:00Z'), agg_value: 0.8 }, + ], + }); + await engine.connect(); + + const result = await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'avg', + }); + + expect(result.metricName).toBe('cpu.usage'); + expect(result.timeseries).toHaveLength(2); + expect(result.timeseries[0].bucket).toEqual(new Date('2024-01-01T00:00:00Z')); + expect(result.timeseries[0].value).toBe(0.5); + expect(result.timeseries[1].value).toBe(0.8); + expect(result.executionTimeMs).toBeGreaterThanOrEqual(0); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('time_bucket($1, time)'); + expect(sql).toContain('AVG(value)'); + expect(sql).toContain('GROUP BY'); + expect(sql).toContain('ORDER BY bucket ASC'); + + const params = mockQuery.mock.calls[0][1] as unknown[]; + expect(params[0]).toBe('1 hour'); + }); + + it('should use SUM aggregation', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 10 }], + }); + await engine.connect(); + + await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'sum', + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('SUM(value)'); + }); + + it('should use MIN aggregation', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 0.1 }], + }); + await engine.connect(); + + await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'min', + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('MIN(value)'); + }); + + it('should use MAX aggregation', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 0.99 }], + }); + await engine.connect(); + + await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'max', + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('MAX(value)'); + }); + + it('should use COUNT aggregation', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 42 }], + }); + await engine.connect(); + + await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'count', + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('COUNT(*)'); + }); + + it('should use last aggregation (array_agg)', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 0.9 }], + }); + await engine.connect(); + + await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'last', + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('(array_agg(value ORDER BY time DESC))[1]'); + }); + + it('should include groupBy columns', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 0.5, label_0: 'server-1' }, + { bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 0.8, label_0: 'server-2' }, + ], + }); + await engine.connect(); + + await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'avg', + groupBy: ['host'], + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('attributes->>'); + expect(sql).toContain('AS label_0'); + expect(sql).toContain('GROUP BY bucket, label_0'); + + const params = mockQuery.mock.calls[0][1] as unknown[]; + expect(params).toContain('host'); + }); + + it('should include attribute filtering', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [{ bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 0.5 }], + }); + await engine.connect(); + + await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'avg', + attributes: { env: 'prod' }, + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('attributes @>'); + expect(sql).toContain('::jsonb'); + + const params = mockQuery.mock.calls[0][1] as unknown[]; + expect(params).toContain(JSON.stringify({ env: 'prod' })); + }); + + it('should return timeseries with labels when groupBy is used', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { bucket: new Date('2024-01-01T00:00:00Z'), agg_value: 0.5, label_0: 'server-1' }, + { bucket: new Date('2024-01-01T01:00:00Z'), agg_value: 0.8, label_0: 'server-2' }, + ], + }); + await engine.connect(); + + const result = await engine.aggregateMetrics({ + ...baseParams, + aggregation: 'avg', + groupBy: ['host'], + }); + + expect(result.timeseries).toHaveLength(2); + expect(result.timeseries[0].labels).toEqual({ host: 'server-1' }); + expect(result.timeseries[1].labels).toEqual({ host: 'server-2' }); + }); + }); + + // ========================================================================= + // getMetricNames + // ========================================================================= + + describe('getMetricNames', () => { + it('should return distinct metric names and types', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { metric_name: 'cpu.usage', metric_type: 'gauge' }, + { metric_name: 'http.requests', metric_type: 'sum' }, + ], + }); + await engine.connect(); + + const result = await engine.getMetricNames({ + projectId: 'proj-1', + }); + + expect(result.names).toHaveLength(2); + expect(result.names[0]).toEqual({ name: 'cpu.usage', type: 'gauge' }); + expect(result.names[1]).toEqual({ name: 'http.requests', type: 'sum' }); + expect(result.executionTimeMs).toBeGreaterThanOrEqual(0); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('SELECT DISTINCT metric_name, metric_type'); + expect(sql).toContain('FROM public.metrics'); + expect(sql).toContain('ORDER BY metric_name ASC'); + }); + + it('should filter by project and optional time range', async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + await engine.connect(); + + await engine.getMetricNames({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('project_id = ANY'); + expect(sql).toContain('time >='); + expect(sql).toContain('time <='); + + const params = mockQuery.mock.calls[0][1] as unknown[]; + expect(params[0]).toEqual(['proj-1']); + expect(params[1]).toEqual(new Date('2024-01-01')); + expect(params[2]).toEqual(new Date('2024-01-02')); + }); + }); + + // ========================================================================= + // getMetricLabelKeys + // ========================================================================= + + describe('getMetricLabelKeys', () => { + it('should return distinct attribute keys', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { key: 'env' }, + { key: 'host' }, + { key: 'region' }, + ], + }); + await engine.connect(); + + const result = await engine.getMetricLabelKeys({ + projectId: 'proj-1', + metricName: 'cpu.usage', + }); + + expect(result.keys).toEqual(['env', 'host', 'region']); + expect(result.executionTimeMs).toBeGreaterThanOrEqual(0); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('SELECT DISTINCT jsonb_object_keys(attributes) AS key'); + expect(sql).toContain('FROM public.metrics'); + expect(sql).toContain('ORDER BY key ASC'); + }); + + it('should include NULL check in WHERE clause', async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + await engine.connect(); + + await engine.getMetricLabelKeys({ + projectId: 'proj-1', + metricName: 'cpu.usage', + }); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('attributes IS NOT NULL'); + }); + }); + + // ========================================================================= + // getMetricLabelValues + // ========================================================================= + + describe('getMetricLabelValues', () => { + it('should return distinct attribute values for a label key', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { value: 'server-1' }, + { value: 'server-2' }, + { value: 'server-3' }, + ], + }); + await engine.connect(); + + const result = await engine.getMetricLabelValues( + { projectId: 'proj-1', metricName: 'cpu.usage' }, + 'host', + ); + + expect(result.values).toEqual(['server-1', 'server-2', 'server-3']); + expect(result.executionTimeMs).toBeGreaterThanOrEqual(0); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain('SELECT DISTINCT attributes->>'); + expect(sql).toContain('AS value'); + expect(sql).toContain('FROM public.metrics'); + expect(sql).toContain('ORDER BY value ASC'); + + // Check the jsonb has-key operator is in WHERE + expect(sql).toContain('attributes ?'); + }); + + it('should filter out null values from results', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { value: 'server-1' }, + { value: null }, + { value: 'server-3' }, + ], + }); + await engine.connect(); + + const result = await engine.getMetricLabelValues( + { projectId: 'proj-1', metricName: 'cpu.usage' }, + 'host', + ); + + expect(result.values).toEqual(['server-1', 'server-3']); + }); + }); + + // ========================================================================= + // deleteMetricsByTimeRange + // ========================================================================= + + describe('deleteMetricsByTimeRange', () => { + it('should delete exemplars first, then metrics', async () => { + // Delete exemplars + mockQuery.mockResolvedValueOnce({ rowCount: 0 }); + // Delete metrics + mockQuery.mockResolvedValueOnce({ rowCount: 5 }); + await engine.connect(); + + const result = await engine.deleteMetricsByTimeRange({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }); + + expect(result.deleted).toBe(5); + expect(result.executionTimeMs).toBeGreaterThanOrEqual(0); + expect(mockQuery).toHaveBeenCalledTimes(2); + + // First call: delete exemplars + const exSql = mockQuery.mock.calls[0][0] as string; + expect(exSql).toContain('DELETE FROM public.metric_exemplars'); + expect(exSql).toContain('metric_id IN'); + expect(exSql).toContain('SELECT id FROM public.metrics'); + + // Second call: delete metrics + const metricSql = mockQuery.mock.calls[1][0] as string; + expect(metricSql).toContain('DELETE FROM public.metrics'); + expect(metricSql).toContain('project_id = ANY'); + expect(metricSql).toContain('time >='); + expect(metricSql).toContain('time <='); + + // Both queries should receive the same params + const params1 = mockQuery.mock.calls[0][1] as unknown[]; + const params2 = mockQuery.mock.calls[1][1] as unknown[]; + expect(params1).toEqual(params2); + expect(params1[0]).toEqual(['proj-1']); + expect(params1[1]).toEqual(new Date('2024-01-01')); + expect(params1[2]).toEqual(new Date('2024-01-02')); + }); + + it('should include optional metricName and serviceName filters', async () => { + mockQuery.mockResolvedValueOnce({ rowCount: 0 }); + mockQuery.mockResolvedValueOnce({ rowCount: 3 }); + await engine.connect(); + + await engine.deleteMetricsByTimeRange({ + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + metricName: 'cpu.usage', + serviceName: 'api', + }); + + const metricSql = mockQuery.mock.calls[1][0] as string; + expect(metricSql).toContain('metric_name = ANY'); + expect(metricSql).toContain('service_name = ANY'); + + const params = mockQuery.mock.calls[1][1] as unknown[]; + expect(params).toContainEqual(['cpu.usage']); + expect(params).toContainEqual(['api']); + }); + }); +}); From 4dc65d769f35998e2c8b3b73e842f465acf5cfd0 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 19:07:55 +0100 Subject: [PATCH 06/43] feat: create database if it doesn't exist during initialization --- .../src/engines/clickhouse/clickhouse-engine.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/reservoir/src/engines/clickhouse/clickhouse-engine.ts b/packages/reservoir/src/engines/clickhouse/clickhouse-engine.ts index 715968e5..b972cbd7 100644 --- a/packages/reservoir/src/engines/clickhouse/clickhouse-engine.ts +++ b/packages/reservoir/src/engines/clickhouse/clickhouse-engine.ts @@ -133,6 +133,20 @@ export class ClickHouseEngine extends StorageEngine { async initialize(): Promise { if (this.options.skipInitialize) return; + // Create database if it doesn't exist (connect without specifying database) + const bootstrapClient = createClient({ + url: `http://${this.config.host}:${this.config.port}`, + username: this.config.username, + password: this.config.password, + }); + try { + await bootstrapClient.command({ + query: `CREATE DATABASE IF NOT EXISTS ${this.config.database}`, + }); + } finally { + await bootstrapClient.close(); + } + const client = this.getClient(); const t = this.tableName; From 4221347a0ba0164ffd74e44d13600ddfcbceb9e6 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 19:11:45 +0100 Subject: [PATCH 07/43] feat: update Dockerfile to include tsx for scripts and copy source scripts --- packages/backend/Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/backend/Dockerfile b/packages/backend/Dockerfile index ae948820..171db295 100644 --- a/packages/backend/Dockerfile +++ b/packages/backend/Dockerfile @@ -34,8 +34,9 @@ COPY packages/shared/package.json ./packages/shared/ COPY packages/reservoir/package.json ./packages/reservoir/ COPY packages/backend/package.json ./packages/backend/ -# Install production dependencies only +# Install production dependencies + tsx for scripts RUN pnpm install --frozen-lockfile --prod +RUN pnpm --filter '@logtide/backend' add tsx # Copy built files COPY --from=builder /app/packages/shared/dist ./packages/shared/dist @@ -43,6 +44,12 @@ COPY --from=builder /app/packages/reservoir/dist ./packages/reservoir/dist COPY --from=builder /app/packages/backend/dist ./packages/backend/dist COPY --from=builder /app/packages/backend/migrations ./packages/backend/migrations +# Copy source scripts (for tsx runtime execution) +COPY --from=builder /app/packages/backend/src/scripts ./packages/backend/src/scripts +COPY --from=builder /app/packages/backend/tsconfig.json ./packages/backend/tsconfig.json +COPY --from=builder /app/packages/shared/package.json ./packages/shared/package.json +COPY --from=builder /app/packages/reservoir/package.json ./packages/reservoir/package.json + # Copy entrypoint script COPY packages/backend/entrypoint.sh ./packages/backend/entrypoint.sh RUN chmod +x ./packages/backend/entrypoint.sh From 72a5df031c3f5de24904589a057af6095f721ec6 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 19:16:25 +0100 Subject: [PATCH 08/43] add ux restructuring design doc --- .../2026-02-22-ux-restructuring-design.md | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 docs/plans/2026-02-22-ux-restructuring-design.md diff --git a/docs/plans/2026-02-22-ux-restructuring-design.md b/docs/plans/2026-02-22-ux-restructuring-design.md new file mode 100644 index 00000000..0a1f75f6 --- /dev/null +++ b/docs/plans/2026-02-22-ux-restructuring-design.md @@ -0,0 +1,108 @@ +# UX Restructuring Design + +**Date:** 2026-02-22 +**Branch:** feature/0.7-ux +**Status:** Approved + +## Problem + +The current UI has 11 flat sidebar items with no grouping. Observability features (Traces, Metrics, Service Map) are separate islands with no shared context. Alerts and Security overlap confusingly (Sigma rules managed in Alerts, output shown in Security). Project pages duplicate the global log search with a worse, copy-pasted version. Cross-page links are broken. No context (project, time range) persists across page navigation. + +## Design + +### 1. Sidebar Restructuring + +**Before:** Dashboard | Projects | Logs | Traces | Metrics | Service Map | Alerts | Errors | Security | Docs | Settings + +**After:** + +``` +Dashboard (standalone at top) + +── OBSERVE ────────────── + Logs /dashboard/search + Traces /dashboard/traces (absorbs Service Map) + Metrics /dashboard/metrics + Errors /dashboard/errors + +── DETECT ─────────────── + Alerts threshold/anomaly rules + alert history + Security Sigma rules + SIEM dashboard + incidents + +── MANAGE ─────────────── + Projects list + API keys + settings (NO log viewer) + Settings org settings, channels, PII, audit +``` + +- Service Map removed from sidebar → tab inside Traces +- Sigma Rules tab removed from Alerts → sub-page of Security +- Section labels: small gray uppercase text + separator +- Docs link moves to sidebar footer +- Command palette updated to match + +### 2. Global Context Bar + +Persistent bar in the topbar on all Observe pages (`/dashboard/search`, `/dashboard/traces`, `/dashboard/metrics`, `/dashboard/errors`). + +``` +[Logo] OrgName │ Project: [All ▼] Time: [Last 24h ▼] │ 🔔 👤 +``` + +- New `observeContextStore` with `selectedProjects: string[]`, `timeRange: { type, from?, to? }` +- Persisted to `sessionStorage` +- Renders only on Observe routes +- Each Observe page reads from the store (removes per-page project/time selectors) +- Per-page filters (service, level, trace ID) remain local +- Search page keeps its multi-project selector but initializes from the store +- Extract duplicated `getTimeRange()` into shared utility + +### 3. Traces Absorbs Service Map + +Traces page gets a view switcher at the top of results: + +``` +[List] [Map] +``` + +- **List view** (default): current traces table +- **Map view**: full Service Map with side panel, health legend, export PNG +- Stats cards and filters visible in all views +- Clicking a node in Map → switches to List filtered by that service +- Delete `/dashboard/service-map/` route entirely + +### 4. Security Absorbs Sigma Rules + +**Alerts (simplified):** 2 tabs → Alert Rules | History (threshold/anomaly only) + +**Security (expanded):** horizontal sub-nav → Dashboard | Rules | Incidents + +``` +/dashboard/security → SIEM Dashboard +/dashboard/security/rules → Sigma Rules (moved from Alerts) +/dashboard/security/incidents → Incident list +/dashboard/security/incidents/[id] → Incident detail +``` + +- Security empty state links to `/dashboard/security/rules` +- Detection events category `security` → Security +- Detection events category `reliability/database/business` → Alerts History + +### 5. Projects Simplified + +**Before:** 3 tabs → Logs | Alerts | Settings +**After:** 2 tabs → API Keys | Settings + +- Delete project log viewer (`projects/[id]/+page.svelte`, ~937 lines) +- "View Logs" on project card → `/dashboard/search?project={id}` +- Default tab = API Keys + +### 6. Bug Fixes + +- Metrics trace link: navigate to `/dashboard/traces/{traceId}?projectId={id}` +- Traces page reads `service`, `projectId`, `traceId` from URL params +- Search/Traces: show error toast on API failure instead of silent empty +- LogsChart: use browser locale instead of hardcoded `it-IT` +- Search checkboxes: replace raw `` with ShadCN Checkbox +- Add empty states to TopServicesWidget/RecentErrorsWidget +- Fix `effectiveTotalLogs` to use API total +- Remove dead code: `filteredLogs`, `Navigation.svelte`, unreachable `pageSize` From 7a79a9872a69ced9ce18d02c8ad7a8f383bb6d91 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 19:21:42 +0100 Subject: [PATCH 09/43] add ux restructuring implementation plan --- docs/plans/2026-02-22-ux-restructuring.md | 1075 +++++++++++++++++++++ 1 file changed, 1075 insertions(+) create mode 100644 docs/plans/2026-02-22-ux-restructuring.md diff --git a/docs/plans/2026-02-22-ux-restructuring.md b/docs/plans/2026-02-22-ux-restructuring.md new file mode 100644 index 00000000..b2775e29 --- /dev/null +++ b/docs/plans/2026-02-22-ux-restructuring.md @@ -0,0 +1,1075 @@ +# UX Restructuring Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Restructure the LogTide dashboard UX — group sidebar items, add global context bar, merge Service Map into Traces, move Sigma Rules into Security, simplify Projects, fix broken cross-page links. + +**Architecture:** The sidebar gets section labels (Observe/Detect/Manage). A new `observeContextStore` persists project + time range across Observe pages. Service Map becomes a view tab inside Traces. Sigma Rules move from Alerts to Security. Project log viewer is deleted (937 lines of duplication). + +**Tech Stack:** SvelteKit (Svelte 5 runes), TypeScript, Tailwind CSS, ShadCN-Svelte components, Svelte stores (writable/derived pattern from `layoutStore`) + +--- + +## Task 1: Create `observeContextStore` + +**Files:** +- Create: `packages/frontend/src/lib/stores/observe-context.ts` +- Reference: `packages/frontend/src/lib/stores/layout.ts` (follow this pattern exactly) + +**Step 1: Create the store** + +```typescript +import { writable, derived, get } from 'svelte/store'; +import { browser } from '$app/environment'; + +export type TimeRangeType = 'last_hour' | 'last_24h' | 'last_7d' | 'custom'; + +interface ObserveContextState { + selectedProjects: string[]; + timeRangeType: TimeRangeType; + customFrom: string; + customTo: string; +} + +const STORAGE_KEY = 'logtide_observe_context'; + +const DEFAULTS: ObserveContextState = { + selectedProjects: [], + timeRangeType: 'last_24h', + customFrom: '', + customTo: '', +}; + +function loadInitialState(): ObserveContextState { + if (!browser) return DEFAULTS; + try { + const saved = sessionStorage.getItem(STORAGE_KEY); + if (saved) { + const parsed = JSON.parse(saved); + return { ...DEFAULTS, ...parsed }; + } + return DEFAULTS; + } catch { + return DEFAULTS; + } +} + +function persist(state: ObserveContextState) { + if (browser) { + try { + sessionStorage.setItem(STORAGE_KEY, JSON.stringify(state)); + } catch {} + } +} + +function createObserveContextStore() { + const { subscribe, set, update } = writable(loadInitialState()); + + return { + subscribe, + + setProjects: (projects: string[]) => { + update((s) => { + const newState = { ...s, selectedProjects: projects }; + persist(newState); + return newState; + }); + }, + + setTimeRange: (type: TimeRangeType, customFrom?: string, customTo?: string) => { + update((s) => { + const newState = { + ...s, + timeRangeType: type, + customFrom: customFrom || s.customFrom, + customTo: customTo || s.customTo, + }; + persist(newState); + return newState; + }); + }, + + getTimeRange: (): { from: Date; to: Date } => { + const state = get(observeContextStore); + const now = new Date(); + switch (state.timeRangeType) { + case 'last_hour': + return { from: new Date(now.getTime() - 60 * 60 * 1000), to: now }; + case 'last_24h': + return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; + case 'last_7d': + return { from: new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000), to: now }; + case 'custom': { + const from = state.customFrom ? new Date(state.customFrom) : new Date(now.getTime() - 24 * 60 * 60 * 1000); + const to = state.customTo ? new Date(state.customTo) : now; + return { from, to }; + } + default: + return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; + } + }, + + clear: () => { + set(DEFAULTS); + if (browser) sessionStorage.removeItem(STORAGE_KEY); + }, + }; +} + +export const observeContextStore = createObserveContextStore(); + +export const selectedProjects = derived(observeContextStore, ($s) => $s.selectedProjects); +export const timeRangeType = derived(observeContextStore, ($s) => $s.timeRangeType); +``` + +**Step 2: Verify it compiles** + +Run: `cd packages/frontend && npx svelte-check --tsconfig ./tsconfig.json 2>&1 | head -20` + +**Step 3: Commit** + +```bash +git add packages/frontend/src/lib/stores/observe-context.ts +git commit -m "add observe context store" +``` + +--- + +## Task 2: Create `ObserveContextBar` component + +**Files:** +- Create: `packages/frontend/src/lib/components/ObserveContextBar.svelte` +- Reference: `packages/frontend/src/lib/components/TimeRangePicker.svelte` for time range UI pattern +- Reference: `packages/frontend/src/routes/dashboard/search/+page.svelte:908-996` for project multi-select popover pattern + +**Step 1: Create the component** + +This component renders in the topbar on Observe pages. It shows: +- A project multi-select popover (using the same Popover + checkbox pattern from search page) +- A time range selector (using button group like security page, lines 166-191) + +```svelte + + +{#if projects.length > 0} +
+ + + + + + +
+ Projects +
+ + | + +
+
+
+ {#each projects as project} + + {/each} +
+ + + + +
+ {#each timeRangeOptions as opt} + + {/each} +
+
+{/if} +``` + +**Step 2: Verify it compiles** + +Run: `cd packages/frontend && npx svelte-check --tsconfig ./tsconfig.json 2>&1 | head -20` + +**Step 3: Commit** + +```bash +git add packages/frontend/src/lib/components/ObserveContextBar.svelte +git commit -m "add observe context bar component" +``` + +--- + +## Task 3: Wire `ObserveContextBar` into `AppLayout.svelte` + +**Files:** +- Modify: `packages/frontend/src/lib/components/AppLayout.svelte` + +**Step 1: Add import and route detection** + +Add after line 56 (`import { logoPath } from "$lib/utils/theme";`): + +```typescript +import ObserveContextBar from "$lib/components/ObserveContextBar.svelte"; +``` + +Add a derived that checks if the current route is an Observe page. Add inside the ` + +
+
+ +
+
+ +{@render children()} +``` + +Note: The existing `security/+page.svelte` will need its header adjusted since the sub-nav now provides context. + +**Step 2: Create `/dashboard/security/rules/+page.svelte`** + +Move the Sigma Rules tab content from `alerts/+page.svelte` (lines 550-590 and related state/functions) into this new page. The page should include: +- `SigmaRulesList` component +- `SigmaSyncDialog` button and dialog +- `DetectionPacksGalleryDialog` button and dialog +- `SigmaRuleDetailsDialog` for viewing rule details +- All the sigma-related state and loading logic from alerts page + +**Step 3: Simplify Alerts page** + +Remove from `alerts/+page.svelte`: +- The "Sigma Rules" tab and `TabsTrigger` +- All sigma-related imports, state, and functions +- Change `grid-cols-3` to `grid-cols-2` in the `TabsList` +- Keep only "Alert Rules" and "History" tabs +- In History tab, keep showing detection events (reliability/database/business categories) — these are operational, not security + +**Step 4: Fix Security dashboard empty state link** + +In `security/+page.svelte` line 220, change: +```typescript +// Before: +onAction={() => goto('/dashboard/alerts')} +// After: +onAction={() => goto('/dashboard/security/rules')} +``` + +**Step 5: Commit** + +```bash +git add -A +git commit -m "move sigma rules from alerts to security section" +``` + +--- + +## Task 11: Simplify Project detail pages + +**Files:** +- Delete: `packages/frontend/src/routes/dashboard/projects/[id]/+page.svelte` (~937 lines) +- Modify: `packages/frontend/src/routes/dashboard/projects/[id]/+layout.svelte` (change tabs) +- Modify: `packages/frontend/src/routes/dashboard/projects/+page.svelte` (update "View Project" link) + +**Step 1: Update project detail layout tabs** + +In `[id]/+layout.svelte`, replace the 3-tab structure (lines 114-120) with 2 tabs: + +```svelte + + + API Keys & Settings + Alerts + + +``` + +Update `currentTab` derived (lines 44-48): +```typescript +const currentTab = $derived(() => { + if (currentPath.endsWith('/alerts')) return 'alerts'; + return 'settings'; +}); +``` + +Update `handleTabChange` (lines 89-96): +```typescript +function handleTabChange(tab: string) { + const basePath = `/dashboard/projects/${projectId}`; + if (tab === 'settings') { goto(basePath); } + else { goto(`${basePath}/${tab}`); } +} +``` + +**Step 2: Rename settings page to be the default** + +Move `packages/frontend/src/routes/dashboard/projects/[id]/settings/+page.svelte` to `packages/frontend/src/routes/dashboard/projects/[id]/+page.svelte` (replacing the deleted log viewer). + +Or simpler: keep settings at its current route and make the default route redirect to settings: + +Create a new `packages/frontend/src/routes/dashboard/projects/[id]/+page.svelte`: +```svelte + +``` + +**Step 3: Update project list "View Project" button** + +In `projects/+page.svelte`, find the "View Project" link and change it to navigate to settings: +```typescript +// Change href from: +href={`/dashboard/projects/${project.id}`} +// To: +href={`/dashboard/projects/${project.id}/settings`} +``` + +Add a "View Logs" button next to it: +```svelte + +``` + +**Step 4: Commit** + +```bash +git add -A +git commit -m "simplify project pages, remove duplicate log viewer" +``` + +--- + +## Task 12: Bug fixes batch + +**Files:** +- Modify: `packages/frontend/src/routes/dashboard/traces/+page.svelte` (silent error fix) +- Modify: `packages/frontend/src/lib/components/dashboard/LogsChart.svelte` (locale fix) +- Modify: `packages/frontend/src/lib/components/dashboard/TopServicesWidget.svelte` (empty state) +- Modify: `packages/frontend/src/lib/components/dashboard/RecentErrorsWidget.svelte` (empty state) +- Delete: `packages/frontend/src/lib/components/Navigation.svelte` (dead code) + +**Step 1: Fix Traces silent error handling** + +In `traces/+page.svelte`, in the `catch` block of `loadTraces()`, add: +```typescript +toastStore.error('Failed to load traces'); +``` + +**Step 2: Fix LogsChart locale** + +In `LogsChart.svelte`, find the `it-IT` locale usage and replace with `undefined` (uses browser default): +```typescript +// Before: +new Date(label).toLocaleTimeString('it-IT', ...) +// After: +new Date(label).toLocaleTimeString(undefined, ...) +``` + +**Step 3: Add empty states to dashboard widgets** + +In `TopServicesWidget.svelte`, add after the `{#each}` block: +```svelte +{#if services.length === 0} +

No services yet

+{/if} +``` + +Same pattern for `RecentErrorsWidget.svelte`. + +**Step 4: Delete Navigation.svelte** + +```bash +rm packages/frontend/src/lib/components/Navigation.svelte +``` + +Verify it's not imported anywhere: +```bash +grep -r "Navigation" packages/frontend/src/lib/components/ --include="*.svelte" --include="*.ts" +``` + +**Step 5: Commit** + +```bash +git add -A +git commit -m "fix locale bug, silent errors, empty states, remove dead code" +``` + +--- + +## Task 13: Update E2E tests + +**Files:** +- Modify: E2E test files that reference removed/changed routes + +**Step 1: Find affected tests** + +```bash +grep -r "service-map\|/dashboard/projects/.*logs\|sigma.*alerts" packages/frontend/tests/ e2e/ --include="*.ts" -l +``` + +**Step 2: Update route references** + +- Replace `/dashboard/service-map` references with `/dashboard/traces` (Map view) +- Replace `/dashboard/projects/[id]` log viewer references with `/dashboard/search?project=[id]` +- Update Sigma rule test paths from `/dashboard/alerts` to `/dashboard/security/rules` +- Update sidebar navigation selectors from flat list to grouped structure + +**Step 3: Run E2E tests to verify** + +```bash +npx playwright test --reporter=list 2>&1 | tail -30 +``` + +**Step 4: Commit** + +```bash +git add -A +git commit -m "update e2e tests for new navigation structure" +``` + +--- + +## Task 14: Final verification + +**Step 1: Run type checks** + +```bash +cd packages/frontend && npx svelte-check --tsconfig ./tsconfig.json +``` + +**Step 2: Run backend tests** + +```bash +cd packages/backend && npx vitest run 2>&1 | tail -20 +``` + +**Step 3: Build the frontend** + +```bash +cd packages/frontend && npm run build +``` + +**Step 4: Manual smoke test checklist** + +- [ ] Sidebar shows 3 groups (Observe/Detect/Manage) with section labels +- [ ] Context bar appears in topbar on Logs/Traces/Metrics/Errors pages +- [ ] Context bar does NOT appear on Dashboard/Alerts/Security/Projects/Settings +- [ ] Changing project in context bar reloads current Observe page +- [ ] Navigating between Observe pages preserves project + time range selection +- [ ] Traces page has List/Map view toggle +- [ ] Map view shows full Service Map with side panel and export +- [ ] `/dashboard/service-map` returns 404 (deleted) +- [ ] Security page has Dashboard/Rules/Incidents sub-nav +- [ ] Security → Rules shows Sigma rules list with sync button +- [ ] Alerts page has only 2 tabs (Alert Rules / History) +- [ ] Projects page "View Logs" goes to global search with project pre-filtered +- [ ] Project detail opens to Settings/API Keys tab (no Logs tab) +- [ ] Command palette (Cmd+K) shows all pages including Metrics +- [ ] Metrics exemplar trace link navigates to trace detail (not list) + +**Step 5: Commit any remaining fixes** + +```bash +git add -A +git commit -m "final fixes from smoke test" +``` From fa07bc2f327a6ccc8dbba70d81031e1d7dabd6fd Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 20:43:08 +0100 Subject: [PATCH 10/43] add observe context store --- .../src/lib/stores/observe-context.ts | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 packages/frontend/src/lib/stores/observe-context.ts diff --git a/packages/frontend/src/lib/stores/observe-context.ts b/packages/frontend/src/lib/stores/observe-context.ts new file mode 100644 index 00000000..eecbee7d --- /dev/null +++ b/packages/frontend/src/lib/stores/observe-context.ts @@ -0,0 +1,132 @@ +import { writable, derived, get } from 'svelte/store'; +import { browser } from '$app/environment'; + +export type TimeRangeType = 'last_hour' | 'last_24h' | 'last_7d' | 'custom'; + +interface ObserveContextState { + selectedProjects: string[]; + timeRangeType: TimeRangeType; + customFrom: string; + customTo: string; +} + +const STORAGE_KEY = 'logtide_observe_context'; + +const VALID_TIME_RANGE_TYPES: TimeRangeType[] = ['last_hour', 'last_24h', 'last_7d', 'custom']; + +const DEFAULTS: ObserveContextState = { + selectedProjects: [], + timeRangeType: 'last_24h', + customFrom: '', + customTo: '', +}; + +function loadInitialState(): ObserveContextState { + if (!browser) return DEFAULTS; + + try { + const raw = sessionStorage.getItem(STORAGE_KEY); + if (!raw) return DEFAULTS; + + const parsed = JSON.parse(raw); + + const timeRangeType = VALID_TIME_RANGE_TYPES.includes(parsed.timeRangeType) + ? parsed.timeRangeType + : DEFAULTS.timeRangeType; + + const selectedProjects = Array.isArray(parsed.selectedProjects) + ? parsed.selectedProjects.filter((p: unknown) => typeof p === 'string') + : DEFAULTS.selectedProjects; + + const customFrom = typeof parsed.customFrom === 'string' ? parsed.customFrom : DEFAULTS.customFrom; + const customTo = typeof parsed.customTo === 'string' ? parsed.customTo : DEFAULTS.customTo; + + return { selectedProjects, timeRangeType, customFrom, customTo }; + } catch { + return DEFAULTS; + } +} + +function persist(state: ObserveContextState) { + if (browser) { + try { + sessionStorage.setItem(STORAGE_KEY, JSON.stringify(state)); + } catch { + // sessionStorage unavailable + } + } +} + +function createObserveContextStore() { + const initialState = loadInitialState(); + const { subscribe, set, update } = writable(initialState); + + return { + subscribe, + + setProjects: (projects: string[]) => { + update((state) => { + const newState = { ...state, selectedProjects: projects }; + persist(newState); + return newState; + }); + }, + + setTimeRange: (type: TimeRangeType, customFrom?: string, customTo?: string) => { + update((state) => { + const newState = { + ...state, + timeRangeType: type, + customFrom: customFrom ?? (type === 'custom' ? state.customFrom : ''), + customTo: customTo ?? (type === 'custom' ? state.customTo : ''), + }; + persist(newState); + return newState; + }); + }, + + getTimeRange: (): { from: Date; to: Date } => { + const state = get({ subscribe }); + const now = new Date(); + + switch (state.timeRangeType) { + case 'last_hour': + return { from: new Date(now.getTime() - 60 * 60 * 1000), to: now }; + case 'last_24h': + return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; + case 'last_7d': + return { from: new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000), to: now }; + case 'custom': { + const from = state.customFrom ? new Date(state.customFrom) : new Date(now.getTime() - 24 * 60 * 60 * 1000); + const to = state.customTo ? new Date(state.customTo) : now; + return { from, to }; + } + default: + return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; + } + }, + + clear: () => { + set(DEFAULTS); + if (browser) { + try { + sessionStorage.removeItem(STORAGE_KEY); + } catch { + // sessionStorage unavailable + } + } + }, + }; +} + +export const observeContextStore = createObserveContextStore(); + +export const selectedProjects = derived( + observeContextStore, + ($state) => $state.selectedProjects, +); + +export const timeRangeType = derived( + observeContextStore, + ($state) => $state.timeRangeType, +); From f559480ee624e9881b848562281bb2e1dfdf02e1 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 20:51:44 +0100 Subject: [PATCH 11/43] add observe context bar component --- .../lib/components/ObserveContextBar.svelte | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 packages/frontend/src/lib/components/ObserveContextBar.svelte diff --git a/packages/frontend/src/lib/components/ObserveContextBar.svelte b/packages/frontend/src/lib/components/ObserveContextBar.svelte new file mode 100644 index 00000000..5c81cb82 --- /dev/null +++ b/packages/frontend/src/lib/components/ObserveContextBar.svelte @@ -0,0 +1,164 @@ + + +
+ + + + + + +
+
+

Projects

+
+ + / + +
+
+
+
+ {#if loading} +

Loading...

+ {:else if projects.length === 0} +

No projects found

+ {:else} + {#each projects as project} + + {/each} + {/if} +
+ + + + +
+ {#each timeRangeOptions as opt} + + {/each} +
+
From edb63e37a3437172622be776d8abf8eef9274215 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 20:51:47 +0100 Subject: [PATCH 12/43] restructure sidebar, wire context bar --- .../src/lib/components/AppLayout.svelte | 196 +++++++++++------- 1 file changed, 120 insertions(+), 76 deletions(-) diff --git a/packages/frontend/src/lib/components/AppLayout.svelte b/packages/frontend/src/lib/components/AppLayout.svelte index 5069d965..a96a865f 100644 --- a/packages/frontend/src/lib/components/AppLayout.svelte +++ b/packages/frontend/src/lib/components/AppLayout.svelte @@ -38,7 +38,6 @@ import LogOut from "@lucide/svelte/icons/log-out"; import Menu from "@lucide/svelte/icons/menu"; import Shield from "@lucide/svelte/icons/shield"; - import Network from "@lucide/svelte/icons/network"; import Book from "@lucide/svelte/icons/book"; import Github from "@lucide/svelte/icons/github"; import X from "@lucide/svelte/icons/x"; @@ -54,6 +53,7 @@ import FeatureBadge from "$lib/components/FeatureBadge.svelte"; import ThemeToggle from "$lib/components/ThemeToggle.svelte"; import { logoPath } from "$lib/utils/theme"; + import ObserveContextBar from "$lib/components/ObserveContextBar.svelte"; interface Props { children?: import("svelte").Snippet; @@ -230,42 +230,45 @@ }; } - const navigationItems: NavItem[] = [ - { label: "Dashboard", href: "/dashboard", icon: LayoutDashboard }, - { label: "Projects", href: "/dashboard/projects", icon: FolderKanban }, - { label: "Logs", href: "/dashboard/search", icon: FileText }, - { - label: "Traces", - href: "/dashboard/traces", - icon: GitBranch, - badge: { id: 'traces-feature', type: 'new', showUntil: '2025-03-01' } - }, + interface NavSection { + label?: string; + items: NavItem[]; + } + + const navigationSections: NavSection[] = [ { - label: "Metrics", - href: "/dashboard/metrics", - icon: BarChart3, - badge: { id: 'metrics-feature', type: 'new', showUntil: '2026-09-01' } + items: [ + { label: "Dashboard", href: "/dashboard", icon: LayoutDashboard }, + ], }, { - label: "Service Map", - href: "/dashboard/service-map", - icon: Network, + label: "Observe", + items: [ + { label: "Logs", href: "/dashboard/search", icon: FileText }, + { label: "Traces", href: "/dashboard/traces", icon: GitBranch }, + { + label: "Metrics", + href: "/dashboard/metrics", + icon: BarChart3, + badge: { id: 'metrics-feature', type: 'new', showUntil: '2026-09-01' } + }, + { label: "Errors", href: "/dashboard/errors", icon: Bug }, + ], }, - { label: "Alerts", href: "/dashboard/alerts", icon: AlertTriangle }, { - label: "Errors", - href: "/dashboard/errors", - icon: Bug, - badge: { id: 'errors-feature', type: 'new', showUntil: '2025-06-01' } + label: "Detect", + items: [ + { label: "Alerts", href: "/dashboard/alerts", icon: AlertTriangle }, + { label: "Security", href: "/dashboard/security", icon: Shield }, + ], }, { - label: "Security", - href: "/dashboard/security", - icon: Shield, - badge: { id: 'security-feature', type: 'new', showUntil: '2025-06-01' } + label: "Manage", + items: [ + { label: "Projects", href: "/dashboard/projects", icon: FolderKanban }, + { label: "Settings", href: "/dashboard/settings", icon: Settings }, + ], }, - { label: "Docs", href: "https://logtide.dev/docs", icon: Book, external: true }, - { label: "Settings", href: "/dashboard/settings", icon: Settings }, ]; function isActive(href: string): boolean { @@ -278,6 +281,9 @@ ); } + const OBSERVE_PATHS = ['/dashboard/search', '/dashboard/traces', '/dashboard/metrics', '/dashboard/errors']; + let isObservePage = $derived(OBSERVE_PATHS.some((p) => page.url.pathname === p || page.url.pathname.startsWith(p + '/'))); + async function handleLogout() { authStore.clearAuth(); goto("/login"); @@ -325,29 +331,37 @@
{/if} + {#if isObservePage} + + {/if}
From 6a72c777e52464db904e99d797973e89af9d881c Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 20:53:01 +0100 Subject: [PATCH 13/43] update command palette nav items --- packages/frontend/src/lib/components/CommandPalette.svelte | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/frontend/src/lib/components/CommandPalette.svelte b/packages/frontend/src/lib/components/CommandPalette.svelte index 466a0c43..07234993 100644 --- a/packages/frontend/src/lib/components/CommandPalette.svelte +++ b/packages/frontend/src/lib/components/CommandPalette.svelte @@ -12,6 +12,7 @@ import Bug from '@lucide/svelte/icons/bug'; import Shield from '@lucide/svelte/icons/shield'; import Settings from '@lucide/svelte/icons/settings'; + import BarChart3 from '@lucide/svelte/icons/bar-chart-3'; import PanelLeftClose from '@lucide/svelte/icons/panel-left-close'; import Keyboard from '@lucide/svelte/icons/keyboard'; import RefreshCw from '@lucide/svelte/icons/refresh-cw'; @@ -36,12 +37,13 @@ const navItems = [ { label: 'Dashboard', href: '/dashboard', icon: LayoutDashboard, shortcut: 'g d' }, - { label: 'Projects', href: '/dashboard/projects', icon: FolderKanban, shortcut: 'g p' }, { label: 'Logs', href: '/dashboard/search', icon: FileText, shortcut: 'g s' }, { label: 'Traces', href: '/dashboard/traces', icon: GitBranch, shortcut: 'g t' }, - { label: 'Alerts', href: '/dashboard/alerts', icon: AlertTriangle, shortcut: 'g a' }, + { label: 'Metrics', href: '/dashboard/metrics', icon: BarChart3, shortcut: 'g m' }, { label: 'Errors', href: '/dashboard/errors', icon: Bug, shortcut: 'g r' }, + { label: 'Alerts', href: '/dashboard/alerts', icon: AlertTriangle, shortcut: 'g a' }, { label: 'Security', href: '/dashboard/security', icon: Shield, shortcut: 'g e' }, + { label: 'Projects', href: '/dashboard/projects', icon: FolderKanban, shortcut: 'g p' }, { label: 'Settings', href: '/dashboard/settings', icon: Settings, shortcut: 'g x' }, ]; From 51d3abb82a6efe0215f828f69645abafa0c8586d Mon Sep 17 00:00:00 2001 From: Polliog Date: Sun, 22 Feb 2026 21:15:11 +0100 Subject: [PATCH 14/43] merge service map into traces, use observe context store --- .../routes/dashboard/service-map/+page.svelte | 452 ------- .../src/routes/dashboard/traces/+page.svelte | 1065 ++++++++++------- 2 files changed, 639 insertions(+), 878 deletions(-) delete mode 100644 packages/frontend/src/routes/dashboard/service-map/+page.svelte diff --git a/packages/frontend/src/routes/dashboard/service-map/+page.svelte b/packages/frontend/src/routes/dashboard/service-map/+page.svelte deleted file mode 100644 index 540d9b9d..00000000 --- a/packages/frontend/src/routes/dashboard/service-map/+page.svelte +++ /dev/null @@ -1,452 +0,0 @@ - - - - Service Map - LogTide - - -
- -
-
- -

Service Map

-
-

- Visualize service dependencies and correlations across your infrastructure -

-
- - - - - Filters - - -
-
- - { - if (v) { - selectedProject = v; - selectedNode = null; - loadMap(); - } - }} - > - - {projects.find((p) => p.id === selectedProject)?.name || - "Select project"} - - - {#each projects as project} - {project.name} - {/each} - - -
- -
- - { - selectedNode = null; - loadMap(); - }} - /> -
-
- - - - -
-
-
- - Healthy (<1%) -
-
- - Degraded (1-10%) -
-
- - Unhealthy (>10%) -
-
- - Log correlation -
-
- -
- - -
- -
- - - {#if isLoading} -
- -
- {:else if loadError} -
-

{loadError}

-
- {:else if mapData && mapData.nodes.length > 0} - - {:else if mapData} -
-
- -

No service dependencies found

-

- Send traces with parent-child spans or logs with trace_id to - see service relationships -

-
-
- {:else} -
-

Select a project to view the service map

-
- {/if} - - -
- - - {#if selectedNode} - {@const health = getHealthLabel(selectedNode.errorRate)} -
- - -
- {selectedNode.name} - - {health.label} - -
- - - - -
-
-

Error Rate

-

- {(selectedNode.errorRate * 100).toFixed(1)}% -

-
-
-

Avg Latency

-

- {formatLatency(selectedNode.avgLatencyMs)} -

-
-
-

P95 Latency

-

- {selectedNode.p95LatencyMs != null - ? formatLatency(selectedNode.p95LatencyMs) - : "N/A"} -

-
-
-

Total Calls

-

- {selectedNode.totalCalls.toLocaleString()} -

-
-
- - - {#if downstreamEdges.length > 0} -
-

- - Calls to -

-
- {#each downstreamEdges as edge} -
- {edge.target} -
- - {edge.callCount} - - {#if edge.type === "log_correlation"} - log - {/if} -
-
- {/each} -
-
- {/if} - - - {#if upstreamEdges.length > 0} -
-

- - Called by -

-
- {#each upstreamEdges as edge} -
- {edge.source} -
- - {edge.callCount} - - {#if edge.type === "log_correlation"} - log - {/if} -
-
- {/each} -
-
- {/if} - - - - -
- {/if} -
-
diff --git a/packages/frontend/src/routes/dashboard/traces/+page.svelte b/packages/frontend/src/routes/dashboard/traces/+page.svelte index 5e109838..ac545327 100644 --- a/packages/frontend/src/routes/dashboard/traces/+page.svelte +++ b/packages/frontend/src/routes/dashboard/traces/+page.svelte @@ -1,14 +1,19 @@ @@ -387,15 +364,6 @@

- -
- - {#if loading} -
- - Loading Sigma rules... -
- {:else if sigmaRules.length === 0} - - -
- -
-

No Sigma rules yet

-

- Import Sigma rules to automatically create alert rules from community standards -

- - - - {:else} - { - selectedSigmaRule = rule; - showSigmaDetails = true; - }} - /> - {/if} - - {#if loadingHistory || detectionsLoading} @@ -855,26 +774,6 @@ }} /> - - - { - loadAlertRules(); - }} - /> - - { - loadAlertRules(); - }} - /> diff --git a/packages/frontend/src/routes/dashboard/security/+layout.svelte b/packages/frontend/src/routes/dashboard/security/+layout.svelte new file mode 100644 index 00000000..47e7580d --- /dev/null +++ b/packages/frontend/src/routes/dashboard/security/+layout.svelte @@ -0,0 +1,40 @@ + + +
+
+ +
+
+ +{@render children()} diff --git a/packages/frontend/src/routes/dashboard/security/+page.svelte b/packages/frontend/src/routes/dashboard/security/+page.svelte index dc913d57..f1b572ad 100644 --- a/packages/frontend/src/routes/dashboard/security/+page.svelte +++ b/packages/frontend/src/routes/dashboard/security/+page.svelte @@ -217,7 +217,7 @@ title="No security events detected" description="Enable Sigma rules to start detecting security threats in your logs." actionLabel="Manage Sigma Rules" - onAction={() => goto('/dashboard/alerts')} + onAction={() => goto('/dashboard/security/rules')} /> {:else} diff --git a/packages/frontend/src/routes/dashboard/security/rules/+page.svelte b/packages/frontend/src/routes/dashboard/security/rules/+page.svelte new file mode 100644 index 00000000..096bdb35 --- /dev/null +++ b/packages/frontend/src/routes/dashboard/security/rules/+page.svelte @@ -0,0 +1,179 @@ + + + + Sigma Rules - LogTide + + +
+
+
+
+ +

Sigma Rules

+
+

+ Manage security detection rules. Import from SigmaHQ or install detection packs. +

+
+
+ + +
+
+ + {#if loading} +
+ + Loading Sigma rules... +
+ {:else if error} + + + {error} + + + {:else if sigmaRules.length === 0} + + +
+ +
+

No Sigma rules yet

+

+ Import Sigma rules to automatically detect security threats in your logs +

+
+ + +
+ + + {:else} + { + selectedSigmaRule = rule; + showSigmaDetails = true; + }} + /> + {/if} +
+ +{#if $currentOrganization} + + + { + loadSigmaRules(); + }} + /> + + { + loadSigmaRules(); + }} + /> +{/if} From c726da87effa173a7319c3f37f3761a2612b7bf7 Mon Sep 17 00:00:00 2001 From: Polliog Date: Mon, 23 Feb 2026 18:15:50 +0100 Subject: [PATCH 18/43] simplify project pages, remove duplicate log viewer --- .../routes/dashboard/projects/+page.svelte | 13 +- .../dashboard/projects/[id]/+layout.svelte | 10 +- .../dashboard/projects/[id]/+page.svelte | 935 +----------------- 3 files changed, 23 insertions(+), 935 deletions(-) diff --git a/packages/frontend/src/routes/dashboard/projects/+page.svelte b/packages/frontend/src/routes/dashboard/projects/+page.svelte index 4805b089..d50b1533 100644 --- a/packages/frontend/src/routes/dashboard/projects/+page.svelte +++ b/packages/frontend/src/routes/dashboard/projects/+page.svelte @@ -1,5 +1,6 @@ -
- - - - Filters - - -
- -
- -
- - { - if (v && (v === "fulltext" || v === "substring")) { - searchMode = v; - sessionStorage.setItem("logtide_project_search_mode", searchMode); - if (searchQuery) { - debouncedSearch(); - } - } - }} - > - - {searchMode === "fulltext" ? "Full-text" : "Substring"} - - - Full-text - Substring - - -
-
- - -
- - -
- - -
- - - - {#snippet child({ props })} - - {/snippet} - - -
-
- - -
-
-
- {#if isLoadingServices} -
- Loading services... -
- {:else if displayedServices().length === 0} -
- No services available -
- {:else} -
- {#each displayedServices() as service} - {@const hasLogsInTimeRange = availableServices.includes(service)} - - {/each} -
- {/if} -
- - -
-
- - -
- -
- - - - {#snippet child({ props })} - - {/snippet} - - -
-
- - -
-
-
-
- {#each ["debug", "info", "warn", "error", "critical"] as level} - - {/each} -
-
- - -
- - -
- - -
-
- - -
- -
- - -
- -
- - - - - - -
- - {#if effectiveTotalLogs > 0} - {effectiveTotalLogs.toLocaleString()} - {effectiveTotalLogs === 1 ? "log" : "logs"} - {#if liveTail} - (last 100) - {/if} - {:else} - No logs - {/if} - - {#if liveTail} - Live - {/if} -
- - - {#if loading} -
- - Loading logs... -
- {:else if error} -
- {error} -
- {:else if logs.length === 0} - - {:else} -
- - - - Time - Service - Level - Message - Actions - - - - {#each logs as log, i} - {@const globalIndex = i} - - - {formatDateTime(log.time)} - - - - - - - - {log.message} - -
- - -
- - - {#if expandedRows.has(globalIndex)} - - -
-
- Full Message: -
- {log.message} -
-
- {#if log.traceId} -
- Trace ID: - -
- {/if} - {#if log.metadata && Object.keys(log.metadata).length > 0} -
- Metadata: -
- 
{JSON.stringify(log.metadata, null, 2)}
-
-
- {/if} - {#if isErrorLevel(log.level) && log.id} -
- -
- {/if} -
- - - {/if} - {/each} - -
-
- - - {#if !liveTail && logs.length > 0} -
-
- Showing {(currentPage - 1) * pageSize + 1} to {(currentPage - 1) * pageSize + logs.length} logs -
-
- - - Page {currentPage} - - -
-
- {/if} - {/if} - - +
+ Redirecting...
- - - - - - - From 1d6ac8b3ace64918e7a944a757e07d99cfff1f24 Mon Sep 17 00:00:00 2001 From: Polliog Date: Mon, 23 Feb 2026 18:18:05 +0100 Subject: [PATCH 19/43] fix locale bug, silent errors, empty states, remove dead code --- .../src/lib/components/Navigation.svelte | 40 ------------------- .../lib/components/dashboard/LogsChart.svelte | 2 +- .../dashboard/RecentErrorsWidget.svelte | 3 ++ .../dashboard/TopServicesWidget.svelte | 3 ++ .../src/routes/dashboard/traces/+page.svelte | 2 + 5 files changed, 9 insertions(+), 41 deletions(-) delete mode 100644 packages/frontend/src/lib/components/Navigation.svelte diff --git a/packages/frontend/src/lib/components/Navigation.svelte b/packages/frontend/src/lib/components/Navigation.svelte deleted file mode 100644 index 6571dde0..00000000 --- a/packages/frontend/src/lib/components/Navigation.svelte +++ /dev/null @@ -1,40 +0,0 @@ - - - diff --git a/packages/frontend/src/lib/components/dashboard/LogsChart.svelte b/packages/frontend/src/lib/components/dashboard/LogsChart.svelte index 336e8291..cc85a9ca 100644 --- a/packages/frontend/src/lib/components/dashboard/LogsChart.svelte +++ b/packages/frontend/src/lib/components/dashboard/LogsChart.svelte @@ -30,7 +30,7 @@ let chart: echarts.ECharts | null = null; function formatTimeLabel(time: string): string { - return new Date(time).toLocaleTimeString('it-IT', { hour: '2-digit', minute: '2-digit', hour12: false }); + return new Date(time).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit', hour12: false }); } function buildEventSeries(): echarts.SeriesOption[] { diff --git a/packages/frontend/src/lib/components/dashboard/RecentErrorsWidget.svelte b/packages/frontend/src/lib/components/dashboard/RecentErrorsWidget.svelte index 4528e864..92806054 100644 --- a/packages/frontend/src/lib/components/dashboard/RecentErrorsWidget.svelte +++ b/packages/frontend/src/lib/components/dashboard/RecentErrorsWidget.svelte @@ -41,6 +41,9 @@
+ {#if errors.length === 0} +

No recent errors

+ {/if} {#each errors as error, index (`${error.time}-${error.service}-${index}`)}
{/if} - {#if isObservePage} - - {/if}
diff --git a/packages/frontend/src/lib/components/ObserveContextBar.svelte b/packages/frontend/src/lib/components/ObserveContextBar.svelte deleted file mode 100644 index 5c81cb82..00000000 --- a/packages/frontend/src/lib/components/ObserveContextBar.svelte +++ /dev/null @@ -1,164 +0,0 @@ - - -
- - - - - - -
-
-

Projects

-
- - / - -
-
-
-
- {#if loading} -

Loading...

- {:else if projects.length === 0} -

No projects found

- {:else} - {#each projects as project} - - {/each} - {/if} -
- - - - -
- {#each timeRangeOptions as opt} - - {/each} -
-
diff --git a/packages/frontend/src/lib/stores/observe-context.ts b/packages/frontend/src/lib/stores/observe-context.ts deleted file mode 100644 index eecbee7d..00000000 --- a/packages/frontend/src/lib/stores/observe-context.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { writable, derived, get } from 'svelte/store'; -import { browser } from '$app/environment'; - -export type TimeRangeType = 'last_hour' | 'last_24h' | 'last_7d' | 'custom'; - -interface ObserveContextState { - selectedProjects: string[]; - timeRangeType: TimeRangeType; - customFrom: string; - customTo: string; -} - -const STORAGE_KEY = 'logtide_observe_context'; - -const VALID_TIME_RANGE_TYPES: TimeRangeType[] = ['last_hour', 'last_24h', 'last_7d', 'custom']; - -const DEFAULTS: ObserveContextState = { - selectedProjects: [], - timeRangeType: 'last_24h', - customFrom: '', - customTo: '', -}; - -function loadInitialState(): ObserveContextState { - if (!browser) return DEFAULTS; - - try { - const raw = sessionStorage.getItem(STORAGE_KEY); - if (!raw) return DEFAULTS; - - const parsed = JSON.parse(raw); - - const timeRangeType = VALID_TIME_RANGE_TYPES.includes(parsed.timeRangeType) - ? parsed.timeRangeType - : DEFAULTS.timeRangeType; - - const selectedProjects = Array.isArray(parsed.selectedProjects) - ? parsed.selectedProjects.filter((p: unknown) => typeof p === 'string') - : DEFAULTS.selectedProjects; - - const customFrom = typeof parsed.customFrom === 'string' ? parsed.customFrom : DEFAULTS.customFrom; - const customTo = typeof parsed.customTo === 'string' ? parsed.customTo : DEFAULTS.customTo; - - return { selectedProjects, timeRangeType, customFrom, customTo }; - } catch { - return DEFAULTS; - } -} - -function persist(state: ObserveContextState) { - if (browser) { - try { - sessionStorage.setItem(STORAGE_KEY, JSON.stringify(state)); - } catch { - // sessionStorage unavailable - } - } -} - -function createObserveContextStore() { - const initialState = loadInitialState(); - const { subscribe, set, update } = writable(initialState); - - return { - subscribe, - - setProjects: (projects: string[]) => { - update((state) => { - const newState = { ...state, selectedProjects: projects }; - persist(newState); - return newState; - }); - }, - - setTimeRange: (type: TimeRangeType, customFrom?: string, customTo?: string) => { - update((state) => { - const newState = { - ...state, - timeRangeType: type, - customFrom: customFrom ?? (type === 'custom' ? state.customFrom : ''), - customTo: customTo ?? (type === 'custom' ? state.customTo : ''), - }; - persist(newState); - return newState; - }); - }, - - getTimeRange: (): { from: Date; to: Date } => { - const state = get({ subscribe }); - const now = new Date(); - - switch (state.timeRangeType) { - case 'last_hour': - return { from: new Date(now.getTime() - 60 * 60 * 1000), to: now }; - case 'last_24h': - return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; - case 'last_7d': - return { from: new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000), to: now }; - case 'custom': { - const from = state.customFrom ? new Date(state.customFrom) : new Date(now.getTime() - 24 * 60 * 60 * 1000); - const to = state.customTo ? new Date(state.customTo) : now; - return { from, to }; - } - default: - return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; - } - }, - - clear: () => { - set(DEFAULTS); - if (browser) { - try { - sessionStorage.removeItem(STORAGE_KEY); - } catch { - // sessionStorage unavailable - } - } - }, - }; -} - -export const observeContextStore = createObserveContextStore(); - -export const selectedProjects = derived( - observeContextStore, - ($state) => $state.selectedProjects, -); - -export const timeRangeType = derived( - observeContextStore, - ($state) => $state.timeRangeType, -); diff --git a/packages/frontend/src/routes/dashboard/metrics/+page.svelte b/packages/frontend/src/routes/dashboard/metrics/+page.svelte index 9cedff7b..70a8c412 100644 --- a/packages/frontend/src/routes/dashboard/metrics/+page.svelte +++ b/packages/frontend/src/routes/dashboard/metrics/+page.svelte @@ -12,7 +12,9 @@ import { metricsStore } from "$lib/stores/metrics"; import type { MetricAggregateResult } from "$lib/api/metrics"; import { currentOrganization } from "$lib/stores/organization"; - import { observeContextStore, selectedProjects as selectedProjectsStore, timeRangeType as ctxTimeRangeType } from "$lib/stores/observe-context"; + import { ProjectsAPI } from "$lib/api/projects"; + import type { Project } from "@logtide/shared"; + import { authStore } from "$lib/stores/auth"; import { layoutStore } from "$lib/stores/layout"; import { @@ -57,8 +59,41 @@ return unsubscribe; }); - // Derive project from observe context store - let selectedProject = $derived($selectedProjectsStore.length > 0 ? $selectedProjectsStore[0] : null); + // Local project and time range state + let token = $state(null); + authStore.subscribe((state) => { token = state.token; }); + let projectsAPI = $derived(new ProjectsAPI(() => token)); + + let projects = $state([]); + let selectedProject = $state(null); + let timeRangeType = $state<'last_hour' | 'last_24h' | 'last_7d'>('last_24h'); + + async function loadProjects() { + if (!$currentOrganization) return; + try { + const res = await projectsAPI.getProjects($currentOrganization.id); + projects = res.projects; + if (projects.length > 0 && !selectedProject) { + selectedProject = projects[0].id; + } + } catch (e) { + console.error("Failed to load projects:", e); + } + } + + function getTimeRange(): { from: Date; to: Date } { + const now = new Date(); + switch (timeRangeType) { + case 'last_hour': + return { from: new Date(now.getTime() - 60 * 60 * 1000), to: now }; + case 'last_24h': + return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; + case 'last_7d': + return { from: new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000), to: now }; + default: + return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; + } + } // Metrics store state let storeState = $state({ @@ -122,10 +157,7 @@ let lastLoadedOrg = $state(null); onMount(() => { - // Initial load if project is already selected - if (selectedProject) { - loadMetricNames(); - } + loadProjects(); return () => { chart?.dispose(); @@ -171,20 +203,21 @@ if ($currentOrganization.id === lastLoadedOrg) return; lastLoadedOrg = $currentOrganization.id; + selectedProject = null; + loadProjects(); }); - // React to project or time range changes from observe context store + // React to project or time range changes let lastContextKey = $state(null); $effect(() => { - // Track dependencies - const _proj = $selectedProjectsStore; - const _tr = $ctxTimeRangeType; + const _proj = selectedProject; + const _tr = timeRangeType; if (!$currentOrganization || !selectedProject) { return; } - const key = `${$currentOrganization.id}-${selectedProject}-${$ctxTimeRangeType}`; + const key = `${$currentOrganization.id}-${selectedProject}-${timeRangeType}`; if (key === lastContextKey) return; lastContextKey = key; @@ -202,7 +235,7 @@ function loadMetricNames() { if (!selectedProject) return; - const { from, to } = observeContextStore.getTimeRange(); + const { from, to } = getTimeRange(); metricsStore.loadMetricNames( selectedProject, from.toISOString(), @@ -214,7 +247,7 @@ metricsStore.selectMetric(metricName); if (!selectedProject) return; - const { from, to } = observeContextStore.getTimeRange(); + const { from, to } = getTimeRange(); const fromISO = from.toISOString(); const toISO = to.toISOString(); @@ -260,7 +293,7 @@ selectedLabelKey = key; selectedLabelValue = null; if (selectedProject && storeState.selectedMetric) { - const { from, to } = observeContextStore.getTimeRange(); + const { from, to } = getTimeRange(); metricsStore.loadLabelValues( selectedProject, storeState.selectedMetric, @@ -273,7 +306,7 @@ function reloadTimeseries() { if (!selectedProject || !storeState.selectedMetric) return; - const { from, to } = observeContextStore.getTimeRange(); + const { from, to } = getTimeRange(); metricsStore.loadTimeseries( selectedProject, storeState.selectedMetric, @@ -284,7 +317,7 @@ function reloadDataPoints() { if (!selectedProject || !storeState.selectedMetric) return; - const { from, to } = observeContextStore.getTimeRange(); + const { from, to } = getTimeRange(); metricsStore.loadDataPoints( selectedProject, storeState.selectedMetric, @@ -457,6 +490,48 @@
+ +
+ + { + selectedProject = v || null; + }} + > + + {projects.find(p => p.id === selectedProject)?.name || "Select project"} + + + {#each projects as project} + {project.name} + {/each} + + +
+ + +
+ + { + if (v) timeRangeType = v as typeof timeRangeType; + }} + > + + {timeRangeType === 'last_hour' ? 'Last Hour' : timeRangeType === 'last_24h' ? 'Last 24 Hours' : 'Last 7 Days'} + + + Last Hour + Last 24 Hours + Last 7 Days + + +
+
diff --git a/packages/frontend/src/routes/dashboard/search/+page.svelte b/packages/frontend/src/routes/dashboard/search/+page.svelte index d8b102fd..2b64d35f 100644 --- a/packages/frontend/src/routes/dashboard/search/+page.svelte +++ b/packages/frontend/src/routes/dashboard/search/+page.svelte @@ -42,8 +42,6 @@ import TerminalLogView from "$lib/components/TerminalLogView.svelte"; import TimeRangePicker, { type TimeRangeType } from "$lib/components/TimeRangePicker.svelte"; import { layoutStore } from "$lib/stores/layout"; - import { observeContextStore } from "$lib/stores/observe-context"; - import { get } from "svelte/store"; import AlertTriangle from "@lucide/svelte/icons/alert-triangle"; import Download from "@lucide/svelte/icons/download"; import ChevronLeft from "@lucide/svelte/icons/chevron-left"; @@ -401,15 +399,7 @@ projects = response.projects; if (projects.length > 0 && selectedProjects.length === 0) { - // Initialize from observe context store if it has selections - const ctxState = get(observeContextStore); - if (ctxState.selectedProjects.length > 0) { - const validIds = ctxState.selectedProjects.filter(id => projects.some(p => p.id === id)); - selectedProjects = validIds.length > 0 ? validIds : projects.map((p) => p.id); - } else { - selectedProjects = projects.map((p) => p.id); - } - observeContextStore.setProjects(selectedProjects); + selectedProjects = projects.map((p) => p.id); await Promise.all([loadServices(), loadHostnames()]); loadLogs(); } @@ -800,12 +790,6 @@ } async function handleTimeRangeChange() { - // Sync time range back to observe context store - if (timeRangePicker) { - const type = timeRangePicker.getType(); - const custom = timeRangePicker.getCustomValues?.() ?? { from: '', to: '' }; - observeContextStore.setTimeRange(type, custom.from, custom.to); - } await Promise.all([loadServices(), loadHostnames()]); applyFilters(); } @@ -955,7 +939,6 @@ class="flex-1" onclick={async () => { selectedProjects = projects.map((p) => p.id); - observeContextStore.setProjects(selectedProjects); await Promise.all([loadServices(), loadHostnames()]); applyFilters(); }} @@ -968,7 +951,6 @@ class="flex-1" onclick={() => { selectedProjects = []; - observeContextStore.setProjects([]); availableServices = []; availableHostnames = []; applyFilters(); @@ -999,7 +981,6 @@ (id) => id !== project.id, ); } - observeContextStore.setProjects(selectedProjects); await Promise.all([loadServices(), loadHostnames()]); applyFilters(); }} diff --git a/packages/frontend/src/routes/dashboard/traces/+page.svelte b/packages/frontend/src/routes/dashboard/traces/+page.svelte index 9617f242..a6978745 100644 --- a/packages/frontend/src/routes/dashboard/traces/+page.svelte +++ b/packages/frontend/src/routes/dashboard/traces/+page.svelte @@ -4,7 +4,8 @@ import { goto } from "$app/navigation"; import { currentOrganization } from "$lib/stores/organization"; import { authStore } from "$lib/stores/auth"; - import { observeContextStore, selectedProjects, timeRangeType as ctxTimeRangeType } from "$lib/stores/observe-context"; + import { ProjectsAPI } from "$lib/api/projects"; + import type { Project } from "@logtide/shared"; import { tracesAPI, type TraceRecord, @@ -89,8 +90,39 @@ token = state.token; }); - // Derive project from observe context store - let selectedProject = $derived($selectedProjects.length > 0 ? $selectedProjects[0] : null); + // Local project and time range state + let projects = $state([]); + let selectedProject = $state(null); + let timeRangeType = $state<'last_hour' | 'last_24h' | 'last_7d'>('last_24h'); + + let projectsAPI = $derived(new ProjectsAPI(() => token)); + + async function loadProjects() { + if (!$currentOrganization) return; + try { + const res = await projectsAPI.getProjects($currentOrganization.id); + projects = res.projects; + if (projects.length > 0 && !selectedProject) { + selectedProject = projects[0].id; + } + } catch (e) { + console.error("Failed to load projects:", e); + } + } + + function getTimeRange(): { from: Date; to: Date } { + const now = new Date(); + switch (timeRangeType) { + case 'last_hour': + return { from: new Date(now.getTime() - 60 * 60 * 1000), to: now }; + case 'last_24h': + return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; + case 'last_7d': + return { from: new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000), to: now }; + default: + return { from: new Date(now.getTime() - 24 * 60 * 60 * 1000), to: now }; + } + } // Pagination let pageSize = $state(25); @@ -109,14 +141,10 @@ } if (urlProjectId) { - observeContextStore.setProjects([urlProjectId]); + selectedProject = urlProjectId; } - // Initial load if project is already selected - if (selectedProject) { - loadTraces(); - loadServices(); - } + loadProjects(); }); // React to organization changes @@ -129,13 +157,14 @@ if ($currentOrganization.id === lastLoadedOrg) return; lastLoadedOrg = $currentOrganization.id; + selectedProject = null; + loadProjects(); }); - // React to project or time range changes from observe context store + // React to project or time range changes $effect(() => { - // Track dependencies - const _proj = $selectedProjects; - const _tr = $ctxTimeRangeType; + const _proj = selectedProject; + const _tr = timeRangeType; if (!selectedProject) { traces = []; @@ -165,7 +194,7 @@ isLoading = true; try { - const timeRange = observeContextStore.getTimeRange(); + const timeRange = getTimeRange(); const offset = (currentPage - 1) * pageSize; const response = await tracesAPI.getTraces({ @@ -220,7 +249,7 @@ mapLoadError = null; try { - const { from, to } = observeContextStore.getTimeRange(); + const { from, to } = getTimeRange(); mapData = await tracesAPI.getServiceMap( selectedProject, from.toISOString(), @@ -441,60 +470,100 @@
- - {#if activeView === 'list'} - - - - Filters - - -
-
- - { - selectedService = v || null; - applyFilters(); - }} - > - - {selectedService || "All services"} - - - All services - {#each availableServices as service} - {service} - {/each} - - -
+ + + + Filters + + +
+
+ + { + selectedProject = v || null; + }} + > + + {projects.find(p => p.id === selectedProject)?.name || "Select project"} + + + {#each projects as project} + {project.name} + {/each} + + +
-
- - { - errorOnly = v === "error"; - applyFilters(); - }} - > - - {errorOnly ? "Errors only" : "All traces"} - - - All traces - Errors only - - -
+
+ + { + if (v) timeRangeType = v as typeof timeRangeType; + }} + > + + {timeRangeType === 'last_hour' ? 'Last Hour' : timeRangeType === 'last_24h' ? 'Last 24 Hours' : 'Last 7 Days'} + + + Last Hour + Last 24 Hours + Last 7 Days + +
- - +
+ + { + selectedService = v || null; + applyFilters(); + }} + > + + {selectedService || "All services"} + + + All services + {#each availableServices as service} + {service} + {/each} + + +
+ +
+ + { + errorOnly = v === "error"; + applyFilters(); + }} + > + + {errorOnly ? "Errors only" : "All traces"} + + + All traces + Errors only + + +
+
+ + + + + {#if activeView === 'list'} From f81dbd8f8eb9cc6a7b483a00d24449905e9c6f4b Mon Sep 17 00:00:00 2001 From: Polliog Date: Mon, 23 Feb 2026 19:50:22 +0100 Subject: [PATCH 21/43] refactor: remove layout store dependencies and simplify page structure in settings --- .../routes/dashboard/settings/+layout.svelte | 191 ++++ .../routes/dashboard/settings/+page.svelte | 862 +----------------- .../dashboard/settings/audit-log/+page.svelte | 53 +- .../dashboard/settings/channels/+page.svelte | 36 +- .../dashboard/settings/general/+page.svelte | 258 ++++++ .../dashboard/settings/members/+page.svelte | 543 +++++++++++ .../dashboard/settings/patterns/+page.svelte | 35 +- .../settings/pii-masking/+page.svelte | 37 +- 8 files changed, 1006 insertions(+), 1009 deletions(-) create mode 100644 packages/frontend/src/routes/dashboard/settings/+layout.svelte create mode 100644 packages/frontend/src/routes/dashboard/settings/general/+page.svelte create mode 100644 packages/frontend/src/routes/dashboard/settings/members/+page.svelte diff --git a/packages/frontend/src/routes/dashboard/settings/+layout.svelte b/packages/frontend/src/routes/dashboard/settings/+layout.svelte new file mode 100644 index 00000000..e5006b16 --- /dev/null +++ b/packages/frontend/src/routes/dashboard/settings/+layout.svelte @@ -0,0 +1,191 @@ + + +
+ +
+
+ + Settings + + {currentPageLabel} +
+

Settings

+

+ Manage settings for {currentOrg?.name || 'your organization'} +

+
+ +
+ + + + + + + +
+ {@render children?.()} +
+
+
diff --git a/packages/frontend/src/routes/dashboard/settings/+page.svelte b/packages/frontend/src/routes/dashboard/settings/+page.svelte index 85faee92..0c0568ed 100644 --- a/packages/frontend/src/routes/dashboard/settings/+page.svelte +++ b/packages/frontend/src/routes/dashboard/settings/+page.svelte @@ -1,868 +1,8 @@ - - - Organization Settings - LogTide - - -
-
-

Organization Settings

-
- -

- Manage settings for {currentOrg?.name || 'your organization'} -

-
-
- - - - Organization Information - Update your organization details - - -
{ e.preventDefault(); saveOrganization(); }} class="space-y-4"> -
- - - {#if !isOwner} -

Only the owner can edit the organization name

- {/if} -
- -
- - -

- Auto-generated from organization name. This cannot be edited manually. -

-
- -
- -