diff --git a/docs/specs/canonical-ai-telemetry-v1.md b/docs/specs/canonical-ai-telemetry-v1.md index b694d83..5c10ef1 100644 --- a/docs/specs/canonical-ai-telemetry-v1.md +++ b/docs/specs/canonical-ai-telemetry-v1.md @@ -291,6 +291,44 @@ type ProviderProbeEventDataV1 = { SignalOps derives health from windows of attempts and probes. A producer cannot directly set the projected health state. +### 6.6 Cost reconciliation + +```ts +type CostReconciliationEventDataV1 = { + scope: "provider_period" | "route_period" + provider: { providerKey: string; providerVendor: string } + period: { start: string; end: string } + cost: { amount: DecimalString; currency: string; source: "billing_reconciled" } + basis: { + billSource: string + billReference?: string + units?: number + unit?: string + unitPrice?: number + } + route?: Route + resource: Resource + attributes?: Record +} +``` + +A reconciliation observation states money a provider billed for a closed period, taken from the +provider's own billing API (`basis.billSource`, for example `fal.models.usage` or +`openai.organization.costs`). It is immutable: a later period is a new event, never an edit, and a +producer re-run emits the same event id so ingest deduplicates it. + +Reconciliation is deliberately separate from attempt cost: + +- It carries no operation or attempt identity. Projections MUST NOT synthesize lifecycle state from + it and MUST NOT add it to attempt-derived cost totals. +- Producers MUST only reconcile periods whose billing lag has passed (fal hourly buckets at least + 90 minutes old, OpenAI daily buckets at least 25 hours old), so a reconciled number never changes + after emission. +- `scope: "route_period"` requires `route`; `scope: "provider_period"` forbids it and reconciles the + whole provider for the period. +- The cockpit compares billed money with the attempt-level estimate for the same period and + provider and shows the delta; the estimate covers only operations that carried cost evidence. + ## 7. Canonical example ```json @@ -415,6 +453,10 @@ MUST NOT turn a successful customer operation into a failure. - V1 is closed-world. Adding, removing, or renaming any field changes the accepted instance set and requires v2. - Changing a field's meaning, an accepted enum, or money/status semantics also requires v2. +- Introducing a brand-new event type is additive-compatible when every existing valid instance + stays valid, no existing type gains or loses semantics, and the type ships with valid and invalid + fixtures; `com.signalops.ai.cost.reconciliation.v1` entered v1 under this rule. Changing or + removing an existing type, field, or accepted enum member remains v2-gated. - SignalOps MUST retain v1 ingestion for at least one documented migration window after v2 ships. - Client adapters MUST pin a contract version and pass shared fixtures before release. - The existing prototype event shapes are v0. SignalOps may provide a v0-to-v1 compatibility diff --git a/packages/contracts/src/v1.ts b/packages/contracts/src/v1.ts index 1363e22..14bb6bc 100644 --- a/packages/contracts/src/v1.ts +++ b/packages/contracts/src/v1.ts @@ -7,6 +7,7 @@ export const SIGNALOPS_V1_EVENT_TYPES = [ "com.signalops.ai.attempt.started.v1", "com.signalops.ai.attempt.terminal.v1", "com.signalops.ai.provider.probe.v1", + "com.signalops.ai.cost.reconciliation.v1", ] as const; export type SignalOpsEventTypeV1 = (typeof SIGNALOPS_V1_EVENT_TYPES)[number]; @@ -168,6 +169,53 @@ export type SignalOpsProviderProbeDataV1 = { attributes?: SignalOpsAttributesV1; }; +export type SignalOpsReconciliationScopeV1 = "provider_period" | "route_period"; + +export type SignalOpsReconciliationProviderV1 = { + providerKey: string; + providerVendor: string; +}; + +export type SignalOpsReconciliationPeriodV1 = { + start: string; + end: string; +}; + +export type SignalOpsReconciliationBasisV1 = { + billSource: string; + billReference?: string; + units?: number; + unit?: string; + unitPrice?: number; +}; + +export type SignalOpsReconciliationCostV1 = { + amount: string; + currency: string; + source: "billing_reconciled"; +}; + +export type SignalOpsCostReconciliationDataV1 = + | { + scope: "provider_period"; + provider: SignalOpsReconciliationProviderV1; + period: SignalOpsReconciliationPeriodV1; + cost: SignalOpsReconciliationCostV1; + basis: SignalOpsReconciliationBasisV1; + resource: SignalOpsResourceV1; + attributes?: SignalOpsAttributesV1; + } + | { + scope: "route_period"; + provider: SignalOpsReconciliationProviderV1; + period: SignalOpsReconciliationPeriodV1; + cost: SignalOpsReconciliationCostV1; + basis: SignalOpsReconciliationBasisV1; + route: SignalOpsRouteV1; + resource: SignalOpsResourceV1; + attributes?: SignalOpsAttributesV1; + }; + type SignalOpsEventEnvelopeV1 = { specversion: "1.0"; id: string; @@ -201,6 +249,10 @@ export type SignalOpsEventV1 = | SignalOpsEventEnvelopeV1< "com.signalops.ai.provider.probe.v1", SignalOpsProviderProbeDataV1 + > + | SignalOpsEventEnvelopeV1< + "com.signalops.ai.cost.reconciliation.v1", + SignalOpsCostReconciliationDataV1 >; export type SignalOpsContractIssueV1 = { diff --git a/schemas/ai-telemetry/v1/event.schema.json b/schemas/ai-telemetry/v1/event.schema.json index 618091a..74a7a46 100644 --- a/schemas/ai-telemetry/v1/event.schema.json +++ b/schemas/ai-telemetry/v1/event.schema.json @@ -32,14 +32,15 @@ "com.signalops.ai.operation.terminal.v1", "com.signalops.ai.attempt.started.v1", "com.signalops.ai.attempt.terminal.v1", - "com.signalops.ai.provider.probe.v1" + "com.signalops.ai.provider.probe.v1", + "com.signalops.ai.cost.reconciliation.v1" ] }, "subject": { "type": "string", "maxLength": 180, "pattern": "^(operation|provider)/[A-Za-z0-9][A-Za-z0-9._:/-]*$", - "description": "Must exactly identify data.operation.id or, for probes, data.route.providerKey; the portable semantic validator enforces the cross-field equality." + "description": "Must exactly identify data.operation.id, or for probes and reconciliation events data.route.providerKey / data.provider.providerKey; the portable semantic validator enforces the cross-field equality." }, "time": { "$ref": "#/$defs/timestamp" }, "datacontenttype": { "const": "application/json" }, @@ -111,6 +112,18 @@ "data": { "$ref": "#/$defs/providerProbeData" } } } + }, + { + "if": { + "properties": { "type": { "const": "com.signalops.ai.cost.reconciliation.v1" } }, + "required": ["type"] + }, + "then": { + "properties": { + "subject": { "type": "string", "pattern": "^provider/" }, + "data": { "$ref": "#/$defs/costReconciliationData" } + } + } } ], "$defs": { @@ -277,17 +290,18 @@ "additionalProperties": false, "required": ["amount", "currency", "source"], "properties": { - "amount": { - "type": "string", - "pattern": "^(0|[1-9][0-9]*)(\\.[0-9]{1,12})?$", - "maxLength": 40 - }, + "amount": { "$ref": "#/$defs/decimalString" }, "currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, "source": { "enum": ["provider_reported", "catalog_estimate", "billing_reconciled"] } } }, + "decimalString": { + "type": "string", + "pattern": "^(0|[1-9][0-9]*)(\\.[0-9]{1,12})?$", + "maxLength": 40 + }, "resource": { "type": "object", "additionalProperties": false, @@ -416,6 +430,84 @@ "resource": { "$ref": "#/$defs/resource" }, "attributes": { "$ref": "#/$defs/attributes" } } + }, + "billingReconciledCost": { + "type": "object", + "additionalProperties": false, + "required": ["amount", "currency", "source"], + "properties": { + "amount": { "$ref": "#/$defs/decimalString" }, + "currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, + "source": { "const": "billing_reconciled" } + } + }, + "reconciliationPeriod": { + "type": "object", + "additionalProperties": false, + "required": ["start", "end"], + "properties": { + "start": { "$ref": "#/$defs/timestamp" }, + "end": { "$ref": "#/$defs/timestamp" } + } + }, + "reconciliationBasis": { + "type": "object", + "additionalProperties": false, + "required": ["billSource"], + "properties": { + "billSource": { "$ref": "#/$defs/identifier" }, + "billReference": { "$ref": "#/$defs/identifier" }, + "units": { "type": "number", "minimum": 0 }, + "unit": { "$ref": "#/$defs/identifier" }, + "unitPrice": { "type": "number", "minimum": 0 } + } + }, + "reconciliationProvider": { + "type": "object", + "additionalProperties": false, + "required": ["providerKey", "providerVendor"], + "properties": { + "providerKey": { "$ref": "#/$defs/identifier" }, + "providerVendor": { "$ref": "#/$defs/identifier" } + } + }, + "costReconciliationData": { + "type": "object", + "additionalProperties": false, + "required": ["scope", "provider", "period", "cost", "basis", "resource"], + "properties": { + "scope": { "enum": ["provider_period", "route_period"] }, + "provider": { "$ref": "#/$defs/reconciliationProvider" }, + "period": { "$ref": "#/$defs/reconciliationPeriod" }, + "cost": { "$ref": "#/$defs/billingReconciledCost" }, + "basis": { "$ref": "#/$defs/reconciliationBasis" }, + "route": { "$ref": "#/$defs/route" }, + "resource": { "$ref": "#/$defs/resource" }, + "attributes": { "$ref": "#/$defs/attributes" } + }, + "allOf": [ + { + "if": { + "properties": { "scope": { "const": "route_period" } }, + "required": ["scope"] + }, + "then": { + "required": ["route"], + "properties": { "route": { "$ref": "#/$defs/route" } } + } + }, + { + "if": { + "properties": { "scope": { "const": "provider_period" } }, + "required": ["scope"] + }, + "then": { + "required": [], + "properties": {}, + "not": { "required": ["route"] } + } + } + ] } } } diff --git a/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-missing-period-end.json b/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-missing-period-end.json new file mode 100644 index 0000000..1e7441f --- /dev/null +++ b/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-missing-period-end.json @@ -0,0 +1,32 @@ +{ + "specversion": "1.0", + "id": "phosphene:reconcile:missing-period-end:2026-08-22T00:00:00.000Z:v1", + "source": "urn:phosphene:production:billing-reconciler", + "type": "com.signalops.ai.cost.reconciliation.v1", + "subject": "provider/primary-fal", + "time": "2026-08-23T05:00:00.000Z", + "datacontenttype": "application/json", + "dataschema": "https://signalops.cc/schemas/ai-telemetry/v1", + "data": { + "scope": "provider_period", + "provider": { + "providerKey": "primary-fal", + "providerVendor": "fal" + }, + "period": { + "start": "2026-08-22T00:00:00.000Z" + }, + "cost": { + "amount": "0.105", + "currency": "USD", + "source": "billing_reconciled" + }, + "basis": { + "billSource": "fal.models.usage" + }, + "resource": { + "environment": "production", + "service": "billing-reconciler" + } + } +} diff --git a/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-provider-period-with-route.json b/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-provider-period-with-route.json new file mode 100644 index 0000000..8dfae2c --- /dev/null +++ b/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-provider-period-with-route.json @@ -0,0 +1,37 @@ +{ + "specversion": "1.0", + "id": "phosphene:reconcile:provider-period-with-route:2026-08-22T00:00:00.000Z:v1", + "source": "urn:phosphene:production:billing-reconciler", + "type": "com.signalops.ai.cost.reconciliation.v1", + "subject": "provider/primary-fal", + "time": "2026-08-23T05:00:00.000Z", + "datacontenttype": "application/json", + "dataschema": "https://signalops.cc/schemas/ai-telemetry/v1", + "data": { + "scope": "provider_period", + "provider": { + "providerKey": "primary-fal", + "providerVendor": "fal" + }, + "period": { + "start": "2026-08-22T00:00:00.000Z", + "end": "2026-08-23T00:00:00.000Z" + }, + "cost": { + "amount": "0.105", + "currency": "USD", + "source": "billing_reconciled" + }, + "basis": { + "billSource": "fal.models.usage" + }, + "route": { + "providerKey": "primary-fal", + "modelKey": "flux-2-pro" + }, + "resource": { + "environment": "production", + "service": "billing-reconciler" + } + } +} diff --git a/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-route-period-no-route.json b/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-route-period-no-route.json new file mode 100644 index 0000000..6e2a031 --- /dev/null +++ b/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-route-period-no-route.json @@ -0,0 +1,33 @@ +{ + "specversion": "1.0", + "id": "phosphene:reconcile:route-period-missing-route:2026-08-22T00:00:00.000Z:v1", + "source": "urn:phosphene:production:billing-reconciler", + "type": "com.signalops.ai.cost.reconciliation.v1", + "subject": "provider/primary-fal", + "time": "2026-08-23T05:00:00.000Z", + "datacontenttype": "application/json", + "dataschema": "https://signalops.cc/schemas/ai-telemetry/v1", + "data": { + "scope": "route_period", + "provider": { + "providerKey": "primary-fal", + "providerVendor": "fal" + }, + "period": { + "start": "2026-08-22T00:00:00.000Z", + "end": "2026-08-23T00:00:00.000Z" + }, + "cost": { + "amount": "0.053", + "currency": "USD", + "source": "billing_reconciled" + }, + "basis": { + "billSource": "fal.models.usage" + }, + "resource": { + "environment": "production", + "service": "billing-reconciler" + } + } +} diff --git a/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-wrong-source.json b/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-wrong-source.json new file mode 100644 index 0000000..303439d --- /dev/null +++ b/schemas/ai-telemetry/v1/fixtures/invalid/reconciliation-wrong-source.json @@ -0,0 +1,33 @@ +{ + "specversion": "1.0", + "id": "phosphene:reconcile:invalid-source:2026-08-22T00:00:00.000Z:v1", + "source": "urn:phosphene:production:billing-reconciler", + "type": "com.signalops.ai.cost.reconciliation.v1", + "subject": "provider/primary-fal", + "time": "2026-08-23T05:00:00.000Z", + "datacontenttype": "application/json", + "dataschema": "https://signalops.cc/schemas/ai-telemetry/v1", + "data": { + "scope": "provider_period", + "provider": { + "providerKey": "primary-fal", + "providerVendor": "fal" + }, + "period": { + "start": "2026-08-22T00:00:00.000Z", + "end": "2026-08-23T00:00:00.000Z" + }, + "cost": { + "amount": "0.105", + "currency": "USD", + "source": "provider_reported" + }, + "basis": { + "billSource": "fal.models.usage" + }, + "resource": { + "environment": "production", + "service": "billing-reconciler" + } + } +} diff --git a/schemas/ai-telemetry/v1/fixtures/valid/cost-reconciliation-route-period.json b/schemas/ai-telemetry/v1/fixtures/valid/cost-reconciliation-route-period.json new file mode 100644 index 0000000..1a436ea --- /dev/null +++ b/schemas/ai-telemetry/v1/fixtures/valid/cost-reconciliation-route-period.json @@ -0,0 +1,46 @@ +{ + "specversion": "1.0", + "id": "phosphene:reconcile:primary-fal:flux-2-pro:2026-08-22T00:00:00.000Z:v1", + "source": "urn:phosphene:production:billing-reconciler", + "type": "com.signalops.ai.cost.reconciliation.v1", + "subject": "provider/primary-fal", + "time": "2026-08-23T05:01:00.000Z", + "datacontenttype": "application/json", + "dataschema": "https://signalops.cc/schemas/ai-telemetry/v1", + "data": { + "scope": "route_period", + "provider": { + "providerKey": "primary-fal", + "providerVendor": "fal" + }, + "period": { + "start": "2026-08-23T04:00:00.000Z", + "end": "2026-08-23T05:00:00.000Z" + }, + "cost": { + "amount": "0.053", + "currency": "USD", + "source": "billing_reconciled" + }, + "basis": { + "billSource": "fal.models.usage", + "billReference": "fal-route-20260822", + "units": 1, + "unit": "image", + "unitPrice": 0.053 + }, + "route": { + "providerKey": "primary-fal", + "providerVendor": "fal", + "modelKey": "flux-2-pro", + "providerModelKey": "fal-ai/flux-2-pro", + "region": "iad" + }, + "resource": { + "environment": "production", + "service": "billing-reconciler", + "release": "de4918457d6009cfe90a10222439881af8be8e14", + "region": "hetzner" + } + } +} diff --git a/schemas/ai-telemetry/v1/fixtures/valid/cost-reconciliation.json b/schemas/ai-telemetry/v1/fixtures/valid/cost-reconciliation.json new file mode 100644 index 0000000..62c3a2b --- /dev/null +++ b/schemas/ai-telemetry/v1/fixtures/valid/cost-reconciliation.json @@ -0,0 +1,38 @@ +{ + "specversion": "1.0", + "id": "phosphene:reconcile:primary-fal:2026-08-22T00:00:00.000Z:v1", + "source": "urn:phosphene:production:billing-reconciler", + "type": "com.signalops.ai.cost.reconciliation.v1", + "subject": "provider/primary-fal", + "time": "2026-08-23T05:00:00.000Z", + "datacontenttype": "application/json", + "dataschema": "https://signalops.cc/schemas/ai-telemetry/v1", + "data": { + "scope": "provider_period", + "provider": { + "providerKey": "primary-fal", + "providerVendor": "fal" + }, + "period": { + "start": "2026-08-23T04:00:00.000Z", + "end": "2026-08-23T05:00:00.000Z" + }, + "cost": { + "amount": "0.105", + "currency": "USD", + "source": "billing_reconciled" + }, + "basis": { + "billSource": "fal.models.usage", + "billReference": "fal-day-20260822", + "units": 2, + "unit": "image" + }, + "resource": { + "environment": "production", + "service": "billing-reconciler", + "release": "de4918457d6009cfe90a10222439881af8be8e14", + "region": "hetzner" + } + } +} diff --git a/schemas/ai-telemetry/v1/semantic-validation.mjs b/schemas/ai-telemetry/v1/semantic-validation.mjs index d7bf352..104902d 100644 --- a/schemas/ai-telemetry/v1/semantic-validation.mjs +++ b/schemas/ai-telemetry/v1/semantic-validation.mjs @@ -21,6 +21,11 @@ export function validateSignalOpsEventSemanticsV1(input) { if (route && typeof route.providerKey === "string") { expectedSubject = `provider/${route.providerKey}`; } + } else if (input.type === "com.signalops.ai.cost.reconciliation.v1") { + const provider = isRecord(data.provider) ? data.provider : undefined; + if (provider && typeof provider.providerKey === "string") { + expectedSubject = `provider/${provider.providerKey}`; + } } else if (input.type.startsWith("com.signalops.ai.")) { const operation = isRecord(data.operation) ? data.operation : undefined; if (operation && typeof operation.id === "string") { diff --git a/scripts/test-live-cockpit-v1.mjs b/scripts/test-live-cockpit-v1.mjs index f00e98d..cb9a7a9 100644 --- a/scripts/test-live-cockpit-v1.mjs +++ b/scripts/test-live-cockpit-v1.mjs @@ -616,6 +616,7 @@ const snapshotFixture = { costEvidence: { observed: 0, total: 4, ratio: 0 }, operationCostEvidence: { observed: 0, total: 3, ratio: 0 }, }, + reconciliation: { lastReconciledAt: null, currencies: [], periods: [] }, environments: ["production"], timeline: timelineFixture, providers: [], diff --git a/scripts/test-signalops-ops-v1.mjs b/scripts/test-signalops-ops-v1.mjs index ff036ec..1274f16 100644 --- a/scripts/test-signalops-ops-v1.mjs +++ b/scripts/test-signalops-ops-v1.mjs @@ -430,4 +430,82 @@ const effective = effectiveProviderHealthV1(quietRoute); assert.equal(effective.live, false); assert.equal(effective.status, quietRoute.windowHealth.status); +// Reconciliation events are immutable observations of provider-billed money. They MUST NOT +// synthesize operations or attempts, MUST NOT alter coverage, MUST NOT enter attempt-derived +// `costByCurrency`, and MUST populate the dedicated `reconciliation` block with the matching +// attempt-terminal estimated cost for the same providerKey and `[start, end)` window. +const reconciliation = await fixture("cost-reconciliation"); +const reconciliationRoute = await fixture("cost-reconciliation-route-period"); +const reconciliationRecords = [ + ...records, + { + tenantId, + receivedAt, + event: structuredClone(reconciliation), + }, + { + tenantId, + receivedAt, + event: structuredClone(reconciliationRoute), + }, +]; +const reconciliationSnapshot = buildSignalOpsOpsSnapshotV1({ + tenantId, + tenantName: "Phosphene", + range: "24h", + records: reconciliationRecords, + now: new Date("2026-08-23T12:00:00.000Z"), +}); +assert.equal( + reconciliationSnapshot.totals.operations, + 1, + "Reconciliation events must not synthesize operations.", +); +assert.equal( + reconciliationSnapshot.totals.attempts, + 1, + "Reconciliation events must not synthesize attempts.", +); +assert.equal( + reconciliationSnapshot.totals.events, + 5, + "Reconciliation events are observed in `totals.events` without becoming operations/attempts.", +); +assert.equal( + reconciliationSnapshot.coverage.providerAttempts.observed, + 1, + "Coverage must not count reconciliation events as provider attempts.", +); +assert.equal( + reconciliationSnapshot.coverage.attemptLifecycle.observed, + 0, + "Coverage must not count reconciliation events as attempt lifecycle.", +); +assert.deepEqual( + reconciliationSnapshot.coverage.operationCostEvidence, + { observed: 1, total: 1, ratio: 1 }, + "Coverage must be unaffected by reconciliation events.", +); +assert.deepEqual( + reconciliationSnapshot.totals.costByCurrency, + [{ currency: "USD", provider_reported: 0, catalog_estimate: 0.053, billing_reconciled: 0 }], + "Reconciliation cost must never enter attempt-derived `costByCurrency`.", +); +assert.equal( + reconciliationSnapshot.reconciliation.lastReconciledAt, + reconciliationRoute.time, +); +assert.deepEqual( + reconciliationSnapshot.reconciliation.currencies.map((row) => row.currency).sort(), + ["USD"], +); +assert.equal(reconciliationSnapshot.reconciliation.periods.length, 2); +const providerPeriod = reconciliationSnapshot.reconciliation.periods.find( + (row) => row.billSource === "fal.models.usage" && row.providerKey === "primary-fal", +); +assert.ok(providerPeriod); +assert.equal(providerPeriod.billed, 0.105); +assert.equal(providerPeriod.estimated, 0.053, "Estimated = sum of attempt-terminal costs in window."); +assert.equal(providerPeriod.delta, 0.105 - 0.053); + console.log("signalops ops v1 checks passed"); diff --git a/src/components/live-cockpit.tsx b/src/components/live-cockpit.tsx index c56d4fe..04d1ece 100644 --- a/src/components/live-cockpit.tsx +++ b/src/components/live-cockpit.tsx @@ -2074,6 +2074,51 @@ export function LiveCockpit() { +
+ + {snapshot.reconciliation.periods.length === 0 ? ( + + ) : ( +
+
+ {snapshot.reconciliation.currencies.map((row) => ( +
+

{row.currency}

+

+ billed {formatMoney(row.billed, row.currency)} · estimated {formatMoney(row.estimated, row.currency)} · delta {formatMoney(row.delta, row.currency)} +

+
+ ))} +
+
+ {snapshot.reconciliation.periods.slice(0, 12).map((period) => ( +
+
+

{period.providerVendor} · {period.providerKey}

+

{timelineLabel(period.start, displayedRange)}

+
+

+ billed {formatMoney(period.billed, period.currency)} · estimated {formatMoney(period.estimated, period.currency)} · delta {formatMoney(period.delta, period.currency)} + {period.units !== undefined ? ` · ${period.units} ${period.unit ?? "units"}` : ""} +

+
+ ))} +
+

+ Estimates cover only operations that carried cost evidence; billed totals come from provider billing APIs after their reporting lag. +

+
+ )} +
+
+ 0 + ? [ + `- Billing reconciliation: ${snapshot.reconciliation.currencies + .map( + (row) => + `${row.currency} billed ${row.billed.toFixed(2)} vs estimated ${row.estimated.toFixed(2)} (delta ${row.delta.toFixed(2)})`, + ) + .join(", ")}; last reconciled ${snapshot.reconciliation.lastReconciledAt ?? "unknown"}`, + ] + : []), `- Retained view: ${retainedMatches} matches across ${indexedRows} indexed rows`, `- Filters: ${filters.length > 0 ? filters.map(markdownCodeV1).join(", ") : "none"}`, "", diff --git a/src/lib/signalops/v1/operation-trace.ts b/src/lib/signalops/v1/operation-trace.ts index 78391ae..ed3c618 100644 --- a/src/lib/signalops/v1/operation-trace.ts +++ b/src/lib/signalops/v1/operation-trace.ts @@ -14,7 +14,8 @@ import type { type OperationEventV1 = Exclude< SignalOpsEventV1, - { type: "com.signalops.ai.provider.probe.v1" } + | { type: "com.signalops.ai.provider.probe.v1" } + | { type: "com.signalops.ai.cost.reconciliation.v1" } >; type OperationAcceptedEventV1 = Extract< diff --git a/src/lib/signalops/v1/ops-snapshot.ts b/src/lib/signalops/v1/ops-snapshot.ts index 1261a13..9b339f3 100644 --- a/src/lib/signalops/v1/ops-snapshot.ts +++ b/src/lib/signalops/v1/ops-snapshot.ts @@ -33,7 +33,13 @@ type AttemptTerminalEvent = Extract< type NonProbeEvent = Exclude< SignalOpsEventV1, - { type: "com.signalops.ai.provider.probe.v1" } + | { type: "com.signalops.ai.provider.probe.v1" } + | { type: "com.signalops.ai.cost.reconciliation.v1" } +>; + +type ReconciliationEvent = Extract< + SignalOpsEventV1, + { type: "com.signalops.ai.cost.reconciliation.v1" } >; export type SignalOpsOpsRangeV1 = "24h" | "7d" | "30d" | "90d"; @@ -204,6 +210,35 @@ export type SignalOpsOpsSnapshotV1 = { failureBreakdown: SignalOpsFailureSnapshotV1[]; recentOperations: SignalOpsOperationSnapshotV1[]; recentFailedOperations: SignalOpsOperationSnapshotV1[]; + reconciliation: SignalOpsReconciliationBlockV1; +}; + +export type SignalOpsReconciliationPeriodRowV1 = { + providerKey: string; + providerVendor: string; + start: string; + end: string; + billSource: string; + currency: string; + billed: number; + estimated: number; + delta: number; + units?: number; + unit?: string; + billReference?: string; +}; + +export type SignalOpsReconciliationCurrencyRowV1 = { + currency: string; + billed: number; + estimated: number; + delta: number; +}; + +export type SignalOpsReconciliationBlockV1 = { + lastReconciledAt: string | null; + currencies: SignalOpsReconciliationCurrencyRowV1[]; + periods: SignalOpsReconciliationPeriodRowV1[]; }; type OperationState = { @@ -259,6 +294,14 @@ const providerExcludedFailures = new Set([ "client_configuration", ]); +type AttemptTerminalCostV1 = { + time: string; + currency: string; + amount: number; +}; + +const MAX_RECONCILIATION_PERIODS_V1 = 50; + export function rangeStartV1(range: SignalOpsOpsRangeV1, now = new Date()): string { const hours = range === "24h" ? 24 : range === "7d" ? 24 * 7 : range === "30d" ? 24 * 30 : 24 * 90; @@ -361,6 +404,77 @@ function durationBetween(start: string | undefined, end: string): number | null return Number.isFinite(duration) && duration >= 0 ? duration : null; } +function buildReconciliationBlockV1( + events: readonly ReconciliationEvent[], + attemptTerminalCosts: ReadonlyMap, +): SignalOpsReconciliationBlockV1 { + if (events.length === 0) { + return { lastReconciledAt: null, currencies: [], periods: [] }; + } + const sorted = [...events].sort((left, right) => + left.time.localeCompare(right.time) || + left.id.localeCompare(right.id), + ); + const lastReconciledAt = sorted.at(-1)?.time ?? null; + const periods: SignalOpsReconciliationPeriodRowV1[] = []; + const currencyMap = new Map(); + for (const event of sorted) { + const startMs = Date.parse(event.data.period.start); + const endMs = Date.parse(event.data.period.end); + if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs <= startMs) continue; + const estimatedRows = attemptTerminalCosts.get(event.data.provider.providerKey) ?? []; + let estimated = 0; + for (const row of estimatedRows) { + const ts = Date.parse(row.time); + if (Number.isFinite(ts) && ts >= startMs && ts < endMs) { + estimated += Number.isFinite(row.amount) ? row.amount : 0; + } + } + const billed = Number(event.data.cost.amount); + const safeBilled = Number.isFinite(billed) ? billed : 0; + const currency = event.data.cost.currency; + const period: SignalOpsReconciliationPeriodRowV1 = { + providerKey: event.data.provider.providerKey, + providerVendor: event.data.provider.providerVendor, + start: event.data.period.start, + end: event.data.period.end, + billSource: event.data.basis.billSource, + currency, + billed: safeBilled, + estimated, + delta: safeBilled - estimated, + ...(event.data.basis.billReference !== undefined + ? { billReference: event.data.basis.billReference } + : {}), + ...(event.data.basis.units !== undefined ? { units: event.data.basis.units } : {}), + ...(event.data.basis.unit !== undefined ? { unit: event.data.basis.unit } : {}), + }; + periods.push(period); + const row = currencyMap.get(currency) ?? { + currency, + billed: 0, + estimated: 0, + delta: 0, + }; + row.billed += safeBilled; + row.estimated += estimated; + row.delta = row.billed - row.estimated; + currencyMap.set(currency, row); + } + periods.sort( + (left, right) => + right.start.localeCompare(left.start) || + left.providerKey.localeCompare(right.providerKey), + ); + return { + lastReconciledAt, + currencies: [...currencyMap.values()].sort((left, right) => + left.currency.localeCompare(right.currency), + ), + periods: periods.slice(0, MAX_RECONCILIATION_PERIODS_V1), + }; +} + function providerHealth( row: ProviderAccumulator, now: Date, @@ -428,7 +542,13 @@ function attemptInstrumentationStartV1( record.event.type === "com.signalops.ai.attempt.started.v1" || record.event.type === "com.signalops.ai.attempt.terminal.v1", )?.event; - if (!firstAttempt || firstAttempt.type === "com.signalops.ai.provider.probe.v1") return null; + if ( + !firstAttempt || + firstAttempt.type !== "com.signalops.ai.attempt.started.v1" && + firstAttempt.type !== "com.signalops.ai.attempt.terminal.v1" + ) { + return null; + } const accepted = operations.get(firstAttempt.data.operation.id)?.accepted?.time; return accepted && accepted < firstAttempt.time ? accepted : firstAttempt.time; } @@ -466,6 +586,7 @@ export function buildSignalOpsOpsSnapshotV1(input: { const attemptIdentity = new Map(); const operationIdentity = new Map(); const probes: Extract[] = []; + const reconciliations: ReconciliationEvent[] = []; const environments = new Set( records.map((record) => record.event.data.resource.environment), ); @@ -480,6 +601,15 @@ export function buildSignalOpsOpsSnapshotV1(input: { continue; } + // Reconciliation events are immutable observations of provider-billed money, not lifecycle + // facts. Exclude them BEFORE any code reads `event.data.operation.id` so they can never + // synthesize operations or attempts, do not alter coverage, and never enter + // `totals.costByCurrency` alongside attempt-derived attempt costs. + if (event.type === "com.signalops.ai.cost.reconciliation.v1") { + reconciliations.push(event); + continue; + } + const operationId = event.data.operation.id; const operationSignature = `${event.data.operation.kind}\u0000${event.data.operation.logicalModelKey ?? ""}`; const knownOperationSignature = operationIdentity.get(operationId); @@ -535,6 +665,10 @@ export function buildSignalOpsOpsSnapshotV1(input: { const providerRows = new Map(); const totalCosts = new Map(); const costedOperations = new Set(); + // Per-provider-key accumulator of attempt-terminal cost evidence keyed by terminal time. + // Reconciliation events use this to compute `estimated = sum(amount)` for matching + // [period.start, period.end) windows. Reconciliation cost itself never enters here. + const attemptTerminalCostsByProvider = new Map(); const timelineState = createTimelineV1(input.range, now); const modelAccumulators = new Map(); const failureAccumulators = new Map(); @@ -594,6 +728,13 @@ export function buildSignalOpsOpsSnapshotV1(input: { event.data.cost.source, event.data.cost.amount, ); + const list = attemptTerminalCostsByProvider.get(event.data.route.providerKey) ?? []; + list.push({ + time: event.time, + currency: event.data.cost.currency, + amount: Number(event.data.cost.amount), + }); + attemptTerminalCostsByProvider.set(event.data.route.providerKey, list); } row.recentOutcomes.push({ occurredAt: event.time, @@ -887,5 +1028,6 @@ export function buildSignalOpsOpsSnapshotV1(input: { recentFailedOperations: recentOperations .filter((operation) => isFailedOperationStatusV1(operation.status)) .slice(0, 50), + reconciliation: buildReconciliationBlockV1(reconciliations, attemptTerminalCostsByProvider), }; } diff --git a/src/lib/signalops/v1/projection-repository.ts b/src/lib/signalops/v1/projection-repository.ts index 40ae4a4..540cae6 100644 --- a/src/lib/signalops/v1/projection-repository.ts +++ b/src/lib/signalops/v1/projection-repository.ts @@ -54,6 +54,9 @@ function isMatchingSnapshot( row.snapshot.providers.every((provider) => provider?.windowHealth !== undefined) && Number.isFinite(row.snapshot?.coverage?.attemptLifecycle?.observed) && Array.isArray(row.snapshot?.failureBreakdown) && + row.snapshot.reconciliation !== undefined && + Array.isArray(row.snapshot.reconciliation.currencies) && + Array.isArray(row.snapshot.reconciliation.periods) && Date.parse(row.projected_at) >= now.getTime() - 60_000 ); }