Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/specs/signalops-observability-platform.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,23 @@ Implementation contract: [`signalops-slo-incidents-v1.md`](signalops-slo-inciden

### Increment 4 — cost and fleet intelligence

- Spend-efficiency lens in the cockpit: cost evidence split into succeeded work,
retry tax, waste on failed operations, and cancelled operations, plus cost per
succeeded operation.
- Two versioned spend objectives: `cost_evidence_coverage` (share of operations
carrying cost evidence) and `reconciliation_delta_ratio` (billed versus
estimated drift per reconciliation period), both with low-sample states.
- Daily provider price watch (Phosphene `scripts/provider-price-watch.ts`):
diffs fal published prices against the stored `AppSetting` baseline and flags
moves beyond 5 %, unit flips, currency flips, and endpoints that left the
catalog. Unreachable or unusable prices carry the baseline forward instead of
deleting it; a run where every endpoint fails throws rather than rewriting
history; the first run seeds the baseline without reporting changes.
- Policy metrics are a closed union (`SignalOpsSloMetricV1`) and the only policy
write path patches thresholds of registry-derived policies, so no database
migration is required for the new objectives: an out-of-registry metric cannot
be stored through the application.

- Add immutable cost-adjustment observations in the next compatible contract version.
- Reconcile estimates, provider reports, and billing facts without mutating execution history.
- Add per-operation, per-route, and per-model unit economics and anomaly detection.
Expand Down
1 change: 1 addition & 0 deletions scripts/test-live-cockpit-v1.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,7 @@ const snapshotFixture = {
operationCostEvidence: { observed: 0, total: 3, ratio: 0 },
},
reconciliation: { lastReconciledAt: null, currencies: [], periods: [] },
spendEfficiency: { currencies: [] },
environments: ["production"],
timeline: timelineFixture,
providers: [],
Expand Down
27 changes: 25 additions & 2 deletions scripts/test-signalops-incidents-v1.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,26 @@ snapshot.coverage.providerAttempts = { observed: 50, total: 100, ratio: 0.5 };
snapshot.coverage.failureClassification = { observed: 5, total: 6, ratio: 5 / 6 };
snapshot.freshness.lastReceivedAt = "2026-08-23T23:00:00.000Z";
snapshot.projection.sourceEventCount = 200;
snapshot.coverage.operationCostEvidence = { observed: 40, total: 100, ratio: 0.4 };
snapshot.reconciliation = {
lastReconciledAt: "2026-08-23T12:00:00.000Z",
currencies: [{ currency: "USD", billed: 3, estimated: 1.5, delta: 1.5 }],
periods: [0, 1, 2].map((index) => ({
providerKey: "fal",
providerVendor: "fal",
scope: "route_period",
start: `2026-08-23T0${index + 2}:00:00.000Z`,
end: `2026-08-23T0${index + 3}:00:00.000Z`,
billSource: "fal.models.usage",
currency: "USD",
billed: 1,
estimated: 0.5,
delta: 0.5,
})),
};

const defaults = defaultSignalOpsSloPoliciesV1(tenantId);
assert.equal(defaults.length, 5);
assert.equal(defaults.length, 7);
assert.equal(defaults.find((policy) => policy.metric === "signal_freshness_ms")?.enabled, false);

const evaluations = evaluateSignalOpsSloPoliciesV1({ snapshot, policies: defaults, now });
Expand All @@ -52,6 +69,8 @@ assert.deepEqual(
["provider_attempt_coverage", "breached", "critical"],
["failure_classification_coverage", "breached", "warning"],
["signal_freshness_ms", "disabled", null],
["cost_evidence_coverage", "breached", "critical"],
["reconciliation_delta_ratio", "breached", "warning"],
],
);

Expand All @@ -61,6 +80,8 @@ lowSample.totals.succeeded = 1;
lowSample.totals.failed = 2;
lowSample.coverage.providerAttempts = { observed: 0, total: 3, ratio: 0 };
lowSample.coverage.failureClassification = { observed: 0, total: 2, ratio: 0 };
lowSample.coverage.operationCostEvidence = { observed: 0, total: 3, ratio: 0 };
lowSample.reconciliation = { lastReconciledAt: null, currencies: [], periods: [] };
assert.equal(
evaluateSignalOpsSloPoliciesV1({ snapshot: lowSample, policies: defaults, now }).find(
(evaluation) => evaluation.policy.metric === "operation_success_rate",
Expand Down Expand Up @@ -159,8 +180,10 @@ await assert.rejects(
const decisions = evaluateSignalOpsIncidentsV1(snapshot, defaults).filter((item) =>
item.metric.startsWith("slo:"),
);
assert.equal(decisions.length, 4);
assert.equal(decisions.length, 6);
assert.ok(decisions.every((item) => typeof item.evidence.observedValue === "number"));
assert.ok(decisions.some((item) => item.metric === "slo:cost_evidence_coverage"));
assert.ok(decisions.some((item) => item.metric === "slo:reconciliation_delta_ratio"));

const reliability = decisions.find((item) => item.metric === "slo:operation_success_rate");
assert.ok(reliability);
Expand Down
88 changes: 88 additions & 0 deletions scripts/test-signalops-ops-v1.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -547,4 +547,92 @@ assert.equal(
"provider_period must sum every route of the provider.",
);


// Spend efficiency: money must be attributed to the operation outcome that
// consumed it, with retries separated from first attempts.
const efficiencyAccepted = structuredClone(accepted);
const efficiencyAttemptOne = structuredClone(attemptTerminal);
const efficiencyAttemptTwo = structuredClone(attemptTerminal);
const efficiencyTerminal = structuredClone(operationTerminal);
const failedEfficiencyAccepted = structuredClone(accepted);
const failedEfficiencyAttempt = structuredClone(attemptTerminal);
const failedEfficiencyTerminal = structuredClone(operationTerminal);
const cancelledAccepted = structuredClone(accepted);
const cancelledAttempt = structuredClone(attemptTerminal);
const cancelledTerminal = structuredClone(operationTerminal);

efficiencyAccepted.data.operation.id = "op_eff_s";
efficiencyAttemptOne.data.operation.id = "op_eff_s";
efficiencyAttemptOne.id = "evt_eff_s_a1";
efficiencyAttemptOne.data.attempt = { id: "att_eff_s_1", number: 1 };
efficiencyAttemptOne.data.cost = { amount: "0.05", currency: "USD", source: "catalog_estimate" };
efficiencyAttemptTwo.data.operation.id = "op_eff_s";
efficiencyAttemptTwo.id = "evt_eff_s_a2";
efficiencyAttemptTwo.data.attempt = { id: "att_eff_s_2", number: 2 };
efficiencyAttemptTwo.data.cost = { amount: "0.02", currency: "USD", source: "catalog_estimate" };
efficiencyTerminal.data.operation.id = "op_eff_s";
efficiencyTerminal.id = "evt_eff_s_term";
efficiencyTerminal.data.outcome = { status: "succeeded" };

failedEfficiencyAccepted.data.operation.id = "op_eff_f";
failedEfficiencyAttempt.data.operation.id = "op_eff_f";
failedEfficiencyAttempt.id = "evt_eff_f_a1";
failedEfficiencyAttempt.data.attempt = { id: "att_eff_f_1", number: 1 };
failedEfficiencyAttempt.data.cost = { amount: "0.10", currency: "USD", source: "catalog_estimate" };
failedEfficiencyTerminal.data.operation.id = "op_eff_f";
failedEfficiencyTerminal.id = "evt_eff_f_term";
failedEfficiencyTerminal.data.outcome = {
status: "failed",
failure: { category: "provider", responsibility: "provider", code: "provider_http_500", retryable: false },
};

cancelledAccepted.data.operation.id = "op_eff_c";
cancelledAttempt.data.operation.id = "op_eff_c";
cancelledAttempt.id = "evt_eff_c_a1";
cancelledAttempt.data.attempt = { id: "att_eff_c_1", number: 1 };
cancelledAttempt.data.cost = { amount: "0.07", currency: "USD", source: "catalog_estimate" };
cancelledTerminal.data.operation.id = "op_eff_c";
cancelledTerminal.id = "evt_eff_c_term";
cancelledTerminal.data.outcome = { status: "cancelled" };

const efficiencySnapshot = buildSignalOpsOpsSnapshotV1({
tenantId,
range: "24h",
now: new Date("2026-08-23T12:00:00.000Z"),
records: [
efficiencyAccepted,
efficiencyAttemptOne,
efficiencyAttemptTwo,
efficiencyTerminal,
failedEfficiencyAccepted,
failedEfficiencyAttempt,
failedEfficiencyTerminal,
cancelledAccepted,
cancelledAttempt,
cancelledTerminal,
].map((event, index) => ({
tenantId,
event,
payloadDigest: `eff-${index}`,
receivedAt,
})),
});
assert.deepEqual(
efficiencySnapshot.spendEfficiency.currencies,
[
{
currency: "USD",
succeededCost: 0.07,
wastedCost: 0.1,
cancelledCost: 0.07,
retryTaxCost: 0.02,
succeededOperations: 1,
wastedOperations: 1,
cancelledOperations: 1,
costPerSucceededOperation: 0.07,
},
],
"Spend efficiency must attribute cost evidence to operation outcomes and separate retry money.",
);

console.log("signalops ops v1 checks passed");
33 changes: 33 additions & 0 deletions src/components/live-cockpit.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2119,6 +2119,39 @@ export function LiveCockpit() {
</Panel>
</section>

<section className="mt-4">
<Panel
title="Spend efficiency"
subtitle={
snapshot.spendEfficiency.currencies.length === 0
? "No cost evidence in this window yet"
: "Where cost evidence went: succeeded work, retries, and operations that produced nothing"
}
>
{snapshot.spendEfficiency.currencies.length === 0 ? (
<EmptyState text="Spend efficiency appears once operations carry cost evidence." />
) : (
<div className="grid gap-2 lg:grid-cols-2">
{snapshot.spendEfficiency.currencies.map((row) => (
<div key={row.currency} className="rounded-lg border border-[var(--border)] px-3 py-2">
<div className="flex items-center justify-between gap-2">
<p className="font-mono text-[10px] font-bold uppercase text-[var(--mute)]">{row.currency}</p>
<p className="font-mono text-[10px] text-[var(--text-dim)]">
{row.costPerSucceededOperation === null
? "—"
: `${formatMoney(row.costPerSucceededOperation, row.currency)} / succeeded op`}
</p>
</div>
<p className="mt-1 font-mono text-[10px] leading-relaxed text-[var(--text-dim)]">
succeeded {formatMoney(row.succeededCost, row.currency)} over {formatNumber(row.succeededOperations)} ops · retry tax {formatMoney(row.retryTaxCost, row.currency)} · wasted on failures {formatMoney(row.wastedCost, row.currency)} over {formatNumber(row.wastedOperations)} ops · cancelled {formatMoney(row.cancelledCost, row.currency)} over {formatNumber(row.cancelledOperations)} ops
</p>
</div>
))}
</div>
)}
</Panel>
</section>

<CockpitWindowPulse
range={displayedRange}
timeline={snapshot.timeline}
Expand Down
89 changes: 89 additions & 0 deletions src/lib/signalops/v1/ops-snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,37 @@ export type SignalOpsOpsSnapshotV1 = {
recentOperations: SignalOpsOperationSnapshotV1[];
recentFailedOperations: SignalOpsOperationSnapshotV1[];
reconciliation: SignalOpsReconciliationBlockV1;
spendEfficiency: SignalOpsSpendEfficiencyBlockV1;
};

export type SignalOpsSpendEfficiencyCurrencyV1 = {
currency: string;
/** Cost evidence attached to attempts of operations that succeeded. */
succeededCost: number;
/** Cost evidence attached to attempts of operations that failed, expired or were abandoned. */
wastedCost: number;
/** Cost evidence attached to attempts of customer-cancelled operations. */
cancelledCost: number;
/** Subset of succeededCost paid for attempts numbered 2 and above. */
retryTaxCost: number;
succeededOperations: number;
wastedOperations: number;
cancelledOperations: number;
costPerSucceededOperation: number | null;
};

export type SignalOpsSpendEfficiencyBlockV1 = {
currencies: SignalOpsSpendEfficiencyCurrencyV1[];
};

type SignalOpsSpendEfficiencyAccumulatorV1 = {
succeededCost: number;
wastedCost: number;
cancelledCost: number;
retryTaxCost: number;
succeededOperations: number;
wastedOperations: number;
cancelledOperations: number;
};

export type SignalOpsReconciliationPeriodRowV1 = {
Expand Down Expand Up @@ -684,6 +715,11 @@ export function buildSignalOpsOpsSnapshotV1(input: {
let terminalAttempts = 0;
let costedTerminalAttempts = 0;
let retryableFailures = 0;
// Per-operation cost evidence, split into first-attempt and retry money, so
// the spend-efficiency block can attribute it to the operation's outcome.
const costByOperation = new Map<string, Map<string, { total: number; retry: number }>>();
const spendEfficiency = new Map<string, SignalOpsSpendEfficiencyAccumulatorV1>();
const roundUsd = (value: number): number => Math.round(value * 1_000_000) / 1_000_000;
for (const attempt of attempts.values()) {
const event = attempt.terminal;
const bucket = timelineBucketV1(
Expand Down Expand Up @@ -727,6 +763,15 @@ export function buildSignalOpsOpsSnapshotV1(input: {
if (event.data.cost) {
costedTerminalAttempts += 1;
costedOperations.add(attempt.operationId);
const costAmount = Number(event.data.cost.amount);
if (Number.isFinite(costAmount)) {
const perCurrency = costByOperation.get(attempt.operationId) ?? new Map();
const entry = perCurrency.get(event.data.cost.currency) ?? { total: 0, retry: 0 };
entry.total += costAmount;
if (event.data.attempt.number >= 2) entry.retry += costAmount;
perCurrency.set(event.data.cost.currency, entry);
costByOperation.set(attempt.operationId, perCurrency);
}
addCost(row.costs, event.data.cost.currency, event.data.cost.source, event.data.cost.amount);
addCost(totalCosts, event.data.cost.currency, event.data.cost.source, event.data.cost.amount);
addCost(
Expand Down Expand Up @@ -866,6 +911,32 @@ export function buildSignalOpsOpsSnapshotV1(input: {
else if (status === "cancelled") cancelledOperations += 1;
else if (status === "stalled") stalledOperations += 1;
else if (failed) failedOperations += 1;
const operationCosts = costByOperation.get(operationId);
if (operationCosts && status !== "running" && status !== "stalled") {
for (const [currency, amounts] of operationCosts) {
const acc = spendEfficiency.get(currency) ?? {
succeededCost: 0,
wastedCost: 0,
cancelledCost: 0,
retryTaxCost: 0,
succeededOperations: 0,
wastedOperations: 0,
cancelledOperations: 0,
};
if (status === "succeeded") {
acc.succeededCost += amounts.total;
acc.retryTaxCost += amounts.retry;
acc.succeededOperations += 1;
} else if (status === "cancelled") {
acc.cancelledCost += amounts.total;
acc.cancelledOperations += 1;
} else if (failed) {
acc.wastedCost += amounts.total;
acc.wastedOperations += 1;
}
spendEfficiency.set(currency, acc);
}
}
if (durationMs !== null) operationDurations.push(durationMs);
operationBucket.operations += 1;
if (failed) operationBucket.failedOperations += 1;
Expand Down Expand Up @@ -1037,5 +1108,23 @@ export function buildSignalOpsOpsSnapshotV1(input: {
.filter((operation) => isFailedOperationStatusV1(operation.status))
.slice(0, 50),
reconciliation: buildReconciliationBlockV1(reconciliations, attemptTerminalCostsByProvider),
spendEfficiency: {
currencies: [...spendEfficiency.entries()]
.map(([currency, acc]) => ({
currency,
succeededCost: roundUsd(acc.succeededCost),
wastedCost: roundUsd(acc.wastedCost),
cancelledCost: roundUsd(acc.cancelledCost),
retryTaxCost: roundUsd(acc.retryTaxCost),
succeededOperations: acc.succeededOperations,
wastedOperations: acc.wastedOperations,
cancelledOperations: acc.cancelledOperations,
costPerSucceededOperation:
acc.succeededOperations > 0
? roundUsd(acc.succeededCost / acc.succeededOperations)
: null,
}))
.sort((left, right) => left.currency.localeCompare(right.currency)),
},
};
}
1 change: 1 addition & 0 deletions src/lib/signalops/v1/projection-repository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ function isMatchingSnapshot(
Number.isFinite(row.snapshot?.coverage?.attemptLifecycle?.observed) &&
Array.isArray(row.snapshot?.failureBreakdown) &&
row.snapshot.reconciliation !== undefined &&
Array.isArray(row.snapshot?.spendEfficiency?.currencies) &&
Array.isArray(row.snapshot.reconciliation.currencies) &&
Array.isArray(row.snapshot.reconciliation.periods) &&
Date.parse(row.projected_at) >= now.getTime() - 60_000
Expand Down
Loading
Loading